complete the demo

grad_edge/matrix_grad.h

@@ -22,22 +22,42 @@ struct edge_network
 {
 	edge_network(int input, int output){
     }
-	Matrix forward(Matrix data,Matrix weights)
+	Matrix forward(Matrix data,Matrix weights,Matrix bais)
 	{
 	//	cout<<"data: ";
 	//	cout_mat(data);
 	//	cout<<"weights: "<<endl;
+	//	cout_mat(weights);	
+		Matrix output = mul(get_T(weights),data);		
+		output = add(output,bais,0);
+		Matrix output1 = e_sigmoid(output);
+	//	cout<<"----------forward data--------"<<endl;
+	//	cout_mat(data);
+	//	cout<<"---------weight1---------"<<endl;
+	//	cout_mat(weights);
+	//	cout<<"---------bais----------"<<endl;
+	//	cout_mat(bais);
+	//	cout<<"---------output--------"<<endl;
+	//	cout_mat(output1);		
+		return output1;
+	}
+	Matrix forward_without_act(Matrix data,Matrix weights,Matrix bais)
+	{
+		Matrix output = mul(get_T(weights),data);		
+		output = add(output,bais,0);
+	//	Matrix output1 = e_sigmoid(output);
+	//	cout<<"----------forward data--------"<<endl;
+	//	cout_mat(data);
+	//	cout<<"---------weight1---------"<<endl;
 	//	cout_mat(weights);
-		Matrix output = mul(weights,data);
-		output = e_sigmoid(output);
-	//	cout<<"forward output: ";
-	//	cout_mat(output);
-	//	cout<<"----------forward finish--------"<<endl;
+	//	cout<<"---------bais----------"<<endl;
+	//	cout_mat(bais);
+	//	cout<<"---------output--------"<<endl;
+	//	cout_mat(output);		
 		return output;
 	}
 	Matrix backward(Matrix grad_next, Matrix output_before,Matrix weights)
 	{
-		
 	//	cout<<"-----------backward-------------"<<endl;
 	//	cout_mat(grad_next);
 	//	cout<<"-----------grad_next------------"<<endl;
@@ -47,7 +67,23 @@ struct edge_network
 	//	cout<<"-----------TTTTTweights--------------"<<endl;
 	//	cout_mat(padding(mul_simple(mul(get_T(weights),grad_next),output_before),2,2));
 //		cout<<"[[[[[";
-		return mul_simple(mul(get_T(weights),grad_next),output_before);
+	//	cout<<"grad_next:"<<endl;
+	//	cout_mat(grad_next);
+	//	cout<<"weights"<<endl;
+	//	cout_mat(weights);
+		for(int index = 0;index<output_before.row;index++)
+		{
+			Node z = output_before.matrix[index][0];
+			Node sigmoid = 1/(1+(1/exp(z)));
+			change_va(output_before,index,0,sigmoid.gradient(z));
+		}
+	//	cout<<"output_before"<<endl;
+	//	cout_mat(output_before);
+	//	cout<<"mul(weights,grad_next)"<<endl;
+	//	cout_mat(mul(weights,grad_next));
+	//	cout<<"grad_backward: "<<endl;
+	//	cout_mat(mul_simple(mul(weights,grad_next),output_before));
+		return mul_simple(mul(weights,grad_next),output_before);
 
 	}
 	Matrix end_layer_backward(Matrix label,Matrix acti_val)
@@ -57,21 +93,25 @@ struct edge_network
 		int index_x,index_y;
 		for(index_x=0;index_x<loss_act.row;index_x++)
 		{
+		//	cout<<"index: "<<index_x;
 		Node t1 = label.matrix[index_x][0],z31 =acti_val.matrix[index_x][0];
-		Node a13 = 1+exp(z31);
+	//	cout<<"t1: "<<t1<<endl;
+		Node a13 = 1/(1+(1/exp(z31)));
 		Node loss = 0.5*(pow((t1-a13),2));
-		Node act = 1/(1+exp(z31));
+		Node act = 1/(1+(1/exp(z31)));		
 		act_output.matrix[index_x][0] = act.gradient(z31);
 		loss_act.matrix[index_x][0] = loss.gradient(a13);
 	//	cout<<"z31: "<<z31<<endl;
 	//	cout<<"z31_grad: "<<act.gradient(z31)<<endl;
-	//	cout<<"a13: "<<a13<<"t1: "<<t1<<endl;
-		cout<<"loss: "<<loss<<endl;	
-	//	cout<<"gradient: "<<loss.gradient(a13)<<endl;
-		}		
+//		cout<<"a13: "<<a13<<"t1: "<<t1<<endl;
+		cout<<"loss: "<<loss<<endl;
+//		cout<<"a13_gradient: "<<loss.gradient(a13)<<endl;
+		}
 		Matrix mid_grad_end = mul_simple(loss_act,act_output);
-	//	cout<<"[[[[[[[[";
-	//	cout_mat(mid_grad_end);
+		
+//		cout<<"mid_grad_end";
+
+//		cout_mat(mid_grad_end);
 		return mid_grad_end;
 	}
 };

main.cpp

@@ -15,7 +15,7 @@ using namespace std;
 clock_t start, stop;
 double duration;
 int main()
-{
+{/*
 	welcome();	
 	string path = "./data/new_data2.csv";
 	Matrix data = read_csv(path);
@@ -45,37 +45,58 @@ int main()
 	}
 	stop = clock();
     printf("%f\n", (double)(stop - start) / CLOCKS_PER_SEC);
+	*/
 	cout<<"------------autodiff for neraul network-----------"<<endl;
-	Matrix data_mine = CreateRandMat(2,1);
-	Matrix label = CreateMatrix(2,1);
+	Matrix data_mine = CreateMatrix(2,1);
+	change_va(data_mine,0,0,0.55);
+	change_va(data_mine,1,0,0.2);
+	cout<<"data mine"<<endl;
+	cout_mat(data_mine);
+	cout<<"data mine"<<endl;
+	Matrix label = CreateRandMat(2,1);
+	change_va(label,0,0,0.4);
+	change_va(label,1,0,0.8);
+
 	Matrix weight1 = CreateRandMat(2,2);
+	change_va(weight1,0,0,0.1);
+	change_va(weight1,0,1,0.2);
+	change_va(weight1,1,0,0.2);
+	change_va(weight1,1,1,0.4);
+	cout<<"weight1";
+	cout_mat(weight1);
+	cout<<"weight1";
+	Matrix bais1 = ones(2,1);
 	Matrix weight2 = CreateRandMat(2,2);
+	change_va(weight2,0,0,0.5);
+	change_va(weight2,1,0,0.6);
+	change_va(weight2,0,1,0.7);
+	change_va(weight2,1,1,0.8);
+	Matrix bais2 = ones(2,1);	
 	Matrix weight3 = CreateRandMat(2,2);
+	Matrix bais3 = ones(2,1);	
 	Matrix weight4 = CreateRandMat(2,2);
-	for(int epoch = 0;epoch<20;epoch++)
+	Matrix bais4 = ones(2,1);	
+	for(int epoch = 0;epoch<10;epoch++)
 	{
-		cout<<"---------epoch: "<<epoch<<"------------"<<endl;
+	cout<<"---------epoch: "<<epoch<<"------------"<<endl;
 //	cout_mat(weight1);
-	edge_network sequaltial(2,2);
-	Matrix output1 = sequaltial.forward(data_mine,weight1);
-	Matrix output2 = sequaltial.forward(output1,weight2);
-	Matrix output3 = sequaltial.forward(output2,weight3);
-	Matrix output4 = sequaltial.forward(output3,weight4);
-	Matrix output_end = sequaltial.end_layer_backward(label,output4);
-	//get the forward
-	Matrix backward1 = sequaltial.backward(output_end,output3,weight4);
-	Matrix grad_w1w2 = mul_simple(backward1,data_mine);
-	Matrix backward2 = sequaltial.backward(backward1,output2,weight3);
-	Matrix grad_w3w4 = mul_simple(backward2,data_mine);
-	Matrix backward3 = sequaltial.backward(backward2,output1,weight2);
-	Matrix grad_w5w6 = mul_simple(backward3,data_mine);
-	Matrix backward4 = sequaltial.backward(backward3,output4,weight1);
-	Matrix grad_w7w8 = mul_simple(backward4,data_mine);
-	weight1 = subtract(weight1,times_mat(0.0001,padding(grad_w1w2,2,2)));
-	weight2 = subtract(weight2,times_mat(0.0001,padding(grad_w3w4,2,2)));
-	weight3 = subtract(weight3,times_mat(0.0001,padding(grad_w5w6,2,2)));
-	weight4 = subtract(weight4,times_mat(0.0001,padding(grad_w7w8,2,2)));
+	int input_dim = 2;
+	int output_dim = 2;
+	edge_network sequaltial(input_dim,output_dim);
+	Matrix output1 = sequaltial.forward(data_mine,weight1,bais1);
+	Matrix output1_without_act = sequaltial.forward_without_act(data_mine,weight1,bais1);
+	Matrix output2 = sequaltial.forward(output1,weight2,bais2);
+	Matrix output2_without_act = sequaltial.forward_without_act(output1,weight2,bais2);	
+	Matrix output_end = sequaltial.end_layer_backward(label,output2_without_act);
+	Matrix backward3 = sequaltial.backward(output_end,output1_without_act,weight2);
+	Matrix weight_2_grad = mul(output_end,get_T(output1));
+	Matrix weight_1_grad = mul(backward3,get_T(data_mine));
+	weight1 = subtract(weight1,times_mat(0.001,weight_1_grad));
+	bais1 = subtract(bais1,times_mat(0.001,backward3));
+	weight2 = subtract(weight2,times_mat(0.001,weight_2_grad));
+	bais2 = subtract(bais2,times_mat(0.001,output_end));
 	}
+	
 	return 0;
 }