add self activation and loss

README.md

@@ -24,10 +24,6 @@
 > Apple LLVM version 10.0.1 (clang-1001.0.46.4)
 > Target: x86_64-apple-darwin18.7.0
 > Thread model: posix
->
-> 实现的:
->
-> 进入到clone 之后的目录
 
 ## 安装编译
 

grad_edge/matrix_grad.h

@@ -11,12 +11,9 @@ using namespace std;
 float sigmoid(float x){
     return 1/(1+exp(x));
 }
-Node mean_square_error(Matrix mid1,Matrix mid2){
-    Node loss;
-    for(size_t i=0 ; i<mid2.row ; i++){
-        loss += pow(mid1.matrix[i]-mid2.matrix[i], 2);
-    }
-    return loss;
+Node sigmoid(Node z){
+   Node sigmoid = 1/(1+(1/exp(z)));
+    return sigmoid;
 }
 struct edge_network
 {
@@ -56,7 +53,7 @@ struct edge_network
 	//	cout_mat(output);		
 		return output;
 	}
-	Matrix backward(Matrix grad_next, Matrix output_before,Matrix weights)
+	Matrix backward(Matrix grad_next, Matrix output_before,Matrix weights,Node p_(Node))
 	{
 	//	cout<<"-----------backward-------------"<<endl;
 	//	cout_mat(grad_next);
@@ -67,26 +64,27 @@ struct edge_network
 	//	cout<<"-----------TTTTTweights--------------"<<endl;
 	//	cout_mat(padding(mul_simple(mul(get_T(weights),grad_next),output_before),2,2));
 //		cout<<"[[[[[";
-	//	cout<<"grad_next:"<<endl;
-	//	cout_mat(grad_next);
-	//	cout<<"weights"<<endl;
-	//	cout_mat(weights);
+		cout<<"grad_next:"<<endl;
+		cout_mat(grad_next);
+		cout<<"weights"<<endl;
+		cout_mat(weights);
 		for(int index = 0;index<output_before.row;index++)
 		{
 			Node z = output_before.matrix[index][0];
-			Node sigmoid = 1/(1+(1/exp(z)));
-			change_va(output_before,index,0,sigmoid.gradient(z));
+//			Node sigmoid =  1/(1+(1/exp(z)));
+			Node anyone = p_(z);
+			change_va(output_before,index,0,anyone.gradient(z));
+			cout<<"grad_edge: "<<anyone.gradient(z)<<endl;
 		}
-	//	cout<<"output_before"<<endl;
-	//	cout_mat(output_before);
-	//	cout<<"mul(weights,grad_next)"<<endl;
-	//	cout_mat(mul(weights,grad_next));
-	//	cout<<"grad_backward: "<<endl;
-	//	cout_mat(mul_simple(mul(weights,grad_next),output_before));
+		cout<<"output_before"<<endl;
+		cout_mat(output_before);
+		cout<<"mul(weights,grad_next)"<<endl;
+		cout_mat(mul(weights,grad_next));
+		cout<<"grad_backward: "<<endl;
+		cout_mat(mul_simple(mul(weights,grad_next),output_before));
 		return mul_simple(mul(weights,grad_next),output_before);
-
 	}
-	Matrix end_layer_backward(Matrix label,Matrix acti_val)
+	Matrix end_layer_backward(Matrix label,Matrix acti_val,Node loss_fun(Node,Node),Node act(Node))
 	{
 		Matrix loss_act = CreateMatrix(acti_val.row,acti_val.col);
 		Matrix act_output = CreateMatrix(acti_val.row,acti_val.col);
@@ -97,7 +95,7 @@ struct edge_network
 		Node t1 = label.matrix[index_x][0],z31 =acti_val.matrix[index_x][0];
 	//	cout<<"t1: "<<t1<<endl;
 		Node a13 = 1/(1+(1/exp(z31)));
-		Node loss = 0.5*(pow((t1-a13),2));
+		Node loss = loss_fun(t1,a13);
 		Node act = 1/(1+(1/exp(z31)));		
 		act_output.matrix[index_x][0] = act.gradient(z31);
 		loss_act.matrix[index_x][0] = loss.gradient(a13);

main.cpp

@@ -14,6 +14,20 @@
 using namespace std;
 clock_t start, stop;
 double duration;
+Node z=1;
+Node t1 = 1,a13 = 1;
+Node loss_act(Node t1,Node a13)
+{
+	 Node loss = 0.5*(pow((t1-a13),2));
+	 return loss;
+}
+Node sigmoid_act(Node z)
+{
+Node sigmoid_act = 1/(1+(1/exp(z)));
+return sigmoid_act;
+}
+Node (*loss)(Node,Node) = loss_act;
+Node (*act)(Node) = sigmoid_act;
 int main()
 {/*
 	welcome();	
@@ -47,36 +61,38 @@ int main()
     printf("%f\n", (double)(stop - start) / CLOCKS_PER_SEC);
 	*/
 	cout<<"------------autodiff for neraul network-----------"<<endl;
+
 	Matrix data_mine = CreateMatrix(2,1);
-	/*change_va(data_mine,0,0,0.55);
-	change_va(data_mine,1,0,0.2);*/
+	change_va(data_mine,0,0,0.55);
+	change_va(data_mine,1,0,0.2);
 	cout<<"data mine"<<endl;
 	cout_mat(data_mine);
 	cout<<"data mine"<<endl;
 	Matrix label = CreateRandMat(2,1);
-  /*  change_va(label,0,0,0.4);
+    change_va(label,0,0,0.4);
 	change_va(label,1,0,0.8);
-	*/
+	
 	Matrix weight1 = CreateRandMat(2,2);
-	/*change_va(weight1,0,0,0.1);
+	change_va(weight1,0,0,0.1);
 	change_va(weight1,0,1,0.2);
 	change_va(weight1,1,0,0.2);
 	change_va(weight1,1,1,0.4);
-	*/
+	
 	cout<<"weight1";
 	cout_mat(weight1);
 	cout<<"weight1";
 	Matrix bais1 = ones(2,1);
 	Matrix weight2 = CreateRandMat(2,2);
-	//change_va(weight2,0,0,0.5);
-	//change_va(weight2,1,0,0.6);
-  //  change_va(weight2,0,1,0.7);
-//	change_va(weight2,1,1,0.8);
+	change_va(weight2,0,0,0.5);
+	change_va(weight2,1,0,0.6);
+    change_va(weight2,0,1,0.7);
+	change_va(weight2,1,1,0.8);
 	Matrix bais2 = ones(2,1);
-	for(int epoch = 0;epoch<1000;epoch++)
+	for(int epoch = 0;epoch<1;epoch++)
 	{
 	cout<<"---------epoch: "<<epoch<<"------------"<<endl;
-//	cout_mat(weight1);
+	cout_mat(weight1);
+
 	int input_dim = 2;
 	int output_dim = 2;
 	edge_network sequaltial(input_dim,output_dim);
@@ -86,10 +102,10 @@ int main()
 	//output1_without_act without activation;
 	Matrix output2 = sequaltial.forward(output1,weight2,bais2);
 	Matrix output2_without_act = sequaltial.forward_without_act(output1,weight2,bais2);	
-	Matrix output_end = sequaltial.end_layer_backward(label,output2_without_act);
+	Matrix output_end = sequaltial.end_layer_backward(label,output2_without_act,*loss,*act);
 	//last output layer should feed the value without activation;
 	//output_end : the gradient of loss layer;
-	Matrix backward3 = sequaltial.backward(output_end,output1_without_act,weight2);
+	Matrix backward3 = sequaltial.backward(output_end,output1_without_act,weight2,*act);
 	//backward3: the gradient of the behind layer;
 	Matrix weight_2_grad = mul(output_end,get_T(output1));
 	Matrix weight_1_grad = mul(backward3,get_T(data_mine));
@@ -98,7 +114,6 @@ int main()
 	weight2 = subtract(weight2,times_mat(0.001,weight_2_grad));
 	bais2 = subtract(bais2,times_mat(0.001,output_end));
 	}
-	
 	return 0;
 }