Improve error handling if decompression of an ec point fails, and cleanup

ec_curve.c (unify comments, etc).

Submitted by: Nils Larsch
Reviewed by: Bodo Moeller, Geoff Thorpe

crypto/bn/bn.h

@@ -783,6 +783,7 @@ void ERR_load_BN_strings(void);
 #define BN_R_NOT_IMPLEMENTED				 116
 #define BN_R_NOT_INITIALIZED				 107
 #define BN_R_NO_INVERSE					 108
+#define BN_R_NO_SOLUTION				 117
 #define BN_R_P_IS_NOT_PRIME				 112
 #define BN_R_TOO_MANY_ITERATIONS			 113
 #define BN_R_TOO_MANY_TEMPORARY_VARIABLES		 109

crypto/bn/bn_err.c

@@ -116,6 +116,7 @@ static ERR_STRING_DATA BN_str_reasons[]=
 {BN_R_NOT_IMPLEMENTED                    ,"not implemented"},
 {BN_R_NOT_INITIALIZED                    ,"not initialized"},
 {BN_R_NO_INVERSE                         ,"no inverse"},
+{BN_R_NO_SOLUTION                        ,"no solution"},
 {BN_R_P_IS_NOT_PRIME                     ,"p is not prime"},
 {BN_R_TOO_MANY_ITERATIONS                ,"too many iterations"},
 {BN_R_TOO_MANY_TEMPORARY_VARIABLES       ,"too many temporary variables"},

crypto/bn/bn_gf2m.c

@@ -988,7 +988,11 @@ int BN_GF2m_mod_solve_quad_arr(BIGNUM *r, const BIGNUM *a_, const unsigned int p
 	
 	if (!BN_GF2m_mod_sqr_arr(w, z, p, ctx)) goto err;
 	if (!BN_GF2m_add(w, z, w)) goto err;
-	if (BN_GF2m_cmp(w, a)) goto err;
+	if (BN_GF2m_cmp(w, a))
+		{
+		BNerr(BN_F_BN_GF2M_MOD_SOLVE_QUAD_ARR, BN_R_NO_SOLUTION);
+		goto err;
+		}
 
 	if (!BN_copy(r, z)) goto err;
 	bn_check_top(r);

crypto/ec/ec2_smpt.c

@@ -77,6 +77,9 @@ int ec_GF2m_simple_set_compressed_coordinates(const EC_GROUP *group, EC_POINT *p
 	BIGNUM *tmp, *x, *y, *z;
 	int ret = 0, z0;
 
+	/* clear error queue */
+	ERR_clear_error();
+
 	if (ctx == NULL)
 		{
 		ctx = new_ctx = BN_CTX_new();
@@ -104,7 +107,19 @@ int ec_GF2m_simple_set_compressed_coordinates(const EC_GROUP *group, EC_POINT *p
 		if (!group->meth->field_div(group, tmp, &group->b, tmp, ctx)) goto err;
 		if (!BN_GF2m_add(tmp, &group->a, tmp)) goto err;
 		if (!BN_GF2m_add(tmp, x, tmp)) goto err;
-		if (!BN_GF2m_mod_solve_quad_arr(z, tmp, group->poly, ctx)) goto err;
+		if (!BN_GF2m_mod_solve_quad_arr(z, tmp, group->poly, ctx))
+			{
+			unsigned long err = ERR_peek_last_error();
+			
+			if (ERR_GET_LIB(err) == ERR_LIB_BN && ERR_GET_REASON(err) == BN_R_NO_SOLUTION)
+				{
+				ERR_clear_error();
+				ECerr(EC_F_EC_GF2M_SIMPLE_SET_COMPRESSED_COORDINATES, EC_R_INVALID_COMPRESSED_POINT);
+				}
+			else
+				ECerr(EC_F_EC_GF2M_SIMPLE_SET_COMPRESSED_COORDINATES, ERR_R_BN_LIB);
+			goto err;
+			}
 		z0 = (BN_is_odd(z)) ? 1 : 0;
 		if (!group->meth->field_mul(group, y, x, z, ctx)) goto err;
 		if (z0 != y_bit)

crypto/ec/ecp_smpl.c

@@ -640,6 +640,9 @@ int ec_GFp_simple_set_compressed_coordinates(const EC_GROUP *group, EC_POINT *po
 	BIGNUM *tmp1, *tmp2, *x, *y;
 	int ret = 0;
 
+	/* clear error queue*/
+	ERR_clear_error();
+
 	if (ctx == NULL)
 		{
 		ctx = new_ctx = BN_CTX_new();
@@ -711,11 +714,11 @@ int ec_GFp_simple_set_compressed_coordinates(const EC_GROUP *group, EC_POINT *po
 	
 	if (!BN_mod_sqrt(y, tmp1, &group->field, ctx))
 		{
-		unsigned long err = ERR_peek_error();
+		unsigned long err = ERR_peek_last_error();
 		
 		if (ERR_GET_LIB(err) == ERR_LIB_BN && ERR_GET_REASON(err) == BN_R_NOT_A_SQUARE)
 			{
-			(void)ERR_get_error();
+			ERR_clear_error();
 			ECerr(EC_F_EC_GFP_SIMPLE_SET_COMPRESSED_COORDINATES, EC_R_INVALID_COMPRESSED_POINT);
 			}
 		else