Rework error handling from asn1_do_lock method.

Reviewed-by: NRich Salz <rsalz@openssl.org>
Reviewed-by: NMatt Caswell <matt@openssl.org>

crypto/asn1/tasn_fre.c

@@ -89,7 +89,7 @@ static void asn1_item_embed_free(ASN1_VALUE **pval, const ASN1_ITEM *it,
 
     case ASN1_ITYPE_NDEF_SEQUENCE:
     case ASN1_ITYPE_SEQUENCE:
-        if (asn1_do_lock(pval, -1, it) > 0)
+        if (asn1_do_lock(pval, -1, it) != 0) /* if error or ref-counter > 0 */
             return;
         if (asn1_cb) {
             i = asn1_cb(ASN1_OP_FREE_PRE, pval, it, NULL);

crypto/asn1/tasn_new.c

@@ -123,7 +123,9 @@ int asn1_item_embed_new(ASN1_VALUE **pval, const ASN1_ITEM *it, int embed)
             if (*pval == NULL)
                 goto memerr;
         }
-        asn1_do_lock(pval, 0, it);
+        /* 0 : init. lock */
+        if (asn1_do_lock(pval, 0, it) < 0)
+            goto memerr;
         asn1_enc_init(pval, it);
         for (i = 0, tt = it->templates; i < it->tcount; tt++, i++) {
             pseqval = asn1_get_field_ptr(pval, tt);

crypto/asn1/tasn_utl.c

@@ -46,13 +46,14 @@ int asn1_set_choice_selector(ASN1_VALUE **pval, int value,
 }
 
 /*
- * Do reference counting. The value 'op' decides what to do. if it is +1
- * then the count is incremented. If op is 0 count is set to 1. If op is -1
- * count is decremented and the return value is the current reference count
- * or 0 if no reference count exists.
- * FIXME: return and manage any error from inside this method
+ * Do atomic reference counting. The value 'op' decides what to do.
+ * If it is +1 then the count is incremented.
+ * If |op| is 0, lock is initialised and count is set to 1.
+ * If |op| is -1, count is decremented and the return value is the current
+ * reference count or 0 if no reference count is active.
+ * It returns -1 on initialisation error.
+ * Used by ASN1_SEQUENCE construct of X509, X509_REQ, X509_CRL objects
  */
-
 int asn1_do_lock(ASN1_VALUE **pval, int op, const ASN1_ITEM *it)
 {
     const ASN1_AUX *aux;
@@ -70,18 +71,21 @@ int asn1_do_lock(ASN1_VALUE **pval, int op, const ASN1_ITEM *it)
         *lck = 1;
         *lock = CRYPTO_THREAD_lock_new();
         if (*lock == NULL) {
-            /* FIXME: should report an error (-1) at this point */
-            return 0;
+            ASN1err(ASN1_F_ASN1_DO_LOCK, ERR_R_MALLOC_FAILURE);
+            return -1;
         }
         return 1;
     }
-    CRYPTO_atomic_add(lck, op, &ret, *lock);
+    if (CRYPTO_atomic_add(lck, op, &ret, *lock) < 0)
+        return -1;  /* failed */
 #ifdef REF_PRINT
     fprintf(stderr, "%p:%4d:%s\n", it, *lck, it->sname);
 #endif
     REF_ASSERT_ISNT(ret < 0);
-    if (ret == 0)
+    if (ret == 0) {
         CRYPTO_THREAD_lock_free(*lock);
+        *lock = NULL;
+    }
     return ret;
 }
 

crypto/include/internal/x509_int.h

@@ -64,15 +64,15 @@ struct X509_crl_info_st {
     X509_NAME *issuer;          /* CRL issuer name */
     ASN1_TIME *lastUpdate;      /* lastUpdate field */
     ASN1_TIME *nextUpdate;      /* nextUpdate field: optional */
-    STACK_OF(X509_REVOKED) *revoked; /* revoked entries: optional */
+    STACK_OF(X509_REVOKED) *revoked;        /* revoked entries: optional */
     STACK_OF(X509_EXTENSION) *extensions;   /* extensions: optional */
-    ASN1_ENCODING enc;          /* encoding of signed portion of CRL */
+    ASN1_ENCODING enc;                      /* encoding of signed portion of CRL */
 };
 
 struct X509_crl_st {
     X509_CRL_INFO crl;          /* signed CRL data */
     X509_ALGOR sig_alg;         /* CRL signature algorithm */
-    ASN1_BIT_STRING signature; /* CRL signature */
+    ASN1_BIT_STRING signature;  /* CRL signature */
     int references;
     int flags;
     /*

include/openssl/asn1.h

@@ -885,6 +885,7 @@ void ERR_load_ASN1_strings(void);
 # define ASN1_F_ASN1_D2I_READ_BIO                         107
 # define ASN1_F_ASN1_DIGEST                               184
 # define ASN1_F_ASN1_DO_ADB                               110
+# define ASN1_F_ASN1_DO_LOCK                              233
 # define ASN1_F_ASN1_DUP                                  111
 # define ASN1_F_ASN1_EX_C2I                               204
 # define ASN1_F_ASN1_FIND_END                             190