Check for overlows and error return from ASN1_object_size()

Reviewed-by: NRichard Levitte <levitte@openssl.org>

crypto/asn1/a_object.c

@@ -26,7 +26,7 @@ int i2d_ASN1_OBJECT(const ASN1_OBJECT *a, unsigned char **pp)
         return (0);
 
     objsize = ASN1_object_size(0, a->length, V_ASN1_OBJECT);
-    if (pp == NULL)
+    if (pp == NULL || objsize == -1)
         return objsize;
 
     p = *pp;

crypto/asn1/tasn_enc.c

@@ -153,17 +153,19 @@ int ASN1_item_ex_i2d(ASN1_VALUE **pval, unsigned char **out,
         for (i = 0, tt = it->templates; i < it->tcount; tt++, i++) {
             const ASN1_TEMPLATE *seqtt;
             ASN1_VALUE **pseqval;
+            int tmplen;
             seqtt = asn1_do_adb(pval, tt, 1);
             if (!seqtt)
                 return 0;
             pseqval = asn1_get_field_ptr(pval, seqtt);
-            /* FIXME: check for errors in enhanced version */
-            seqcontlen += asn1_template_ex_i2d(pseqval, NULL, seqtt,
-                                               -1, aclass);
+            tmplen = asn1_template_ex_i2d(pseqval, NULL, seqtt, -1, aclass);
+            if (tmplen == -1 || (tmplen > INT_MAX - seqcontlen))
+                return -1;
+            seqcontlen += tmplen;
         }
 
         seqlen = ASN1_object_size(ndef, seqcontlen, tag);
-        if (!out)
+        if (!out || seqlen == -1)
             return seqlen;
         /* Output SEQUENCE header */
         ASN1_put_object(out, ndef, seqcontlen, tag, aclass);
@@ -280,19 +282,24 @@ static int asn1_template_ex_i2d(ASN1_VALUE **pval, unsigned char **out,
         /* Determine total length of items */
         skcontlen = 0;
         for (i = 0; i < sk_ASN1_VALUE_num(sk); i++) {
+            int tmplen;
             skitem = sk_ASN1_VALUE_value(sk, i);
-            skcontlen += ASN1_item_ex_i2d(&skitem, NULL,
-                                          ASN1_ITEM_ptr(tt->item),
-                                          -1, iclass);
+            tmplen = ASN1_item_ex_i2d(&skitem, NULL, ASN1_ITEM_ptr(tt->item),
+                                      -1, iclass);
+            if (tmplen == -1 || (skcontlen > INT_MAX - tmplen))
+                return -1;
+            skcontlen += tmplen;
         }
         sklen = ASN1_object_size(ndef, skcontlen, sktag);
+        if (sklen == -1)
+            return -1;
         /* If EXPLICIT need length of surrounding tag */
         if (flags & ASN1_TFLG_EXPTAG)
             ret = ASN1_object_size(ndef, sklen, ttag);
         else
             ret = sklen;
 
-        if (!out)
+        if (!out || ret == -1)
             return ret;
 
         /* Now encode this lot... */
@@ -321,7 +328,7 @@ static int asn1_template_ex_i2d(ASN1_VALUE **pval, unsigned char **out,
             return 0;
         /* Find length of EXPLICIT tag */
         ret = ASN1_object_size(ndef, i, ttag);
-        if (out) {
+        if (out && ret != -1) {
             /* Output tag and item */
             ASN1_put_object(out, ndef, i, ttag, tclass);
             ASN1_item_ex_i2d(pval, out, ASN1_ITEM_ptr(tt->item), -1, iclass);