Various tidies/fixes:

Make streaming support in cms cleaner.

Note errors in various S/MIME functions if CMS_final() fails.

Add streaming support for enveloped data.

apps/cms.c

@@ -116,7 +116,6 @@ int MAIN(int argc, char **argv)
 	char *passargin = NULL, *passin = NULL;
 	char *inrand = NULL;
 	int need_rand = 0;
-	int indef = 0;
 	const EVP_MD *sign_md = NULL;
 	int informat = FORMAT_SMIME, outformat = FORMAT_SMIME;
         int keyform = FORMAT_PEM;
@@ -232,11 +231,11 @@ int MAIN(int argc, char **argv)
 		else if (!strcmp (*args, "-no_attr_verify"))
 				flags |= CMS_NO_ATTR_VERIFY;
 		else if (!strcmp (*args, "-stream"))
-				indef = 1;
+				flags |= CMS_STREAM;
 		else if (!strcmp (*args, "-indef"))
-				indef = 1;
+				flags |= CMS_STREAM;
 		else if (!strcmp (*args, "-noindef"))
-				indef = 0;
+				flags &= ~CMS_STREAM;
 		else if (!strcmp (*args, "-nooldmime"))
 				flags |= CMS_NOOLDMIMETYPE;
 		else if (!strcmp (*args, "-crlfeol"))
@@ -726,32 +725,22 @@ int MAIN(int argc, char **argv)
 
 	if (operation == SMIME_DATA_CREATE)
 		{
-		if (indef)
-			flags |= CMS_STREAM;
 		cms = CMS_data_create(in, flags);
 		}
 	else if (operation == SMIME_DIGEST_CREATE)
 		{
-		if (indef)
-			flags |= CMS_STREAM;
 		cms = CMS_digest_create(in, sign_md, flags);
 		}
 	else if (operation == SMIME_COMPRESS)
 		{
-		if (indef)
-			flags |= CMS_STREAM;
 		cms = CMS_compress(in, -1, flags);
 		}
 	else if (operation == SMIME_ENCRYPT)
 		{
-		if (indef)
-			flags |= CMS_STREAM;
 		cms = CMS_encrypt(encerts, in, cipher, flags);
 		}
 	else if (operation == SMIME_ENCRYPTED_ENCRYPT)
 		{
-		if (indef)
-			flags |= CMS_STREAM;
 		cms = CMS_EncryptedData_encrypt(in, cipher,
 						secret_key, secret_keylen,
 						flags);
@@ -766,11 +755,9 @@ int MAIN(int argc, char **argv)
 			{
 			if (flags & CMS_DETACHED)
 				{
-				if (outformat == FORMAT_SMIME)
-					flags |= CMS_STREAM;
+				if (outformat != FORMAT_SMIME)
+					flags &= ~CMS_STREAM;
 				}
-			else if (indef)
-				flags |= CMS_STREAM;
 			flags |= CMS_PARTIAL;
 			cms = CMS_sign(NULL, NULL, other, in, flags);
 			if (!cms)

crypto/cms/cms_io.c

@@ -89,20 +89,6 @@ int i2d_CMS_bio(BIO *bp, CMS_ContentInfo *cms)
 
 IMPLEMENT_PEM_rw_const(CMS, CMS_ContentInfo, PEM_STRING_CMS, CMS_ContentInfo)
 
-#if 0
-/* Streaming encode support for CMS */
-
-static BIO *cmsbio_init(ASN1_VALUE *val, BIO *out)
-	{
-	return CMS_dataInit((CMS_ContentInfo *)val, out);
-	}
-
-static int cmsbio_final(ASN1_VALUE *val, BIO *cmsbio)
-	{
-	return CMS_dataFinal((CMS_ContentInfo *)val, cmsbio);
-	}
-#endif
-
 BIO *BIO_new_CMS(BIO *out, CMS_ContentInfo *cms) 
 	{
 	return BIO_new_NDEF(out, (ASN1_VALUE *)cms,

crypto/cms/cms_lib.c

@@ -184,6 +184,7 @@ int CMS_dataFinal(CMS_ContentInfo *cms, BIO *cmsbio)
 		{
 
 		case NID_pkcs7_data:
+		case NID_pkcs7_enveloped:
 		case NID_pkcs7_encrypted:
 		case NID_id_smime_ct_compressedData:
 		/* Nothing to do */

crypto/cms/cms_smime.c

@@ -447,8 +447,8 @@ CMS_ContentInfo *CMS_sign(X509 *signcert, EVP_PKEY *pkey, STACK_OF(X509) *certs,
 
 	if ((flags & (CMS_STREAM|CMS_PARTIAL)) || CMS_final(cms, data, flags))
 		return cms;
-
-	return cms;
+	else
+		goto err;
 
 	merr:
 	CMSerr(CMS_F_CMS_SIGN, ERR_R_MALLOC_FAILURE);
@@ -483,8 +483,8 @@ CMS_ContentInfo *CMS_encrypt(STACK_OF(X509) *certs, BIO *data,
 
 	if ((flags & (CMS_STREAM|CMS_PARTIAL)) || CMS_final(cms, data, flags))
 		return cms;
-
-	return cms;
+	else
+		goto err;
 
 	merr:
 	CMSerr(CMS_F_CMS_ENCRYPT, ERR_R_MALLOC_FAILURE);
@@ -530,6 +530,7 @@ int CMS_decrypt(CMS_ContentInfo *cms, EVP_PKEY *pk, X509 *cert,
 					break;
 				if (cert)
 					return 0;
+				ERR_clear_error();
 				}
 			}