Fix error handling in i2d* functions.

CLA: trivial
Reviewed-by: NKurt Roeckx <kurt@roeckx.be>
Reviewed-by: NBernd Edlinger <bernd.edlinger@hotmail.de>
(Merged from https://github.com/openssl/openssl/pull/4600)

crypto/x509/x_pubkey.c

@@ -206,7 +206,7 @@ int i2d_PUBKEY(EVP_PKEY *a, unsigned char **pp)
     if (!a)
         return 0;
     if (!X509_PUBKEY_set(&xpk, a))
-        return 0;
+        return -1;
     ret = i2d_X509_PUBKEY(xpk, pp);
     X509_PUBKEY_free(xpk);
     return ret;
@@ -246,7 +246,7 @@ int i2d_RSA_PUBKEY(RSA *a, unsigned char **pp)
     pktmp = EVP_PKEY_new();
     if (pktmp == NULL) {
         ASN1err(ASN1_F_I2D_RSA_PUBKEY, ERR_R_MALLOC_FAILURE);
-        return 0;
+        return -1;
     }
     EVP_PKEY_set1_RSA(pktmp, a);
     ret = i2d_PUBKEY(pktmp, pp);
@@ -286,7 +286,7 @@ int i2d_DSA_PUBKEY(DSA *a, unsigned char **pp)
     pktmp = EVP_PKEY_new();
     if (pktmp == NULL) {
         ASN1err(ASN1_F_I2D_DSA_PUBKEY, ERR_R_MALLOC_FAILURE);
-        return 0;
+        return -1;
     }
     EVP_PKEY_set1_DSA(pktmp, a);
     ret = i2d_PUBKEY(pktmp, pp);
@@ -325,7 +325,7 @@ int i2d_EC_PUBKEY(EC_KEY *a, unsigned char **pp)
         return 0;
     if ((pktmp = EVP_PKEY_new()) == NULL) {
         ASN1err(ASN1_F_I2D_EC_PUBKEY, ERR_R_MALLOC_FAILURE);
-        return 0;
+        return -1;
     }
     EVP_PKEY_set1_EC_KEY(pktmp, a);
     ret = i2d_PUBKEY(pktmp, pp);