BIO_lookup_ex: Do not retry on EAI_MEMORY

We should not retry on EAI_MEMORY as that error is most probably
fatal and not depending on AI_ADDRCONFIG hint.

Also report the error from the first call if the second call fails
as that one would be most probably the more interesting one.
Reviewed-by: NPaul Dale <paul.dale@oracle.com>
Reviewed-by: NRichard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/9535)

(cherry picked from commit 91cb81d40a8102c3d8667629661be8d6937db82b)

crypto/bio/b_addr.c

@@ -675,7 +675,7 @@ int BIO_lookup_ex(const char *host, const char *service, int lookup_type,
 
     if (1) {
 #ifdef AI_PASSIVE
-        int gai_ret = 0;
+        int gai_ret = 0, old_ret = 0;
         struct addrinfo hints;
 
         memset(&hints, 0, sizeof(hints));
@@ -683,12 +683,12 @@ int BIO_lookup_ex(const char *host, const char *service, int lookup_type,
         hints.ai_family = family;
         hints.ai_socktype = socktype;
         hints.ai_protocol = protocol;
-#ifdef AI_ADDRCONFIG
-#ifdef AF_UNSPEC
+# ifdef AI_ADDRCONFIG
+#  ifdef AF_UNSPEC
         if (family == AF_UNSPEC)
-#endif
+#  endif
             hints.ai_flags |= AI_ADDRCONFIG;
-#endif
+# endif
 
         if (lookup_type == BIO_LOOKUP_SERVER)
             hints.ai_flags |= AI_PASSIVE;
@@ -703,6 +703,11 @@ int BIO_lookup_ex(const char *host, const char *service, int lookup_type,
             SYSerr(SYS_F_GETADDRINFO, get_last_socket_error());
             BIOerr(BIO_F_BIO_LOOKUP_EX, ERR_R_SYS_LIB);
             break;
+# endif
+# ifdef EAI_MEMORY
+        case EAI_MEMORY:
+            BIOerr(BIO_F_BIO_LOOKUP_EX, ERR_R_MALLOC_FAILURE);
+            break;
 # endif
         case 0:
             ret = 1;             /* Success */
@@ -712,11 +717,12 @@ int BIO_lookup_ex(const char *host, const char *service, int lookup_type,
             if (hints.ai_flags & AI_ADDRCONFIG) {
                 hints.ai_flags &= ~AI_ADDRCONFIG;
                 hints.ai_flags |= AI_NUMERICHOST;
+                old_ret = gai_ret;
                 goto retry;
             }
 # endif
             BIOerr(BIO_F_BIO_LOOKUP_EX, ERR_R_SYS_LIB);
-            ERR_add_error_data(1, gai_strerror(gai_ret));
+            ERR_add_error_data(1, gai_strerror(old_ret ? old_ret : gai_ret));
             break;
         }
     } else {