Give a better error if an attempt is made to set a zero length groups list

Previously we indicated this as a malloc failure which isn't very
helpful.
Reviewed-by: NKurt Roeckx <kurt@roeckx.be>
(Merged from https://github.com/openssl/openssl/pull/7479)

(cherry picked from commit 680bd131b69d57e891888ab70d300176a5a16617)

doc/man3/SSL_CTX_set1_curves.pod

@@ -32,6 +32,9 @@ SSL_set1_curves_list, SSL_get1_curves, SSL_get_shared_curve
 
 =head1 DESCRIPTION
 
+For all of the functions below that set the supported groups there must be at
+least one group in the list.
+
 SSL_CTX_set1_groups() sets the supported groups for B<ctx> to B<glistlen>
 groups in the array B<glist>. The array consist of all NIDs of groups in
 preference order. For a TLS client the groups are used directly in the

ssl/t1_lib.c

@@ -343,6 +343,10 @@ int tls1_set_groups(uint16_t **pext, size_t *pextlen,
      */
     unsigned long dup_list = 0;
 
+    if (ngroups == 0) {
+        SSLerr(SSL_F_TLS1_SET_GROUPS, SSL_R_BAD_LENGTH);
+        return 0;
+    }
     if ((glist = OPENSSL_malloc(ngroups * sizeof(*glist))) == NULL) {
         SSLerr(SSL_F_TLS1_SET_GROUPS, ERR_R_MALLOC_FAILURE);
         return 0;