NGX-2040 - fix wildcard match on punycode/IDNA DNS names

- bugfix: should not treat '--' as invalid domain substring.
- '-' should not be the first letter of a domain
Signed-off-by: NViktor Dukhovni <viktor@openssl.org>
Reviewed-by: NRich Salz <rsalz@openssl.org>

crypto/x509v3/v3_utl.c

@@ -840,7 +840,8 @@ static const unsigned char *valid_star(const unsigned char *p, size_t len,
             state = LABEL_START;
             ++dots;
         } else if (p[i] == '-') {
-            if ((state & LABEL_HYPHEN) != 0)
+            /* no domain/subdomain starts with '-' */
+            if ((state & LABEL_START) != 0)
                 return NULL;
             state |= LABEL_HYPHEN;
         } else

test/v3nametest.c

@@ -6,12 +6,16 @@
 static const char *const names[] = {
     "a", "b", ".", "*", "@",
     ".a", "a.", ".b", "b.", ".*", "*.", "*@", "@*", "a@", "@a", "b@", "..",
+    "-example.com", "example-.com",
     "@@", "**", "*.com", "*com", "*.*.com", "*com", "com*", "*example.com",
     "*@example.com", "test@*.example.com", "example.com", "www.example.com",
     "test.www.example.com", "*.example.com", "*.www.example.com",
     "test.*.example.com", "www.*.com",
     ".www.example.com", "*www.example.com",
     "example.net", "xn--rger-koa.example.com",
+    "*.xn--rger-koa.example.com", "www.xn--rger-koa.example.com",
+    "*.good--example.com", "www.good--example.com",
+    "*.xn--bar.com", "xn--foo.xn--bar.com",
     "a.example.com", "b.example.com",
     "postmaster@example.com", "Postmaster@example.com",
     "postmaster@EXAMPLE.COM",
@@ -27,6 +31,9 @@ static const char *const exceptions[] = {
     "set CN: host: [*.www.example.com] matches [.www.example.com]",
     "set CN: host: [*www.example.com] matches [www.example.com]",
     "set CN: host: [test.www.example.com] matches [.www.example.com]",
+    "set CN: host: [*.xn--rger-koa.example.com] matches [www.xn--rger-koa.example.com]",
+    "set CN: host: [*.xn--bar.com] matches [xn--foo.xn--bar.com]",
+    "set CN: host: [*.good--example.com] matches [www.good--example.com]",
     "set CN: host-no-wildcards: [*.www.example.com] matches [.www.example.com]",
     "set CN: host-no-wildcards: [test.www.example.com] matches [.www.example.com]",
     "set emailAddress: email: [postmaster@example.com] does not match [Postmaster@example.com]",
@@ -43,6 +50,9 @@ static const char *const exceptions[] = {
     "set dnsName: host: [*.www.example.com] matches [.www.example.com]",
     "set dnsName: host: [*www.example.com] matches [www.example.com]",
     "set dnsName: host: [test.www.example.com] matches [.www.example.com]",
+    "set dnsName: host: [*.xn--rger-koa.example.com] matches [www.xn--rger-koa.example.com]",
+    "set dnsName: host: [*.xn--bar.com] matches [xn--foo.xn--bar.com]",
+    "set dnsName: host: [*.good--example.com] matches [www.good--example.com]",
     "set rfc822Name: email: [postmaster@example.com] does not match [Postmaster@example.com]",
     "set rfc822Name: email: [Postmaster@example.com] does not match [postmaster@example.com]",
     "set rfc822Name: email: [Postmaster@example.com] does not match [postmaster@EXAMPLE.COM]",