8174873: Improved certificate procesing

Reviewed-by: jnimeh, ahgross, rhalade

src/share/classes/sun/security/util/HostnameChecker.java

 /*
- * Copyright (c) 2002, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,6 +34,7 @@ import java.security.Principal;
 import java.security.cert.*;
 
 import javax.security.auth.x500.X500Principal;
+import javax.net.ssl.SNIHostName;
 
 import sun.security.ssl.Krb5Helper;
 import sun.security.x509.X500Name;
@@ -186,6 +187,15 @@ public class HostnameChecker {
      */
     private void matchDNS(String expectedName, X509Certificate cert)
             throws CertificateException {
+        // Check that the expected name is a valid domain name.
+        try {
+            // Using the checking implemented in SNIHostName
+            SNIHostName sni = new SNIHostName(expectedName);
+        } catch (IllegalArgumentException iae) {
+            throw new CertificateException(
+                "Illegal given domain name: " + expectedName, iae);
+        }
+
         Collection<List<?>> subjAltNames = cert.getSubjectAlternativeNames();
         if (subjAltNames != null) {
             boolean foundDNS = false;
@@ -257,6 +267,18 @@ public class HostnameChecker {
      * may contain the wildcard character *
      */
     private boolean isMatched(String name, String template) {
+        // check the validity of the domain name template.
+        try {
+            // Replacing wildcard character '*' with 'x' so as to check
+            // the domain name template validity.
+            //
+            // Using the checking implemented in SNIHostName
+            SNIHostName sni = new SNIHostName(template.replace('*', 'x'));
+        } catch (IllegalArgumentException iae) {
+            // It would be nice to add debug log if not matching.
+            return false;
+        }
+
         if (checkType == TYPE_TLS) {
             return matchAllWildcards(name, template);
         } else if (checkType == TYPE_LDAP) {