8207250: setUseClientMode post handshake with the same value as before does not throw IAE

Reviewed-by: xuelei

src/java.base/share/classes/sun/security/ssl/TransportContext.java

@@ -393,6 +393,13 @@ class TransportContext implements ConnectionContext, Closeable {
     }
 
     void setUseClientMode(boolean useClientMode) {
+        // Once handshaking has begun, the mode can not be reset for the
+        // life of this engine.
+        if (handshakeContext != null || isNegotiated) {
+            throw new IllegalArgumentException(
+                    "Cannot change mode after SSL traffic has started");
+        }
+
         /*
          * If we need to change the client mode and the enabled
          * protocols and cipher suites haven't specifically been
@@ -400,13 +407,6 @@ class TransportContext implements ConnectionContext, Closeable {
          * default ones.
          */
         if (sslConfig.isClientMode != useClientMode) {
-            // Once handshaking has begun, the mode can not be reset for the
-            // life of this engine.
-            if (handshakeContext != null || isNegotiated) {
-                throw new IllegalArgumentException(
-                    "Cannot change mode after SSL traffic has started");
-            }
-
             if (sslContext.isDefaultProtocolVesions(
                     sslConfig.enabledProtocols)) {
                 sslConfig.enabledProtocols =

test/jdk/sun/security/ssl/SSLEngineImpl/EngineEnforceUseClientMode.java

 /*
- * Copyright (c) 2004, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2004, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,7 +28,7 @@
 
 /*
  * @test
- * @bug 4980882
+ * @bug 4980882 8207250
  * @summary SSLEngine should enforce setUseClientMode
  * @run main/othervm EngineEnforceUseClientMode
  * @author Brad R. Wetmore
@@ -190,14 +190,18 @@ public class EngineEnforceUseClientMode {
                 checkTransfer(appOut1, appIn2);
                 checkTransfer(appOut2, appIn1);
 
+                // Should not be able to set mode now, no matter if
+                // it is the same of different.
                 System.out.println("Try changing modes...");
-                try {
-                    ssle2.setUseClientMode(true);
-                    throw new RuntimeException(
-                        "setUseClientMode():  " +
-                        "Didn't catch the exception properly");
-                } catch (IllegalArgumentException e) {
-                    System.out.println("Caught the correct exception.");
+                for (boolean b : new Boolean[] {true, false}) {
+                    try {
+                        ssle2.setUseClientMode(b);
+                        throw new RuntimeException(
+                                "setUseClientMode(" + b + "):  " +
+                                        "Didn't catch the exception properly");
+                    } catch (IllegalArgumentException e) {
+                        System.out.println("Caught the correct exception.");
+                    }
                 }
 
                 return;