7118809: rcache deadlock

Reviewed-by: valeriep

src/share/classes/sun/security/krb5/internal/rcache/CacheTable.java

@@ -31,8 +31,6 @@
 package sun.security.krb5.internal.rcache;
 
 import java.util.Hashtable;
-import sun.security.krb5.internal.KerberosTime;
-
 
 /**
  * This class implements Hashtable to store the replay caches.
@@ -60,12 +58,15 @@ public class CacheTable extends Hashtable<String,ReplayCache> {
             }
             rc = new ReplayCache(principal, this);
             rc.put(time, currTime);
-            super.put(principal, rc);
+            if (!rc.isEmpty()) {
+                super.put(principal, rc);
+            }
         }
         else {
             rc.put(time, currTime);
-            // re-insert the entry, since rc.put could have removed the entry
-            super.put(principal, rc);
+            if (rc.isEmpty()) {
+                super.remove(rc);
+            }
             if (DEBUG) {
                 System.out.println("replay cache found.");
             }

src/share/classes/sun/security/krb5/internal/rcache/ReplayCache.java

@@ -31,8 +31,6 @@
 
 package sun.security.krb5.internal.rcache;
 
-import sun.security.krb5.KrbException;
-import sun.security.krb5.Config;
 import sun.security.krb5.internal.Krb5;
 import java.util.LinkedList;
 import java.util.ListIterator;
@@ -48,10 +46,13 @@ public class ReplayCache extends LinkedList<AuthTime> {
 
     private static final long serialVersionUID = 2997933194993803994L;
 
+    // These 3 fields are now useless, keep for serialization compatibility
     private String principal;
     private CacheTable table;
     private int nap = 10 * 60 * 1000; //10 minutes break
+
     private boolean DEBUG = Krb5.DEBUG;
+
     /**
      * Constructs a ReplayCache for a client principal in specified <code>CacheTable</code>.
      * @param p client principal name.
@@ -125,20 +126,11 @@ public class ReplayCache extends LinkedList<AuthTime> {
         if (DEBUG) {
             printList();
         }
-
-        // if there are no entries in the replay cache,
-        // remove the replay cache from the table.
-        if (this.size() == 0) {
-            table.remove(principal);
-        }
-        if (DEBUG) {
-            printList();
-        }
     }
 
 
     /**
-     * Printes out the debug message.
+     * Prints out the debug message.
      */
     private void printList() {
         Object[] total = toArray();

test/sun/security/krb5/auto/Context.java

@@ -76,7 +76,6 @@ public class Context {
 
     private Subject s;
     private ExtendedGSSContext x;
-    private boolean f;      // context established?
     private String name;
     private GSSCredential cred;     // see static method delegated().
 
@@ -194,7 +193,6 @@ public class Context {
                 return null;
             }
         }, null);
-        f = false;
     }
 
     /**
@@ -228,7 +226,6 @@ public class Context {
                 return null;
             }
         }, null);
-        f = false;
     }
 
     /**
@@ -502,6 +499,29 @@ public class Context {
         return sb.toString();
     }
 
+    public byte[] take(final byte[] in) throws Exception {
+        return doAs(new Action() {
+            @Override
+            public byte[] run(Context me, byte[] input) throws Exception {
+                if (me.x.isEstablished()) {
+                    System.out.println(name + " side established");
+                    if (input != null) {
+                        throw new Exception("Context established but " +
+                                "still receive token at " + name);
+                    }
+                    return null;
+                } else {
+                    System.out.println(name + " call initSecContext");
+                    if (me.x.isInitiator()) {
+                        return me.x.initSecContext(input, 0, input.length);
+                    } else {
+                        return me.x.acceptSecContext(input, 0, input.length);
+                    }
+                }
+            }
+        }, in);
+    }
+
     /**
      * Handshake (security context establishment process) between two Contexts
      * @param c the initiator
@@ -510,54 +530,9 @@ public class Context {
      */
     static public void handshake(final Context c, final Context s) throws Exception {
         byte[] t = new byte[0];
-        while (!c.f || !s.f) {
-            t = c.doAs(new Action() {
-                @Override
-                public byte[] run(Context me, byte[] input) throws Exception {
-                    if (me.x.isEstablished()) {
-                        me.f = true;
-                        System.out.println(c.name + " side established");
-                        if (input != null) {
-                            throw new Exception("Context established but " +
-                                    "still receive token at " + c.name);
-                        }
-                        return null;
-                    } else {
-                        System.out.println(c.name + " call initSecContext");
-                        if (usingStream) {
-                            ByteArrayOutputStream os = new ByteArrayOutputStream();
-                            me.x.initSecContext(new ByteArrayInputStream(input), os);
-                            return os.size() == 0 ? null : os.toByteArray();
-                        } else {
-                            return me.x.initSecContext(input, 0, input.length);
-                        }
-                    }
-                }
-            }, t);
-
-            t = s.doAs(new Action() {
-                @Override
-                public byte[] run(Context me, byte[] input) throws Exception {
-                    if (me.x.isEstablished()) {
-                        me.f = true;
-                        System.out.println(s.name + " side established");
-                        if (input != null) {
-                            throw new Exception("Context established but " +
-                                    "still receive token at " + s.name);
-                        }
-                        return null;
-                    } else {
-                        System.out.println(s.name + " called acceptSecContext");
-                        if (usingStream) {
-                            ByteArrayOutputStream os = new ByteArrayOutputStream();
-                            me.x.acceptSecContext(new ByteArrayInputStream(input), os);
-                            return os.size() == 0 ? null : os.toByteArray();
-                        } else {
-                            return me.x.acceptSecContext(input, 0, input.length);
-                        }
-                    }
-                }
-            }, t);
+        while (!c.x.isEstablished() || !s.x.isEstablished()) {
+            t = c.take(t);
+            t = s.take(t);
         }
     }
 }

test/sun/security/krb5/auto/ReplayCache.java

+/*
+ * Copyright 2012 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+/*
+ * @test
+ * @bug 7118809
+ * @run main/othervm ReplayCache
+ * @summary rcache deadlock
+ */
+
+import org.ietf.jgss.GSSException;
+import sun.security.jgss.GSSUtil;
+import sun.security.krb5.KrbException;
+import sun.security.krb5.internal.Krb5;
+
+public class ReplayCache {
+
+    public static void main(String[] args)
+            throws Exception {
+
+        new OneKDC(null).writeJAASConf();
+
+        Context c, s;
+        c = Context.fromJAAS("client");
+        s = Context.fromJAAS("server");
+
+        c.startAsClient(OneKDC.SERVER, GSSUtil.GSS_KRB5_MECH_OID);
+        s.startAsServer(GSSUtil.GSS_KRB5_MECH_OID);
+
+        byte[] first = c.take(new byte[0]);
+        s.take(first);
+
+        s.startAsServer(GSSUtil.GSS_KRB5_MECH_OID);
+        try {
+            s.take(first);  // Replay the last token sent
+            throw new Exception("This method should fail");
+        } catch (GSSException gsse) {
+            KrbException ke = (KrbException)gsse.getCause();
+            if (ke.returnCode() != Krb5.KRB_AP_ERR_REPEAT) {
+                throw gsse;
+            }
+        }
+    }
+}