7165807: Non optimized initialization of NSS crypto library leads to scalability issues

Reviewed-by: mullan, valeriep

make/sun/security/pkcs11/mapfile-vers

@@ -102,7 +102,7 @@ SUNWprivate_1.1 {
 		Java_sun_security_pkcs11_Secmod_nssGetLibraryHandle;
 		Java_sun_security_pkcs11_Secmod_nssLoadLibrary;
 		Java_sun_security_pkcs11_Secmod_nssVersionCheck;
-		Java_sun_security_pkcs11_Secmod_nssInit;
+		Java_sun_security_pkcs11_Secmod_nssInitialize;
 		Java_sun_security_pkcs11_Secmod_nssGetModuleList;
 
 	local:

makefiles/mapfiles/libj2pkcs11/mapfile-vers

 #
-# Copyright (c) 2003, 2012, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -102,7 +102,7 @@ SUNWprivate_1.1 {
 		Java_sun_security_pkcs11_Secmod_nssGetLibraryHandle;
 		Java_sun_security_pkcs11_Secmod_nssLoadLibrary;
 		Java_sun_security_pkcs11_Secmod_nssVersionCheck;
-		Java_sun_security_pkcs11_Secmod_nssInit;
+		Java_sun_security_pkcs11_Secmod_nssInitialize;
 		Java_sun_security_pkcs11_Secmod_nssGetModuleList;
 
 	local:

src/share/classes/sun/security/pkcs11/Config.java

@@ -197,6 +197,10 @@ final class Config {
     // (false).
     private boolean useEcX963Encoding = false;
 
+    // Flag to indicate whether NSS should favour performance (false) or
+    // memory footprint (true).
+    private boolean nssOptimizeSpace = false;
+
     private Config(String filename, InputStream in) throws IOException {
         if (in == null) {
             if (filename.startsWith("--")) {
@@ -329,6 +333,10 @@ final class Config {
         return useEcX963Encoding;
     }
 
+    boolean getNssOptimizeSpace() {
+        return nssOptimizeSpace;
+    }
+
     private static String expand(final String s) throws IOException {
         try {
             return PropertyExpander.expand(s);
@@ -451,6 +459,8 @@ final class Config {
                 nssUseSecmodTrust = parseBooleanEntry(word);
             } else if (word.equals("useEcX963Encoding")) {
                 useEcX963Encoding = parseBooleanEntry(word);
+            } else if (word.equals("nssOptimizeSpace")) {
+                nssOptimizeSpace = parseBooleanEntry(word);
             } else {
                 throw new ConfigurationException
                         ("Unknown keyword '" + word + "', line " + st.lineno());

src/share/classes/sun/security/pkcs11/Secmod.java

@@ -158,11 +158,17 @@ public final class Secmod {
      */
     public void initialize(String configDir, String nssLibDir)
             throws IOException {
-        initialize(DbMode.READ_WRITE, configDir, nssLibDir);
+        initialize(DbMode.READ_WRITE, configDir, nssLibDir, false);
     }
 
-    public synchronized void initialize(DbMode dbMode, String configDir, String nssLibDir)
+    public void initialize(DbMode dbMode, String configDir, String nssLibDir)
             throws IOException {
+        initialize(dbMode, configDir, nssLibDir, false);
+    }
+
+    public synchronized void initialize(DbMode dbMode, String configDir,
+        String nssLibDir, boolean nssOptimizeSpace) throws IOException {
+
         if (isInitialized()) {
             throw new IOException("NSS is already initialized");
         }
@@ -211,7 +217,8 @@ public final class Secmod {
         }
 
         if (DEBUG) System.out.println("dir: " + configDir);
-        boolean initok = nssInit(dbMode.functionName, nssHandle, configDir);
+        boolean initok = nssInitialize(dbMode.functionName, nssHandle,
+            configDir, nssOptimizeSpace);
         if (DEBUG) System.out.println("init: " + initok);
         if (initok == false) {
             throw new IOException("NSS initialization failed");
@@ -764,7 +771,7 @@ public final class Secmod {
 
     private static native boolean nssVersionCheck(long handle, String minVersion);
 
-    private static native boolean nssInit(String functionName, long handle, String configDir);
+    private static native boolean nssInitialize(String functionName, long handle, String configDir, boolean nssOptimizeSpace);
 
     private static native Object nssGetModuleList(long handle, String libDir);
 

src/share/classes/sun/security/pkcs11/SunPKCS11.java

 /*
- * Copyright (c) 2003, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -167,6 +167,7 @@ public final class SunPKCS11 extends AuthProvider {
             try {
                 String nssLibraryDirectory = config.getNssLibraryDirectory();
                 String nssSecmodDirectory = config.getNssSecmodDirectory();
+                boolean nssOptimizeSpace = config.getNssOptimizeSpace();
 
                 if (secmod.isInitialized()) {
                     if (nssSecmodDirectory != null) {
@@ -204,7 +205,7 @@ public final class SunPKCS11 extends AuthProvider {
                         }
                     }
                     secmod.initialize(nssDbMode, nssSecmodDirectory,
-                        nssLibraryDirectory);
+                        nssLibraryDirectory, nssOptimizeSpace);
                 }
             } catch (IOException e) {
                 // XXX which exception to throw

src/share/native/sun/security/pkcs11/j2secmod.c

 /*
- * Copyright (c) 2005, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -51,20 +51,63 @@ JNIEXPORT jboolean JNICALL Java_sun_security_pkcs11_Secmod_nssVersionCheck
     return (res == 0) ? JNI_FALSE : JNI_TRUE;
 }
 
-JNIEXPORT jboolean JNICALL Java_sun_security_pkcs11_Secmod_nssInit
-  (JNIEnv *env, jclass thisClass, jstring jFunctionName, jlong jHandle, jstring jConfigDir)
+/*
+ * Initializes NSS.
+ * The NSS_INIT_OPTIMIZESPACE flag is supplied by the caller.
+ * The NSS_Init* functions are mapped to the NSS_Initialize function.
+ */
+JNIEXPORT jboolean JNICALL Java_sun_security_pkcs11_Secmod_nssInitialize
+  (JNIEnv *env, jclass thisClass, jstring jFunctionName, jlong jHandle, jstring jConfigDir, jboolean jNssOptimizeSpace)
 {
-    const char *functionName = (*env)->GetStringUTFChars(env, jFunctionName, NULL);
-    const char *configDir = (jConfigDir == NULL) ? NULL : (*env)->GetStringUTFChars(env, jConfigDir, NULL);
-    FPTR_Init init = (FPTR_Init)findFunction(env, jHandle, functionName);
-    int res;
-
-    (*env)->ReleaseStringUTFChars(env, jFunctionName, functionName);
-    if (init == NULL) {
-        return JNI_FALSE;
+    const char *functionName =
+        (*env)->GetStringUTFChars(env, jFunctionName, NULL);
+    const char *configDir = (jConfigDir == NULL)
+        ? NULL : (*env)->GetStringUTFChars(env, jConfigDir, NULL);
+    FPTR_Initialize initialize =
+        (FPTR_Initialize)findFunction(env, jHandle, "NSS_Initialize");
+    int res = 0;
+    unsigned int flags = 0x00;
+
+    if (jNssOptimizeSpace == JNI_TRUE) {
+        flags = 0x20; // NSS_INIT_OPTIMIZESPACE flag
     }
 
-    res = init(configDir);
+    if (initialize != NULL) {
+        /*
+         * If the NSS_Init function is requested then call NSS_Initialize to
+         * open the Cert, Key and Security Module databases, read only.
+         */
+        if (strcmp("NSS_Init", functionName) == 0) {
+            flags = flags | 0x01; // NSS_INIT_READONLY flag
+            res = initialize(configDir, "", "", "secmod.db", flags);
+
+        /*
+         * If the NSS_InitReadWrite function is requested then call
+         * NSS_Initialize to open the Cert, Key and Security Module databases,
+         * read/write.
+         */
+        } else if (strcmp("NSS_InitReadWrite", functionName) == 0) {
+            res = initialize(configDir, "", "", "secmod.db", flags);
+
+        /*
+         * If the NSS_NoDB_Init function is requested then call
+         * NSS_Initialize without creating Cert, Key or Security Module
+         * databases.
+         */
+        } else if (strcmp("NSS_NoDB_Init", functionName) == 0) {
+            flags = flags | 0x02  // NSS_INIT_NOCERTDB flag
+                          | 0x04  // NSS_INIT_NOMODDB flag
+                          | 0x08  // NSS_INIT_FORCEOPEN flag
+                          | 0x10; // NSS_INIT_NOROOTINIT flag
+            res = initialize("", "", "", "", flags);
+
+        } else {
+            res = 2;
+        }
+    } else {
+        res = 1;
+    }
+    (*env)->ReleaseStringUTFChars(env, jFunctionName, functionName);
     if (configDir != NULL) {
         (*env)->ReleaseStringUTFChars(env, jConfigDir, configDir);
     }

src/solaris/native/sun/security/pkcs11/j2secmod_md.h

 /*
- * Copyright (c) 2005, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,9 +25,14 @@
 
 // in nss.h:
 // extern PRBool NSS_VersionCheck(const char *importedVersion);
-// extern SECStatus NSS_Init(const char *configdir);
+// extern SECStatus NSS_Initialize(const char *configdir,
+//     const char *certPrefix, const char *keyPrefix,
+//     const char *secmodName, PRUint32 flags);
+
 typedef int (*FPTR_VersionCheck)(const char *importedVersion);
-typedef int (*FPTR_Init)(const char *configdir);
+typedef int (*FPTR_Initialize)(const char *configdir,
+        const char *certPrefix, const char *keyPrefix,
+        const char *secmodName, unsigned int flags);
 
 // in secmod.h
 //extern SECMODModule *SECMOD_LoadModule(char *moduleSpec,SECMODModule *parent,

src/windows/native/sun/security/pkcs11/j2secmod_md.h

 /*
- * Copyright (c) 2005, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,9 +27,14 @@
 
 // in nss.h:
 // extern PRBool NSS_VersionCheck(const char *importedVersion);
-// extern SECStatus NSS_Init(const char *configdir);
+// extern SECStatus NSS_Initialize(const char *configdir,
+//      const char *certPrefix, const char *keyPrefix,
+//      const char *secmodName, PRUint32 flags);
+
 typedef int __declspec(dllimport) (*FPTR_VersionCheck)(const char *importedVersion);
-typedef int __declspec(dllimport) (*FPTR_Init)(const char *configdir);
+typedef int __declspec(dllimport) (*FPTR_Initialize)(const char *configdir,
+        const char *certPrefix, const char *keyPrefix,
+        const char *secmodName, unsigned int flags);
 
 // in secmod.h
 //extern SECMODModule *SECMOD_LoadModule(char *moduleSpec,SECMODModule *parent,