6861984: solaris version of libsaproc.so should support SA_ALTROOT directly

Reviewed-by: kvn, twisti

agent/make/saenv.sh

@@ -48,6 +48,8 @@ if [ "$OS" = "Linux" ]; then
      CPU=i386
    fi
 else
+   LD_AUDIT_32=$STARTDIR/../src/os/solaris/proc/`uname -p`/libsaproc_audit.so
+   export LD_AUDIT_32
    SA_LIBPATH=$STARTDIR/../src/os/solaris/proc/`uname -p`:$STARTDIR/solaris/`uname -p`
    OPTIONS="-Dsa.library.path=$SA_LIBPATH -Dsun.jvm.hotspot.debugger.useProcDebugger"
    CPU=sparc

agent/make/saenv64.sh

@@ -43,6 +43,8 @@ else
   fi
 fi
 
+LD_AUDIT_64=$STARTDIR/../src/os/solaris/proc/$CPU/libsaproc_audit.so
+export LD_AUDIT_64
 SA_LIBPATH=$STARTDIR/../src/os/solaris/proc/$CPU:$STARTDIR/solaris/$CPU
 
 OPTIONS="-Dsa.library.path=$SA_LIBPATH -Dsun.jvm.hotspot.debugger.useProcDebugger"

agent/src/os/solaris/proc/Makefile

@@ -56,24 +56,28 @@ i386:: javahomecheck
 	@javah -classpath $(CLASSES_DIR) -jni sun.jvm.hotspot.debugger.proc.ProcDebuggerLocal
 	CC -G -KPIC -I${JAVA_HOME}/include -I${JAVA_HOME}/include/solaris saproc.cpp \
            -M mapfile -o $@/libsaproc.so -ldemangle
+	CC -o $@/libsaproc_audit.so -G -Kpic -z defs saproc_audit.cpp -lmapmalloc -ldl -lc 
 
 amd64:: javahomecheck
 	$(MKDIRS) $@
 	@javah -classpath $(CLASSES_DIR) -jni sun.jvm.hotspot.debugger.proc.ProcDebuggerLocal
 	CC -G -KPIC -xarch=amd64 -I${JAVA_HOME}/include -I${JAVA_HOME}/include/solaris saproc.cpp \
            -M mapfile -o $@/libsaproc.so -ldemangle
+	CC -xarch=amd64 -o $@/libsaproc_audit.so -G -Kpic -z defs saproc_audit.cpp -lmapmalloc -ldl -lc 
 
 sparc:: javahomecheck
 	$(MKDIRS) $@
 	@javah -classpath $(CLASSES_DIR) -jni sun.jvm.hotspot.debugger.proc.ProcDebuggerLocal
 	CC -G -KPIC -xarch=v8  -I${JAVA_HOME}/include -I${JAVA_HOME}/include/solaris saproc.cpp \
            -M mapfile -o $@/libsaproc.so -ldemangle
+	CC -xarch=v8 -o $@/libsaproc_audit.so -G -Kpic -z defs saproc_audit.cpp -lmapmalloc -ldl -lc 
 
 sparcv9:: javahomecheck
 	$(MKDIRS) $@
 	@javah -classpath $(CLASSES_DIR) -jni sun.jvm.hotspot.debugger.proc.ProcDebuggerLocal
 	CC -G -KPIC -xarch=v9 -I${JAVA_HOME}/include -I${JAVA_HOME}/include/solaris saproc.cpp \
            -M mapfile -o $@/libsaproc.so -ldemangle
+	CC -xarch=v9 -o $@/libsaproc_audit.so -G -Kpic -z defs saproc_audit.cpp -lmapmalloc -ldl -lc 
 
 clean::
 	$(RM) -rf sun_jvm_hotspot_debugger_proc_ProcDebuggerLocal.h

agent/src/os/solaris/proc/mapfile

@@ -45,6 +45,8 @@ SUNWprivate_1.1 {
 		Java_sun_jvm_hotspot_debugger_proc_ProcDebuggerLocal_resume0;
 		Java_sun_jvm_hotspot_debugger_proc_ProcDebuggerLocal_suspend0;
 		Java_sun_jvm_hotspot_debugger_proc_ProcDebuggerLocal_writeBytesToProcess0;
+                # this is needed by saproc_audit.c to redirect opens in libproc.so
+                libsaproc_open;
 local:
 	*;
 };

agent/src/os/solaris/proc/saproc.cpp

@@ -214,11 +214,17 @@ static void init_alt_root() {
   }
 }
 
-static int find_file_hook(const char * name, int elf_checksum) {
+// This function is a complete substitute for the open system call
+// since it's also used to override open calls from libproc to
+// implement as a pathmap style facility for the SA.  If libproc
+// starts using other interfaces then this might have to extended to
+// cover other calls.
+extern "C" int libsaproc_open(const char * name, int oflag, ...) {
+  if (oflag == O_RDONLY) {
     init_alt_root();
 
     if (_libsaproc_debug) {
-    printf("libsaproc DEBUG: find_file_hook %s 0x%x\n", name, elf_checksum);
+      printf("libsaproc DEBUG: libsaproc_open %s\n", name);
     }
 
     if (alt_root_len > 0) {
@@ -230,7 +236,7 @@ static int find_file_hook(const char * name, int elf_checksum) {
       fd = open(alt_path, O_RDONLY);
       if (fd >= 0) {
         if (_libsaproc_debug) {
-        printf("libsaproc DEBUG: find_file_hook substituted %s\n", alt_path);
+          printf("libsaproc DEBUG: libsaproc_open substituted %s\n", alt_path);
         }
         return fd;
       }
@@ -241,23 +247,26 @@ static int find_file_hook(const char * name, int elf_checksum) {
         fd = open(alt_path, O_RDONLY);
         if (fd >= 0) {
           if (_libsaproc_debug) {
-          printf("libsaproc DEBUG: find_file_hook substituted %s\n", alt_path);
+            printf("libsaproc DEBUG: libsaproc_open substituted %s\n", alt_path);
           }
           return fd;
         }
       }
     }
-  return -1;
-}
+  }
 
-static int pathmap_open(const char* name) {
-  int fd = open(name, O_RDONLY);
-  if (fd < 0) {
-    fd = find_file_hook(name, 0);
+  {
+    mode_t mode;
+    va_list ap;
+    va_start(ap, oflag);
+    mode = va_arg(ap, mode_t);
+    va_end(ap);
+
+    return open(name, oflag, mode);
   }
-  return fd;
 }
 
+
 static void * pathmap_dlopen(const char * name, int mode) {
   init_alt_root();
 
@@ -608,7 +617,7 @@ init_classsharing_workaround(void *cd, const prmap_t* pmap, const char* obj_name
   print_debug("looking for %s\n", classes_jsa);
 
   // open the classes[_g].jsa
-  int fd = pathmap_open(classes_jsa);
+  int fd = libsaproc_open(classes_jsa, O_RDONLY);
   if (fd < 0) {
     char errMsg[ERR_MSG_SIZE];
     sprintf(errMsg, "can't open shared archive file %s", classes_jsa);
@@ -1209,8 +1218,6 @@ JNIEXPORT jstring JNICALL Java_sun_jvm_hotspot_debugger_proc_ProcDebuggerLocal_d
   return res;
 }
 
-typedef int (*find_file_hook_t)(const char *, int elf_checksum);
-
 /*
  * Class:       sun_jvm_hotspot_debugger_proc_ProcDebuggerLocal
  * Method:      initIDs
@@ -1230,16 +1237,6 @@ JNIEXPORT void JNICALL Java_sun_jvm_hotspot_debugger_proc_ProcDebuggerLocal_init
   if (libproc_handle == 0)
      THROW_NEW_DEBUGGER_EXCEPTION("can't load libproc.so, if you are using Solaris 5.7 or below, copy libproc.so from 5.8!");
 
-  // If possible, set shared object find file hook.
-  void (*set_hook)(find_file_hook_t) = (void(*)(find_file_hook_t))dlsym(libproc_handle, "Pset_find_file_hook");
-  if (set_hook) {
-    // we found find file hook symbol, set up our hook function.
-    set_hook(find_file_hook);
-  } else if (getenv(SA_ALTROOT)) {
-    printf("libsaproc WARNING: %s set, but can't set file hook. " \
-           "Did you use right version of libproc.so?\n", SA_ALTROOT);
-  }
-
   p_ps_prochandle_ID = env->GetFieldID(clazz, "p_ps_prochandle", "J");
   CHECK_EXCEPTION;
 

agent/src/os/solaris/proc/saproc_audit.cpp

+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+#include <link.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <varargs.h>
+
+// This class sets up an interposer on open calls from libproc.so to
+// support a pathmap facility in the SA.
+
+static uintptr_t* libproc_cookie;
+static uintptr_t* libc_cookie;
+static uintptr_t* libsaproc_cookie;
+
+
+uint_t
+la_version(uint_t version)
+{
+  return (LAV_CURRENT);
+}
+
+
+uint_t
+la_objopen(Link_map * lmp, Lmid_t lmid, uintptr_t * cookie)
+{
+  if (strstr(lmp->l_name, "/libproc.so") != NULL) {
+    libproc_cookie = cookie;
+    return LA_FLG_BINDFROM;
+  }
+  if (strstr(lmp->l_name, "/libc.so") != NULL) {
+    libc_cookie = cookie;
+    return LA_FLG_BINDTO;
+  }
+  if (strstr(lmp->l_name, "/libsaproc.so") != NULL) {
+    libsaproc_cookie = cookie;
+    return LA_FLG_BINDTO | LA_FLG_BINDFROM;
+  }
+  return 0;
+}
+
+
+#if     defined(_LP64)
+uintptr_t
+la_symbind64(Elf64_Sym *symp, uint_t symndx, uintptr_t *refcook,
+             uintptr_t *defcook, uint_t *sb_flags, const char *sym_name)
+#else
+uintptr_t
+la_symbind32(Elf32_Sym *symp, uint_t symndx, uintptr_t *refcook,
+             uintptr_t *defcook, uint_t *sb_flags)
+#endif
+{
+#if     !defined(_LP64)
+  const char      *sym_name = (const char *)symp->st_name;
+#endif
+  if (strcmp(sym_name, "open") == 0 && refcook == libproc_cookie) {
+    // redirect all open calls from libproc.so through libsaproc_open which will
+    // try the alternate library locations first.
+    void* handle = dlmopen(LM_ID_BASE, "libsaproc.so", RTLD_NOLOAD);
+    if (handle == NULL) {
+      fprintf(stderr, "libsaproc_audit.so: didn't find libsaproc.so during linking\n");
+    } else {
+      uintptr_t libsaproc_open = (uintptr_t)dlsym(handle, "libsaproc_open");
+      if (libsaproc_open == 0) {
+        fprintf(stderr, "libsaproc_audit.so: didn't find libsaproc_open during linking\n");
+      } else {
+        return libsaproc_open;
+      }
+    }
+  }
+  return symp->st_value;
+}