8008088: SA can hang the VM

Reviewed-by: mgronlun, sla, dholmes

agent/src/os/bsd/libproc_impl.c

 /*
- * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -91,6 +91,14 @@ void print_debug(const char* format,...) {
    }
 }
 
+void print_error(const char* format,...) {
+  va_list alist;
+  va_start(alist, format);
+  fputs("ERROR: ", stderr);
+  vfprintf(stderr, format, alist);
+  va_end(alist);
+}
+
 bool is_debug() {
    return _libsaproc_debug;
 }

agent/src/os/bsd/libproc_impl.h

 /*
- * Copyright (c) 2003, 2005, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -107,6 +107,7 @@ struct ps_prochandle {
 int pathmap_open(const char* name);
 
 void print_debug(const char* format,...);
+void print_error(const char* format,...);
 bool is_debug();
 
 typedef bool (*thread_info_callback)(struct ps_prochandle* ph, pthread_t pid, lwpid_t lwpid);

agent/src/os/bsd/ps_proc.c

 /*
- * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -129,42 +129,66 @@ static bool process_get_lwp_info(struct ps_prochandle *ph, lwpid_t lwp_id, void
   return (errno == 0)? true: false;
 }
 
-// attach to a process/thread specified by "pid"
-static bool ptrace_attach(pid_t pid) {
-  if (ptrace(PT_ATTACH, pid, NULL, 0) < 0) {
-    print_debug("ptrace(PTRACE_ATTACH, ..) failed for %d\n", pid);
+static bool ptrace_continue(pid_t pid, int signal) {
+  // pass the signal to the process so we don't swallow it
+  if (ptrace(PTRACE_CONT, pid, NULL, signal) < 0) {
+    print_debug("ptrace(PTRACE_CONT, ..) failed for %d\n", pid);
     return false;
-  } else {
-    int ret;
-    int status;
-    do {
-      // Wait for debuggee to stop.
-      ret = waitpid(pid, &status, 0);
-      if (ret >= 0) {
-        if (WIFSTOPPED(status)) {
-          // Debuggee stopped.
+  }
+  return true;
+}
+
+// waits until the ATTACH has stopped the process
+// by signal SIGSTOP
+static bool ptrace_waitpid(pid_t pid) {
+  int ret;
+  int status;
+  do {
+    // Wait for debuggee to stop.
+    ret = waitpid(pid, &status, 0);
+    if (ret >= 0) {
+      if (WIFSTOPPED(status)) {
+        // Any signal will stop the thread, make sure it is SIGSTOP. Otherwise SIGSTOP
+        // will still be pending and delivered when the process is DETACHED and the process
+        // will go to sleep.
+        if (WSTOPSIG(status) == SIGSTOP) {
+          // Debuggee stopped by SIGSTOP.
           return true;
-        } else {
-          print_debug("waitpid(): Child process exited/terminated (status = 0x%x)\n", status);
+        }
+        if (!ptrace_continue(pid, WSTOPSIG(status))) {
+          print_error("Failed to correctly attach to VM. VM might HANG! [PTRACE_CONT failed, stopped by %d]\n", WSTOPSIG(status));
           return false;
         }
       } else {
-        switch (errno) {
-          case EINTR:
-            continue;
-            break;
-          case ECHILD:
-            print_debug("waitpid() failed. Child process pid (%d) does not exist \n", pid);
-            break;
-          case EINVAL:
-            print_debug("waitpid() failed. Invalid options argument.\n");
-            break;
-          default:
-            print_debug("waitpid() failed. Unexpected error %d\n",errno);
-        }
+        print_debug("waitpid(): Child process exited/terminated (status = 0x%x)\n", status);
         return false;
       }
-    } while(true);
+    } else {
+      switch (errno) {
+        case EINTR:
+          continue;
+          break;
+        case ECHILD:
+          print_debug("waitpid() failed. Child process pid (%d) does not exist \n", pid);
+          break;
+        case EINVAL:
+          print_debug("waitpid() failed. Invalid options argument.\n");
+          break;
+        default:
+          print_debug("waitpid() failed. Unexpected error %d\n",errno);
+      }
+      return false;
+    }
+  } while(true);
+}
+
+// attach to a process/thread specified by "pid"
+static bool ptrace_attach(pid_t pid) {
+  if (ptrace(PT_ATTACH, pid, NULL, 0) < 0) {
+    print_debug("ptrace(PTRACE_ATTACH, ..) failed for %d\n", pid);
+    return false;
+  } else {
+    return ptrace_waitpid(pid);
   }
 }
 

agent/src/os/linux/libproc_impl.c

 /*
- * Copyright (c) 2003, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -92,6 +92,14 @@ void print_debug(const char* format,...) {
    }
 }
 
+void print_error(const char* format,...) {
+  va_list alist;
+  va_start(alist, format);
+  fputs("ERROR: ", stderr);
+  vfprintf(stderr, format, alist);
+  va_end(alist);
+}
+
 bool is_debug() {
    return _libsaproc_debug;
 }

agent/src/os/linux/libproc_impl.h

 /*
- * Copyright (c) 2003, 2005, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -105,6 +105,7 @@ struct ps_prochandle {
 int pathmap_open(const char* name);
 
 void print_debug(const char* format,...);
+void print_error(const char* format,...);
 bool is_debug();
 
 typedef bool (*thread_info_callback)(struct ps_prochandle* ph, pthread_t pid, lwpid_t lwpid);

agent/src/os/linux/ps_proc.c

 /*
- * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <signal.h>
 #include <errno.h>
 #include <sys/ptrace.h>
 #include "libproc_impl.h"
@@ -142,46 +143,71 @@ static bool process_get_lwp_regs(struct ps_prochandle* ph, pid_t pid, struct use
 
 }
 
-// attach to a process/thread specified by "pid"
-static bool ptrace_attach(pid_t pid) {
-  if (ptrace(PTRACE_ATTACH, pid, NULL, NULL) < 0) {
-    print_debug("ptrace(PTRACE_ATTACH, ..) failed for %d\n", pid);
+static bool ptrace_continue(pid_t pid, int signal) {
+  // pass the signal to the process so we don't swallow it
+  if (ptrace(PTRACE_CONT, pid, NULL, signal) < 0) {
+    print_debug("ptrace(PTRACE_CONT, ..) failed for %d\n", pid);
     return false;
-  } else {
-    int ret;
-    int status;
-    do {
-      // Wait for debuggee to stop.
-      ret = waitpid(pid, &status, 0);
-      if (ret == -1 && errno == ECHILD) {
-        // try cloned process.
-        ret = waitpid(pid, &status, __WALL);
-      }
-      if (ret >= 0) {
-        if (WIFSTOPPED(status)) {
-          // Debuggee stopped.
+  }
+  return true;
+}
+
+// waits until the ATTACH has stopped the process
+// by signal SIGSTOP
+static bool ptrace_waitpid(pid_t pid) {
+  int ret;
+  int status;
+  while (true) {
+    // Wait for debuggee to stop.
+    ret = waitpid(pid, &status, 0);
+    if (ret == -1 && errno == ECHILD) {
+      // try cloned process.
+      ret = waitpid(pid, &status, __WALL);
+    }
+    if (ret >= 0) {
+      if (WIFSTOPPED(status)) {
+        // Any signal will stop the thread, make sure it is SIGSTOP. Otherwise SIGSTOP
+        // will still be pending and delivered when the process is DETACHED and the process
+        // will go to sleep.
+        if (WSTOPSIG(status) == SIGSTOP) {
+          // Debuggee stopped by SIGSTOP.
           return true;
-        } else {
-          print_debug("waitpid(): Child process exited/terminated (status = 0x%x)\n", status);
+        }
+        if (!ptrace_continue(pid, WSTOPSIG(status))) {
+          print_error("Failed to correctly attach to VM. VM might HANG! [PTRACE_CONT failed, stopped by %d]\n", WSTOPSIG(status));
           return false;
         }
       } else {
-        switch (errno) {
-          case EINTR:
-            continue;
-            break;
-          case ECHILD:
-            print_debug("waitpid() failed. Child process pid (%d) does not exist \n", pid);
-            break;
-          case EINVAL:
-            print_debug("waitpid() failed. Invalid options argument.\n");
-            break;
-          default:
-            print_debug("waitpid() failed. Unexpected error %d\n",errno);
-        }
+        print_debug("waitpid(): Child process exited/terminated (status = 0x%x)\n", status);
         return false;
       }
-    } while(true);
+    } else {
+      switch (errno) {
+        case EINTR:
+          continue;
+          break;
+        case ECHILD:
+          print_debug("waitpid() failed. Child process pid (%d) does not exist \n", pid);
+          break;
+        case EINVAL:
+          print_debug("waitpid() failed. Invalid options argument.\n");
+          break;
+        default:
+          print_debug("waitpid() failed. Unexpected error %d\n",errno);
+          break;
+      }
+      return false;
+    }
+  }
+}
+
+// attach to a process/thread specified by "pid"
+static bool ptrace_attach(pid_t pid) {
+  if (ptrace(PTRACE_ATTACH, pid, NULL, NULL) < 0) {
+    print_debug("ptrace(PTRACE_ATTACH, ..) failed for %d\n", pid);
+    return false;
+  } else {
+    return ptrace_waitpid(pid);
   }
 }