8235637: jhsdb jmap from OpenJDK 11.0.5 doesn't work if prelink is enabled

Summary: error handling for "lib_base_diff == 0"
Reviewed-by: ysuenaga, cjplummer, kevinw
Contributed-by: suenaga@oss.nttdata.com

agent/src/os/linux/ps_core.c

@@ -868,6 +868,9 @@ err:
 #define LINK_MAP_LD_OFFSET    offsetof(struct link_map, l_ld)
 #define LINK_MAP_NEXT_OFFSET  offsetof(struct link_map, l_next)
 
+#define INVALID_LOAD_ADDRESS -1L
+#define ZERO_LOAD_ADDRESS 0x0L
+
 // Calculate the load address of shared library
 // on prelink-enabled environment.
 //
@@ -884,7 +887,7 @@ static uintptr_t calc_prelinked_load_address(struct ps_prochandle* ph, int lib_f
   phbuf = read_program_header_table(lib_fd, elf_ehdr);
   if (phbuf == NULL) {
     print_debug("can't read program header of shared object\n");
-    return 0L;
+    return INVALID_LOAD_ADDRESS;
   }
 
   // Get the address of .dynamic section from shared library.
@@ -900,7 +903,7 @@ static uintptr_t calc_prelinked_load_address(struct ps_prochandle* ph, int lib_f
   if (ps_pdread(ph, (psaddr_t)link_map_addr + LINK_MAP_LD_OFFSET,
                &lib_ld, sizeof(uintptr_t)) != PS_OK) {
     print_debug("can't read address of dynamic section in shared object\n");
-    return 0L;
+    return INVALID_LOAD_ADDRESS;
   }
 
   // Return the load address which is calculated by the address of .dynamic
@@ -1011,9 +1014,9 @@ static bool read_shared_lib_info(struct ps_prochandle* ph) {
             // continue with other libraries...
          } else {
             if (read_elf_header(lib_fd, &elf_ehdr)) {
-               if (lib_base_diff == 0x0L) {
+               if (lib_base_diff == ZERO_LOAD_ADDRESS) {
                  lib_base_diff = calc_prelinked_load_address(ph, lib_fd, &elf_ehdr, link_map_addr);
-                 if (lib_base_diff == 0x0L) {
+                 if (lib_base_diff == INVALID_LOAD_ADDRESS) {
                    close(lib_fd);
                    return false;
                  }