gpstate: fix getting version information

We no longer have a filerep-based API to be able to retrieve a mirror's
version. Instead, have the postmaster append a known string
(POSTMASTER_MIRROR_VERSION_DETAIL_MSG) to the detail message when a
client attempts to connect to a mirror. gpstate will then look for this
string to determine a mirror's version. (This is similar to the current
practice of returning replication state in the detail message.)
Co-authored-by: NMark Sliva <msliva@pivotal.io>
Co-authored-by: NJacob Champion <pchampion@pivotal.io>

gpMgmt/sbin/gpgetstatususingtransition.py

@@ -10,6 +10,7 @@ from gppylib.mainUtils import *
 import os, sys
 
 import pickle, base64
+import re
 
 from optparse import Option, OptionGroup, OptionParser, OptionValueError
 
@@ -28,6 +29,11 @@ PQPING_NO_RESPONSE = 2
 PQPING_NO_ATTEMPT = 3
 PQPING_MIRROR_READY = 64
 
+# When attempting to connect to a mirror, the postmaster will report the
+# software version after a short prefix. This is that prefix, which must match
+# that in <postmaster/postmaster.h>.
+POSTMASTER_MIRROR_VERSION_DETAIL_MSG = "- VERSION:"
+
 def _get_segment_status(segment):
     cmd = base.Command('pg_isready for segment',
                        "pg_isready -q -h %s -p %d" % (segment.hostname, segment.port))
@@ -52,6 +58,39 @@ def _get_segment_status(segment):
 
     return None
 
+# Used by _get_segment_version() to find the version string for a mirror.
+_version_regex = re.compile(r'%s (.*)' % POSTMASTER_MIRROR_VERSION_DETAIL_MSG)
+
+def _get_segment_version(seg):
+    try:
+        if seg.role == gparray.ROLE_PRIMARY:
+            dburl = dbconn.DbURL(hostname=seg.hostname, port=seg.port, dbname="template1")
+            conn = dbconn.connect(dburl, utility=True)
+            return dbconn.execSQLForSingleton(conn, "select version()")
+
+        if seg.role == gparray.ROLE_MIRROR:
+            cmd = base.Command("Try connecting to mirror",
+                               "psql -h %s -p %s template1 -c 'select 1'"
+                               %(seg.hostname, seg.port))
+            cmd.run(validateAfter=False)
+            if cmd.results.rc == 0:
+                raise RuntimeError("Connection to mirror succeeded unexpectedly")
+
+            stderr = cmd.results.stderr.splitlines()
+            for line in stderr:
+                match = _version_regex.match(line)
+                if match:
+                    return match.group(1)
+
+            raise RuntimeError("Unexpected error from mirror connection: %s" % cmd.results.stderr)
+
+        logger.error("Invalid role '%s' for dbid %d", seg.role, seg.dbid)
+        return None
+
+    except Exception as ex:
+        logger.error("Could not get segment version for dbid %d", seg.dbid, exc_info=ex)
+        return None
+
 #
 # todo: the file containing this should be renamed since it gets more status than just from transition
 #
@@ -136,9 +175,7 @@ class GpSegStatusProgram:
             for statusRequest in toFetch:
                 data = None
                 if statusRequest == gp.SEGMENT_STATUS__GET_VERSION:
-#                    data = self.getStatusUsingTransition(seg, statusRequest, pidRunningStatus)
-                    if data is not None:
-                        data = data.rstrip()
+                    data = _get_segment_version(seg)
 
                 elif statusRequest == gp.SEGMENT_STATUS__GET_MIRROR_STATUS:
                     data = _get_segment_status(seg)

src/backend/postmaster/postmaster.c

@@ -2619,8 +2619,10 @@ retry1:
 			ereport(FATAL,
 					(errcode(ERRCODE_MIRROR_READY),
 					 errmsg(POSTMASTER_IN_RECOVERY_MSG),
-					 errdetail(POSTMASTER_IN_RECOVERY_DETAIL_MSG " %X/%X",
-						   (uint32) (recptr >> 32), (uint32) recptr)));
+					 errdetail(POSTMASTER_IN_RECOVERY_DETAIL_MSG " %X/%X\n"
+							   POSTMASTER_MIRROR_VERSION_DETAIL_MSG " %s",
+							   (uint32) (recptr >> 32), (uint32) recptr,
+							   TextDatumGetCString(pgsql_version(NULL)))));
 			break;
 		case CAC_OK:
 			break;

src/include/postmaster/postmaster.h

@@ -48,6 +48,8 @@ extern int	postmaster_alive_fds[2];
 #define POSTMASTER_IN_STARTUP_MSG "the database system is starting up"
 #define POSTMASTER_IN_RECOVERY_MSG "the database system is in recovery mode"
 #define POSTMASTER_IN_RECOVERY_DETAIL_MSG "last replayed record at"
+/* gpstate must be updated if this message changes */
+#define POSTMASTER_MIRROR_VERSION_DETAIL_MSG "- VERSION:"
 
 extern const char *progname;
 extern PGDLLIMPORT const char *progname;