Test to verify preservation of checksum configuration after gprecoverseg

Add a behave test to verify that the checksum configuration is preserved
after a segment is recovered using gprecoveseg.
Signed-off-by: NShoaib Lari <slari@pivotal.io>

gpMgmt/test/behave/mgmt_utils/gprecoverseg.feature

@@ -177,3 +177,24 @@ Feature: gprecoverseg tests
         Then gprecoverseg should return a return code of 0
         Then all the segments are running
         And the segments are synchronized
+
+    @multinode
+    @gprecoverseg_checksums
+    Scenario: gprecoverseg should use the same setting for data_checksums for a full recovery
+        Given the database is running
+        And results of the sql "show data_checksums" db "template1" are stored in the context
+        # cause a full recovery AFTER an injected failure on a remote primary
+        And all the segments are running
+        And the segments are synchronized
+        And the information of a "mirror" segment on a remote host is saved
+        And the information of the corresponding primary segment on a remote host is saved
+        And user runs the command "gpfaultinjector  -f filerep_consumer  -m async -y reset" with the saved "primary" segment option
+        And user runs the command "gpfaultinjector  -f filerep_consumer  -m async -y fault" with the saved "primary" segment option
+        Then the saved mirror segment is marked down in config
+        And the saved mirror segment process is still running on that host
+        And user can start transactions
+        When the user runs "gprecoverseg -F -a"
+        Then gprecoverseg should return a return code of 0
+        And all the segments are running
+        # validate the the new segment has the correct setting by getting admin connection to that segment
+        Then the saved primary segment reports the same value for sql "show data_checksums" db "template1" as was saved
\ No newline at end of file

gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py

@@ -2436,6 +2436,15 @@ def impl(context, tname, dbname):
         curs = dbconn.execSQL(conn, sql)
         context.stored_rows = curs.fetchall()
 
+@given('results of the sql "{sql}" db "{dbname}" are stored in the context')
+@when( 'results of the sql "{sql}" db "{dbname}" are stored in the context')
+def impl(context, sql, dbname):
+    context.stored_sql_results = []
+
+    with dbconn.connect(dbconn.DbURL(dbname=dbname)) as conn:
+        curs = dbconn.execSQL(conn, sql)
+        context.stored_sql_results = curs.fetchall()
+
 
 @then('validate that "{dataline}" "{formatter}" seperated by "{delim}" is in the stored rows')
 def impl(context, dataline, formatter, delim):

gpMgmt/test/behave/mgmt_utils/steps/recoverseg_mgmt_utils.py

@@ -5,18 +5,23 @@ from gppylib.commands import gp
 from gppylib.gparray import GpArray
 from test.behave_utils.utils import *
 import platform
+from behave import given, when, then
 
+
+#  todo ONLY implemented for a mirror; change name of step?
 @given('the information of a "{seg}" segment on a remote host is saved')
 @when('the information of a "{seg}" segment on a remote host is saved')
 @then('the information of a "{seg}" segment on a remote host is saved')
 def impl(context, seg):
     if seg == "mirror":
         gparray = GpArray.initFromCatalog(dbconn.DbURL())
-        mirror_segs = [seg for seg in gparray.getDbList() if seg.isSegmentMirror() and seg.getSegmentHostName() != platform.node()]
+        mirror_segs = [seg for seg in gparray.getDbList()
+                       if seg.isSegmentMirror() and seg.getSegmentHostName() != platform.node()]
         context.remote_mirror_segdbId = mirror_segs[0].getSegmentDbId()
         context.remote_mirror_segcid = mirror_segs[0].getSegmentContentId()
-        context.remote_mirror_segdbname = mirror_segs[0].getSegmentHostName()
+        context.remote_mirror_seg_host = mirror_segs[0].getSegmentHostName()
         context.remote_mirror_datadir = mirror_segs[0].getSegmentDataDirectory()
+        context.remote_mirror_seg_port = mirror_segs[0].getSegmentPort()
 
 @given('wait until the segment state of the corresponding primary goes in ChangeTrackingDisabled')
 @when('wait until the segment state of the corresponding primary goes in ChangeTrackingDisabled')
@@ -52,6 +57,7 @@ def impl(context):
             context.remote_pair_primary_port = seg.getSegmentPort()
             context.remote_pair_primary_host = seg.getSegmentHostName()
 
+
 @given('user runs the command "{cmd}" with the saved "{seg}" segment option')
 @when('user runs the command "{cmd}" with the saved "{seg}" segment option')
 @then('user runs the command "{cmd}" with the saved "{seg}" segment option')
@@ -69,20 +75,20 @@ def impl(context, cmd, seg):
 @then('the saved mirror segment process is still running on that host')
 def impl(context):
     cmd = """ps ux | grep "/bin/postgres \-D %s " | grep -v grep""" % (context.remote_mirror_datadir)
-    cmd=Command(name='user command', cmdStr=cmd, ctxt=REMOTE, remoteHost=context.remote_mirror_segdbname)
+    cmd=Command(name='user command', cmdStr=cmd, ctxt=REMOTE, remoteHost=context.remote_mirror_seg_host)
     cmd.run(validateAfter=True)
     res = cmd.get_results()
     if not res.stdout.strip():
-        raise Exception('Mirror segment "%s" not active on "%s"' % (context.remote_mirror_datadir, context.remote_mirror_segdbname))
+        raise Exception('Mirror segment "%s" not active on "%s"' % (context.remote_mirror_datadir, context.remote_mirror_seg_host))
     
 @given('the saved mirror segment is marked down in config')
 @when('the saved mirror segment is marked down in config')
 @then('the saved mirror segment is marked down in config')
 def impl(context):
-    qry = """select count(*) from gp_segment_configuration where status='d' and hostname='%s' and dbid=%s""" % (context.remote_mirror_segdbname, context.remote_mirror_segdbId)
+    qry = """select count(*) from gp_segment_configuration where status='d' and hostname='%s' and dbid=%s""" % (context.remote_mirror_seg_host, context.remote_mirror_segdbId)
     row_count = getRows('template1', qry)[0][0]
     if row_count != 1:
-        raise Exception('Expected mirror segment %s on host %s to be down, but it is running.' % (context.remote_mirror_datadir, context.remote_mirror_segdbname))
+        raise Exception('Expected mirror segment %s on host %s to be down, but it is running.' % (context.remote_mirror_datadir, context.remote_mirror_seg_host))
 
 @given('the mirror with content id "{cid}" is marked down in config')
 @when('the mirror with content id "{cid}" is marked down in config')
@@ -160,6 +166,17 @@ def impl(context, cid):
     '''
     runCommandOnRemoteSegment(context, cid, remove_extra_tid_entry_sql)
 
+
+@then('the saved primary segment reports the same value for sql "{sql_cmd}" db "{dbname}" as was saved')
+def impl(context, sql_cmd, dbname):
+    psql_cmd = "PGDATABASE=\'%s\' PGOPTIONS=\'-c gp_session_role=utility\' psql -t -h %s -p %s -c \"%s\"; " % (
+        dbname, context.remote_pair_primary_host, context.remote_pair_primary_port, sql_cmd)
+    cmd = Command(name='Running Remote command: %s' % psql_cmd, cmdStr = psql_cmd)
+    cmd.run(validateAfter=True)
+    if [cmd.get_results().stdout.strip()] not in context.stored_sql_results:
+        raise Exception("cmd results do not match\n expected: '%s'\n received: '%s'" % (
+            context.stored_sql_results, cmd.get_results().stdout.strip()))
+
 def isSegmentUp(context, dbid):
     qry = """select count(*) from gp_segment_configuration where status='d' and dbid=%s""" % dbid
     row_count = getRows('template1', qry)[0][0]