gpexpand: exclude master-only tables from the template

Gpexpand creates new primary segments by first creating a template from
the master datadir and then copying it to the new segments.  Some
catalog tables are only meaningful on master, such as
gp_segment_configuration, their content are then cleared on each new
segment with the "delete from ..." commands.

This works but is slow because we have to include the content of the
master-only tables in the archive, distribute them via network, and
clear them via the slow "delete from ..." commands -- the "truncate"
command is fast but it is disallowed on catalog tables as filenode must
not be changed for catalog tables.

To make it faster we now exclude these tables from the template
directly, so less data are transferred and there is no need to "delete
from" them explicitly.

gpMgmt/bin/gpexpand

@@ -8,6 +8,7 @@ from gppylib.mainUtils import getProgramName
 
 import copy
 import datetime
+import glob
 import os
 import random
 import sys
@@ -626,8 +627,9 @@ class SegmentTemplate:
         """Builds segment template tar file"""
         self.statusLogger.set_status('BUILD_SEGMENT_TEMPLATE_STARTED', self.tempDir)
         # build segment template should consider tablespace files
-        self._create_template(newTableSpaceInfo)
-        self._fixup_template()
+        excludes = self._list_master_only_files()
+        self._create_template(newTableSpaceInfo, excludes=excludes)
+        self._fixup_template(excludes=excludes)
         self._tar_template()
         self.statusLogger.set_status('BUILD_SEGMENT_TEMPLATE_DONE')
 
@@ -640,7 +642,108 @@ class SegmentTemplate:
         numNewSegments = len(self.gparray.getExpansionSegDbList())
         self.statusLogger.set_status('BUILD_SEGMENTS_DONE', numNewSegments)
 
-    def _create_template(self, newTableSpaceInfo=None):
+    def _list_master_only_files(self):
+        """Build a list of files of master only tables and their indexes.
+
+        The content of master only tables should be cleared or refilled on the
+        new segments, by including an empty copy of the heap & index files less
+        data is transferred via network and there is no need to delete the
+        content on the segments explicitly.
+
+        However empty index files are actually invalid, we need to reindex the
+        master only tables on the new segments to rebuild the index files, this
+        is fast as the heap files are all empty.
+        """
+
+        self.logger.info("Building the file list of master only tables")
+
+        regclasses = ["'%s'::regclass::oid" % tab
+                      for tab in MASTER_ONLY_TABLES]
+        regclasses = "(" + ", ".join(regclasses) + ")"
+        self.logger.debug("master only tables: %s" % regclasses)
+
+        global_sql = """
+/* relation files */
+SELECT pg_catalog.pg_relation_filepath(c.oid)
+  FROM pg_catalog.pg_class c
+ WHERE c.oid IN %s
+   AND c.relfilenode = 0
+
+UNION ALL
+
+/* index files */
+SELECT pg_catalog.pg_relation_filepath(i.indexrelid)
+  FROM pg_catalog.pg_index i
+  JOIN pg_catalog.pg_class c
+    ON i.indexrelid = c.oid
+ WHERE i.indrelid IN %s
+   AND c.relfilenode = 0
+""" % (regclasses, regclasses)
+
+        per_db_sql = """
+/* relation files */
+SELECT pg_catalog.pg_relation_filepath(c.oid)
+  FROM pg_catalog.pg_class c
+ WHERE c.oid IN %s
+   AND c.relfilenode <> 0
+""" % (regclasses)
+
+        result = []
+
+        def add_paths(paths):
+            """we need to include not only the relation/index files,
+            but also visibility, free space and seg files"""
+            for path in paths:
+                # ${filenode}: the relation file itself
+                result.append(os.path.join('.', path))
+                # ${filenode}_vm: the visibility map
+                # ${filenode}_fsm: the free space map
+                result.extend(glob.glob('./%s_*' % path))
+                # ${filenode}.[1-9][0-9]*: the seg files
+                result.extend(glob.glob('./%s.*' % path))
+
+        # the file list will be passed to pg_basebackup as the exclude list,
+        # it expects the paths are like "./global/1234", so we must chdir to
+        # the master data dir.
+        oldcwd = os.getcwd()
+        os.chdir(self.masterDataDirectory)
+
+        # first list the global master-only tables
+        with dbconn.connect(self.dburl, encoding='UTF8') as conn:
+            paths = [str(row[0])
+                     for row in dbconn.execSQL(conn, global_sql).fetchall()]
+            self.logger.debug("raw files of global master-only tables: %s" % paths)
+            add_paths(paths)
+
+            # also get a list of all the databases
+            databases = [str(row[0])
+                         for row in catalog.getDatabaseList(conn)]
+            self.logger.debug("list of databases: %s" % databases)
+
+        # then list the per-database master-only tables
+        for database in databases:
+            # template0 does not accept connections, so we have no chance to
+            # clear the master-only tables on it, however we expect this
+            # database to be small enough, so it can be safely ignored.
+            if database == 'template0':
+                continue
+
+            dburl = dbconn.DbURL( hostname=self.dburl.pghost
+                                , port=self.dburl.pgport
+                                , dbname=database)
+            with dbconn.connect(dburl, encoding='UTF8') as conn:
+                paths = [str(row[0])
+                         for row in dbconn.execSQL(conn, per_db_sql).fetchall()]
+                self.logger.debug("raw files of per-database master-only tables: %s" % paths)
+                add_paths(paths)
+
+        self.logger.debug("files of master only tables: %s" % result)
+
+        os.chdir(oldcwd)
+
+        return result
+
+    def _create_template(self, newTableSpaceInfo=None, excludes=[]):
         """Creates the schema template that is used by new segments"""
         self.logger.info('Creating segment template')
 
@@ -663,6 +766,7 @@ class SegmentTemplate:
                                host=masterSeg.getSegmentHostName(),
                                port=str(masterSeg.getSegmentPort()),
                                recovery_mode=False,
+                               excludePaths=excludes,
                                target_gp_dbid=dummyDBID)
             cmd.run(validateAfter=True)
         except Exception, msg:
@@ -673,9 +777,11 @@ class SegmentTemplate:
         # then there are no user-created tablespaces in the system,
         # no need to consider tablespace problems in gpexpand.
         if newTableSpaceInfo:
-            self._handle_tablespace_template(dummyDBID, newTableSpaceInfo)
+            self._handle_tablespace_template(dummyDBID, newTableSpaceInfo,
+                                             excludes)
 
-    def _handle_tablespace_template(self, dummyDBID, newTableSpaceInfo):
+    def _handle_tablespace_template(self, dummyDBID, newTableSpaceInfo,
+                                    excludes=[]):
         """
         If there are user-created tablespaces in GreenplumDB cluster, we
         have to pack them into the template. The logic here contains two
@@ -714,16 +820,43 @@ class SegmentTemplate:
                                                "dumps")
         os.mkdir(tablespace_template_dir)
 
+        # build a list of master-only files that are under tablespaces, these
+        # files will be ignored during the dumping.
+        full_excludes = []
+        for pathname in excludes:
+            if not pathname.startswith('./pg_tblspc/'):
+                # only tablespaces are handled here
+                continue
+            # a pathname is like './pg_tblspc/16385/GPDB_7_301911081/16386/6052_vm'
+            # we need to change it to './16385/<dummyDBID>/GPDB_7_301911081/16386/6052_vm'
+            pathname = os.sep.join([pathname.split(os.sep)[2], str(dummyDBID)] +
+                                   pathname.split(os.sep)[3:])
+            full_excludes.append(os.path.join(tablespace_template_dir,
+                                              pathname))
+
         for tbcspc_oid in tbcspc_oids:
             symlink_path = os.path.join(master_tblspc_dir, tbcspc_oid)
             target_path = os.readlink(symlink_path)
             os.mkdir(os.path.join(tablespace_template_dir, tbcspc_oid))
             # the target name for copytree does not impact anything
             shutil.copytree(target_path,
-                        os.path.join(tablespace_template_dir, tbcspc_oid, str(dummyDBID)))
+                            os.path.join(tablespace_template_dir,
+                                         tbcspc_oid, str(dummyDBID)),
+                            shutil.ignore_patterns(full_excludes))
             shutil.rmtree(os.path.join(os.path.dirname(target_path),
                                        str(dummyDBID)))
 
+        # the files of the master only tables are already deleted, now add an
+        # empty copy of them, so they are seen as empty on the new segments,
+        # and we do not need to delete them via sql separately.
+        for fullname in full_excludes:
+            dirname = os.path.dirname(fullname)
+            # the template dir is not expected to be updated concurrently,
+            # so it is safe to use a check-and-create style to create dirs.
+            if not os.path.isdir(dirname):
+                os.makedirs(dirname)
+            open(fullname, 'wb')
+
         with open(os.path.join(self.tempDir,
                                "pg_tblspc",
                                "newTableSpaceInfo.json"), "w") as f:
@@ -847,7 +980,7 @@ class SegmentTemplate:
         self._start_new_primary_segments()
         self._stop_new_primary_segments()
 
-    def _fixup_template(self):
+    def _fixup_template(self, excludes=[]):
         """Copies postgresql.conf and pg_hba.conf files from a valid segment on the system.
         Then modifies the template copy of pg_hba.conf"""
 
@@ -869,6 +1002,27 @@ class SegmentTemplate:
                     remoteHost=self.srcSegHostname)
         cpCmd.run(validateAfter=True)
 
+        # the files of the master only tables are already deleted, now add an
+        # empty copy of them, so they are seen as empty on the new segments,
+        # and we do not need to delete them via sql separately.
+        if excludes:
+            self.logger.info('Creating an empty copy of excluded files')
+            for pathname in excludes:
+                if pathname.startswith('./pg_tblspc/'):
+                    # tablespaces are handled separately
+                    continue
+                fullname = os.path.join(self.tempDir, pathname)
+                filename = os.path.basename(fullname)
+                dirname = os.path.dirname(fullname)
+                if '.' in filename:
+                    # ignore seg files
+                    continue
+                # the template dir is not expected to be updated concurrently,
+                # so it is safe to use a check-and-create style to create dirs.
+                if not os.path.isdir(dirname):
+                    os.makedirs(dirname)
+                open(fullname, 'wb')
+
     def _tar_template(self):
         """Tars up the template files"""
         self.logger.info('Creating schema tar file')
@@ -1369,8 +1523,15 @@ class gpexpand:
         conn.close()
 
         """
-        Connect to each database in each segment and do some cleanup of tables that have stuff in them as a result of copying the segment from the master.
-        Note, this functionality used to be in segcopy and was therefore done just once to the original copy of the master.
+        Connect to each database in each segment and do some cleanup of tables
+        that have stuff in them as a result of copying the segment from the
+        master.  Note, this functionality used to be in segcopy and was
+        therefore done just once to the original copy of the master.
+
+        Need to start the new segments with the system indexes disabled, this
+        is necessary because the master-only catalog tables are copied as empty
+        files, as well as their index files, the indexes are invalid and should
+        not be used until being reindexed.
         """
         for seg in newSegments:
             if seg.isSegmentMirror() == True:
@@ -1386,16 +1547,18 @@ class gpexpand:
                 , ctxt=REMOTE
                 , remoteHost=seg.getSegmentHostName()
                 , pg_ctl_wait=True
-                , timeout=SEGMENT_TIMEOUT_DEFAULT)
+                , timeout=SEGMENT_TIMEOUT_DEFAULT
+                , disableSystemIndexes=True)
             self.pool.addCommand(segStartCmd)
         self.pool.join()
         self.pool.check_results()
 
         """
-        Build the list of delete statements based on the MASTER_ONLY_TABLES
+        Build the list of reindex statements based on the MASTER_ONLY_TABLES
         defined in gpcatalog.py
         """
-        statements = ["delete from pg_catalog.%s" % tab for tab in MASTER_ONLY_TABLES]
+        statements = ["REINDEX TABLE pg_catalog.%s" % tab
+                      for tab in MASTER_ONLY_TABLES]
 
         """
           Connect to each database in the new segments, and clean up the catalog tables.
@@ -1423,6 +1586,43 @@ class gpexpand:
         self.pool.check_results()
         self.pool.getCompletedItems()
 
+        # stop the segments
+        for seg in newSegments:
+            if seg.isSegmentMirror():
+                continue
+            """ Stop all the new segments. """
+            segStartCmd = SegmentStop(
+                name="Stopping new segment dbid %s on host %s." % (str(seg.getSegmentDbId()),
+                                                                   seg.getSegmentHostName())
+                , dataDir=seg.getSegmentDataDirectory()
+                , ctxt=REMOTE
+                , remoteHost=seg.getSegmentHostName()
+                , timeout=SEGMENT_TIMEOUT_DEFAULT)
+            self.pool.addCommand(segStartCmd)
+        self.pool.join()
+        self.pool.check_results()
+
+        # restart the segments with index enabled
+        for seg in newSegments:
+            if seg.isSegmentMirror() == True:
+                continue
+            """ Start all the new segments in utilty mode. """
+            segStartCmd = SegmentStart(
+                name="Starting new segment dbid %s on host %s." % (str(seg.getSegmentDbId()),
+                                                                   seg.getSegmentHostName())
+                , gpdb=seg
+                , numContentsInCluster=self.newPrimaryCount  # Starting seg on it's own.
+                , era=None
+                , mirrormode=MIRROR_MODE_MIRRORLESS
+                , utilityMode=True
+                , ctxt=REMOTE
+                , remoteHost=seg.getSegmentHostName()
+                , pg_ctl_wait=True
+                , timeout=SEGMENT_TIMEOUT_DEFAULT)
+            self.pool.addCommand(segStartCmd)
+        self.pool.join()
+        self.pool.check_results()
+
     # --------------------------------------------------------------------------
     def restore_master(self):
         """Restores the gp_segment_configuration catalog table for rollback"""

gpMgmt/bin/gppylib/commands/gp.py

@@ -210,6 +210,13 @@ class PgCtlBackendOptions(CmdArgs):
         if restricted:  self.append("-c superuser_reserved_connections=%s" % max_connections)
         return self
 
+    def set_disable_system_indexes(self, disable):
+        """
+        @param disable: true if disabling system indexes
+        """
+        if disable: self.append("-P")
+        return self
+
 
 class PgCtlStartArgs(CmdArgs):
     """
@@ -337,7 +344,8 @@ class SegmentStart(Command):
     def __init__(self, name, gpdb, numContentsInCluster, era, mirrormode,
                  utilityMode=False, ctxt=LOCAL, remoteHost=None,
                  pg_ctl_wait=True, timeout=SEGMENT_TIMEOUT_DEFAULT,
-                 specialMode=None, wrapper=None, wrapper_args=None):
+                 specialMode=None, wrapper=None, wrapper_args=None,
+                 disableSystemIndexes=False):
 
         # This is referenced from calling code
         self.segment = gpdb
@@ -350,6 +358,7 @@ class SegmentStart(Command):
         b = PgCtlBackendOptions(port)
         b.set_utility(utilityMode)
         b.set_special(specialMode)
+        b.set_disable_system_indexes(disableSystemIndexes)
 
         # build pg_ctl command
         c = PgCtlStartArgs(datadir, b, era, wrapper, wrapper_args, pg_ctl_wait, timeout)

gpMgmt/test/behave/mgmt_utils/gpexpand.feature

@@ -84,6 +84,7 @@ Feature: expand the cluster by adding more segments
         Then the number of segments have been saved
         When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent"
         Then verify that the cluster has 2 new segments
+        And verify that the master-only tables are empty on one new segment
 
     @gpexpand_no_mirrors
     @gpexpand_host
@@ -100,6 +101,7 @@ Feature: expand the cluster by adding more segments
         Then the number of segments have been saved
         When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent"
         Then verify that the cluster has 2 new segments
+        And verify that the master-only tables are empty on one new segment
 
     @gpexpand_no_mirrors
     @gpexpand_host_and_segment
@@ -116,6 +118,7 @@ Feature: expand the cluster by adding more segments
         Then the number of segments have been saved
         When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent"
         Then verify that the cluster has 4 new segments
+        And verify that the master-only tables are empty on one new segment
 
     @gpexpand_mirrors
     @gpexpand_segment
@@ -130,6 +133,7 @@ Feature: expand the cluster by adding more segments
         And the number of segments have been saved
         When the user runs gpexpand with a static inputfile for a single-node cluster with mirrors
         Then verify that the cluster has 4 new segments
+        And verify that the master-only tables are empty on one new segment
 
     @gpexpand_mirrors
     @gpexpand_segment
@@ -166,6 +170,7 @@ Feature: expand the cluster by adding more segments
         Then the number of segments have been saved
         When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent"
         Then verify that the cluster has 8 new segments
+        And verify that the master-only tables are empty on one new segment
 
     @gpexpand_mirrors
     @gpexpand_host_and_segment
@@ -184,6 +189,7 @@ Feature: expand the cluster by adding more segments
         Then the number of segments have been saved
         When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent"
         Then verify that the cluster has 14 new segments
+        And verify that the master-only tables are empty on one new segment
 
     @gpexpand_mirrors
     @gpexpand_host_and_segment

gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py

@@ -30,6 +30,7 @@ from time import sleep
 from os import path
 
 from gppylib.gparray import GpArray, ROLE_PRIMARY, ROLE_MIRROR
+from gppylib.gpcatalog import MASTER_ONLY_TABLES
 from gppylib.commands.gp import SegmentStart, GpStandbyStart, MasterStop
 from gppylib.commands import gp
 from gppylib.commands.unix import findCmdInPath, Scp
@@ -2388,6 +2389,31 @@ def impl(context, num_of_segments):
             "%s\ndump of gp_segment_configuration: %s" %
             (context.start_data_segments, end_data_segments, rows))
 
+@then('verify that the master-only tables are empty on one new segment')
+def impl(context):
+    dbname = 'gptest'
+
+    # lookup the port of the newest primary segment
+    with dbconn.connect(dbconn.DbURL(dbname=dbname),
+                        unsetSearchPath=False) as conn:
+        query = """SELECT address, port
+                     FROM gp_segment_configuration
+                    WHERE role = 'p'
+                    ORDER BY content DESC
+                    LIMIT 1;"""
+        row = dbconn.execSQLForSingletonRow(conn, query)
+        address = str(row[0])
+        port = int(row[1])
+
+    # verify that all the master-only tables are empty on this new segment
+    with dbconn.connect(dbconn.DbURL(dbname=dbname, hostname=address, port=port),
+                        unsetSearchPath=False, utility=True) as conn:
+        for tab in MASTER_ONLY_TABLES:
+            query = "SELECT count(*) FROM %s;" % tab
+            count = int(dbconn.execSQLForSingleton(conn, query))
+            if count > 0:
+                raise Exception("Master-only table '%s' is not empty on the new segments" % tab)
+
 @given('the cluster is setup for an expansion on hosts "{hostnames}"')
 def impl(context, hostnames):
     hosts = hostnames.split(",")