gpcheckcat: add check for orphaned TOAST tables

There are four main types of orphan TOAST tables:
- Double Orphan TOAST tables due to a missing reltoastrelid in pg_class and a
  missing pg_depend entry.
- Bad Reference Orphaned TOAST tables due to a missing reltoastrelid in
  pg_class.
- Bad Dependency Orphaned TOAST tables due to a missing entry in pg_depend.
- Mismatch Orphaned TOAST tables due to reltoastrelid in pg_class pointing to
  an incorrect TOAST table.

Repair scripts are done on a per-segment basis since catalog changes are
required.

Note: A repair script cannot be generated for mismatch orphan TOAST tables
because the current repair implmentation does not work with two or more tables,
so let the user repair manually for safety. A manual catalogue change is needed
to fix by updating the pg_depend TOAST table entry and setting the refobjid
field to the correct dependent table. Similarly, we do not attempt to
repair double-orphan situations.
Authored-by: NKalen Krempely <kkrempely@pivotal.io>
Co-Authored-by: NMark Sliva <msliva@pivotal.io>
Co-Authored-by: NJamie McAtamney <jmcatamney@pivotal.io>
Co-authored-by: NJimmy Yih <jyih@pivotal.io>

gpMgmt/bin/gpcheckcat

@@ -36,6 +36,7 @@ try:
     from gpcheckcat_modules.leaked_schema_dropper import LeakedSchemaDropper
     from gpcheckcat_modules.repair import Repair
     from gpcheckcat_modules.foreign_key_check import ForeignKeyCheck
+    from gpcheckcat_modules.orphaned_toast_tables_check import OrphanedToastTablesCheck, OrphanToastTableIssue, OrphanedTable
 
 
 except ImportError, e:
@@ -2908,6 +2909,59 @@ def log_unique_index_violations(violations):
     logger.error('\n'.join(log_output))
 
 
+# -------------------------------------------------------------------------------
+def checkOrphanedToastTables():
+    logger.info('-----------------------------------')
+    logger.info('Performing check: checking for orphaned TOAST tables')
+
+    db_connection = connect2(GV.cfg[1], utilityMode=False)
+    checker = OrphanedToastTablesCheck()
+    check_passed = checker.runCheck(db_connection)
+
+    checkname = 'orphaned toast table(s)'
+    if check_passed:
+        logger.info('[OK]  %s' % checkname)
+    else:
+        logger.info('[FAIL]  %s' % checkname)
+        GV.checkStatus = False
+        setError(ERROR_REMOVE)
+
+        # log raw orphan toast table query results only to the log file
+        log_output = ['Orphan toast tables detected for the following scenarios:']
+        for issue_type in checker.issue_types():
+            log_output.append('\n' + issue_type.header)
+            log_output.append('content_id | toast_table_oid | toast_table_name | expected_table_oid | expected_table_name | dependent_table_oid | dependent_table_name')
+            log_output.append('-----------+-----------------+------------------+--------------------+---------------------+---------------------+---------------------')
+            for row in checker.rows_of_type(issue_type):
+                log_output.append('{} | {} | {} | {} | {} | {} | {}'.format(
+                    row['content_id'],
+                    row['toast_table_oid'],
+                    row['toast_table_name'],
+                    row['expected_table_oid'],
+                    row['expected_table_name'],
+                    row['dependent_table_oid'],
+                    row['dependent_table_name']))
+        log_output = '\n'.join(log_output)
+        log_output = log_output + '\n'
+        logger.error(log_output)
+
+        # log fix instructions for the orphaned toast tables to stdout and log file
+        gplog.log_literal(logger, logging.CRITICAL, checker.get_fix_text())
+
+        # log per-orphan table issue and cause to stdout and log file
+        for issue, segments in checker.iterate_issues():
+            cat_issue_obj = CatOrphanToastTableIssue(issue.table.oid,
+                                                     issue.table.catname,
+                                                     issue,
+                                                     segments)
+            error_object = getGPObject(issue.table.oid, issue.table.catname)
+            error_object.addOrphanToastTableIssue(cat_issue_obj)
+
+        do_repair_for_segments(segments_to_repair_statements=checker.add_repair_statements(GV.cfg),
+                               issue_type="orphaned_toast_tables",
+                               description='Repairing orphaned TOAST tables')
+
+
 ############################################################################
 # Help populating repair part for all checked types
 # All functions dependent on results stored in global variable GV
@@ -3032,6 +3086,14 @@ all_checks = {
             "order": 13,
             "online": True
         },
+    "orphaned_toast_tables":
+        {
+            "description": "Check pg_class and pg_depend for orphaned TOAST tables",
+            "fn": checkOrphanedToastTables,
+            "version": 'main',
+            "order": 14,
+            "online": True
+        },
 }
 
 
@@ -3142,6 +3204,7 @@ def runAllChecks():
 
 
 def do_repair(sql_repair_contents, issue_type, description):
+    logger.info("Starting repair of %s with %d issues" % (description, len(sql_repair_contents)))
     if len(sql_repair_contents) == 0:
         return
 
@@ -3154,6 +3217,18 @@ def do_repair(sql_repair_contents, issue_type, description):
 
     print_repair_issues(repair_dir_path)
 
+def do_repair_for_segments(segments_to_repair_statements, issue_type, description):
+    logger.info("Starting repair of %s" % description)
+
+    repair_dir_path = ""
+    try:
+        repair_obj = Repair(context=GV, issue_type=issue_type, desc=description)
+        repair_dir_path = repair_obj.create_segment_repair_scripts(segments_to_repair_statements)
+    except Exception, ex:
+        logger.fatal(str(ex))
+
+    print_repair_issues(repair_dir_path)
+
 
 def do_repair_for_extra(catalog_issues):
     setError(ERROR_REMOVE)
@@ -3163,7 +3238,8 @@ def do_repair_for_extra(catalog_issues):
             repair_obj = Repair(context=GV, issue_type="extra", desc="Removing extra entries")
             repair_dir_path = repair_obj.create_repair_for_extra_missing(catalog_table_obj=catalog_table_obj,
                                                                          issues=issues,
-                                                                         pk_name=pk_name)
+                                                                         pk_name=pk_name,
+                                                                         segments=GV.cfg)
         except Exception, ex:
             logger.fatal(str(ex))
 
@@ -3803,6 +3879,18 @@ class CatDependencyIssue:
             % (self.catname, self.oid, self.content)
         )
 
+# -------------------------------------------------------------------------------
+class CatOrphanToastTableIssue:
+    def __init__(self, oid, catname, issue, segments):
+        self.oid = oid
+        self.catname = catname
+        self.issue = issue
+        self.segments = segments
+
+    def report(self):
+        myprint('%s' % self.issue.description)
+        myprint('''On segment(s) %s table '%s' (oid: %s) %s''' % (', '.join(map(str, sorted(self.segments))), self.catname, self.oid, self.issue.cause))
+
 # -------------------------------------------------------------------------------
 class GPObject:
     def __init__(self, oid, catname):
@@ -3814,6 +3902,7 @@ class GPObject:
         self.aclIssues = {}  # key=issue.catname, value=list of catACLIssue
         self.foreignkeyIssues = {}  # key=issue.catname, value=list of catForeignKeyIssue
         self.dependencyIssues = {} # key=issue.catname, value=list of catDependencyIssue
+        self.orphanToastTableIssues = {} # key=issue.catname, value=list of orphanToastTableIssues
 
     def addDependencyIssue(self, issue):
         if issue.catname in self.dependencyIssues:
@@ -3851,6 +3940,12 @@ class GPObject:
         else:
             self.foreignkeyIssues[issue.catname] = [issue]
 
+    def addOrphanToastTableIssue(self, issue):
+        if issue.catname in self.orphanToastTableIssues:
+            self.orphanToastTableIssues[issue.catname].append(issue)
+        else:
+            self.orphanToastTableIssues[issue.catname] = [issue]
+
     def isTopLevel(self):
         return True
 
@@ -3943,6 +4038,13 @@ class GPObject:
                     each.report()
             myprint('')
 
+        # Report Orphan Toast Table issues
+        if len(self.orphanToastTableIssues):
+            for catname, issues in self.orphanToastTableIssues.iteritems():
+                for each in issues:
+                    each.report()
+            myprint('')
+
         myprint = _myprint
 
         # Collect all tables with missing issues for later reporting
@@ -4168,7 +4270,7 @@ def checkcatReport():
             reportAllIssuesRecursive(child, graph)
 
     buildGraph()
-    reportedCheck = ['duplicate','missing_extraneous','inconsistent','foreign_key','acl']
+    reportedCheck = ['duplicate','missing_extraneous','inconsistent','foreign_key','acl', 'orphaned_toast_tables']
     myprint('')
     myprint('SUMMARY REPORT')
     myprint('===================================================================')

gpMgmt/bin/gpcheckcat_modules/orphan_toast_table_issues.py

+#!/usr/bin/env python
+
+
+class OrphanToastTableIssue(object):
+    def __init__(self, cause, row, table):
+        self.cause = cause
+        self.row = row
+        self.table = table
+
+    def __eq__(self, other):
+        if isinstance(other, OrphanToastTableIssue):
+            return self.cause == other.cause and self.row == self.row
+        return False
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __repr__(self):
+        return "OrphanToastTableIssue(%s,%s)" % (self.cause, self.row)
+
+    def __hash__(self):
+        return hash(self.__repr__())
+
+    fix_text = []
+    header = ''
+    description = ''
+
+    @property
+    def repair_script(self):
+        return ''
+
+
+class DoubleOrphanToastTableIssue(OrphanToastTableIssue):
+    def __init__(self, row, table):
+        cause = ""
+        if row['double_orphan_parent_reltoastrelid'] is None:
+            cause = '''The parent table does not exist. Therefore, the toast table '%(toast_table_name)s' (oid: %(toast_table_oid)s) can likely be dropped.\n'''
+            cause = cause % dict(toast_table_oid=row['toast_table_oid'],
+                                 toast_table_name=row['toast_table_name'])
+        elif row['double_orphan_parent_oid'] and row['double_orphan_parent_toast_oid']:
+            cause = '''with parent table '%(double_orphan_parent_name)s' (oid: %(double_orphan_parent_oid)s). '''
+            cause += '''The parent table already references a valid toast table '%(double_orphan_parent_toast_name)s' (oid: %(double_orphan_parent_toast_oid)s). '''
+            cause += '''\nTo fix, either:\n  1) drop the orphan toast table '%(toast_table_name)s' (oid: %(toast_table_oid)s)\nor:'''
+            cause += '''\n  2a) drop the pg_depend entry for '%(double_orphan_parent_toast_name)s' (oid: %(double_orphan_parent_toast_oid)s)'''
+            cause += '''\n  2b) drop the toast table '%(double_orphan_parent_toast_name)s' (oid: %(double_orphan_parent_toast_oid)s),'''
+            cause += '''\n  2c) add pg_depend entry where objid is '%(toast_table_name)s' (oid: %(toast_table_oid)s) and refobjid is '%(double_orphan_parent_name)s' (oid: %(double_orphan_parent_oid)s) with deptype 'i'.'''
+            cause += '''\n  2d) update the pg_class entry for '%(double_orphan_parent_name)s' (oid: %(double_orphan_parent_oid)s), and set reltoastrelid to '%(toast_table_name)s' (oid: %(toast_table_oid)s).\n'''
+            cause = cause % dict(toast_table_oid=row['toast_table_oid'],
+                                 toast_table_name=row['toast_table_name'],
+                                 double_orphan_parent_oid=row['double_orphan_parent_oid'],
+                                 double_orphan_parent_name=row['double_orphan_parent_name'],
+                                 double_orphan_parent_toast_oid=row['double_orphan_parent_toast_oid'],
+                                 double_orphan_parent_toast_name=row['double_orphan_parent_toast_name'])
+        elif row['double_orphan_parent_oid'] and row['double_orphan_parent_toast_oid'] is None:
+            cause = '''with parent table '%(double_orphan_parent_name)s' (oid: %(double_orphan_parent_oid)s) has no valid toast table. '''
+            cause += '''Verify that the parent table requires a toast table. To fix:'''
+            cause += '''\n  1) update the pg_class entry for '%(double_orphan_parent_name)s' (oid: %(double_orphan_parent_oid)s) and set reltoastrelid to '%(toast_table_name)s' (oid: %(toast_table_oid)s).'''
+            cause += '''\n  2) add pg_depend entry where objid is '%(toast_table_name)s' (oid: %(toast_table_oid)s) and refobjid is '%(double_orphan_parent_name)s' (oid: %(double_orphan_parent_oid)s) with deptype 'i'.\n'''
+            cause = cause % dict(toast_table_oid=row['toast_table_oid'],
+                                 toast_table_name=row['toast_table_name'],
+                                 double_orphan_parent_oid=row['double_orphan_parent_oid'],
+                                 double_orphan_parent_name=row['double_orphan_parent_name'])
+
+        OrphanToastTableIssue.__init__(self, cause, row, table)
+
+    header = 'Double Orphan TOAST tables due to a missing reltoastrelid in pg_class and a missing pg_depend entry.'
+    fix_text = [
+        header, 
+        '''  A manual catalog change is needed to fix. Attempt to determine the original dependent table's OID from the name of the TOAST table.\n''',
+        '''  If the dependent table has a valid OID and exists, the update its pg_class entry with the correct reltoastrelid and adds a pg_depend entry.\n''',
+        '''  If the dependent table doesn't exist, delete the associated TOAST table.\n''',
+        '''  If the dependent table is invalid, the associated TOAST table has been renamed. A manual catalog change is needed.\n'''
+    ]
+    description = 'Found a "double orphan" orphaned TOAST table caused by missing reltoastrelid and missing pg_depend entry.'
+
+    @property
+    def repair_script(self):
+        # There are multiple ways a double orphan toast table can occur and thus be fixed.
+        # We need DBA input to determine how it is broken in order to safely fix it.
+        # Since, at this time, we cannot get user input, we chose not to generate a repair script
+        # in order to be conservative.
+        return ''
+
+class ReferenceOrphanToastTableIssue(OrphanToastTableIssue):
+    def __init__(self, row, table):
+        cause = '''has an orphaned TOAST table '%s' (OID: %s).''' % (row['toast_table_name'], row['toast_table_oid'])
+        OrphanToastTableIssue.__init__(self, cause, row, table)
+
+    header = 'Bad Reference Orphaned TOAST tables due to a missing reltoastrelid in pg_class'
+    description = 'Found a "bad reference" orphaned TOAST table caused by missing a reltoastrelid in pg_class.'
+    fix_text = [
+        header,
+        '''  To fix, run the generated repair script which updates a pg_class entry using the correct dependent table OID for reltoastrelid.\n'''
+    ]
+
+    @property
+    def repair_script(self):
+        return '''UPDATE "pg_class" SET reltoastrelid = %d WHERE oid = %s;''' % (
+            self.row["toast_table_oid"], self.row["dependent_table_oid"])
+
+
+class DependencyOrphanToastTableIssue(OrphanToastTableIssue):
+    def __init__(self, row, table):
+        cause = '''has an orphaned TOAST table '%s' (OID: %s).'''% (row['toast_table_name'], row['toast_table_oid'])
+        OrphanToastTableIssue.__init__(self, cause, row, table)
+
+    header = 'Bad Dependency Orphaned TOAST tables due to a missing pg_depend entry'
+    description = 'Found a "bad dependency" orphaned TOAST table caused by missing a pg_depend entry.'
+    fix_text = [
+        header,
+        '''  To fix, run the generated repair script which inserts a pg_depend entry using the correct dependent table OID for refobjid.\n'''
+    ]
+
+    @property
+    def repair_script(self):
+        # 1259 is the reserved oid for pg_class and 'i' means internal dependency; these are safe to hard-code
+        return '''INSERT INTO pg_depend VALUES (1259, %d, 0, 1259, %d, 0, 'i');''' % (
+            self.row["toast_table_oid"], self.row["expected_table_oid"])
+
+
+class MismatchOrphanToastTableIssue(OrphanToastTableIssue):
+    def __init__(self, row, table):
+        cause = '''has an orphaned TOAST table '%s' (OID: %s). Expected dependent table to be '%s' (OID: %s).''' % (
+            row['toast_table_name'], row['toast_table_oid'], row['expected_table_name'], row['expected_table_oid'])
+        OrphanToastTableIssue.__init__(self, cause, row, table)
+
+    header = 'Mismatch Orphaned TOAST tables due to reltoastrelid in pg_class pointing to an incorrect TOAST table'
+    description = 'Found a "mismatched" orphaned TOAST table caused by a reltoastrelid in pg_class pointing to an incorrect TOAST table. A manual catalog change is needed.'
+    fix_text = [
+        header,
+        '''  A manual catalog change is needed to fix by updating the pg_depend TOAST table entry and setting the refobjid field to the correct dependent table.\n'''
+    ]
+
+    @property
+    def repair_script(self):
+        # There can be a cyclic reference in reltoastrelid in pg_class which
+        # is difficult to determine the valid toast table (if any).
+        # Therefore, no repair script can safely be generated and
+        # not loose customer data.
+        return ''

gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py

+#!/usr/bin/env python
+
+import itertools
+
+from collections import defaultdict, namedtuple
+from gppylib.db import dbconn
+from gpcheckcat_modules.orphan_toast_table_issues import OrphanToastTableIssue, DoubleOrphanToastTableIssue, ReferenceOrphanToastTableIssue, DependencyOrphanToastTableIssue, MismatchOrphanToastTableIssue
+
+
+OrphanedTable = namedtuple('OrphanedTable', 'oid catname')
+
+
+class OrphanedToastTablesCheck:
+    def __init__(self):
+        self._issues = []
+
+        # Normally, there's a "loop" between a table and its TOAST table:
+        # - The table's reltoastrelid field in pg_class points to its TOAST table
+        # - The TOAST table has an entry in pg_depend pointing to its table
+        # This can break and orphan a TOAST table in one of three ways:
+        # - The reltoastrelid entry is set to 0
+        # - The reltoastrelid entry is set to a different oid value
+        # - The pg_depend entry is missing
+        # - The reltoastrelid entry is wrong *and* the pg_depend entry is missing
+        # The following query attempts to "follow" the loop from pg_class to
+        # pg_depend back to pg_class, and if the table oids don't match and/or
+        # one is missing, the TOAST table is considered to be an orphan.
+        self.orphaned_toast_tables_query = """
+SELECT
+    gp_segment_id AS content_id,
+    toast_table_oid,
+    toast_table_name,
+    expected_table_oid,
+    expected_table_name,
+    dependent_table_oid,
+    dependent_table_name,
+    double_orphan_parent_oid,
+    double_orphan_parent_name,
+    double_orphan_parent_reltoastrelid,
+    double_orphan_parent_toast_oid,
+    double_orphan_parent_toast_name
+FROM (
+    SELECT
+        tst.gp_segment_id,
+        tst.oid AS toast_table_oid,
+        tst.relname AS toast_table_name,
+        tbl.oid AS expected_table_oid,
+        tbl.relname AS expected_table_name,
+        dep.refobjid AS dependent_table_oid,
+        dep.refobjid::regclass::text AS dependent_table_name,
+        dbl.oid AS double_orphan_parent_oid,
+        dbl.relname AS double_orphan_parent_name,
+        dbl.reltoastrelid AS double_orphan_parent_reltoastrelid,
+        dbl_tst.oid AS double_orphan_parent_toast_oid,
+        dbl_tst.relname AS double_orphan_parent_toast_name
+    FROM
+        pg_class tst
+        LEFT JOIN pg_depend dep ON tst.oid = dep.objid
+        LEFT JOIN pg_class tbl ON tst.oid = tbl.reltoastrelid
+        LEFT JOIN pg_class dbl
+            ON trim('pg_toast.pg_toast_' FROM tst.oid::regclass::text)::int::regclass::oid = dbl.oid
+        LEFT JOIN pg_class dbl_tst ON dbl.reltoastrelid = dbl_tst.oid
+    WHERE tst.relkind='t'
+        AND	(
+            tbl.oid IS NULL
+            OR refobjid IS NULL
+            OR tbl.oid != dep.refobjid
+        )
+        AND (
+            tbl.relnamespace IS NULL
+            OR tbl.relnamespace != (SELECT oid FROM pg_namespace WHERE nspname = 'pg_catalog')
+        )
+    UNION ALL
+    SELECT
+        tst.gp_segment_id,
+        tst.oid AS toast_table_oid,
+        tst.relname AS toast_table_name,
+        tbl.oid AS expected_table_oid,
+        tbl.relname AS expected_table_name,
+        dep.refobjid AS dependent_table_oid,
+        dep.refobjid::regclass::text AS dependent_table_name,
+        dbl.oid AS double_orphan_parent_oid,
+        dbl.relname AS double_orphan_parent_name,
+        dbl.reltoastrelid AS double_orphan_parent_reltoastrelid,
+        dbl.reltoastrelid AS double_orphan_parent_toast_oid,
+        dbl_tst.relname AS double_orphan_parent_toast_name
+    FROM gp_dist_random('pg_class') tst
+        LEFT JOIN gp_dist_random('pg_depend') dep ON tst.oid = dep.objid AND tst.gp_segment_id = dep.gp_segment_id
+        LEFT JOIN gp_dist_random('pg_class') tbl ON tst.oid = tbl.reltoastrelid AND tst.gp_segment_id = tbl.gp_segment_id
+        LEFT JOIN gp_dist_random('pg_class') dbl
+            ON trim('pg_toast.pg_toast_' FROM tst.oid::regclass::text)::int::regclass::oid = dbl.oid 
+            AND tst.gp_segment_id = dbl.gp_segment_id
+        LEFT JOIN pg_class dbl_tst ON dbl.reltoastrelid = dbl_tst.oid AND tst.gp_segment_id = dbl_tst.gp_segment_id
+    WHERE tst.relkind='t'
+        AND (
+            tbl.oid IS NULL
+            OR refobjid IS NULL
+            OR tbl.oid != dep.refobjid
+        )
+        AND (
+            tbl.relnamespace IS NULL
+            OR tbl.relnamespace != (SELECT oid FROM pg_namespace WHERE nspname = 'pg_catalog')
+        )
+    ORDER BY toast_table_oid, expected_table_oid, dependent_table_oid, gp_segment_id
+) AS subquery
+GROUP BY gp_segment_id, toast_table_oid, toast_table_name, expected_table_oid, expected_table_name, dependent_table_oid, dependent_table_name,
+    double_orphan_parent_oid,
+    double_orphan_parent_name,
+    double_orphan_parent_reltoastrelid,
+    double_orphan_parent_toast_oid,
+    double_orphan_parent_toast_name;
+"""
+
+    def runCheck(self, db_connection):
+        orphaned_toast_tables = db_connection.query(self.orphaned_toast_tables_query).dictresult()
+        if len(orphaned_toast_tables) == 0:
+            return True
+
+        for row in orphaned_toast_tables:
+            if row['expected_table_oid'] is None and row['dependent_table_oid'] is None:
+                table = OrphanedTable(row['toast_table_oid'], row['toast_table_name'])
+                issue_type = DoubleOrphanToastTableIssue
+
+            elif row['expected_table_oid'] is None:
+                table = OrphanedTable(row['dependent_table_oid'], row['dependent_table_name'])
+                issue_type = ReferenceOrphanToastTableIssue
+
+            elif row['dependent_table_oid'] is None:
+                table = OrphanedTable(row['expected_table_oid'], row['expected_table_name'])
+                issue_type = DependencyOrphanToastTableIssue
+
+            else:
+                table = OrphanedTable(row['dependent_table_oid'], row['dependent_table_name'])
+                issue_type = MismatchOrphanToastTableIssue
+
+            issue = issue_type(row, table)
+            self._issues.append(issue)
+
+        return False
+
+    def iterate_issues(self):
+        """
+        Yields an OrphanToastTableIssue instance, and the set of segment IDs
+        that exhibit that issue, for every issue found by the checker. For
+        instance, if table 'my_table' has a double-orphan issue on segments 1
+        and 2, and a dependency-orphan issue on segment -1, iterate_issues()
+        will generate two tuples:
+
+            ( DoubleOrphanToastTableIssue('my_table', ...),     { 1, 2 } )
+            ( DependencyOrphanToastTableIssue('my_table', ...), { -1 }   )
+
+        The OrphanToastTableIssue instance internally corresponds to a segment
+        that exhibits the issue, but in cases where there is more than one
+        segment with the problem, *which* segment it corresponds to is not
+        defined. (For instance, in the above example, the
+        DoubleOrphanToastTableIssue's row['content_id'] could be either 1 or 2.)
+
+        It's assumed that this is not a problem (and if it becomes a problem,
+        this implementation will need a rework).
+        """
+        # The hard part here is that our "model" has one issue per segment,
+        # whereas we want to present one issue per (table, issue type) pair as
+        # our "view".
+        #
+        # We use itertools.groupby() as the core. This requires us to sort our
+        # list by (table, issue type) before performing the grouping, for the
+        # same reason that you need to perform `sort` in a `sort | uniq`
+        # pipeline.
+        def issue_key(issue):
+            return (issue.table.oid, type(issue))
+
+        def key_sort(this, that):
+            return cmp(hash(issue_key(this)), hash(issue_key(that)))
+
+        sorted_issues = sorted(self._issues, key_sort)
+
+        for _, group in itertools.groupby(sorted_issues, issue_key):
+            issues = list(group)
+            yield issues[0], { i.row['content_id'] for i in issues }
+
+    def rows_of_type(self, issue_cls):
+        return [ issue.row for issue in self._issues if isinstance(issue, issue_cls) ]
+
+    def issue_types(self):
+        return { type(issue) for issue in self._issues }
+
+    def get_fix_text(self):
+        log_output = ['\nORPHAN TOAST TABLE FIXES:',
+                      '===================================================================']
+        for issue_type in self.issue_types():
+            log_output += issue_type.fix_text
+        return '\n'.join(log_output)
+
+    def add_repair_statements(self, segments):
+        content_id_to_segment_map = self._get_content_id_to_segment_map(segments)
+
+        for issue in self._issues:
+            if issue.repair_script:
+                content_id_to_segment_map[issue.row['content_id']]['repair_statements'].append(issue.repair_script)
+
+        segments_with_repair_statements = filter(lambda segment: len(segment['repair_statements']) > 0, content_id_to_segment_map.values())
+        for segment in segments_with_repair_statements:
+            segment['repair_statements'] = ["SET allow_system_table_mods=true;"] + segment['repair_statements']
+
+        return segments_with_repair_statements
+
+    @staticmethod
+    def _get_content_id_to_segment_map(segments):
+        content_id_to_segment = {}
+        for segment in segments.values():
+            segment['repair_statements'] = []
+            content_id_to_segment[segment['content']] = segment
+
+        return content_id_to_segment

gpMgmt/test/behave/mgmt_utils/gpcheckcat.feature

@@ -425,3 +425,137 @@ Feature: gpcheckcat tests
         Then gpcheckcat should print "Name of test which found this issue: dependency_pg_type" to stdout
         Then gpcheckcat should print "Table pg_type has a dependency issue on oid .* at content 0" to stdout
         And the user runs "dropdb gpcheckcat_dependency"
+
+    @orphaned_toast
+    Scenario: gpcheckcat should repair "bad reference" orphaned toast tables (caused by missing reltoastrelid)
+        Given the database "gpcheckcat_orphans" is broken with "bad reference" orphaned toast tables
+        When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+        Then gpcheckcat should return a return code of 1
+        And gpcheckcat should print "catalog issue\(s\) found , repair script\(s\) generated" to stdout
+        And gpcheckcat should print "To fix, run the generated repair script which updates a pg_class entry using the correct dependent table OID for reltoastrelid" to stdout
+        And run all the repair scripts in the dir "repair_dir"
+        When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+        And gpcheckcat should print "Found no catalog issue" to stdout
+        And the user runs "dropdb gpcheckcat_orphans"
+        And the path "repair_dir" is removed from current working directory
+
+    @orphaned_toast
+    Scenario: gpcheckcat should repair "bad dependency" orphaned toast tables (caused by missing pg_depend entry)
+        Given the database "gpcheckcat_orphans" is broken with "bad dependency" orphaned toast tables
+        When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+        Then gpcheckcat should return a return code of 1
+        And gpcheckcat should print "catalog issue\(s\) found , repair script\(s\) generated" to stdout
+        And gpcheckcat should print "To fix, run the generated repair script which inserts a pg_depend entry using the correct dependent table OID for refobjid" to stdout
+        And run all the repair scripts in the dir "repair_dir"
+        When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+        And gpcheckcat should print "Found no catalog issue" to stdout
+        And the user runs "dropdb gpcheckcat_orphans"
+        And the path "repair_dir" is removed from current working directory
+
+    @orphaned_toast
+    Scenario: gpcheckcat should log and not attempt to repair "double orphan - no parent" orphaned toast tables (caused by both missing reltoastrelid and missing pg_depend entry)
+        Given the database "gpcheckcat_orphans" is broken with "double orphan - no parent" orphaned toast tables
+        When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+        Then gpcheckcat should return a return code of 1
+        And gpcheckcat should print "catalog issue\(s\) found , repair script\(s\) generated" to stdout
+        And gpcheckcat should print "The parent table does not exist. Therefore, the toast table" to stdout
+        And run all the repair scripts in the dir "repair_dir"
+        When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+        And gpcheckcat should print "catalog issue\(s\) found , repair script\(s\) generated" to stdout
+        And the user runs "dropdb gpcheckcat_orphans"
+        And the path "repair_dir" is removed from current working directory
+
+    @orphaned_toast
+    Scenario: gpcheckcat should log and not attempt to repair "double orphan - valid parent" orphaned toast tables (caused by both missing reltoastrelid and missing pg_depend entry)
+        Given the database "gpcheckcat_orphans" is broken with "double orphan - valid parent" orphaned toast tables
+        When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+        Then gpcheckcat should return a return code of 1
+        And gpcheckcat should print "catalog issue\(s\) found , repair script\(s\) generated" to stdout
+        And gpcheckcat should print "The parent table already references a valid toast table" to stdout
+        And run all the repair scripts in the dir "repair_dir"
+        When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+        And gpcheckcat should print "catalog issue\(s\) found , repair script\(s\) generated" to stdout
+        And the user runs "dropdb gpcheckcat_orphans"
+        And the path "repair_dir" is removed from current working directory
+
+    @orphaned_toast
+    Scenario: gpcheckcat should log and not attempt to repair "double orphan - invalid parent" orphaned toast tables (caused by both missing reltoastrelid and missing pg_depend entry)
+        Given the database "gpcheckcat_orphans" is broken with "double orphan - invalid parent" orphaned toast tables
+        When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+        Then gpcheckcat should return a return code of 1
+        And gpcheckcat should print "catalog issue\(s\) found , repair script\(s\) generated" to stdout
+        And gpcheckcat should print "Verify that the parent table requires a toast table." to stdout
+        And run all the repair scripts in the dir "repair_dir"
+        When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+        And gpcheckcat should print "catalog issue\(s\) found , repair script\(s\) generated" to stdout
+        And the user runs "dropdb gpcheckcat_orphans"
+        And the path "repair_dir" is removed from current working directory
+
+    @orphaned_toast
+    Scenario: gpcheckcat should log and not repair "mismatched non-cyclic" orphaned toast tables (caused by non-matching reltoastrelid)
+        Given the database "gpcheckcat_orphans" is broken with "mismatched non-cyclic" orphaned toast tables
+        When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+        Then gpcheckcat should return a return code of 1
+        And gpcheckcat should print "catalog issue\(s\) found , repair script\(s\) generated" to stdout
+        And gpcheckcat should print "A manual catalog change is needed to fix by updating the pg_depend TOAST table entry and setting the refobjid field to the correct dependent table" to stdout
+        And run all the repair scripts in the dir "repair_dir"
+        When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+        And gpcheckcat should print "catalog issue\(s\) found , repair script\(s\) generated" to stdout
+        And gpcheckcat should print "A manual catalog change is needed" to stdout
+        And the user runs "dropdb gpcheckcat_orphans"
+        And the path "repair_dir" is removed from current working directory
+
+    @orphaned_toast
+    Scenario: gpcheckcat should log and not attempt to repair "mismatched cyclic" orphaned toast tables
+        Given the database "gpcheckcat_orphans" is broken with "mismatched cyclic" orphaned toast tables
+        When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+        Then gpcheckcat should return a return code of 1
+        And gpcheckcat should print "catalog issue\(s\) found , repair script\(s\) generated" to stdout
+        And gpcheckcat should print "A manual catalog change is needed to fix by updating the pg_depend TOAST table entry and setting the refobjid field to the correct dependent table" to stdout
+        And run all the repair scripts in the dir "repair_dir"
+        When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+        And gpcheckcat should print "catalog issue\(s\) found , repair script\(s\) generated" to stdout
+        And gpcheckcat should print "A manual catalog change is needed" to stdout
+        And the user runs "dropdb gpcheckcat_orphans"
+        And the path "repair_dir" is removed from current working directory
+
+    @orphaned_toast
+    Scenario: gpcheckcat should repair orphaned toast tables that are only orphaned on some segments
+        Given the database "gpcheckcat_orphans" is broken with "bad reference" orphaned toast tables only on segments with content IDs "0, 1"
+        When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+        Then gpcheckcat should return a return code of 1
+        And gpcheckcat should print "On segment\(s\) 0, 1 table" to stdout
+        And gpcheckcat should print "catalog issue\(s\) found , repair script\(s\) generated" to stdout
+        And run all the repair scripts in the dir "repair_dir"
+        When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+        And gpcheckcat should print "Found no catalog issue" to stdout
+        And the user runs "dropdb gpcheckcat_orphans"
+        And the path "repair_dir" is removed from current working directory
+
+    @orphaned_toast
+    Scenario: gpcheckcat should repair orphaned toast tables that are only orphaned on the master
+		# TODO: should we just combine this into the test above?
+        Given the database "gpcheckcat_orphans" is broken with "bad reference" orphaned toast tables only on segments with content IDs "-1"
+        When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+        Then gpcheckcat should return a return code of 1
+        And gpcheckcat should print "On segment\(s\) -1 table" to stdout
+        And gpcheckcat should print "catalog issue\(s\) found , repair script\(s\) generated" to stdout
+        And run all the repair scripts in the dir "repair_dir"
+        When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+        And gpcheckcat should print "Found no catalog issue" to stdout
+        And the user runs "dropdb gpcheckcat_orphans"
+        And the path "repair_dir" is removed from current working directory
+
+    @orphaned_toast
+    Scenario: gpcheckcat should repair tables that are orphaned in different ways per segment
+        Given the database "gpcheckcat_orphans" has a table that is orphaned in multiple ways
+         When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+         Then gpcheckcat should return a return code of 1
+          And gpcheckcat should print "Found a \"bad reference\" orphaned TOAST table caused by missing a reltoastrelid in pg_class." to stdout
+          And gpcheckcat should print "Found a \"bad dependency\" orphaned TOAST table caused by missing a pg_depend entry." to stdout
+          And gpcheckcat should print "catalog issue\(s\) found , repair script\(s\) generated" to stdout
+          And run all the repair scripts in the dir "repair_dir"
+         When the user runs "gpcheckcat -R orphaned_toast_tables -g repair_dir gpcheckcat_orphans"
+         Then gpcheckcat should print "Found no catalog issue" to stdout
+          And the user runs "dropdb gpcheckcat_orphans"
+          And the path "repair_dir" is removed from current working directory

gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py

@@ -2613,3 +2613,155 @@ PARTITION BY RANGE (year)
 """
         dbconn.execSQL(conn, query)
         conn.commit()
+
+@given('the database "{dbname}" is broken with "{broken}" orphaned toast tables only on segments with content IDs "{contentIDs}"')
+def break_orphaned_toast_tables(context, dbname, broken, contentIDs=None):
+    drop_database_if_exists(context, dbname)
+    create_database(context, dbname)
+
+    sql = ''
+    if broken == 'bad reference':
+        sql = '''
+DROP TABLE IF EXISTS bad_reference;
+CREATE TABLE bad_reference (a text);
+UPDATE pg_class SET reltoastrelid = 0 WHERE relname = 'bad_reference';'''
+    if broken == 'mismatched non-cyclic':
+        sql = '''
+DROP TABLE IF EXISTS mismatch_one;
+CREATE TABLE mismatch_one (a text);
+
+DROP TABLE IF EXISTS mismatch_two;
+CREATE TABLE mismatch_two (a text);
+
+DROP TABLE IF EXISTS mismatch_three;
+CREATE TABLE mismatch_three (a text);
+
+-- 1 -> 2 -> 3
+UPDATE pg_class SET reltoastrelid = (SELECT reltoastrelid FROM pg_class WHERE relname = 'mismatch_two') WHERE relname = 'mismatch_one';
+UPDATE pg_class SET reltoastrelid = (SELECT reltoastrelid FROM pg_class WHERE relname = 'mismatch_three') WHERE relname = 'mismatch_two';'''
+    if broken == 'mismatched cyclic':
+        sql = '''
+DROP TABLE IF EXISTS mismatch_fixed;
+CREATE TABLE mismatch_fixed (a text);
+
+DROP TABLE IF EXISTS mismatch_one;
+CREATE TABLE mismatch_one (a text);
+
+DROP TABLE IF EXISTS mismatch_two;
+CREATE TABLE mismatch_two (a text);
+
+DROP TABLE IF EXISTS mismatch_three;
+CREATE TABLE mismatch_three (a text);
+        
+-- fixed -> 1 -> 2 -> 3 -> 1
+UPDATE pg_class SET reltoastrelid = (SELECT reltoastrelid FROM pg_class WHERE relname = 'mismatch_one') WHERE relname = 'mismatch_fixed'; -- "save" the reltoastrelid
+UPDATE pg_class SET reltoastrelid = (SELECT reltoastrelid FROM pg_class WHERE relname = 'mismatch_two') WHERE relname = 'mismatch_one';
+UPDATE pg_class SET reltoastrelid = (SELECT reltoastrelid FROM pg_class WHERE relname = 'mismatch_three') WHERE relname = 'mismatch_two';
+UPDATE pg_class SET reltoastrelid = (SELECT reltoastrelid FROM pg_class WHERE relname = 'mismatch_fixed') WHERE relname = 'mismatch_three';'''
+    if broken == "bad dependency":
+        sql = '''
+DROP TABLE IF EXISTS bad_dependency;
+CREATE TABLE bad_dependency (a text);
+
+DELETE FROM pg_depend WHERE refobjid = 'bad_dependency'::regclass;'''
+    if broken == "double orphan - no parent":
+        sql = '''
+DROP TABLE IF EXISTS double_orphan_no_parent;
+CREATE TABLE double_orphan_no_parent (a text);
+
+DELETE FROM pg_depend WHERE refobjid = 'double_orphan_no_parent'::regclass;
+DROP TABLE double_orphan_no_parent;'''
+    if broken == "double orphan - valid parent":
+        sql = '''
+DROP TABLE IF EXISTS double_orphan_valid_parent;
+CREATE TABLE double_orphan_valid_parent (a text);
+
+-- save double_orphan_valid_parent toast table oid
+CREATE TEMP TABLE first_orphan_toast AS
+    SELECT oid, relname FROM pg_class WHERE oid = (SELECT reltoastrelid FROM pg_class WHERE oid = 'double_orphan_valid_parent'::regclass);
+
+-- create a orphan toast table
+DELETE FROM pg_depend WHERE objid = (SELECT oid FROM first_orphan_toast);
+
+DROP TABLE double_orphan_valid_parent;
+
+-- recreate double_orphan_valid_parent table to create a second valid toast table
+CREATE TABLE double_orphan_valid_parent (a text);
+
+-- save the second toast table oid from recreating double_orphan_valid_parent
+CREATE TEMP TABLE second_orphan_toast AS
+    SELECT oid, relname FROM pg_class WHERE oid = (SELECT reltoastrelid FROM pg_class WHERE oid = 'double_orphan_valid_parent'::regclass);
+
+-- swap the first_orphan_toast table with a temp name
+UPDATE pg_class SET relname = (SELECT relname || '_temp' FROM second_orphan_toast)
+    WHERE oid = (SELECT oid FROM first_orphan_toast);
+
+-- swap second_orphan_toast table with the original name of valid_parent toast table
+UPDATE pg_class SET relname = (SELECT relname FROM first_orphan_toast)
+     WHERE oid = (SELECT oid FROM second_orphan_toast);
+
+-- swap the temp name with the first_orphan_toast table
+UPDATE pg_class SET relname = (SELECT relname FROM second_orphan_toast)
+     WHERE oid = (SELECT oid FROM first_orphan_toast);'''
+    if broken == "double orphan - invalid parent":
+        sql = '''
+DROP TABLE IF EXISTS double_orphan_invalid_parent;
+CREATE TABLE double_orphan_invalid_parent (a text);
+
+DELETE FROM pg_depend
+    WHERE objid = (SELECT reltoastrelid FROM pg_class WHERE relname = 'double_orphan_invalid_parent')
+    AND refobjid = (SELECT oid FROM pg_class where relname = 'double_orphan_invalid_parent');
+
+UPDATE pg_class SET reltoastrelid = 0 WHERE relname = 'double_orphan_invalid_parent';'''
+
+    dbURLs = [dbconn.DbURL(dbname=dbname)]
+
+    if contentIDs:
+        dbURLs = []
+
+        seg_config_sql = '''SELECT port,hostname FROM gp_segment_configuration WHERE role='p' AND content IN (%s);''' % contentIDs
+        for port, hostname in getRows(dbname, seg_config_sql):
+            dbURLs.append(dbconn.DbURL(dbname=dbname, hostname=hostname, port=port))
+
+    for dbURL in dbURLs:
+        utility = True if contentIDs else False
+        with dbconn.connect(dbURL, allowSystemTableMods=True, utility=utility) as conn:
+            dbconn.execSQL(conn, sql)
+            conn.commit()
+
+@given('the database "{dbname}" is broken with "{broken}" orphaned toast tables')
+def impl(context, dbname, broken):
+    break_orphaned_toast_tables(context, dbname, broken)
+
+@given('the database "{dbname}" has a table that is orphaned in multiple ways')
+def impl(context, dbname):
+    drop_database_if_exists(context, dbname)
+    create_database(context, dbname)
+
+    master = dbconn.DbURL(dbname=dbname)
+    gparray = GpArray.initFromCatalog(master)
+
+    primary0 = gparray.segmentPairs[0].primaryDB
+    primary1 = gparray.segmentPairs[1].primaryDB
+
+    seg0 = dbconn.DbURL(dbname=dbname, hostname=primary0.hostname, port=primary0.port)
+    seg1 = dbconn.DbURL(dbname=dbname, hostname=primary1.hostname, port=primary1.port)
+
+    with dbconn.connect(master, allowSystemTableMods=True) as conn:
+        dbconn.execSQL(conn, """
+            DROP TABLE IF EXISTS borked;
+            CREATE TABLE borked (a text);
+        """)
+        conn.commit()
+
+    with dbconn.connect(seg0, utility=True, allowSystemTableMods=True) as conn:
+        dbconn.execSQL(conn, """
+            DELETE FROM pg_depend WHERE refobjid = 'borked'::regclass;
+        """)
+        conn.commit()
+
+    with dbconn.connect(seg1, utility=True, allowSystemTableMods=True) as conn:
+        dbconn.execSQL(conn, """
+            UPDATE pg_class SET reltoastrelid = 0 WHERE oid = 'borked'::regclass;
+        """)
+        conn.commit()