Skip to content

G
Gpdb

Greenplum / Gpdb

通知 7
Star 1
Fork 0
G
Gpdb

体验新版 GitCode，发现更多精彩内容 >>

提交 189a3547 编写于 作者: D Daniel Gustafsson
浏览文件
操作

Break out Greenplum specific parts from pg_upgrade 

This attempts to minimize the diff to upstream in preparation of
merging with a later version of pg_upgrade. In some cases this
re-introduces quirks from small style cleanups we've performed,
but it will only be for a short period until the merge happens.

PostgreSQL have in recent versions moved to just having a version.c
file for version specific function. In this commit we move Greenplum
specific code to version_gp.c, since we would do that sooner or later
anyways.

No functionality is altered with this, this is limited to refactoring.
上级 3f561d42
隐藏空白更改
内联 并排
Showing with 1219 addition and 879 deletion
contrib/pg_upgrade/Makefile
浏览文件 @ 189a3547
......@@ -10,7 +10,9 @@ OBJS = check.o controldata.o dump.o exec.o file.o function.o info.o \
option.o page.o pg_upgrade.o relfilenode.o server.o \
tablespace.o util.o version.o version_old_8_3.o $(WIN32RES)
OBJS += aotable.o gpdb4_heap_convert.o version_old_gpdb4.o oid_dump.o
# Source files specific to Greenplum
OBJS += aotable.o gpdb4_heap_convert.o version_gp.o oid_dump.o \
check_gp.o file_gp.o reporting.o
PG_CPPFLAGS = -DFRONTEND -DDLSUFFIX=\"$(DLSUFFIX)\" -I$(srcdir) -I$(libpq_srcdir)
PG_LIBS = $(libpq_pgport)
......
contrib/pg_upgrade/check.c
浏览文件 @ 189a3547
......@@ -20,10 +20,6 @@ static void check_proper_datallowconn(ClusterInfo *cluster);
static void check_for_isn_and_int8_passing_mismatch(ClusterInfo *cluster);
static void check_for_reg_data_type_usage(ClusterInfo *cluster);
static void check_external_partition(void);
static void check_covering_aoindex(void);
static void check_hash_partition_usage(void);
static void check_partition_indexes(void);
/*
......@@ -107,19 +103,11 @@ check_old_cluster(bool live_check,
check_for_reg_data_type_usage(&old_cluster);
check_for_isn_and_int8_passing_mismatch(&old_cluster);
check_external_partition();
check_covering_aoindex();
check_hash_partition_usage();
check_partition_indexes();
check_greenplum();
/* old = PG 8.3 checks? */
/*
* All of these checks have been disabled in GPDB, since we're using
* this to upgrade only to 8.3. Needs to be removed when we merge
* with PostgreSQL 8.4.
*
* GPDB_91_MERGE_FIXME: that comment doesn't make sense to me; what checks
* have been disabled?
* Upgrading from Greenplum 4.3.x. Upgrading from one version of 4.3.x
* to another 4.3.x version is not supported.
*
* The change to name datatype's alignment was backported to GPDB 5.0,
* so we need to check even when upgradeing to GPDB 5.0.
......@@ -132,12 +120,16 @@ check_old_cluster(bool live_check,
old_GPDB4_check_no_free_aoseg(&old_cluster);
}
/*
* Upgrades from GPDB 5.x
*/
if (GET_MAJOR_VERSION(old_cluster.major_version) <= 803 &&
GET_MAJOR_VERSION(new_cluster.major_version) >= 804)
{
old_8_3_check_for_name_data_type_usage(&old_cluster);
old_8_3_check_for_tsquery_usage(&old_cluster);
old_8_3_check_ltree_usage(&old_cluster);
check_hash_partition_usage();
if (user_opts.check)
{
old_8_3_rebuild_tsvector_tables(&old_cluster, true);
......@@ -844,384 +836,3 @@ check_proper_datallowconn(ClusterInfo *cluster)
check_ok();
}
/*
* check_external_partition
*
* External tables cannot be included in the partitioning hierarchy during the
* initial definition with CREATE TABLE, they must be defined separately and
* injected via ALTER TABLE EXCHANGE. The partitioning system catalogs are
* however not replicated onto the segments which means ALTER TABLE EXCHANGE
* is prohibited in utility mode. This means that pg_upgrade cannot upgrade a
* cluster containing external partitions, they must be handled manually
* before/after the upgrade.
*
* Check for the existence of external partitions and refuse the upgrade if
* found.
*/
static void
check_external_partition(void)
{
char query[QUERY_ALLOC];
char output_path[MAXPGPATH];
FILE *script = NULL;
bool found = false;
int dbnum;
prep_status("Checking for external tables used in partitioning");
snprintf(output_path, sizeof(output_path), "%s/external_partitions.txt",
os_info.cwd);
/*
* We need to query the inheritance catalog rather than the partitioning
* catalogs since they are not available on the segments.
*/
snprintf(query, sizeof(query),
"SELECT cc.relname, c.relname AS partname, c.relnamespace "
"FROM pg_inherits i "
" JOIN pg_class c ON (i.inhrelid = c.oid AND c.relstorage = '%c') "
" JOIN pg_class cc ON (i.inhparent = cc.oid);",
RELSTORAGE_EXTERNAL);
for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
{
PGresult *res;
int ntups;
int rowno;
DbInfo *active_db = &old_cluster.dbarr.dbs[dbnum];
PGconn *conn;
conn = connectToServer(&old_cluster, active_db->db_name);
res = executeQueryOrDie(conn, query);
ntups = PQntuples(res);
if (ntups > 0)
{
found = true;
if (script == NULL && (script = fopen(output_path, "w")) == NULL)
pg_log(PG_FATAL, "Could not create necessary file: %s\n",
output_path);
for (rowno = 0; rowno < ntups; rowno++)
{
fprintf(script, "External partition \"%s\" in relation \"%s\"\n",
PQgetvalue(res, rowno, PQfnumber(res, "partname")),
PQgetvalue(res, rowno, PQfnumber(res, "relname")));
}
}
PQclear(res);
PQfinish(conn);
}
if (found)
{
fclose(script);
pg_log(PG_REPORT, "fatal\n");
pg_log(PG_FATAL,
"| Your installation contains partitioned tables with external\n"
"| tables as partitions. These partitions need to be removed\n"
"| from the partition hierarchy before the upgrade. A list of\n"
"| external partitions to remove is in the file:\n"
"| \t%s\n\n", output_path);
}
else
{
check_ok();
}
}
/*
* check_covering_aoindex
*
* A partitioned AO table which had an index created on the parent relation,
* and an AO partition exchanged into the hierarchy without any indexes will
* break upgrades due to the way pg_dump generates DDL.
*
* create table t (a integer, b text, c integer)
* with (appendonly=true)
* distributed by (a)
* partition by range(c) (start(1) end(3) every(1));
* create index t_idx on t (b);
*
* At this point, the t_idx index has created AO blockdir relations for all
* partitions. We now exchange a new table into the hierarchy which has no
* index defined:
*
* create table t_exch (a integer, b text, c integer)
* with (appendonly=true)
* distributed by (a);
* alter table t exchange partition for (rank(1)) with table t_exch;
*
* The partition which was swapped into the hierarchy with EXCHANGE does not
* have any indexes and thus no AO blockdir relation. This is in itself not
* a problem, but when pg_dump generates DDL for the above situation it will
* create the index in such a way that it covers the entire hierarchy, as in
* its original state. The below snippet illustrates the dumped DDL:
*
* create table t ( ... )
* ...
* partition by (... );
* create index t_idx on t ( ... );
*
* This creates a problem for the Oid synchronization in pg_upgrade since it
* expects to find a preassigned Oid for the AO blockdir relations for each
* partition. A longer term solution would be to generate DDL in pg_dump which
* creates the current state, but for the time being we disallow upgrades on
* cluster which exhibits this.
*/
static void
check_covering_aoindex(void)
{
char query[QUERY_ALLOC];
char output_path[MAXPGPATH];
FILE *script = NULL;
bool found = false;
int dbnum;
prep_status("Checking for non-covering indexes on partitioned AO tables");
snprintf(output_path, sizeof(output_path), "%s/mismatched_aopartition_indexes.txt",
os_info.cwd);
snprintf(query, sizeof(query),
"SELECT DISTINCT ao.relid, inh.inhrelid "
"FROM pg_catalog.pg_appendonly ao "
" JOIN pg_catalog.pg_inherits inh "
" ON (inh.inhparent = ao.relid) "
" JOIN pg_catalog.pg_appendonly aop "
" ON (inh.inhrelid = aop.relid AND aop.blkdirrelid = 0) "
" JOIN pg_catalog.pg_index i "
" ON (i.indrelid = ao.relid) "
"WHERE ao.blkdirrelid <> 0;");
for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
{
PGresult *res;
PGconn *conn;
int ntups;
int rowno;
DbInfo *active_db = &old_cluster.dbarr.dbs[dbnum];
conn = connectToServer(&old_cluster, active_db->db_name);
res = executeQueryOrDie(conn, query);
ntups = PQntuples(res);
if (ntups > 0)
{
found = true;
if (script == NULL && (script = fopen(output_path, "w")) == NULL)
pg_log(PG_FATAL, "Could not create necessary file: %s\n",
output_path);
for (rowno = 0; rowno < ntups; rowno++)
{
fprintf(script, "Mismatched index on partition %s in relation %s\n",
PQgetvalue(res, rowno, PQfnumber(res, "inhrelid")),
PQgetvalue(res, rowno, PQfnumber(res, "relid")));
}
}
PQclear(res);
PQfinish(conn);
}
if (found)
{
fclose(script);
pg_log(PG_REPORT, "fatal\n");
pg_log(PG_FATAL,
"| Your installation contains partitioned append-only tables\n"
"| with an index defined on the partition parent which isn't\n"
"| present on all partition members. These indexes must be\n"
"| dropped before the upgrade. A list of relations, and the\n"
"| partitions in question is in the file:\n"
"| \t%s\n\n", output_path);
}
else
{
check_ok();
}
}
/*
* check_hash_partition_usage()
* 8.3 -> 8.4
*
* Hash partitioning was never officially supported in GPDB5 and was removed
* in GPDB6, but better check just in case someone has found the hidden GUC
* and used them anyway.
*
* The hash algorithm was changed in 8.4, so upgrading is impossible anyway.
* This is basically the same problem as with hash indexes in PostgreSQL.
*/
static void
check_hash_partition_usage(void)
{
int dbnum;
FILE *script = NULL;
bool found = false;
char output_path[MAXPGPATH];
prep_status("Checking for hash partitioned tables");
snprintf(output_path, sizeof(output_path), "%s/hash_partitioned_tables.txt",
os_info.cwd);
for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
{
PGresult *res;
bool db_used = false;
int ntups;
int rowno;
int i_nspname,
i_relname;
DbInfo *active_db = &old_cluster.dbarr.dbs[dbnum];
PGconn *conn = connectToServer(&old_cluster, active_db->db_name);
res = executeQueryOrDie(conn,
"SELECT n.nspname, c.relname "
"FROM pg_catalog.pg_partition p, pg_catalog.pg_class c, pg_catalog.pg_namespace n "
"WHERE p.parrelid = c.oid AND c.relnamespace = n.oid "
"AND parkind = 'h'");
ntups = PQntuples(res);
i_nspname = PQfnumber(res, "nspname");
i_relname = PQfnumber(res, "relname");
for (rowno = 0; rowno < ntups; rowno++)
{
found = true;
if (script == NULL && (script = fopen(output_path, "w")) == NULL)
pg_log(PG_FATAL, "Could not create necessary file: %s\n", output_path);
if (!db_used)
{
fprintf(script, "Database: %s\n", active_db->db_name);
db_used = true;
}
fprintf(script, " %s.%s\n",
PQgetvalue(res, rowno, i_nspname),
PQgetvalue(res, rowno, i_relname));
}
PQclear(res);
PQfinish(conn);
}
if (found)
{
fclose(script);
pg_log(PG_REPORT, "fatal\n");
pg_log(PG_FATAL,
"| Your installation contains hash partitioned tables.\n"
"| Upgrading hash partitioned tables is not supported,\n"
"| so this cluster cannot currently be upgraded. You\n"
"| can remove the problem tables and restart the\n"
"| migration. A list of the problem tables is in the\n"
"| file:\n"
"| \t%s\n\n", output_path);
}
else
check_ok();
}
/*
* check_partition_indexes
*
* There are numerous pitfalls surrounding indexes on partition hierarchies,
* so rather than trying to cover all the cornercases we disallow indexes on
* partitioned tables altogether during the upgrade. Since we in any case
* invalidate the indexes forcing a REINDEX, there is little to be gained by
* handling them for the end-user.
*/
static void
check_partition_indexes(void)
{
int dbnum;
FILE *script = NULL;
bool found = false;
char output_path[MAXPGPATH];
prep_status("Checking for indexes on partitioned tables");
snprintf(output_path, sizeof(output_path), "%s/partitioned_tables_indexes.txt",
os_info.cwd);
for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
{
PGresult *res;
bool db_used = false;
int ntups;
int rowno;
int i_nspname;
int i_relname;
int i_indexes;
DbInfo *active_db = &old_cluster.dbarr.dbs[dbnum];
PGconn *conn = connectToServer(&old_cluster, active_db->db_name);
res = executeQueryOrDie(conn,
"WITH partitions AS ("
" SELECT DISTINCT n.nspname, "
" c.relname "
" FROM pg_catalog.pg_partition p "
" JOIN pg_catalog.pg_class c ON (p.parrelid = c.oid) "
" JOIN pg_catalog.pg_namespace n ON (n.oid = c.relnamespace) "
" UNION "
" SELECT n.nspname, "
" partitiontablename AS relname "
" FROM pg_catalog.pg_partitions p "
" JOIN pg_catalog.pg_class c ON (p.partitiontablename = c.relname) "
" JOIN pg_catalog.pg_namespace n ON (n.oid = c.relnamespace) "
") "
"SELECT nspname, "
" relname, "
" count(indexname) AS indexes "
"FROM partitions "
" JOIN pg_catalog.pg_indexes ON (relname = tablename AND "
" nspname = schemaname) "
"GROUP BY nspname, relname "
"ORDER BY relname");
ntups = PQntuples(res);
i_nspname = PQfnumber(res, "nspname");
i_relname = PQfnumber(res, "relname");
i_indexes = PQfnumber(res, "indexes");
for (rowno = 0; rowno < ntups; rowno++)
{
found = true;
if (script == NULL && (script = fopen(output_path, "w")) == NULL)
pg_log(PG_FATAL, "Could not create necessary file: %s\n", output_path);
if (!db_used)
{
fprintf(script, "Database: %s\n", active_db->db_name);
db_used = true;
}
fprintf(script, " %s.%s has %s index(es)\n",
PQgetvalue(res, rowno, i_nspname),
PQgetvalue(res, rowno, i_relname),
PQgetvalue(res, rowno, i_indexes));
}
PQclear(res);
PQfinish(conn);
}
if (found)
{
fclose(script);
pg_log(PG_REPORT, "fatal\n");
pg_log(PG_FATAL,
"| Your installation contains partitioned tables with\n"
"| indexes defined on them. Indexes on partition parents,\n"
"| as well as children, must be dropped before upgrade.\n"
"| A list of the problem tables is in the file:\n"
"| \t%s\n\n", output_path);
}
else
check_ok();
}
contrib/pg_upgrade/check_gp.c 0 → 100644
浏览文件 @ 189a3547
/*
* check_gp.c
*
* Greenplum specific server checks and output routines
*
* Any compatibility checks which are version dependent (testing for issues in
* specific versions of Greenplum) should be placed in their respective
* version_old_gpdb{MAJORVERSION}.c file. The checks in this file supplement
* the checks present in check.c, which is the upstream file for performing
* checks against a PostgreSQL cluster.
*
* Copyright (c) 2010, PostgreSQL Global Development Group
* Copyright (c) 2017-Present Pivotal Software, Inc
*/
#include "pg_upgrade.h"
static void check_external_partition(void);
static void check_covering_aoindex(void);
static void check_partition_indexes(void);
/*
* check_greenplum
*
* Rather than exporting all checks, we export a single API function which in
* turn is responsible for running Greenplum checks. This function should be
* executed after all PostgreSQL checks. The order of the checks should not
* matter.
*/
void
check_greenplum(void)
{
/*
* Upgrading within a major version is a handy feature of pg_upgrade, but
* we don't allow it for within 4.3.x clusters, 4.3.x can only be an old
* version to be upgraded from.
*/
if (GET_MAJOR_VERSION(old_cluster.major_version) <= 802 &&
GET_MAJOR_VERSION(new_cluster.major_version) <= 802)
{
pg_log(PG_FATAL,
"old and new cluster cannot both be Greenplum 4.3.x installations\n");
}
check_external_partition();
check_covering_aoindex();
check_partition_indexes();
}
/*
* check_external_partition
*
* External tables cannot be included in the partitioning hierarchy during the
* initial definition with CREATE TABLE, they must be defined separately and
* injected via ALTER TABLE EXCHANGE. The partitioning system catalogs are
* however not replicated onto the segments which means ALTER TABLE EXCHANGE
* is prohibited in utility mode. This means that pg_upgrade cannot upgrade a
* cluster containing external partitions, they must be handled manually
* before/after the upgrade.
*
* Check for the existence of external partitions and refuse the upgrade if
* found.
*/
static void
check_external_partition(void)
{
char query[QUERY_ALLOC];
char output_path[MAXPGPATH];
FILE *script = NULL;
bool found = false;
int dbnum;
prep_status("Checking for external tables used in partitioning");
snprintf(output_path, sizeof(output_path), "%s/external_partitions.txt",
os_info.cwd);
/*
* We need to query the inheritance catalog rather than the partitioning
* catalogs since they are not available on the segments.
*/
snprintf(query, sizeof(query),
"SELECT cc.relname, c.relname AS partname, c.relnamespace "
"FROM pg_inherits i "
" JOIN pg_class c ON (i.inhrelid = c.oid AND c.relstorage = '%c') "
" JOIN pg_class cc ON (i.inhparent = cc.oid);",
RELSTORAGE_EXTERNAL);
for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
{
PGresult *res;
int ntups;
int rowno;
DbInfo *active_db = &old_cluster.dbarr.dbs[dbnum];
PGconn *conn;
conn = connectToServer(&old_cluster, active_db->db_name);
res = executeQueryOrDie(conn, query);
ntups = PQntuples(res);
if (ntups > 0)
{
found = true;
if (script == NULL && (script = fopen(output_path, "w")) == NULL)
pg_log(PG_FATAL, "Could not create necessary file: %s\n",
output_path);
for (rowno = 0; rowno < ntups; rowno++)
{
fprintf(script, "External partition \"%s\" in relation \"%s\"\n",
PQgetvalue(res, rowno, PQfnumber(res, "partname")),
PQgetvalue(res, rowno, PQfnumber(res, "relname")));
}
}
PQclear(res);
PQfinish(conn);
}
if (found)
{
fclose(script);
pg_log(PG_REPORT, "fatal\n");
pg_log(PG_FATAL,
"| Your installation contains partitioned tables with external\n"
"| tables as partitions. These partitions need to be removed\n"
"| from the partition hierarchy before the upgrade. A list of\n"
"| external partitions to remove is in the file:\n"
"| \t%s\n\n", output_path);
}
else
{
check_ok();
}
}
/*
* check_covering_aoindex
*
* A partitioned AO table which had an index created on the parent relation,
* and an AO partition exchanged into the hierarchy without any indexes will
* break upgrades due to the way pg_dump generates DDL.
*
* create table t (a integer, b text, c integer)
* with (appendonly=true)
* distributed by (a)
* partition by range(c) (start(1) end(3) every(1));
* create index t_idx on t (b);
*
* At this point, the t_idx index has created AO blockdir relations for all
* partitions. We now exchange a new table into the hierarchy which has no
* index defined:
*
* create table t_exch (a integer, b text, c integer)
* with (appendonly=true)
* distributed by (a);
* alter table t exchange partition for (rank(1)) with table t_exch;
*
* The partition which was swapped into the hierarchy with EXCHANGE does not
* have any indexes and thus no AO blockdir relation. This is in itself not
* a problem, but when pg_dump generates DDL for the above situation it will
* create the index in such a way that it covers the entire hierarchy, as in
* its original state. The below snippet illustrates the dumped DDL:
*
* create table t ( ... )
* ...
* partition by (... );
* create index t_idx on t ( ... );
*
* This creates a problem for the Oid synchronization in pg_upgrade since it
* expects to find a preassigned Oid for the AO blockdir relations for each
* partition. A longer term solution would be to generate DDL in pg_dump which
* creates the current state, but for the time being we disallow upgrades on
* cluster which exhibits this.
*/
static void
check_covering_aoindex(void)
{
char query[QUERY_ALLOC];
char output_path[MAXPGPATH];
FILE *script = NULL;
bool found = false;
int dbnum;
prep_status("Checking for non-covering indexes on partitioned AO tables");
snprintf(output_path, sizeof(output_path), "%s/mismatched_aopartition_indexes.txt",
os_info.cwd);
snprintf(query, sizeof(query),
"SELECT DISTINCT ao.relid, inh.inhrelid "
"FROM pg_catalog.pg_appendonly ao "
" JOIN pg_catalog.pg_inherits inh "
" ON (inh.inhparent = ao.relid) "
" JOIN pg_catalog.pg_appendonly aop "
" ON (inh.inhrelid = aop.relid AND aop.blkdirrelid = 0) "
" JOIN pg_catalog.pg_index i "
" ON (i.indrelid = ao.relid) "
"WHERE ao.blkdirrelid <> 0;");
for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
{
PGresult *res;
PGconn *conn;
int ntups;
int rowno;
DbInfo *active_db = &old_cluster.dbarr.dbs[dbnum];
conn = connectToServer(&old_cluster, active_db->db_name);
res = executeQueryOrDie(conn, query);
ntups = PQntuples(res);
if (ntups > 0)
{
found = true;
if (script == NULL && (script = fopen(output_path, "w")) == NULL)
pg_log(PG_FATAL, "Could not create necessary file: %s\n",
output_path);
for (rowno = 0; rowno < ntups; rowno++)
{
fprintf(script, "Mismatched index on partition %s in relation %s\n",
PQgetvalue(res, rowno, PQfnumber(res, "inhrelid")),
PQgetvalue(res, rowno, PQfnumber(res, "relid")));
}
}
PQclear(res);
PQfinish(conn);
}
if (found)
{
fclose(script);
pg_log(PG_REPORT, "fatal\n");
pg_log(PG_FATAL,
"| Your installation contains partitioned append-only tables\n"
"| with an index defined on the partition parent which isn't\n"
"| present on all partition members. These indexes must be\n"
"| dropped before the upgrade. A list of relations, and the\n"
"| partitions in question is in the file:\n"
"| \t%s\n\n", output_path);
}
else
{
check_ok();
}
}
/*
* check_partition_indexes
*
* There are numerous pitfalls surrounding indexes on partition hierarchies,
* so rather than trying to cover all the cornercases we disallow indexes on
* partitioned tables altogether during the upgrade. Since we in any case
* invalidate the indexes forcing a REINDEX, there is little to be gained by
* handling them for the end-user.
*/
static void
check_partition_indexes(void)
{
int dbnum;
FILE *script = NULL;
bool found = false;
char output_path[MAXPGPATH];
prep_status("Checking for indexes on partitioned tables");
snprintf(output_path, sizeof(output_path), "%s/partitioned_tables_indexes.txt",
os_info.cwd);
for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
{
PGresult *res;
bool db_used = false;
int ntups;
int rowno;
int i_nspname;
int i_relname;
int i_indexes;
DbInfo *active_db = &old_cluster.dbarr.dbs[dbnum];
PGconn *conn = connectToServer(&old_cluster, active_db->db_name);
res = executeQueryOrDie(conn,
"WITH partitions AS ("
" SELECT DISTINCT n.nspname, "
" c.relname "
" FROM pg_catalog.pg_partition p "
" JOIN pg_catalog.pg_class c ON (p.parrelid = c.oid) "
" JOIN pg_catalog.pg_namespace n ON (n.oid = c.relnamespace) "
" UNION "
" SELECT n.nspname, "
" partitiontablename AS relname "
" FROM pg_catalog.pg_partitions p "
" JOIN pg_catalog.pg_class c ON (p.partitiontablename = c.relname) "
" JOIN pg_catalog.pg_namespace n ON (n.oid = c.relnamespace) "
") "
"SELECT nspname, "
" relname, "
" count(indexname) AS indexes "
"FROM partitions "
" JOIN pg_catalog.pg_indexes ON (relname = tablename AND "
" nspname = schemaname) "
"GROUP BY nspname, relname "
"ORDER BY relname");
ntups = PQntuples(res);
i_nspname = PQfnumber(res, "nspname");
i_relname = PQfnumber(res, "relname");
i_indexes = PQfnumber(res, "indexes");
for (rowno = 0; rowno < ntups; rowno++)
{
found = true;
if (script == NULL && (script = fopen(output_path, "w")) == NULL)
pg_log(PG_FATAL, "Could not create necessary file: %s\n", output_path);
if (!db_used)
{
fprintf(script, "Database: %s\n", active_db->db_name);
db_used = true;
}
fprintf(script, " %s.%s has %s index(es)\n",
PQgetvalue(res, rowno, i_nspname),
PQgetvalue(res, rowno, i_relname),
PQgetvalue(res, rowno, i_indexes));
}
PQclear(res);
PQfinish(conn);
}
if (found)
{
fclose(script);
pg_log(PG_REPORT, "fatal\n");
pg_log(PG_FATAL,
"| Your installation contains partitioned tables with\n"
"| indexes defined on them. Indexes on partition parents,\n"
"| as well as children, must be dropped before upgrade.\n"
"| A list of the problem tables is in the file:\n"
"| \t%s\n\n", output_path);
}
else
check_ok();
}
contrib/pg_upgrade/dump.c
浏览文件 @ 189a3547
......@@ -27,37 +27,6 @@ generate_old_dump(void)
check_ok();
}
static char *
get_preassigned_oids_for_db(char *line)
{
char *dbname;
int dbnum;
// FIXME: parsing the dump like this is madness.
// We should use a dump file for each database to
// begin with. (like more recent version of PostgreSQL).
if (strncmp(line, "\\connect ", strlen("\\connect ")) != 0)
return NULL;
dbname = line + strlen("\\connect ");
/* strip newline */
if (strlen(dbname) <= 1)
return NULL;
dbname[strlen(dbname) - 1] = '\0';
for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
{
DbInfo *olddb = &old_cluster.dbarr.dbs[dbnum];
if (strcmp(olddb->db_name, dbname) == 0)
{
/* Found it! */
return olddb->reserved_oids;
}
}
return NULL;
}
/*
* split_old_dump
......@@ -93,6 +62,7 @@ split_old_dump(void)
* Open all files in binary mode to avoid line end translation on Windows,
* both for input and output.
*/
snprintf(filename, sizeof(filename), "%s/%s", os_info.cwd, ALL_DUMP_FILE);
if ((all_dump = fopen(filename, PG_BINARY_R)) == NULL)
pg_log(PG_FATAL, "Cannot open dump file %s\n", filename);
......@@ -116,9 +86,7 @@ split_old_dump(void)
if (current_output == globals_dump && start_of_line &&
suppressed_username &&
strncmp(line, "\\connect ", strlen("\\connect ")) == 0)
{
current_output = db_dump;
}
/* output unless we are recreating our own username */
if (current_output != globals_dump || !start_of_line ||
......
contrib/pg_upgrade/file.c
浏览文件 @ 189a3547
......@@ -11,9 +11,6 @@
#include <fcntl.h>
#include "storage/bufpage.h"
#include "storage/checksum.h"
#include "storage/checksum_impl.h"
#ifndef WIN32
......@@ -108,92 +105,6 @@ copyAndUpdateFile(pageCnvCtx *pageConverter,
}
}
/*
* rewriteHeapPageWithChecksum
*
* Copies a relation file from src to dst and sets the data checksum in the
* page headers in the process. We are not using a pageConverter, even though
* that would make sense, since pageConverter are deprecated and removed in
* upstream and would give us merge headaches.
*/
void
rewriteHeapPageChecksum(const char *fromfile, const char *tofile,
const char *schemaName, const char *relName)
{
int src_fd;
int dst_fd;
int blkno;
int bytesRead;
int totalBytesRead;
char *buf;
ssize_t writesize;
struct stat statbuf;
/*
* transfer_relfile() should never call us unless requested by the data
* checksum option but better doublecheck before we start rewriting data.
*/
if (user_opts.checksum_mode == CHECKSUM_NONE)
pg_log(PG_FATAL, "error, incorrect checksum configuration detected.\n");
if ((src_fd = open(fromfile, O_RDONLY | PG_BINARY, 0)) < 0)
pg_log(PG_FATAL,
"error while rewriting relation \"%s.%s\": could not open file \"%s\": %s\n",
schemaName, relName, fromfile, strerror(errno));
if (fstat(src_fd, &statbuf) != 0)
pg_log(PG_FATAL,
"error while rewriting relation \"%s.%s\": could not stat file \"%s\": %s\n",
schemaName, relName, fromfile, strerror(errno));
if ((dst_fd = open(tofile, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, S_IRUSR | S_IWUSR)) < 0)
pg_log(PG_FATAL,
"error while rewriting relation \"%s.%s\": could not create file \"%s\": %s\n",
schemaName, relName, tofile, strerror(errno));
blkno = 0;
totalBytesRead = 0;
buf = (char *) pg_malloc(BLCKSZ);
while ((bytesRead = read(src_fd, buf, BLCKSZ)) == BLCKSZ)
{
Size page_size;
page_size = PageGetPageSize((PageHeader) buf);
if (!PageSizeIsValid(page_size) && page_size != 0)
pg_log(PG_FATAL,
"error while rewriting relation \"%s.%s\": invalid page size detected (%zd)\n",
schemaName, relName, page_size);
if (!PageIsNew(buf))
{
if (user_opts.checksum_mode == CHECKSUM_ADD)
((PageHeader) buf)->pd_checksum = pg_checksum_page(buf, blkno);
else
memset(&(((PageHeader) buf)->pd_checksum), 0, sizeof(uint16));
}
writesize = write(dst_fd, buf, BLCKSZ);
if (writesize != BLCKSZ)
pg_log(PG_FATAL, "error when rewriting relation \"%s.%s\": %s",
schemaName, relName, strerror(errno));
blkno++;
totalBytesRead += BLCKSZ;
}
if (totalBytesRead != statbuf.st_size)
pg_log(PG_FATAL,
"error when rewriting relation \"%s.%s\": torn read on file \"%s\"%c %s\n",
schemaName, relName, fromfile,
(errno != 0 ? ':' : ' '), strerror(errno));
pg_free(buf);
close(dst_fd);
close(src_fd);
}
/*
* linkAndUpdateFile()
......
contrib/pg_upgrade/file_gp.c 0 → 100644
浏览文件 @ 189a3547
/*
* file.c
*
* Greenplum specific file system operations
*
* Portions Copyright (c) 2010, PostgreSQL Global Development Group
* Portions Copyright (c) 2016-Present, Pivotal Software Inc
*/
#include "pg_upgrade.h"
#include "storage/bufpage.h"
#include "storage/checksum.h"
#include "storage/checksum_impl.h"
/*
* In upgrading from GPDB4, copy the pg_distributedlog over in
* vanilla. The assumption that this works needs to be verified
*/
void
copy_distributedlog(void)
{
char old_dlog_path[MAXPGPATH];
char new_dlog_path[MAXPGPATH];
prep_status("Deleting new distributedlog");
snprintf(old_dlog_path, sizeof(old_dlog_path), "%s/pg_distributedlog", old_cluster.pgdata);
snprintf(new_dlog_path, sizeof(new_dlog_path), "%s/pg_distributedlog", new_cluster.pgdata);
if (rmtree(new_dlog_path, true) != true)
pg_log(PG_FATAL, "Unable to delete directory %s\n", new_dlog_path);
check_ok();
prep_status("Copying old distributedlog to new server");
/* libpgport's copydir() doesn't work in FRONTEND code */
#ifndef WIN32
exec_prog(true, SYSTEMQUOTE "%s \"%s\" \"%s\"" SYSTEMQUOTE,
"cp -Rf",
#else
/* flags: everything, no confirm, quiet, overwrite read-only */
exec_prog(true, SYSTEMQUOTE "%s \"%s\" \"%s\\\"" SYSTEMQUOTE,
"xcopy /e /y /q /r",
#endif
old_dlog_path, new_dlog_path);
check_ok();
}
/*
* rewriteHeapPageWithChecksum
*
* Copies a relation file from src to dst and sets the data checksum in the
* page headers in the process. We are not using a pageConverter, even though
* that would make sense, since pageConverter are deprecated and removed in
* upstream and would give us merge headaches.
*/
void
rewriteHeapPageChecksum(const char *fromfile, const char *tofile,
const char *schemaName, const char *relName)
{
int src_fd;
int dst_fd;
int blkno;
int bytesRead;
int totalBytesRead;
char *buf;
ssize_t writesize;
struct stat statbuf;
/*
* transfer_relfile() should never call us unless requested by the data
* checksum option but better doublecheck before we start rewriting data.
*/
if (user_opts.checksum_mode == CHECKSUM_NONE)
pg_log(PG_FATAL, "error, incorrect checksum configuration detected.\n");
if ((src_fd = open(fromfile, O_RDONLY | PG_BINARY, 0)) < 0)
pg_log(PG_FATAL,
"error while rewriting relation \"%s.%s\": could not open file \"%s\": %s\n",
schemaName, relName, fromfile, strerror(errno));
if (fstat(src_fd, &statbuf) != 0)
pg_log(PG_FATAL,
"error while rewriting relation \"%s.%s\": could not stat file \"%s\": %s\n",
schemaName, relName, fromfile, strerror(errno));
if ((dst_fd = open(tofile, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, S_IRUSR | S_IWUSR)) < 0)
pg_log(PG_FATAL,
"error while rewriting relation \"%s.%s\": could not create file \"%s\": %s\n",
schemaName, relName, tofile, strerror(errno));
blkno = 0;
totalBytesRead = 0;
buf = (char *) pg_malloc(BLCKSZ);
while ((bytesRead = read(src_fd, buf, BLCKSZ)) == BLCKSZ)
{
Size page_size;
page_size = PageGetPageSize((PageHeader) buf);
if (!PageSizeIsValid(page_size) && page_size != 0)
pg_log(PG_FATAL,
"error while rewriting relation \"%s.%s\": invalid page size detected (%zd)\n",
schemaName, relName, page_size);
if (!PageIsNew(buf))
{
if (user_opts.checksum_mode == CHECKSUM_ADD)
((PageHeader) buf)->pd_checksum = pg_checksum_page(buf, blkno);
else
memset(&(((PageHeader) buf)->pd_checksum), 0, sizeof(uint16));
}
writesize = write(dst_fd, buf, BLCKSZ);
if (writesize != BLCKSZ)
pg_log(PG_FATAL, "error when rewriting relation \"%s.%s\": %s",
schemaName, relName, strerror(errno));
blkno++;
totalBytesRead += BLCKSZ;
}
if (totalBytesRead != statbuf.st_size)
pg_log(PG_FATAL,
"error when rewriting relation \"%s.%s\": torn read on file \"%s\"%c %s\n",
schemaName, relName, fromfile,
(errno != 0 ? ':' : ' '), strerror(errno));
pg_free(buf);
close(dst_fd);
close(src_fd);
}
contrib/pg_upgrade/function.c
浏览文件 @ 189a3547
......@@ -271,7 +271,7 @@ get_loadable_libraries(void)
"SELECT DISTINCT probin "
"FROM pg_catalog.pg_proc "
"WHERE prolang = 13 /* C */ AND "
" probin IS NOT NULL AND "
" probin IS NOT NULL AND "
" %s "
" oid >= %u;",
pg83_str,
......
contrib/pg_upgrade/info.c
浏览文件 @ 189a3547
......@@ -325,69 +325,6 @@ get_db_infos(ClusterInfo *cluster)
cluster->dbarr.ndbs = ntups;
}
/*
* get_numeric_types()
*
* queries the cluster for all types based on NUMERIC, as well as NUMERIC
* itself, and return an InvalidOid terminated array of pg_type Oids for
* these types.
*/
static Oid *
get_numeric_types(PGconn *conn)
{
char query[QUERY_ALLOC];
PGresult *res;
Oid *result;
int result_count = 0;
int iterator = 0;
/*
* We don't know beforehands how many types based on NUMERIC that we will
* find so allocate space that "should be enough". Once processed we can
* shrink the allocation or we could've put these on the stack and moved
* to a heap based allocation at that point - but even at a too large
* array we still waste very little memory in the grand scheme of things
* so keep it simple and leave it be with an overflow check instead.
*/
result = pg_malloc(sizeof(Oid) * NUMERIC_ALLOC);
memset(result, InvalidOid, NUMERIC_ALLOC);
result[result_count++] = 1700; /* 1700 == NUMERICOID */
/*
* iterator is a trailing pointer into the array which traverses the
* array one by one while result_count fills it - and can do so by
* adding n items per loop iteration. Once iterator has caught up with
* result_count we know that no more pg_type tuples are of interest.
* This is a handcoded version of WITH RECURSIVE and can be replaced
* by an actual recursive CTE once GPDB supports these.
*/
while (iterator < result_count && result[iterator] != InvalidOid)
{
snprintf(query, sizeof(query),
"SELECT typ.oid AS typoid, base.oid AS baseoid "
"FROM pg_type base "
" JOIN pg_type typ ON base.oid = typ.typbasetype "
"WHERE base.typbasetype = '%u'::pg_catalog.oid;",
result[iterator++]);
res = executeQueryOrDie(conn, query);
if (PQntuples(res) > 0)
{
result[result_count++] = atooid(PQgetvalue(res, 0, PQfnumber(res, "typoid")));
result[result_count++] = atooid(PQgetvalue(res, 0, PQfnumber(res, "baseoid")));
}
PQclear(res);
if (result_count == NUMERIC_ALLOC - 1)
pg_log(PG_FATAL, "Too many NUMERIC types found");
}
return result;
}
/*
* get_rel_infos()
*
......@@ -963,6 +900,63 @@ free_db_and_rel_infos(DbInfoArr *db_arr)
}
#if 0 /* Not used in GPDB */
/*
* relarr_lookup_rel()
*
* Searches "relname" in rel_arr. Returns the *real* pointer to the
* RelInfo structure.
*/
static RelInfo *
relarr_lookup_rel(migratorContext *ctx, RelInfoArr *rel_arr,
const char *nspname, const char *relname,
Cluster whichCluster)
{
int relnum;
if (!rel_arr || !relname)
return NULL;
for (relnum = 0; relnum < rel_arr->nrels; relnum++)
{
if (strcmp(rel_arr->rels[relnum].nspname, nspname) == 0 &&
strcmp(rel_arr->rels[relnum].relname, relname) == 0)
return &rel_arr->rels[relnum];
}
pg_log(ctx, PG_FATAL, "Could not find %s.%s in %s cluster\n",
nspname, relname, CLUSTERNAME(whichCluster));
return NULL;
}
/*
* relarr_lookup_reloid()
*
* Returns a pointer to the RelInfo structure for the
* given oid or NULL if the desired entry cannot be
* found.
*/
static RelInfo *
relarr_lookup_reloid(migratorContext *ctx, RelInfoArr *rel_arr, Oid oid,
Cluster whichCluster)
{
int relnum;
if (!rel_arr || !oid)
return NULL;
for (relnum = 0; relnum < rel_arr->nrels; relnum++)
{
if (rel_arr->rels[relnum].reloid == oid)
return &rel_arr->rels[relnum];
}
pg_log(ctx, PG_FATAL, "Could not find %d in %s cluster\n",
oid, CLUSTERNAME(whichCluster));
return NULL;
}
#endif
static void
free_rel_infos(RelInfoArr *rel_arr)
{
......
contrib/pg_upgrade/oid_dump.c
浏览文件 @ 189a3547
......@@ -52,7 +52,7 @@
* into memory, and inject it to the pg_dump output, just after the \connect
* line.
*
* Copyright (c) 2017, Pivotal Software Inc
* Copyright (c) 2017-Present, Pivotal Software Inc
*/
#include "pg_upgrade.h"
......@@ -359,3 +359,37 @@ dump_rows(PQExpBuffer buf, FILE *file, PGconn *conn,
destroyPQExpBuffer(buf);
}
char *
get_preassigned_oids_for_db(char *line)
{
char *dbname;
int dbnum;
/*
* FIXME: parsing the dump like this is madness. We should use a dump file
* for each database to begin with. (like more recent version of
* PostgreSQL).
*/
if (strncmp(line, "\\connect ", strlen("\\connect ")) != 0)
return NULL;
dbname = line + strlen("\\connect ");
/* strip newline */
if (strlen(dbname) <= 1)
return NULL;
dbname[strlen(dbname) - 1] = '\0';
for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
{
DbInfo *olddb = &old_cluster.dbarr.dbs[dbnum];
if (strcmp(olddb->db_name, dbname) == 0)
{
/* Found it! */
return olddb->reserved_oids;
}
}
return NULL;
}
contrib/pg_upgrade/pg_upgrade.c
浏览文件 @ 189a3547
......@@ -49,7 +49,6 @@ static void prepare_new_cluster(void);
static void prepare_new_databases(void);
static void create_new_objects(void);
static void copy_clog_xlog_xid(void);
static void copy_distributedlog(void);
static void set_frozenxids(void);
static void setup(char *argv0, bool live_check);
static void cleanup(void);
......@@ -508,38 +507,6 @@ create_new_objects(void)
dump_new_oids();
}
/*
* In upgrading from GPDB4, copy the pg_distributedlog over in
* vanilla. The assumption that this works needs to be verified
*/
static void
copy_distributedlog(void)
{
char old_dlog_path[MAXPGPATH];
char new_dlog_path[MAXPGPATH];
prep_status("Deleting new distributedlog");
snprintf(old_dlog_path, sizeof(old_dlog_path), "%s/pg_distributedlog", old_cluster.pgdata);
snprintf(new_dlog_path, sizeof(new_dlog_path), "%s/pg_distributedlog", new_cluster.pgdata);
if (rmtree(new_dlog_path, true) != true)
pg_log(PG_FATAL, "Unable to delete directory %s\n", new_dlog_path);
check_ok();
prep_status("Copying old distributedlog to new server");
/* libpgport's copydir() doesn't work in FRONTEND code */
#ifndef WIN32
exec_prog(true, SYSTEMQUOTE "%s \"%s\" \"%s\"" SYSTEMQUOTE,
"cp -Rf",
#else
/* flags: everything, no confirm, quiet, overwrite read-only */
exec_prog(true, SYSTEMQUOTE "%s \"%s\" \"%s\\\"" SYSTEMQUOTE,
"xcopy /e /y /q /r",
#endif
old_dlog_path, new_dlog_path);
check_ok();
}
static void
copy_clog_xlog_xid(void)
{
......
contrib/pg_upgrade/pg_upgrade.h
浏览文件 @ 189a3547
......@@ -470,8 +470,6 @@ const char *linkAndUpdateFile(pageCnvCtx *pageConverter, const char *src,
const char *dst);
void check_hard_link(void);
void rewriteHeapPageChecksum(const char *fromfile, const char *tofile,
const char *schemaName, const char *relName);
/* function.c */
......@@ -500,13 +498,6 @@ void get_pg_database_relfilenode(ClusterInfo *cluster);
const char *transfer_all_new_dbs(DbInfoArr *olddb_arr,
DbInfoArr *newdb_arr, char *old_pgdata, char *new_pgdata);
/* aotable.c */
void restore_aosegment_tables(void);
/* gpdb4_heap_convert.c */
const char *convert_gpdb4_heap_file(const char *src, const char *dst,
bool has_numerics, AttInfo *atts, int natts);
/* tablespace.c */
void init_tablespaces(void);
......@@ -538,16 +529,12 @@ void pg_free(void *ptr);
const char *getErrorText(int errNum);
unsigned int str2uint(const char *str);
void pg_putenv(const char *var, const char *val);
void report_progress(ClusterInfo *cluster, progress_type op, char *fmt,...);
void close_progress(void);
/* version.c */
void new_9_0_populate_pg_largeobject_metadata(ClusterInfo *cluster,
bool check_mode);
void new_gpdb5_0_invalidate_indexes(bool check_mode);
void new_gpdb_invalidate_bitmap_indexes(bool check_mode);
/* version_old_8_3.c */
......@@ -560,15 +547,6 @@ void old_8_3_invalidate_bpchar_pattern_ops_indexes(ClusterInfo *cluster,
bool check_mode);
char *old_8_3_create_sequence_script(ClusterInfo *cluster);
/* version_old_gpdb4.c */
void old_GPDB4_check_for_money_data_type_usage(ClusterInfo *cluster);
void old_GPDB4_check_no_free_aoseg(ClusterInfo *cluster);
/* oid_dump.c */
void dump_new_oids(void);
void get_old_oids(void);
void slurp_oid_files(void);
/*
* Hack to make backend macros that check for assertions to work.
*/
......@@ -580,3 +558,43 @@ void slurp_oid_files(void);
#undef Assert
#endif
#define Assert(condition) ((void) (true || (condition)))
/* aotable.c */
void restore_aosegment_tables(void);
/* gpdb4_heap_convert.c */
const char *convert_gpdb4_heap_file(const char *src, const char *dst,
bool has_numerics, AttInfo *atts, int natts);
/* file_gp.c */
void copy_distributedlog(void);
void rewriteHeapPageChecksum(const char *fromfile, const char *tofile,
const char *schemaName, const char *relName);
/* version_old_gpdb4.c */
void old_GPDB4_check_for_money_data_type_usage(ClusterInfo *cluster);
void old_GPDB4_check_no_free_aoseg(ClusterInfo *cluster);
void check_hash_partition_usage(void);
void new_gpdb5_0_invalidate_indexes(bool check_mode);
void new_gpdb_invalidate_bitmap_indexes(bool check_mode);
Oid *get_numeric_types(PGconn *conn);
/* oid_dump.c */
void dump_new_oids(void);
void get_old_oids(void);
void slurp_oid_files(void);
char *get_preassigned_oids_for_db(char *line);
/* check_gp.c */
void check_greenplum(void);
/* reporting.c */
void report_progress(ClusterInfo *cluster, progress_type op, char *fmt,...);
void close_progress(void);
contrib/pg_upgrade/reporting.c 0 → 100644
浏览文件 @ 189a3547
/*
* reporting.c
*
* runtime reporting functions
*
* Copyright (c) 2017-Present, Pivotal Software Inc.
*/
#include "pg_upgrade.h"
#include <time.h>
static FILE *progress_file = NULL;
static int progress_id = 0;
static int progress_counter = 0;
static unsigned long progress_prev = 0;
/* Number of operations per progress report file */
#define OP_PER_PROGRESS 25
#define TS_PER_PROGRESS (5 * 1000000)
static char *
opname(progress_type op)
{
char *ret = "unknown";
switch(op)
{
case CHECK:
ret = "check";
break;
case SCHEMA_DUMP:
ret = "dump";
break;
case SCHEMA_RESTORE:
ret = "restore";
break;
case FILE_MAP:
ret = "map";
break;
case FILE_COPY:
ret = "copy";
break;
case FIXUP:
ret = "fixup";
break;
case ABORT:
ret = "error";
break;
case DONE:
ret = "done";
break;
default:
break;
}
return ret;
}
static unsigned long
epoch_us(void)
{
struct timeval tv;
gettimeofday(&tv, NULL);
return (tv.tv_sec) * 1000000 + tv.tv_usec;
}
void
report_progress(ClusterInfo *cluster, progress_type op, char *fmt,...)
{
va_list args;
char message[MAX_STRING];
char filename[MAXPGPATH];
unsigned long ts;
if (!log_opts.progress)
return;
ts = epoch_us();
va_start(args, fmt);
vsnprintf(message, sizeof(message), fmt, args);
va_end(args);
if (!progress_file)
{
snprintf(filename, sizeof(filename), "%s/%d.inprogress",
os_info.cwd, ++progress_id);
if ((progress_file = fopen(filename, "w")) == NULL)
pg_log(PG_FATAL, "Could not create progress file: %s\n", filename);
}
fprintf(progress_file, "%lu;%s;%s;%s;\n",
epoch_us(), CLUSTER_NAME(cluster), opname(op), message);
progress_counter++;
/*
* Swap the progress report to a new file if we have exceeded the max
* number of operations per file as well as the minumum time per report. We
* want to avoid too frequent reports while still providing timely feedback
* to the user.
*/
if ((progress_counter > OP_PER_PROGRESS) && (ts > progress_prev + TS_PER_PROGRESS))
close_progress();
}
void
close_progress(void)
{
char old[MAXPGPATH];
char new[MAXPGPATH];
if (!log_opts.progress || !progress_file)
return;
snprintf(old, sizeof(old), "%s/%d.inprogress", os_info.cwd, progress_id);
snprintf(new, sizeof(new), "%s/%d.done", os_info.cwd, progress_id);
fclose(progress_file);
progress_file = NULL;
rename(old, new);
progress_counter = 0;
progress_prev = epoch_us();
}
contrib/pg_upgrade/util.c
浏览文件 @ 189a3547
......@@ -10,16 +10,7 @@
#include "pg_upgrade.h"
#include <signal.h>
#include <time.h>
static FILE *progress_file = NULL;
static int progress_id = 0;
static int progress_counter = 0;
static unsigned long progress_prev = 0;
/* Number of operations per progress report file */
#define OP_PER_PROGRESS 25
#define TS_PER_PROGRESS (5 * 1000000)
LogOpts log_opts;
......@@ -289,110 +280,3 @@ pg_putenv(const char *var, const char *val)
#endif
}
}
static char *
opname(progress_type op)
{
char *ret = "unknown";
switch(op)
{
case CHECK:
ret = "check";
break;
case SCHEMA_DUMP:
ret = "dump";
break;
case SCHEMA_RESTORE:
ret = "restore";
break;
case FILE_MAP:
ret = "map";
break;
case FILE_COPY:
ret = "copy";
break;
case FIXUP:
ret = "fixup";
break;
case ABORT:
ret = "error";
break;
case DONE:
ret = "done";
break;
default:
break;
}
return ret;
}
static unsigned long
epoch_us(void)
{
struct timeval tv;
gettimeofday(&tv, NULL);
return (tv.tv_sec) * 1000000 + tv.tv_usec;
}
void
report_progress(ClusterInfo *cluster, progress_type op, char *fmt,...)
{
va_list args;
char message[MAX_STRING];
char filename[MAXPGPATH];
unsigned long ts;
if (!log_opts.progress)
return;
ts = epoch_us();
va_start(args, fmt);
vsnprintf(message, sizeof(message), fmt, args);
va_end(args);
if (!progress_file)
{
snprintf(filename, sizeof(filename), "%s/%d.inprogress",
os_info.cwd, ++progress_id);
if ((progress_file = fopen(filename, "w")) == NULL)
pg_log(PG_FATAL, "Could not create progress file: %s\n", filename);
}
fprintf(progress_file, "%lu;%s;%s;%s;\n",
epoch_us(), CLUSTER_NAME(cluster), opname(op), message);
progress_counter++;
/*
* Swap the progress report to a new file if we have exceeded the max
* number of operations per file as well as the minumum time per report. We
* want to avoid too frequent reports while still providing timely feedback
* to the user.
*/
if ((progress_counter > OP_PER_PROGRESS) && (ts > progress_prev + TS_PER_PROGRESS))
close_progress();
}
void
close_progress(void)
{
char old[MAXPGPATH];
char new[MAXPGPATH];
if (!log_opts.progress || !progress_file)
return;
snprintf(old, sizeof(old), "%s/%d.inprogress", os_info.cwd, progress_id);
snprintf(new, sizeof(new), "%s/%d.done", os_info.cwd, progress_id);
fclose(progress_file);
progress_file = NULL;
rename(old, new);
progress_counter = 0;
progress_prev = epoch_us();
}
contrib/pg_upgrade/version.c
浏览文件 @ 189a3547
......@@ -89,125 +89,3 @@ new_9_0_populate_pg_largeobject_metadata(ClusterInfo *cluster, bool check_mode)
else
check_ok();
}
/*
* new_gpdb5_0_invalidate_indexes()
* new >= GPDB 5.0, old <= GPDB 4.3
*
* GPDB 5.0 follows the PostgreSQL 8.3 page format, while GPDB 4.3 used
* the 8.2 format. A new field was added to the page header, so we need
* mark all indexes as invalid.
*/
void
new_gpdb5_0_invalidate_indexes(bool check_mode)
{
int dbnum;
FILE *script = NULL;
char output_path[MAXPGPATH];
prep_status("Invalidating indexes in new cluster");
snprintf(output_path, sizeof(output_path), "%s/reindex_all.sql",
os_info.cwd);
if (!check_mode)
{
if ((script = fopen(output_path, "w")) == NULL)
pg_log(PG_FATAL, "Could not create necessary file: %s\n",
output_path);
}
for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
{
DbInfo *olddb = &old_cluster.dbarr.dbs[dbnum];
PGconn *conn = connectToServer(&new_cluster, olddb->db_name);
char query[QUERY_ALLOC];
/*
* GPDB doesn't allow hacking the catalogs without setting
* allow_system_table_mods first.
*/
PQclear(executeQueryOrDie(conn, "set allow_system_table_mods='dml'"));
/*
* check_mode doesn't do much interesting for this but at least
* we'll know we are allowed to change allow_system_table_mods
* which is required
*/
if (!check_mode)
{
snprintf(query, sizeof(query),
"UPDATE pg_index SET indisvalid = false WHERE indexrelid >= %u",
FirstNormalObjectId);
PQclear(executeQueryOrDie(conn, query));
fprintf(script, "\\connect %s\n",
quote_identifier(olddb->db_name));
fprintf(script, "REINDEX DATABASE %s;\n",
quote_identifier(olddb->db_name));
}
PQfinish(conn);
}
if (!check_mode)
fclose(script);
report_status(PG_WARNING, "warning");
if (check_mode)
pg_log(PG_WARNING, "\n"
"| All indexes have different internal formats\n"
"| between your old and new clusters so they must\n"
"| be reindexed with the REINDEX command. After\n"
"| migration, you will be given REINDEX instructions.\n\n");
else
pg_log(PG_WARNING, "\n"
"| All indexes have different internal formats\n"
"| between your old and new clusters so they must\n"
"| be reindexed with the REINDEX command.\n"
"| The file:\n"
"| \t%s\n"
"| when executed by psql by the database super-user\n"
"| will recreate all invalid indexes.\n\n",
output_path);
}
/*
* new_gpdb_invalidate_bitmap_indexes()
*
* TODO: We are currently missing the support to migrate over bitmap indexes.
* Hence, mark all bitmap indexes as invalid.
*/
void
new_gpdb_invalidate_bitmap_indexes(bool check_mode)
{
int dbnum;
prep_status("Invalidating indexes in new cluster");
for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
{
DbInfo *olddb = &old_cluster.dbarr.dbs[dbnum];
PGconn *conn = connectToServer(&new_cluster, olddb->db_name);
/*
* GPDB doesn't allow hacking the catalogs without setting
* allow_system_table_mods first.
*/
PQclear(executeQueryOrDie(conn, "set allow_system_table_mods='dml'"));
/*
* check_mode doesn't do much interesting for this but at least
* we'll know we are allowed to change allow_system_table_mods
* which is required
*/
if (!check_mode)
{
PQclear(executeQueryOrDie(conn,
"UPDATE pg_index SET indisvalid = false FROM pg_class c WHERE c.oid = indexrelid AND indexrelid >= %u AND relam = 3013;",
FirstNormalObjectId));
}
PQfinish(conn);
}
check_ok();
}
contrib/pg_upgrade/version_old_gpdb4.c → contrib/pg_upgrade/version_gp.c
浏览文件 @ 189a3547
/*
* version_old_gpdb4.c
*
* GPDB-version-specific routines
* GPDB-version-specific routines for upgrades from Greenplum 4.3.x
*
* Copyright (c) 2016, Pivotal Software Inc
* Copyright (c) 2016-Present Pivotal Software, Inc
*/
#include "pg_upgrade.h"
......@@ -199,3 +199,268 @@ old_GPDB4_check_no_free_aoseg(ClusterInfo *cluster)
else
check_ok();
}
/*
* check_hash_partition_usage()
* 8.3 -> 8.4
*
* Hash partitioning was never officially supported in GPDB5 and was removed
* in GPDB6, but better check just in case someone has found the hidden GUC
* and used them anyway.
*
* The hash algorithm was changed in 8.4, so upgrading is impossible anyway.
* This is basically the same problem as with hash indexes in PostgreSQL.
*/
void
check_hash_partition_usage(void)
{
int dbnum;
FILE *script = NULL;
bool found = false;
char output_path[MAXPGPATH];
prep_status("Checking for hash partitioned tables");
snprintf(output_path, sizeof(output_path), "%s/hash_partitioned_tables.txt",
os_info.cwd);
for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
{
PGresult *res;
bool db_used = false;
int ntups;
int rowno;
int i_nspname,
i_relname;
DbInfo *active_db = &old_cluster.dbarr.dbs[dbnum];
PGconn *conn = connectToServer(&old_cluster, active_db->db_name);
res = executeQueryOrDie(conn,
"SELECT n.nspname, c.relname "
"FROM pg_catalog.pg_partition p, pg_catalog.pg_class c, pg_catalog.pg_namespace n "
"WHERE p.parrelid = c.oid AND c.relnamespace = n.oid "
"AND parkind = 'h'");
ntups = PQntuples(res);
i_nspname = PQfnumber(res, "nspname");
i_relname = PQfnumber(res, "relname");
for (rowno = 0; rowno < ntups; rowno++)
{
found = true;
if (script == NULL && (script = fopen(output_path, "w")) == NULL)
pg_log(PG_FATAL, "Could not create necessary file: %s\n", output_path);
if (!db_used)
{
fprintf(script, "Database: %s\n", active_db->db_name);
db_used = true;
}
fprintf(script, " %s.%s\n",
PQgetvalue(res, rowno, i_nspname),
PQgetvalue(res, rowno, i_relname));
}
PQclear(res);
PQfinish(conn);
}
if (found)
{
fclose(script);
pg_log(PG_REPORT, "fatal\n");
pg_log(PG_FATAL,
"| Your installation contains hash partitioned tables.\n"
"| Upgrading hash partitioned tables is not supported,\n"
"| so this cluster cannot currently be upgraded. You\n"
"| can remove the problem tables and restart the\n"
"| migration. A list of the problem tables is in the\n"
"| file:\n"
"| \t%s\n\n", output_path);
}
else
check_ok();
}
/*
* new_gpdb5_0_invalidate_indexes()
* new >= GPDB 5.0, old <= GPDB 4.3
*
* GPDB 5.0 follows the PostgreSQL 8.3 page format, while GPDB 4.3 used
* the 8.2 format. A new field was added to the page header, so we need
* mark all indexes as invalid.
*/
void
new_gpdb5_0_invalidate_indexes(bool check_mode)
{
int dbnum;
FILE *script = NULL;
char output_path[MAXPGPATH];
prep_status("Invalidating indexes in new cluster");
snprintf(output_path, sizeof(output_path), "%s/reindex_all.sql",
os_info.cwd);
if (!check_mode)
{
if ((script = fopen(output_path, "w")) == NULL)
pg_log(PG_FATAL, "Could not create necessary file: %s\n",
output_path);
}
for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
{
DbInfo *olddb = &old_cluster.dbarr.dbs[dbnum];
PGconn *conn = connectToServer(&new_cluster, olddb->db_name);
char query[QUERY_ALLOC];
/*
* GPDB doesn't allow hacking the catalogs without setting
* allow_system_table_mods first.
*/
PQclear(executeQueryOrDie(conn, "set allow_system_table_mods='dml'"));
/*
* check_mode doesn't do much interesting for this but at least
* we'll know we are allowed to change allow_system_table_mods
* which is required
*/
if (!check_mode)
{
snprintf(query, sizeof(query),
"UPDATE pg_index SET indisvalid = false WHERE indexrelid >= %u",
FirstNormalObjectId);
PQclear(executeQueryOrDie(conn, query));
fprintf(script, "\\connect %s\n",
quote_identifier(olddb->db_name));
fprintf(script, "REINDEX DATABASE %s;\n",
quote_identifier(olddb->db_name));
}
PQfinish(conn);
}
if (!check_mode)
fclose(script);
report_status(PG_WARNING, "warning");
if (check_mode)
pg_log(PG_WARNING, "\n"
"| All indexes have different internal formats\n"
"| between your old and new clusters so they must\n"
"| be reindexed with the REINDEX command. After\n"
"| migration, you will be given REINDEX instructions.\n\n");
else
pg_log(PG_WARNING, "\n"
"| All indexes have different internal formats\n"
"| between your old and new clusters so they must\n"
"| be reindexed with the REINDEX command.\n"
"| The file:\n"
"| \t%s\n"
"| when executed by psql by the database super-user\n"
"| will recreate all invalid indexes.\n\n",
output_path);
}
/*
* new_gpdb_invalidate_bitmap_indexes()
*
* TODO: We are currently missing the support to migrate over bitmap indexes.
* Hence, mark all bitmap indexes as invalid.
*/
void
new_gpdb_invalidate_bitmap_indexes(bool check_mode)
{
int dbnum;
prep_status("Invalidating indexes in new cluster");
for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
{
DbInfo *olddb = &old_cluster.dbarr.dbs[dbnum];
PGconn *conn = connectToServer(&new_cluster, olddb->db_name);
/*
* GPDB doesn't allow hacking the catalogs without setting
* allow_system_table_mods first.
*/
PQclear(executeQueryOrDie(conn, "set allow_system_table_mods='dml'"));
/*
* check_mode doesn't do much interesting for this but at least
* we'll know we are allowed to change allow_system_table_mods
* which is required
*/
if (!check_mode)
{
PQclear(executeQueryOrDie(conn,
"UPDATE pg_index SET indisvalid = false FROM pg_class c WHERE c.oid = indexrelid AND indexrelid >= %u AND relam = 3013;",
FirstNormalObjectId));
}
PQfinish(conn);
}
check_ok();
}
/*
* get_numeric_types()
*
* queries the cluster for all types based on NUMERIC, as well as NUMERIC
* itself, and return an InvalidOid terminated array of pg_type Oids for
* these types.
*/
Oid *
get_numeric_types(PGconn *conn)
{
char query[QUERY_ALLOC];
PGresult *res;
Oid *result;
int result_count = 0;
int iterator = 0;
/*
* We don't know beforehands how many types based on NUMERIC that we will
* find so allocate space that "should be enough". Once processed we can
* shrink the allocation or we could've put these on the stack and moved
* to a heap based allocation at that point - but even at a too large
* array we still waste very little memory in the grand scheme of things
* so keep it simple and leave it be with an overflow check instead.
*/
result = pg_malloc(sizeof(Oid) * NUMERIC_ALLOC);
memset(result, InvalidOid, NUMERIC_ALLOC);
result[result_count++] = 1700; /* 1700 == NUMERICOID */
/*
* iterator is a trailing pointer into the array which traverses the
* array one by one while result_count fills it - and can do so by
* adding n items per loop iteration. Once iterator has caught up with
* result_count we know that no more pg_type tuples are of interest.
* This is a handcoded version of WITH RECURSIVE and can be replaced
* by an actual recursive CTE once GPDB supports these.
*/
while (iterator < result_count && result[iterator] != InvalidOid)
{
snprintf(query, sizeof(query),
"SELECT typ.oid AS typoid, base.oid AS baseoid "
"FROM pg_type base "
" JOIN pg_type typ ON base.oid = typ.typbasetype "
"WHERE base.typbasetype = '%u'::pg_catalog.oid;",
result[iterator++]);
res = executeQueryOrDie(conn, query);
if (PQntuples(res) > 0)
{
result[result_count++] = atooid(PQgetvalue(res, 0, PQfnumber(res, "typoid")));
result[result_count++] = atooid(PQgetvalue(res, 0, PQfnumber(res, "baseoid")));
}
PQclear(res);
if (result_count == NUMERIC_ALLOC - 1)
pg_log(PG_FATAL, "Too many NUMERIC types found");
}
return result;
}
contrib/pg_upgrade/version_old_8_3.c
浏览文件 @ 189a3547
......@@ -54,8 +54,8 @@ old_8_3_check_for_name_data_type_usage(ClusterInfo *cluster)
"FROM pg_catalog.pg_class c, "
" pg_catalog.pg_namespace n, "
" pg_catalog.pg_attribute a "
"WHERE c.relkind='r' AND "
" c.oid = a.attrelid AND "
"WHERE c.oid = a.attrelid AND "
" c.relkind='r' AND "
" a.attnum > 1 AND "
" NOT a.attisdropped AND "
" a.atttypid = 'pg_catalog.name'::pg_catalog.regtype AND "
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或