From 9d701e624f4b9386cbd99519dab7936afe3d5aed Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Fri, 21 Aug 2020 20:48:59 +0900 Subject: [PATCH 001/589] Rework EXPLAIN for planner's buffer usage. Commit ce77abe63c allowed EXPLAIN (BUFFERS) to report the information on buffer usage during planning phase. However three issues were reported regarding this feature. (1) Previously, EXPLAIN option BUFFERS required ANALYZE. So the query had to be actually executed by specifying ANALYZE even when we want to see only the planner's buffer usage. This was inconvenient especially when the query was write one like DELETE. (2) EXPLAIN included the planner's buffer usage in summary information. So SUMMARY option had to be enabled to report that. Also this format was confusing. (3) The output structure for planning information was not consistent between TEXT format and the others. For example, "Planning" tag was output in JSON format, but not in TEXT format. For (1), this commit allows us to perform EXPLAIN (BUFFERS) without ANALYZE to report the planner's buffer usage. For (2), this commit changed EXPLAIN output so that the planner's buffer usage is reported before summary information. For (3), this commit made the output structure for planning information more consistent between the formats. Back-patch to v13 where the planner's buffer usage was allowed to be reported in EXPLAIN. Reported-by: Pierre Giraud, David Rowley Author: Fujii Masao Reviewed-by: David Rowley, Julien Rouhaud, Pierre Giraud Discussion: https://postgr.es/m/07b226e6-fa49-687f-b110-b7c37572f69e@dalibo.com --- doc/src/sgml/ref/explain.sgml | 3 +- src/backend/commands/explain.c | 46 +++++++++++----------- src/test/regress/expected/explain.out | 55 +++++++++++++++++++++++++-- src/test/regress/sql/explain.sql | 2 + 4 files changed, 77 insertions(+), 29 deletions(-) diff --git a/doc/src/sgml/ref/explain.sgml b/doc/src/sgml/ref/explain.sgml index 1c19e254dc24..906b2ccd50a2 100644 --- a/doc/src/sgml/ref/explain.sgml +++ b/doc/src/sgml/ref/explain.sgml @@ -187,8 +187,7 @@ ROLLBACK; query processing. The number of blocks shown for an upper-level node includes those used by all its child nodes. In text - format, only non-zero values are printed. This parameter may only be - used when ANALYZE is also enabled. It defaults to + format, only non-zero values are printed. It defaults to FALSE. diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 30e0a7ee7f21..c98c9b5547c5 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -116,7 +116,8 @@ static void show_instrumentation_count(const char *qlabel, int which, static void show_foreignscan_info(ForeignScanState *fsstate, ExplainState *es); static void show_eval_params(Bitmapset *bms_params, ExplainState *es); static const char *explain_get_index_name(Oid indexId); -static void show_buffer_usage(ExplainState *es, const BufferUsage *usage); +static void show_buffer_usage(ExplainState *es, const BufferUsage *usage, + bool planning); static void show_wal_usage(ExplainState *es, const WalUsage *usage); static void ExplainIndexScanDetails(Oid indexid, ScanDirection indexorderdir, ExplainState *es); @@ -221,11 +222,6 @@ ExplainQuery(ParseState *pstate, ExplainStmt *stmt, parser_errposition(pstate, opt->location))); } - if (es->buffers && !es->analyze) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("EXPLAIN option BUFFERS requires ANALYZE"))); - if (es->wal && !es->analyze) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), @@ -586,8 +582,13 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, /* Create textual dump of plan tree */ ExplainPrintPlan(es, queryDesc); - if (es->summary && (planduration || bufusage)) + /* Show buffer usage in planning */ + if (bufusage) + { ExplainOpenGroup("Planning", "Planning", true, es); + show_buffer_usage(es, bufusage, true); + ExplainCloseGroup("Planning", "Planning", true, es); + } if (es->summary && planduration) { @@ -596,19 +597,6 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, ExplainPropertyFloat("Planning Time", "ms", 1000.0 * plantime, 3, es); } - /* Show buffer usage */ - if (es->summary && bufusage) - { - if (es->format == EXPLAIN_FORMAT_TEXT) - es->indent++; - show_buffer_usage(es, bufusage); - if (es->format == EXPLAIN_FORMAT_TEXT) - es->indent--; - } - - if (es->summary && (planduration || bufusage)) - ExplainCloseGroup("Planning", "Planning", true, es); - /* Print info about runtime of triggers */ if (es->analyze) ExplainPrintTriggers(es, queryDesc); @@ -1996,7 +1984,7 @@ ExplainNode(PlanState *planstate, List *ancestors, /* Show buffer/WAL usage */ if (es->buffers && planstate->instrument) - show_buffer_usage(es, &planstate->instrument->bufusage); + show_buffer_usage(es, &planstate->instrument->bufusage, false); if (es->wal && planstate->instrument) show_wal_usage(es, &planstate->instrument->walusage); @@ -2015,7 +2003,7 @@ ExplainNode(PlanState *planstate, List *ancestors, ExplainOpenWorker(n, es); if (es->buffers) - show_buffer_usage(es, &instrument->bufusage); + show_buffer_usage(es, &instrument->bufusage, false); if (es->wal) show_wal_usage(es, &instrument->walusage); ExplainCloseWorker(n, es); @@ -3301,7 +3289,7 @@ explain_get_index_name(Oid indexId) * Show buffer usage details. */ static void -show_buffer_usage(ExplainState *es, const BufferUsage *usage) +show_buffer_usage(ExplainState *es, const BufferUsage *usage, bool planning) { if (es->format == EXPLAIN_FORMAT_TEXT) { @@ -3317,6 +3305,15 @@ show_buffer_usage(ExplainState *es, const BufferUsage *usage) usage->temp_blks_written > 0); bool has_timing = (!INSTR_TIME_IS_ZERO(usage->blk_read_time) || !INSTR_TIME_IS_ZERO(usage->blk_write_time)); + bool show_planning = (planning && (has_shared || + has_local || has_temp || has_timing)); + + if (show_planning) + { + ExplainIndentText(es); + appendStringInfoString(es->str, "Planning:\n"); + es->indent++; + } /* Show only positive counter values. */ if (has_shared || has_local || has_temp) @@ -3386,6 +3383,9 @@ show_buffer_usage(ExplainState *es, const BufferUsage *usage) INSTR_TIME_GET_MILLISEC(usage->blk_write_time)); appendStringInfoChar(es->str, '\n'); } + + if (show_planning) + es->indent--; } else { diff --git a/src/test/regress/expected/explain.out b/src/test/regress/expected/explain.out index 96baba038c2e..a1ee6c679256 100644 --- a/src/test/regress/expected/explain.out +++ b/src/test/regress/expected/explain.out @@ -106,7 +106,6 @@ select explain_filter('explain (analyze, buffers, format json) select * from int "Temp Written Blocks": N + }, + "Planning": { + - "Planning Time": N.N, + "Shared Hit Blocks": N, + "Shared Read Blocks": N, + "Shared Dirtied Blocks": N, + @@ -118,6 +117,7 @@ select explain_filter('explain (analyze, buffers, format json) select * from int "Temp Read Blocks": N, + "Temp Written Blocks": N + }, + + "Planning Time": N.N, + "Triggers": [ + ], + "Execution Time": N.N + @@ -155,7 +155,6 @@ select explain_filter('explain (analyze, buffers, format xml) select * from int8 N + + + - N.N + N + N + N+ @@ -167,6 +166,7 @@ select explain_filter('explain (analyze, buffers, format xml) select * from int8 N + N + + + N.N + + + N.N + @@ -201,7 +201,6 @@ select explain_filter('explain (analyze, buffers, format yaml) select * from int Temp Read Blocks: N + Temp Written Blocks: N + Planning: + - Planning Time: N.N + Shared Hit Blocks: N + Shared Read Blocks: N + Shared Dirtied Blocks: N + @@ -212,10 +211,58 @@ select explain_filter('explain (analyze, buffers, format yaml) select * from int Local Written Blocks: N + Temp Read Blocks: N + Temp Written Blocks: N + + Planning Time: N.N + Triggers: + Execution Time: N.N (1 row) +select explain_filter('explain (buffers, format text) select * from int8_tbl i8'); + explain_filter +--------------------------------------------------------- + Seq Scan on int8_tbl i8 (cost=N.N..N.N rows=N width=N) +(1 row) + +select explain_filter('explain (buffers, format json) select * from int8_tbl i8'); + explain_filter +------------------------------------ + [ + + { + + "Plan": { + + "Node Type": "Seq Scan", + + "Parallel Aware": false, + + "Relation Name": "int8_tbl",+ + "Alias": "i8", + + "Startup Cost": N.N, + + "Total Cost": N.N, + + "Plan Rows": N, + + "Plan Width": N, + + "Shared Hit Blocks": N, + + "Shared Read Blocks": N, + + "Shared Dirtied Blocks": N, + + "Shared Written Blocks": N, + + "Local Hit Blocks": N, + + "Local Read Blocks": N, + + "Local Dirtied Blocks": N, + + "Local Written Blocks": N, + + "Temp Read Blocks": N, + + "Temp Written Blocks": N + + }, + + "Planning": { + + "Shared Hit Blocks": N, + + "Shared Read Blocks": N, + + "Shared Dirtied Blocks": N, + + "Shared Written Blocks": N, + + "Local Hit Blocks": N, + + "Local Read Blocks": N, + + "Local Dirtied Blocks": N, + + "Local Written Blocks": N, + + "Temp Read Blocks": N, + + "Temp Written Blocks": N + + } + + } + + ] +(1 row) + -- SETTINGS option -- We have to ignore other settings that might be imposed by the environment, -- so printing the whole Settings field unfortunately won't do. @@ -402,7 +449,6 @@ select jsonb_pretty( "Shared Written Blocks": 0 + }, + "Planning": { + - "Planning Time": 0.0, + "Local Hit Blocks": 0, + "Temp Read Blocks": 0, + "Local Read Blocks": 0, + @@ -416,6 +462,7 @@ select jsonb_pretty( }, + "Triggers": [ + ], + + "Planning Time": 0.0, + "Execution Time": 0.0 + } + ] diff --git a/src/test/regress/sql/explain.sql b/src/test/regress/sql/explain.sql index dce2a3420724..01783c607aa0 100644 --- a/src/test/regress/sql/explain.sql +++ b/src/test/regress/sql/explain.sql @@ -57,6 +57,8 @@ select explain_filter('explain (analyze, buffers, format text) select * from int select explain_filter('explain (analyze, buffers, format json) select * from int8_tbl i8'); select explain_filter('explain (analyze, buffers, format xml) select * from int8_tbl i8'); select explain_filter('explain (analyze, buffers, format yaml) select * from int8_tbl i8'); +select explain_filter('explain (buffers, format text) select * from int8_tbl i8'); +select explain_filter('explain (buffers, format json) select * from int8_tbl i8'); -- SETTINGS option -- We have to ignore other settings that might be imposed by the environment, From eabba4a3eb71b3886d0ec581155df6202b96b15a Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Sat, 22 Aug 2020 01:22:55 +0900 Subject: [PATCH 002/589] Fix explain regression test failure. Commit 9d701e624f caused the regression test for EXPLAIN to fail on the buildfarm member prion. This happened because of instability of test output, i.e., in text format, whether "Planning:" line is output varies depending on the system state. This commit updated the regression test so that it ignores that "Planning:" line to produce more stable test output and get rid of the test failure. Back-patch to v13. Author: Fujii Masao Discussion: https://postgr.es/m/1803897.1598021621@sss.pgh.pa.us --- src/test/regress/expected/explain.out | 3 +++ src/test/regress/sql/explain.sql | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/test/regress/expected/explain.out b/src/test/regress/expected/explain.out index a1ee6c679256..dc7ab2ce8bfb 100644 --- a/src/test/regress/expected/explain.out +++ b/src/test/regress/expected/explain.out @@ -23,6 +23,9 @@ begin -- Ignore text-mode buffers output because it varies depending -- on the system state CONTINUE WHEN (ln ~ ' +Buffers: .*'); + -- Ignore text-mode "Planning:" line because whether it's output + -- varies depending on the system state + CONTINUE WHEN (ln = 'Planning:'); return next ln; end loop; end; diff --git a/src/test/regress/sql/explain.sql b/src/test/regress/sql/explain.sql index 01783c607aa0..c79116c927b1 100644 --- a/src/test/regress/sql/explain.sql +++ b/src/test/regress/sql/explain.sql @@ -25,6 +25,9 @@ begin -- Ignore text-mode buffers output because it varies depending -- on the system state CONTINUE WHEN (ln ~ ' +Buffers: .*'); + -- Ignore text-mode "Planning:" line because whether it's output + -- varies depending on the system state + CONTINUE WHEN (ln = 'Planning:'); return next ln; end loop; end; From 50289819230d8ddad510879ee4793b04a05cf13b Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 21 Aug 2020 15:00:42 -0400 Subject: [PATCH 003/589] Fix handling of CREATE TABLE LIKE with inheritance. If a CREATE TABLE command uses both LIKE and traditional inheritance, Vars in CHECK constraints and expression indexes that are absorbed from a LIKE parent table tended to get mis-numbered, resulting in wrong answers and/or bizarre error messages (though probably not any actual crashes, thanks to validation occurring in the executor). In v12 and up, the same could happen to Vars in GENERATED expressions, even in cases with no LIKE clause but multiple traditional-inheritance parents. The cause of the problem for LIKE is that parse_utilcmd.c supposed it could renumber such Vars correctly during transformCreateStmt(), which it cannot since we have not yet accounted for columns added via inheritance. Fix that by postponing processing of LIKE INCLUDING CONSTRAINTS, DEFAULTS, GENERATED, INDEXES till after we've performed DefineRelation(). The error with GENERATED and multiple inheritance is a simple oversight in MergeAttributes(); it knows it has to renumber Vars in inherited CHECK constraints, but forgot to apply the same processing to inherited GENERATED expressions (a/k/a defaults). Per bug #16272 from Tom Gottfried. The non-GENERATED variants of the issue are ancient, presumably dating right back to the addition of CREATE TABLE LIKE; hence back-patch to all supported branches. Discussion: https://postgr.es/m/16272-6e32da020e9a9381@postgresql.org --- src/backend/commands/tablecmds.c | 126 +++++- src/backend/parser/parse_utilcmd.c | 381 +++++++++++------- src/backend/tcop/utility.c | 36 +- src/include/nodes/parsenodes.h | 1 + src/include/parser/parse_utilcmd.h | 2 + .../expected/create_table.out | 2 + .../test_ddl_deparse/test_ddl_deparse.c | 3 + .../regress/expected/create_table_like.out | 40 +- src/test/regress/sql/create_table_like.sql | 19 +- 9 files changed, 435 insertions(+), 175 deletions(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index cd989c95e517..790c09c522e4 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -405,6 +405,8 @@ static bool ConstraintImpliedByRelConstraint(Relation scanrel, List *testConstraint, List *provenConstraint); static ObjectAddress ATExecColumnDefault(Relation rel, const char *colName, Node *newDefault, LOCKMODE lockmode); +static ObjectAddress ATExecCookedColumnDefault(Relation rel, AttrNumber attnum, + Node *newDefault); static ObjectAddress ATExecAddIdentity(Relation rel, const char *colName, Node *def, LOCKMODE lockmode); static ObjectAddress ATExecSetIdentity(Relation rel, const char *colName, @@ -2054,8 +2056,8 @@ storage_name(char c) * 'schema' is the column/attribute definition for the table. (It's a list * of ColumnDef's.) It is destructively changed. * 'supers' is a list of OIDs of parent relations, already locked by caller. - * 'relpersistence' is a persistence type of the table. - * 'is_partition' tells if the table is a partition + * 'relpersistence' is the persistence type of the table. + * 'is_partition' tells if the table is a partition. * * Output arguments: * 'supconstr' receives a list of constraints belonging to the parents, @@ -2218,7 +2220,11 @@ MergeAttributes(List *schema, List *supers, char relpersistence, TupleDesc tupleDesc; TupleConstr *constr; AttrMap *newattmap; + List *inherited_defaults; + List *cols_with_defaults; AttrNumber parent_attno; + ListCell *lc1; + ListCell *lc2; /* caller already got lock */ relation = table_open(parent, NoLock); @@ -2304,6 +2310,9 @@ MergeAttributes(List *schema, List *supers, char relpersistence, */ newattmap = make_attrmap(tupleDesc->natts); + /* We can't process inherited defaults until newattmap is complete. */ + inherited_defaults = cols_with_defaults = NIL; + for (parent_attno = 1; parent_attno <= tupleDesc->natts; parent_attno++) { @@ -2359,7 +2368,7 @@ MergeAttributes(List *schema, List *supers, char relpersistence, get_collation_name(defCollId), get_collation_name(attribute->attcollation)))); - /* Copy storage parameter */ + /* Copy/check storage parameter */ if (def->storage == 0) def->storage = attribute->attstorage; else if (def->storage != attribute->attstorage) @@ -2410,7 +2419,7 @@ MergeAttributes(List *schema, List *supers, char relpersistence, } /* - * Copy default if any + * Locate default if any */ if (attribute->atthasdef) { @@ -2432,23 +2441,59 @@ MergeAttributes(List *schema, List *supers, char relpersistence, Assert(this_default != NULL); /* - * If default expr could contain any vars, we'd need to fix - * 'em, but it can't; so default is ready to apply to child. - * - * If we already had a default from some prior parent, check - * to see if they are the same. If so, no problem; if not, - * mark the column as having a bogus default. Below, we will - * complain if the bogus default isn't overridden by the child - * schema. + * If it's a GENERATED default, it might contain Vars that + * need to be mapped to the inherited column(s)' new numbers. + * We can't do that till newattmap is ready, so just remember + * all the inherited default expressions for the moment. */ - Assert(def->raw_default == NULL); - if (def->cooked_default == NULL) - def->cooked_default = this_default; - else if (!equal(def->cooked_default, this_default)) - { - def->cooked_default = &bogus_marker; - have_bogus_defaults = true; - } + inherited_defaults = lappend(inherited_defaults, this_default); + cols_with_defaults = lappend(cols_with_defaults, def); + } + } + + /* + * Now process any inherited default expressions, adjusting attnos + * using the completed newattmap map. + */ + forboth(lc1, inherited_defaults, lc2, cols_with_defaults) + { + Node *this_default = (Node *) lfirst(lc1); + ColumnDef *def = (ColumnDef *) lfirst(lc2); + bool found_whole_row; + + /* Adjust Vars to match new table's column numbering */ + this_default = map_variable_attnos(this_default, + 1, 0, + newattmap, + InvalidOid, &found_whole_row); + + /* + * For the moment we have to reject whole-row variables. We could + * convert them, if we knew the new table's rowtype OID, but that + * hasn't been assigned yet. (A variable could only appear in a + * generation expression, so the error message is correct.) + */ + if (found_whole_row) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot convert whole-row table reference"), + errdetail("Generation expression for column \"%s\" contains a whole-row reference to table \"%s\".", + def->colname, + RelationGetRelationName(relation)))); + + /* + * If we already had a default from some prior parent, check to + * see if they are the same. If so, no problem; if not, mark the + * column as having a bogus default. Below, we will complain if + * the bogus default isn't overridden by the child schema. + */ + Assert(def->raw_default == NULL); + if (def->cooked_default == NULL) + def->cooked_default = this_default; + else if (!equal(def->cooked_default, this_default)) + { + def->cooked_default = &bogus_marker; + have_bogus_defaults = true; } } @@ -2667,7 +2712,6 @@ MergeAttributes(List *schema, List *supers, char relpersistence, def->raw_default = newdef->raw_default; def->cooked_default = newdef->cooked_default; } - } else { @@ -3781,6 +3825,7 @@ AlterTableGetLockLevel(List *cmds) * Theoretically, these could be ShareRowExclusiveLock. */ case AT_ColumnDefault: + case AT_CookedColumnDefault: case AT_AlterConstraint: case AT_AddIndex: /* from ADD CONSTRAINT */ case AT_AddIndexConstraint: @@ -4040,6 +4085,13 @@ ATPrepCmd(List **wqueue, Relation rel, AlterTableCmd *cmd, /* No command-specific prep needed */ pass = cmd->def ? AT_PASS_ADD_OTHERCONSTR : AT_PASS_DROP; break; + case AT_CookedColumnDefault: /* add a pre-cooked default */ + /* This is currently used only in CREATE TABLE */ + /* (so the permission check really isn't necessary) */ + ATSimplePermissions(rel, ATT_TABLE | ATT_FOREIGN_TABLE); + /* This command never recurses */ + pass = AT_PASS_ADD_OTHERCONSTR; + break; case AT_AddIdentity: ATSimplePermissions(rel, ATT_TABLE | ATT_VIEW | ATT_FOREIGN_TABLE); /* This command never recurses */ @@ -4398,6 +4450,9 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, case AT_ColumnDefault: /* ALTER COLUMN DEFAULT */ address = ATExecColumnDefault(rel, cmd->name, cmd->def, lockmode); break; + case AT_CookedColumnDefault: /* add a pre-cooked default */ + address = ATExecCookedColumnDefault(rel, cmd->num, cmd->def); + break; case AT_AddIdentity: cmd = ATParseTransformCmd(wqueue, tab, rel, cmd, false, lockmode, cur_pass, context); @@ -6859,6 +6914,35 @@ ATExecColumnDefault(Relation rel, const char *colName, return address; } +/* + * Add a pre-cooked default expression. + * + * Return the address of the affected column. + */ +static ObjectAddress +ATExecCookedColumnDefault(Relation rel, AttrNumber attnum, + Node *newDefault) +{ + ObjectAddress address; + + /* We assume no checking is required */ + + /* + * Remove any old default for the column. We use RESTRICT here for + * safety, but at present we do not expect anything to depend on the + * default. (In ordinary cases, there could not be a default in place + * anyway, but it's possible when combining LIKE with inheritance.) + */ + RemoveAttrDefault(RelationGetRelid(rel), attnum, DROP_RESTRICT, false, + true); + + (void) StoreAttrDefault(rel, attnum, newDefault, true, false); + + ObjectAddressSubSet(address, RelationRelationId, + RelationGetRelid(rel), attnum); + return address; +} + /* * ALTER TABLE ALTER COLUMN ADD IDENTITY * diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index 25abc544fc72..6c49554defbc 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -86,7 +86,6 @@ typedef struct List *ckconstraints; /* CHECK constraints */ List *fkconstraints; /* FOREIGN KEY constraints */ List *ixconstraints; /* index-creating constraints */ - List *inh_indexes; /* cloned indexes from INCLUDING INDEXES */ List *extstats; /* cloned extended statistics */ List *blist; /* "before list" of things to do before * creating the table */ @@ -154,6 +153,9 @@ static Const *transformPartitionBoundValue(ParseState *pstate, Node *con, * Returns a List of utility commands to be done in sequence. One of these * will be the transformed CreateStmt, but there may be additional actions * to be done before and after the actual DefineRelation() call. + * In addition to normal utility commands such as AlterTableStmt and + * IndexStmt, the result list may contain TableLikeClause(s), representing + * the need to perform additional parse analysis after DefineRelation(). * * SQL allows constraints to be scattered all over, so thumb through * the columns and collect all constraints into one place. @@ -241,7 +243,6 @@ transformCreateStmt(CreateStmt *stmt, const char *queryString) cxt.ckconstraints = NIL; cxt.fkconstraints = NIL; cxt.ixconstraints = NIL; - cxt.inh_indexes = NIL; cxt.extstats = NIL; cxt.blist = NIL; cxt.alist = NIL; @@ -917,18 +918,18 @@ transformTableConstraint(CreateStmtContext *cxt, Constraint *constraint) * transformTableLikeClause * * Change the LIKE portion of a CREATE TABLE statement into - * column definitions which recreate the user defined column portions of - * . + * column definitions that recreate the user defined column portions of + * . Also, if there are any LIKE options that we can't fully + * process at this point, add the TableLikeClause to cxt->alist, which + * will cause utility.c to call expandTableLikeClause() after the new + * table has been created. */ static void transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_clause) { AttrNumber parent_attno; - AttrNumber new_attno; Relation relation; TupleDesc tupleDesc; - TupleConstr *constr; - AttrMap *attmap; AclResult aclresult; char *comment; ParseCallbackState pcbstate; @@ -942,6 +943,7 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("LIKE is not supported for creating foreign tables"))); + /* Open the relation referenced by the LIKE clause */ relation = relation_openrv(table_like_clause->relation, AccessShareLock); if (relation->rd_rel->relkind != RELKIND_RELATION && @@ -978,37 +980,11 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla } tupleDesc = RelationGetDescr(relation); - constr = tupleDesc->constr; - - /* - * Initialize column number map for map_variable_attnos(). We need this - * since dropped columns in the source table aren't copied, so the new - * table can have different column numbers. - */ - attmap = make_attrmap(tupleDesc->natts); - - /* - * We must fill the attmap now so that it can be used to process generated - * column default expressions in the per-column loop below. - */ - new_attno = 1; - for (parent_attno = 1; parent_attno <= tupleDesc->natts; - parent_attno++) - { - Form_pg_attribute attribute = TupleDescAttr(tupleDesc, - parent_attno - 1); - - /* - * Ignore dropped columns in the parent. attmap entry is left zero. - */ - if (attribute->attisdropped) - continue; - - attmap->attnums[parent_attno - 1] = list_length(cxt->columns) + (new_attno++); - } /* * Insert the copied attributes into the cxt for the new table definition. + * We must do this now so that they appear in the table in the relative + * position where the LIKE clause is, as required by SQL99. */ for (parent_attno = 1; parent_attno <= tupleDesc->natts; parent_attno++) @@ -1052,52 +1028,12 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla cxt->columns = lappend(cxt->columns, def); /* - * Copy default, if present and it should be copied. We have separate - * options for plain default expressions and GENERATED defaults. + * Although we don't transfer the column's default/generation + * expression now, we need to mark it GENERATED if appropriate. */ - if (attribute->atthasdef && - (attribute->attgenerated ? - (table_like_clause->options & CREATE_TABLE_LIKE_GENERATED) : - (table_like_clause->options & CREATE_TABLE_LIKE_DEFAULTS))) - { - Node *this_default = NULL; - AttrDefault *attrdef; - int i; - bool found_whole_row; - - /* Find default in constraint structure */ - Assert(constr != NULL); - attrdef = constr->defval; - for (i = 0; i < constr->num_defval; i++) - { - if (attrdef[i].adnum == parent_attno) - { - this_default = stringToNode(attrdef[i].adbin); - break; - } - } - Assert(this_default != NULL); - - def->cooked_default = map_variable_attnos(this_default, - 1, 0, - attmap, - InvalidOid, &found_whole_row); - - /* - * Prevent this for the same reason as for constraints below. Note - * that defaults cannot contain any vars, so it's OK that the - * error message refers to generated columns. - */ - if (found_whole_row) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot convert whole-row table reference"), - errdetail("Generation expression for column \"%s\" contains a whole-row reference to table \"%s\".", - attributeName, - RelationGetRelationName(relation)))); - + if (attribute->atthasdef && attribute->attgenerated && + (table_like_clause->options & CREATE_TABLE_LIKE_GENERATED)) def->generated = attribute->attgenerated; - } /* * Copy identity if requested @@ -1145,14 +1081,191 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla } } + /* + * We cannot yet deal with defaults, CHECK constraints, or indexes, since + * we don't yet know what column numbers the copied columns will have in + * the finished table. If any of those options are specified, add the + * LIKE clause to cxt->alist so that expandTableLikeClause will be called + * after we do know that. + */ + if (table_like_clause->options & + (CREATE_TABLE_LIKE_DEFAULTS | + CREATE_TABLE_LIKE_GENERATED | + CREATE_TABLE_LIKE_CONSTRAINTS | + CREATE_TABLE_LIKE_INDEXES)) + cxt->alist = lappend(cxt->alist, table_like_clause); + + /* + * We may copy extended statistics if requested, since the representation + * of CreateStatsStmt doesn't depend on column numbers. + */ + if (table_like_clause->options & CREATE_TABLE_LIKE_STATISTICS) + { + List *parent_extstats; + ListCell *l; + + parent_extstats = RelationGetStatExtList(relation); + + foreach(l, parent_extstats) + { + Oid parent_stat_oid = lfirst_oid(l); + CreateStatsStmt *stats_stmt; + + stats_stmt = generateClonedExtStatsStmt(cxt->relation, + RelationGetRelid(relation), + parent_stat_oid); + + /* Copy comment on statistics object, if requested */ + if (table_like_clause->options & CREATE_TABLE_LIKE_COMMENTS) + { + comment = GetComment(parent_stat_oid, StatisticExtRelationId, 0); + + /* + * We make use of CreateStatsStmt's stxcomment option, so as + * not to need to know now what name the statistics will have. + */ + stats_stmt->stxcomment = comment; + } + + cxt->extstats = lappend(cxt->extstats, stats_stmt); + } + + list_free(parent_extstats); + } + + /* + * Close the parent rel, but keep our AccessShareLock on it until xact + * commit. That will prevent someone else from deleting or ALTERing the + * parent before we can run expandTableLikeClause. + */ + table_close(relation, NoLock); +} + +/* + * expandTableLikeClause + * + * Process LIKE options that require knowing the final column numbers + * assigned to the new table's columns. This executes after we have + * run DefineRelation for the new table. It returns a list of utility + * commands that should be run to generate indexes etc. + */ +List * +expandTableLikeClause(RangeVar *heapRel, TableLikeClause *table_like_clause) +{ + List *result = NIL; + List *atsubcmds = NIL; + AttrNumber parent_attno; + Relation relation; + Relation childrel; + TupleDesc tupleDesc; + TupleConstr *constr; + AttrMap *attmap; + char *comment; + + /* + * Open the relation referenced by the LIKE clause. We should still have + * the table lock obtained by transformTableLikeClause (and this'll throw + * an assertion failure if not). Hence, no need to recheck privileges + * etc. + */ + relation = relation_openrv(table_like_clause->relation, NoLock); + + tupleDesc = RelationGetDescr(relation); + constr = tupleDesc->constr; + + /* + * Open the newly-created child relation; we have lock on that too. + */ + childrel = relation_openrv(heapRel, NoLock); + + /* + * Construct a map from the LIKE relation's attnos to the child rel's. + * This re-checks type match etc, although it shouldn't be possible to + * have a failure since both tables are locked. + */ + attmap = build_attrmap_by_name(RelationGetDescr(childrel), + tupleDesc); + + /* + * Process defaults, if required. + */ + if ((table_like_clause->options & + (CREATE_TABLE_LIKE_DEFAULTS | CREATE_TABLE_LIKE_GENERATED)) && + constr != NULL) + { + AttrDefault *attrdef = constr->defval; + + for (parent_attno = 1; parent_attno <= tupleDesc->natts; + parent_attno++) + { + Form_pg_attribute attribute = TupleDescAttr(tupleDesc, + parent_attno - 1); + + /* + * Ignore dropped columns in the parent. + */ + if (attribute->attisdropped) + continue; + + /* + * Copy default, if present and it should be copied. We have + * separate options for plain default expressions and GENERATED + * defaults. + */ + if (attribute->atthasdef && + (attribute->attgenerated ? + (table_like_clause->options & CREATE_TABLE_LIKE_GENERATED) : + (table_like_clause->options & CREATE_TABLE_LIKE_DEFAULTS))) + { + Node *this_default = NULL; + AlterTableCmd *atsubcmd; + bool found_whole_row; + + /* Find default in constraint structure */ + for (int i = 0; i < constr->num_defval; i++) + { + if (attrdef[i].adnum == parent_attno) + { + this_default = stringToNode(attrdef[i].adbin); + break; + } + } + Assert(this_default != NULL); + + atsubcmd = makeNode(AlterTableCmd); + atsubcmd->subtype = AT_CookedColumnDefault; + atsubcmd->num = attmap->attnums[parent_attno - 1]; + atsubcmd->def = map_variable_attnos(this_default, + 1, 0, + attmap, + InvalidOid, + &found_whole_row); + + /* + * Prevent this for the same reason as for constraints below. + * Note that defaults cannot contain any vars, so it's OK that + * the error message refers to generated columns. + */ + if (found_whole_row) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot convert whole-row table reference"), + errdetail("Generation expression for column \"%s\" contains a whole-row reference to table \"%s\".", + NameStr(attribute->attname), + RelationGetRelationName(relation)))); + + atsubcmds = lappend(atsubcmds, atsubcmd); + } + } + } + /* * Copy CHECK constraints if requested, being careful to adjust attribute * numbers so they match the child. */ if ((table_like_clause->options & CREATE_TABLE_LIKE_CONSTRAINTS) && - tupleDesc->constr) + constr != NULL) { - TupleConstr *constr = tupleDesc->constr; int ccnum; for (ccnum = 0; ccnum < constr->num_check; ccnum++) @@ -1160,9 +1273,10 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla char *ccname = constr->check[ccnum].ccname; char *ccbin = constr->check[ccnum].ccbin; bool ccnoinherit = constr->check[ccnum].ccnoinherit; - Constraint *n = makeNode(Constraint); Node *ccbin_node; bool found_whole_row; + Constraint *n; + AlterTableCmd *atsubcmd; ccbin_node = map_variable_attnos(stringToNode(ccbin), 1, 0, @@ -1183,13 +1297,22 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla ccname, RelationGetRelationName(relation)))); + n = makeNode(Constraint); n->contype = CONSTR_CHECK; n->conname = pstrdup(ccname); n->location = -1; n->is_no_inherit = ccnoinherit; n->raw_expr = NULL; n->cooked_expr = nodeToString(ccbin_node); - cxt->ckconstraints = lappend(cxt->ckconstraints, n); + + /* We can skip validation, since the new table should be empty. */ + n->skip_validation = true; + n->initially_valid = true; + + atsubcmd = makeNode(AlterTableCmd); + atsubcmd->subtype = AT_AddConstraint; + atsubcmd->def = (Node *) n; + atsubcmds = lappend(atsubcmds, atsubcmd); /* Copy comment on constraint */ if ((table_like_clause->options & CREATE_TABLE_LIKE_COMMENTS) && @@ -1201,18 +1324,34 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla CommentStmt *stmt = makeNode(CommentStmt); stmt->objtype = OBJECT_TABCONSTRAINT; - stmt->object = (Node *) list_make3(makeString(cxt->relation->schemaname), - makeString(cxt->relation->relname), + stmt->object = (Node *) list_make3(makeString(heapRel->schemaname), + makeString(heapRel->relname), makeString(n->conname)); stmt->comment = comment; - cxt->alist = lappend(cxt->alist, stmt); + result = lappend(result, stmt); } } } /* - * Likewise, copy indexes if requested + * If we generated any ALTER TABLE actions above, wrap them into a single + * ALTER TABLE command. Stick it at the front of the result, so it runs + * before any CommentStmts we made above. + */ + if (atsubcmds) + { + AlterTableStmt *atcmd = makeNode(AlterTableStmt); + + atcmd->relation = copyObject(heapRel); + atcmd->cmds = atsubcmds; + atcmd->objtype = OBJECT_TABLE; + atcmd->missing_ok = false; + result = lcons(atcmd, result); + } + + /* + * Process indexes if required. */ if ((table_like_clause->options & CREATE_TABLE_LIKE_INDEXES) && relation->rd_rel->relhasindex) @@ -1231,7 +1370,7 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla parent_index = index_open(parent_index_oid, AccessShareLock); /* Build CREATE INDEX statement to recreate the parent_index */ - index_stmt = generateClonedIndexStmt(cxt->relation, + index_stmt = generateClonedIndexStmt(heapRel, parent_index, attmap, NULL); @@ -1248,49 +1387,14 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla index_stmt->idxcomment = comment; } - /* Save it in the inh_indexes list for the time being */ - cxt->inh_indexes = lappend(cxt->inh_indexes, index_stmt); + result = lappend(result, index_stmt); index_close(parent_index, AccessShareLock); } } - /* - * Likewise, copy extended statistics if requested - */ - if (table_like_clause->options & CREATE_TABLE_LIKE_STATISTICS) - { - List *parent_extstats; - ListCell *l; - - parent_extstats = RelationGetStatExtList(relation); - - foreach(l, parent_extstats) - { - Oid parent_stat_oid = lfirst_oid(l); - CreateStatsStmt *stats_stmt; - - stats_stmt = generateClonedExtStatsStmt(cxt->relation, - RelationGetRelid(relation), - parent_stat_oid); - - /* Copy comment on statistics object, if requested */ - if (table_like_clause->options & CREATE_TABLE_LIKE_COMMENTS) - { - comment = GetComment(parent_stat_oid, StatisticExtRelationId, 0); - - /* - * We make use of CreateStatsStmt's stxcomment option, so as - * not to need to know now what name the statistics will have. - */ - stats_stmt->stxcomment = comment; - } - - cxt->extstats = lappend(cxt->extstats, stats_stmt); - } - - list_free(parent_extstats); - } + /* Done with child rel */ + table_close(childrel, NoLock); /* * Close the parent rel, but keep our AccessShareLock on it until xact @@ -1298,6 +1402,8 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla * parent before the child is committed. */ table_close(relation, NoLock); + + return result; } static void @@ -1590,7 +1696,7 @@ generateClonedIndexStmt(RangeVar *heapRel, Relation source_idx, attmap, InvalidOid, &found_whole_row); - /* As in transformTableLikeClause, reject whole-row variables */ + /* As in expandTableLikeClause, reject whole-row variables */ if (found_whole_row) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1699,7 +1805,7 @@ generateClonedIndexStmt(RangeVar *heapRel, Relation source_idx, attmap, InvalidOid, &found_whole_row); - /* As in transformTableLikeClause, reject whole-row variables */ + /* As in expandTableLikeClause, reject whole-row variables */ if (found_whole_row) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1897,24 +2003,6 @@ transformIndexConstraints(CreateStmtContext *cxt) indexlist = lappend(indexlist, index); } - /* Add in any indexes defined by LIKE ... INCLUDING INDEXES */ - foreach(lc, cxt->inh_indexes) - { - index = (IndexStmt *) lfirst(lc); - - if (index->primary) - { - if (cxt->pkey != NULL) - ereport(ERROR, - (errcode(ERRCODE_INVALID_TABLE_DEFINITION), - errmsg("multiple primary keys for table \"%s\" are not allowed", - cxt->relation->relname))); - cxt->pkey = index; - } - - indexlist = lappend(indexlist, index); - } - /* * Scan the index list and remove any redundant index specifications. This * can happen if, for instance, the user writes UNIQUE PRIMARY KEY. A @@ -3115,7 +3203,6 @@ transformAlterTableStmt(Oid relid, AlterTableStmt *stmt, cxt.ckconstraints = NIL; cxt.fkconstraints = NIL; cxt.ixconstraints = NIL; - cxt.inh_indexes = NIL; cxt.extstats = NIL; cxt.blist = NIL; cxt.alist = NIL; diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 9b0c376c8cb5..6154d2c8c63b 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -1197,6 +1197,28 @@ ProcessUtilitySlow(ParseState *pstate, secondaryObject, stmt); } + else if (IsA(stmt, TableLikeClause)) + { + /* + * Do delayed processing of LIKE options. This + * will result in additional sub-statements for us + * to process. We can just tack those onto the + * to-do list. + */ + TableLikeClause *like = (TableLikeClause *) stmt; + RangeVar *rv = ((CreateStmt *) parsetree)->relation; + List *morestmts; + + morestmts = expandTableLikeClause(rv, like); + stmts = list_concat(stmts, morestmts); + + /* + * We don't need a CCI now, besides which the "l" + * list pointer is now possibly invalid, so just + * skip the CCI test below. + */ + continue; + } else { /* @@ -1405,6 +1427,7 @@ ProcessUtilitySlow(ParseState *pstate, IndexStmt *stmt = (IndexStmt *) parsetree; Oid relid; LOCKMODE lockmode; + bool is_alter_table; if (stmt->concurrent) PreventInTransactionBlock(isTopLevel, @@ -1466,6 +1489,17 @@ ProcessUtilitySlow(ParseState *pstate, list_free(inheritors); } + /* + * If the IndexStmt is already transformed, it must have + * come from generateClonedIndexStmt, which in current + * usage means it came from expandTableLikeClause rather + * than from original parse analysis. And that means we + * must treat it like ALTER TABLE ADD INDEX, not CREATE. + * (This is a bit grotty, but currently it doesn't seem + * worth adding a separate bool field for the purpose.) + */ + is_alter_table = stmt->transformed; + /* Run parse analysis ... */ stmt = transformIndexStmt(relid, stmt, queryString); @@ -1477,7 +1511,7 @@ ProcessUtilitySlow(ParseState *pstate, InvalidOid, /* no predefined OID */ InvalidOid, /* no parent index */ InvalidOid, /* no parent constraint */ - false, /* is_alter_table */ + is_alter_table, true, /* check_rights */ true, /* check_not_in_use */ false, /* skip_build */ diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 151bcdb7ef5b..47d4c07306d0 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1786,6 +1786,7 @@ typedef enum AlterTableType AT_AddColumnRecurse, /* internal to commands/tablecmds.c */ AT_AddColumnToView, /* implicitly via CREATE OR REPLACE VIEW */ AT_ColumnDefault, /* alter column default */ + AT_CookedColumnDefault, /* add a pre-cooked column default */ AT_DropNotNull, /* alter column drop not null */ AT_SetNotNull, /* alter column set not null */ AT_DropExpression, /* alter column drop expression */ diff --git a/src/include/parser/parse_utilcmd.h b/src/include/parser/parse_utilcmd.h index 1a5e0b83a7a5..bc3d66ed8814 100644 --- a/src/include/parser/parse_utilcmd.h +++ b/src/include/parser/parse_utilcmd.h @@ -31,6 +31,8 @@ extern void transformRuleStmt(RuleStmt *stmt, const char *queryString, extern List *transformCreateSchemaStmt(CreateSchemaStmt *stmt); extern PartitionBoundSpec *transformPartitionBound(ParseState *pstate, Relation parent, PartitionBoundSpec *spec); +extern List *expandTableLikeClause(RangeVar *heapRel, + TableLikeClause *table_like_clause); extern IndexStmt *generateClonedIndexStmt(RangeVar *heapRel, Relation source_idx, const struct AttrMap *attmap, diff --git a/src/test/modules/test_ddl_deparse/expected/create_table.out b/src/test/modules/test_ddl_deparse/expected/create_table.out index c7c9bf8971f3..0f2a2c164eb5 100644 --- a/src/test/modules/test_ddl_deparse/expected/create_table.out +++ b/src/test/modules/test_ddl_deparse/expected/create_table.out @@ -135,6 +135,8 @@ CREATE TABLE like_fkey_table ( INCLUDING STORAGE ); NOTICE: DDL test: type simple, tag CREATE TABLE +NOTICE: DDL test: type alter table, tag ALTER TABLE +NOTICE: subcommand: ALTER COLUMN SET DEFAULT (precooked) NOTICE: DDL test: type simple, tag CREATE INDEX NOTICE: DDL test: type simple, tag CREATE INDEX -- Volatile table types diff --git a/src/test/modules/test_ddl_deparse/test_ddl_deparse.c b/src/test/modules/test_ddl_deparse/test_ddl_deparse.c index b7bdb88ce7f7..def4e39f19de 100644 --- a/src/test/modules/test_ddl_deparse/test_ddl_deparse.c +++ b/src/test/modules/test_ddl_deparse/test_ddl_deparse.c @@ -111,6 +111,9 @@ get_altertable_subcmdtypes(PG_FUNCTION_ARGS) case AT_ColumnDefault: strtype = "ALTER COLUMN SET DEFAULT"; break; + case AT_CookedColumnDefault: + strtype = "ALTER COLUMN SET DEFAULT (precooked)"; + break; case AT_DropNotNull: strtype = "DROP NOT NULL"; break; diff --git a/src/test/regress/expected/create_table_like.out b/src/test/regress/expected/create_table_like.out index 655e8e41dd90..e3edbd8b511c 100644 --- a/src/test/regress/expected/create_table_like.out +++ b/src/test/regress/expected/create_table_like.out @@ -160,7 +160,9 @@ SELECT * FROM test_like_gen_3; DROP TABLE test_like_gen_1, test_like_gen_2, test_like_gen_3; -- also test generated column with a "forward" reference (bug #16342) -CREATE TABLE test_like_4 (b int DEFAULT 42, c int GENERATED ALWAYS AS (a * 2) STORED, a int); +CREATE TABLE test_like_4 (b int DEFAULT 42, + c int GENERATED ALWAYS AS (a * 2) STORED, + a int CHECK (a > 0)); \d test_like_4 Table "public.test_like_4" Column | Type | Collation | Nullable | Default @@ -168,6 +170,8 @@ CREATE TABLE test_like_4 (b int DEFAULT 42, c int GENERATED ALWAYS AS (a * 2) ST b | integer | | | 42 c | integer | | | generated always as (a * 2) stored a | integer | | | +Check constraints: + "test_like_4_a_check" CHECK (a > 0) CREATE TABLE test_like_4a (LIKE test_like_4); CREATE TABLE test_like_4b (LIKE test_like_4 INCLUDING DEFAULTS); @@ -233,7 +237,32 @@ SELECT a, b, c FROM test_like_4d; 11 | 42 | 22 (1 row) +-- Test renumbering of Vars when combining LIKE with inheritance +CREATE TABLE test_like_5 (x point, y point, z point); +CREATE TABLE test_like_5x (p int CHECK (p > 0), + q int GENERATED ALWAYS AS (p * 2) STORED); +CREATE TABLE test_like_5c (LIKE test_like_4 INCLUDING ALL) + INHERITS (test_like_5, test_like_5x); +\d test_like_5c + Table "public.test_like_5c" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+------------------------------------ + x | point | | | + y | point | | | + z | point | | | + p | integer | | | + q | integer | | | generated always as (p * 2) stored + b | integer | | | 42 + c | integer | | | generated always as (a * 2) stored + a | integer | | | +Check constraints: + "test_like_4_a_check" CHECK (a > 0) + "test_like_5x_p_check" CHECK (p > 0) +Inherits: test_like_5, + test_like_5x + DROP TABLE test_like_4, test_like_4a, test_like_4b, test_like_4c, test_like_4d; +DROP TABLE test_like_5, test_like_5x, test_like_5c; CREATE TABLE inhg (x text, LIKE inhx INCLUDING INDEXES, y text); /* copies indexes */ INSERT INTO inhg VALUES (5, 10); INSERT INTO inhg VALUES (20, 10); -- should fail @@ -269,9 +298,10 @@ ALTER TABLE ctlt1 ALTER COLUMN a SET STORAGE MAIN; CREATE TABLE ctlt2 (c text); ALTER TABLE ctlt2 ALTER COLUMN c SET STORAGE EXTERNAL; COMMENT ON COLUMN ctlt2.c IS 'C'; -CREATE TABLE ctlt3 (a text CHECK (length(a) < 5), c text); +CREATE TABLE ctlt3 (a text CHECK (length(a) < 5), c text CHECK (length(c) < 7)); ALTER TABLE ctlt3 ALTER COLUMN c SET STORAGE EXTERNAL; ALTER TABLE ctlt3 ALTER COLUMN a SET STORAGE MAIN; +CREATE INDEX ctlt3_fnidx ON ctlt3 ((a || c)); COMMENT ON COLUMN ctlt3.a IS 'A3'; COMMENT ON COLUMN ctlt3.c IS 'C'; COMMENT ON CONSTRAINT ctlt3_a_check ON ctlt3 IS 't3_a_check'; @@ -327,10 +357,11 @@ NOTICE: merging multiple inherited definitions of column "a" Check constraints: "ctlt1_a_check" CHECK (length(a) > 2) "ctlt3_a_check" CHECK (length(a) < 5) + "ctlt3_c_check" CHECK (length(c) < 7) Inherits: ctlt1, ctlt3 -CREATE TABLE ctlt13_like (LIKE ctlt3 INCLUDING CONSTRAINTS INCLUDING COMMENTS INCLUDING STORAGE) INHERITS (ctlt1); +CREATE TABLE ctlt13_like (LIKE ctlt3 INCLUDING CONSTRAINTS INCLUDING INDEXES INCLUDING COMMENTS INCLUDING STORAGE) INHERITS (ctlt1); NOTICE: merging column "a" with inherited definition \d+ ctlt13_like Table "public.ctlt13_like" @@ -339,9 +370,12 @@ NOTICE: merging column "a" with inherited definition a | text | | not null | | main | | A3 b | text | | | | extended | | c | text | | | | external | | C +Indexes: + "ctlt13_like_expr_idx" btree ((a || c)) Check constraints: "ctlt1_a_check" CHECK (length(a) > 2) "ctlt3_a_check" CHECK (length(a) < 5) + "ctlt3_c_check" CHECK (length(c) < 7) Inherits: ctlt1 SELECT description FROM pg_description, pg_constraint c WHERE classoid = 'pg_constraint'::regclass AND objoid = c.oid AND c.conrelid = 'ctlt13_like'::regclass; diff --git a/src/test/regress/sql/create_table_like.sql b/src/test/regress/sql/create_table_like.sql index 6981ac0cbeee..f0a8a56b76fa 100644 --- a/src/test/regress/sql/create_table_like.sql +++ b/src/test/regress/sql/create_table_like.sql @@ -66,7 +66,9 @@ SELECT * FROM test_like_gen_3; DROP TABLE test_like_gen_1, test_like_gen_2, test_like_gen_3; -- also test generated column with a "forward" reference (bug #16342) -CREATE TABLE test_like_4 (b int DEFAULT 42, c int GENERATED ALWAYS AS (a * 2) STORED, a int); +CREATE TABLE test_like_4 (b int DEFAULT 42, + c int GENERATED ALWAYS AS (a * 2) STORED, + a int CHECK (a > 0)); \d test_like_4 CREATE TABLE test_like_4a (LIKE test_like_4); CREATE TABLE test_like_4b (LIKE test_like_4 INCLUDING DEFAULTS); @@ -84,7 +86,17 @@ SELECT a, b, c FROM test_like_4c; \d test_like_4d INSERT INTO test_like_4d (a) VALUES(11); SELECT a, b, c FROM test_like_4d; + +-- Test renumbering of Vars when combining LIKE with inheritance +CREATE TABLE test_like_5 (x point, y point, z point); +CREATE TABLE test_like_5x (p int CHECK (p > 0), + q int GENERATED ALWAYS AS (p * 2) STORED); +CREATE TABLE test_like_5c (LIKE test_like_4 INCLUDING ALL) + INHERITS (test_like_5, test_like_5x); +\d test_like_5c + DROP TABLE test_like_4, test_like_4a, test_like_4b, test_like_4c, test_like_4d; +DROP TABLE test_like_5, test_like_5x, test_like_5c; CREATE TABLE inhg (x text, LIKE inhx INCLUDING INDEXES, y text); /* copies indexes */ INSERT INTO inhg VALUES (5, 10); @@ -119,9 +131,10 @@ CREATE TABLE ctlt2 (c text); ALTER TABLE ctlt2 ALTER COLUMN c SET STORAGE EXTERNAL; COMMENT ON COLUMN ctlt2.c IS 'C'; -CREATE TABLE ctlt3 (a text CHECK (length(a) < 5), c text); +CREATE TABLE ctlt3 (a text CHECK (length(a) < 5), c text CHECK (length(c) < 7)); ALTER TABLE ctlt3 ALTER COLUMN c SET STORAGE EXTERNAL; ALTER TABLE ctlt3 ALTER COLUMN a SET STORAGE MAIN; +CREATE INDEX ctlt3_fnidx ON ctlt3 ((a || c)); COMMENT ON COLUMN ctlt3.a IS 'A3'; COMMENT ON COLUMN ctlt3.c IS 'C'; COMMENT ON CONSTRAINT ctlt3_a_check ON ctlt3 IS 't3_a_check'; @@ -138,7 +151,7 @@ CREATE TABLE ctlt1_inh (LIKE ctlt1 INCLUDING CONSTRAINTS INCLUDING COMMENTS) INH SELECT description FROM pg_description, pg_constraint c WHERE classoid = 'pg_constraint'::regclass AND objoid = c.oid AND c.conrelid = 'ctlt1_inh'::regclass; CREATE TABLE ctlt13_inh () INHERITS (ctlt1, ctlt3); \d+ ctlt13_inh -CREATE TABLE ctlt13_like (LIKE ctlt3 INCLUDING CONSTRAINTS INCLUDING COMMENTS INCLUDING STORAGE) INHERITS (ctlt1); +CREATE TABLE ctlt13_like (LIKE ctlt3 INCLUDING CONSTRAINTS INCLUDING INDEXES INCLUDING COMMENTS INCLUDING STORAGE) INHERITS (ctlt1); \d+ ctlt13_like SELECT description FROM pg_description, pg_constraint c WHERE classoid = 'pg_constraint'::regclass AND objoid = c.oid AND c.conrelid = 'ctlt13_like'::regclass; From bfd78c0b41c5d59e6850dee412f32748da0a3c11 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Fri, 21 Aug 2020 18:29:37 -0400 Subject: [PATCH 004/589] docs: add COMMENT examples for new features, rename rtree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported-by: Jürgen Purtz Discussion: https://postgr.es/m/15ec5428-d46a-1725-f38d-44986a977abb@purtz.de Author: Jürgen Purtz Backpatch-through: 11 --- doc/src/sgml/ref/comment.sgml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/ref/comment.sgml b/doc/src/sgml/ref/comment.sgml index 965c5a40ad72..fd7492a25567 100644 --- a/doc/src/sgml/ref/comment.sgml +++ b/doc/src/sgml/ref/comment.sgml @@ -306,7 +306,7 @@ COMMENT ON TABLE mytable IS NULL; Some more examples: -COMMENT ON ACCESS METHOD rtree IS 'R-Tree access method'; +COMMENT ON ACCESS METHOD gin IS 'GIN index access method'; COMMENT ON AGGREGATE my_aggregate (double precision) IS 'Computes sample variance'; COMMENT ON CAST (text AS int4) IS 'Allow casts from text to int4'; COMMENT ON COLLATION "fr_CA" IS 'Canadian French'; @@ -316,6 +316,7 @@ COMMENT ON CONSTRAINT bar_col_cons ON bar IS 'Constrains column col'; COMMENT ON CONSTRAINT dom_col_constr ON DOMAIN dom IS 'Constrains col of domain'; COMMENT ON DATABASE my_database IS 'Development Database'; COMMENT ON DOMAIN my_domain IS 'Email Address Domain'; +COMMENT ON EVENT TRIGGER abort_ddl IS 'Aborts all DDL commands'; COMMENT ON EXTENSION hstore IS 'implements the hstore data type'; COMMENT ON FOREIGN DATA WRAPPER mywrapper IS 'my foreign data wrapper'; COMMENT ON FOREIGN TABLE my_foreign_table IS 'Employee Information in other database'; @@ -330,12 +331,15 @@ COMMENT ON OPERATOR CLASS int4ops USING btree IS '4 byte integer operators for b COMMENT ON OPERATOR FAMILY integer_ops USING btree IS 'all integer operators for btrees'; COMMENT ON POLICY my_policy ON mytable IS 'Filter rows by users'; COMMENT ON PROCEDURE my_proc (integer, integer) IS 'Runs a report'; +COMMENT ON PUBLICATION alltables IS 'Publishes all operations on all tables'; COMMENT ON ROLE my_role IS 'Administration group for finance tables'; +COMMENT ON ROUTINE my_routine (integer, integer) IS 'Runs a routine (which is a function or procedure)'; COMMENT ON RULE my_rule ON my_table IS 'Logs updates of employee records'; COMMENT ON SCHEMA my_schema IS 'Departmental data'; COMMENT ON SEQUENCE my_sequence IS 'Used to generate primary keys'; COMMENT ON SERVER myserver IS 'my foreign server'; COMMENT ON STATISTICS my_statistics IS 'Improves planner row estimations'; +COMMENT ON SUBSCRIPTION alltables IS 'Subscription for all operations on all tables'; COMMENT ON TABLE my_schema.my_table IS 'Employee Information'; COMMENT ON TABLESPACE my_tablespace IS 'Tablespace for indexes'; COMMENT ON TEXT SEARCH CONFIGURATION my_config IS 'Special word filtering'; From 2a9f37243b0b0b3621f1851a6a8644d4ca2749d6 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Fri, 21 Aug 2020 20:23:09 -0400 Subject: [PATCH 005/589] docs: improve description of how to handle multiple databases This is a redesign of the intro to the managing databases chapter. Discussion: https://postgr.es/m/159586122762.680.1361378513036616007@wrigleys.postgresql.org Author: David G. Johnston Backpatch-through: 9.5 --- doc/src/sgml/manage-ag.sgml | 50 ++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/doc/src/sgml/manage-ag.sgml b/doc/src/sgml/manage-ag.sgml index 01453e6dae72..74055a470655 100644 --- a/doc/src/sgml/manage-ag.sgml +++ b/doc/src/sgml/manage-ag.sgml @@ -33,21 +33,41 @@ - When connecting to the database server, a client must specify in - its connection request the name of the database it wants to connect - to. It is not possible to access more than one database per - connection. However, an application is not restricted in the number of - connections it opens to the same or other databases. Databases are - physically separated and access control is managed at the - connection level. If one PostgreSQL server - instance is to house projects or users that should be separate and - for the most part unaware of each other, it is therefore - recommended to put them into separate databases. If the projects - or users are interrelated and should be able to use each other's - resources, they should be put in the same database but possibly - into separate schemas. Schemas are a purely logical structure and who can - access what is managed by the privilege system. More information about - managing schemas is in . + When connecting to the database server, a client must specify the + database name in its connection request. + It is not possible to access more than one database per + connection. However, clients can open multiple connections to + the same database, or different databases. + Database-level security has two components: access control + (see ), managed at the + connection level, and authorization control + (see ), managed via the grant system. + Foreign data wrappers (see ) + allow for objects within one database to act as proxies for objects in + other database or clusters. + The older dblink module (see ) provides a similar capability. + By default, all users can connect to all databases using all connection methods. + + + + If one PostgreSQL server cluster is planned to contain + unrelated projects or users that should be, for the most part, unaware + of each other, it is recommended to put them into separate databases and + adjust authorizations and access controls accordingly. + If the projects or users are interrelated, and thus should be able to use + each other's resources, they should be put in the same database but probably + into separate schemas; this provides a modular structure with namespace + isolation and authorization control. + More information about managing schemas is in . + + + + While multiple databases can be created within a single cluster, it is advised + to consider carefully whether the benefits outweigh the risks and limitations. + In particular, the impact that having a shared WAL (see ) + has on backup and recovery options. While individual databases in the cluster + are isolated when considered from the user's perspective, they are closely bound + from the database administrator's point-of-view. From c3a288649e152612791121fa6d17a1322b8f2814 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Sat, 22 Aug 2020 22:26:10 +0900 Subject: [PATCH 006/589] doc: Fix format, incorrect structure names and markup inconsistencies Author: Alexander Lakhin Discussion: https://postgr.es/m/a2345841-10a5-4eef-257c-02302347cf39@gmail.com Backpatch-through: 13 --- doc/src/sgml/datetime.sgml | 8 ++++---- doc/src/sgml/func.sgml | 2 +- doc/src/sgml/libpq.sgml | 29 ++++++++++++++++------------- doc/src/sgml/monitoring.sgml | 8 ++++---- doc/src/sgml/protocol.sgml | 5 +++-- 5 files changed, 28 insertions(+), 24 deletions(-) diff --git a/doc/src/sgml/datetime.sgml b/doc/src/sgml/datetime.sgml index bbf50b76f8c3..39fbc39cb0dd 100644 --- a/doc/src/sgml/datetime.sgml +++ b/doc/src/sgml/datetime.sgml @@ -564,8 +564,8 @@ - PostgreSQL can accept time zone specifications that - are written according to the POSIX standard's rules + PostgreSQL can accept time zone specifications + that are written according to the POSIX standard's rules for the TZ environment variable. POSIX time zone specifications are inadequate to deal with the complexity of real-world time zone history, @@ -635,8 +635,8 @@ or -). The positive sign is used for zones west of Greenwich. (Note that this is the opposite of the ISO-8601 sign convention used elsewhere in - PostgreSQL.) hh can have - one or two digits; mm + PostgreSQL.) hh + can have one or two digits; mm and ss (if used) must have two. diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 9a4ac5a1ea36..51ec5281c0b3 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -14101,7 +14101,7 @@ SELECT xmltable.* size_sq_km float PATH 'SIZE[@unit = "sq_km"]', size_other text PATH 'concat(SIZE[@unit!="sq_km"], " ", SIZE[@unit!="sq_km"]/@unit)', - premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified') ; + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); id | ordinality | COUNTRY_NAME | country_id | size_sq_km | size_other | premier_name ----+------------+--------------+------------+------------+--------------+--------------- diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index f7b765f76dc9..72c42407790b 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -781,7 +781,7 @@ PGPing PQping(const char *conninfo); PQsetSSLKeyPassHook_OpenSSL lets an application override - libpq's default + libpq's default handling of encrypted client certificate key files using or interactive prompting. @@ -793,20 +793,23 @@ void PQsetSSLKeyPassHook_OpenSSL(PQsslKeyPassHook_OpenSSL_type hook); int callback_fn(char *buf, int size, PGconn *conn); - which libpq will then call instead of - its default PQdefaultSSLKeyPassHook_OpenSSL handler. The callback - should determine the password for the key and copy it to result-buffer - buf of size size. The string in - buf must be null-terminated. The callback must return the length of - the password stored in buf excluding the null terminator. - On failure, the callback should set buf[0] = '\0' and return 0. - See PQdefaultSSLKeyPassHook_OpenSSL in libpq's - source code for an example. - - + which libpq will then call + instead of its default + PQdefaultSSLKeyPassHook_OpenSSL handler. The + callback should determine the password for the key and copy it to + result-buffer buf of size + size. The string in buf + must be null-terminated. The callback must return the length of the + password stored in buf excluding the null + terminator. On failure, the callback should set + buf[0] = '\0' and return 0. See + PQdefaultSSLKeyPassHook_OpenSSL in + libpq's source code for an example. + + If the user specified an explicit key location, - its path will be in conn->pgsslkey when the callback + its path will be in conn->sslkey when the callback is invoked. This will be empty if the default key path is being used. For keys that are engine specifiers, it is up to engine implementations whether they use the OpenSSL password callback or define their own handling. diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 304c49f07b76..0f11375c8529 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -4444,7 +4444,7 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i name text - name of the SLRU + Name of the SLRU @@ -4648,7 +4648,7 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i argument. The argument can be bgwriter to reset all the counters shown in the pg_stat_bgwriter - view,or archiver to reset all the counters shown in + view, or archiver to reset all the counters shown in the pg_stat_archiver view. @@ -5188,8 +5188,8 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid, finalizing analyze - The command is updating pg_class. When this phase is completed, - ANALYZE will end. + The command is updating pg_class. When this + phase is completed, ANALYZE will end. diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 8b00235a5161..0c7087397d73 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -1742,8 +1742,9 @@ simple query protocol can be used. For the purpose of testing replication commands, you can make a replication - connection via psql or any other libpq-using - tool with a connection string including the replication option, + connection via psql or any other + libpq-using tool with a connection string including + the replication option, e.g.: psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" From 5b02d68e758307e0ae8fae4d7bbcd687f1dd6ce1 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 22 Aug 2020 12:34:17 -0400 Subject: [PATCH 007/589] Fix ALTER TABLE's scheduling rules for AT_AddConstraint subcommands. Commit 1281a5c90 rearranged the logic in this area rather drastically, and it broke the case of adding a foreign key constraint in the same ALTER that adds the pkey or unique constraint it depends on. While self-referential fkeys are surely a pretty niche case, this used to work so we shouldn't break it. To fix, reorganize the scheduling rules in ATParseTransformCmd so that a transformed AT_AddConstraint subcommand will be delayed into a later pass in all cases, not only when it's been spit out as a side-effect of parsing some other command type. Also tweak the logic so that we won't run ATParseTransformCmd twice while doing this. It seems to work even without that, but it's surely wasting cycles to do so. Per bug #16589 from Jeremy Evans. Back-patch to v13 where the new code was introduced. Discussion: https://postgr.es/m/16589-31c8d981ca503896@postgresql.org --- src/backend/commands/tablecmds.c | 151 ++++++++++++---------- src/test/regress/expected/alter_table.out | 36 ++++++ src/test/regress/sql/alter_table.sql | 14 ++ 3 files changed, 135 insertions(+), 66 deletions(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 790c09c522e4..d2b15a3387b0 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -4513,9 +4513,12 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, lockmode); break; case AT_AddConstraint: /* ADD CONSTRAINT */ - cmd = ATParseTransformCmd(wqueue, tab, rel, cmd, false, lockmode, - cur_pass, context); - /* Might not have gotten AddConstraint back from parse transform */ + /* Transform the command only during initial examination */ + if (cur_pass == AT_PASS_ADD_CONSTR) + cmd = ATParseTransformCmd(wqueue, tab, rel, cmd, + false, lockmode, + cur_pass, context); + /* Depending on constraint type, might be no more work to do now */ if (cmd != NULL) address = ATExecAddConstraint(wqueue, tab, rel, @@ -4523,9 +4526,12 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, false, false, lockmode); break; case AT_AddConstraintRecurse: /* ADD CONSTRAINT with recursion */ - cmd = ATParseTransformCmd(wqueue, tab, rel, cmd, true, lockmode, - cur_pass, context); - /* Might not have gotten AddConstraint back from parse transform */ + /* Transform the command only during initial examination */ + if (cur_pass == AT_PASS_ADD_CONSTR) + cmd = ATParseTransformCmd(wqueue, tab, rel, cmd, + true, lockmode, + cur_pass, context); + /* Depending on constraint type, might be no more work to do now */ if (cmd != NULL) address = ATExecAddConstraint(wqueue, tab, rel, @@ -4787,75 +4793,88 @@ ATParseTransformCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, foreach(lc, atstmt->cmds) { AlterTableCmd *cmd2 = lfirst_node(AlterTableCmd, lc); + int pass; + + /* + * This switch need only cover the subcommand types that can be added + * by parse_utilcmd.c; otherwise, we'll use the default strategy of + * executing the subcommand immediately, as a substitute for the + * original subcommand. (Note, however, that this does cause + * AT_AddConstraint subcommands to be rescheduled into later passes, + * which is important for index and foreign key constraints.) + * + * We assume we needn't do any phase-1 checks for added subcommands. + */ + switch (cmd2->subtype) + { + case AT_SetNotNull: + /* Need command-specific recursion decision */ + ATPrepSetNotNull(wqueue, rel, cmd2, + recurse, false, + lockmode, context); + pass = AT_PASS_COL_ATTRS; + break; + case AT_AddIndex: + /* This command never recurses */ + /* No command-specific prep needed */ + pass = AT_PASS_ADD_INDEX; + break; + case AT_AddIndexConstraint: + /* This command never recurses */ + /* No command-specific prep needed */ + pass = AT_PASS_ADD_INDEXCONSTR; + break; + case AT_AddConstraint: + /* Recursion occurs during execution phase */ + if (recurse) + cmd2->subtype = AT_AddConstraintRecurse; + switch (castNode(Constraint, cmd2->def)->contype) + { + case CONSTR_PRIMARY: + case CONSTR_UNIQUE: + case CONSTR_EXCLUSION: + pass = AT_PASS_ADD_INDEXCONSTR; + break; + default: + pass = AT_PASS_ADD_OTHERCONSTR; + break; + } + break; + case AT_AlterColumnGenericOptions: + /* This command never recurses */ + /* No command-specific prep needed */ + pass = AT_PASS_MISC; + break; + default: + pass = cur_pass; + break; + } - if (newcmd == NULL && - (cmd->subtype == cmd2->subtype || - (cmd->subtype == AT_AddConstraintRecurse && - cmd2->subtype == AT_AddConstraint))) + if (pass < cur_pass) + { + /* Cannot schedule into a pass we already finished */ + elog(ERROR, "ALTER TABLE scheduling failure: too late for pass %d", + pass); + } + else if (pass > cur_pass) { - /* Found the transformed version of our subcommand */ - cmd2->subtype = cmd->subtype; /* copy recursion flag */ - newcmd = cmd2; + /* OK, queue it up for later */ + tab->subcmds[pass] = lappend(tab->subcmds[pass], cmd2); } else { - int pass; - /* - * Schedule added subcommand appropriately. We assume we needn't - * do any phase-1 checks for it. This switch only has to cover - * the subcommand types that can be added by parse_utilcmd.c. + * We should see at most one subcommand for the current pass, + * which is the transformed version of the original subcommand. */ - switch (cmd2->subtype) + if (newcmd == NULL && cmd->subtype == cmd2->subtype) { - case AT_SetNotNull: - /* Need command-specific recursion decision */ - ATPrepSetNotNull(wqueue, rel, cmd2, - recurse, false, - lockmode, context); - pass = AT_PASS_COL_ATTRS; - break; - case AT_AddIndex: - /* This command never recurses */ - /* No command-specific prep needed */ - pass = AT_PASS_ADD_INDEX; - break; - case AT_AddIndexConstraint: - /* This command never recurses */ - /* No command-specific prep needed */ - pass = AT_PASS_ADD_INDEXCONSTR; - break; - case AT_AddConstraint: - /* Recursion occurs during execution phase */ - if (recurse) - cmd2->subtype = AT_AddConstraintRecurse; - switch (castNode(Constraint, cmd2->def)->contype) - { - case CONSTR_PRIMARY: - case CONSTR_UNIQUE: - case CONSTR_EXCLUSION: - pass = AT_PASS_ADD_INDEXCONSTR; - break; - default: - pass = AT_PASS_ADD_OTHERCONSTR; - break; - } - break; - case AT_AlterColumnGenericOptions: - /* This command never recurses */ - /* No command-specific prep needed */ - pass = AT_PASS_MISC; - break; - default: - elog(ERROR, "unexpected AlterTableType: %d", - (int) cmd2->subtype); - pass = AT_PASS_UNSET; - break; + /* Found the transformed version of our subcommand */ + newcmd = cmd2; } - /* Must be for a later pass than we're currently doing */ - if (pass <= cur_pass) - elog(ERROR, "ALTER TABLE scheduling failure"); - tab->subcmds[pass] = lappend(tab->subcmds[pass], cmd2); + else + elog(ERROR, "ALTER TABLE scheduling failure: bogus item for pass %d", + pass); } } diff --git a/src/test/regress/expected/alter_table.out b/src/test/regress/expected/alter_table.out index 6f90eae2f8ce..f56615393ec3 100644 --- a/src/test/regress/expected/alter_table.out +++ b/src/test/regress/expected/alter_table.out @@ -3678,6 +3678,42 @@ ALTER TABLE ataddindex Indexes: "ataddindex_expr_excl" EXCLUDE USING btree ((f1 ~~ 'a'::text) WITH =) +DROP TABLE ataddindex; +CREATE TABLE ataddindex(id int, ref_id int); +ALTER TABLE ataddindex + ADD PRIMARY KEY (id), + ADD FOREIGN KEY (ref_id) REFERENCES ataddindex; +\d ataddindex + Table "public.ataddindex" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + id | integer | | not null | + ref_id | integer | | | +Indexes: + "ataddindex_pkey" PRIMARY KEY, btree (id) +Foreign-key constraints: + "ataddindex_ref_id_fkey" FOREIGN KEY (ref_id) REFERENCES ataddindex(id) +Referenced by: + TABLE "ataddindex" CONSTRAINT "ataddindex_ref_id_fkey" FOREIGN KEY (ref_id) REFERENCES ataddindex(id) + +DROP TABLE ataddindex; +CREATE TABLE ataddindex(id int, ref_id int); +ALTER TABLE ataddindex + ADD UNIQUE (id), + ADD FOREIGN KEY (ref_id) REFERENCES ataddindex (id); +\d ataddindex + Table "public.ataddindex" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + id | integer | | | + ref_id | integer | | | +Indexes: + "ataddindex_id_key" UNIQUE CONSTRAINT, btree (id) +Foreign-key constraints: + "ataddindex_ref_id_fkey" FOREIGN KEY (ref_id) REFERENCES ataddindex(id) +Referenced by: + TABLE "ataddindex" CONSTRAINT "ataddindex_ref_id_fkey" FOREIGN KEY (ref_id) REFERENCES ataddindex(id) + DROP TABLE ataddindex; -- unsupported constraint types for partitioned tables CREATE TABLE partitioned ( diff --git a/src/test/regress/sql/alter_table.sql b/src/test/regress/sql/alter_table.sql index ce6401d80d28..4cc55d852513 100644 --- a/src/test/regress/sql/alter_table.sql +++ b/src/test/regress/sql/alter_table.sql @@ -2252,6 +2252,20 @@ ALTER TABLE ataddindex \d ataddindex DROP TABLE ataddindex; +CREATE TABLE ataddindex(id int, ref_id int); +ALTER TABLE ataddindex + ADD PRIMARY KEY (id), + ADD FOREIGN KEY (ref_id) REFERENCES ataddindex; +\d ataddindex +DROP TABLE ataddindex; + +CREATE TABLE ataddindex(id int, ref_id int); +ALTER TABLE ataddindex + ADD UNIQUE (id), + ADD FOREIGN KEY (ref_id) REFERENCES ataddindex (id); +\d ataddindex +DROP TABLE ataddindex; + -- unsupported constraint types for partitioned tables CREATE TABLE partitioned ( a int, From 4d346def1555ea55b3adf76fc4afa3d3495ecfdd Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 22 Aug 2020 14:46:40 -0400 Subject: [PATCH 008/589] Avoid pushing quals down into sub-queries that have grouping sets. The trouble with doing this is that an apparently-constant subquery output column isn't really constant if it is a grouping column that appears in only some of the grouping sets. A qual using such a column would be subject to incorrect const-folding after push-down, as seen in bug #16585 from Paul Sivash. To fix, just disable qual pushdown altogether if the sub-query has nonempty groupingSets. While we could imagine far less restrictive solutions, there is not much point in working harder right now, because subquery_planner() won't move HAVING clauses to WHERE within such a subquery. If the qual stays in HAVING it's not going to be a lot more useful than if we'd kept it at the outer level. Having said that, this restriction could be removed if we used a parsetree representation that distinguished such outputs from actual constants, which is something I hope to do in future. Hence, make the patch a minimal addition rather than integrating it more tightly (e.g. by renumbering the existing items in subquery_is_pushdown_safe's comment). Back-patch to 9.5 where grouping sets were introduced. Discussion: https://postgr.es/m/16585-9d8c340d23ade8c1@postgresql.org --- src/backend/optimizer/path/allpaths.c | 15 ++++++++++ src/test/regress/expected/groupingsets.out | 32 ++++++++++++++++++++++ src/test/regress/sql/groupingsets.sql | 16 +++++++++++ 3 files changed, 63 insertions(+) diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 6da0dcd61cec..0eeff804bcf0 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -3182,6 +3182,17 @@ standard_join_search(PlannerInfo *root, int levels_needed, List *initial_rels) * volatile qual could succeed for some SRF output rows and fail for others, * a behavior that cannot occur if it's evaluated before SRF expansion. * + * 6. If the subquery has nonempty grouping sets, we cannot push down any + * quals. The concern here is that a qual referencing a "constant" grouping + * column could get constant-folded, which would be improper because the value + * is potentially nullable by grouping-set expansion. This restriction could + * be removed if we had a parsetree representation that shows that such + * grouping columns are not really constant. (There are other ideas that + * could be used to relax this restriction, but that's the approach most + * likely to get taken in the future. Note that there's not much to be gained + * so long as subquery_planner can't move HAVING clauses to WHERE within such + * a subquery.) + * * In addition, we make several checks on the subquery's output columns to see * if it is safe to reference them in pushed-down quals. If output column k * is found to be unsafe to reference, we set safetyInfo->unsafeColumns[k] @@ -3226,6 +3237,10 @@ subquery_is_pushdown_safe(Query *subquery, Query *topquery, if (subquery->limitOffset != NULL || subquery->limitCount != NULL) return false; + /* Check point 6 */ + if (subquery->groupClause && subquery->groupingSets) + return false; + /* Check points 3, 4, and 5 */ if (subquery->distinctClause || subquery->hasWindowFuncs || diff --git a/src/test/regress/expected/groupingsets.out b/src/test/regress/expected/groupingsets.out index 03ada654bb57..701d52b465d5 100644 --- a/src/test/regress/expected/groupingsets.out +++ b/src/test/regress/expected/groupingsets.out @@ -434,6 +434,38 @@ select x, not x as not_x, q2 from | | 4567890123456789 (5 rows) +-- check qual push-down rules for a subquery with grouping sets +explain (verbose, costs off) +select * from ( + select 1 as x, q1, sum(q2) + from int8_tbl i1 + group by grouping sets(1, 2) +) ss +where x = 1 and q1 = 123; + QUERY PLAN +-------------------------------------------- + Subquery Scan on ss + Output: ss.x, ss.q1, ss.sum + Filter: ((ss.x = 1) AND (ss.q1 = 123)) + -> GroupAggregate + Output: (1), i1.q1, sum(i1.q2) + Group Key: 1 + Sort Key: i1.q1 + Group Key: i1.q1 + -> Seq Scan on public.int8_tbl i1 + Output: 1, i1.q1, i1.q2 +(10 rows) + +select * from ( + select 1 as x, q1, sum(q2) + from int8_tbl i1 + group by grouping sets(1, 2) +) ss +where x = 1 and q1 = 123; + x | q1 | sum +---+----+----- +(0 rows) + -- simple rescan tests select a, b, sum(v.x) from (values (1),(2)) v(x), gstest_data(v.x) diff --git a/src/test/regress/sql/groupingsets.sql b/src/test/regress/sql/groupingsets.sql index e6c28743a441..d4e5628eba8d 100644 --- a/src/test/regress/sql/groupingsets.sql +++ b/src/test/regress/sql/groupingsets.sql @@ -172,6 +172,22 @@ select x, not x as not_x, q2 from group by grouping sets(x, q2) order by x, q2; +-- check qual push-down rules for a subquery with grouping sets +explain (verbose, costs off) +select * from ( + select 1 as x, q1, sum(q2) + from int8_tbl i1 + group by grouping sets(1, 2) +) ss +where x = 1 and q1 = 123; + +select * from ( + select 1 as x, q1, sum(q2) + from int8_tbl i1 + group by grouping sets(1, 2) +) ss +where x = 1 and q1 = 123; + -- simple rescan tests select a, b, sum(v.x) From a3c66de6c5e1ee9dd41ce1454496568622fb7712 Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Mon, 24 Aug 2020 08:16:19 +0530 Subject: [PATCH 009/589] Improve the vacuum error context phase information. We were displaying the wrong phase information for 'info' message in the index clean up phase because we were switching to the previous phase a bit early. We were also not displaying context information for heap phase unless the block number is valid which is fine for error cases but for messages at 'info' or lower error level it appears to be inconsistent with index phase information. Reported-by: Sawada Masahiko Author: Sawada Masahiko Reviewed-by: Amit Kapila Backpatch-through: 13, where it was introduced Discussion: https://postgr.es/m/CA+fd4k4HcbhPnCs7paRTw1K-AHin8y4xKomB9Ru0ATw0UeTy2w@mail.gmail.com --- src/backend/access/heap/vacuumlazy.c | 52 +++++++++++++++++----------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 44e2224dd557..8de31bf071b8 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -1662,6 +1662,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); + /* Clear the block number information */ + vacrelstats->blkno = InvalidBlockNumber; + pfree(frozen); /* save stats for use later */ @@ -1879,6 +1882,9 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) npages++; } + /* Clear the block number information */ + vacrelstats->blkno = InvalidBlockNumber; + if (BufferIsValid(vmbuffer)) { ReleaseBuffer(vmbuffer); @@ -2496,30 +2502,30 @@ lazy_cleanup_index(Relation indrel, *stats = index_vacuum_cleanup(&ivinfo, *stats); + if (*stats) + { + if (IsParallelWorker()) + msg = gettext_noop("index \"%s\" now contains %.0f row versions in %u pages as reported by parallel vacuum worker"); + else + msg = gettext_noop("index \"%s\" now contains %.0f row versions in %u pages"); + + ereport(elevel, + (errmsg(msg, + RelationGetRelationName(indrel), + (*stats)->num_index_tuples, + (*stats)->num_pages), + errdetail("%.0f index row versions were removed.\n" + "%u index pages have been deleted, %u are currently reusable.\n" + "%s.", + (*stats)->tuples_removed, + (*stats)->pages_deleted, (*stats)->pages_free, + pg_rusage_show(&ru0)))); + } + /* Revert back to the old phase information for error traceback */ restore_vacuum_error_info(vacrelstats, &saved_err_info); pfree(vacrelstats->indname); vacrelstats->indname = NULL; - - if (!(*stats)) - return; - - if (IsParallelWorker()) - msg = gettext_noop("index \"%s\" now contains %.0f row versions in %u pages as reported by parallel vacuum worker"); - else - msg = gettext_noop("index \"%s\" now contains %.0f row versions in %u pages"); - - ereport(elevel, - (errmsg(msg, - RelationGetRelationName(indrel), - (*stats)->num_index_tuples, - (*stats)->num_pages), - errdetail("%.0f index row versions were removed.\n" - "%u index pages have been deleted, %u are currently reusable.\n" - "%s.", - (*stats)->tuples_removed, - (*stats)->pages_deleted, (*stats)->pages_free, - pg_rusage_show(&ru0)))); } /* @@ -3582,12 +3588,18 @@ vacuum_error_callback(void *arg) if (BlockNumberIsValid(errinfo->blkno)) errcontext("while scanning block %u of relation \"%s.%s\"", errinfo->blkno, errinfo->relnamespace, errinfo->relname); + else + errcontext("while scanning relation \"%s.%s\"", + errinfo->relnamespace, errinfo->relname); break; case VACUUM_ERRCB_PHASE_VACUUM_HEAP: if (BlockNumberIsValid(errinfo->blkno)) errcontext("while vacuuming block %u of relation \"%s.%s\"", errinfo->blkno, errinfo->relnamespace, errinfo->relname); + else + errcontext("while vacuuming relation \"%s.%s\"", + errinfo->relnamespace, errinfo->relname); break; case VACUUM_ERRCB_PHASE_VACUUM_INDEX: From 77c1537f512e6ac2513f8695c795dc94cbf207ee Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 24 Aug 2020 16:46:52 +0900 Subject: [PATCH 010/589] doc: Fix some markups for support functions of index AMs All the documentation of index AMs has been using for local_relopts. This is a structure, so is a much better choice. Alexander has found the inconsistency for btree, while I have spotted the rest when applying the concept of consistency to the docs. Author: Alexander Lakhin, Michael Paquier Reviewed-by: Tom Lane Discussion: https://postgr.es/m/20200822133022.GC24782@paquier.xyz --- doc/src/sgml/brin.sgml | 2 +- doc/src/sgml/btree.sgml | 2 +- doc/src/sgml/gin.sgml | 2 +- doc/src/sgml/gist.sgml | 2 +- doc/src/sgml/spgist.sgml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/src/sgml/brin.sgml b/doc/src/sgml/brin.sgml index 55b6272db62e..b9d596e3c4e7 100644 --- a/doc/src/sgml/brin.sgml +++ b/doc/src/sgml/brin.sgml @@ -576,7 +576,7 @@ typedef struct BrinOpcInfo The options function is passed a pointer to a - local_relopts struct, which needs to be + local_relopts struct, which needs to be filled with a set of operator class specific options. The options can be accessed from other support functions using the PG_HAS_OPCLASS_OPTIONS() and diff --git a/doc/src/sgml/btree.sgml b/doc/src/sgml/btree.sgml index d03ee4d6fa0d..435b7cb24da9 100644 --- a/doc/src/sgml/btree.sgml +++ b/doc/src/sgml/btree.sgml @@ -566,7 +566,7 @@ equalimage(opcintype oid) returns bool options(relopts local_relopts *) returns void - The function is passed a pointer to a local_relopts + The function is passed a pointer to a local_relopts struct, which needs to be filled with a set of operator class specific options. The options can be accessed from other support functions using the PG_HAS_OPCLASS_OPTIONS() and diff --git a/doc/src/sgml/gin.sgml b/doc/src/sgml/gin.sgml index 07114f77199c..2d862669c337 100644 --- a/doc/src/sgml/gin.sgml +++ b/doc/src/sgml/gin.sgml @@ -412,7 +412,7 @@ The options function is passed a pointer to a - local_relopts struct, which needs to be + local_relopts struct, which needs to be filled with a set of operator class specific options. The options can be accessed from other support functions using the PG_HAS_OPCLASS_OPTIONS() and diff --git a/doc/src/sgml/gist.sgml b/doc/src/sgml/gist.sgml index 5d970ee9f2f4..a505815f4ec5 100644 --- a/doc/src/sgml/gist.sgml +++ b/doc/src/sgml/gist.sgml @@ -962,7 +962,7 @@ LANGUAGE C STRICT; - The function is passed a pointer to a local_relopts + The function is passed a pointer to a local_relopts struct, which needs to be filled with a set of operator class specific options. The options can be accessed from other support functions using the PG_HAS_OPCLASS_OPTIONS() and diff --git a/doc/src/sgml/spgist.sgml b/doc/src/sgml/spgist.sgml index 5d6e893d4918..b86302e4efde 100644 --- a/doc/src/sgml/spgist.sgml +++ b/doc/src/sgml/spgist.sgml @@ -897,7 +897,7 @@ LANGUAGE C STRICT; - The function is passed a pointer to a local_relopts + The function is passed a pointer to a local_relopts struct, which needs to be filled with a set of operator class specific options. The options can be accessed from other support functions using the PG_HAS_OPCLASS_OPTIONS() and From 7f055fba3fa99d807837a229967fd6c5dd720530 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 25 Aug 2020 07:29:05 +0200 Subject: [PATCH 011/589] doc: Fix up title case This fixes some instances that were missed in earlier processings and that now look a bit strange because they are inconsistent with nearby titles. --- doc/src/sgml/dml.sgml | 2 +- doc/src/sgml/func.sgml | 2 +- doc/src/sgml/plpgsql.sgml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/src/sgml/dml.sgml b/doc/src/sgml/dml.sgml index 97a773095540..3844e34a7dcc 100644 --- a/doc/src/sgml/dml.sgml +++ b/doc/src/sgml/dml.sgml @@ -262,7 +262,7 @@ DELETE FROM products; - Returning Data From Modified Rows + Returning Data from Modified Rows RETURNING diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 51ec5281c0b3..bbbffd9d5bbc 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -6876,7 +6876,7 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); - Differences From XQuery (<literal>LIKE_REGEX</literal>) + Differences from XQuery (<literal>LIKE_REGEX</literal>) LIKE_REGEX diff --git a/doc/src/sgml/plpgsql.sgml b/doc/src/sgml/plpgsql.sgml index d5c1654b16e4..815912666dd0 100644 --- a/doc/src/sgml/plpgsql.sgml +++ b/doc/src/sgml/plpgsql.sgml @@ -1657,7 +1657,7 @@ END; - Returning From a Function + Returning from a Function There are two commands available that allow you to return data From ff60394a8c9a7af8b32de420ccb54a20a0f019c1 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 25 Aug 2020 09:53:12 -0400 Subject: [PATCH 012/589] docs: client certificates are always sent to the server They are not "requested" by the server. Reported-by: Kyotaro Horiguchi Discussion: https://postgr.es/m/20200825.155320.986648039251743210.horikyota.ntt@gmail.com Backpatch-through: 9.5 --- doc/src/sgml/libpq.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index 72c42407790b..92556c7ce0cc 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -7880,7 +7880,7 @@ ldap://ldap.acme.com/cn=dbserver,cn=hosts?pgconnectinfo?base?(objectclass=*) ~/.postgresql/postgresql.crt client certificate - requested by server + sent to server From c34605daed563fcade07a9f45bcf440459599c00 Mon Sep 17 00:00:00 2001 From: David Rowley Date: Wed, 26 Aug 2020 10:51:36 +1200 Subject: [PATCH 013/589] Fixup some misusages of bms_num_members() It's a bit inefficient to test if a Bitmapset is empty by counting all the members and seeing if that number is zero. It's much better just to use bms_is_empty(). Likewise for checking if there are at least two members, just use bms_membership(), which does not need to do anything more after finding two members. Discussion: https://postgr.es/m/CAApHDvpvwm_QjbDOb5xga%2BKmX9XkN9xQavNGm3SvDbVnCYOerQ%40mail.gmail.com Reviewed-by: Tomas Vondra --- src/backend/optimizer/path/clausesel.c | 3 +-- src/backend/statistics/dependencies.c | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/backend/optimizer/path/clausesel.c b/src/backend/optimizer/path/clausesel.c index a3ebe10592d0..37a735b06bba 100644 --- a/src/backend/optimizer/path/clausesel.c +++ b/src/backend/optimizer/path/clausesel.c @@ -164,8 +164,7 @@ clauselist_selectivity_simple(PlannerInfo *root, * directly to clause_selectivity(). None of what we might do below is * relevant. */ - if ((list_length(clauses) == 1) && - bms_num_members(estimatedclauses) == 0) + if (list_length(clauses) == 1 && bms_is_empty(estimatedclauses)) return clause_selectivity(root, (Node *) linitial(clauses), varRelid, jointype, sjinfo); diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c index 3e37e2758ca0..4e30abb67437 100644 --- a/src/backend/statistics/dependencies.c +++ b/src/backend/statistics/dependencies.c @@ -1246,7 +1246,7 @@ dependencies_clauselist_selectivity(PlannerInfo *root, * of clauses. We must return 1.0 so the calling function's selectivity is * unaffected. */ - if (bms_num_members(clauses_attnums) < 2) + if (bms_membership(clauses_attnums) != BMS_MULTIPLE) { bms_free(clauses_attnums); pfree(list_attnums); @@ -1273,18 +1273,18 @@ dependencies_clauselist_selectivity(PlannerInfo *root, { StatisticExtInfo *stat = (StatisticExtInfo *) lfirst(l); Bitmapset *matched; - int num_matched; + BMS_Membership membership; /* skip statistics that are not of the correct type */ if (stat->kind != STATS_EXT_DEPENDENCIES) continue; matched = bms_intersect(clauses_attnums, stat->keys); - num_matched = bms_num_members(matched); + membership = bms_membership(matched); bms_free(matched); /* skip objects matching fewer than two attributes from clauses */ - if (num_matched < 2) + if (membership != BMS_MULTIPLE) continue; func_dependencies[nfunc_dependencies] From 29dd6d8bc631eebc3e50493c115f7a215f03bd0a Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Wed, 26 Aug 2020 10:50:02 +0900 Subject: [PATCH 014/589] Prevent non-superusers from reading pg_backend_memory_contexts, by default. pg_backend_memory_contexts view contains some internal information of memory contexts. Since exposing them to any users by default may cause security issue, this commit allows only superusers to read this view, by default, like we do for pg_shmem_allocations view. Bump catalog version. Author: Atsushi Torikoshi Reviewed-by: Michael Paquier, Fujii Masao Discussion: https://postgr.es/m/1414992.1597849297@sss.pgh.pa.us --- doc/src/sgml/catalogs.sgml | 4 ++++ src/backend/catalog/system_views.sql | 3 +++ src/include/catalog/catversion.h | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 1232b24e74cf..9fe260ecff7f 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -9697,6 +9697,10 @@ SCRAM-SHA-256$<iteration count>:&l + + By default, the pg_backend_memory_contexts view can be + read only by superusers. + diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index ba5a23ac2524..a2d61302f9e8 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -557,6 +557,9 @@ REVOKE EXECUTE ON FUNCTION pg_get_shmem_allocations() FROM PUBLIC; CREATE VIEW pg_backend_memory_contexts AS SELECT * FROM pg_get_backend_memory_contexts(); +REVOKE ALL ON pg_backend_memory_contexts FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION pg_get_backend_memory_contexts() FROM PUBLIC; + -- Statistics views CREATE VIEW pg_stat_all_tables AS diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 3e6779763000..573f1841b73d 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 202008191 +#define CATALOG_VERSION_NO 202008261 #endif From 50db5964ee333bc148e0c8844ffafaf585c719c6 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Wed, 26 Aug 2020 10:51:31 +0900 Subject: [PATCH 015/589] Move codes for pg_backend_memory_contexts from mmgr/mcxt.c to adt/mcxtfuncs.c. Previously the codes for pg_backend_memory_contexts were in src/backend/utils/mmgr/mcxt.c. This commit moves them to src/backend/utils/adt/mcxtfuncs.c so that mcxt.c basically includes only the low-level interface for memory contexts. Author: Atsushi Torikoshi Reviewed-by: Michael Paquier, Fujii Masao Discussion: https://postgr.es/m/20200819135545.GC19121@paquier.xyz --- src/backend/utils/adt/Makefile | 1 + src/backend/utils/adt/mcxtfuncs.c | 157 ++++++++++++++++++++++++++++++ src/backend/utils/mmgr/mcxt.c | 137 -------------------------- 3 files changed, 158 insertions(+), 137 deletions(-) create mode 100644 src/backend/utils/adt/mcxtfuncs.c diff --git a/src/backend/utils/adt/Makefile b/src/backend/utils/adt/Makefile index 5d2aca8cfe6f..54d5c3794726 100644 --- a/src/backend/utils/adt/Makefile +++ b/src/backend/utils/adt/Makefile @@ -57,6 +57,7 @@ OBJS = \ lockfuncs.o \ mac.o \ mac8.o \ + mcxtfuncs.o \ misc.o \ name.o \ network.o \ diff --git a/src/backend/utils/adt/mcxtfuncs.c b/src/backend/utils/adt/mcxtfuncs.c new file mode 100644 index 000000000000..50e1b07ff02c --- /dev/null +++ b/src/backend/utils/adt/mcxtfuncs.c @@ -0,0 +1,157 @@ +/*------------------------------------------------------------------------- + * + * mcxtfuncs.c + * Functions to show backend memory context. + * + * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/utils/adt/mcxtfuncs.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "funcapi.h" +#include "miscadmin.h" +#include "mb/pg_wchar.h" +#include "utils/builtins.h" + +/* ---------- + * The max bytes for showing identifiers of MemoryContext. + * ---------- + */ +#define MEMORY_CONTEXT_IDENT_DISPLAY_SIZE 1024 + +/* + * PutMemoryContextsStatsTupleStore + * One recursion level for pg_get_backend_memory_contexts. + */ +static void +PutMemoryContextsStatsTupleStore(Tuplestorestate *tupstore, + TupleDesc tupdesc, MemoryContext context, + const char *parent, int level) +{ +#define PG_GET_BACKEND_MEMORY_CONTEXTS_COLS 9 + + Datum values[PG_GET_BACKEND_MEMORY_CONTEXTS_COLS]; + bool nulls[PG_GET_BACKEND_MEMORY_CONTEXTS_COLS]; + MemoryContextCounters stat; + MemoryContext child; + const char *name; + const char *ident; + + AssertArg(MemoryContextIsValid(context)); + + name = context->name; + ident = context->ident; + + /* + * To be consistent with logging output, we label dynahash contexts + * with just the hash table name as with MemoryContextStatsPrint(). + */ + if (ident && strcmp(name, "dynahash") == 0) + { + name = ident; + ident = NULL; + } + + /* Examine the context itself */ + memset(&stat, 0, sizeof(stat)); + (*context->methods->stats) (context, NULL, (void *) &level, &stat); + + memset(values, 0, sizeof(values)); + memset(nulls, 0, sizeof(nulls)); + + if (name) + values[0] = CStringGetTextDatum(name); + else + nulls[0] = true; + + if (ident) + { + int idlen = strlen(ident); + char clipped_ident[MEMORY_CONTEXT_IDENT_DISPLAY_SIZE]; + + /* + * Some identifiers such as SQL query string can be very long, + * truncate oversize identifiers. + */ + if (idlen >= MEMORY_CONTEXT_IDENT_DISPLAY_SIZE) + idlen = pg_mbcliplen(ident, idlen, MEMORY_CONTEXT_IDENT_DISPLAY_SIZE - 1); + + memcpy(clipped_ident, ident, idlen); + clipped_ident[idlen] = '\0'; + values[1] = CStringGetTextDatum(clipped_ident); + } + else + nulls[1] = true; + + if (parent) + values[2] = CStringGetTextDatum(parent); + else + nulls[2] = true; + + values[3] = Int32GetDatum(level); + values[4] = Int64GetDatum(stat.totalspace); + values[5] = Int64GetDatum(stat.nblocks); + values[6] = Int64GetDatum(stat.freespace); + values[7] = Int64GetDatum(stat.freechunks); + values[8] = Int64GetDatum(stat.totalspace - stat.freespace); + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + + for (child = context->firstchild; child != NULL; child = child->nextchild) + { + PutMemoryContextsStatsTupleStore(tupstore, tupdesc, + child, name, level + 1); + } +} + +/* + * pg_get_backend_memory_contexts + * SQL SRF showing backend memory context. + */ +Datum +pg_get_backend_memory_contexts(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + TupleDesc tupdesc; + Tuplestorestate *tupstore; + MemoryContext per_query_ctx; + MemoryContext oldcontext; + + /* check to see if caller supports us returning a tuplestore */ + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that cannot accept a set"))); + if (!(rsinfo->allowedModes & SFRM_Materialize)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("materialize mode required, but it is not allowed in this context"))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; + oldcontext = MemoryContextSwitchTo(per_query_ctx); + + tupstore = tuplestore_begin_heap(true, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; + + MemoryContextSwitchTo(oldcontext); + + PutMemoryContextsStatsTupleStore(tupstore, tupdesc, + TopMemoryContext, NULL, 0); + + /* clean up and return the tuplestore */ + tuplestore_donestoring(tupstore); + + return (Datum) 0; +} diff --git a/src/backend/utils/mmgr/mcxt.c b/src/backend/utils/mmgr/mcxt.c index d9bb2499db75..88c76f290cea 100644 --- a/src/backend/utils/mmgr/mcxt.c +++ b/src/backend/utils/mmgr/mcxt.c @@ -21,10 +21,8 @@ #include "postgres.h" -#include "funcapi.h" #include "mb/pg_wchar.h" #include "miscadmin.h" -#include "utils/builtins.h" #include "utils/memdebug.h" #include "utils/memutils.h" @@ -69,11 +67,6 @@ static void MemoryContextStatsPrint(MemoryContext context, void *passthru, #define AssertNotInCriticalSection(context) \ Assert(CritSectionCount == 0 || (context)->allowInCritSection) -/* ---------- - * The max bytes for showing identifiers of MemoryContext. - * ---------- - */ -#define MEMORY_CONTEXT_IDENT_DISPLAY_SIZE 1024 /***************************************************************************** * EXPORTED ROUTINES * @@ -1228,133 +1221,3 @@ pchomp(const char *in) n--; return pnstrdup(in, n); } - -/* - * PutMemoryContextsStatsTupleStore - * One recursion level for pg_get_backend_memory_contexts. - */ -static void -PutMemoryContextsStatsTupleStore(Tuplestorestate *tupstore, - TupleDesc tupdesc, MemoryContext context, - const char *parent, int level) -{ -#define PG_GET_BACKEND_MEMORY_CONTEXTS_COLS 9 - - Datum values[PG_GET_BACKEND_MEMORY_CONTEXTS_COLS]; - bool nulls[PG_GET_BACKEND_MEMORY_CONTEXTS_COLS]; - MemoryContextCounters stat; - MemoryContext child; - const char *name; - const char *ident; - - AssertArg(MemoryContextIsValid(context)); - - name = context->name; - ident = context->ident; - - /* - * To be consistent with logging output, we label dynahash contexts - * with just the hash table name as with MemoryContextStatsPrint(). - */ - if (ident && strcmp(name, "dynahash") == 0) - { - name = ident; - ident = NULL; - } - - /* Examine the context itself */ - memset(&stat, 0, sizeof(stat)); - (*context->methods->stats) (context, NULL, (void *) &level, &stat); - - memset(values, 0, sizeof(values)); - memset(nulls, 0, sizeof(nulls)); - - if (name) - values[0] = CStringGetTextDatum(name); - else - nulls[0] = true; - - if (ident) - { - int idlen = strlen(ident); - char clipped_ident[MEMORY_CONTEXT_IDENT_DISPLAY_SIZE]; - - /* - * Some identifiers such as SQL query string can be very long, - * truncate oversize identifiers. - */ - if (idlen >= MEMORY_CONTEXT_IDENT_DISPLAY_SIZE) - idlen = pg_mbcliplen(ident, idlen, MEMORY_CONTEXT_IDENT_DISPLAY_SIZE - 1); - - memcpy(clipped_ident, ident, idlen); - clipped_ident[idlen] = '\0'; - values[1] = CStringGetTextDatum(clipped_ident); - } - else - nulls[1] = true; - - if (parent) - values[2] = CStringGetTextDatum(parent); - else - nulls[2] = true; - - values[3] = Int32GetDatum(level); - values[4] = Int64GetDatum(stat.totalspace); - values[5] = Int64GetDatum(stat.nblocks); - values[6] = Int64GetDatum(stat.freespace); - values[7] = Int64GetDatum(stat.freechunks); - values[8] = Int64GetDatum(stat.totalspace - stat.freespace); - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - - for (child = context->firstchild; child != NULL; child = child->nextchild) - { - PutMemoryContextsStatsTupleStore(tupstore, tupdesc, - child, name, level + 1); - } -} - -/* - * pg_get_backend_memory_contexts - * SQL SRF showing backend memory context. - */ -Datum -pg_get_backend_memory_contexts(PG_FUNCTION_ARGS) -{ - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - MemoryContext per_query_ctx; - MemoryContext oldcontext; - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* Build a tuple descriptor for our result type */ - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); - - per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; - oldcontext = MemoryContextSwitchTo(per_query_ctx); - - tupstore = tuplestore_begin_heap(true, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - PutMemoryContextsStatsTupleStore(tupstore, tupdesc, - TopMemoryContext, NULL, 0); - - /* clean up and return the tuplestore */ - tuplestore_donestoring(tupstore); - - return (Datum) 0; -} From adc8fc6167aa3f68b951ddd60ea32a62b13f18d6 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Wed, 26 Aug 2020 10:52:02 +0900 Subject: [PATCH 016/589] Add regression test for pg_backend_memory_contexts. Author: Atsushi Torikoshi Reviewed-by: Michael Paquier, Fujii Masao Discussion: https://postgr.es/m/20200819135545.GC19121@paquier.xyz --- src/test/regress/expected/sysviews.out | 9 +++++++++ src/test/regress/sql/sysviews.sql | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out index 06c4c3e47637..1cffc3349d60 100644 --- a/src/test/regress/expected/sysviews.out +++ b/src/test/regress/expected/sysviews.out @@ -19,6 +19,15 @@ select count(*) >= 0 as ok from pg_available_extensions; t (1 row) +-- The entire output of pg_backend_memory_contexts is not stable, +-- we test only the existance and basic condition of TopMemoryContext. +select name, ident, parent, level, total_bytes >= free_bytes + from pg_backend_memory_contexts where level = 0; + name | ident | parent | level | ?column? +------------------+-------+--------+-------+---------- + TopMemoryContext | | | 0 | t +(1 row) + -- At introduction, pg_config had 23 entries; it may grow select count(*) > 20 as ok from pg_config; ok diff --git a/src/test/regress/sql/sysviews.sql b/src/test/regress/sql/sysviews.sql index 28e412b73530..ac4a0e1cbba7 100644 --- a/src/test/regress/sql/sysviews.sql +++ b/src/test/regress/sql/sysviews.sql @@ -12,6 +12,11 @@ select count(*) >= 0 as ok from pg_available_extension_versions; select count(*) >= 0 as ok from pg_available_extensions; +-- The entire output of pg_backend_memory_contexts is not stable, +-- we test only the existance and basic condition of TopMemoryContext. +select name, ident, parent, level, total_bytes >= free_bytes + from pg_backend_memory_contexts where level = 0; + -- At introduction, pg_config had 23 entries; it may grow select count(*) > 20 as ok from pg_config; From 808e13b282efa7e7ac7b78e886aca5684f4bccd3 Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Wed, 26 Aug 2020 07:36:43 +0530 Subject: [PATCH 017/589] Extend the BufFile interface. Allow BufFile to support temporary files that can be used by the single backend when the corresponding files need to be survived across the transaction and need to be opened and closed multiple times. Such files need to be created as a member of a SharedFileSet. Additionally, this commit implements the interface for BufFileTruncate to allow files to be truncated up to a particular offset and extends the BufFileSeek API to support the SEEK_END case. This also adds an option to provide a mode while opening the shared BufFiles instead of always opening in read-only mode. These enhancements in BufFile interface are required for the upcoming patch to allow the replication apply worker, to handle streamed in-progress transactions. Author: Dilip Kumar, Amit Kapila Reviewed-by: Amit Kapila Tested-by: Neha Sharma Discussion: https://postgr.es/m/688b0b7f-2f6c-d827-c27b-216a8e3ea700@2ndquadrant.com --- doc/src/sgml/monitoring.sgml | 4 + src/backend/postmaster/pgstat.c | 3 + src/backend/storage/file/buffile.c | 129 ++++++++++++++++++++-- src/backend/storage/file/fd.c | 9 +- src/backend/storage/file/sharedfileset.c | 105 ++++++++++++++++-- src/backend/utils/sort/logtape.c | 4 +- src/backend/utils/sort/sharedtuplestore.c | 2 +- src/include/pgstat.h | 1 + src/include/storage/buffile.h | 4 +- src/include/storage/fd.h | 2 +- src/include/storage/sharedfileset.h | 4 +- 11 files changed, 239 insertions(+), 28 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 0f11375c8529..17a0df697848 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -1202,6 +1202,10 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser BufFileWrite Waiting for a write to a buffered file. + + BufFileTruncate + Waiting for a buffered file to be truncated. + ControlFileRead Waiting for a read from the pg_control diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 73ce944fb1ce..8116b2361430 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -3940,6 +3940,9 @@ pgstat_get_wait_io(WaitEventIO w) case WAIT_EVENT_BUFFILE_WRITE: event_name = "BufFileWrite"; break; + case WAIT_EVENT_BUFFILE_TRUNCATE: + event_name = "BufFileTruncate"; + break; case WAIT_EVENT_CONTROL_FILE_READ: event_name = "ControlFileRead"; break; diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c index 2d7a08232089..d581f96eda98 100644 --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c @@ -32,10 +32,14 @@ * (by opening multiple fd.c temporary files). This is an essential feature * for sorts and hashjoins on large amounts of data. * - * BufFile supports temporary files that can be made read-only and shared with - * other backends, as infrastructure for parallel execution. Such files need - * to be created as a member of a SharedFileSet that all participants are - * attached to. + * BufFile supports temporary files that can be shared with other backends, as + * infrastructure for parallel execution. Such files need to be created as a + * member of a SharedFileSet that all participants are attached to. + * + * BufFile also supports temporary files that can be used by the single backend + * when the corresponding files need to be survived across the transaction and + * need to be opened and closed multiple times. Such files need to be created + * as a member of a SharedFileSet. *------------------------------------------------------------------------- */ @@ -277,7 +281,7 @@ BufFileCreateShared(SharedFileSet *fileset, const char *name) * backends and render it read-only. */ BufFile * -BufFileOpenShared(SharedFileSet *fileset, const char *name) +BufFileOpenShared(SharedFileSet *fileset, const char *name, int mode) { BufFile *file; char segment_name[MAXPGPATH]; @@ -301,7 +305,7 @@ BufFileOpenShared(SharedFileSet *fileset, const char *name) } /* Try to load a segment. */ SharedSegmentName(segment_name, name, nfiles); - files[nfiles] = SharedFileSetOpen(fileset, segment_name); + files[nfiles] = SharedFileSetOpen(fileset, segment_name, mode); if (files[nfiles] <= 0) break; ++nfiles; @@ -321,7 +325,7 @@ BufFileOpenShared(SharedFileSet *fileset, const char *name) file = makeBufFileCommon(nfiles); file->files = files; - file->readOnly = true; /* Can't write to files opened this way */ + file->readOnly = (mode == O_RDONLY) ? true : false; file->fileset = fileset; file->name = pstrdup(name); @@ -666,11 +670,21 @@ BufFileSeek(BufFile *file, int fileno, off_t offset, int whence) newFile = file->curFile; newOffset = (file->curOffset + file->pos) + offset; break; -#ifdef NOT_USED case SEEK_END: - /* could be implemented, not needed currently */ + + /* + * The file size of the last file gives us the end offset of that + * file. + */ + newFile = file->numFiles - 1; + newOffset = FileSize(file->files[file->numFiles - 1]); + if (newOffset < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not determine size of temporary file \"%s\" from BufFile \"%s\": %m", + FilePathName(file->files[file->numFiles - 1]), + file->name))); break; -#endif default: elog(ERROR, "invalid whence: %d", whence); return EOF; @@ -838,3 +852,98 @@ BufFileAppend(BufFile *target, BufFile *source) return startBlock; } + +/* + * Truncate a BufFile created by BufFileCreateShared up to the given fileno and + * the offset. + */ +void +BufFileTruncateShared(BufFile *file, int fileno, off_t offset) +{ + int numFiles = file->numFiles; + int newFile = fileno; + off_t newOffset = file->curOffset; + char segment_name[MAXPGPATH]; + int i; + + /* + * Loop over all the files up to the given fileno and remove the files + * that are greater than the fileno and truncate the given file up to the + * offset. Note that we also remove the given fileno if the offset is 0 + * provided it is not the first file in which we truncate it. + */ + for (i = file->numFiles - 1; i >= fileno; i--) + { + if ((i != fileno || offset == 0) && i != 0) + { + SharedSegmentName(segment_name, file->name, i); + FileClose(file->files[i]); + if (!SharedFileSetDelete(file->fileset, segment_name, true)) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not delete shared fileset \"%s\": %m", + segment_name))); + numFiles--; + newOffset = MAX_PHYSICAL_FILESIZE; + + /* + * This is required to indicate that we have deleted the given + * fileno. + */ + if (i == fileno) + newFile--; + } + else + { + if (FileTruncate(file->files[i], offset, + WAIT_EVENT_BUFFILE_TRUNCATE) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not truncate file \"%s\": %m", + FilePathName(file->files[i])))); + newOffset = offset; + } + } + + file->numFiles = numFiles; + + /* + * If the truncate point is within existing buffer then we can just adjust + * pos within buffer. + */ + if (newFile == file->curFile && + newOffset >= file->curOffset && + newOffset <= file->curOffset + file->nbytes) + { + /* No need to reset the current pos if the new pos is greater. */ + if (newOffset <= file->curOffset + file->pos) + file->pos = (int) (newOffset - file->curOffset); + + /* Adjust the nbytes for the current buffer. */ + file->nbytes = (int) (newOffset - file->curOffset); + } + else if (newFile == file->curFile && + newOffset < file->curOffset) + { + /* + * The truncate point is within the existing file but prior to the + * current position, so we can forget the current buffer and reset the + * current position. + */ + file->curOffset = newOffset; + file->pos = 0; + file->nbytes = 0; + } + else if (newFile < file->curFile) + { + /* + * The truncate point is prior to the current file, so need to reset + * the current position accordingly. + */ + file->curFile = newFile; + file->curOffset = newOffset; + file->pos = 0; + file->nbytes = 0; + } + /* Nothing to do, if the truncate point is beyond current file. */ +} diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c index 5f6420efb2d7..f376a97ed677 100644 --- a/src/backend/storage/file/fd.c +++ b/src/backend/storage/file/fd.c @@ -1743,18 +1743,17 @@ PathNameCreateTemporaryFile(const char *path, bool error_on_failure) /* * Open a file that was created with PathNameCreateTemporaryFile, possibly in * another backend. Files opened this way don't count against the - * temp_file_limit of the caller, are read-only and are automatically closed - * at the end of the transaction but are not deleted on close. + * temp_file_limit of the caller, are automatically closed at the end of the + * transaction but are not deleted on close. */ File -PathNameOpenTemporaryFile(const char *path) +PathNameOpenTemporaryFile(const char *path, int mode) { File file; ResourceOwnerEnlargeFiles(CurrentResourceOwner); - /* We open the file read-only. */ - file = PathNameOpenFile(path, O_RDONLY | PG_BINARY); + file = PathNameOpenFile(path, mode | PG_BINARY); /* If no such file, then we don't raise an error. */ if (file <= 0 && errno != ENOENT) diff --git a/src/backend/storage/file/sharedfileset.c b/src/backend/storage/file/sharedfileset.c index 16b7594756c6..65fd8ff5c0c3 100644 --- a/src/backend/storage/file/sharedfileset.c +++ b/src/backend/storage/file/sharedfileset.c @@ -13,6 +13,10 @@ * files can be discovered by name, and a shared ownership semantics so that * shared files survive until the last user detaches. * + * SharedFileSets can be used by backends when the temporary files need to be + * opened/closed multiple times and the underlying files need to survive across + * transactions. + * *------------------------------------------------------------------------- */ @@ -25,25 +29,36 @@ #include "common/hashfn.h" #include "miscadmin.h" #include "storage/dsm.h" +#include "storage/ipc.h" #include "storage/sharedfileset.h" #include "utils/builtins.h" +static List *filesetlist = NIL; + static void SharedFileSetOnDetach(dsm_segment *segment, Datum datum); +static void SharedFileSetDeleteOnProcExit(int status, Datum arg); static void SharedFileSetPath(char *path, SharedFileSet *fileset, Oid tablespace); static void SharedFilePath(char *path, SharedFileSet *fileset, const char *name); static Oid ChooseTablespace(const SharedFileSet *fileset, const char *name); /* - * Initialize a space for temporary files that can be opened for read-only - * access by other backends. Other backends must attach to it before - * accessing it. Associate this SharedFileSet with 'seg'. Any contained - * files will be deleted when the last backend detaches. + * Initialize a space for temporary files that can be opened by other backends. + * Other backends must attach to it before accessing it. Associate this + * SharedFileSet with 'seg'. Any contained files will be deleted when the + * last backend detaches. + * + * We can also use this interface if the temporary files are used only by + * single backend but the files need to be opened and closed multiple times + * and also the underlying files need to survive across transactions. For + * such cases, dsm segment 'seg' should be passed as NULL. Callers are + * expected to explicitly remove such files by using SharedFileSetDelete/ + * SharedFileSetDeleteAll or we remove such files on proc exit. * * Files will be distributed over the tablespaces configured in * temp_tablespaces. * * Under the covers the set is one or more directories which will eventually - * be deleted when there are no backends attached. + * be deleted. */ void SharedFileSetInit(SharedFileSet *fileset, dsm_segment *seg) @@ -84,7 +99,25 @@ SharedFileSetInit(SharedFileSet *fileset, dsm_segment *seg) } /* Register our cleanup callback. */ - on_dsm_detach(seg, SharedFileSetOnDetach, PointerGetDatum(fileset)); + if (seg) + on_dsm_detach(seg, SharedFileSetOnDetach, PointerGetDatum(fileset)); + else + { + static bool registered_cleanup = false; + + if (!registered_cleanup) + { + /* + * We must not have registered any fileset before registering the + * fileset clean up. + */ + Assert(filesetlist == NIL); + on_proc_exit(SharedFileSetDeleteOnProcExit, 0); + registered_cleanup = true; + } + + filesetlist = lcons((void *) fileset, filesetlist); + } } /* @@ -147,13 +180,13 @@ SharedFileSetCreate(SharedFileSet *fileset, const char *name) * another backend. */ File -SharedFileSetOpen(SharedFileSet *fileset, const char *name) +SharedFileSetOpen(SharedFileSet *fileset, const char *name, int mode) { char path[MAXPGPATH]; File file; SharedFilePath(path, fileset, name); - file = PathNameOpenTemporaryFile(path); + file = PathNameOpenTemporaryFile(path, mode); return file; } @@ -192,6 +225,9 @@ SharedFileSetDeleteAll(SharedFileSet *fileset) SharedFileSetPath(dirpath, fileset, fileset->tablespaces[i]); PathNameDeleteTemporaryDir(dirpath); } + + /* Unregister the shared fileset */ + SharedFileSetUnregister(fileset); } /* @@ -222,6 +258,59 @@ SharedFileSetOnDetach(dsm_segment *segment, Datum datum) SharedFileSetDeleteAll(fileset); } +/* + * Callback function that will be invoked on the process exit. This will + * process the list of all the registered sharedfilesets and delete the + * underlying files. + */ +static void +SharedFileSetDeleteOnProcExit(int status, Datum arg) +{ + ListCell *l; + + /* Loop over all the pending shared fileset entry */ + foreach(l, filesetlist) + { + SharedFileSet *fileset = (SharedFileSet *) lfirst(l); + + SharedFileSetDeleteAll(fileset); + } + + filesetlist = NIL; +} + +/* + * Unregister the shared fileset entry registered for cleanup on proc exit. + */ +void +SharedFileSetUnregister(SharedFileSet *input_fileset) +{ + bool found = false; + ListCell *l; + + /* + * If the caller is following the dsm based cleanup then we don't maintain + * the filesetlist so return. + */ + if (filesetlist == NIL) + return; + + foreach(l, filesetlist) + { + SharedFileSet *fileset = (SharedFileSet *) lfirst(l); + + /* Remove the entry from the list */ + if (input_fileset == fileset) + { + filesetlist = list_delete_cell(filesetlist, l); + found = true; + break; + } + } + + Assert(found); +} + /* * Build the path for the directory holding the files backing a SharedFileSet * in a given tablespace. diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c index 5517e59c50fd..788815cdab6c 100644 --- a/src/backend/utils/sort/logtape.c +++ b/src/backend/utils/sort/logtape.c @@ -78,6 +78,8 @@ #include "postgres.h" +#include + #include "storage/buffile.h" #include "utils/builtins.h" #include "utils/logtape.h" @@ -551,7 +553,7 @@ ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared, lt = <s->tapes[i]; pg_itoa(i, filename); - file = BufFileOpenShared(fileset, filename); + file = BufFileOpenShared(fileset, filename, O_RDONLY); filesize = BufFileSize(file); /* diff --git a/src/backend/utils/sort/sharedtuplestore.c b/src/backend/utils/sort/sharedtuplestore.c index 6537a4303b12..b83fb50dac8f 100644 --- a/src/backend/utils/sort/sharedtuplestore.c +++ b/src/backend/utils/sort/sharedtuplestore.c @@ -559,7 +559,7 @@ sts_parallel_scan_next(SharedTuplestoreAccessor *accessor, void *meta_data) sts_filename(name, accessor, accessor->read_participant); accessor->read_file = - BufFileOpenShared(accessor->fileset, name); + BufFileOpenShared(accessor->fileset, name, O_RDONLY); } /* Seek and load the chunk header. */ diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 13872013823e..807a9c1edf6e 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -916,6 +916,7 @@ typedef enum WAIT_EVENT_BASEBACKUP_READ = PG_WAIT_IO, WAIT_EVENT_BUFFILE_READ, WAIT_EVENT_BUFFILE_WRITE, + WAIT_EVENT_BUFFILE_TRUNCATE, WAIT_EVENT_CONTROL_FILE_READ, WAIT_EVENT_CONTROL_FILE_SYNC, WAIT_EVENT_CONTROL_FILE_SYNC_UPDATE, diff --git a/src/include/storage/buffile.h b/src/include/storage/buffile.h index f4752bab0da5..fc34c49522da 100644 --- a/src/include/storage/buffile.h +++ b/src/include/storage/buffile.h @@ -48,7 +48,9 @@ extern long BufFileAppend(BufFile *target, BufFile *source); extern BufFile *BufFileCreateShared(SharedFileSet *fileset, const char *name); extern void BufFileExportShared(BufFile *file); -extern BufFile *BufFileOpenShared(SharedFileSet *fileset, const char *name); +extern BufFile *BufFileOpenShared(SharedFileSet *fileset, const char *name, + int mode); extern void BufFileDeleteShared(SharedFileSet *fileset, const char *name); +extern void BufFileTruncateShared(BufFile *file, int fileno, off_t offset); #endif /* BUFFILE_H */ diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 8cd125d7dfaa..e209f047e853 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -94,7 +94,7 @@ extern mode_t FileGetRawMode(File file); /* Operations used for sharing named temporary files */ extern File PathNameCreateTemporaryFile(const char *name, bool error_on_failure); -extern File PathNameOpenTemporaryFile(const char *name); +extern File PathNameOpenTemporaryFile(const char *path, int mode); extern bool PathNameDeleteTemporaryFile(const char *name, bool error_on_failure); extern void PathNameCreateTemporaryDir(const char *base, const char *name); extern void PathNameDeleteTemporaryDir(const char *name); diff --git a/src/include/storage/sharedfileset.h b/src/include/storage/sharedfileset.h index 2d6cf077e51d..d5edb600af96 100644 --- a/src/include/storage/sharedfileset.h +++ b/src/include/storage/sharedfileset.h @@ -37,9 +37,11 @@ typedef struct SharedFileSet extern void SharedFileSetInit(SharedFileSet *fileset, dsm_segment *seg); extern void SharedFileSetAttach(SharedFileSet *fileset, dsm_segment *seg); extern File SharedFileSetCreate(SharedFileSet *fileset, const char *name); -extern File SharedFileSetOpen(SharedFileSet *fileset, const char *name); +extern File SharedFileSetOpen(SharedFileSet *fileset, const char *name, + int mode); extern bool SharedFileSetDelete(SharedFileSet *fileset, const char *name, bool error_on_failure); extern void SharedFileSetDeleteAll(SharedFileSet *fileset); +extern void SharedFileSetUnregister(SharedFileSet *input_fileset); #endif From 7e453634bb62f06a048f5562ba59d52aa1f28d12 Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Wed, 26 Aug 2020 09:40:52 +0530 Subject: [PATCH 018/589] Add additional information in the vacuum error context. The additional information added will be an offset number for heap operations. This information will help us in finding the exact tuple due to which the error has occurred. Author: Mahendra Singh Thalor and Amit Kapila Reviewed-by: Sawada Masahiko, Justin Pryzby and Amit Kapila Discussion: https://postgr.es/m/CAKYtNApK488TDF4bMbw+1QH8HJf9cxdNDXquhU50TK5iv_FtCQ@mail.gmail.com --- src/backend/access/heap/pruneheap.c | 19 +++++- src/backend/access/heap/vacuumlazy.c | 91 ++++++++++++++++++++++------ src/include/access/heapam.h | 3 +- 3 files changed, 90 insertions(+), 23 deletions(-) diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 3ad4222cb8af..bc510e2e9b36 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -188,7 +188,7 @@ heap_page_prune_opt(Relation relation, Buffer buffer) /* OK to prune */ (void) heap_page_prune(relation, buffer, vistest, limited_xmin, limited_ts, - true, &ignore); + true, &ignore, NULL); } /* And release buffer lock */ @@ -213,6 +213,9 @@ heap_page_prune_opt(Relation relation, Buffer buffer) * send its own new total to pgstats, and we don't want this delta applied * on top of that.) * + * off_loc is the offset location required by the caller to use in error + * callback. + * * Returns the number of tuples deleted from the page and sets * latestRemovedXid. */ @@ -221,7 +224,8 @@ heap_page_prune(Relation relation, Buffer buffer, GlobalVisState *vistest, TransactionId old_snap_xmin, TimestampTz old_snap_ts, - bool report_stats, TransactionId *latestRemovedXid) + bool report_stats, TransactionId *latestRemovedXid, + OffsetNumber *off_loc) { int ndeleted = 0; Page page = BufferGetPage(buffer); @@ -262,6 +266,13 @@ heap_page_prune(Relation relation, Buffer buffer, if (prstate.marked[offnum]) continue; + /* + * Set the offset number so that we can display it along with any + * error that occurred while processing this tuple. + */ + if (off_loc) + *off_loc = offnum; + /* Nothing to do if slot is empty or already dead */ itemid = PageGetItemId(page, offnum); if (!ItemIdIsUsed(itemid) || ItemIdIsDead(itemid)) @@ -271,6 +282,10 @@ heap_page_prune(Relation relation, Buffer buffer, ndeleted += heap_prune_chain(buffer, offnum, &prstate); } + /* Clear the offset information once we have processed the given page. */ + if (off_loc) + *off_loc = InvalidOffsetNumber; + /* Any error while applying the changes is critical */ START_CRIT_SECTION(); diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 8de31bf071b8..a0da444af0ea 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -316,6 +316,7 @@ typedef struct LVRelStats /* Used for error callback */ char *indname; BlockNumber blkno; /* used only for heap operations */ + OffsetNumber offnum; /* used only for heap operations */ VacErrPhase phase; } LVRelStats; @@ -323,6 +324,7 @@ typedef struct LVRelStats typedef struct LVSavedErrInfo { BlockNumber blkno; + OffsetNumber offnum; VacErrPhase phase; } LVSavedErrInfo; @@ -341,7 +343,8 @@ static void lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, Relation *Irel, int nindexes, bool aggressive); static void lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats); -static bool lazy_check_needs_freeze(Buffer buf, bool *hastup); +static bool lazy_check_needs_freeze(Buffer buf, bool *hastup, + LVRelStats *vacrelstats); static void lazy_vacuum_all_indexes(Relation onerel, Relation *Irel, IndexBulkDeleteResult **stats, LVRelStats *vacrelstats, LVParallelState *lps, @@ -364,6 +367,7 @@ static void lazy_record_dead_tuple(LVDeadTuples *dead_tuples, static bool lazy_tid_reaped(ItemPointer itemptr, void *state); static int vac_cmp_itemptr(const void *left, const void *right); static bool heap_page_is_all_visible(Relation rel, Buffer buf, + LVRelStats *vacrelstats, TransactionId *visibility_cutoff_xid, bool *all_frozen); static void lazy_parallel_vacuum_indexes(Relation *Irel, IndexBulkDeleteResult **stats, LVRelStats *vacrelstats, LVParallelState *lps, @@ -396,7 +400,8 @@ static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); static void vacuum_error_callback(void *arg); static void update_vacuum_error_info(LVRelStats *errinfo, LVSavedErrInfo *saved_err_info, - int phase, BlockNumber blkno); + int phase, BlockNumber blkno, + OffsetNumber offnum); static void restore_vacuum_error_info(LVRelStats *errinfo, const LVSavedErrInfo *saved_err_info); @@ -547,7 +552,8 @@ heap_vacuum_rel(Relation onerel, VacuumParams *params, * revert to the previous phase. */ update_vacuum_error_info(vacrelstats, NULL, VACUUM_ERRCB_PHASE_TRUNCATE, - vacrelstats->nonempty_pages); + vacrelstats->nonempty_pages, + InvalidOffsetNumber); lazy_truncate_heap(onerel, vacrelstats); } @@ -960,7 +966,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); update_vacuum_error_info(vacrelstats, NULL, VACUUM_ERRCB_PHASE_SCAN_HEAP, - blkno); + blkno, InvalidOffsetNumber); if (blkno == next_unskippable_block) { @@ -1129,7 +1135,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, * to use lazy_check_needs_freeze() for both situations, though. */ LockBuffer(buf, BUFFER_LOCK_SHARE); - if (!lazy_check_needs_freeze(buf, &hastup)) + if (!lazy_check_needs_freeze(buf, &hastup, vacrelstats)) { UnlockReleaseBuffer(buf); vacrelstats->scanned_pages++; @@ -1244,7 +1250,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, */ tups_vacuumed += heap_page_prune(onerel, buf, vistest, false, InvalidTransactionId, 0, - &vacrelstats->latestRemovedXid); + &vacrelstats->latestRemovedXid, + &vacrelstats->offnum); /* * Now scan the page to collect vacuumable items and check for tuples @@ -1267,6 +1274,11 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, { ItemId itemid; + /* + * Set the offset number so that we can display it along with any + * error that occurred while processing this tuple. + */ + vacrelstats->offnum = offnum; itemid = PageGetItemId(page, offnum); /* Unused items require no processing, but we count 'em */ @@ -1468,6 +1480,12 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, } } /* scan along page */ + /* + * Clear the offset information once we have processed all the tuples + * on the page. + */ + vacrelstats->offnum = InvalidOffsetNumber; + /* * If we froze any tuples, mark the buffer dirty, and write a WAL * record recording the changes. We must log the changes to be @@ -1845,7 +1863,7 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) /* Update error traceback information */ update_vacuum_error_info(vacrelstats, &saved_err_info, VACUUM_ERRCB_PHASE_VACUUM_HEAP, - InvalidBlockNumber); + InvalidBlockNumber, InvalidOffsetNumber); pg_rusage_init(&ru0); npages = 0; @@ -1927,7 +1945,7 @@ lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer, /* Update error traceback information */ update_vacuum_error_info(vacrelstats, &saved_err_info, VACUUM_ERRCB_PHASE_VACUUM_HEAP, - blkno); + blkno, InvalidOffsetNumber); START_CRIT_SECTION(); @@ -1979,7 +1997,8 @@ lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer, * dirty, exclusively locked, and, if needed, a full page image has been * emitted in the log_heap_clean() above. */ - if (heap_page_is_all_visible(onerel, buffer, &visibility_cutoff_xid, + if (heap_page_is_all_visible(onerel, buffer, vacrelstats, + &visibility_cutoff_xid, &all_frozen)) PageSetAllVisible(page); @@ -2018,7 +2037,7 @@ lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer, * Also returns a flag indicating whether page contains any tuples at all. */ static bool -lazy_check_needs_freeze(Buffer buf, bool *hastup) +lazy_check_needs_freeze(Buffer buf, bool *hastup, LVRelStats *vacrelstats) { Page page = BufferGetPage(buf); OffsetNumber offnum, @@ -2043,6 +2062,11 @@ lazy_check_needs_freeze(Buffer buf, bool *hastup) { ItemId itemid; + /* + * Set the offset number so that we can display it along with any + * error that occurred while processing this tuple. + */ + vacrelstats->offnum = offnum; itemid = PageGetItemId(page, offnum); /* this should match hastup test in count_nondeletable_pages() */ @@ -2057,10 +2081,13 @@ lazy_check_needs_freeze(Buffer buf, bool *hastup) if (heap_tuple_needs_freeze(tupleheader, FreezeLimit, MultiXactCutoff, buf)) - return true; + break; } /* scan along page */ - return false; + /* Clear the offset information once we have processed the given page. */ + vacrelstats->offnum = InvalidOffsetNumber; + + return (offnum <= maxoff); } /* @@ -2438,7 +2465,7 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, vacrelstats->indname = pstrdup(RelationGetRelationName(indrel)); update_vacuum_error_info(vacrelstats, &saved_err_info, VACUUM_ERRCB_PHASE_VACUUM_INDEX, - InvalidBlockNumber); + InvalidBlockNumber, InvalidOffsetNumber); /* Do bulk deletion */ *stats = index_bulk_delete(&ivinfo, *stats, @@ -2498,7 +2525,7 @@ lazy_cleanup_index(Relation indrel, vacrelstats->indname = pstrdup(RelationGetRelationName(indrel)); update_vacuum_error_info(vacrelstats, &saved_err_info, VACUUM_ERRCB_PHASE_INDEX_CLEANUP, - InvalidBlockNumber); + InvalidBlockNumber, InvalidOffsetNumber); *stats = index_vacuum_cleanup(&ivinfo, *stats); @@ -2522,7 +2549,7 @@ lazy_cleanup_index(Relation indrel, pg_rusage_show(&ru0)))); } - /* Revert back to the old phase information for error traceback */ + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrelstats, &saved_err_info); pfree(vacrelstats->indname); vacrelstats->indname = NULL; @@ -2964,6 +2991,7 @@ vac_cmp_itemptr(const void *left, const void *right) */ static bool heap_page_is_all_visible(Relation rel, Buffer buf, + LVRelStats *vacrelstats, TransactionId *visibility_cutoff_xid, bool *all_frozen) { @@ -2988,6 +3016,11 @@ heap_page_is_all_visible(Relation rel, Buffer buf, ItemId itemid; HeapTupleData tuple; + /* + * Set the offset number so that we can display it along with any + * error that occurred while processing this tuple. + */ + vacrelstats->offnum = offnum; itemid = PageGetItemId(page, offnum); /* Unused or redirect line pointers are of no interest */ @@ -3065,6 +3098,9 @@ heap_page_is_all_visible(Relation rel, Buffer buf, } } /* scan along page */ + /* Clear the offset information once we have processed the given page. */ + vacrelstats->offnum = InvalidOffsetNumber; + return all_visible; } @@ -3586,8 +3622,14 @@ vacuum_error_callback(void *arg) { case VACUUM_ERRCB_PHASE_SCAN_HEAP: if (BlockNumberIsValid(errinfo->blkno)) - errcontext("while scanning block %u of relation \"%s.%s\"", - errinfo->blkno, errinfo->relnamespace, errinfo->relname); + { + if (OffsetNumberIsValid(errinfo->offnum)) + errcontext("while scanning block %u and offset %u of relation \"%s.%s\"", + errinfo->blkno, errinfo->offnum, errinfo->relnamespace, errinfo->relname); + else + errcontext("while scanning block %u of relation \"%s.%s\"", + errinfo->blkno, errinfo->relnamespace, errinfo->relname); + } else errcontext("while scanning relation \"%s.%s\"", errinfo->relnamespace, errinfo->relname); @@ -3595,8 +3637,14 @@ vacuum_error_callback(void *arg) case VACUUM_ERRCB_PHASE_VACUUM_HEAP: if (BlockNumberIsValid(errinfo->blkno)) - errcontext("while vacuuming block %u of relation \"%s.%s\"", - errinfo->blkno, errinfo->relnamespace, errinfo->relname); + { + if (OffsetNumberIsValid(errinfo->offnum)) + errcontext("while vacuuming block %u and offset %u of relation \"%s.%s\"", + errinfo->blkno, errinfo->offnum, errinfo->relnamespace, errinfo->relname); + else + errcontext("while vacuuming block %u of relation \"%s.%s\"", + errinfo->blkno, errinfo->relnamespace, errinfo->relname); + } else errcontext("while vacuuming relation \"%s.%s\"", errinfo->relnamespace, errinfo->relname); @@ -3631,15 +3679,17 @@ vacuum_error_callback(void *arg) */ static void update_vacuum_error_info(LVRelStats *errinfo, LVSavedErrInfo *saved_err_info, int phase, - BlockNumber blkno) + BlockNumber blkno, OffsetNumber offnum) { if (saved_err_info) { + saved_err_info->offnum = errinfo->offnum; saved_err_info->blkno = errinfo->blkno; saved_err_info->phase = errinfo->phase; } errinfo->blkno = blkno; + errinfo->offnum = offnum; errinfo->phase = phase; } @@ -3650,5 +3700,6 @@ static void restore_vacuum_error_info(LVRelStats *errinfo, const LVSavedErrInfo *saved_err_info) { errinfo->blkno = saved_err_info->blkno; + errinfo->offnum = saved_err_info->offnum; errinfo->phase = saved_err_info->phase; } diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index ba77013f64f2..92b19dba324f 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -178,7 +178,8 @@ extern int heap_page_prune(Relation relation, Buffer buffer, struct GlobalVisState *vistest, TransactionId limited_oldest_xmin, TimestampTz limited_oldest_ts, - bool report_stats, TransactionId *latestRemovedXid); + bool report_stats, TransactionId *latestRemovedXid, + OffsetNumber *off_loc); extern void heap_page_prune_execute(Buffer buffer, OffsetNumber *redirected, int nredirected, OffsetNumber *nowdead, int ndead, From fe7fd4e9613f58262d30782a34b01cc0c4cbbeb5 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 26 Aug 2020 20:42:27 +0900 Subject: [PATCH 019/589] Add regression tests for REPLICA IDENTITY with dropped indexes REPLICA IDENTITY USING INDEX behaves the same way as NOTHING if the associated index is dropped, even if there is a primary key that could be used as a fallback for the changes generated. There have never been any tests to cover such scenarios, so this commit closes the gap. Author: Michael Paquier Reviewed-by: Masahiko Sawada, Rahila Syed, Euler Taveira Discussion: https://postgr.es/m/20200522035028.GO2355@paquier.xyz --- contrib/test_decoding/expected/ddl.out | 71 +++++++++++++++++++++++++- contrib/test_decoding/sql/ddl.sql | 31 +++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/contrib/test_decoding/expected/ddl.out b/contrib/test_decoding/expected/ddl.out index d79cd316b79f..4ff0044c7879 100644 --- a/contrib/test_decoding/expected/ddl.out +++ b/contrib/test_decoding/expected/ddl.out @@ -565,6 +565,35 @@ UPDATE table_with_unique_not_null SET data = 3 WHERE data = 2; UPDATE table_with_unique_not_null SET id = -id; UPDATE table_with_unique_not_null SET id = -id; DELETE FROM table_with_unique_not_null WHERE data = 3; +-- check tables with dropped indexes used in REPLICA IDENTITY +-- table with primary key +CREATE TABLE table_dropped_index_with_pk (a int PRIMARY KEY, b int, c int); +CREATE UNIQUE INDEX table_dropped_index_with_pk_idx + ON table_dropped_index_with_pk(a); +ALTER TABLE table_dropped_index_with_pk REPLICA IDENTITY + USING INDEX table_dropped_index_with_pk_idx; +DROP INDEX table_dropped_index_with_pk_idx; +INSERT INTO table_dropped_index_with_pk VALUES (1,1,1), (2,2,2), (3,3,3); +UPDATE table_dropped_index_with_pk SET a = 4 WHERE a = 1; +UPDATE table_dropped_index_with_pk SET b = 5 WHERE a = 2; +UPDATE table_dropped_index_with_pk SET b = 6, c = 7 WHERE a = 3; +DELETE FROM table_dropped_index_with_pk WHERE b = 1; +DELETE FROM table_dropped_index_with_pk WHERE a = 3; +DROP TABLE table_dropped_index_with_pk; +-- table without primary key +CREATE TABLE table_dropped_index_no_pk (a int NOT NULL, b int, c int); +CREATE UNIQUE INDEX table_dropped_index_no_pk_idx + ON table_dropped_index_no_pk(a); +ALTER TABLE table_dropped_index_no_pk REPLICA IDENTITY + USING INDEX table_dropped_index_no_pk_idx; +DROP INDEX table_dropped_index_no_pk_idx; +INSERT INTO table_dropped_index_no_pk VALUES (1,1,1), (2,2,2), (3,3,3); +UPDATE table_dropped_index_no_pk SET a = 4 WHERE a = 1; +UPDATE table_dropped_index_no_pk SET b = 5 WHERE a = 2; +UPDATE table_dropped_index_no_pk SET b = 6, c = 7 WHERE a = 3; +DELETE FROM table_dropped_index_no_pk WHERE b = 1; +DELETE FROM table_dropped_index_no_pk WHERE a = 3; +DROP TABLE table_dropped_index_no_pk; -- check toast support BEGIN; CREATE SEQUENCE toasttable_rand_seq START 79 INCREMENT 1499; -- portable "random" @@ -682,6 +711,46 @@ SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'inc table public.table_with_unique_not_null: DELETE: id[integer]:4 COMMIT BEGIN + table public.table_dropped_index_with_pk: INSERT: a[integer]:1 b[integer]:1 c[integer]:1 + table public.table_dropped_index_with_pk: INSERT: a[integer]:2 b[integer]:2 c[integer]:2 + table public.table_dropped_index_with_pk: INSERT: a[integer]:3 b[integer]:3 c[integer]:3 + COMMIT + BEGIN + table public.table_dropped_index_with_pk: UPDATE: a[integer]:4 b[integer]:1 c[integer]:1 + COMMIT + BEGIN + table public.table_dropped_index_with_pk: UPDATE: a[integer]:2 b[integer]:5 c[integer]:2 + COMMIT + BEGIN + table public.table_dropped_index_with_pk: UPDATE: a[integer]:3 b[integer]:6 c[integer]:7 + COMMIT + BEGIN + table public.table_dropped_index_with_pk: DELETE: (no-tuple-data) + COMMIT + BEGIN + table public.table_dropped_index_with_pk: DELETE: (no-tuple-data) + COMMIT + BEGIN + table public.table_dropped_index_no_pk: INSERT: a[integer]:1 b[integer]:1 c[integer]:1 + table public.table_dropped_index_no_pk: INSERT: a[integer]:2 b[integer]:2 c[integer]:2 + table public.table_dropped_index_no_pk: INSERT: a[integer]:3 b[integer]:3 c[integer]:3 + COMMIT + BEGIN + table public.table_dropped_index_no_pk: UPDATE: a[integer]:4 b[integer]:1 c[integer]:1 + COMMIT + BEGIN + table public.table_dropped_index_no_pk: UPDATE: a[integer]:2 b[integer]:5 c[integer]:2 + COMMIT + BEGIN + table public.table_dropped_index_no_pk: UPDATE: a[integer]:3 b[integer]:6 c[integer]:7 + COMMIT + BEGIN + table public.table_dropped_index_no_pk: DELETE: (no-tuple-data) + COMMIT + BEGIN + table public.table_dropped_index_no_pk: DELETE: (no-tuple-data) + COMMIT + BEGIN table public.toasttable: INSERT: id[integer]:1 toasted_col1[text]:'12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000' rand1[double precision]:79 toasted_col2[text]:null rand2[double precision]:1578 COMMIT BEGIN @@ -690,7 +759,7 @@ SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'inc BEGIN table public.toasttable: UPDATE: id[integer]:1 toasted_col1[text]:'12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000' rand1[double precision]:79 toasted_col2[text]:null rand2[double precision]:1578 COMMIT -(103 rows) +(143 rows) INSERT INTO toasttable(toasted_col1) SELECT string_agg(g.i::text, '') FROM generate_series(1, 2000) g(i); -- update of second column, first column unchanged diff --git a/contrib/test_decoding/sql/ddl.sql b/contrib/test_decoding/sql/ddl.sql index 2c4823e57805..1b3866d01530 100644 --- a/contrib/test_decoding/sql/ddl.sql +++ b/contrib/test_decoding/sql/ddl.sql @@ -345,6 +345,37 @@ UPDATE table_with_unique_not_null SET id = -id; UPDATE table_with_unique_not_null SET id = -id; DELETE FROM table_with_unique_not_null WHERE data = 3; +-- check tables with dropped indexes used in REPLICA IDENTITY +-- table with primary key +CREATE TABLE table_dropped_index_with_pk (a int PRIMARY KEY, b int, c int); +CREATE UNIQUE INDEX table_dropped_index_with_pk_idx + ON table_dropped_index_with_pk(a); +ALTER TABLE table_dropped_index_with_pk REPLICA IDENTITY + USING INDEX table_dropped_index_with_pk_idx; +DROP INDEX table_dropped_index_with_pk_idx; +INSERT INTO table_dropped_index_with_pk VALUES (1,1,1), (2,2,2), (3,3,3); +UPDATE table_dropped_index_with_pk SET a = 4 WHERE a = 1; +UPDATE table_dropped_index_with_pk SET b = 5 WHERE a = 2; +UPDATE table_dropped_index_with_pk SET b = 6, c = 7 WHERE a = 3; +DELETE FROM table_dropped_index_with_pk WHERE b = 1; +DELETE FROM table_dropped_index_with_pk WHERE a = 3; +DROP TABLE table_dropped_index_with_pk; + +-- table without primary key +CREATE TABLE table_dropped_index_no_pk (a int NOT NULL, b int, c int); +CREATE UNIQUE INDEX table_dropped_index_no_pk_idx + ON table_dropped_index_no_pk(a); +ALTER TABLE table_dropped_index_no_pk REPLICA IDENTITY + USING INDEX table_dropped_index_no_pk_idx; +DROP INDEX table_dropped_index_no_pk_idx; +INSERT INTO table_dropped_index_no_pk VALUES (1,1,1), (2,2,2), (3,3,3); +UPDATE table_dropped_index_no_pk SET a = 4 WHERE a = 1; +UPDATE table_dropped_index_no_pk SET b = 5 WHERE a = 2; +UPDATE table_dropped_index_no_pk SET b = 6, c = 7 WHERE a = 3; +DELETE FROM table_dropped_index_no_pk WHERE b = 1; +DELETE FROM table_dropped_index_no_pk WHERE a = 3; +DROP TABLE table_dropped_index_no_pk; + -- check toast support BEGIN; CREATE SEQUENCE toasttable_rand_seq START 79 INCREMENT 1499; -- portable "random" From e942af7b8261cd8070d0eeaf518dbc1a664859fd Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 26 Aug 2020 17:08:11 -0400 Subject: [PATCH 020/589] Suppress compiler warning in non-cassert builds. Oversight in 808e13b28, reported by Bruce Momjian. Discussion: https://postgr.es/m/20200826160251.GB21909@momjian.us --- src/backend/storage/file/sharedfileset.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/backend/storage/file/sharedfileset.c b/src/backend/storage/file/sharedfileset.c index 65fd8ff5c0c3..8b96e81fffff 100644 --- a/src/backend/storage/file/sharedfileset.c +++ b/src/backend/storage/file/sharedfileset.c @@ -285,7 +285,6 @@ SharedFileSetDeleteOnProcExit(int status, Datum arg) void SharedFileSetUnregister(SharedFileSet *input_fileset) { - bool found = false; ListCell *l; /* @@ -303,12 +302,12 @@ SharedFileSetUnregister(SharedFileSet *input_fileset) if (input_fileset == fileset) { filesetlist = list_delete_cell(filesetlist, l); - found = true; - break; + return; } } - Assert(found); + /* Should have found a match */ + Assert(false); } /* From 77c7267c37f7fa8e5e48abda4798afdbecb2b95a Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 27 Aug 2020 16:40:34 +0900 Subject: [PATCH 021/589] Fix comment in procarray.c The description of GlobalVisDataRels was missing, GlobalVisCatalogRels being mentioned instead. Author: Jim Nasby Discussion: https://postgr.es/m/8e06c883-2858-1fd4-07c5-560c28b08dcd@amazon.com --- src/backend/storage/ipc/procarray.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c index 45eab7e5a622..a023090fbbd3 100644 --- a/src/backend/storage/ipc/procarray.c +++ b/src/backend/storage/ipc/procarray.c @@ -146,7 +146,7 @@ typedef struct ProcArrayStruct * I.e. the difference to GlobalVisSharedRels is that * snapshot in other databases are ignored. * - * 3) GlobalVisCatalogRels, which only considers an XID's + * 3) GlobalVisDataRels, which only considers an XID's * effects visible-to-everyone if neither snapshots in the current * database, nor a replication slot's xmin consider XID as running. * From 10564ee02ca380f8d614eabc4e80c5d39ea4edad Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 27 Aug 2020 17:36:13 -0400 Subject: [PATCH 022/589] Fix code for re-finding scan position in a multicolumn GIN index. collectMatchBitmap() needs to re-find the index tuple it was previously looking at, after transiently dropping lock on the index page it's on. The tuple should still exist and be at its prior position or somewhere to the right of that, since ginvacuum never removes tuples but concurrent insertions could add one. However, there was a thinko in that logic, to the effect of expecting any inserted tuples to have the same index "attnum" as what we'd been scanning. Since there's no physical separation of tuples with different attnums, it's not terribly hard to devise scenarios where this fails, leading to transient "lost saved point in index" errors. (While I've duplicated this with manual testing, it seems impossible to make a reproducible test case with our available testing technology.) Fix by just continuing the scan when the attnum doesn't match. While here, improve the error message used if we do fail, so that it matches the wording used in btree for a similar case. collectMatchBitmap()'s posting-tree code path was previously not exercised at all by our regression tests. While I can't make a regression test that exhibits the bug, I can at least improve the code coverage here, so do that. The test case I made for this is an extension of one added by 4b754d6c1, so it only works in HEAD and v13; didn't seem worth trying hard to back-patch it. Per bug #16595 from Jesse Kinkead. This has been broken since multicolumn capability was added to GIN (commit 27cb66fdf), so back-patch to all supported branches. Discussion: https://postgr.es/m/16595-633118be8eef9ce2@postgresql.org --- src/backend/access/gin/ginget.c | 28 +++++++------ src/test/regress/expected/gin.out | 65 +++++++++++++++++++++++++++++++ src/test/regress/sql/gin.sql | 24 ++++++++++++ 3 files changed, 105 insertions(+), 12 deletions(-) diff --git a/src/backend/access/gin/ginget.c b/src/backend/access/gin/ginget.c index 7bdcbc858e39..2cfccdedcf59 100644 --- a/src/backend/access/gin/ginget.c +++ b/src/backend/access/gin/ginget.c @@ -264,24 +264,28 @@ collectMatchBitmap(GinBtreeData *btree, GinBtreeStack *stack, /* Search forward to re-find idatum */ for (;;) { - Datum newDatum; - GinNullCategory newCategory; - if (moveRightIfItNeeded(btree, stack, snapshot) == false) - elog(ERROR, "lost saved point in index"); /* must not happen !!! */ + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("failed to re-find tuple within index \"%s\"", + RelationGetRelationName(btree->index)))); page = BufferGetPage(stack->buffer); itup = (IndexTuple) PageGetItem(page, PageGetItemId(page, stack->off)); - if (gintuple_get_attrnum(btree->ginstate, itup) != attnum) - elog(ERROR, "lost saved point in index"); /* must not happen !!! */ - newDatum = gintuple_get_key(btree->ginstate, itup, - &newCategory); + if (gintuple_get_attrnum(btree->ginstate, itup) == attnum) + { + Datum newDatum; + GinNullCategory newCategory; + + newDatum = gintuple_get_key(btree->ginstate, itup, + &newCategory); - if (ginCompareEntries(btree->ginstate, attnum, - newDatum, newCategory, - idatum, icategory) == 0) - break; /* Found! */ + if (ginCompareEntries(btree->ginstate, attnum, + newDatum, newCategory, + idatum, icategory) == 0) + break; /* Found! */ + } stack->off++; } diff --git a/src/test/regress/expected/gin.out b/src/test/regress/expected/gin.out index 83de5220fb9c..b335466fc4ba 100644 --- a/src/test/regress/expected/gin.out +++ b/src/test/regress/expected/gin.out @@ -199,6 +199,71 @@ from i @> '{1}' and j @> '{10}' | 2 | 0 | t (10 rows) +reset enable_seqscan; +reset enable_bitmapscan; +-- re-purpose t_gin_test_tbl to test scans involving posting trees +insert into t_gin_test_tbl select array[1, g, g/10], array[2, g, g/10] + from generate_series(1, 20000) g; +select gin_clean_pending_list('t_gin_test_tbl_i_j_idx') is not null; + ?column? +---------- + t +(1 row) + +analyze t_gin_test_tbl; +set enable_seqscan = off; +set enable_bitmapscan = on; +explain (costs off) +select count(*) from t_gin_test_tbl where j @> array[50]; + QUERY PLAN +--------------------------------------------------------- + Aggregate + -> Bitmap Heap Scan on t_gin_test_tbl + Recheck Cond: (j @> '{50}'::integer[]) + -> Bitmap Index Scan on t_gin_test_tbl_i_j_idx + Index Cond: (j @> '{50}'::integer[]) +(5 rows) + +select count(*) from t_gin_test_tbl where j @> array[50]; + count +------- + 11 +(1 row) + +explain (costs off) +select count(*) from t_gin_test_tbl where j @> array[2]; + QUERY PLAN +--------------------------------------------------------- + Aggregate + -> Bitmap Heap Scan on t_gin_test_tbl + Recheck Cond: (j @> '{2}'::integer[]) + -> Bitmap Index Scan on t_gin_test_tbl_i_j_idx + Index Cond: (j @> '{2}'::integer[]) +(5 rows) + +select count(*) from t_gin_test_tbl where j @> array[2]; + count +------- + 20000 +(1 row) + +explain (costs off) +select count(*) from t_gin_test_tbl where j @> '{}'::int[]; + QUERY PLAN +--------------------------------------------------------- + Aggregate + -> Bitmap Heap Scan on t_gin_test_tbl + Recheck Cond: (j @> '{}'::integer[]) + -> Bitmap Index Scan on t_gin_test_tbl_i_j_idx + Index Cond: (j @> '{}'::integer[]) +(5 rows) + +select count(*) from t_gin_test_tbl where j @> '{}'::int[]; + count +------- + 20006 +(1 row) + reset enable_seqscan; reset enable_bitmapscan; drop table t_gin_test_tbl; diff --git a/src/test/regress/sql/gin.sql b/src/test/regress/sql/gin.sql index abe35752652a..efb8ef3e964c 100644 --- a/src/test/regress/sql/gin.sql +++ b/src/test/regress/sql/gin.sql @@ -138,4 +138,28 @@ from reset enable_seqscan; reset enable_bitmapscan; +-- re-purpose t_gin_test_tbl to test scans involving posting trees +insert into t_gin_test_tbl select array[1, g, g/10], array[2, g, g/10] + from generate_series(1, 20000) g; + +select gin_clean_pending_list('t_gin_test_tbl_i_j_idx') is not null; + +analyze t_gin_test_tbl; + +set enable_seqscan = off; +set enable_bitmapscan = on; + +explain (costs off) +select count(*) from t_gin_test_tbl where j @> array[50]; +select count(*) from t_gin_test_tbl where j @> array[50]; +explain (costs off) +select count(*) from t_gin_test_tbl where j @> array[2]; +select count(*) from t_gin_test_tbl where j @> array[2]; +explain (costs off) +select count(*) from t_gin_test_tbl where j @> '{}'::int[]; +select count(*) from t_gin_test_tbl where j @> '{}'::int[]; + +reset enable_seqscan; +reset enable_bitmapscan; + drop table t_gin_test_tbl; From 924123a87f40c12063a2bb2500805447cddc02a3 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 28 Aug 2020 08:16:32 +0200 Subject: [PATCH 023/589] passwordcheck: Log cracklib diagnostics When calling cracklib to check the password, the diagnostic from cracklib was thrown away. This would hide essential information such as no dictionary being installed. Change this to show the cracklib error message using errdetail_log(). Reviewed-by: Daniel Gustafsson Reviewed-by: Laurenz Albe Discussion: https://www.postgresql.org/message-id/flat/f7266133-618a-0adc-52ef-f43c78806b0e%402ndquadrant.com --- contrib/passwordcheck/passwordcheck.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/contrib/passwordcheck/passwordcheck.c b/contrib/passwordcheck/passwordcheck.c index d5f9d14b0109..70f056232fe7 100644 --- a/contrib/passwordcheck/passwordcheck.c +++ b/contrib/passwordcheck/passwordcheck.c @@ -91,6 +91,9 @@ check_password(const char *username, int i; bool pwd_has_letter, pwd_has_nonletter; +#ifdef USE_CRACKLIB + const char *reason; +#endif /* enforce minimum length */ if (pwdlen < MIN_PWD_LENGTH) @@ -125,10 +128,11 @@ check_password(const char *username, #ifdef USE_CRACKLIB /* call cracklib to check password */ - if (FascistCheck(password, CRACKLIB_DICTPATH)) + if ((reason = FascistCheck(password, CRACKLIB_DICTPATH))) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("password is easily cracked"))); + errmsg("password is easily cracked"), + errdetail_log("cracklib diagnostic: %s", reason))); #endif } From 42aaed60c83ff51aa736f50ad96e43653fc539da Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 28 Aug 2020 08:19:12 +0200 Subject: [PATCH 024/589] doc: Update cracklib URL Author: Daniel Gustafsson Reviewed-by: Laurenz Albe Discussion: https://www.postgresql.org/message-id/flat/f7266133-618a-0adc-52ef-f43c78806b0e%402ndquadrant.com --- doc/src/sgml/passwordcheck.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/passwordcheck.sgml b/doc/src/sgml/passwordcheck.sgml index 4128b6cc4f6f..0d89bb95b9de 100644 --- a/doc/src/sgml/passwordcheck.sgml +++ b/doc/src/sgml/passwordcheck.sgml @@ -25,7 +25,7 @@ You can adapt this module to your needs by changing the source code. For example, you can use - CrackLib + CrackLib to check passwords — this only requires uncommenting two lines in the Makefile and rebuilding the module. (We cannot include CrackLib From 7a1cd5260aa20bc13aec8960a57904b5623d1830 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Fri, 28 Aug 2020 16:54:59 +0900 Subject: [PATCH 025/589] doc: Rework tables for built-in operator classes of index AMs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tables listing all the operator classes available for BRIN, GIN, GiST and SP-GiST had a confusing format where the same operator could be listed multiple times, for different data types. This improves the shape of these tables by adding the types associated to each operator, for their associated operator class. Each table included previously the data type that could be used for an operator class in an extra column. This is removed to reduce the width of the tables as this is now described within each operator. This also makes the tables fit better in the PDF documentation. Reported-by: osdba Author: Michael Paquier Reviewed-by: Álvaro Herrera, Tom Lane, Bruce Momjian Discussion: https://postgr.es/m/38d55061.9604.173b32c60ec.Coremail.mailtch@163.com --- doc/src/sgml/brin.sgml | 616 ++++++++++++++++++--------------------- doc/src/sgml/gin.sgml | 75 ++--- doc/src/sgml/gist.sgml | 225 ++++++-------- doc/src/sgml/spgist.sgml | 202 ++++++------- 4 files changed, 504 insertions(+), 614 deletions(-) diff --git a/doc/src/sgml/brin.sgml b/doc/src/sgml/brin.sgml index b9d596e3c4e7..4420794e5bb5 100644 --- a/doc/src/sgml/brin.sgml +++ b/doc/src/sgml/brin.sgml @@ -120,354 +120,292 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was Built-in <acronym>BRIN</acronym> Operator Classes - - - - + Name - Indexed Data Type Indexable Operators - int8_minmax_ops - bigint - - < - <= - = - >= - > - - - - bit_minmax_ops - bit - - < - <= - = - >= - > - - - - varbit_minmax_ops - bit varying - - < - <= - = - >= - > - - - - box_inclusion_ops - box - - << - &< - && - &> - >> - ~= - @> - <@ - &<| - <<| - |>> - |&> - - - - bytea_minmax_ops - bytea - - < - <= - = - >= - > - - - - bpchar_minmax_ops - character - - < - <= - = - >= - > - - - - char_minmax_ops - "char" - - < - <= - = - >= - > - - - - date_minmax_ops - date - - < - <= - = - >= - > - - - - float8_minmax_ops - double precision - - < - <= - = - >= - > - - - - inet_minmax_ops - inet - - < - <= - = - >= - > - - - - network_inclusion_ops - inet - - && - >>= - <<= - = - >> - << - - - - int4_minmax_ops - integer - - < - <= - = - >= - > - - - - interval_minmax_ops - interval - - < - <= - = - >= - > - - - - macaddr_minmax_ops - macaddr - - < - <= - = - >= - > - - - - macaddr8_minmax_ops - macaddr8 - - < - <= - = - >= - > - - - - name_minmax_ops - name - - < - <= - = - >= - > - - - - numeric_minmax_ops - numeric - - < - <= - = - >= - > - - - - pg_lsn_minmax_ops - pg_lsn - - < - <= - = - >= - > - - - - oid_minmax_ops - oid - - < - <= - = - >= - > - - - - range_inclusion_ops - any range type - - << - &< - && - &> - >> - @> - <@ - -|- - = - < - <= - = - > - >= - - - - float4_minmax_ops - real - - < - <= - = - >= - > - - - - int2_minmax_ops - smallint - - < - <= - = - >= - > - - - - text_minmax_ops - text - - < - <= - = - >= - > - - - - tid_minmax_ops - tid - - < - <= - = - >= - > - - - - timestamp_minmax_ops - timestamp without time zone - - < - <= - = - >= - > - - - - timestamptz_minmax_ops - timestamp with time zone - - < - <= - = - >= - > - - - - time_minmax_ops - time without time zone - - < - <= - = - >= - > - - - - timetz_minmax_ops - time with time zone - - < - <= - = - >= - > - - - - uuid_minmax_ops - uuid - - < - <= - = - >= - > - + bit_minmax_ops + = (bit,bit) + + < (bit,bit) + > (bit,bit) + <= (bit,bit) + >= (bit,bit) + + + box_inclusion_ops + @> (box,point) + + << (box,box) + &< (box,box) + &> (box,box) + >> (box,box) + <@ (box,box) + @> (box,box) + ~= (box,box) + && (box,box) + <<| (box,box) + &<| (box,box) + |&> (box,box) + |>> (box,box) + + + bpchar_minmax_ops + = (character,character) + + < (character,character) + <= (character,character) + > (character,character) + >= (character,character) + + + bytea_minmax_ops + = (bytea,bytea) + + < (bytea,bytea) + <= (bytea,bytea) + > (bytea,bytea) + >= (bytea,bytea) + + + char_minmax_ops + = ("char","char") + + < ("char","char") + <= ("char","char") + > ("char","char") + >= ("char","char") + + + date_minmax_ops + = (date,date) + + < (date,date) + <= (date,date) + > (date,date) + >= (date,date) + + + float4_minmax_ops + = (float4,float4) + + < (float4,float4) + > (float4,float4) + <= (float4,float4) + >= (float4,float4) + + + float8_minmax_ops + = (float8,float8) + + < (float8,float8) + <= (float8,float8) + > (float8,float8) + >= (float8,float8) + + + inet_inclusion_ops + << (inet,inet) + + <<= (inet,inet) + >> (inet,inet) + >>= (inet,inet) + = (inet,inet) + && (inet,inet) + + + inet_minmax_ops + = (inet,inet) + + < (inet,inet) + <= (inet,inet) + > (inet,inet) + >= (inet,inet) + + + int2_minmax_ops + = (int2,int2) + + < (int2,int2) + > (int2,int2) + <= (int2,int2) + >= (int2,int2) + + + int4_minmax_ops + = (int4,int4) + + < (int4,int4) + > (int4,int4) + <= (int4,int4) + >= (int4,int4) + + + int8_minmax_ops + = (bigint,bigint) + + < (bigint,bigint) + > (bigint,bigint) + <= (bigint,bigint) + >= (bigint,bigint) + + + interval_minmax_ops + = (interval,interval) + + < (interval,interval) + <= (interval,interval) + > (interval,interval) + >= (interval,interval) + + + macaddr_minmax_ops + = (macaddr,macaddr) + + < (macaddr,macaddr) + <= (macaddr,macaddr) + > (macaddr,macaddr) + >= (macaddr,macaddr) + + + macaddr8_minmax_ops + = (macaddr8,macaddr8) + + < (macaddr8,macaddr8) + <= (macaddr8,macaddr8) + > (macaddr8,macaddr8) + >= (macaddr8,macaddr8) + + + name_minmax_ops + = (name,name) + + < (name,name) + <= (name,name) + > (name,name) + >= (name,name) + + + numeric_minmax_ops + = (numeric,numeric) + + < (numeric,numeric) + <= (numeric,numeric) + > (numeric,numeric) + >= (numeric,numeric) + + + oid_minmax_ops + = (oid,oid) + + < (oid,oid) + > (oid,oid) + <= (oid,oid) + >= (oid,oid) + + + pg_lsn_minmax_ops + = (pg_lsn,pg_lsn) + + < (pg_lsn,pg_lsn) + > (pg_lsn,pg_lsn) + <= (pg_lsn,pg_lsn) + >= (pg_lsn,pg_lsn) + + + range_inclusion_ops + = (anyrange,anyrange) + + < (anyrange,anyrange) + <= (anyrange,anyrange) + >= (anyrange,anyrange) + > (anyrange,anyrange) + && (anyrange,anyrange) + @> (anyrange,anyelement) + @> (anyrange,anyrange) + <@ (anyrange,anyrange) + << (anyrange,anyrange) + >> (anyrange,anyrange) + &< (anyrange,anyrange) + &> (anyrange,anyrange) + -|- (anyrange,anyrange) + + + text_minmax_ops + = (text,text) + + < (text,text) + <= (text,text) + > (text,text) + >= (text,text) + + + tid_minmax_ops + = (tid,tid) + + < (tid,tid) + > (tid,tid) + <= (tid,tid) + >= (tid,tid) + + + timestamp_minmax_ops + = (timestamp,timestamp) + + < (timestamp,timestamp) + <= (timestamp,timestamp) + > (timestamp,timestamp) + >= (timestamp,timestamp) + + + timestamptz_minmax_ops + = (timestamptz,timestamptz) + + < (timestamptz,timestamptz) + <= (timestamptz,timestamptz) + > (timestamptz,timestamptz) + >= (timestamptz,timestamptz) + + + time_minmax_ops + = (time,time) + + < (time,time) + <= (time,time) + > (time,time) + >= (time,time) + + + timetz_minmax_ops + = (timetz,timetz) + + < (timetz,timetz) + <= (timetz,timetz) + > (timetz,timetz) + >= (timetz,timetz) + + + uuid_minmax_ops + = (uuid,uuid) + + < (uuid,uuid) + > (uuid,uuid) + <= (uuid,uuid) + >= (uuid,uuid) + + + varbit_minmax_ops + = (varbit,varbit) + < (varbit,varbit) + > (varbit,varbit) + <= (varbit,varbit) + >= (varbit,varbit)
diff --git a/doc/src/sgml/gin.sgml b/doc/src/sgml/gin.sgml index 2d862669c337..5c8d4d52757c 100644 --- a/doc/src/sgml/gin.sgml +++ b/doc/src/sgml/gin.sgml @@ -75,53 +75,62 @@ Built-in <acronym>GIN</acronym> Operator Classes - + Name - Indexed Data Type Indexable Operators - array_ops - anyarray - - && - <@ - = - @> - + array_ops + && (anyarray,anyarray) - jsonb_ops - jsonb - - ? - ?& - ?| - @> - @? - @@ - + @> (anyarray,anyarray) - jsonb_path_ops - jsonb - - @> - @? - @@ - + <@ (anyarray,anyarray) - tsvector_ops - tsvector - - @@ - @@@ - + = (anyarray,anyarray) + + + jsonb_ops + @> (jsonb,jsonb) + + + @? (jsonb,jsonpath) + + + @@ (jsonb,jsonpath) + + + ? (jsonb,text) + + + ?| (jsonb,text[]) + + + ?& (jsonb,text[]) + + + jsonb_path_ops + @> (jsonb,jsonb) + + + @? (jsonb,jsonpath) + + + @@ (jsonb,jsonpath) + + + tsvector_ops + @@ (tsvector,tsquery) + + + @@@ (tsvector,tsquery) diff --git a/doc/src/sgml/gist.sgml b/doc/src/sgml/gist.sgml index a505815f4ec5..f9226e7a35cb 100644 --- a/doc/src/sgml/gist.sgml +++ b/doc/src/sgml/gist.sgml @@ -53,157 +53,126 @@
Built-in <acronym>GiST</acronym> Operator Classes - + Name - Indexed Data Type Indexable Operators Ordering Operators - box_ops - box - - && - &> - &< - &<| - >> - << - <<| - <@ - @> - @ - |&> - |>> - ~ - ~= - - - <-> - + box_ops + << (box,box) + <-> (box,point) + &< (box,box) + && (box,box) + &> (box,box) + >> (box,box) + ~= (box,box) + @> (box,box) + <@ (box,box) + &<| (box,box) + <<| (box,box) + |>> (box,box) + |&> (box,box) + ~ (box,box) + @ (box,box) + - circle_ops - circle - - && - &> - &< - &<| - >> - << - <<| - <@ - @> - @ - |&> - |>> - ~ - ~= - - - <-> - + circle_ops + << (circle,circle) + <-> (circle,point) + &< (circle,circle) + &> (circle,circle) + >> (circle,circle) + <@ (circle,circle) + @> (circle,circle) + ~= (circle,circle) + && (circle,circle) + |>> (circle,circle) + <<| (circle,circle) + &<| (circle,circle) + |&> (circle,circle) + @ (circle,circle) + ~ (circle,circle) + - inet_ops - inet, cidr - - && - >> - >>= - > - >= - <> - << - <<= - < - <= - = - - - + inet_ops + << (inet,inet) + + <<= (inet,inet) + >> (inet,inet) + >>= (inet,inet) + = (inet,inet) + <> (inet,inet) + < (inet,inet) + <= (inet,inet) + > (inet,inet) + >= (inet,inet) + && (inet,inet) + - point_ops - point - - >> - >^ - << - <@ - <@ - <@ - <^ - ~= - - - <-> - + point_ops + >^ (point,point) + <-> (point,point) + << (point,point) + >> (point,point) + <^ (point,point) + ~= (point,point) + <@ (point,box) + <@ (point,polygon) + <@ (point,circle) + - poly_ops - polygon - - && - &> - &< - &<| - >> - << - <<| - <@ - @> - @ - |&> - |>> - ~ - ~= - - - <-> - + poly_ops + << (polygon,polygon) + <-> (polygon,point) + &< (polygon,polygon) + &> (polygon,polygon) + >> (polygon,polygon) + <@ (polygon,polygon) + @> (polygon,polygon) + ~= (polygon,polygon) + && (polygon,polygon) + <<| (polygon,polygon) + &<| (polygon,polygon) + |&> (polygon,polygon) + |>> (polygon,polygon) + @ (polygon,polygon) + ~ (polygon,polygon) + - range_ops - any range type - - && - &> - &< - >> - << - <@ - -|- - = - @> - @> - - - + range_ops + = (anyrange,anyrange) + + && (anyrange,anyrange) + @> (anyrange,anyelement) + @> (anyrange,anyrange) + <@ (anyrange,anyrange) + << (anyrange,anyrange) + >> (anyrange,anyrange) + &< (anyrange,anyrange) + &> (anyrange,anyrange) + -|- (anyrange,anyrange) + - tsquery_ops - tsquery - - <@ - @> - - - + tsquery_ops + <@ (tsquery,tsquery) + + @> (tsquery,tsquery) - tsvector_ops - tsvector - - @@ - - - + tsvector_ops + @@ (tsvector,tsquery) + diff --git a/doc/src/sgml/spgist.sgml b/doc/src/sgml/spgist.sgml index b86302e4efde..68d09951d9fc 100644 --- a/doc/src/sgml/spgist.sgml +++ b/doc/src/sgml/spgist.sgml @@ -64,142 +64,116 @@
Built-in <acronym>SP-GiST</acronym> Operator Classes - + Name - Indexed Data Type Indexable Operators Ordering Operators - kd_point_ops - point - - << - <@ - <^ - >> - >^ - ~= - - - <-> - + box_ops + << (box,box) + <-> (box,point) + &< (box,box) + &> (box,box) + >> (box,box) + <@ (box,box) + @> (box,box) + ~= (box,box) + && (box,box) + <<| (box,box) + &<| (box,box) + |&> (box,box) + |>> (box,box) + - quad_point_ops - point - - << - <@ - <^ - >> - >^ - ~= - - - <-> - + kd_point_ops + >^ (point,point) + <-> (point,point) + << (point,point) + >> (point,point) + <^ (point,point) + ~= (point,point) + <@ (point,box) + - range_ops - any range type - - && - &< - &> - -|- - << - <@ - = - >> - @> - - - + network_ops + << (inet,inet) + + <<= (inet,inet) + >> (inet,inet) + >>= (inet,inet) + = (inet,inet) + <> (inet,inet) + < (inet,inet) + <= (inet,inet) + > (inet,inet) + >= (inet,inet) + && (inet,inet) + - box_ops - box - - << - &< - && - &> - >> - ~= - @> - <@ - &<| - <<| - |>> - |&> - - - <-> - + poly_ops + << (polygon,polygon) + <-> (polygon,point) + &< (polygon,polygon) + &> (polygon,polygon) + >> (polygon,polygon) + <@ (polygon,polygon) + @> (polygon,polygon) + ~= (polygon,polygon) + && (polygon,polygon) + <<| (polygon,polygon) + &<| (polygon,polygon) + |>> (polygon,polygon) + |&> (polygon,polygon) + - poly_ops - polygon - - << - &< - && - &> - >> - ~= - @> - <@ - &<| - <<| - |>> - |&> - - - <-> - + quad_point_ops + >^ (point,point) + <-> (point,point) + << (point,point) + >> (point,point) + <^ (point,point) + ~= (point,point) + <@ (point,box) + - text_ops - text - - < - <= - = - > - >= - ~<=~ - ~<~ - ~>=~ - ~>~ - ^@ - - - + range_ops + = (anyrange,anyrange) + + && (anyrange,anyrange) + @> (anyrange,anyelement) + @> (anyrange,anyrange) + <@ (anyrange,anyrange) + << (anyrange,anyrange) + >> (anyrange,anyrange) + &< (anyrange,anyrange) + &> (anyrange,anyrange) + -|- (anyrange,anyrange) + - inet_ops - inet, cidr - - && - >> - >>= - > - >= - <> - << - <<= - < - <= - = - - - + text_ops + = (text,text) + + < (text,text) + <= (text,text) + > (text,text) + >= (text,text) + ~<~ (text,text) + ~<=~ (text,text) + ~>=~ (text,text) + ~>~ (text,text) + ^@ (text,text)
From 9511fb37ac78c77736e5483118265f7e83cd9f3c Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Sun, 30 Aug 2020 14:14:34 +0900 Subject: [PATCH 026/589] Reset indisreplident for an invalid index in DROP INDEX CONCURRENTLY A failure when dropping concurrently an index used in a replica identity could leave in pg_index an index marked as !indisvalid and indisreplident. Reindexing this index would switch back indisvalid to true, and if the replica identity of the parent relation was switched to use a different index, it would be possible to finish with more than one index marked as indisreplident. If that were to happen, this could mess up with the relation cache as an incorrect index could be used for the replica identity. Indexes marked as invalid are discarded as candidates for the replica identity, as of RelationGetIndexList(), so similarly to what is done with indisclustered, resetting indisreplident when the index is marked as invalid keeps things consistent. REINDEX CONCURRENTLY's swapping already resets the flag for the old index, while the new index inherits the value of the old index to-be-dropped, so only DROP INDEX was an issue. Even if this is a bug, the sequence able to reproduce a problem requires a failure while running DROP INDEX CONCURRENTLY, something unlikely going to happen in the field, so no backpatch is done. Author: Michael Paquier Reviewed-by: Dmitry Dolgov Discussion: https://postgr.es/m/20200827025721.GN2017@paquier.xyz --- src/backend/catalog/index.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 1be27eec52e6..62e487bb4c8a 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -1512,7 +1512,6 @@ index_concurrently_swap(Oid newIndexId, Oid oldIndexId, const char *oldName) /* Preserve indisreplident in the new index */ newIndexForm->indisreplident = oldIndexForm->indisreplident; - oldIndexForm->indisreplident = false; /* Preserve indisclustered in the new index */ newIndexForm->indisclustered = oldIndexForm->indisclustered; @@ -1524,6 +1523,7 @@ index_concurrently_swap(Oid newIndexId, Oid oldIndexId, const char *oldName) newIndexForm->indisvalid = true; oldIndexForm->indisvalid = false; oldIndexForm->indisclustered = false; + oldIndexForm->indisreplident = false; CatalogTupleUpdate(pg_index, &oldIndexTuple->t_self, oldIndexTuple); CatalogTupleUpdate(pg_index, &newIndexTuple->t_self, newIndexTuple); @@ -3349,10 +3349,13 @@ index_set_state_flags(Oid indexId, IndexStateFlagsAction action) * CONCURRENTLY that failed partway through.) * * Note: the CLUSTER logic assumes that indisclustered cannot be - * set on any invalid index, so clear that flag too. + * set on any invalid index, so clear that flag too. Similarly, + * ALTER TABLE assumes that indisreplident cannot be set for + * invalid indexes. */ indexForm->indisvalid = false; indexForm->indisclustered = false; + indexForm->indisreplident = false; break; case INDEX_DROP_SET_DEAD: @@ -3364,6 +3367,8 @@ index_set_state_flags(Oid indexId, IndexStateFlagsAction action) * the index at all. */ Assert(!indexForm->indisvalid); + Assert(!indexForm->indisclustered); + Assert(!indexForm->indisreplident); indexForm->indisready = false; indexForm->indislive = false; break; From 3d351d916b20534f973eda760cde17d96545d4c4 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 30 Aug 2020 12:21:51 -0400 Subject: [PATCH 027/589] Redefine pg_class.reltuples to be -1 before the first VACUUM or ANALYZE. Historically, we've considered the state with relpages and reltuples both zero as indicating that we do not know the table's tuple density. This is problematic because it's impossible to distinguish "never yet vacuumed" from "vacuumed and seen to be empty". In particular, a user cannot use VACUUM or ANALYZE to override the planner's normal heuristic that an empty table should not be believed to be empty because it is probably about to get populated. That heuristic is a good safety measure, so I don't care to abandon it, but there should be a way to override it if the table is indeed intended to stay empty. Hence, represent the initial state of ignorance by setting reltuples to -1 (relpages is still set to zero), and apply the minimum-ten-pages heuristic only when reltuples is still -1. If the table is empty, VACUUM or ANALYZE (but not CREATE INDEX) will override that to reltuples = relpages = 0, and then we'll plan on that basis. This requires a bunch of fiddly little changes, but we can get rid of some ugly kluges that were formerly needed to maintain the old definition. One notable point is that FDWs' GetForeignRelSize methods will see baserel->tuples = -1 when no ANALYZE has been done on the foreign table. That seems like a net improvement, since those methods were formerly also in the dark about what baserel->tuples = 0 really meant. Still, it is an API change. I bumped catversion because code predating this change would get confused by seeing reltuples = -1. Discussion: https://postgr.es/m/F02298E0-6EF4-49A1-BCB6-C484794D9ACC@thebuild.com --- contrib/file_fdw/file_fdw.c | 2 +- contrib/pgstattuple/pgstatapprox.c | 3 +++ contrib/postgres_fdw/postgres_fdw.c | 7 +++-- doc/src/sgml/catalogs.sgml | 4 +++ doc/src/sgml/fdwhandler.sgml | 3 ++- src/backend/access/gin/ginvacuum.c | 2 +- src/backend/access/heap/vacuumlazy.c | 38 +++++++++++---------------- src/backend/access/nbtree/nbtree.c | 1 + src/backend/access/table/tableam.c | 22 +++++++--------- src/backend/catalog/heap.c | 4 +-- src/backend/catalog/index.c | 9 +++++++ src/backend/commands/vacuum.c | 14 +++++----- src/backend/optimizer/path/allpaths.c | 6 ++++- src/backend/optimizer/util/plancat.c | 11 +++----- src/backend/postmaster/autovacuum.c | 4 +++ src/backend/rewrite/rewriteDefine.c | 2 +- src/backend/utils/cache/relcache.c | 4 +-- src/include/access/genam.h | 4 +-- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_class.h | 4 +-- 20 files changed, 77 insertions(+), 69 deletions(-) diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index fbcf7ca9c91e..072a6dc1c163 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -996,7 +996,7 @@ estimate_size(PlannerInfo *root, RelOptInfo *baserel, /* * Estimate the number of tuples in the file. */ - if (baserel->pages > 0) + if (baserel->tuples >= 0 && baserel->pages > 0) { /* * We have # of pages and # of tuples from pg_class (that is, from a diff --git a/contrib/pgstattuple/pgstatapprox.c b/contrib/pgstattuple/pgstatapprox.c index 3a99333d4435..23306e11a78d 100644 --- a/contrib/pgstattuple/pgstatapprox.c +++ b/contrib/pgstattuple/pgstatapprox.c @@ -195,6 +195,9 @@ statapprox_heap(Relation rel, output_type *stat) stat->tuple_count = vac_estimate_reltuples(rel, nblocks, scanned, stat->tuple_count); + /* It's not clear if we could get -1 here, but be safe. */ + stat->tuple_count = Max(stat->tuple_count, 0); + /* * Calculate percentages if the relation has one or more pages. */ diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index 9fc53cad6803..a31abce7c996 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -692,15 +692,14 @@ postgresGetForeignRelSize(PlannerInfo *root, else { /* - * If the foreign table has never been ANALYZEd, it will have relpages - * and reltuples equal to zero, which most likely has nothing to do - * with reality. We can't do a whole lot about that if we're not + * If the foreign table has never been ANALYZEd, it will have + * reltuples < 0, meaning "unknown". We can't do much if we're not * allowed to consult the remote server, but we can use a hack similar * to plancat.c's treatment of empty relations: use a minimum size * estimate of 10 pages, and divide by the column-datatype-based width * estimate to get the corresponding number of tuples. */ - if (baserel->pages == 0 && baserel->tuples == 0) + if (baserel->tuples < 0) { baserel->pages = 10; baserel->tuples = diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 9fe260ecff7f..1d1b8ce8fb12 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -1977,6 +1977,10 @@ SCRAM-SHA-256$<iteration count>:&l the planner. It is updated by VACUUM, ANALYZE, and a few DDL commands such as CREATE INDEX. + If the table has never yet been vacuumed or + analyzed, reltuples + contains -1 indicating that the row count is + unknown. diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml index 74793035d7f5..72fa1272120d 100644 --- a/doc/src/sgml/fdwhandler.sgml +++ b/doc/src/sgml/fdwhandler.sgml @@ -130,7 +130,8 @@ GetForeignRelSize(PlannerInfo *root, (The initial value is from pg_class.reltuples which represents the total row count seen by the - last ANALYZE.) + last ANALYZE; it will be -1 if + no ANALYZE has been done on this foreign table.) diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c index 9cd6638df621..0935a6d9e53d 100644 --- a/src/backend/access/gin/ginvacuum.c +++ b/src/backend/access/gin/ginvacuum.c @@ -727,7 +727,7 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) * entries. This is bogus if the index is partial, but it's real hard to * tell how many distinct heap entries are referenced by a GIN index. */ - stats->num_index_tuples = info->num_heap_tuples; + stats->num_index_tuples = Max(info->num_heap_tuples, 0); stats->estimated_count = info->estimated_count; /* diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index a0da444af0ea..53b1a952543b 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -208,7 +208,8 @@ typedef struct LVShared * live tuples in the index vacuum case or the new live tuples in the * index cleanup case. * - * estimated_count is true if reltuples is an estimated value. + * estimated_count is true if reltuples is an estimated value. (Note that + * reltuples could be -1 in this case, indicating we have no idea.) */ double reltuples; bool estimated_count; @@ -567,31 +568,19 @@ heap_vacuum_rel(Relation onerel, VacuumParams *params, /* * Update statistics in pg_class. * - * A corner case here is that if we scanned no pages at all because every - * page is all-visible, we should not update relpages/reltuples, because - * we have no new information to contribute. In particular this keeps us - * from replacing relpages=reltuples=0 (which means "unknown tuple - * density") with nonzero relpages and reltuples=0 (which means "zero - * tuple density") unless there's some actual evidence for the latter. + * In principle new_live_tuples could be -1 indicating that we (still) + * don't know the tuple count. In practice that probably can't happen, + * since we'd surely have scanned some pages if the table is new and + * nonempty. * - * It's important that we use tupcount_pages and not scanned_pages for the - * check described above; scanned_pages counts pages where we could not - * get cleanup lock, and which were processed only for frozenxid purposes. - * - * We do update relallvisible even in the corner case, since if the table - * is all-visible we'd definitely like to know that. But clamp the value - * to be not more than what we're setting relpages to. + * For safety, clamp relallvisible to be not more than what we're setting + * relpages to. * * Also, don't change relfrozenxid/relminmxid if we skipped any pages, * since then we don't know for certain that all tuples have a newer xmin. */ new_rel_pages = vacrelstats->rel_pages; new_live_tuples = vacrelstats->new_live_tuples; - if (vacrelstats->tupcount_pages == 0 && new_rel_pages > 0) - { - new_rel_pages = vacrelstats->old_rel_pages; - new_live_tuples = vacrelstats->old_live_tuples; - } visibilitymap_count(onerel, &new_rel_allvisible, NULL); if (new_rel_allvisible > new_rel_pages) @@ -612,7 +601,7 @@ heap_vacuum_rel(Relation onerel, VacuumParams *params, /* report results to the stats collector, too */ pgstat_report_vacuum(RelationGetRelid(onerel), onerel->rd_rel->relisshared, - new_live_tuples, + Max(new_live_tuples, 0), vacrelstats->new_dead_tuples); pgstat_progress_end_command(); @@ -1695,9 +1684,12 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, vacrelstats->tupcount_pages, live_tuples); - /* also compute total number of surviving heap entries */ + /* + * Also compute the total number of surviving heap entries. In the + * (unlikely) scenario that new_live_tuples is -1, take it as zero. + */ vacrelstats->new_rel_tuples = - vacrelstats->new_live_tuples + vacrelstats->new_dead_tuples; + Max(vacrelstats->new_live_tuples, 0) + vacrelstats->new_dead_tuples; /* * Release any remaining pin on visibility map page. @@ -2434,7 +2426,7 @@ lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, * dead_tuples, and update running statistics. * * reltuples is the number of heap tuples to be passed to the - * bulkdelete callback. + * bulkdelete callback. It's always assumed to be estimated. */ static void lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c index 8fa6ac7296b9..c822b49a7102 100644 --- a/src/backend/access/nbtree/nbtree.c +++ b/src/backend/access/nbtree/nbtree.c @@ -853,6 +853,7 @@ _bt_vacuum_needs_cleanup(IndexVacuumInfo *info) prev_num_heap_tuples = metad->btm_last_cleanup_num_heap_tuples; if (cleanup_scale_factor <= 0 || + info->num_heap_tuples < 0 || prev_num_heap_tuples <= 0 || (info->num_heap_tuples - prev_num_heap_tuples) / prev_num_heap_tuples >= cleanup_scale_factor) diff --git a/src/backend/access/table/tableam.c b/src/backend/access/table/tableam.c index c63831976575..6438c457161a 100644 --- a/src/backend/access/table/tableam.c +++ b/src/backend/access/table/tableam.c @@ -701,18 +701,14 @@ table_block_relation_estimate_size(Relation rel, int32 *attr_widths, * doesn't happen instantaneously, and it won't happen at all for cases * such as temporary tables.) * - * We approximate "never vacuumed" by "has relpages = 0", which means this - * will also fire on genuinely empty relations. Not great, but - * fortunately that's a seldom-seen case in the real world, and it - * shouldn't degrade the quality of the plan too much anyway to err in - * this direction. + * We test "never vacuumed" by seeing whether reltuples < 0. * * If the table has inheritance children, we don't apply this heuristic. * Totally empty parent tables are quite common, so we should be willing * to believe that they are empty. */ if (curpages < 10 && - relpages == 0 && + reltuples < 0 && !rel->rd_rel->relhassubclass) curpages = 10; @@ -727,17 +723,17 @@ table_block_relation_estimate_size(Relation rel, int32 *attr_widths, } /* estimate number of tuples from previous tuple density */ - if (relpages > 0) + if (reltuples >= 0 && relpages > 0) density = reltuples / (double) relpages; else { /* - * When we have no data because the relation was truncated, estimate - * tuple width from attribute datatypes. We assume here that the - * pages are completely full, which is OK for tables (since they've - * presumably not been VACUUMed yet) but is probably an overestimate - * for indexes. Fortunately get_relation_info() can clamp the - * overestimate to the parent table's size. + * When we have no data because the relation was never yet vacuumed, + * estimate tuple width from attribute datatypes. We assume here that + * the pages are completely full, which is OK for tables but is + * probably an overestimate for indexes. Fortunately + * get_relation_info() can clamp the overestimate to the parent + * table's size. * * Note: this code intentionally disregards alignment considerations, * because (a) that would be gilding the lily considering how crude diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index f2ca686397eb..abd5bdb866b3 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -1015,7 +1015,7 @@ AddNewRelationTuple(Relation pg_class_desc, case RELKIND_TOASTVALUE: /* The relation is real, but as yet empty */ new_rel_reltup->relpages = 0; - new_rel_reltup->reltuples = 0; + new_rel_reltup->reltuples = -1; new_rel_reltup->relallvisible = 0; break; case RELKIND_SEQUENCE: @@ -1027,7 +1027,7 @@ AddNewRelationTuple(Relation pg_class_desc, default: /* Views, etc, have no disk storage */ new_rel_reltup->relpages = 0; - new_rel_reltup->reltuples = 0; + new_rel_reltup->reltuples = -1; new_rel_reltup->relallvisible = 0; break; } diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 62e487bb4c8a..d0ec9a4b9c80 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -2722,6 +2722,15 @@ index_update_stats(Relation rel, /* Should this be a more comprehensive test? */ Assert(rd_rel->relkind != RELKIND_PARTITIONED_INDEX); + /* + * As a special hack, if we are dealing with an empty table and the + * existing reltuples is -1, we leave that alone. This ensures that + * creating an index as part of CREATE TABLE doesn't cause the table to + * prematurely look like it's been vacuumed. + */ + if (reltuples == 0 && rd_rel->reltuples < 0) + reltuples = -1; + /* Apply required updates, if any, to copied tuple */ dirty = false; diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 23eb605d4cb2..308a51d95d7a 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -1128,8 +1128,8 @@ vacuum_set_xid_limits(Relation rel, * live tuples seen; but if we did not, we should not blindly extrapolate * from that number, since VACUUM may have scanned a quite nonrandom * subset of the table. When we have only partial information, we take - * the old value of pg_class.reltuples as a measurement of the - * tuple density in the unscanned pages. + * the old value of pg_class.reltuples/pg_class.relpages as a measurement + * of the tuple density in the unscanned pages. * * Note: scanned_tuples should count only *live* tuples, since * pg_class.reltuples is defined that way. @@ -1152,18 +1152,16 @@ vac_estimate_reltuples(Relation relation, /* * If scanned_pages is zero but total_pages isn't, keep the existing value - * of reltuples. (Note: callers should avoid updating the pg_class - * statistics in this situation, since no new information has been - * provided.) + * of reltuples. (Note: we might be returning -1 in this case.) */ if (scanned_pages == 0) return old_rel_tuples; /* - * If old value of relpages is zero, old density is indeterminate; we - * can't do much except scale up scanned_tuples to match total_pages. + * If old density is unknown, we can't do much except scale up + * scanned_tuples to match total_pages. */ - if (old_rel_pages == 0) + if (old_rel_tuples < 0 || old_rel_pages == 0) return floor((scanned_tuples / scanned_pages) * total_pages + 0.5); /* diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 0eeff804bcf0..b399592ff815 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -912,7 +912,11 @@ set_foreign_size(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) /* ... but do not let it set the rows estimate to zero */ rel->rows = clamp_row_est(rel->rows); - /* also, make sure rel->tuples is not insane relative to rel->rows */ + /* + * Also, make sure rel->tuples is not insane relative to rel->rows. + * Notably, this ensures sanity if pg_class.reltuples contains -1 and the + * FDW doesn't do anything to replace that. + */ rel->tuples = Max(rel->tuples, rel->rows); } diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index 25545029d7ad..f9d0d67aa75a 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -974,11 +974,6 @@ estimate_rel_size(Relation rel, int32 *attr_widths, /* it has storage, ok to call the smgr */ curpages = RelationGetNumberOfBlocks(rel); - /* coerce values in pg_class to more desirable types */ - relpages = (BlockNumber) rel->rd_rel->relpages; - reltuples = (double) rel->rd_rel->reltuples; - relallvisible = (BlockNumber) rel->rd_rel->relallvisible; - /* report estimated # pages */ *pages = curpages; /* quick exit if rel is clearly empty */ @@ -988,6 +983,7 @@ estimate_rel_size(Relation rel, int32 *attr_widths, *allvisfrac = 0; break; } + /* coerce values in pg_class to more desirable types */ relpages = (BlockNumber) rel->rd_rel->relpages; reltuples = (double) rel->rd_rel->reltuples; @@ -1006,12 +1002,12 @@ estimate_rel_size(Relation rel, int32 *attr_widths, } /* estimate number of tuples from previous tuple density */ - if (relpages > 0) + if (reltuples >= 0 && relpages > 0) density = reltuples / (double) relpages; else { /* - * When we have no data because the relation was truncated, + * If we have no data because the relation was never vacuumed, * estimate tuple width from attribute datatypes. We assume * here that the pages are completely full, which is OK for * tables (since they've presumably not been VACUUMed yet) but @@ -1059,6 +1055,7 @@ estimate_rel_size(Relation rel, int32 *attr_widths, break; case RELKIND_FOREIGN_TABLE: /* Just use whatever's in pg_class */ + /* Note that FDW must cope if reltuples is -1! */ *pages = rel->rd_rel->relpages; *tuples = rel->rd_rel->reltuples; *allvisfrac = 0; diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index c6ec657a9367..1b8cd7bacd43 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -3080,6 +3080,10 @@ relation_needs_vacanalyze(Oid relid, instuples = tabentry->inserts_since_vacuum; anltuples = tabentry->changes_since_analyze; + /* If the table hasn't yet been vacuumed, take reltuples as zero */ + if (reltuples < 0) + reltuples = 0; + vacthresh = (float4) vac_base_thresh + vac_scale_factor * reltuples; vacinsthresh = (float4) vac_ins_base_thresh + vac_ins_scale_factor * reltuples; anlthresh = (float4) anl_base_thresh + anl_scale_factor * reltuples; diff --git a/src/backend/rewrite/rewriteDefine.c b/src/backend/rewrite/rewriteDefine.c index 9989df110746..8ef0917021cf 100644 --- a/src/backend/rewrite/rewriteDefine.c +++ b/src/backend/rewrite/rewriteDefine.c @@ -621,7 +621,7 @@ DefineQueryRewrite(const char *rulename, classForm->relam = InvalidOid; classForm->reltablespace = InvalidOid; classForm->relpages = 0; - classForm->reltuples = 0; + classForm->reltuples = -1; classForm->relallvisible = 0; classForm->reltoastrelid = InvalidOid; classForm->relhasindex = false; diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index a2453cf1f421..96ecad02ddb1 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -1870,7 +1870,7 @@ formrdesc(const char *relationName, Oid relationReltype, relation->rd_rel->relreplident = REPLICA_IDENTITY_NOTHING; relation->rd_rel->relpages = 0; - relation->rd_rel->reltuples = 0; + relation->rd_rel->reltuples = -1; relation->rd_rel->relallvisible = 0; relation->rd_rel->relkind = RELKIND_RELATION; relation->rd_rel->relnatts = (int16) natts; @@ -3692,7 +3692,7 @@ RelationSetNewRelfilenode(Relation relation, char persistence) if (relation->rd_rel->relkind != RELKIND_SEQUENCE) { classform->relpages = 0; /* it's empty until further notice */ - classform->reltuples = 0; + classform->reltuples = -1; classform->relallvisible = 0; } classform->relfrozenxid = freezeXid; diff --git a/src/include/access/genam.h b/src/include/access/genam.h index 931257bd8172..68d90f5141d6 100644 --- a/src/include/access/genam.h +++ b/src/include/access/genam.h @@ -38,8 +38,8 @@ typedef struct IndexBuildResult * * num_heap_tuples is accurate only when estimated_count is false; * otherwise it's just an estimate (currently, the estimate is the - * prior value of the relation's pg_class.reltuples field). It will - * always just be an estimate during ambulkdelete. + * prior value of the relation's pg_class.reltuples field, so it could + * even be -1). It will always just be an estimate during ambulkdelete. */ typedef struct IndexVacuumInfo { diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 573f1841b73d..52ca61f8a8e8 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 202008261 +#define CATALOG_VERSION_NO 202008301 #endif diff --git a/src/include/catalog/pg_class.h b/src/include/catalog/pg_class.h index 78b33b2a7f9b..679eec34439b 100644 --- a/src/include/catalog/pg_class.h +++ b/src/include/catalog/pg_class.h @@ -62,8 +62,8 @@ CATALOG(pg_class,1259,RelationRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(83,Relat /* # of blocks (not always up-to-date) */ int32 relpages BKI_DEFAULT(0); - /* # of tuples (not always up-to-date) */ - float4 reltuples BKI_DEFAULT(0); + /* # of tuples (not always up-to-date; -1 means "unknown") */ + float4 reltuples BKI_DEFAULT(-1); /* # of all-visible blocks (not always up-to-date) */ int32 relallvisible BKI_DEFAULT(0); From 6ca547cf75ef6e922476c51a3fb5e253eef5f1b6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 30 Aug 2020 14:37:24 -0400 Subject: [PATCH 028/589] Mark factorial operator, and postfix operators in general, as deprecated. Per discussion, we're planning to remove parser support for postfix operators in order to simplify the grammar. So it behooves us to put out a deprecation notice at least one release before that. There is only one built-in postfix operator, ! for factorial. Label it deprecated in the docs and in pg_description, and adjust some examples that formerly relied on it. (The sister prefix operator !! is also deprecated. We don't really have to remove that one, but since we're suggesting that people use factorial() instead, it seems better to remove both operators.) Also state in the CREATE OPERATOR ref page that postfix operators in general are going away. Although this changes the initial contents of pg_description, I did not force a catversion bump; it doesn't seem essential. In v13, also back-patch 4c5cf5431, so that there's someplace for the s to point to. Mark Dilger and John Naylor, with some adjustments by me Discussion: https://postgr.es/m/BE2DF53D-251A-4E26-972F-930E523580E9@enterprisedb.com --- doc/src/sgml/func.sgml | 6 ++++-- doc/src/sgml/ref/create_operator.sgml | 9 ++++++++- doc/src/sgml/syntax.sgml | 23 ++--------------------- doc/src/sgml/typeconv.sgml | 17 ++++++++--------- src/include/catalog/pg_operator.dat | 4 ++-- src/include/catalog/pg_proc.dat | 1 + 6 files changed, 25 insertions(+), 35 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index bbbffd9d5bbc..b9f591296a5d 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -1055,6 +1055,7 @@ repeat('Pg', 4) PgPgPgPg Factorial + (deprecated, use factorial() instead) 5 ! @@ -1068,7 +1069,8 @@ repeat('Pg', 4) PgPgPgPg numeric - Factorial (as a prefix operator) + Factorial as a prefix operator + (deprecated, use factorial() instead) !! 5 @@ -1349,7 +1351,7 @@ repeat('Pg', 4) PgPgPgPg - + factorial factorial ( bigint ) diff --git a/doc/src/sgml/ref/create_operator.sgml b/doc/src/sgml/ref/create_operator.sgml index d5c385c087f5..66c34e0072f0 100644 --- a/doc/src/sgml/ref/create_operator.sgml +++ b/doc/src/sgml/ref/create_operator.sgml @@ -87,11 +87,18 @@ CREATE OPERATOR name ( At least one of LEFTARG and RIGHTARG must be defined. For - binary operators, both must be defined. For right unary + binary operators, both must be defined. For right unary operators, only LEFTARG should be defined, while for left unary operators only RIGHTARG should be defined. + + + Right unary, also called postfix, operators are deprecated and will be + removed in PostgreSQL version 14. + + + The function_name function must have been previously defined using CREATE diff --git a/doc/src/sgml/syntax.sgml b/doc/src/sgml/syntax.sgml index 2f993ca2e037..0ee303cb87f3 100644 --- a/doc/src/sgml/syntax.sgml +++ b/doc/src/sgml/syntax.sgml @@ -977,27 +977,8 @@ CAST ( 'string' AS type ) Most operators have the same precedence and are left-associative. The precedence and associativity of the operators is hard-wired into the parser. - - - - You will - sometimes need to add parentheses when using combinations of - binary and unary operators. For instance: - -SELECT 5 ! - 6; - - will be parsed as: - -SELECT 5 ! (- 6); - - because the parser has no idea — until it is too late - — that ! is defined as a postfix operator, - not an infix one. To get the desired behavior in this case, you - must write: - -SELECT (5 !) - 6; - - This is the price one pays for extensibility. + Add parentheses if you want an expression with multiple operators + to be parsed in some other way than what the precedence rules imply. diff --git a/doc/src/sgml/typeconv.sgml b/doc/src/sgml/typeconv.sgml index 8900d0eb3832..98662fc91fb6 100644 --- a/doc/src/sgml/typeconv.sgml +++ b/doc/src/sgml/typeconv.sgml @@ -354,20 +354,19 @@ Some examples follow. -Factorial Operator Type Resolution +Square Root Operator Type Resolution -There is only one factorial operator (postfix !) +There is only one square root operator (prefix |/) defined in the standard catalog, and it takes an argument of type -bigint. +double precision. The scanner assigns an initial type of integer to the argument in this query expression: -SELECT 40 ! AS "40 factorial"; - - 40 factorial --------------------------------------------------- - 815915283247897734345611269596115894272000000000 +SELECT |/ 40 AS "square root of 40"; + square root of 40 +------------------- + 6.324555320336759 (1 row) @@ -375,7 +374,7 @@ So the parser does a type conversion on the operand and the query is equivalent to: -SELECT CAST(40 AS bigint) ! AS "40 factorial"; +SELECT |/ CAST(40 AS double precision) AS "square root of 40"; diff --git a/src/include/catalog/pg_operator.dat b/src/include/catalog/pg_operator.dat index 5b0e063655d3..4f8b9865effc 100644 --- a/src/include/catalog/pg_operator.dat +++ b/src/include/catalog/pg_operator.dat @@ -218,10 +218,10 @@ oprname => '>=', oprleft => 'xid8', oprright => 'xid8', oprresult => 'bool', oprcom => '<=(xid8,xid8)', oprnegate => '<(xid8,xid8)', oprcode => 'xid8ge', oprrest => 'scalargesel', oprjoin => 'scalargejoinsel' }, -{ oid => '388', descr => 'factorial', +{ oid => '388', descr => 'deprecated, use factorial() instead', oprname => '!', oprkind => 'r', oprleft => 'int8', oprright => '0', oprresult => 'numeric', oprcode => 'numeric_fac' }, -{ oid => '389', descr => 'deprecated, use ! instead', +{ oid => '389', descr => 'deprecated, use factorial() instead', oprname => '!!', oprkind => 'l', oprleft => '0', oprright => 'int8', oprresult => 'numeric', oprcode => 'numeric_fac' }, { oid => '385', descr => 'equal', diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 27989971db74..1dd325e0e6fd 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -328,6 +328,7 @@ proname => 'unknownout', prorettype => 'cstring', proargtypes => 'unknown', prosrc => 'unknownout' }, { oid => '111', + descr => 'implementation of deprecated ! and !! factorial operators', proname => 'numeric_fac', prorettype => 'numeric', proargtypes => 'int8', prosrc => 'numeric_fac' }, From 3a788db60108fed2e51f62a79a5f425401300338 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Mon, 31 Aug 2020 13:03:54 +0200 Subject: [PATCH 029/589] Fix docs bug stating file_fdw requires absolute paths It has always (since the first commit) worked with relative paths, so use the same wording as other parts of the documentation. Author: Bruce Momjian Discussion: https://postgr.es/m/CABUevExx-hm=cit+A9LeKBH39srvk8Y2tEZeEAj5mP8YfzNKUg@mail.gmail.com --- doc/src/sgml/file-fdw.sgml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/file-fdw.sgml b/doc/src/sgml/file-fdw.sgml index ed028e4ec942..d985ef0a069f 100644 --- a/doc/src/sgml/file-fdw.sgml +++ b/doc/src/sgml/file-fdw.sgml @@ -28,7 +28,8 @@ - Specifies the file to be read. Must be an absolute path name. + Specifies the file to be read. Relative paths are relative to the + data directory. Either filename or program must be specified, but not both. From 243a3b92a67519d4da1712b4fbad194bded3afb5 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 31 Aug 2020 13:20:04 -0400 Subject: [PATCH 030/589] doc: clarify the useful features of procedures This was not clearly documented when procedures were added in PG 11. Reported-by: Robin Abbi Discussion: https://postgr.es/m/CAGmg_NX327KKVuJmbWZD=pGutYFxzZjX1rU+3ji8UuX=8ONn9Q@mail.gmail.com Backpatch-through: 11 --- doc/src/sgml/xfunc.sgml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index 6de464c65457..732d93552127 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -84,8 +84,11 @@ A procedure is a database object similar to a function. The difference is that a procedure does not return a value, so there is no return type declaration. While a function is called as part of a query or DML - command, a procedure is called explicitly using - the statement. + command, a procedure is called in isolation using + the command. If the CALL command is not + part of an explicit transaction, a procedure in many server-side + languages can commit, rollback, and begin new transactions during + its execution, which is not possible in functions. From 47c427d006267d752fee4655543ec99dabe9e61d Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 31 Aug 2020 13:43:05 -0400 Subject: [PATCH 031/589] docs: improve 'capitals' inheritance example Adds constraints and improves wording. Reported-by: 2552891@gmail.com Discussion: https://postgr.es/m/159586122762.680.1361378513036616007@wrigleys.postgresql.org Backpatch-through: 9.5 --- doc/src/sgml/advanced.sgml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/advanced.sgml b/doc/src/sgml/advanced.sgml index d77312600f7b..2d4ab85d450c 100644 --- a/doc/src/sgml/advanced.sgml +++ b/doc/src/sgml/advanced.sgml @@ -616,7 +616,7 @@ CREATE TABLE cities ( ); CREATE TABLE capitals ( - state char(2) + state char(2) UNIQUE NOT NULL ) INHERITS (cities); @@ -630,7 +630,8 @@ CREATE TABLE capitals ( text, a native PostgreSQL type for variable length character strings. The capitals table has - an extra column, state, which shows their states. In + an additional column, state, which shows its + state abbreviation. In PostgreSQL, a table can inherit from zero or more other tables. From 06eba0fd10b1c19d578b90f6ab792834fe9a7418 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 31 Aug 2020 13:49:17 -0400 Subject: [PATCH 032/589] doc: improve description of subscripting of arrays It wasn't clear the non-integers are cast to integers for subscripting, rather than throwing an error. Reported-by: sean@materialize.io Discussion: https://postgr.es/m/159538675800.624.7728794628229799531@wrigleys.postgresql.org Backpatch-through: 9.5 --- doc/src/sgml/syntax.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/syntax.sgml b/doc/src/sgml/syntax.sgml index 0ee303cb87f3..b0ae5d2e127e 100644 --- a/doc/src/sgml/syntax.sgml +++ b/doc/src/sgml/syntax.sgml @@ -1359,7 +1359,7 @@ CREATE FUNCTION dept(text) RETURNS dept (Here, the brackets [ ] are meant to appear literally.) Each subscript is itself an expression, - which must yield an integer value. + which will be rounded to the nearest integer value. From 1e0512ff23e600e9bc19e7f1a1c5ce0597c7bd47 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 31 Aug 2020 13:58:00 -0400 Subject: [PATCH 033/589] C comment: remove mention of use of t_hoff WAL structure member Reported-by: Antonin Houska Discussion: https://postgr.es/m/21643.1595353537@antos Backpatch-through: 9.5 --- src/include/access/heapam_xlog.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h index aa17f7df84d4..15251941128a 100644 --- a/src/include/access/heapam_xlog.h +++ b/src/include/access/heapam_xlog.h @@ -137,8 +137,6 @@ typedef struct xl_heap_truncate * or updated tuple in WAL; we can save a few bytes by reconstructing the * fields that are available elsewhere in the WAL record, or perhaps just * plain needn't be reconstructed. These are the fields we must store. - * NOTE: t_hoff could be recomputed, but we may as well store it because - * it will come for free due to alignment considerations. */ typedef struct xl_heap_header { From 70e791f47e976a810254d52d38fbc33acdc8705f Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 31 Aug 2020 15:23:19 -0400 Subject: [PATCH 034/589] docs: replace "stable storage" with "durable" in descriptions For PG, "durable storage" has a clear meaning, while "stable storage" does not, so use the former. Discussion: https://postgr.es/m/20200817165222.GA31806@momjian.us Backpatch-through: 9.5 --- doc/src/sgml/config.sgml | 2 +- doc/src/sgml/monitoring.sgml | 48 ++++++++++++++++++------------------ 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7a7177c55083..218aa5ae04ec 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -2740,7 +2740,7 @@ include_dir 'conf.d' ensure data preservation even if a standby instance of PostgreSQL were to crash, but not if the standby suffers an operating-system-level crash, since the data has not - necessarily reached stable storage on the standby. + necessarily reached durable storage on the standby. Finally, the setting local causes commits to wait for local flush to disk, but not for replication. This is not usually desirable when synchronous replication is in use, but is provided for diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 17a0df697848..1e5ff0cd57f8 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -1214,12 +1214,12 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser ControlFileSync Waiting for the pg_control file to reach - stable storage. + durable storage. ControlFileSyncUpdate Waiting for an update to the pg_control file - to reach stable storage. + to reach durable storage. ControlFileWrite @@ -1250,12 +1250,12 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser DataFileFlush - Waiting for a relation data file to reach stable storage. + Waiting for a relation data file to reach durable storage. DataFileImmediateSync Waiting for an immediate synchronization of a relation data file to - stable storage. + durable storage. DataFilePrefetch @@ -1268,7 +1268,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser DataFileSync - Waiting for changes to a relation data file to reach stable storage. + Waiting for changes to a relation data file to reach durable storage. DataFileTruncate @@ -1285,7 +1285,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser LockFileAddToDataDirSync - Waiting for data to reach stable storage while adding a line to the + Waiting for data to reach durable storage while adding a line to the data directory lock file. @@ -1300,7 +1300,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser LockFileCreateSync - Waiting for data to reach stable storage while creating the data + Waiting for data to reach durable storage while creating the data directory lock file. @@ -1315,12 +1315,12 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser LogicalRewriteCheckpointSync - Waiting for logical rewrite mappings to reach stable storage + Waiting for logical rewrite mappings to reach durable storage during a checkpoint. LogicalRewriteMappingSync - Waiting for mapping data to reach stable storage during a logical + Waiting for mapping data to reach durable storage during a logical rewrite. @@ -1330,7 +1330,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser LogicalRewriteSync - Waiting for logical rewrite mappings to reach stable + Waiting for logical rewrite mappings to reach durable storage. @@ -1348,7 +1348,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser RelationMapSync - Waiting for the relation map file to reach stable storage. + Waiting for the relation map file to reach durable storage. RelationMapWrite @@ -1373,12 +1373,12 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser ReplicationSlotRestoreSync - Waiting for a replication slot control file to reach stable storage + Waiting for a replication slot control file to reach durable storage while restoring it to memory. ReplicationSlotSync - Waiting for a replication slot control file to reach stable + Waiting for a replication slot control file to reach durable storage. @@ -1387,7 +1387,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser SLRUFlushSync - Waiting for SLRU data to reach stable storage during a checkpoint + Waiting for SLRU data to reach durable storage during a checkpoint or database shutdown. @@ -1396,7 +1396,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser SLRUSync - Waiting for SLRU data to reach stable storage following a page + Waiting for SLRU data to reach durable storage following a page write. @@ -1411,7 +1411,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser SnapbuildSync Waiting for a serialized historical catalog snapshot to reach - stable storage. + durable storage. SnapbuildWrite @@ -1421,7 +1421,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser TimelineHistoryFileSync Waiting for a timeline history file received via streaming - replication to reach stable storage. + replication to reach durable storage. TimelineHistoryFileWrite @@ -1434,7 +1434,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser TimelineHistorySync - Waiting for a newly created timeline history file to reach stable + Waiting for a newly created timeline history file to reach durable storage. @@ -1448,7 +1448,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser TwophaseFileSync - Waiting for a two phase state file to reach stable storage. + Waiting for a two phase state file to reach durable storage. TwophaseFileWrite @@ -1456,7 +1456,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser WALBootstrapSync - Waiting for WAL to reach stable storage during + Waiting for WAL to reach durable storage during bootstrapping. @@ -1471,7 +1471,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser WALCopySync Waiting for a new WAL segment created by copying an existing one to - reach stable storage. + reach durable storage. WALCopyWrite @@ -1480,7 +1480,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser WALInitSync - Waiting for a newly initialized WAL file to reach stable + Waiting for a newly initialized WAL file to reach durable storage. @@ -1498,11 +1498,11 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser WALSync - Waiting for a WAL file to reach stable storage. + Waiting for a WAL file to reach durable storage. WALSyncMethodAssign - Waiting for data to reach stable storage while assigning a new + Waiting for data to reach durable storage while assigning a new WAL sync method. From b1ae70b3b4fd54220a0901eaf3dd4c5ca0827108 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 31 Aug 2020 16:21:03 -0400 Subject: [PATCH 035/589] docs: clarify intermediate certificate creation instructions Specifically, explain the v3_ca openssl specification. Discussion: https://postgr.es/m/20200824175653.GA32411@momjian.us Backpatch-through: 9.5 --- doc/src/sgml/runtime.sgml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index c8698898f325..a01add94b7fd 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -2193,8 +2193,10 @@ pg_dumpall -p 5432 | psql -d postgres -p 5433 The certificates of intermediate certificate authorities can also be appended to the file. Doing this avoids the necessity of storing intermediate certificates on clients, assuming the root and - intermediate certificates were created with v3_ca - extensions. This allows easier expiration of intermediate certificates. + intermediate certificates were created with v3_ca + extensions. (This sets the certificate's basic constraint of + CA to true.) + This allows easier expiration of intermediate certificates. From de2d1920ddf54ec8dbcc87821ade5346f0cf8926 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 31 Aug 2020 16:59:59 -0400 Subject: [PATCH 036/589] doc: cross-link file-fdw and CSV config log sections There is an file-fdw example that reads the server config file, so cross link them. Reported-by: Oleg Samoilov Discussion: https://postgr.es/m/159800192078.2886.10431506404995508950@wrigleys.postgresql.org Backpatch-through: 9.5 --- doc/src/sgml/config.sgml | 2 ++ doc/src/sgml/file-fdw.sgml | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 218aa5ae04ec..a068f33b6107 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -7075,6 +7075,8 @@ CREATE TABLE postgres_log COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; + It is also possible to access the file as a foreign table, using + the supplied module. diff --git a/doc/src/sgml/file-fdw.sgml b/doc/src/sgml/file-fdw.sgml index d985ef0a069f..599952ed3d7d 100644 --- a/doc/src/sgml/file-fdw.sgml +++ b/doc/src/sgml/file-fdw.sgml @@ -216,7 +216,9 @@ One of the obvious uses for file_fdw is to make the PostgreSQL activity log available as a table for querying. To - do this, first you must be logging to a CSV file, which here we + do this, first you must be logging to a CSV file, + which here we will call pglog.csv. First, install file_fdw as an extension: From 9524fa1aa557b19c9f049b3d79dc93e53cb9d57b Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 31 Aug 2020 17:05:53 -0400 Subject: [PATCH 037/589] docs: in mapping SQL to C data types, timestamp isn't a pointer It is an int64. Reported-by: ajulien@shaktiware.fr Discussion: https://postgr.es/m/159845038271.24995.15682121015698255155@wrigleys.postgresql.org Backpatch-through: 9.5 --- doc/src/sgml/xfunc.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index 732d93552127..e0692e993f59 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -2226,7 +2226,7 @@ memcpy(destination->data, buffer, 40); timestamp - Timestamp* + Timestamp datatype/timestamp.h From 0ebe82a9413fcd5621266ec57d9f4bc50f0457d8 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Mon, 31 Aug 2020 17:04:40 -0400 Subject: [PATCH 038/589] doc: Update partitioning limitation on BEFORE triggers Reported-by: Erwin Brandstetter Discussion: https://postgr.es/m/CAGHENJ6Le7S3qJJx2TvWvTwRNS3N=BtoNeb7AF2rZvfNBMeQcg@mail.gmail.com --- doc/src/sgml/ddl.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index f45c951b2b08..d3bf38fa60b4 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -4065,8 +4065,8 @@ ALTER INDEX measurement_city_id_logdate_key - BEFORE ROW triggers, if necessary, must be defined - on individual partitions, not the partitioned table. + BEFORE ROW triggers cannot change which partition + is the final destination for a new row. From 50ed605b3e311f26497cea979887a8deed3eabf6 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 31 Aug 2020 17:36:23 -0400 Subject: [PATCH 039/589] pg_upgrade doc: mention saving postgresql.conf.auto files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also mention files included by postgresql.conf. Reported-by: Álvaro Herrera Discussion: https://postgr.es/m/08AD4526-75AB-457B-B2DD-099663F28040@yesql.se Backpatch-through: 9.5 --- doc/src/sgml/ref/pgupgrade.sgml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml index 6779a5bddcf3..74f27130ec5c 100644 --- a/doc/src/sgml/ref/pgupgrade.sgml +++ b/doc/src/sgml/ref/pgupgrade.sgml @@ -516,9 +516,10 @@ pg_upgrade.exe Save any configuration files from the old standbys' configuration - directories you need to keep, e.g. postgresql.conf, - pg_hba.conf, because these will be overwritten or - removed in the next step. + directories you need to keep, e.g. postgresql.conf + (and any files included by it), postgresql.auto.conf, + pg_hba.conf, because these will be overwritten + or removed in the next step. @@ -606,7 +607,8 @@ rsync --archive --delete --hard-links --size-only --no-inc-recursive /vol1/pg_tb If you modified pg_hba.conf, restore its original settings. It might also be necessary to adjust other configuration files in the new - cluster to match the old cluster, e.g. postgresql.conf. + cluster to match the old cluster, e.g., postgresql.conf + (and any files included by it), postgresql.auto.conf. From 953c64e0f67d44b1db97a3d33b329ccd4691f4c6 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 31 Aug 2020 18:33:37 -0400 Subject: [PATCH 040/589] doc: add commas after 'i.e.' and 'e.g.' This follows the American format, https://jakubmarian.com/comma-after-i-e-and-e-g/. There is no intention of requiring this format for future text, but making existing text consistent every few years makes sense. Discussion: https://postgr.es/m/20200825183619.GA22369@momjian.us Backpatch-through: 9.5 --- doc/src/sgml/bki.sgml | 2 +- doc/src/sgml/config.sgml | 16 +++++++-------- doc/src/sgml/datatype.sgml | 4 ++-- doc/src/sgml/ddl.sgml | 4 ++-- doc/src/sgml/ecpg.sgml | 6 +++--- doc/src/sgml/extend.sgml | 4 ++-- doc/src/sgml/func.sgml | 8 ++++---- doc/src/sgml/glossary.sgml | 2 +- doc/src/sgml/high-availability.sgml | 8 ++++---- doc/src/sgml/indexam.sgml | 2 +- doc/src/sgml/install-windows.sgml | 4 ++-- doc/src/sgml/libpq.sgml | 14 ++++++------- doc/src/sgml/logicaldecoding.sgml | 4 ++-- doc/src/sgml/monitoring.sgml | 2 +- doc/src/sgml/mvcc.sgml | 2 +- doc/src/sgml/parallel.sgml | 4 ++-- doc/src/sgml/perform.sgml | 2 +- doc/src/sgml/postgres-fdw.sgml | 2 +- doc/src/sgml/protocol.sgml | 4 ++-- doc/src/sgml/queries.sgml | 4 ++-- doc/src/sgml/ref/create_database.sgml | 4 ++-- doc/src/sgml/ref/create_event_trigger.sgml | 2 +- doc/src/sgml/ref/create_function.sgml | 6 +++--- doc/src/sgml/ref/create_procedure.sgml | 2 +- doc/src/sgml/ref/create_statistics.sgml | 2 +- doc/src/sgml/ref/create_table.sgml | 4 ++-- doc/src/sgml/ref/initdb.sgml | 2 +- doc/src/sgml/ref/pg_dump.sgml | 2 +- doc/src/sgml/ref/pg_dumpall.sgml | 2 +- doc/src/sgml/ref/pg_restore.sgml | 2 +- doc/src/sgml/ref/pg_rewind.sgml | 2 +- doc/src/sgml/ref/pgbench.sgml | 10 ++++----- doc/src/sgml/ref/pgupgrade.sgml | 24 +++++++++++----------- doc/src/sgml/ref/postgres-ref.sgml | 2 +- doc/src/sgml/ref/prepare.sgml | 2 +- doc/src/sgml/ref/psql-ref.sgml | 6 +++--- doc/src/sgml/replication-origins.sgml | 6 +++--- doc/src/sgml/runtime.sgml | 2 +- doc/src/sgml/sepgsql.sgml | 2 +- doc/src/sgml/sources.sgml | 4 ++-- doc/src/sgml/sslinfo.sgml | 4 ++-- doc/src/sgml/tableam.sgml | 2 +- doc/src/sgml/textsearch.sgml | 4 ++-- doc/src/sgml/wal.sgml | 2 +- doc/src/sgml/xfunc.sgml | 6 +++--- doc/src/sgml/xml2.sgml | 2 +- 46 files changed, 103 insertions(+), 103 deletions(-) diff --git a/doc/src/sgml/bki.sgml b/doc/src/sgml/bki.sgml index 272a10a98c5a..4e696d1d3ed0 100644 --- a/doc/src/sgml/bki.sgml +++ b/doc/src/sgml/bki.sgml @@ -459,7 +459,7 @@ Use of symbolic references is enabled in a particular catalog column by attaching BKI_LOOKUP(lookuprule) to the column's definition, where lookuprule - is the name of the referenced catalog, e.g. pg_proc. + is the name of the referenced catalog, e.g., pg_proc. BKI_LOOKUP can be attached to columns of type Oid, regproc, oidvector, or Oid[]; in the latter two cases it implies performing a diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index a068f33b6107..606e80df0ed0 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -424,7 +424,7 @@ include_dir 'directory' start with the . character are also ignored, to prevent mistakes since such files are hidden on some platforms. Multiple files within an include directory are processed in file name order - (according to C locale rules, i.e. numbers before letters, and + (according to C locale rules, i.e., numbers before letters, and uppercase letters before lowercase ones). @@ -1085,7 +1085,7 @@ include_dir 'conf.d' With this parameter enabled, you can still create ordinary global users. Simply append @ when specifying the user - name in the client, e.g. joe@. The @ + name in the client, e.g., joe@. The @ will be stripped off before the user name is looked up by the server. @@ -3319,7 +3319,7 @@ include_dir 'conf.d' disabled, but the server continues to accumulate WAL segment files in the expectation that a command will soon be provided. Setting archive_command to a command that does nothing but - return true, e.g. /bin/true (REM on + return true, e.g., /bin/true (REM on Windows), effectively disables archiving, but also breaks the chain of WAL files needed for archive recovery, so it should only be used in unusual circumstances. @@ -3564,7 +3564,7 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows This parameter specifies that recovery should end as soon as a - consistent state is reached, i.e. as early as possible. When restoring + consistent state is reached, i.e., as early as possible. When restoring from an online backup, this means the point where taking the backup ended. @@ -3610,7 +3610,7 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows variable has been set earlier in the configuration file). Preferred style is to use a numeric offset from UTC, or you can write a full time zone name, - e.g. Europe/Helsinki not EEST. + e.g., Europe/Helsinki not EEST. @@ -4992,7 +4992,7 @@ ANY num_sync ( for more information. @@ -1645,7 +1645,7 @@ include $(PGXS) NO_INSTALLCHECK - don't define an installcheck target, useful e.g. if tests require special configuration, or don't use pg_regress + don't define an installcheck target, useful e.g., if tests require special configuration, or don't use pg_regress diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index b9f591296a5d..2efd80baa450 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -7642,9 +7642,9 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); In to_timestamp and to_date, - if the year format specification is less than four digits, e.g. + if the year format specification is less than four digits, e.g., YYY, and the supplied year is less than four digits, - the year will be adjusted to be nearest to the year 2020, e.g. + the year will be adjusted to be nearest to the year 2020, e.g., 95 becomes 1995. @@ -26386,7 +26386,7 @@ FOR EACH ROW EXECUTE FUNCTION suppress_redundant_updates_trigger(); objsubid integer - Sub-object ID (e.g. attribute number for a column) + Sub-object ID (e.g., attribute number for a column) command_tag @@ -26476,7 +26476,7 @@ FOR EACH ROW EXECUTE FUNCTION suppress_redundant_updates_trigger(); objsubid integer - Sub-object ID (e.g. attribute number for a column) + Sub-object ID (e.g., attribute number for a column) original diff --git a/doc/src/sgml/glossary.sgml b/doc/src/sgml/glossary.sgml index 9d2385031ca4..41f3e5ee86ba 100644 --- a/doc/src/sgml/glossary.sgml +++ b/doc/src/sgml/glossary.sgml @@ -708,7 +708,7 @@ Contains the values of row - attributes (i.e. the data) for a + attributes (i.e., the data) for a relation. The heap is realized within one or more file segments diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index d6f79fc435ea..ab9e6d87ce7a 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -219,7 +219,7 @@ protocol to make nodes agree on a serializable transactional order. this is unacceptable, either the middleware or the application must query such values from a single server and then use those values in write queries. Another option is to use this replication - option with a traditional primary-standby setup, i.e. data modification + option with a traditional primary-standby setup, i.e., data modification queries are sent only to the primary and are propagated to the standby servers via primary-standby replication, not by the replication middleware. Care must also be taken that all @@ -662,7 +662,7 @@ protocol to make nodes agree on a serializable transactional order. Set up continuous archiving on the primary to an archive directory accessible from the standby, as described in . The archive location should be - accessible from the standby even when the primary is down, i.e. it should + accessible from the standby even when the primary is down, i.e., it should reside on the standby server itself or another trusted server, not on the primary server. @@ -2232,7 +2232,7 @@ LOG: database system is ready to accept read only connections - Data Definition Language (DDL): e.g. CREATE INDEX + Data Definition Language (DDL): e.g., CREATE INDEX @@ -2298,7 +2298,7 @@ LOG: database system is ready to accept read only connections WAL file control commands will not work during recovery, - e.g. pg_start_backup, pg_switch_wal etc. + e.g., pg_start_backup, pg_switch_wal etc. diff --git a/doc/src/sgml/indexam.sgml b/doc/src/sgml/indexam.sgml index 1aea4db707d5..390c49eb6aba 100644 --- a/doc/src/sgml/indexam.sgml +++ b/doc/src/sgml/indexam.sgml @@ -386,7 +386,7 @@ amcanreturn (Relation indexRelation, int attno); linkend="indexes-index-only-scans">index-only scans on the given column, by returning the indexed column values for an index entry in the form of an IndexTuple. The attribute number - is 1-based, i.e. the first column's attno is 1. Returns true if supported, + is 1-based, i.e., the first column's attno is 1. Returns true if supported, else false. If the access method does not support index-only scans at all, the amcanreturn field in its IndexAmRoutine struct can be set to NULL. diff --git a/doc/src/sgml/install-windows.sgml b/doc/src/sgml/install-windows.sgml index 0d885d0a25a3..11f5957aca5b 100644 --- a/doc/src/sgml/install-windows.sgml +++ b/doc/src/sgml/install-windows.sgml @@ -117,7 +117,7 @@ same purpose. From the Visual Studio Command Prompt, you can change the targeted CPU architecture, build type, and target OS by using the - vcvarsall.bat command, e.g. + vcvarsall.bat command, e.g., vcvarsall.bat x64 10.0.10240.0 to target Windows 10 with a 64-bit release build. See -help for the other options of vcvarsall.bat. All commands should be run from @@ -250,7 +250,7 @@ $ENV{MSBFLAGS}="/m"; installations C:\Program Files\GnuWin32. Consider installing into C:\GnuWin32 or use the NTFS short name path to GnuWin32 in your PATH environment setting - (e.g. C:\PROGRA~1\GnuWin32). + (e.g., C:\PROGRA~1\GnuWin32). diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index 92556c7ce0cc..a397073526f4 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -355,7 +355,7 @@ PostgresPollingStatusType PQconnectPoll(PGconn *conn); Conversely, if PQconnectPoll(conn) last returned PGRES_POLLING_WRITING, wait until the socket is ready to write, then call PQconnectPoll(conn) again. - On the first iteration, i.e. if you have yet to call + On the first iteration, i.e., if you have yet to call PQconnectPoll, behave as if it last returned PGRES_POLLING_WRITING. Continue this loop until PQconnectPoll(conn) returns @@ -994,7 +994,7 @@ postgresql://%2Fvar%2Flib%2Fpostgresql/dbname hostaddr, and port options accept a comma-separated list of values. The same number of elements must be given in each option that is specified, such - that e.g. the first hostaddr corresponds to the first host name, + that e.g., the first hostaddr corresponds to the first host name, the second hostaddr corresponds to the second host name, and so forth. As an exception, if only one port is specified, it applies to all the hosts. @@ -1022,7 +1022,7 @@ postgresql://%2Fvar%2Flib%2Fpostgresql/dbname If a password file is used, you can have different passwords for different hosts. All the other connection options are the same for every - host in the list; it is not possible to e.g. specify different + host in the list; it is not possible to e.g., specify different usernames for different hosts. @@ -1227,7 +1227,7 @@ postgresql://%2Fvar%2Flib%2Fpostgresql/dbname Maximum wait for connection, in seconds (write as a decimal integer, - e.g. 10). Zero, negative, or not specified means + e.g., 10). Zero, negative, or not specified means wait indefinitely. The minimum allowed timeout is 2 seconds, therefore a value of 1 is interpreted as 2. This timeout applies separately to each host name or IP address. @@ -2426,7 +2426,7 @@ const char *PQsslAttribute(const PGconn *conn, const char *attribute_name); cipher - A short name of the ciphersuite used, e.g. + A short name of the ciphersuite used, e.g., "DHE-RSA-DES-CBC3-SHA". The names are specific to each SSL implementation. @@ -4821,7 +4821,7 @@ int PQflush(PGconn *conn); again. Repeat until returns 0. (It is necessary to check for read-ready and drain the input with , - because the server can block trying to send us data, e.g. NOTICE + because the server can block trying to send us data, e.g., NOTICE messages, and won't read our data until we read its.) Once returns 0, wait for the socket to be read-ready and then read the response as described above. @@ -7737,7 +7737,7 @@ ldap://ldap.acme.com/cn=dbserver,cn=hosts?pgconnectinfo?base?(objectclass=*) For a connection to be known SSL-secured, SSL usage must be configured on both the client and the server before the connection is made. If it is only configured on the server, the client may end up - sending sensitive information (e.g. passwords) before + sending sensitive information (e.g., passwords) before it knows that the server requires high security. In libpq, secure connections can be ensured by setting the sslmode parameter to verify-full or diff --git a/doc/src/sgml/logicaldecoding.sgml b/doc/src/sgml/logicaldecoding.sgml index 1571d71a5b6c..2765ed550eae 100644 --- a/doc/src/sgml/logicaldecoding.sgml +++ b/doc/src/sgml/logicaldecoding.sgml @@ -882,7 +882,7 @@ OutputPluginWrite(ctx, true); Streaming of Large Transactions for Logical Decoding - The basic output plugin callbacks (e.g. begin_cb, + The basic output plugin callbacks (e.g., begin_cb, change_cb, commit_cb and message_cb) are only invoked when the transaction actually commits. The changes are still decoded from the transaction @@ -958,7 +958,7 @@ stream_commit_cb(...); <-- commit of the streamed transaction currently used for decoded changes) is selected and streamed. However, in some cases we still have to spill to the disk even if streaming is enabled because if we cross the memory limit but we still have not decoded the - complete tuple e.g. only decoded toast table insert but not the main table + complete tuple e.g., only decoded toast table insert but not the main table insert. diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 1e5ff0cd57f8..d973e1149aa9 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -206,7 +206,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser When the server shuts down cleanly, a permanent copy of the statistics data is stored in the pg_stat subdirectory, so that statistics can be retained across server restarts. When recovery is - performed at server start (e.g. after immediate shutdown, server crash, + performed at server start (e.g., after immediate shutdown, server crash, and point-in-time recovery), all statistics counters are reset. diff --git a/doc/src/sgml/mvcc.sgml b/doc/src/sgml/mvcc.sgml index d127c0b9ad8e..6920913a260f 100644 --- a/doc/src/sgml/mvcc.sgml +++ b/doc/src/sgml/mvcc.sgml @@ -269,7 +269,7 @@ In PostgreSQL, you can request any of the four standard transaction isolation levels, but internally only - three distinct isolation levels are implemented, i.e. PostgreSQL's + three distinct isolation levels are implemented, i.e., PostgreSQL's Read Uncommitted mode behaves like Read Committed. This is because it is the only sensible way to map the standard isolation levels to PostgreSQL's multiversion concurrency control architecture. diff --git a/doc/src/sgml/parallel.sgml b/doc/src/sgml/parallel.sgml index 712450c22ae8..e31bd9d3cebd 100644 --- a/doc/src/sgml/parallel.sgml +++ b/doc/src/sgml/parallel.sgml @@ -433,7 +433,7 @@ EXPLAIN SELECT * FROM pgbench_accounts WHERE filler LIKE '%x%'; . Of course, this plan may turn out to be slower than the serial plan which the planner preferred, but this will not always be the case. If you don't get a parallel - plan even with very small values of these settings (e.g. after setting + plan even with very small values of these settings (e.g., after setting them both to zero), there may be some reason why the query planner is unable to generate a parallel plan for your query. See and @@ -531,7 +531,7 @@ EXPLAIN SELECT * FROM pgbench_accounts WHERE filler LIKE '%x%'; Functions and aggregates must be marked PARALLEL UNSAFE if they write to the database, access sequences, change the transaction state - even temporarily (e.g. a PL/pgSQL function which establishes an + even temporarily (e.g., a PL/pgSQL function which establishes an EXCEPTION block to catch errors), or make persistent changes to settings. Similarly, functions must be marked PARALLEL RESTRICTED if they access temporary tables, client connection state, diff --git a/doc/src/sgml/perform.sgml b/doc/src/sgml/perform.sgml index bfe6c5150247..1cd9f5092db1 100644 --- a/doc/src/sgml/perform.sgml +++ b/doc/src/sgml/perform.sgml @@ -1903,7 +1903,7 @@ SELECT * FROM x, y, a, b, c WHERE something AND somethingelse; Place the database cluster's data directory in a memory-backed - file system (i.e. RAM disk). This eliminates all + file system (i.e., RAM disk). This eliminates all database disk I/O, but limits data storage to the amount of available memory (and perhaps swap). diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml index eab2cc937845..4efaf35d3c4e 100644 --- a/doc/src/sgml/postgres-fdw.sgml +++ b/doc/src/sgml/postgres-fdw.sgml @@ -150,7 +150,7 @@ A superuser may override this check on a per-user-mapping basis by setting - the user mapping option password_required 'false', e.g. + the user mapping option password_required 'false', e.g., ALTER USER MAPPING FOR some_non_superuser SERVER loopback_nopw OPTIONS (ADD password_required 'false'); diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 0c7087397d73..7da2feb2ae51 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -28,7 +28,7 @@ the server, the connection will be rejected (for example, this would occur if the client requested protocol version 4.0, which does not exist as of this writing). If the minor version requested by the client is not - supported by the server (e.g. the client requests version 3.1, but the + supported by the server (e.g., the client requests version 3.1, but the server supports only 3.0), the server may either reject the connection or may respond with a NegotiateProtocolVersion message containing the highest minor protocol version which it supports. The client may then choose either @@ -422,7 +422,7 @@ by the client, but does support an earlier version of the protocol; this message indicates the highest supported minor version. This message will also be sent if the client requested unsupported protocol - options (i.e. beginning with _pq_.) in the + options (i.e., beginning with _pq_.) in the startup packet. This message will be followed by an ErrorResponse or a message indicating the success or failure of authentication. diff --git a/doc/src/sgml/queries.sgml b/doc/src/sgml/queries.sgml index 572e9682733a..67ca71e56490 100644 --- a/doc/src/sgml/queries.sgml +++ b/doc/src/sgml/queries.sgml @@ -1256,7 +1256,7 @@ GROUPING SETS ( ( ) ) - This is commonly used for analysis over hierarchical data; e.g. total + This is commonly used for analysis over hierarchical data; e.g., total salary by department, division, and company-wide total. @@ -1265,7 +1265,7 @@ GROUPING SETS ( CUBE ( e1, e2, ... ) - represents the given list and all of its possible subsets (i.e. the power + represents the given list and all of its possible subsets (i.e., the power set). Thus CUBE ( a, b, c ) diff --git a/doc/src/sgml/ref/create_database.sgml b/doc/src/sgml/ref/create_database.sgml index d116b321bce9..420576c5e83e 100644 --- a/doc/src/sgml/ref/create_database.sgml +++ b/doc/src/sgml/ref/create_database.sgml @@ -139,7 +139,7 @@ CREATE DATABASE name Collation order (LC_COLLATE) to use in the new database. - This affects the sort order applied to strings, e.g. in queries with + This affects the sort order applied to strings, e.g., in queries with ORDER BY, as well as the order used in indexes on text columns. The default is to use the collation order of the template database. See below for additional restrictions. @@ -151,7 +151,7 @@ CREATE DATABASE name Character classification (LC_CTYPE) to use in the new - database. This affects the categorization of characters, e.g. lower, + database. This affects the categorization of characters, e.g., lower, upper and digit. The default is to use the character classification of the template database. See below for additional restrictions. diff --git a/doc/src/sgml/ref/create_event_trigger.sgml b/doc/src/sgml/ref/create_event_trigger.sgml index 52ba746166be..becd31bcadf7 100644 --- a/doc/src/sgml/ref/create_event_trigger.sgml +++ b/doc/src/sgml/ref/create_event_trigger.sgml @@ -86,7 +86,7 @@ CREATE EVENT TRIGGER name A list of values for the associated filter_variable for which the trigger should fire. For TAG, this means a - list of command tags (e.g. 'DROP FUNCTION'). + list of command tags (e.g., 'DROP FUNCTION'). diff --git a/doc/src/sgml/ref/create_function.sgml b/doc/src/sgml/ref/create_function.sgml index f81cedc8233e..97285b757841 100644 --- a/doc/src/sgml/ref/create_function.sgml +++ b/doc/src/sgml/ref/create_function.sgml @@ -257,7 +257,7 @@ CREATE [ OR REPLACE ] FUNCTION The name of the language that the function is implemented in. It can be sql, c, internal, or the name of a user-defined - procedural language, e.g. plpgsql. Enclosing the + procedural language, e.g., plpgsql. Enclosing the name in single quotes is deprecated and requires matching case. @@ -431,11 +431,11 @@ CREATE [ OR REPLACE ] FUNCTION Functions should be labeled parallel unsafe if they modify any database state, or if they make changes to the transaction such as using sub-transactions, or if they access sequences or attempt to make - persistent changes to settings (e.g. setval). They should + persistent changes to settings (e.g., setval). They should be labeled as parallel restricted if they access temporary tables, client connection state, cursors, prepared statements, or miscellaneous backend-local state which the system cannot synchronize in parallel mode - (e.g. setseed cannot be executed other than by the group + (e.g., setseed cannot be executed other than by the group leader because a change made by another process would not be reflected in the leader). In general, if a function is labeled as being safe when it is restricted or unsafe, or if it is labeled as being restricted when diff --git a/doc/src/sgml/ref/create_procedure.sgml b/doc/src/sgml/ref/create_procedure.sgml index 0ea6513cb588..d225695626cb 100644 --- a/doc/src/sgml/ref/create_procedure.sgml +++ b/doc/src/sgml/ref/create_procedure.sgml @@ -164,7 +164,7 @@ CREATE [ OR REPLACE ] PROCEDURE The name of the language that the procedure is implemented in. It can be sql, c, internal, or the name of a user-defined - procedural language, e.g. plpgsql. Enclosing the + procedural language, e.g., plpgsql. Enclosing the name in single quotes is deprecated and requires matching case. diff --git a/doc/src/sgml/ref/create_statistics.sgml b/doc/src/sgml/ref/create_statistics.sgml index 5b583aacb433..4363be50c3c4 100644 --- a/doc/src/sgml/ref/create_statistics.sgml +++ b/doc/src/sgml/ref/create_statistics.sgml @@ -131,7 +131,7 @@ CREATE STATISTICS [ IF NOT EXISTS ] statistics_na Examples - Create table t1 with two functionally dependent columns, i.e. + Create table t1 with two functionally dependent columns, i.e., knowledge of a value in the first column is sufficient for determining the value in the other column. Then functional dependency statistics are built on those columns: diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index dc688c415fa8..087cad184c0c 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -914,7 +914,7 @@ WITH ( MODULUS numeric_literal, REM one or more columns on which the uniqueness is not enforced. Note that although the constraint is not enforced on the included columns, it still depends on them. Consequently, some operations on these columns - (e.g. DROP COLUMN) can cause cascaded constraint and + (e.g., DROP COLUMN) can cause cascaded constraint and index deletion. @@ -960,7 +960,7 @@ WITH ( MODULUS numeric_literal, REM of columns to be specified which will be included in the non-key portion of the index. Although uniqueness is not enforced on the included columns, the constraint still depends on them. Consequently, some operations on the - included columns (e.g. DROP COLUMN) can cause cascaded + included columns (e.g., DROP COLUMN) can cause cascaded constraint and index deletion. diff --git a/doc/src/sgml/ref/initdb.sgml b/doc/src/sgml/ref/initdb.sgml index b6a55ce105f9..385ac2515061 100644 --- a/doc/src/sgml/ref/initdb.sgml +++ b/doc/src/sgml/ref/initdb.sgml @@ -86,7 +86,7 @@ PostgreSQL documentation initdb initializes the database cluster's default locale and character set encoding. The character set encoding, collation order (LC_COLLATE) and character set classes - (LC_CTYPE, e.g. upper, lower, digit) can be set separately + (LC_CTYPE, e.g., upper, lower, digit) can be set separately for a database when it is created. initdb determines those settings for the template1 database, which will serve as the default for all other databases. diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 7a37fd80455c..0b2e2de87b6d 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -759,7 +759,7 @@ PostgreSQL documentation - Use conditional commands (i.e. add an IF EXISTS + Use conditional commands (i.e., add an IF EXISTS clause) when cleaning database objects. This option is not valid unless is also specified. diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml index 541269d376d7..43abc530a0f6 100644 --- a/doc/src/sgml/ref/pg_dumpall.sgml +++ b/doc/src/sgml/ref/pg_dumpall.sgml @@ -321,7 +321,7 @@ PostgreSQL documentation - Use conditional commands (i.e. add an IF EXISTS + Use conditional commands (i.e., add an IF EXISTS clause) to drop databases and other objects. This option is not valid unless is also specified. diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml index b942cb238b1b..27eab2f02a52 100644 --- a/doc/src/sgml/ref/pg_restore.sgml +++ b/doc/src/sgml/ref/pg_restore.sgml @@ -572,7 +572,7 @@ PostgreSQL documentation - Use conditional commands (i.e. add an IF EXISTS + Use conditional commands (i.e., add an IF EXISTS clause) to drop database objects. This option is not valid unless is also specified. diff --git a/doc/src/sgml/ref/pg_rewind.sgml b/doc/src/sgml/ref/pg_rewind.sgml index 440eed7d4b71..ae23badc08c4 100644 --- a/doc/src/sgml/ref/pg_rewind.sgml +++ b/doc/src/sgml/ref/pg_rewind.sgml @@ -73,7 +73,7 @@ PostgreSQL documentation from the WAL archive to the pg_wal directory, or run pg_rewind with the -c option to automatically retrieve them from the WAL archive. The use of - pg_rewind is not limited to failover, e.g. a standby + pg_rewind is not limited to failover, e.g., a standby server can be promoted, run some write transactions, and then rewinded to become a standby again. diff --git a/doc/src/sgml/ref/pgbench.sgml b/doc/src/sgml/ref/pgbench.sgml index 9f3bb5fce65c..75575b6f0604 100644 --- a/doc/src/sgml/ref/pgbench.sgml +++ b/doc/src/sgml/ref/pgbench.sgml @@ -617,7 +617,7 @@ pgbench options d transaction to finish. The wait time is called the schedule lag time, and its average and maximum are also reported separately. The transaction latency with respect to the actual transaction start time, - i.e. the time spent executing the transaction in the database, can be + i.e., the time spent executing the transaction in the database, can be computed by subtracting the schedule lag time from the reported latency. @@ -767,7 +767,7 @@ pgbench options d client per thread and there are no external or data dependencies. From a statistical viewpoint reproducing runs exactly is a bad idea because it can hide the performance variability or improve performance unduly, - e.g. by hitting the same pages as a previous run. + e.g., by hitting the same pages as a previous run. However, it may also be of great help for debugging, for instance re-running a tricky case which leads to an error. Use wisely. @@ -787,7 +787,7 @@ pgbench options d Remember to take the sampling rate into account when processing the log file. For example, when computing TPS values, you need to multiply - the numbers accordingly (e.g. with 0.01 sample rate, you'll only get + the numbers accordingly (e.g., with 0.01 sample rate, you'll only get 1/100 of the actual TPS). @@ -1991,7 +1991,7 @@ f(x) = PHI(2.0 * parameter * (x - mu) / (max - min + 1)) / 2.0 / parameter, that is a relative 1.0 / parameter around the mean; for instance, if parameter is 4.0, 67% of values are drawn from the - middle quarter (1.0 / 4.0) of the interval (i.e. from + middle quarter (1.0 / 4.0) of the interval (i.e., from 3.0 / 8.0 to 5.0 / 8.0) and 95% from the middle half (2.0 / 4.0) of the interval (second and third quartiles). The minimum allowed parameter @@ -2186,7 +2186,7 @@ END; and max_lag, are only present if the option is used. They provide statistics about the time each transaction had to wait for the - previous one to finish, i.e. the difference between each transaction's + previous one to finish, i.e., the difference between each transaction's scheduled start time and the time it actually started. The very last field, skipped, is only present if the option is used, too. diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml index 74f27130ec5c..b59c5697a36c 100644 --- a/doc/src/sgml/ref/pgupgrade.sgml +++ b/doc/src/sgml/ref/pgupgrade.sgml @@ -41,8 +41,8 @@ PostgreSQL documentation pg_upgrade (formerly called pg_migrator) allows data stored in PostgreSQL data files to be upgraded to a later PostgreSQL major version without the data dump/reload typically required for - major version upgrades, e.g. from 9.5.8 to 9.6.4 or from 10.7 to 11.2. - It is not required for minor version upgrades, e.g. from 9.6.2 to 9.6.3 + major version upgrades, e.g., from 9.5.8 to 9.6.4 or from 10.7 to 11.2. + It is not required for minor version upgrades, e.g., from 9.6.2 to 9.6.3 or from 10.1 to 10.2. @@ -60,7 +60,7 @@ PostgreSQL documentation pg_upgrade does its best to - make sure the old and new clusters are binary-compatible, e.g. by + make sure the old and new clusters are binary-compatible, e.g., by checking for compatible compile-time settings, including 32/64-bit binaries. It is important that any external modules are also binary compatible, though this cannot @@ -239,13 +239,13 @@ PostgreSQL documentation Optionally move the old cluster - If you are using a version-specific installation directory, e.g. + If you are using a version-specific installation directory, e.g., /opt/PostgreSQL/&majorversion;, you do not need to move the old cluster. The graphical installers all use version-specific installation directories. - If your installation directory is not version-specific, e.g. + If your installation directory is not version-specific, e.g., /usr/local/pgsql, it is necessary to move the current PostgreSQL install directory so it does not interfere with the new PostgreSQL installation. Once the current PostgreSQL server is shut down, it is safe to rename the @@ -303,9 +303,9 @@ make prefix=/usr/local/pgsql.new install Install any custom shared object files (or DLLs) used by the old cluster - into the new cluster, e.g. pgcrypto.so, + into the new cluster, e.g., pgcrypto.so, whether they are from contrib - or some other source. Do not install the schema definitions, e.g. + or some other source. Do not install the schema definitions, e.g., CREATE EXTENSION pgcrypto, because these will be upgraded from the old cluster. Also, any custom full text search files (dictionary, synonym, @@ -516,7 +516,7 @@ pg_upgrade.exe Save any configuration files from the old standbys' configuration - directories you need to keep, e.g. postgresql.conf + directories you need to keep, e.g., postgresql.conf (and any files included by it), postgresql.auto.conf, pg_hba.conf, because these will be overwritten or removed in the next step. @@ -543,7 +543,7 @@ rsync --archive --delete --hard-links --size-only --no-inc-recursive old_cluster on the standby. The directory structure under the specified directories on the primary and standbys must match. Consult the rsync manual page for details on specifying the - remote directory, e.g. + remote directory, e.g., rsync --archive --delete --hard-links --size-only --no-inc-recursive /opt/PostgreSQL/9.5 \ @@ -669,7 +669,7 @@ psql --username=postgres --file=script.sql postgres pg_upgrade completes. (Automatic deletion is not possible if you have user-defined tablespaces inside the old data directory.) You can also delete the old installation directories - (e.g. bin, share). + (e.g., bin, share). @@ -793,7 +793,7 @@ psql --username=postgres --file=script.sql postgres If you are upgrading a pre-PostgreSQL 9.2 cluster that uses a configuration-file-only directory, you must pass the real data directory location to pg_upgrade, and - pass the configuration directory location to the server, e.g. + pass the configuration directory location to the server, e.g., -d /real-data-directory -o '-D /configuration-directory'. @@ -815,7 +815,7 @@ psql --username=postgres --file=script.sql postgres copy with any changes to make it consistent. ( is necessary because rsync only has file modification-time granularity of one second.) You might want to exclude some - files, e.g. postmaster.pid, as documented in postmaster.pid, as documented in . If your file system supports file system snapshots or copy-on-write file copies, you can use that to make a backup of the old cluster and tablespaces, though the snapshot diff --git a/doc/src/sgml/ref/postgres-ref.sgml b/doc/src/sgml/ref/postgres-ref.sgml index 6e62f54c597c..806949df42b7 100644 --- a/doc/src/sgml/ref/postgres-ref.sgml +++ b/doc/src/sgml/ref/postgres-ref.sgml @@ -821,7 +821,7 @@ PostgreSQL documentation To start postgres with a specific - port, e.g. 1234: + port, e.g., 1234: $ postgres -p 1234 diff --git a/doc/src/sgml/ref/prepare.sgml b/doc/src/sgml/ref/prepare.sgml index 5ec86aee10d3..1e484f6d2021 100644 --- a/doc/src/sgml/ref/prepare.sgml +++ b/doc/src/sgml/ref/prepare.sgml @@ -73,7 +73,7 @@ PREPARE name [ ( psql returns 0 to the shell if it - finished normally, 1 if a fatal error of its own occurs (e.g. out of memory, + finished normally, 1 if a fatal error of its own occurs (e.g., out of memory, file not found), 2 if the connection to the server went bad and the session was not interactive, and 3 if an error occurred in a script and the variable ON_ERROR_STOP was set. @@ -3011,7 +3011,7 @@ lo_import 152801 In latex-longtable format, this controls the proportional width of each column containing a left-aligned data type. It is specified as a whitespace-separated list of values, - e.g. '0.2 0.2 0.6'. Unspecified output columns + e.g., '0.2 0.2 0.6'. Unspecified output columns use the last specified value. @@ -4469,7 +4469,7 @@ testdb=> \set PROMPT1 '%[%033[1;33;40m%]%n@%/%R%[%033[0m%]%# ' psql starts up. Tab-completion is also supported, although the completion logic makes no claim to be an SQL parser. The queries generated by tab-completion - can also interfere with other SQL commands, e.g. SET + can also interfere with other SQL commands, e.g., SET TRANSACTION ISOLATION LEVEL. If for some reason you do not like the tab completion, you can turn it off by putting this in a file named diff --git a/doc/src/sgml/replication-origins.sgml b/doc/src/sgml/replication-origins.sgml index a03ce76e2ef3..7e02c4605b25 100644 --- a/doc/src/sgml/replication-origins.sgml +++ b/doc/src/sgml/replication-origins.sgml @@ -31,7 +31,7 @@ which is what should be used to refer to the origin across systems, is free-form text. It should be used in a way that makes conflicts between replication origins created by different replication solutions - unlikely; e.g. by prefixing the replication solution's name to it. + unlikely; e.g., by prefixing the replication solution's name to it. The OID is used only to avoid having to store the long version in situations where space efficiency is important. It should never be shared across systems. @@ -68,7 +68,7 @@ manner. Replay progress for all replication origins can be seen in the pg_replication_origin_status - view. An individual origin's progress, e.g. when resuming + view. An individual origin's progress, e.g., when resuming replication, can be acquired using pg_replication_origin_progress() for any origin or @@ -86,7 +86,7 @@ output plugin callbacks (see ) generated by the session is tagged with the replication origin of the generating session. This allows treating them differently in the output - plugin, e.g. ignoring all but locally-originating rows. Additionally + plugin, e.g., ignoring all but locally-originating rows. Additionally the filter_by_origin_cb callback can be used to filter the logical decoding change stream based on the diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index a01add94b7fd..6cda39f3abfc 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -1865,7 +1865,7 @@ pg_dumpall -p 5432 | psql -d postgres -p 5433 be migrated in-place from one major PostgreSQL version to another. Upgrades can be performed in minutes, particularly with mode. It requires steps similar to - pg_dumpall above, e.g. starting/stopping the server, + pg_dumpall above, e.g., starting/stopping the server, running initdb. The pg_upgrade documentation outlines the necessary steps. diff --git a/doc/src/sgml/sepgsql.sgml b/doc/src/sgml/sepgsql.sgml index 0d6b1b733dc9..9961569afc8a 100644 --- a/doc/src/sgml/sepgsql.sgml +++ b/doc/src/sgml/sepgsql.sgml @@ -530,7 +530,7 @@ UPDATE t1 SET x = 2, y = func1(y) WHERE z = 100; commands. SELinux provides a feature to allow trusted code to run using a security label different from that of the client, generally for the purpose of providing highly controlled access to - sensitive data (e.g. rows might be omitted, or the precision of stored + sensitive data (e.g., rows might be omitted, or the precision of stored values might be reduced). Whether or not a function acts as a trusted procedure is controlled by its security label and the operating system security policy. For example: diff --git a/doc/src/sgml/sources.sgml b/doc/src/sgml/sources.sgml index 283c3e03573a..998e7d5fba19 100644 --- a/doc/src/sgml/sources.sgml +++ b/doc/src/sgml/sources.sgml @@ -913,7 +913,7 @@ BETTER: unrecognized node type: 42 Both, macros with arguments and static inline functions, may be used. The latter are preferable if there are - multiple-evaluation hazards when written as a macro, as e.g. the + multiple-evaluation hazards when written as a macro, as e.g., the case with #define Max(x, y) ((x) > (y) ? (x) : (y)) @@ -924,7 +924,7 @@ BETTER: unrecognized node type: 42 When the definition of an inline function references symbols - (i.e. variables, functions) that are only available as part of the + (i.e., variables, functions) that are only available as part of the backend, the function may not be visible when included from frontend code. diff --git a/doc/src/sgml/sslinfo.sgml b/doc/src/sgml/sslinfo.sgml index c237d4ba95db..7d3fcb716707 100644 --- a/doc/src/sgml/sslinfo.sgml +++ b/doc/src/sgml/sslinfo.sgml @@ -53,7 +53,7 @@ - Returns the name of the protocol used for the SSL connection (e.g. TLSv1.0 + Returns the name of the protocol used for the SSL connection (e.g., TLSv1.0 TLSv1.1, or TLSv1.2). @@ -69,7 +69,7 @@ Returns the name of the cipher used for the SSL connection - (e.g. DHE-RSA-AES256-SHA). + (e.g., DHE-RSA-AES256-SHA). diff --git a/doc/src/sgml/tableam.sgml b/doc/src/sgml/tableam.sgml index efd141d3f5cc..a4fed6ea577a 100644 --- a/doc/src/sgml/tableam.sgml +++ b/doc/src/sgml/tableam.sgml @@ -77,7 +77,7 @@ necessary for each tuple to have a tuple identifier (TID) consisting of a block number and an item number (see also ). It is not strictly necessary that the - sub-parts of TIDs have the same meaning they e.g. have + sub-parts of TIDs have the same meaning they e.g., have for heap, but if bitmap scan support is desired (it is optional), the block number needs to provide locality. diff --git a/doc/src/sgml/textsearch.sgml b/doc/src/sgml/textsearch.sgml index 64c7ddb94aa5..2ebdf02bfaf0 100644 --- a/doc/src/sgml/textsearch.sgml +++ b/doc/src/sgml/textsearch.sgml @@ -3710,7 +3710,7 @@ SELECT plainto_tsquery('supernovae stars'); - A GiST index can be covering, i.e. use the INCLUDE + A GiST index can be covering, i.e., use the INCLUDE clause. Included columns can have data types without any GiST operator class. Included attributes will be stored uncompressed. @@ -3736,7 +3736,7 @@ SELECT plainto_tsquery('supernovae stars'); allows the implementation of very fast searches with online update. Partitioning can be done at the database level using table inheritance, or by distributing documents over - servers and collecting external search results, e.g. via Foreign Data access. The latter is possible because ranking functions use only local information. diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml index 7a13d8d5025e..d1c3893b147c 100644 --- a/doc/src/sgml/wal.sgml +++ b/doc/src/sgml/wal.sgml @@ -266,7 +266,7 @@ overhead can reduce performance, especially if journaling causes file system data to be flushed to disk. Fortunately, data flushing during journaling can - often be disabled with a file system mount option, e.g. + often be disabled with a file system mount option, e.g., data=writeback on a Linux ext3 file system. Journaled file systems do improve boot speed after a crash. diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index e0692e993f59..0f60a4a0ab6d 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -152,9 +152,9 @@ Besides SELECT queries, the commands can include data modification queries (INSERT, UPDATE, and DELETE), as well as - other SQL commands. (You cannot use transaction control commands, e.g. + other SQL commands. (You cannot use transaction control commands, e.g., COMMIT, SAVEPOINT, and some utility - commands, e.g. VACUUM, in SQL functions.) + commands, e.g., VACUUM, in SQL functions.) However, the final command must be a SELECT or have a RETURNING clause that returns whatever is @@ -3389,7 +3389,7 @@ if (!ptr) exceptions. Any exceptions must be caught and appropriate errors passed back to the C interface. If possible, compile C++ with to eliminate exceptions entirely; in such - cases, you must check for failures in your C++ code, e.g. check for + cases, you must check for failures in your C++ code, e.g., check for NULL returned by new(). diff --git a/doc/src/sgml/xml2.sgml b/doc/src/sgml/xml2.sgml index a61ec44337f2..584bb3e923fa 100644 --- a/doc/src/sgml/xml2.sgml +++ b/doc/src/sgml/xml2.sgml @@ -303,7 +303,7 @@ AS t(article_id integer, author text, page_count integer, title text); The calling SELECT statement doesn't necessarily have to be just SELECT * — it can reference the output columns by name or join them to other tables. The function produces a - virtual table with which you can perform any operation you wish (e.g. + virtual table with which you can perform any operation you wish (e.g., aggregation, joining, sorting etc). So we could also have: SELECT t.title, p.fullname, p.email From ab3c6d41552411ea2fe4904ec8294951c52c113d Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 31 Aug 2020 18:48:38 -0400 Subject: [PATCH 041/589] doc: document how the backup manifest is transferred Reported-by: Bernd Helmle Discussion: https://postgr.es/m/31acf8b0f1f701d53245e0cae38abdf5c3a0d559.camel@oopsware.de Backpatch-through: 13 --- doc/src/sgml/protocol.sgml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 7da2feb2ae51..c3cb7b4255fd 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2666,8 +2666,10 @@ The commands accepted in replication mode are: ustar interchange format specified in the POSIX 1003.1-2008 standard) dump of the tablespace contents, except that the two trailing blocks of zeroes specified in the standard are omitted. - After the tar data is complete, a final ordinary result set will be sent, - containing the WAL end position of the backup, in the same format as + After the tar data is complete, and if a backup manifest was requested, + another CopyResponse result is sent, containing the manifest data for the + current base backup. In any case, a final ordinary result set will be + sent, containing the WAL end position of the backup, in the same format as the start position. From 4ab77697f67aa5b90b032b9175b46901859da6d7 Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Tue, 1 Sep 2020 08:11:39 +0530 Subject: [PATCH 042/589] Fix the SharedFileSetUnregister API. Commit 808e13b282 introduced a few APIs to extend the existing Buffile interface. In SharedFileSetDeleteOnProcExit, it tries to delete the list element while traversing the list with 'foreach' construct which makes the behavior of list traversal unpredictable. Author: Amit Kapila Reviewed-by: Dilip Kumar Tested-by: Dilip Kumar and Neha Sharma Discussion: https://postgr.es/m/CAA4eK1JhLatVcQ2OvwA_3s0ih6Hx9+kZbq107cXVsSWWukH7vA@mail.gmail.com --- src/backend/storage/file/sharedfileset.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/backend/storage/file/sharedfileset.c b/src/backend/storage/file/sharedfileset.c index 8b96e81fffff..859c22e79b62 100644 --- a/src/backend/storage/file/sharedfileset.c +++ b/src/backend/storage/file/sharedfileset.c @@ -266,12 +266,16 @@ SharedFileSetOnDetach(dsm_segment *segment, Datum datum) static void SharedFileSetDeleteOnProcExit(int status, Datum arg) { - ListCell *l; - - /* Loop over all the pending shared fileset entry */ - foreach(l, filesetlist) + /* + * Remove all the pending shared fileset entries. We don't use foreach() here + * because SharedFileSetDeleteAll will remove the current element in + * filesetlist. Though we have used foreach_delete_current() to remove the + * element from filesetlist it could only fix up the state of one of the + * loops, see SharedFileSetUnregister. + */ + while (list_length(filesetlist) > 0) { - SharedFileSet *fileset = (SharedFileSet *) lfirst(l); + SharedFileSet *fileset = (SharedFileSet *) linitial(filesetlist); SharedFileSetDeleteAll(fileset); } @@ -301,7 +305,7 @@ SharedFileSetUnregister(SharedFileSet *input_fileset) /* Remove the entry from the list */ if (input_fileset == fileset) { - filesetlist = list_delete_cell(filesetlist, l); + filesetlist = foreach_delete_current(filesetlist, l); return; } } From b55b4dad99e99d5306744a4e8ef8021fa3a922e4 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 1 Sep 2020 13:14:44 -0400 Subject: [PATCH 043/589] Teach libpq to handle arbitrary-length lines in .pgpass files. Historically there's been a hard-wired assumption here that no line of a .pgpass file could be as long as NAMEDATALEN*5 bytes. That's a bit shaky to start off with, because (a) there's no reason to suppose that host names fit in NAMEDATALEN, and (b) this figure fails to allow for backslash escape characters. However, it fails completely if someone wants to use a very long password, and we're now hearing reports of people wanting to use "security tokens" that can run up to several hundred bytes. Another angle is that the file is specified to allow comment lines, but there's no reason to assume that long comment lines aren't possible. Rather than guessing at what might be a more suitable limit, let's replace the fixed-size buffer with an expansible PQExpBuffer. That adds one malloc/free cycle to the typical use-case, but that's surely pretty cheap relative to the I/O this code has to do. Also, add TAP test cases to exercise this code, because there was no test coverage before. This reverts most of commit 2eb3bc588, as there's no longer a need for a warning message about overlength .pgpass lines. (I kept the explicit check for comment lines, though.) In HEAD and v13, this also fixes an oversight in 74a308cf5: there's not much point in explicit_bzero'ing the line buffer if we only do so in two of the three exit paths. Back-patch to all supported branches, except that the test case only goes back to v10 where src/test/authentication/ was added. Discussion: https://postgr.es/m/4187382.1598909041@sss.pgh.pa.us --- src/interfaces/libpq/fe-connect.c | 119 ++++++++++------------ src/test/authentication/t/001_password.pl | 29 +++++- 2 files changed, 79 insertions(+), 69 deletions(-) diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 7bee9dd2014c..724076a3103e 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -6937,10 +6937,7 @@ passwordFromFile(const char *hostname, const char *port, const char *dbname, { FILE *fp; struct stat stat_buf; - int line_number = 0; - -#define LINELEN NAMEDATALEN*5 - char buf[LINELEN]; + PQExpBufferData buf; if (dbname == NULL || dbname[0] == '\0') return NULL; @@ -6996,89 +6993,77 @@ passwordFromFile(const char *hostname, const char *port, const char *dbname, if (fp == NULL) return NULL; + /* Use an expansible buffer to accommodate any reasonable line length */ + initPQExpBuffer(&buf); + while (!feof(fp) && !ferror(fp)) { - char *t = buf, - *ret, - *p1, - *p2; - int len; - int buflen; + /* Make sure there's a reasonable amount of room in the buffer */ + if (!enlargePQExpBuffer(&buf, 128)) + break; - if (fgets(buf, sizeof(buf), fp) == NULL) + /* Read some data, appending it to what we already have */ + if (fgets(buf.data + buf.len, buf.maxlen - buf.len, fp) == NULL) break; + buf.len += strlen(buf.data + buf.len); - line_number++; - buflen = strlen(buf); - if (buflen >= sizeof(buf) - 1 && buf[buflen - 1] != '\n') + /* If we don't yet have a whole line, loop around to read more */ + if (!(buf.len > 0 && buf.data[buf.len - 1] == '\n') && !feof(fp)) + continue; + + /* ignore comments */ + if (buf.data[0] != '#') { - char rest[LINELEN]; - int restlen; + char *t = buf.data; + int len; - /* - * Warn if this password setting line is too long, because it's - * unexpectedly truncated. - */ - if (buf[0] != '#') - fprintf(stderr, - libpq_gettext("WARNING: line %d too long in password file \"%s\"\n"), - line_number, pgpassfile); + /* strip trailing newline and carriage return */ + len = pg_strip_crlf(t); - /* eat rest of the line */ - while (!feof(fp) && !ferror(fp)) + if (len > 0 && + (t = pwdfMatchesString(t, hostname)) != NULL && + (t = pwdfMatchesString(t, port)) != NULL && + (t = pwdfMatchesString(t, dbname)) != NULL && + (t = pwdfMatchesString(t, username)) != NULL) { - if (fgets(rest, sizeof(rest), fp) == NULL) - break; - restlen = strlen(rest); - if (restlen < sizeof(rest) - 1 || rest[restlen - 1] == '\n') - break; - } - } - - /* ignore comments */ - if (buf[0] == '#') - continue; - - /* strip trailing newline and carriage return */ - len = pg_strip_crlf(buf); + /* Found a match. */ + char *ret, + *p1, + *p2; - if (len == 0) - continue; + ret = strdup(t); - if ((t = pwdfMatchesString(t, hostname)) == NULL || - (t = pwdfMatchesString(t, port)) == NULL || - (t = pwdfMatchesString(t, dbname)) == NULL || - (t = pwdfMatchesString(t, username)) == NULL) - continue; + fclose(fp); + explicit_bzero(buf.data, buf.maxlen); + termPQExpBuffer(&buf); - /* Found a match. */ - ret = strdup(t); - fclose(fp); + if (!ret) + { + /* Out of memory. XXX: an error message would be nice. */ + return NULL; + } - if (!ret) - { - /* Out of memory. XXX: an error message would be nice. */ - explicit_bzero(buf, sizeof(buf)); - return NULL; - } + /* De-escape password. */ + for (p1 = p2 = ret; *p1 != ':' && *p1 != '\0'; ++p1, ++p2) + { + if (*p1 == '\\' && p1[1] != '\0') + ++p1; + *p2 = *p1; + } + *p2 = '\0'; - /* De-escape password. */ - for (p1 = p2 = ret; *p1 != ':' && *p1 != '\0'; ++p1, ++p2) - { - if (*p1 == '\\' && p1[1] != '\0') - ++p1; - *p2 = *p1; + return ret; + } } - *p2 = '\0'; - return ret; + /* No match, reset buffer to prepare for next line. */ + buf.len = 0; } fclose(fp); - explicit_bzero(buf, sizeof(buf)); + explicit_bzero(buf.data, buf.maxlen); + termPQExpBuffer(&buf); return NULL; - -#undef LINELEN } diff --git a/src/test/authentication/t/001_password.pl b/src/test/authentication/t/001_password.pl index 1305de0051a6..1b4323fe2a60 100644 --- a/src/test/authentication/t/001_password.pl +++ b/src/test/authentication/t/001_password.pl @@ -17,7 +17,7 @@ } else { - plan tests => 10; + plan tests => 13; } @@ -45,7 +45,9 @@ sub test_role $status_string = 'success' if ($expected_res eq 0); - my $res = $node->psql('postgres', undef, extra_params => [ '-U', $role ]); + local $Test::Builder::Level = $Test::Builder::Level + 1; + + my $res = $node->psql('postgres', undef, extra_params => [ '-U', $role, '-w' ]); is($res, $expected_res, "authentication $status_string for method $method, role $role"); return; @@ -96,3 +98,26 @@ sub test_role reset_pg_hba($node, 'scram-sha-256'); $ENV{"PGCHANNELBINDING"} = 'require'; test_role($node, 'scram_role', 'scram-sha-256', 2); + +# Test .pgpass processing; but use a temp file, don't overwrite the real one! +my $pgpassfile = "${TestLib::tmp_check}/pgpass"; + +delete $ENV{"PGPASSWORD"}; +delete $ENV{"PGCHANNELBINDING"}; +$ENV{"PGPASSFILE"} = $pgpassfile; + +append_to_file($pgpassfile, qq! +# This very long comment is just here to exercise handling of long lines in the file. This very long comment is just here to exercise handling of long lines in the file. This very long comment is just here to exercise handling of long lines in the file. This very long comment is just here to exercise handling of long lines in the file. This very long comment is just here to exercise handling of long lines in the file. +*:*:postgres:scram_role:pass:this is not part of the password. +!); +chmod 0600, $pgpassfile or die; + +reset_pg_hba($node, 'password'); +test_role($node, 'scram_role', 'password from pgpass', 0); +test_role($node, 'md5_role', 'password from pgpass', 2); + +append_to_file($pgpassfile, qq! +*:*:*:md5_role:p\\ass +!); + +test_role($node, 'md5_role', 'password from pgpass', 0); From afc7e0ad556a4f720c466cb4815fc77d310fc50a Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Tue, 1 Sep 2020 13:40:43 -0400 Subject: [PATCH 044/589] Raise error on concurrent drop of partitioned index We were already raising an error for DROP INDEX CONCURRENTLY on a partitioned table, albeit a different and confusing one: ERROR: DROP INDEX CONCURRENTLY must be first action in transaction Change that to throw a more comprehensible error: ERROR: cannot drop partitioned index \"%s\" concurrently Michael Paquier authored the test case for indexes on temporary partitioned tables. Backpatch to 11, where indexes on partitioned tables were added. Reported-by: Jan Mussler Reviewed-by: Michael Paquier Discussion: https://postgr.es/m/16594-d2956ca909585067@postgresql.org --- doc/src/sgml/ref/drop_index.sgml | 2 ++ src/backend/commands/tablecmds.c | 11 +++++++++++ src/test/regress/expected/indexing.out | 20 ++++++++++++++++++++ src/test/regress/sql/indexing.sql | 13 +++++++++++++ 4 files changed, 46 insertions(+) diff --git a/doc/src/sgml/ref/drop_index.sgml b/doc/src/sgml/ref/drop_index.sgml index 0aedd71bd68d..85cf23bca20a 100644 --- a/doc/src/sgml/ref/drop_index.sgml +++ b/doc/src/sgml/ref/drop_index.sgml @@ -57,6 +57,8 @@ DROP INDEX [ CONCURRENTLY ] [ IF EXISTS ] nameDROP INDEX commands can be performed within a transaction block, but DROP INDEX CONCURRENTLY cannot. + Lastly, indexes on partitioned tables cannot be dropped using this + option. For temporary tables, DROP INDEX is always diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index d2b15a3387b0..3e57c7f9e1db 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -1373,6 +1373,17 @@ RemoveRelations(DropStmt *drop) flags |= PERFORM_DELETION_CONCURRENTLY; } + /* + * Concurrent index drop cannot be used with partitioned indexes, + * either. + */ + if ((flags & PERFORM_DELETION_CONCURRENTLY) != 0 && + get_rel_relkind(relOid) == RELKIND_PARTITIONED_INDEX) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot drop partitioned index \"%s\" concurrently", + rel->relname))); + /* OK, we're ready to delete this one */ obj.classId = RelationRelationId; obj.objectId = relOid; diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81b8..7e78a07af8b8 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -174,6 +174,8 @@ create table idxpart1 partition of idxpart for values from (0) to (10); drop index idxpart1_a_idx; -- no way ERROR: cannot drop index idxpart1_a_idx because index idxpart_a_idx requires it HINT: You can drop index idxpart_a_idx instead. +drop index concurrently idxpart_a_idx; -- unsupported +ERROR: cannot drop partitioned index "idxpart_a_idx" concurrently drop index idxpart_a_idx; -- both indexes go away select relname, relkind from pg_class where relname like 'idxpart%' order by relname; @@ -194,6 +196,24 @@ select relname, relkind from pg_class (2 rows) drop table idxpart; +-- DROP behavior with temporary partitioned indexes +create temp table idxpart_temp (a int) partition by range (a); +create index on idxpart_temp(a); +create temp table idxpart1_temp partition of idxpart_temp + for values from (0) to (10); +drop index idxpart1_temp_a_idx; -- error +ERROR: cannot drop index idxpart1_temp_a_idx because index idxpart_temp_a_idx requires it +HINT: You can drop index idxpart_temp_a_idx instead. +-- non-concurrent drop is enforced here, so it is a valid case. +drop index concurrently idxpart_temp_a_idx; +select relname, relkind from pg_class + where relname like 'idxpart_temp%' order by relname; + relname | relkind +--------------+--------- + idxpart_temp | p +(1 row) + +drop table idxpart_temp; -- ALTER INDEX .. ATTACH, error cases create table idxpart (a int, b int) partition by range (a, b); create table idxpart1 partition of idxpart for values from (0, 0) to (10, 10); diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b92..42f398b67c2c 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -92,6 +92,7 @@ create table idxpart (a int) partition by range (a); create index on idxpart (a); create table idxpart1 partition of idxpart for values from (0) to (10); drop index idxpart1_a_idx; -- no way +drop index concurrently idxpart_a_idx; -- unsupported drop index idxpart_a_idx; -- both indexes go away select relname, relkind from pg_class where relname like 'idxpart%' order by relname; @@ -101,6 +102,18 @@ select relname, relkind from pg_class where relname like 'idxpart%' order by relname; drop table idxpart; +-- DROP behavior with temporary partitioned indexes +create temp table idxpart_temp (a int) partition by range (a); +create index on idxpart_temp(a); +create temp table idxpart1_temp partition of idxpart_temp + for values from (0) to (10); +drop index idxpart1_temp_a_idx; -- error +-- non-concurrent drop is enforced here, so it is a valid case. +drop index concurrently idxpart_temp_a_idx; +select relname, relkind from pg_class + where relname like 'idxpart_temp%' order by relname; +drop table idxpart_temp; + -- ALTER INDEX .. ATTACH, error cases create table idxpart (a int, b int) partition by range (a, b); create table idxpart1 partition of idxpart for values from (0, 0) to (10, 10); From db864c3c36035e4620afd114c783af7d777d78b0 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 1 Sep 2020 17:00:10 -0400 Subject: [PATCH 045/589] doc: clarify that max_wal_size is "during" checkpoints Previous wording was "between". Reported-by: Pavel Luzanov Discussion: https://postgr.es/m/26906a54-d7cb-2f8e-eed7-e31660024694@postgrespro.ru Backpatch-through: 9.5 --- doc/src/sgml/config.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 606e80df0ed0..c4ba49ffaf58 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3219,7 +3219,7 @@ include_dir 'conf.d' - Maximum size to let the WAL grow to between automatic WAL + Maximum size to let the WAL grow during automatic checkpoints. This is a soft limit; WAL size can exceed max_wal_size under special circumstances, such as heavy load, a failing archive_command, or a high From a7212be8b9e0885ee769e8c55f99ef742cda487b Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 1 Sep 2020 18:37:12 -0400 Subject: [PATCH 046/589] Set cutoff xmin more aggressively when vacuuming a temporary table. Since other sessions aren't allowed to look into a temporary table of our own session, we do not need to worry about the global xmin horizon when setting the vacuum XID cutoff. Indeed, if we're not inside a transaction block, we may set oldestXmin to be the next XID, because there cannot be any in-doubt tuples in a temp table, nor any tuples that are dead but still visible to some snapshot of our transaction. (VACUUM, of course, is never inside a transaction block; but we need to test that because CLUSTER shares the same code.) This approach allows us to always clean out a temp table completely during VACUUM, independently of concurrent activity. Aside from being useful in its own right, that simplifies building reproducible test cases. Discussion: https://postgr.es/m/3490536.1598629609@sss.pgh.pa.us --- src/backend/access/heap/vacuumlazy.c | 1 + src/backend/commands/cluster.c | 28 ++++++----- src/backend/commands/vacuum.c | 73 +++++++++++++++++++--------- src/include/commands/cluster.h | 3 +- src/include/commands/vacuum.h | 1 + 5 files changed, 70 insertions(+), 36 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 53b1a952543b..92389e6666bc 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -471,6 +471,7 @@ heap_vacuum_rel(Relation onerel, VacuumParams *params, params->freeze_table_age, params->multixact_freeze_min_age, params->multixact_freeze_table_age, + true, /* we must be a top-level command */ &OldestXmin, &FreezeLimit, &xidFullScanLimit, &MultiXactCutoff, &mxactFullScanLimit); diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index 04d12a7ece69..0d647e912c47 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -67,10 +67,13 @@ typedef struct } RelToCluster; -static void rebuild_relation(Relation OldHeap, Oid indexOid, bool verbose); +static void rebuild_relation(Relation OldHeap, Oid indexOid, + bool isTopLevel, bool verbose); static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, - bool verbose, bool *pSwapToastByContent, - TransactionId *pFreezeXid, MultiXactId *pCutoffMulti); + bool isTopLevel, bool verbose, + bool *pSwapToastByContent, + TransactionId *pFreezeXid, + MultiXactId *pCutoffMulti); static List *get_tables_to_cluster(MemoryContext cluster_context); @@ -170,7 +173,7 @@ cluster(ClusterStmt *stmt, bool isTopLevel) table_close(rel, NoLock); /* Do the job. */ - cluster_rel(tableOid, indexOid, stmt->options); + cluster_rel(tableOid, indexOid, stmt->options, isTopLevel); } else { @@ -219,7 +222,8 @@ cluster(ClusterStmt *stmt, bool isTopLevel) PushActiveSnapshot(GetTransactionSnapshot()); /* Do the job. */ cluster_rel(rvtc->tableOid, rvtc->indexOid, - stmt->options | CLUOPT_RECHECK); + stmt->options | CLUOPT_RECHECK, + isTopLevel); PopActiveSnapshot(); CommitTransactionCommand(); } @@ -250,7 +254,7 @@ cluster(ClusterStmt *stmt, bool isTopLevel) * and error messages should refer to the operation as VACUUM not CLUSTER. */ void -cluster_rel(Oid tableOid, Oid indexOid, int options) +cluster_rel(Oid tableOid, Oid indexOid, int options, bool isTopLevel) { Relation OldHeap; bool verbose = ((options & CLUOPT_VERBOSE) != 0); @@ -400,7 +404,7 @@ cluster_rel(Oid tableOid, Oid indexOid, int options) TransferPredicateLocksToHeapRelation(OldHeap); /* rebuild_relation does all the dirty work */ - rebuild_relation(OldHeap, indexOid, verbose); + rebuild_relation(OldHeap, indexOid, isTopLevel, verbose); /* NB: rebuild_relation does table_close() on OldHeap */ @@ -545,11 +549,12 @@ mark_index_clustered(Relation rel, Oid indexOid, bool is_internal) * * OldHeap: table to rebuild --- must be opened and exclusive-locked! * indexOid: index to cluster by, or InvalidOid to rewrite in physical order. + * isTopLevel: should be passed down from ProcessUtility. * * NB: this routine closes OldHeap at the right time; caller should not. */ static void -rebuild_relation(Relation OldHeap, Oid indexOid, bool verbose) +rebuild_relation(Relation OldHeap, Oid indexOid, bool isTopLevel, bool verbose) { Oid tableOid = RelationGetRelid(OldHeap); Oid tableSpace = OldHeap->rd_rel->reltablespace; @@ -577,7 +582,7 @@ rebuild_relation(Relation OldHeap, Oid indexOid, bool verbose) AccessExclusiveLock); /* Copy the heap data into the new table in the desired order */ - copy_table_data(OIDNewHeap, tableOid, indexOid, verbose, + copy_table_data(OIDNewHeap, tableOid, indexOid, isTopLevel, verbose, &swap_toast_by_content, &frozenXid, &cutoffMulti); /* @@ -728,7 +733,8 @@ make_new_heap(Oid OIDOldHeap, Oid NewTableSpace, char relpersistence, * *pCutoffMulti receives the MultiXactId used as a cutoff point. */ static void -copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, bool verbose, +copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, + bool isTopLevel, bool verbose, bool *pSwapToastByContent, TransactionId *pFreezeXid, MultiXactId *pCutoffMulti) { @@ -826,7 +832,7 @@ copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, bool verbose, * Since we're going to rewrite the whole table anyway, there's no reason * not to be aggressive about this. */ - vacuum_set_xid_limits(OldHeap, 0, 0, 0, 0, + vacuum_set_xid_limits(OldHeap, 0, 0, 0, 0, isTopLevel, &OldestXmin, &FreezeXid, NULL, &MultiXactCutoff, NULL); diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 308a51d95d7a..ddeec870d811 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -907,6 +907,9 @@ get_all_vacuum_rels(int options) /* * vacuum_set_xid_limits() -- compute oldestXmin and freeze cutoff points * + * Input parameters are the target relation, applicable freeze age settings, + * and isTopLevel which should be passed down from ProcessUtility. + * * The output parameters are: * - oldestXmin is the cutoff value used to distinguish whether tuples are * DEAD or RECENTLY_DEAD (see HeapTupleSatisfiesVacuum). @@ -931,6 +934,7 @@ vacuum_set_xid_limits(Relation rel, int freeze_table_age, int multixact_freeze_min_age, int multixact_freeze_table_age, + bool isTopLevel, TransactionId *oldestXmin, TransactionId *freezeLimit, TransactionId *xidFullScanLimit, @@ -946,32 +950,53 @@ vacuum_set_xid_limits(Relation rel, MultiXactId mxactLimit; MultiXactId safeMxactLimit; - /* - * We can always ignore processes running lazy vacuum. This is because we - * use these values only for deciding which tuples we must keep in the - * tables. Since lazy vacuum doesn't write its XID anywhere (usually no - * XID assigned), it's safe to ignore it. In theory it could be - * problematic to ignore lazy vacuums in a full vacuum, but keep in mind - * that only one vacuum process can be working on a particular table at - * any time, and that each vacuum is always an independent transaction. - */ - *oldestXmin = GetOldestNonRemovableTransactionId(rel); - - if (OldSnapshotThresholdActive()) + if (RELATION_IS_LOCAL(rel) && !IsInTransactionBlock(isTopLevel)) + { + /* + * If we are processing a temp relation (which by prior checks must be + * one belonging to our session), and we are not inside any + * transaction block, then there can be no tuples in the rel that are + * still in-doubt, nor can there be any that are dead but possibly + * still interesting to some snapshot our session holds. We don't + * need to care whether other sessions could see such tuples, either. + * So we can aggressively set the cutoff xmin to be the nextXid. + */ + *oldestXmin = ReadNewTransactionId(); + } + else { - TransactionId limit_xmin; - TimestampTz limit_ts; + /* + * Otherwise, calculate the cutoff xmin normally. + * + * We can always ignore processes running lazy vacuum. This is + * because we use these values only for deciding which tuples we must + * keep in the tables. Since lazy vacuum doesn't write its XID + * anywhere (usually no XID assigned), it's safe to ignore it. In + * theory it could be problematic to ignore lazy vacuums in a full + * vacuum, but keep in mind that only one vacuum process can be + * working on a particular table at any time, and that each vacuum is + * always an independent transaction. + */ + *oldestXmin = GetOldestNonRemovableTransactionId(rel); - if (TransactionIdLimitedForOldSnapshots(*oldestXmin, rel, &limit_xmin, &limit_ts)) + if (OldSnapshotThresholdActive()) { - /* - * TODO: We should only set the threshold if we are pruning on the - * basis of the increased limits. Not as crucial here as it is for - * opportunistic pruning (which often happens at a much higher - * frequency), but would still be a significant improvement. - */ - SetOldSnapshotThresholdTimestamp(limit_ts, limit_xmin); - *oldestXmin = limit_xmin; + TransactionId limit_xmin; + TimestampTz limit_ts; + + if (TransactionIdLimitedForOldSnapshots(*oldestXmin, rel, + &limit_xmin, &limit_ts)) + { + /* + * TODO: We should only set the threshold if we are pruning on + * the basis of the increased limits. Not as crucial here as + * it is for opportunistic pruning (which often happens at a + * much higher frequency), but would still be a significant + * improvement. + */ + SetOldSnapshotThresholdTimestamp(limit_ts, limit_xmin); + *oldestXmin = limit_xmin; + } } } @@ -1905,7 +1930,7 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params) cluster_options |= CLUOPT_VERBOSE; /* VACUUM FULL is now a variant of CLUSTER; see cluster.c */ - cluster_rel(relid, InvalidOid, cluster_options); + cluster_rel(relid, InvalidOid, cluster_options, true); } else table_relation_vacuum(onerel, params, vac_strategy); diff --git a/src/include/commands/cluster.h b/src/include/commands/cluster.h index e05884781b94..1eb144204b6a 100644 --- a/src/include/commands/cluster.h +++ b/src/include/commands/cluster.h @@ -19,7 +19,8 @@ extern void cluster(ClusterStmt *stmt, bool isTopLevel); -extern void cluster_rel(Oid tableOid, Oid indexOid, int options); +extern void cluster_rel(Oid tableOid, Oid indexOid, int options, + bool isTopLevel); extern void check_index_is_clusterable(Relation OldHeap, Oid indexOid, bool recheck, LOCKMODE lockmode); extern void mark_index_clustered(Relation rel, Oid indexOid, bool is_internal); diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index a4cd7214009c..d9475c99890c 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -267,6 +267,7 @@ extern void vacuum_set_xid_limits(Relation rel, int freeze_min_age, int freeze_table_age, int multixact_freeze_min_age, int multixact_freeze_table_age, + bool isTopLevel, TransactionId *oldestXmin, TransactionId *freezeLimit, TransactionId *xidFullScanLimit, From 4c51a2d1e4b750bc11b8de9a85b079a14f798741 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 1 Sep 2020 18:40:37 -0400 Subject: [PATCH 047/589] Improve test coverage of ginvacuum.c. Add a test case that exercises vacuum's deletion of empty GIN posting pages. Since this is a temp table, it should now work reliably to delete a bunch of rows and immediately VACUUM. Before the preceding commit, this would not have had the desired effect, at least not in parallel regression tests. Discussion: https://postgr.es/m/3490536.1598629609@sss.pgh.pa.us --- src/test/regress/expected/gin.out | 21 +++++++++++++++++++++ src/test/regress/sql/gin.sql | 8 ++++++++ 2 files changed, 29 insertions(+) diff --git a/src/test/regress/expected/gin.out b/src/test/regress/expected/gin.out index b335466fc4ba..6402e89c7ff1 100644 --- a/src/test/regress/expected/gin.out +++ b/src/test/regress/expected/gin.out @@ -264,6 +264,27 @@ select count(*) from t_gin_test_tbl where j @> '{}'::int[]; 20006 (1 row) +-- test vacuuming of posting trees +delete from t_gin_test_tbl where j @> array[2]; +vacuum t_gin_test_tbl; +select count(*) from t_gin_test_tbl where j @> array[50]; + count +------- + 0 +(1 row) + +select count(*) from t_gin_test_tbl where j @> array[2]; + count +------- + 0 +(1 row) + +select count(*) from t_gin_test_tbl where j @> '{}'::int[]; + count +------- + 6 +(1 row) + reset enable_seqscan; reset enable_bitmapscan; drop table t_gin_test_tbl; diff --git a/src/test/regress/sql/gin.sql b/src/test/regress/sql/gin.sql index efb8ef3e964c..5194afcc1f56 100644 --- a/src/test/regress/sql/gin.sql +++ b/src/test/regress/sql/gin.sql @@ -159,6 +159,14 @@ explain (costs off) select count(*) from t_gin_test_tbl where j @> '{}'::int[]; select count(*) from t_gin_test_tbl where j @> '{}'::int[]; +-- test vacuuming of posting trees +delete from t_gin_test_tbl where j @> array[2]; +vacuum t_gin_test_tbl; + +select count(*) from t_gin_test_tbl where j @> array[50]; +select count(*) from t_gin_test_tbl where j @> array[2]; +select count(*) from t_gin_test_tbl where j @> '{}'::int[]; + reset enable_seqscan; reset enable_bitmapscan; From 1d65416661bbb0b165865a521ce038ffb61b12ad Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 2 Sep 2020 09:08:12 +0900 Subject: [PATCH 048/589] Improve handling of dropped relations for REINDEX DATABASE/SCHEMA/SYSTEM When multiple relations are reindexed, a scan of pg_class is done first to build the list of relations to work on. However the REINDEX logic has never checked if a relation listed still exists when beginning the work on it, causing for example sudden cache lookup failures. This commit adds safeguards against dropped relations for REINDEX, similarly to VACUUM or CLUSTER where we try to open the relation, ignoring it if it is missing. A new option is added to the REINDEX routines to control if a missed relation is OK to ignore or not. An isolation test, based on REINDEX SCHEMA, is added for the concurrent and non-concurrent cases. Author: Michael Paquier Reviewed-by: Anastasia Lubennikova Discussion: https://postgr.es/m/20200813043805.GE11663@paquier.xyz --- src/backend/access/table/table.c | 34 ++++++++++ src/backend/catalog/index.c | 34 ++++++++-- src/backend/commands/indexcmds.c | 63 +++++++++++++++++-- src/include/access/table.h | 1 + src/include/nodes/parsenodes.h | 1 + .../isolation/expected/reindex-schema.out | 17 +++++ src/test/isolation/isolation_schedule | 1 + src/test/isolation/specs/reindex-schema.spec | 32 ++++++++++ 8 files changed, 174 insertions(+), 9 deletions(-) create mode 100644 src/test/isolation/expected/reindex-schema.out create mode 100644 src/test/isolation/specs/reindex-schema.spec diff --git a/src/backend/access/table/table.c b/src/backend/access/table/table.c index 1aa01a54b341..7c29091e6c19 100644 --- a/src/backend/access/table/table.c +++ b/src/backend/access/table/table.c @@ -57,6 +57,40 @@ table_open(Oid relationId, LOCKMODE lockmode) return r; } + +/* ---------------- + * try_table_open - open a table relation by relation OID + * + * Same as table_open, except return NULL instead of failing + * if the relation does not exist. + * ---------------- + */ +Relation +try_table_open(Oid relationId, LOCKMODE lockmode) +{ + Relation r; + + r = try_relation_open(relationId, lockmode); + + /* leave if table does not exist */ + if (!r) + return NULL; + + if (r->rd_rel->relkind == RELKIND_INDEX || + r->rd_rel->relkind == RELKIND_PARTITIONED_INDEX) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("\"%s\" is an index", + RelationGetRelationName(r)))); + else if (r->rd_rel->relkind == RELKIND_COMPOSITE_TYPE) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("\"%s\" is a composite type", + RelationGetRelationName(r)))); + + return r; +} + /* ---------------- * table_openrv - open a table relation specified * by a RangeVar node diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index d0ec9a4b9c80..1d662e9af433 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -3437,8 +3437,20 @@ reindex_index(Oid indexId, bool skip_constraint_checks, char persistence, * Open and lock the parent heap relation. ShareLock is sufficient since * we only need to be sure no schema or data changes are going on. */ - heapId = IndexGetRelation(indexId, false); - heapRelation = table_open(heapId, ShareLock); + heapId = IndexGetRelation(indexId, + (options & REINDEXOPT_MISSING_OK) != 0); + /* if relation is missing, leave */ + if (!OidIsValid(heapId)) + return; + + if ((options & REINDEXOPT_MISSING_OK) != 0) + heapRelation = try_table_open(heapId, ShareLock); + else + heapRelation = table_open(heapId, ShareLock); + + /* if relation is gone, leave */ + if (!heapRelation) + return; if (progress) { @@ -3672,7 +3684,14 @@ reindex_relation(Oid relid, int flags, int options) * to prevent schema and data changes in it. The lock level used here * should match ReindexTable(). */ - rel = table_open(relid, ShareLock); + if ((options & REINDEXOPT_MISSING_OK) != 0) + rel = try_table_open(relid, ShareLock); + else + rel = table_open(relid, ShareLock); + + /* if relation is gone, leave */ + if (!rel) + return false; /* * This may be useful when implemented someday; but that day is not today. @@ -3771,7 +3790,14 @@ reindex_relation(Oid relid, int flags, int options) * still hold the lock on the main table. */ if ((flags & REINDEX_REL_PROCESS_TOAST) && OidIsValid(toast_relid)) - result |= reindex_relation(toast_relid, flags, options); + { + /* + * Note that this should fail if the toast relation is missing, so + * reset REINDEXOPT_MISSING_OK. + */ + result |= reindex_relation(toast_relid, flags, + options & ~(REINDEXOPT_MISSING_OK)); + } return result; } diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 254dbcdce52b..b3a92381f95f 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -2766,9 +2766,19 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, /* functions in indexes may want a snapshot set */ PushActiveSnapshot(GetTransactionSnapshot()); + /* check if the relation still exists */ + if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(relid))) + { + PopActiveSnapshot(); + CommitTransactionCommand(); + continue; + } + if (concurrent && get_rel_persistence(relid) != RELPERSISTENCE_TEMP) { - (void) ReindexRelationConcurrently(relid, options); + (void) ReindexRelationConcurrently(relid, + options | + REINDEXOPT_MISSING_OK); /* ReindexRelationConcurrently() does the verbose output */ } else @@ -2778,7 +2788,9 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, result = reindex_relation(relid, REINDEX_REL_PROCESS_TOAST | REINDEX_REL_CHECK_CONSTRAINTS, - options | REINDEXOPT_REPORT_PROGRESS); + options | + REINDEXOPT_REPORT_PROGRESS | + REINDEXOPT_MISSING_OK); if (result && (options & REINDEXOPT_VERBOSE)) ereport(INFO, @@ -2893,7 +2905,17 @@ ReindexRelationConcurrently(Oid relationOid, int options) errmsg("cannot reindex system catalogs concurrently"))); /* Open relation to get its indexes */ - heapRelation = table_open(relationOid, ShareUpdateExclusiveLock); + if ((options & REINDEXOPT_MISSING_OK) != 0) + { + heapRelation = try_table_open(relationOid, + ShareUpdateExclusiveLock); + /* leave if relation does not exist */ + if (!heapRelation) + break; + } + else + heapRelation = table_open(relationOid, + ShareUpdateExclusiveLock); /* Add all the valid indexes of relation to list */ foreach(lc, RelationGetIndexList(heapRelation)) @@ -2978,7 +3000,13 @@ ReindexRelationConcurrently(Oid relationOid, int options) } case RELKIND_INDEX: { - Oid heapId = IndexGetRelation(relationOid, false); + Oid heapId = IndexGetRelation(relationOid, + (options & REINDEXOPT_MISSING_OK) != 0); + Relation heapRelation; + + /* if relation is missing, leave */ + if (!OidIsValid(heapId)) + break; if (IsCatalogRelationOid(heapId)) ereport(ERROR, @@ -2995,6 +3023,25 @@ ReindexRelationConcurrently(Oid relationOid, int options) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot reindex invalid index on TOAST table concurrently"))); + /* + * Check if parent relation can be locked and if it exists, + * this needs to be done at this stage as the list of indexes + * to rebuild is not complete yet, and REINDEXOPT_MISSING_OK + * should not be used once all the session locks are taken. + */ + if ((options & REINDEXOPT_MISSING_OK) != 0) + { + heapRelation = try_table_open(heapId, + ShareUpdateExclusiveLock); + /* leave if relation does not exist */ + if (!heapRelation) + break; + } + else + heapRelation = table_open(heapId, + ShareUpdateExclusiveLock); + table_close(heapRelation, NoLock); + /* Save the list of relation OIDs in private context */ oldcontext = MemoryContextSwitchTo(private_context); @@ -3025,7 +3072,13 @@ ReindexRelationConcurrently(Oid relationOid, int options) break; } - /* Definitely no indexes, so leave */ + /* + * Definitely no indexes, so leave. Any checks based on + * REINDEXOPT_MISSING_OK should be done only while the list of indexes to + * work on is built as the session locks taken before this transaction + * commits will make sure that they cannot be dropped by a concurrent + * session until this operation completes. + */ if (indexIds == NIL) { PopActiveSnapshot(); diff --git a/src/include/access/table.h b/src/include/access/table.h index cf0ef7b337ba..68dc4d62c03f 100644 --- a/src/include/access/table.h +++ b/src/include/access/table.h @@ -22,6 +22,7 @@ extern Relation table_open(Oid relationId, LOCKMODE lockmode); extern Relation table_openrv(const RangeVar *relation, LOCKMODE lockmode); extern Relation table_openrv_extended(const RangeVar *relation, LOCKMODE lockmode, bool missing_ok); +extern Relation try_table_open(Oid relationId, LOCKMODE lockmode); extern void table_close(Relation relation, LOCKMODE lockmode); #endif /* TABLE_H */ diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 47d4c07306d0..23840bb8e64e 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -3352,6 +3352,7 @@ typedef struct ConstraintsSetStmt /* Reindex options */ #define REINDEXOPT_VERBOSE (1 << 0) /* print progress info */ #define REINDEXOPT_REPORT_PROGRESS (1 << 1) /* report pgstat progress */ +#define REINDEXOPT_MISSING_OK (2 << 1) /* skip missing relations */ typedef enum ReindexObjectType { diff --git a/src/test/isolation/expected/reindex-schema.out b/src/test/isolation/expected/reindex-schema.out new file mode 100644 index 000000000000..0884e7541236 --- /dev/null +++ b/src/test/isolation/expected/reindex-schema.out @@ -0,0 +1,17 @@ +Parsed test spec with 3 sessions + +starting permutation: begin1 lock1 reindex2 drop3 end1 +step begin1: BEGIN; +step lock1: LOCK reindex_schema.tab_locked IN SHARE UPDATE EXCLUSIVE MODE; +step reindex2: REINDEX SCHEMA reindex_schema; +step drop3: DROP TABLE reindex_schema.tab_dropped; +step end1: COMMIT; +step reindex2: <... completed> + +starting permutation: begin1 lock1 reindex_conc2 drop3 end1 +step begin1: BEGIN; +step lock1: LOCK reindex_schema.tab_locked IN SHARE UPDATE EXCLUSIVE MODE; +step reindex_conc2: REINDEX SCHEMA CONCURRENTLY reindex_schema; +step drop3: DROP TABLE reindex_schema.tab_dropped; +step end1: COMMIT; +step reindex_conc2: <... completed> diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index 218c87b24bf4..6acbb695ecea 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -49,6 +49,7 @@ test: lock-committed-update test: lock-committed-keyupdate test: update-locked-tuple test: reindex-concurrently +test: reindex-schema test: propagate-lock-delete test: tuplelock-conflict test: tuplelock-update diff --git a/src/test/isolation/specs/reindex-schema.spec b/src/test/isolation/specs/reindex-schema.spec new file mode 100644 index 000000000000..feff8a5ec05d --- /dev/null +++ b/src/test/isolation/specs/reindex-schema.spec @@ -0,0 +1,32 @@ +# REINDEX with schemas +# +# Check that concurrent drop of relations while doing a REINDEX +# SCHEMA allows the command to work. + +setup +{ + CREATE SCHEMA reindex_schema; + CREATE TABLE reindex_schema.tab_locked (a int PRIMARY KEY); + CREATE TABLE reindex_schema.tab_dropped (a int PRIMARY KEY); +} + +teardown +{ + DROP SCHEMA reindex_schema CASCADE; +} + +session "s1" +step "begin1" { BEGIN; } +step "lock1" { LOCK reindex_schema.tab_locked IN SHARE UPDATE EXCLUSIVE MODE; } +step "end1" { COMMIT; } + +session "s2" +step "reindex2" { REINDEX SCHEMA reindex_schema; } +step "reindex_conc2" { REINDEX SCHEMA CONCURRENTLY reindex_schema; } + +session "s3" +step "drop3" { DROP TABLE reindex_schema.tab_dropped; } + +# The table can be dropped while reindex is waiting. +permutation "begin1" "lock1" "reindex2" "drop3" "end1" +permutation "begin1" "lock1" "reindex_conc2" "drop3" "end1" From 05c16b827f3ae66ec718e9af49e4d96906fa2ffb Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Tue, 1 Sep 2020 20:43:23 -0400 Subject: [PATCH 049/589] Fix typo in comment Introduced by 8b08f7d4820f; backpatch to 11. Discussion: https://postgr.es/m/20200812214918.GA30353@alvherre.pgsql --- src/bin/pg_dump/pg_dump.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index da97b731b159..2f051b83d911 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -368,7 +368,7 @@ typedef struct _indxInfo * contains both key and nonkey attributes */ bool indisclustered; bool indisreplident; - Oid parentidx; /* if partitioned, parent index OID */ + Oid parentidx; /* if a partition, parent index OID */ SimplePtrList partattaches; /* if partitioned, partition attach objects */ /* if there is an associated constraint object, its dumpId: */ From be9788e9989a0744ba60ab100153340fd123b786 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Wed, 2 Sep 2020 10:55:55 +0900 Subject: [PATCH 050/589] Avoid unnecessary acquisition of SyncRepLock in transaction commit time. In SyncRepWaitForLSN() routine called in transaction commit time, SyncRepLock is necessary to atomically both check the shared sync_standbys_defined flag and operate the sync replication wait-queue. On the other hand, when the flag is false, the lock is not necessary because the wait-queue is not touched. But due to the changes by commit 48c9f49265, previously the lock was taken whatever the flag was. This could cause unnecessary performance overhead in every transaction commit time. Therefore this commit avoids that unnecessary aquisition of SyncRepLock. Author: Fujii Masao Reviewed-by: Asim Praveen, Masahiko Sawada, Discussion: https://postgr.es/m/20200406050332.nsscfqjzk2d57zyx@alap3.anarazel.de --- src/backend/replication/syncrep.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/backend/replication/syncrep.c b/src/backend/replication/syncrep.c index df1e341c7642..6e8c76537af0 100644 --- a/src/backend/replication/syncrep.c +++ b/src/backend/replication/syncrep.c @@ -158,18 +158,30 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit) */ Assert(InterruptHoldoffCount > 0); + /* + * Fast exit if user has not requested sync replication, or there are no + * sync replication standby names defined. + * + * Since this routine gets called every commit time, it's important to + * exit quickly if sync replication is not requested. So we check + * WalSndCtl->sync_standbys_defined flag without the lock and exit + * immediately if it's false. If it's true, we need to check it again later + * while holding the lock, to check the flag and operate the sync rep + * queue atomically. This is necessary to avoid the race condition + * described in SyncRepUpdateSyncStandbysDefined(). On the other + * hand, if it's false, the lock is not necessary because we don't touch + * the queue. + */ + if (!SyncRepRequested() || + !((volatile WalSndCtlData *) WalSndCtl)->sync_standbys_defined) + return; + /* Cap the level for anything other than commit to remote flush only. */ if (commit) mode = SyncRepWaitMode; else mode = Min(SyncRepWaitMode, SYNC_REP_WAIT_FLUSH); - /* - * Fast exit if user has not requested sync replication. - */ - if (!SyncRepRequested()) - return; - Assert(SHMQueueIsDetached(&(MyProc->syncRepLinks))); Assert(WalSndCtl != NULL); From 01767533e37fa255722cc0088ed67a0309875963 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 2 Sep 2020 14:56:59 +0900 Subject: [PATCH 051/589] Fix thinko with definition of REINDEXOPT_MISSING_OK This had no direct consequences, but let's be consistent and it would be confusing when adding new flags. Oversight in 1d65416. Reported-by: Justin Pryzby Discussion: https://postgr.es/m/20200902024148.GB20149@telsasoft.com --- src/include/nodes/parsenodes.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 23840bb8e64e..d52c563305ae 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -3352,7 +3352,7 @@ typedef struct ConstraintsSetStmt /* Reindex options */ #define REINDEXOPT_VERBOSE (1 << 0) /* print progress info */ #define REINDEXOPT_REPORT_PROGRESS (1 << 1) /* report pgstat progress */ -#define REINDEXOPT_MISSING_OK (2 << 1) /* skip missing relations */ +#define REINDEXOPT_MISSING_OK (1 << 2) /* skip missing relations */ typedef enum ReindexObjectType { From 07f386ede026ae8c3f2adeba0c22139df19bf2ff Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 2 Sep 2020 16:59:22 +0900 Subject: [PATCH 052/589] Add access method names to \d[i|m|t]+ in psql Listing a full set of relations with those psql meta-commands, without a matching pattern, has never showed the access method associated with each relation. This commit adds the access method of tables, indexes and matviews, masking it for relation kinds where it does not apply. Note that when HIDE_TABLEAM is enabled, the information does not show up. This is available when connecting to a backend version of at least 12, where table AMs have been introduced. Author: Georgios Kokolatos Reviewed-by: Vignesh C, Michael Paquier, Justin Pryzby Discussion: https://postgr.es/m/svaS1VTOEscES9CLKVTeKItjJP1EEJuBhTsA0ESOdlnbXeQSgycYwVlliL5zt8Jwcfo4ATYDXtEqsExxjkSkkhCSTCL8fnRgaCAJdr0unUg=@protonmail.com --- doc/src/sgml/ref/psql-ref.sgml | 4 +- src/bin/psql/describe.c | 18 +++++++- src/test/regress/expected/psql.out | 74 +++++++++++++++++++++++++++--- src/test/regress/sql/psql.sql | 24 ++++++++-- 4 files changed, 108 insertions(+), 12 deletions(-) diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml index 201946990f43..e1e2236a71de 100644 --- a/doc/src/sgml/ref/psql-ref.sgml +++ b/doc/src/sgml/ref/psql-ref.sgml @@ -1183,7 +1183,9 @@ testdb=> columns of the table are shown, as is the presence of OIDs in the table, the view definition if the relation is a view, a non-default replica - identity setting. + identity setting and the + access method name + if the relation has an access method. diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index d81f1575bf4c..0266fc5fa858 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -3678,7 +3678,7 @@ listTables(const char *tabtypes, const char *pattern, bool verbose, bool showSys PGresult *res; printQueryOpt myopt = pset.popt; int cols_so_far; - bool translate_columns[] = {false, false, true, false, false, false, false, false}; + bool translate_columns[] = {false, false, true, false, false, false, false, false, false}; /* If tabtypes is empty, we default to \dtvmsE (but see also command.c) */ if (!(showTables || showIndexes || showViews || showMatViews || showSeq || showForeign)) @@ -3751,6 +3751,16 @@ listTables(const char *tabtypes, const char *pattern, bool verbose, bool showSys * to; this might change with future additions to the output columns. */ + /* + * Access methods exist for tables, materialized views and indexes. + * This has been introduced in PostgreSQL 12 for tables. + */ + if (pset.sversion >= 120000 && !pset.hide_tableam && + (showTables || showMatViews || showIndexes)) + appendPQExpBuffer(&buf, + ",\n am.amname as \"%s\"", + gettext_noop("Access Method")); + /* * As of PostgreSQL 9.0, use pg_table_size() to show a more accurate * size of a table, including FSM, VM and TOAST tables. @@ -3772,6 +3782,12 @@ listTables(const char *tabtypes, const char *pattern, bool verbose, bool showSys appendPQExpBufferStr(&buf, "\nFROM pg_catalog.pg_class c" "\n LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace"); + + if (pset.sversion >= 120000 && !pset.hide_tableam && + (showTables || showMatViews || showIndexes)) + appendPQExpBufferStr(&buf, + "\n LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam"); + if (showIndexes) appendPQExpBufferStr(&buf, "\n LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid" diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out index 555d464f9182..daac0ff49d6e 100644 --- a/src/test/regress/expected/psql.out +++ b/src/test/regress/expected/psql.out @@ -2795,20 +2795,28 @@ Argument data types | numeric Type | func \pset tuples_only false --- check conditional tableam display --- Create a heap2 table am handler with heapam handler +-- check conditional am display +\pset expanded off +CREATE SCHEMA tableam_display; +CREATE ROLE regress_display_role; +ALTER SCHEMA tableam_display OWNER TO regress_display_role; +SET search_path TO tableam_display; CREATE ACCESS METHOD heap_psql TYPE TABLE HANDLER heap_tableam_handler; +SET ROLE TO regress_display_role; +-- Use only relations with a physical size of zero. CREATE TABLE tbl_heap_psql(f1 int, f2 char(100)) using heap_psql; CREATE TABLE tbl_heap(f1 int, f2 char(100)) using heap; +CREATE VIEW view_heap_psql AS SELECT f1 from tbl_heap_psql; +CREATE MATERIALIZED VIEW mat_view_heap_psql USING heap_psql AS SELECT f1 from tbl_heap_psql; \d+ tbl_heap_psql - Table "public.tbl_heap_psql" + Table "tableam_display.tbl_heap_psql" Column | Type | Collation | Nullable | Default | Storage | Stats target | Description --------+----------------+-----------+----------+---------+----------+--------------+------------- f1 | integer | | | | plain | | f2 | character(100) | | | | extended | | \d+ tbl_heap - Table "public.tbl_heap" + Table "tableam_display.tbl_heap" Column | Type | Collation | Nullable | Default | Storage | Stats target | Description --------+----------------+-----------+----------+---------+----------+--------------+------------- f1 | integer | | | | plain | | @@ -2816,7 +2824,7 @@ CREATE TABLE tbl_heap(f1 int, f2 char(100)) using heap; \set HIDE_TABLEAM off \d+ tbl_heap_psql - Table "public.tbl_heap_psql" + Table "tableam_display.tbl_heap_psql" Column | Type | Collation | Nullable | Default | Storage | Stats target | Description --------+----------------+-----------+----------+---------+----------+--------------+------------- f1 | integer | | | | plain | | @@ -2824,16 +2832,68 @@ CREATE TABLE tbl_heap(f1 int, f2 char(100)) using heap; Access method: heap_psql \d+ tbl_heap - Table "public.tbl_heap" + Table "tableam_display.tbl_heap" Column | Type | Collation | Nullable | Default | Storage | Stats target | Description --------+----------------+-----------+----------+---------+----------+--------------+------------- f1 | integer | | | | plain | | f2 | character(100) | | | | extended | | Access method: heap +-- AM is displayed for tables, indexes and materialized views. +\d+ + List of relations + Schema | Name | Type | Owner | Persistence | Access Method | Size | Description +-----------------+--------------------+-------------------+----------------------+-------------+---------------+---------+------------- + tableam_display | mat_view_heap_psql | materialized view | regress_display_role | permanent | heap_psql | 0 bytes | + tableam_display | tbl_heap | table | regress_display_role | permanent | heap | 0 bytes | + tableam_display | tbl_heap_psql | table | regress_display_role | permanent | heap_psql | 0 bytes | + tableam_display | view_heap_psql | view | regress_display_role | permanent | | 0 bytes | +(4 rows) + +\dt+ + List of relations + Schema | Name | Type | Owner | Persistence | Access Method | Size | Description +-----------------+---------------+-------+----------------------+-------------+---------------+---------+------------- + tableam_display | tbl_heap | table | regress_display_role | permanent | heap | 0 bytes | + tableam_display | tbl_heap_psql | table | regress_display_role | permanent | heap_psql | 0 bytes | +(2 rows) + +\dm+ + List of relations + Schema | Name | Type | Owner | Persistence | Access Method | Size | Description +-----------------+--------------------+-------------------+----------------------+-------------+---------------+---------+------------- + tableam_display | mat_view_heap_psql | materialized view | regress_display_role | permanent | heap_psql | 0 bytes | +(1 row) + +-- But not for views and sequences. +\dv+ + List of relations + Schema | Name | Type | Owner | Persistence | Size | Description +-----------------+----------------+------+----------------------+-------------+---------+------------- + tableam_display | view_heap_psql | view | regress_display_role | permanent | 0 bytes | +(1 row) + \set HIDE_TABLEAM on -DROP TABLE tbl_heap, tbl_heap_psql; +\d+ + List of relations + Schema | Name | Type | Owner | Persistence | Size | Description +-----------------+--------------------+-------------------+----------------------+-------------+---------+------------- + tableam_display | mat_view_heap_psql | materialized view | regress_display_role | permanent | 0 bytes | + tableam_display | tbl_heap | table | regress_display_role | permanent | 0 bytes | + tableam_display | tbl_heap_psql | table | regress_display_role | permanent | 0 bytes | + tableam_display | view_heap_psql | view | regress_display_role | permanent | 0 bytes | +(4 rows) + +RESET ROLE; +RESET search_path; +DROP SCHEMA tableam_display CASCADE; +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table tableam_display.tbl_heap_psql +drop cascades to table tableam_display.tbl_heap +drop cascades to view tableam_display.view_heap_psql +drop cascades to materialized view tableam_display.mat_view_heap_psql DROP ACCESS METHOD heap_psql; +DROP ROLE regress_display_role; -- test numericlocale (as best we can without control of psql's locale) \pset format aligned \pset expanded off diff --git a/src/test/regress/sql/psql.sql b/src/test/regress/sql/psql.sql index 5a160809807b..47b28d2a0763 100644 --- a/src/test/regress/sql/psql.sql +++ b/src/test/regress/sql/psql.sql @@ -455,20 +455,38 @@ select 1 where false; \df exp \pset tuples_only false --- check conditional tableam display +-- check conditional am display +\pset expanded off --- Create a heap2 table am handler with heapam handler +CREATE SCHEMA tableam_display; +CREATE ROLE regress_display_role; +ALTER SCHEMA tableam_display OWNER TO regress_display_role; +SET search_path TO tableam_display; CREATE ACCESS METHOD heap_psql TYPE TABLE HANDLER heap_tableam_handler; +SET ROLE TO regress_display_role; +-- Use only relations with a physical size of zero. CREATE TABLE tbl_heap_psql(f1 int, f2 char(100)) using heap_psql; CREATE TABLE tbl_heap(f1 int, f2 char(100)) using heap; +CREATE VIEW view_heap_psql AS SELECT f1 from tbl_heap_psql; +CREATE MATERIALIZED VIEW mat_view_heap_psql USING heap_psql AS SELECT f1 from tbl_heap_psql; \d+ tbl_heap_psql \d+ tbl_heap \set HIDE_TABLEAM off \d+ tbl_heap_psql \d+ tbl_heap +-- AM is displayed for tables, indexes and materialized views. +\d+ +\dt+ +\dm+ +-- But not for views and sequences. +\dv+ \set HIDE_TABLEAM on -DROP TABLE tbl_heap, tbl_heap_psql; +\d+ +RESET ROLE; +RESET search_path; +DROP SCHEMA tableam_display CASCADE; DROP ACCESS METHOD heap_psql; +DROP ROLE regress_display_role; -- test numericlocale (as best we can without control of psql's locale) From fd5e3b291415e6cf55408af1282585c945464c8f Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 2 Sep 2020 15:17:33 +0200 Subject: [PATCH 053/589] Remove unused parameter unused since 39bd3fd1db6f3aa3764d4a1bebcd71c4e9c00281 Discussion: https://www.postgresql.org/message-id/flat/511bb100-f829-ba21-2f10-9f952ec06ead%402ndquadrant.com --- src/pl/plpgsql/src/pl_gram.y | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/pl/plpgsql/src/pl_gram.y b/src/pl/plpgsql/src/pl_gram.y index 5a7e1a44442b..8227bf0449fc 100644 --- a/src/pl/plpgsql/src/pl_gram.y +++ b/src/pl/plpgsql/src/pl_gram.y @@ -107,7 +107,7 @@ static void check_labels(const char *start_label, const char *end_label, int end_location); static PLpgSQL_expr *read_cursor_args(PLpgSQL_var *cursor, - int until, const char *expected); + int until); static List *read_raise_options(void); static void check_raise_parameters(PLpgSQL_stmt_raise *stmt); @@ -1414,8 +1414,7 @@ for_control : for_variable K_IN /* collect cursor's parameters if any */ new->argquery = read_cursor_args(cursor, - K_LOOP, - "LOOP"); + K_LOOP); /* create loop's private RECORD variable */ new->var = (PLpgSQL_variable *) @@ -2129,7 +2128,7 @@ stmt_open : K_OPEN cursor_variable else { /* predefined cursor query, so read args */ - new->argquery = read_cursor_args($2, ';', ";"); + new->argquery = read_cursor_args($2, ';'); } $$ = (PLpgSQL_stmt *)new; @@ -3773,7 +3772,7 @@ check_labels(const char *start_label, const char *end_label, int end_location) * parens). */ static PLpgSQL_expr * -read_cursor_args(PLpgSQL_var *cursor, int until, const char *expected) +read_cursor_args(PLpgSQL_var *cursor, int until) { PLpgSQL_expr *expr; PLpgSQL_row *row; From 66f163068030b5c5fe792a0daee27822dac43791 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 2 Sep 2020 18:23:56 -0400 Subject: [PATCH 054/589] Add string_to_table() function. This splits a string at occurrences of a delimiter. It is exactly like string_to_array() except for producing a set of values instead of an array of values. Thus, the relationship of these two functions is the same as between regexp_split_to_table() and regexp_split_to_array(). Although the same results could be had from unnest(string_to_array()), this is somewhat faster than that, and anyway it seems reasonable to have it for symmetry with the regexp functions. Pavel Stehule, reviewed by Peter Smith Discussion: https://postgr.es/m/CAFj8pRD8HOpjq2TqeTBhSo_QkzjLOhXzGCpKJ4nCs7Y9SQkuPw@mail.gmail.com --- doc/src/sgml/func.sgml | 90 +++++++---- src/backend/utils/adt/varlena.c | 214 +++++++++++++++++++-------- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.dat | 18 ++- src/test/regress/expected/arrays.out | 108 ++++++++++++++ src/test/regress/sql/arrays.sql | 15 ++ 6 files changed, 354 insertions(+), 93 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 2efd80baa450..e2e618791ee0 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -3220,7 +3220,7 @@ repeat('Pg', 4) PgPgPgPg Splits string using a POSIX regular - expression as the delimiter; see + expression as the delimiter, producing an array of results; see . @@ -3239,7 +3239,7 @@ repeat('Pg', 4) PgPgPgPg Splits string using a POSIX regular - expression as the delimiter; see + expression as the delimiter, producing a set of results; see . @@ -3460,6 +3460,65 @@ repeat('Pg', 4) PgPgPgPg + + + + string_to_array + + string_to_array ( string text, delimiter text , null_string text ) + text[] + + + Splits the string at occurrences + of delimiter and forms the resulting fields + into a text array. + If delimiter is NULL, + each character in the string will become a + separate element in the array. + If delimiter is an empty string, then + the string is treated as a single field. + If null_string is supplied and is + not NULL, fields matching that string are + replaced by NULL. + + + string_to_array('xx~~yy~~zz', '~~', 'yy') + {xx,NULL,zz} + + + + + + + string_to_table + + string_to_table ( string text, delimiter text , null_string text ) + setof text + + + Splits the string at occurrences + of delimiter and returns the resulting fields + as a set of text rows. + If delimiter is NULL, + each character in the string will become a + separate row of the result. + If delimiter is an empty string, then + the string is treated as a single field. + If null_string is supplied and is + not NULL, fields matching that string are + replaced by NULL. + + + string_to_table('xx~^~yy~^~zz', '~^~', 'yy') + + + xx + NULL + zz + + + + @@ -17819,33 +17878,6 @@ SELECT NULLIF(value, '(none)') ... - - - - string_to_array - - string_to_array ( string text, delimiter text , null_string text ) - text[] - - - Splits the string at occurrences - of delimiter and forms the remaining data - into a text array. - If delimiter is NULL, - each character in the string will become a - separate element in the array. - If delimiter is an empty string, then - the string is treated as a single field. - If null_string is supplied and is - not NULL, fields matching that string are converted - to NULL entries. - - - string_to_array('xx~~yy~~zz', '~~', 'yy') - {xx,NULL,zz} - - - diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index df10bfb906ed..d7bc33054174 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -26,6 +26,7 @@ #include "lib/hyperloglog.h" #include "libpq/pqformat.h" #include "miscadmin.h" +#include "nodes/execnodes.h" #include "parser/scansup.h" #include "port/pg_bswap.h" #include "regex/regex.h" @@ -92,6 +93,17 @@ typedef struct pg_locale_t locale; } VarStringSortSupport; +/* + * Output data for split_text(): we output either to an array or a table. + * tupstore and tupdesc must be set up in advance to output to a table. + */ +typedef struct +{ + ArrayBuildState *astate; + Tuplestorestate *tupstore; + TupleDesc tupdesc; +} SplitTextOutputData; + /* * This should be large enough that most strings will fit, but small enough * that we feel comfortable putting it on the stack @@ -139,7 +151,11 @@ static bytea *bytea_substring(Datum str, bool length_not_specified); static bytea *bytea_overlay(bytea *t1, bytea *t2, int sp, int sl); static void appendStringInfoText(StringInfo str, const text *t); -static Datum text_to_array_internal(PG_FUNCTION_ARGS); +static bool split_text(FunctionCallInfo fcinfo, SplitTextOutputData *tstate); +static void split_text_accum_result(SplitTextOutputData *tstate, + text *field_value, + text *null_string, + Oid collation); static text *array_to_text_internal(FunctionCallInfo fcinfo, ArrayType *v, const char *fldsep, const char *null_string); static StringInfo makeStringAggState(FunctionCallInfo fcinfo); @@ -4564,13 +4580,13 @@ replace_text_regexp(text *src_text, void *regexp, } /* - * split_text + * split_part * parse input string * return ord item (1 based) * based on provided field separator */ Datum -split_text(PG_FUNCTION_ARGS) +split_part(PG_FUNCTION_ARGS) { text *inputstring = PG_GETARG_TEXT_PP(0); text *fldsep = PG_GETARG_TEXT_PP(1); @@ -4599,7 +4615,6 @@ split_text(PG_FUNCTION_ARGS) /* empty field separator */ if (fldsep_len < 1) { - text_position_cleanup(&state); /* if first field, return input string, else empty string */ if (fldnum == 1) PG_RETURN_TEXT_P(inputstring); @@ -4679,7 +4694,19 @@ text_isequal(text *txt1, text *txt2, Oid collid) Datum text_to_array(PG_FUNCTION_ARGS) { - return text_to_array_internal(fcinfo); + SplitTextOutputData tstate; + + /* For array output, tstate should start as all zeroes */ + memset(&tstate, 0, sizeof(tstate)); + + if (!split_text(fcinfo, &tstate)) + PG_RETURN_NULL(); + + if (tstate.astate == NULL) + PG_RETURN_ARRAYTYPE_P(construct_empty_array(TEXTOID)); + + PG_RETURN_ARRAYTYPE_P(makeArrayResult(tstate.astate, + CurrentMemoryContext)); } /* @@ -4693,30 +4720,90 @@ text_to_array(PG_FUNCTION_ARGS) Datum text_to_array_null(PG_FUNCTION_ARGS) { - return text_to_array_internal(fcinfo); + return text_to_array(fcinfo); +} + +/* + * text_to_table + * parse input string and return table of elements, + * based on provided field separator + */ +Datum +text_to_table(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsi = (ReturnSetInfo *) fcinfo->resultinfo; + SplitTextOutputData tstate; + MemoryContext old_cxt; + + /* check to see if caller supports us returning a tuplestore */ + if (rsi == NULL || !IsA(rsi, ReturnSetInfo)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that cannot accept a set"))); + if (!(rsi->allowedModes & SFRM_Materialize)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("materialize mode required, but it is not allowed in this context"))); + + /* OK, prepare tuplestore in per-query memory */ + old_cxt = MemoryContextSwitchTo(rsi->econtext->ecxt_per_query_memory); + + tstate.astate = NULL; + tstate.tupdesc = CreateTupleDescCopy(rsi->expectedDesc); + tstate.tupstore = tuplestore_begin_heap(true, false, work_mem); + + MemoryContextSwitchTo(old_cxt); + + (void) split_text(fcinfo, &tstate); + + tuplestore_donestoring(tstate.tupstore); + + rsi->returnMode = SFRM_Materialize; + rsi->setResult = tstate.tupstore; + rsi->setDesc = tstate.tupdesc; + + return (Datum) 0; +} + +/* + * text_to_table_null + * parse input string and return table of elements, + * based on provided field separator and null string + * + * This is a separate entry point only to prevent the regression tests from + * complaining about different argument sets for the same internal function. + */ +Datum +text_to_table_null(PG_FUNCTION_ARGS) +{ + return text_to_table(fcinfo); } /* - * common code for text_to_array and text_to_array_null functions + * Common code for text_to_array, text_to_array_null, text_to_table + * and text_to_table_null functions. * * These are not strict so we have to test for null inputs explicitly. + * Returns false if result is to be null, else returns true. + * + * Note that if the result is valid but empty (zero elements), we return + * without changing *tstate --- caller must handle that case, too. */ -static Datum -text_to_array_internal(PG_FUNCTION_ARGS) +static bool +split_text(FunctionCallInfo fcinfo, SplitTextOutputData *tstate) { text *inputstring; text *fldsep; text *null_string; + Oid collation = PG_GET_COLLATION(); int inputstring_len; int fldsep_len; char *start_ptr; text *result_text; - bool is_null; - ArrayBuildState *astate = NULL; /* when input string is NULL, then result is NULL too */ if (PG_ARGISNULL(0)) - PG_RETURN_NULL(); + return false; inputstring = PG_GETARG_TEXT_PP(0); @@ -4743,35 +4830,19 @@ text_to_array_internal(PG_FUNCTION_ARGS) inputstring_len = VARSIZE_ANY_EXHDR(inputstring); fldsep_len = VARSIZE_ANY_EXHDR(fldsep); - /* return empty array for empty input string */ + /* return empty set for empty input string */ if (inputstring_len < 1) - PG_RETURN_ARRAYTYPE_P(construct_empty_array(TEXTOID)); + return true; - /* - * empty field separator: return the input string as a one-element - * array - */ + /* empty field separator: return input string as a one-element set */ if (fldsep_len < 1) { - Datum elems[1]; - bool nulls[1]; - int dims[1]; - int lbs[1]; - - /* single element can be a NULL too */ - is_null = null_string ? text_isequal(inputstring, null_string, PG_GET_COLLATION()) : false; - - elems[0] = PointerGetDatum(inputstring); - nulls[0] = is_null; - dims[0] = 1; - lbs[0] = 1; - /* XXX: this hardcodes assumptions about the text type */ - PG_RETURN_ARRAYTYPE_P(construct_md_array(elems, nulls, - 1, dims, lbs, - TEXTOID, -1, false, TYPALIGN_INT)); + split_text_accum_result(tstate, inputstring, + null_string, collation); + return true; } - text_position_setup(inputstring, fldsep, PG_GET_COLLATION(), &state); + text_position_setup(inputstring, fldsep, collation, &state); start_ptr = VARDATA_ANY(inputstring); @@ -4797,16 +4868,12 @@ text_to_array_internal(PG_FUNCTION_ARGS) chunk_len = end_ptr - start_ptr; } - /* must build a temp text datum to pass to accumArrayResult */ + /* build a temp text datum to pass to split_text_accum_result */ result_text = cstring_to_text_with_len(start_ptr, chunk_len); - is_null = null_string ? text_isequal(result_text, null_string, PG_GET_COLLATION()) : false; /* stash away this field */ - astate = accumArrayResult(astate, - PointerGetDatum(result_text), - is_null, - TEXTOID, - CurrentMemoryContext); + split_text_accum_result(tstate, result_text, + null_string, collation); pfree(result_text); @@ -4821,16 +4888,12 @@ text_to_array_internal(PG_FUNCTION_ARGS) else { /* - * When fldsep is NULL, each character in the inputstring becomes an - * element in the result array. The separator is effectively the - * space between characters. + * When fldsep is NULL, each character in the input string becomes a + * separate element in the result set. The separator is effectively + * the space between characters. */ inputstring_len = VARSIZE_ANY_EXHDR(inputstring); - /* return empty array for empty input string */ - if (inputstring_len < 1) - PG_RETURN_ARRAYTYPE_P(construct_empty_array(TEXTOID)); - start_ptr = VARDATA_ANY(inputstring); while (inputstring_len > 0) @@ -4839,16 +4902,12 @@ text_to_array_internal(PG_FUNCTION_ARGS) CHECK_FOR_INTERRUPTS(); - /* must build a temp text datum to pass to accumArrayResult */ + /* build a temp text datum to pass to split_text_accum_result */ result_text = cstring_to_text_with_len(start_ptr, chunk_len); - is_null = null_string ? text_isequal(result_text, null_string, PG_GET_COLLATION()) : false; /* stash away this field */ - astate = accumArrayResult(astate, - PointerGetDatum(result_text), - is_null, - TEXTOID, - CurrentMemoryContext); + split_text_accum_result(tstate, result_text, + null_string, collation); pfree(result_text); @@ -4857,8 +4916,47 @@ text_to_array_internal(PG_FUNCTION_ARGS) } } - PG_RETURN_ARRAYTYPE_P(makeArrayResult(astate, - CurrentMemoryContext)); + return true; +} + +/* + * Add text item to result set (table or array). + * + * This is also responsible for checking to see if the item matches + * the null_string, in which case we should emit NULL instead. + */ +static void +split_text_accum_result(SplitTextOutputData *tstate, + text *field_value, + text *null_string, + Oid collation) +{ + bool is_null = false; + + if (null_string && text_isequal(field_value, null_string, collation)) + is_null = true; + + if (tstate->tupstore) + { + Datum values[1]; + bool nulls[1]; + + values[0] = PointerGetDatum(field_value); + nulls[0] = is_null; + + tuplestore_putvalues(tstate->tupstore, + tstate->tupdesc, + values, + nulls); + } + else + { + tstate->astate = accumArrayResult(tstate->astate, + PointerGetDatum(field_value), + is_null, + TEXTOID, + CurrentMemoryContext); + } } /* diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 52ca61f8a8e8..c807f83baddd 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 202008301 +#define CATALOG_VERSION_NO 202009021 #endif diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 1dd325e0e6fd..687509ba9265 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -1561,16 +1561,24 @@ { oid => '383', proname => 'array_cat', proisstrict => 'f', prorettype => 'anyarray', proargtypes => 'anyarray anyarray', prosrc => 'array_cat' }, -{ oid => '394', descr => 'split delimited text into text[]', +{ oid => '394', descr => 'split delimited text', proname => 'string_to_array', proisstrict => 'f', prorettype => '_text', proargtypes => 'text text', prosrc => 'text_to_array' }, +{ oid => '376', descr => 'split delimited text, with null string', + proname => 'string_to_array', proisstrict => 'f', prorettype => '_text', + proargtypes => 'text text text', prosrc => 'text_to_array_null' }, +{ oid => '8432', descr => 'split delimited text', + proname => 'string_to_table', proisstrict => 'f', prorows => '1000', + proretset => 't', prorettype => 'text', proargtypes => 'text text', + prosrc => 'text_to_table' }, +{ oid => '8433', descr => 'split delimited text, with null string', + proname => 'string_to_table', proisstrict => 'f', prorows => '1000', + proretset => 't', prorettype => 'text', proargtypes => 'text text text', + prosrc => 'text_to_table_null' }, { oid => '395', descr => 'concatenate array elements, using delimiter, into text', proname => 'array_to_string', provolatile => 's', prorettype => 'text', proargtypes => 'anyarray text', prosrc => 'array_to_text' }, -{ oid => '376', descr => 'split delimited text into text[], with null string', - proname => 'string_to_array', proisstrict => 'f', prorettype => '_text', - proargtypes => 'text text text', prosrc => 'text_to_array_null' }, { oid => '384', descr => 'concatenate array elements, using delimiter and null string, into text', proname => 'array_to_string', proisstrict => 'f', provolatile => 's', @@ -3547,7 +3555,7 @@ prosrc => 'regexp_matches' }, { oid => '2088', descr => 'split string by field_sep and return field_num', proname => 'split_part', prorettype => 'text', - proargtypes => 'text text int4', prosrc => 'split_text' }, + proargtypes => 'text text int4', prosrc => 'split_part' }, { oid => '2765', descr => 'split string by pattern', proname => 'regexp_split_to_table', prorows => '1000', proretset => 't', prorettype => 'text', proargtypes => 'text text', diff --git a/src/test/regress/expected/arrays.out b/src/test/regress/expected/arrays.out index c730563f0386..f9d9ad6aef90 100644 --- a/src/test/regress/expected/arrays.out +++ b/src/test/regress/expected/arrays.out @@ -1755,6 +1755,114 @@ select string_to_array('1,2,3,4,*,6', ',', '*'); {1,2,3,4,NULL,6} (1 row) +select v, v is null as "is null" from string_to_table('1|2|3', '|') g(v); + v | is null +---+--------- + 1 | f + 2 | f + 3 | f +(3 rows) + +select v, v is null as "is null" from string_to_table('1|2|3|', '|') g(v); + v | is null +---+--------- + 1 | f + 2 | f + 3 | f + | f +(4 rows) + +select v, v is null as "is null" from string_to_table('1||2|3||', '||') g(v); + v | is null +-----+--------- + 1 | f + 2|3 | f + | f +(3 rows) + +select v, v is null as "is null" from string_to_table('1|2|3', '') g(v); + v | is null +-------+--------- + 1|2|3 | f +(1 row) + +select v, v is null as "is null" from string_to_table('', '|') g(v); + v | is null +---+--------- +(0 rows) + +select v, v is null as "is null" from string_to_table('1|2|3', NULL) g(v); + v | is null +---+--------- + 1 | f + | | f + 2 | f + | | f + 3 | f +(5 rows) + +select v, v is null as "is null" from string_to_table(NULL, '|') g(v); + v | is null +---+--------- +(0 rows) + +select v, v is null as "is null" from string_to_table('abc', '') g(v); + v | is null +-----+--------- + abc | f +(1 row) + +select v, v is null as "is null" from string_to_table('abc', '', 'abc') g(v); + v | is null +---+--------- + | t +(1 row) + +select v, v is null as "is null" from string_to_table('abc', ',') g(v); + v | is null +-----+--------- + abc | f +(1 row) + +select v, v is null as "is null" from string_to_table('abc', ',', 'abc') g(v); + v | is null +---+--------- + | t +(1 row) + +select v, v is null as "is null" from string_to_table('1,2,3,4,,6', ',') g(v); + v | is null +---+--------- + 1 | f + 2 | f + 3 | f + 4 | f + | f + 6 | f +(6 rows) + +select v, v is null as "is null" from string_to_table('1,2,3,4,,6', ',', '') g(v); + v | is null +---+--------- + 1 | f + 2 | f + 3 | f + 4 | f + | t + 6 | f +(6 rows) + +select v, v is null as "is null" from string_to_table('1,2,3,4,*,6', ',', '*') g(v); + v | is null +---+--------- + 1 | f + 2 | f + 3 | f + 4 | f + | t + 6 | f +(6 rows) + select array_to_string(NULL::int4[], ',') IS NULL; ?column? ---------- diff --git a/src/test/regress/sql/arrays.sql b/src/test/regress/sql/arrays.sql index 25dd4e2c6ded..2b689ae88f59 100644 --- a/src/test/regress/sql/arrays.sql +++ b/src/test/regress/sql/arrays.sql @@ -544,6 +544,21 @@ select string_to_array('1,2,3,4,,6', ','); select string_to_array('1,2,3,4,,6', ',', ''); select string_to_array('1,2,3,4,*,6', ',', '*'); +select v, v is null as "is null" from string_to_table('1|2|3', '|') g(v); +select v, v is null as "is null" from string_to_table('1|2|3|', '|') g(v); +select v, v is null as "is null" from string_to_table('1||2|3||', '||') g(v); +select v, v is null as "is null" from string_to_table('1|2|3', '') g(v); +select v, v is null as "is null" from string_to_table('', '|') g(v); +select v, v is null as "is null" from string_to_table('1|2|3', NULL) g(v); +select v, v is null as "is null" from string_to_table(NULL, '|') g(v); +select v, v is null as "is null" from string_to_table('abc', '') g(v); +select v, v is null as "is null" from string_to_table('abc', '', 'abc') g(v); +select v, v is null as "is null" from string_to_table('abc', ',') g(v); +select v, v is null as "is null" from string_to_table('abc', ',', 'abc') g(v); +select v, v is null as "is null" from string_to_table('1,2,3,4,,6', ',') g(v); +select v, v is null as "is null" from string_to_table('1,2,3,4,,6', ',', '') g(v); +select v, v is null as "is null" from string_to_table('1,2,3,4,*,6', ',', '*') g(v); + select array_to_string(NULL::int4[], ',') IS NULL; select array_to_string('{}'::int4[], ','); select array_to_string(array[1,2,3,4,NULL,6], ','); From 464824323e57dc4b397e8b05854d779908b55304 Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Thu, 3 Sep 2020 07:54:07 +0530 Subject: [PATCH 055/589] Add support for streaming to built-in logical replication. To add support for streaming of in-progress transactions into the built-in logical replication, we need to do three things: * Extend the logical replication protocol, so identify in-progress transactions, and allow adding additional bits of information (e.g. XID of subtransactions). * Modify the output plugin (pgoutput) to implement the new stream API callbacks, by leveraging the extended replication protocol. * Modify the replication apply worker, to properly handle streamed in-progress transaction by spilling the data to disk and then replaying them on commit. We however must explicitly disable streaming replication during replication slot creation, even if the plugin supports it. We don't need to replicate the changes accumulated during this phase, and moreover we don't have a replication connection open so we don't have where to send the data anyway. Author: Tomas Vondra, Dilip Kumar and Amit Kapila Reviewed-by: Amit Kapila, Kuntal Ghosh and Ajin Cherian Tested-by: Neha Sharma, Mahendra Singh Thalor and Ajin Cherian Discussion: https://postgr.es/m/688b0b7f-2f6c-d827-c27b-216a8e3ea700@2ndquadrant.com --- doc/src/sgml/monitoring.sgml | 16 + doc/src/sgml/ref/alter_subscription.sgml | 5 +- doc/src/sgml/ref/create_subscription.sgml | 11 + src/backend/catalog/pg_subscription.c | 1 + src/backend/catalog/system_views.sql | 2 +- src/backend/commands/subscriptioncmds.c | 46 +- src/backend/postmaster/pgstat.c | 12 + .../libpqwalreceiver/libpqwalreceiver.c | 4 + src/backend/replication/logical/proto.c | 162 ++- src/backend/replication/logical/worker.c | 952 +++++++++++++++++- src/backend/replication/pgoutput/pgoutput.c | 367 ++++++- src/bin/pg_dump/pg_dump.c | 18 +- src/bin/pg_dump/pg_dump.h | 1 + src/bin/psql/describe.c | 10 +- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_subscription.h | 3 + src/include/pgstat.h | 6 +- src/include/replication/logicalproto.h | 42 +- src/include/replication/walreceiver.h | 1 + src/test/regress/expected/subscription.out | 63 +- src/test/regress/sql/subscription.sql | 15 + src/test/subscription/t/015_stream.pl | 98 ++ src/tools/pgindent/typedefs.list | 3 + 23 files changed, 1766 insertions(+), 74 deletions(-) create mode 100644 src/test/subscription/t/015_stream.pl diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index d973e1149aa9..673a0e73e453 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -1509,6 +1509,22 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser WALWrite Waiting for a write to a WAL file. + + LogicalChangesRead + Waiting for a read from a logical changes file. + + + LogicalChangesWrite + Waiting for a write to a logical changes file. + + + LogicalSubxactRead + Waiting for a read from a logical subxact file. + + + LogicalSubxactWrite + Waiting for a write to a logical subxact file. +
diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index 81c4e70cdf45..a1666b370be9 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -165,8 +165,9 @@ ALTER SUBSCRIPTION name RENAME TO < . See there for more information. The parameters that can be altered are slot_name, - synchronous_commit, and - binary. + synchronous_commit, + binary, and + streaming. diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index cdb22c54feab..b7d7457d004e 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -228,6 +228,17 @@ CREATE SUBSCRIPTION subscription_name + + streaming (boolean) + + + Specifies whether streaming of in-progress transactions should + be enabled for this subscription. By default, all transactions + are fully decoded on the publisher, and only then sent to the + subscriber as a whole. + + + diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c index 90bf5cf0c6de..311d46225adf 100644 --- a/src/backend/catalog/pg_subscription.c +++ b/src/backend/catalog/pg_subscription.c @@ -66,6 +66,7 @@ GetSubscription(Oid subid, bool missing_ok) sub->owner = subform->subowner; sub->enabled = subform->subenabled; sub->binary = subform->subbinary; + sub->stream = subform->substream; /* Get conninfo */ datum = SysCacheGetAttr(SUBSCRIPTIONOID, diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index a2d61302f9e8..ed4f3f142d87 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1128,7 +1128,7 @@ REVOKE ALL ON pg_replication_origin_status FROM public; -- All columns of pg_subscription except subconninfo are readable. REVOKE ALL ON pg_subscription FROM public; -GRANT SELECT (subdbid, subname, subowner, subenabled, subbinary, subslotname, subpublications) +GRANT SELECT (subdbid, subname, subowner, subenabled, subbinary, substream, subslotname, subpublications) ON pg_subscription TO public; diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index 40b6377a8522..1696454c0bbb 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -63,7 +63,8 @@ parse_subscription_options(List *options, bool *copy_data, char **synchronous_commit, bool *refresh, - bool *binary_given, bool *binary) + bool *binary_given, bool *binary, + bool *streaming_given, bool *streaming) { ListCell *lc; bool connect_given = false; @@ -99,6 +100,11 @@ parse_subscription_options(List *options, *binary_given = false; *binary = false; } + if (streaming) + { + *streaming_given = false; + *streaming = false; + } /* Parse options */ foreach(lc, options) @@ -194,6 +200,16 @@ parse_subscription_options(List *options, *binary_given = true; *binary = defGetBoolean(defel); } + else if (strcmp(defel->defname, "streaming") == 0 && streaming) + { + if (*streaming_given) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + + *streaming_given = true; + *streaming = defGetBoolean(defel); + } else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), @@ -337,6 +353,8 @@ CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) bool enabled_given; bool enabled; bool copy_data; + bool streaming; + bool streaming_given; char *synchronous_commit; char *conninfo; char *slotname; @@ -360,7 +378,8 @@ CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) ©_data, &synchronous_commit, NULL, /* no "refresh" */ - &binary_given, &binary); + &binary_given, &binary, + &streaming_given, &streaming); /* * Since creating a replication slot is not transactional, rolling back @@ -427,6 +446,7 @@ CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) values[Anum_pg_subscription_subowner - 1] = ObjectIdGetDatum(owner); values[Anum_pg_subscription_subenabled - 1] = BoolGetDatum(enabled); values[Anum_pg_subscription_subbinary - 1] = BoolGetDatum(binary); + values[Anum_pg_subscription_substream - 1] = BoolGetDatum(streaming); values[Anum_pg_subscription_subconninfo - 1] = CStringGetTextDatum(conninfo); if (slotname) @@ -698,6 +718,8 @@ AlterSubscription(AlterSubscriptionStmt *stmt) char *synchronous_commit; bool binary_given; bool binary; + bool streaming_given; + bool streaming; parse_subscription_options(stmt->options, NULL, /* no "connect" */ @@ -707,7 +729,8 @@ AlterSubscription(AlterSubscriptionStmt *stmt) NULL, /* no "copy_data" */ &synchronous_commit, NULL, /* no "refresh" */ - &binary_given, &binary); + &binary_given, &binary, + &streaming_given, &streaming); if (slotname_given) { @@ -739,6 +762,13 @@ AlterSubscription(AlterSubscriptionStmt *stmt) replaces[Anum_pg_subscription_subbinary - 1] = true; } + if (streaming_given) + { + values[Anum_pg_subscription_substream - 1] = + BoolGetDatum(streaming); + replaces[Anum_pg_subscription_substream - 1] = true; + } + update_tuple = true; break; } @@ -756,7 +786,8 @@ AlterSubscription(AlterSubscriptionStmt *stmt) NULL, /* no "copy_data" */ NULL, /* no "synchronous_commit" */ NULL, /* no "refresh" */ - NULL, NULL); /* no "binary" */ + NULL, NULL, /* no "binary" */ + NULL, NULL); /* no streaming */ Assert(enabled_given); if (!sub->slotname && enabled) @@ -800,8 +831,8 @@ AlterSubscription(AlterSubscriptionStmt *stmt) ©_data, NULL, /* no "synchronous_commit" */ &refresh, - NULL, NULL); /* no "binary" */ - + NULL, NULL, /* no "binary" */ + NULL, NULL); /* no "streaming" */ values[Anum_pg_subscription_subpublications - 1] = publicationListToArray(stmt->publication); replaces[Anum_pg_subscription_subpublications - 1] = true; @@ -843,7 +874,8 @@ AlterSubscription(AlterSubscriptionStmt *stmt) ©_data, NULL, /* no "synchronous_commit" */ NULL, /* no "refresh" */ - NULL, NULL); /* no "binary" */ + NULL, NULL, /* no "binary" */ + NULL, NULL); /* no "streaming" */ AlterSubscription_refresh(sub, copy_data); diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 8116b2361430..5f4b168fd16b 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -4141,6 +4141,18 @@ pgstat_get_wait_io(WaitEventIO w) case WAIT_EVENT_WAL_WRITE: event_name = "WALWrite"; break; + case WAIT_EVENT_LOGICAL_CHANGES_READ: + event_name = "LogicalChangesRead"; + break; + case WAIT_EVENT_LOGICAL_CHANGES_WRITE: + event_name = "LogicalChangesWrite"; + break; + case WAIT_EVENT_LOGICAL_SUBXACT_READ: + event_name = "LogicalSubxactRead"; + break; + case WAIT_EVENT_LOGICAL_SUBXACT_WRITE: + event_name = "LogicalSubxactWrite"; + break; /* no default case, so that compiler will warn */ } diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 8afa5a29b484..ad574099ff70 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -425,6 +425,10 @@ libpqrcv_startstreaming(WalReceiverConn *conn, appendStringInfo(&cmd, "proto_version '%u'", options->proto.logical.proto_version); + if (options->proto.logical.streaming && + PQserverVersion(conn->streamConn) >= 140000) + appendStringInfo(&cmd, ", streaming 'on'"); + pubnames = options->proto.logical.publication_names; pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames); if (!pubnames_str) diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c index 9ff8097bf5fd..eb19142b4865 100644 --- a/src/backend/replication/logical/proto.c +++ b/src/backend/replication/logical/proto.c @@ -138,10 +138,15 @@ logicalrep_read_origin(StringInfo in, XLogRecPtr *origin_lsn) * Write INSERT to the output stream. */ void -logicalrep_write_insert(StringInfo out, Relation rel, HeapTuple newtuple, bool binary) +logicalrep_write_insert(StringInfo out, TransactionId xid, Relation rel, + HeapTuple newtuple, bool binary) { pq_sendbyte(out, 'I'); /* action INSERT */ + /* transaction ID (if not valid, we're not streaming) */ + if (TransactionIdIsValid(xid)) + pq_sendint32(out, xid); + /* use Oid as relation identifier */ pq_sendint32(out, RelationGetRelid(rel)); @@ -177,8 +182,8 @@ logicalrep_read_insert(StringInfo in, LogicalRepTupleData *newtup) * Write UPDATE to the output stream. */ void -logicalrep_write_update(StringInfo out, Relation rel, HeapTuple oldtuple, - HeapTuple newtuple, bool binary) +logicalrep_write_update(StringInfo out, TransactionId xid, Relation rel, + HeapTuple oldtuple, HeapTuple newtuple, bool binary) { pq_sendbyte(out, 'U'); /* action UPDATE */ @@ -186,6 +191,10 @@ logicalrep_write_update(StringInfo out, Relation rel, HeapTuple oldtuple, rel->rd_rel->relreplident == REPLICA_IDENTITY_FULL || rel->rd_rel->relreplident == REPLICA_IDENTITY_INDEX); + /* transaction ID (if not valid, we're not streaming) */ + if (TransactionIdIsValid(xid)) + pq_sendint32(out, xid); + /* use Oid as relation identifier */ pq_sendint32(out, RelationGetRelid(rel)); @@ -247,7 +256,8 @@ logicalrep_read_update(StringInfo in, bool *has_oldtuple, * Write DELETE to the output stream. */ void -logicalrep_write_delete(StringInfo out, Relation rel, HeapTuple oldtuple, bool binary) +logicalrep_write_delete(StringInfo out, TransactionId xid, Relation rel, + HeapTuple oldtuple, bool binary) { Assert(rel->rd_rel->relreplident == REPLICA_IDENTITY_DEFAULT || rel->rd_rel->relreplident == REPLICA_IDENTITY_FULL || @@ -255,6 +265,10 @@ logicalrep_write_delete(StringInfo out, Relation rel, HeapTuple oldtuple, bool b pq_sendbyte(out, 'D'); /* action DELETE */ + /* transaction ID (if not valid, we're not streaming) */ + if (TransactionIdIsValid(xid)) + pq_sendint32(out, xid); + /* use Oid as relation identifier */ pq_sendint32(out, RelationGetRelid(rel)); @@ -295,6 +309,7 @@ logicalrep_read_delete(StringInfo in, LogicalRepTupleData *oldtup) */ void logicalrep_write_truncate(StringInfo out, + TransactionId xid, int nrelids, Oid relids[], bool cascade, bool restart_seqs) @@ -304,6 +319,10 @@ logicalrep_write_truncate(StringInfo out, pq_sendbyte(out, 'T'); /* action TRUNCATE */ + /* transaction ID (if not valid, we're not streaming) */ + if (TransactionIdIsValid(xid)) + pq_sendint32(out, xid); + pq_sendint32(out, nrelids); /* encode and send truncate flags */ @@ -346,12 +365,16 @@ logicalrep_read_truncate(StringInfo in, * Write relation description to the output stream. */ void -logicalrep_write_rel(StringInfo out, Relation rel) +logicalrep_write_rel(StringInfo out, TransactionId xid, Relation rel) { char *relname; pq_sendbyte(out, 'R'); /* sending RELATION */ + /* transaction ID (if not valid, we're not streaming) */ + if (TransactionIdIsValid(xid)) + pq_sendint32(out, xid); + /* use Oid as relation identifier */ pq_sendint32(out, RelationGetRelid(rel)); @@ -396,7 +419,7 @@ logicalrep_read_rel(StringInfo in) * This function will always write base type info. */ void -logicalrep_write_typ(StringInfo out, Oid typoid) +logicalrep_write_typ(StringInfo out, TransactionId xid, Oid typoid) { Oid basetypoid = getBaseType(typoid); HeapTuple tup; @@ -404,6 +427,10 @@ logicalrep_write_typ(StringInfo out, Oid typoid) pq_sendbyte(out, 'Y'); /* sending TYPE */ + /* transaction ID (if not valid, we're not streaming) */ + if (TransactionIdIsValid(xid)) + pq_sendint32(out, xid); + tup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(basetypoid)); if (!HeapTupleIsValid(tup)) elog(ERROR, "cache lookup failed for type %u", basetypoid); @@ -720,3 +747,126 @@ logicalrep_read_namespace(StringInfo in) return nspname; } + +/* + * Write the information for the start stream message to the output stream. + */ +void +logicalrep_write_stream_start(StringInfo out, + TransactionId xid, bool first_segment) +{ + pq_sendbyte(out, 'S'); /* action STREAM START */ + + Assert(TransactionIdIsValid(xid)); + + /* transaction ID (we're starting to stream, so must be valid) */ + pq_sendint32(out, xid); + + /* 1 if this is the first streaming segment for this xid */ + pq_sendbyte(out, first_segment ? 1 : 0); +} + +/* + * Read the information about the start stream message from output stream. + */ +TransactionId +logicalrep_read_stream_start(StringInfo in, bool *first_segment) +{ + TransactionId xid; + + Assert(first_segment); + + xid = pq_getmsgint(in, 4); + *first_segment = (pq_getmsgbyte(in) == 1); + + return xid; +} + +/* + * Write the stop stream message to the output stream. + */ +void +logicalrep_write_stream_stop(StringInfo out) +{ + pq_sendbyte(out, 'E'); /* action STREAM END */ +} + +/* + * Write STREAM COMMIT to the output stream. + */ +void +logicalrep_write_stream_commit(StringInfo out, ReorderBufferTXN *txn, + XLogRecPtr commit_lsn) +{ + uint8 flags = 0; + + pq_sendbyte(out, 'c'); /* action STREAM COMMIT */ + + Assert(TransactionIdIsValid(txn->xid)); + + /* transaction ID */ + pq_sendint32(out, txn->xid); + + /* send the flags field (unused for now) */ + pq_sendbyte(out, flags); + + /* send fields */ + pq_sendint64(out, commit_lsn); + pq_sendint64(out, txn->end_lsn); + pq_sendint64(out, txn->commit_time); +} + +/* + * Read STREAM COMMIT from the output stream. + */ +TransactionId +logicalrep_read_stream_commit(StringInfo in, LogicalRepCommitData *commit_data) +{ + TransactionId xid; + uint8 flags; + + xid = pq_getmsgint(in, 4); + + /* read flags (unused for now) */ + flags = pq_getmsgbyte(in); + + if (flags != 0) + elog(ERROR, "unrecognized flags %u in commit message", flags); + + /* read fields */ + commit_data->commit_lsn = pq_getmsgint64(in); + commit_data->end_lsn = pq_getmsgint64(in); + commit_data->committime = pq_getmsgint64(in); + + return xid; +} + +/* + * Write STREAM ABORT to the output stream. Note that xid and subxid will be + * same for the top-level transaction abort. + */ +void +logicalrep_write_stream_abort(StringInfo out, TransactionId xid, + TransactionId subxid) +{ + pq_sendbyte(out, 'A'); /* action STREAM ABORT */ + + Assert(TransactionIdIsValid(xid) && TransactionIdIsValid(subxid)); + + /* transaction ID */ + pq_sendint32(out, xid); + pq_sendint32(out, subxid); +} + +/* + * Read STREAM ABORT from the output stream. + */ +void +logicalrep_read_stream_abort(StringInfo in, TransactionId *xid, + TransactionId *subxid) +{ + Assert(xid && subxid); + + *xid = pq_getmsgint(in, 4); + *subxid = pq_getmsgint(in, 4); +} diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index b576e342cb7d..812aca80112c 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -18,11 +18,45 @@ * This module includes server facing code and shares libpqwalreceiver * module with walreceiver for providing the libpq specific functionality. * + * + * STREAMED TRANSACTIONS + * --------------------- + * Streamed transactions (large transactions exceeding a memory limit on the + * upstream) are not applied immediately, but instead, the data is written + * to temporary files and then applied at once when the final commit arrives. + * + * Unlike the regular (non-streamed) case, handling streamed transactions has + * to handle aborts of both the toplevel transaction and subtransactions. This + * is achieved by tracking offsets for subtransactions, which is then used + * to truncate the file with serialized changes. + * + * The files are placed in tmp file directory by default, and the filenames + * include both the XID of the toplevel transaction and OID of the + * subscription. This is necessary so that different workers processing a + * remote transaction with the same XID doesn't interfere. + * + * We use BufFiles instead of using normal temporary files because (a) the + * BufFile infrastructure supports temporary files that exceed the OS file size + * limit, (b) provides a way for automatic clean up on the error and (c) provides + * a way to survive these files across local transactions and allow to open and + * close at stream start and close. We decided to use SharedFileSet + * infrastructure as without that it deletes the files on the closure of the + * file and if we decide to keep stream files open across the start/stop stream + * then it will consume a lot of memory (more than 8K for each BufFile and + * there could be multiple such BufFiles as the subscriber could receive + * multiple start/stop streams for different transactions before getting the + * commit). Moreover, if we don't use SharedFileSet then we also need to invent + * a new way to pass filenames to BufFile APIs so that we are allowed to open + * the file we desired across multiple stream-open calls for the same + * transaction. *------------------------------------------------------------------------- */ #include "postgres.h" +#include +#include + #include "access/table.h" #include "access/tableam.h" #include "access/xact.h" @@ -33,7 +67,9 @@ #include "catalog/pg_inherits.h" #include "catalog/pg_subscription.h" #include "catalog/pg_subscription_rel.h" +#include "catalog/pg_tablespace.h" #include "commands/tablecmds.h" +#include "commands/tablespace.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/execPartition.h" @@ -63,7 +99,9 @@ #include "replication/walreceiver.h" #include "replication/worker_internal.h" #include "rewrite/rewriteHandler.h" +#include "storage/buffile.h" #include "storage/bufmgr.h" +#include "storage/fd.h" #include "storage/ipc.h" #include "storage/lmgr.h" #include "storage/proc.h" @@ -71,6 +109,7 @@ #include "tcop/tcopprot.h" #include "utils/builtins.h" #include "utils/catcache.h" +#include "utils/dynahash.h" #include "utils/datum.h" #include "utils/fmgroids.h" #include "utils/guc.h" @@ -99,9 +138,26 @@ typedef struct SlotErrCallbackArg int remote_attnum; } SlotErrCallbackArg; +/* + * Stream xid hash entry. Whenever we see a new xid we create this entry in the + * xidhash and along with it create the streaming file and store the fileset handle. + * The subxact file is created iff there is any subxact info under this xid. This + * entry is used on the subsequent streams for the xid to get the corresponding + * fileset handles, so storing them in hash makes the search faster. + */ +typedef struct StreamXidHash +{ + TransactionId xid; /* xid is the hash key and must be first */ + SharedFileSet *stream_fileset; /* shared file set for stream data */ + SharedFileSet *subxact_fileset; /* shared file set for subxact info */ +} StreamXidHash; + static MemoryContext ApplyMessageContext = NULL; MemoryContext ApplyContext = NULL; +/* per stream context for streaming transactions */ +static MemoryContext LogicalStreamingContext = NULL; + WalReceiverConn *wrconn = NULL; Subscription *MySubscription = NULL; @@ -110,12 +166,66 @@ bool MySubscriptionValid = false; bool in_remote_transaction = false; static XLogRecPtr remote_final_lsn = InvalidXLogRecPtr; +/* fields valid only when processing streamed transaction */ +bool in_streamed_transaction = false; + +static TransactionId stream_xid = InvalidTransactionId; + +/* + * Hash table for storing the streaming xid information along with shared file + * set for streaming and subxact files. + */ +static HTAB *xidhash = NULL; + +/* BufFile handle of the current streaming file */ +static BufFile *stream_fd = NULL; + +typedef struct SubXactInfo +{ + TransactionId xid; /* XID of the subxact */ + int fileno; /* file number in the buffile */ + off_t offset; /* offset in the file */ +} SubXactInfo; + +/* Sub-transaction data for the current streaming transaction */ +typedef struct ApplySubXactData +{ + uint32 nsubxacts; /* number of sub-transactions */ + uint32 nsubxacts_max; /* current capacity of subxacts */ + TransactionId subxact_last; /* xid of the last sub-transaction */ + SubXactInfo *subxacts; /* sub-xact offset in changes file */ +} ApplySubXactData; + +static ApplySubXactData subxact_data = {0, 0, InvalidTransactionId, NULL}; + +static void subxact_filename(char *path, Oid subid, TransactionId xid); +static void changes_filename(char *path, Oid subid, TransactionId xid); + +/* + * Information about subtransactions of a given toplevel transaction. + */ +static void subxact_info_write(Oid subid, TransactionId xid); +static void subxact_info_read(Oid subid, TransactionId xid); +static void subxact_info_add(TransactionId xid); +static inline void cleanup_subxact_info(void); + +/* + * Serialize and deserialize changes for a toplevel transaction. + */ +static void stream_cleanup_files(Oid subid, TransactionId xid); +static void stream_open_file(Oid subid, TransactionId xid, bool first); +static void stream_write_change(char action, StringInfo s); +static void stream_close_file(void); + static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply); static void store_flush_position(XLogRecPtr remote_lsn); static void maybe_reread_subscription(void); +/* prototype needed because of stream_commit */ +static void apply_dispatch(StringInfo s); + static void apply_handle_insert_internal(ResultRelInfo *relinfo, EState *estate, TupleTableSlot *remoteslot); static void apply_handle_update_internal(ResultRelInfo *relinfo, @@ -187,6 +297,42 @@ ensure_transaction(void) return true; } +/* + * Handle streamed transactions. + * + * If in streaming mode (receiving a block of streamed transaction), we + * simply redirect it to a file for the proper toplevel transaction. + * + * Returns true for streamed transactions, false otherwise (regular mode). + */ +static bool +handle_streamed_transaction(const char action, StringInfo s) +{ + TransactionId xid; + + /* not in streaming mode */ + if (!in_streamed_transaction) + return false; + + Assert(stream_fd != NULL); + Assert(TransactionIdIsValid(stream_xid)); + + /* + * We should have received XID of the subxact as the first part of the + * message, so extract it. + */ + xid = pq_getmsgint(s, 4); + + Assert(TransactionIdIsValid(xid)); + + /* Add the new subxact to the array (unless already there). */ + subxact_info_add(xid); + + /* write the change to the current file */ + stream_write_change(action, s); + + return true; +} /* * Executor state preparation for evaluation of constraint expressions, @@ -612,16 +758,335 @@ static void apply_handle_origin(StringInfo s) { /* - * ORIGIN message can only come inside remote transaction and before any - * actual writes. + * ORIGIN message can only come inside streaming transaction or inside + * remote transaction and before any actual writes. */ - if (!in_remote_transaction || - (IsTransactionState() && !am_tablesync_worker())) + if (!in_streamed_transaction && + (!in_remote_transaction || + (IsTransactionState() && !am_tablesync_worker()))) ereport(ERROR, (errcode(ERRCODE_PROTOCOL_VIOLATION), errmsg("ORIGIN message sent out of order"))); } +/* + * Handle STREAM START message. + */ +static void +apply_handle_stream_start(StringInfo s) +{ + bool first_segment; + HASHCTL hash_ctl; + + Assert(!in_streamed_transaction); + + /* + * Start a transaction on stream start, this transaction will be committed + * on the stream stop. We need the transaction for handling the buffile, + * used for serializing the streaming data and subxact info. + */ + ensure_transaction(); + + /* notify handle methods we're processing a remote transaction */ + in_streamed_transaction = true; + + /* extract XID of the top-level transaction */ + stream_xid = logicalrep_read_stream_start(s, &first_segment); + + /* + * Initialize the xidhash table if we haven't yet. This will be used for + * the entire duration of the apply worker so create it in permanent + * context. + */ + if (xidhash == NULL) + { + hash_ctl.keysize = sizeof(TransactionId); + hash_ctl.entrysize = sizeof(StreamXidHash); + hash_ctl.hcxt = ApplyContext; + xidhash = hash_create("StreamXidHash", 1024, &hash_ctl, + HASH_ELEM | HASH_CONTEXT); + } + + /* open the spool file for this transaction */ + stream_open_file(MyLogicalRepWorker->subid, stream_xid, first_segment); + + /* if this is not the first segment, open existing subxact file */ + if (!first_segment) + subxact_info_read(MyLogicalRepWorker->subid, stream_xid); + + pgstat_report_activity(STATE_RUNNING, NULL); +} + +/* + * Handle STREAM STOP message. + */ +static void +apply_handle_stream_stop(StringInfo s) +{ + Assert(in_streamed_transaction); + + /* + * Close the file with serialized changes, and serialize information about + * subxacts for the toplevel transaction. + */ + subxact_info_write(MyLogicalRepWorker->subid, stream_xid); + stream_close_file(); + + /* We must be in a valid transaction state */ + Assert(IsTransactionState()); + + /* Commit the per-stream transaction */ + CommitTransactionCommand(); + + in_streamed_transaction = false; + + /* Reset per-stream context */ + MemoryContextReset(LogicalStreamingContext); + + pgstat_report_activity(STATE_IDLE, NULL); +} + +/* + * Handle STREAM abort message. + */ +static void +apply_handle_stream_abort(StringInfo s) +{ + TransactionId xid; + TransactionId subxid; + + Assert(!in_streamed_transaction); + + logicalrep_read_stream_abort(s, &xid, &subxid); + + /* + * If the two XIDs are the same, it's in fact abort of toplevel xact, so + * just delete the files with serialized info. + */ + if (xid == subxid) + stream_cleanup_files(MyLogicalRepWorker->subid, xid); + else + { + /* + * OK, so it's a subxact. We need to read the subxact file for the + * toplevel transaction, determine the offset tracked for the subxact, + * and truncate the file with changes. We also remove the subxacts + * with higher offsets (or rather higher XIDs). + * + * We intentionally scan the array from the tail, because we're likely + * aborting a change for the most recent subtransactions. + * + * We can't use the binary search here as subxact XIDs won't + * necessarily arrive in sorted order, consider the case where we have + * released the savepoint for multiple subtransactions and then + * performed rollback to savepoint for one of the earlier + * sub-transaction. + */ + + int64 i; + int64 subidx; + BufFile *fd; + bool found = false; + char path[MAXPGPATH]; + StreamXidHash *ent; + + subidx = -1; + ensure_transaction(); + subxact_info_read(MyLogicalRepWorker->subid, xid); + + for (i = subxact_data.nsubxacts; i > 0; i--) + { + if (subxact_data.subxacts[i - 1].xid == subxid) + { + subidx = (i - 1); + found = true; + break; + } + } + + /* + * If it's an empty sub-transaction then we will not find the subxid + * here so just cleanup the subxact info and return. + */ + if (!found) + { + /* Cleanup the subxact info */ + cleanup_subxact_info(); + CommitTransactionCommand(); + return; + } + + Assert((subidx >= 0) && (subidx < subxact_data.nsubxacts)); + + ent = (StreamXidHash *) hash_search(xidhash, + (void *) &xid, + HASH_FIND, + &found); + Assert(found); + + /* open the changes file */ + changes_filename(path, MyLogicalRepWorker->subid, xid); + fd = BufFileOpenShared(ent->stream_fileset, path, O_RDWR); + + /* OK, truncate the file at the right offset */ + BufFileTruncateShared(fd, subxact_data.subxacts[subidx].fileno, + subxact_data.subxacts[subidx].offset); + BufFileClose(fd); + + /* discard the subxacts added later */ + subxact_data.nsubxacts = subidx; + + /* write the updated subxact list */ + subxact_info_write(MyLogicalRepWorker->subid, xid); + CommitTransactionCommand(); + } +} + +/* + * Handle STREAM COMMIT message. + */ +static void +apply_handle_stream_commit(StringInfo s) +{ + TransactionId xid; + StringInfoData s2; + int nchanges; + char path[MAXPGPATH]; + char *buffer = NULL; + bool found; + LogicalRepCommitData commit_data; + StreamXidHash *ent; + MemoryContext oldcxt; + BufFile *fd; + + Assert(!in_streamed_transaction); + + xid = logicalrep_read_stream_commit(s, &commit_data); + + elog(DEBUG1, "received commit for streamed transaction %u", xid); + + ensure_transaction(); + + /* + * Allocate file handle and memory required to process all the messages in + * TopTransactionContext to avoid them getting reset after each message is + * processed. + */ + oldcxt = MemoryContextSwitchTo(TopTransactionContext); + + /* open the spool file for the committed transaction */ + changes_filename(path, MyLogicalRepWorker->subid, xid); + elog(DEBUG1, "replaying changes from file \"%s\"", path); + ent = (StreamXidHash *) hash_search(xidhash, + (void *) &xid, + HASH_FIND, + &found); + Assert(found); + fd = BufFileOpenShared(ent->stream_fileset, path, O_RDONLY); + + buffer = palloc(BLCKSZ); + initStringInfo(&s2); + + MemoryContextSwitchTo(oldcxt); + + remote_final_lsn = commit_data.commit_lsn; + + /* + * Make sure the handle apply_dispatch methods are aware we're in a remote + * transaction. + */ + in_remote_transaction = true; + pgstat_report_activity(STATE_RUNNING, NULL); + + /* + * Read the entries one by one and pass them through the same logic as in + * apply_dispatch. + */ + nchanges = 0; + while (true) + { + int nbytes; + int len; + + CHECK_FOR_INTERRUPTS(); + + /* read length of the on-disk record */ + nbytes = BufFileRead(fd, &len, sizeof(len)); + + /* have we reached end of the file? */ + if (nbytes == 0) + break; + + /* do we have a correct length? */ + if (nbytes != sizeof(len)) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read from streaming transaction's changes file \"%s\": %m", + path))); + + Assert(len > 0); + + /* make sure we have sufficiently large buffer */ + buffer = repalloc(buffer, len); + + /* and finally read the data into the buffer */ + if (BufFileRead(fd, buffer, len) != len) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read from streaming transaction's changes file \"%s\": %m", + path))); + + /* copy the buffer to the stringinfo and call apply_dispatch */ + resetStringInfo(&s2); + appendBinaryStringInfo(&s2, buffer, len); + + /* Ensure we are reading the data into our memory context. */ + oldcxt = MemoryContextSwitchTo(ApplyMessageContext); + + apply_dispatch(&s2); + + MemoryContextReset(ApplyMessageContext); + + MemoryContextSwitchTo(oldcxt); + + nchanges++; + + if (nchanges % 1000 == 0) + elog(DEBUG1, "replayed %d changes from file '%s'", + nchanges, path); + } + + BufFileClose(fd); + + /* + * Update origin state so we can restart streaming from correct position + * in case of crash. + */ + replorigin_session_origin_lsn = commit_data.end_lsn; + replorigin_session_origin_timestamp = commit_data.committime; + + pfree(buffer); + pfree(s2.data); + + CommitTransactionCommand(); + pgstat_report_stat(false); + + store_flush_position(commit_data.end_lsn); + + elog(DEBUG1, "replayed %d (all) changes from file \"%s\"", + nchanges, path); + + in_remote_transaction = false; + + /* Process any tables that are being synchronized in parallel. */ + process_syncing_tables(commit_data.end_lsn); + + /* unlink the files with serialized changes and subxact info */ + stream_cleanup_files(MyLogicalRepWorker->subid, xid); + + pgstat_report_activity(STATE_IDLE, NULL); +} + /* * Handle RELATION message. * @@ -635,6 +1100,9 @@ apply_handle_relation(StringInfo s) { LogicalRepRelation *rel; + if (handle_streamed_transaction('R', s)) + return; + rel = logicalrep_read_rel(s); logicalrep_relmap_update(rel); } @@ -650,6 +1118,9 @@ apply_handle_type(StringInfo s) { LogicalRepTyp typ; + if (handle_streamed_transaction('Y', s)) + return; + logicalrep_read_typ(s, &typ); logicalrep_typmap_update(&typ); } @@ -686,6 +1157,9 @@ apply_handle_insert(StringInfo s) TupleTableSlot *remoteslot; MemoryContext oldctx; + if (handle_streamed_transaction('I', s)) + return; + ensure_transaction(); relid = logicalrep_read_insert(s, &newtup); @@ -801,6 +1275,9 @@ apply_handle_update(StringInfo s) RangeTblEntry *target_rte; MemoryContext oldctx; + if (handle_streamed_transaction('U', s)) + return; + ensure_transaction(); relid = logicalrep_read_update(s, &has_oldtup, &oldtup, @@ -950,6 +1427,9 @@ apply_handle_delete(StringInfo s) TupleTableSlot *remoteslot; MemoryContext oldctx; + if (handle_streamed_transaction('D', s)) + return; + ensure_transaction(); relid = logicalrep_read_delete(s, &oldtup); @@ -1320,6 +1800,9 @@ apply_handle_truncate(StringInfo s) List *relids_logged = NIL; ListCell *lc; + if (handle_streamed_transaction('T', s)) + return; + ensure_transaction(); remote_relids = logicalrep_read_truncate(s, &cascade, &restart_seqs); @@ -1458,6 +1941,22 @@ apply_dispatch(StringInfo s) case 'O': apply_handle_origin(s); break; + /* STREAM START */ + case 'S': + apply_handle_stream_start(s); + break; + /* STREAM END */ + case 'E': + apply_handle_stream_stop(s); + break; + /* STREAM ABORT */ + case 'A': + apply_handle_stream_abort(s); + break; + /* STREAM COMMIT */ + case 'c': + apply_handle_stream_commit(s); + break; default: ereport(ERROR, (errcode(ERRCODE_PROTOCOL_VIOLATION), @@ -1570,6 +2069,14 @@ LogicalRepApplyLoop(XLogRecPtr last_received) "ApplyMessageContext", ALLOCSET_DEFAULT_SIZES); + /* + * This memory context is used for per-stream data when the streaming mode + * is enabled. This context is reset on each stream stop. + */ + LogicalStreamingContext = AllocSetContextCreate(ApplyContext, + "LogicalStreamingContext", + ALLOCSET_DEFAULT_SIZES); + /* mark as idle, before starting to loop */ pgstat_report_activity(STATE_IDLE, NULL); @@ -1674,7 +2181,7 @@ LogicalRepApplyLoop(XLogRecPtr last_received) /* confirm all writes so far */ send_feedback(last_received, false, false); - if (!in_remote_transaction) + if (!in_remote_transaction && !in_streamed_transaction) { /* * If we didn't get any transactions for a while there might be @@ -1938,6 +2445,7 @@ maybe_reread_subscription(void) strcmp(newsub->name, MySubscription->name) != 0 || strcmp(newsub->slotname, MySubscription->slotname) != 0 || newsub->binary != MySubscription->binary || + newsub->stream != MySubscription->stream || !equal(newsub->publications, MySubscription->publications)) { ereport(LOG, @@ -1979,6 +2487,439 @@ subscription_change_cb(Datum arg, int cacheid, uint32 hashvalue) MySubscriptionValid = false; } +/* + * subxact_info_write + * Store information about subxacts for a toplevel transaction. + * + * For each subxact we store offset of it's first change in the main file. + * The file is always over-written as a whole. + * + * XXX We should only store subxacts that were not aborted yet. + */ +static void +subxact_info_write(Oid subid, TransactionId xid) +{ + char path[MAXPGPATH]; + bool found; + Size len; + StreamXidHash *ent; + BufFile *fd; + + Assert(TransactionIdIsValid(xid)); + + /* find the xid entry in the xidhash */ + ent = (StreamXidHash *) hash_search(xidhash, + (void *) &xid, + HASH_FIND, + &found); + /* we must found the entry for its top transaction by this time */ + Assert(found); + + /* + * If there is no subtransaction then nothing to do, but if already have + * subxact file then delete that. + */ + if (subxact_data.nsubxacts == 0) + { + if (ent->subxact_fileset) + { + cleanup_subxact_info(); + SharedFileSetDeleteAll(ent->subxact_fileset); + pfree(ent->subxact_fileset); + ent->subxact_fileset = NULL; + } + return; + } + + subxact_filename(path, subid, xid); + + /* + * Create the subxact file if it not already created, otherwise open the + * existing file. + */ + if (ent->subxact_fileset == NULL) + { + MemoryContext oldctx; + + /* + * We need to maintain shared fileset across multiple stream + * start/stop calls. So, need to allocate it in a persistent context. + */ + oldctx = MemoryContextSwitchTo(ApplyContext); + ent->subxact_fileset = palloc(sizeof(SharedFileSet)); + SharedFileSetInit(ent->subxact_fileset, NULL); + MemoryContextSwitchTo(oldctx); + + fd = BufFileCreateShared(ent->subxact_fileset, path); + } + else + fd = BufFileOpenShared(ent->subxact_fileset, path, O_RDWR); + + len = sizeof(SubXactInfo) * subxact_data.nsubxacts; + + /* Write the subxact count and subxact info */ + BufFileWrite(fd, &subxact_data.nsubxacts, sizeof(subxact_data.nsubxacts)); + BufFileWrite(fd, subxact_data.subxacts, len); + + BufFileClose(fd); + + /* free the memory allocated for subxact info */ + cleanup_subxact_info(); +} + +/* + * subxact_info_read + * Restore information about subxacts of a streamed transaction. + * + * Read information about subxacts into the structure subxact_data that can be + * used later. + */ +static void +subxact_info_read(Oid subid, TransactionId xid) +{ + char path[MAXPGPATH]; + bool found; + Size len; + BufFile *fd; + StreamXidHash *ent; + MemoryContext oldctx; + + Assert(TransactionIdIsValid(xid)); + Assert(!subxact_data.subxacts); + Assert(subxact_data.nsubxacts == 0); + Assert(subxact_data.nsubxacts_max == 0); + + /* Find the stream xid entry in the xidhash */ + ent = (StreamXidHash *) hash_search(xidhash, + (void *) &xid, + HASH_FIND, + &found); + + /* + * If subxact_fileset is not valid that mean we don't have any subxact + * info + */ + if (ent->subxact_fileset == NULL) + return; + + subxact_filename(path, subid, xid); + + fd = BufFileOpenShared(ent->subxact_fileset, path, O_RDONLY); + + /* read number of subxact items */ + if (BufFileRead(fd, &subxact_data.nsubxacts, + sizeof(subxact_data.nsubxacts)) != + sizeof(subxact_data.nsubxacts)) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read from streaming transaction's subxact file \"%s\": %m", + path))); + + len = sizeof(SubXactInfo) * subxact_data.nsubxacts; + + /* we keep the maximum as a power of 2 */ + subxact_data.nsubxacts_max = 1 << my_log2(subxact_data.nsubxacts); + + /* + * Allocate subxact information in the logical streaming context. We need + * this information during the complete stream so that we can add the sub + * transaction info to this. On stream stop we will flush this information + * to the subxact file and reset the logical streaming context. + */ + oldctx = MemoryContextSwitchTo(LogicalStreamingContext); + subxact_data.subxacts = palloc(subxact_data.nsubxacts_max * + sizeof(SubXactInfo)); + MemoryContextSwitchTo(oldctx); + + if ((len > 0) && ((BufFileRead(fd, subxact_data.subxacts, len)) != len)) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read from streaming transaction's subxact file \"%s\": %m", + path))); + + BufFileClose(fd); +} + +/* + * subxact_info_add + * Add information about a subxact (offset in the main file). + */ +static void +subxact_info_add(TransactionId xid) +{ + SubXactInfo *subxacts = subxact_data.subxacts; + int64 i; + + /* We must have a valid top level stream xid and a stream fd. */ + Assert(TransactionIdIsValid(stream_xid)); + Assert(stream_fd != NULL); + + /* + * If the XID matches the toplevel transaction, we don't want to add it. + */ + if (stream_xid == xid) + return; + + /* + * In most cases we're checking the same subxact as we've already seen in + * the last call, so make sure to ignore it (this change comes later). + */ + if (subxact_data.subxact_last == xid) + return; + + /* OK, remember we're processing this XID. */ + subxact_data.subxact_last = xid; + + /* + * Check if the transaction is already present in the array of subxact. We + * intentionally scan the array from the tail, because we're likely adding + * a change for the most recent subtransactions. + * + * XXX Can we rely on the subxact XIDs arriving in sorted order? That + * would allow us to use binary search here. + */ + for (i = subxact_data.nsubxacts; i > 0; i--) + { + /* found, so we're done */ + if (subxacts[i - 1].xid == xid) + return; + } + + /* This is a new subxact, so we need to add it to the array. */ + if (subxact_data.nsubxacts == 0) + { + MemoryContext oldctx; + + subxact_data.nsubxacts_max = 128; + + /* + * Allocate this memory for subxacts in per-stream context, see + * subxact_info_read. + */ + oldctx = MemoryContextSwitchTo(LogicalStreamingContext); + subxacts = palloc(subxact_data.nsubxacts_max * sizeof(SubXactInfo)); + MemoryContextSwitchTo(oldctx); + } + else if (subxact_data.nsubxacts == subxact_data.nsubxacts_max) + { + subxact_data.nsubxacts_max *= 2; + subxacts = repalloc(subxacts, + subxact_data.nsubxacts_max * sizeof(SubXactInfo)); + } + + subxacts[subxact_data.nsubxacts].xid = xid; + + /* + * Get the current offset of the stream file and store it as offset of + * this subxact. + */ + BufFileTell(stream_fd, + &subxacts[subxact_data.nsubxacts].fileno, + &subxacts[subxact_data.nsubxacts].offset); + + subxact_data.nsubxacts++; + subxact_data.subxacts = subxacts; +} + +/* format filename for file containing the info about subxacts */ +static void +subxact_filename(char *path, Oid subid, TransactionId xid) +{ + snprintf(path, MAXPGPATH, "%u-%u.subxacts", subid, xid); +} + +/* format filename for file containing serialized changes */ +static inline void +changes_filename(char *path, Oid subid, TransactionId xid) +{ + snprintf(path, MAXPGPATH, "%u-%u.changes", subid, xid); +} + +/* + * stream_cleanup_files + * Cleanup files for a subscription / toplevel transaction. + * + * Remove files with serialized changes and subxact info for a particular + * toplevel transaction. Each subscription has a separate set of files. + */ +static void +stream_cleanup_files(Oid subid, TransactionId xid) +{ + char path[MAXPGPATH]; + StreamXidHash *ent; + + /* Remove the xid entry from the stream xid hash */ + ent = (StreamXidHash *) hash_search(xidhash, + (void *) &xid, + HASH_REMOVE, + NULL); + /* By this time we must have created the transaction entry */ + Assert(ent != NULL); + + /* Delete the change file and release the stream fileset memory */ + changes_filename(path, subid, xid); + SharedFileSetDeleteAll(ent->stream_fileset); + pfree(ent->stream_fileset); + ent->stream_fileset = NULL; + + /* Delete the subxact file and release the memory, if it exist */ + if (ent->subxact_fileset) + { + subxact_filename(path, subid, xid); + SharedFileSetDeleteAll(ent->subxact_fileset); + pfree(ent->subxact_fileset); + ent->subxact_fileset = NULL; + } +} + +/* + * stream_open_file + * Open a file that we'll use to serialize changes for a toplevel + * transaction. + * + * Open a file for streamed changes from a toplevel transaction identified + * by stream_xid (global variable). If it's the first chunk of streamed + * changes for this transaction, initialize the shared fileset and create the + * buffile, otherwise open the previously created file. + * + * This can only be called at the beginning of a "streaming" block, i.e. + * between stream_start/stream_stop messages from the upstream. + */ +static void +stream_open_file(Oid subid, TransactionId xid, bool first_segment) +{ + char path[MAXPGPATH]; + bool found; + MemoryContext oldcxt; + StreamXidHash *ent; + + Assert(in_streamed_transaction); + Assert(OidIsValid(subid)); + Assert(TransactionIdIsValid(xid)); + Assert(stream_fd == NULL); + + /* create or find the xid entry in the xidhash */ + ent = (StreamXidHash *) hash_search(xidhash, + (void *) &xid, + HASH_ENTER | HASH_FIND, + &found); + Assert(first_segment || found); + changes_filename(path, subid, xid); + elog(DEBUG1, "opening file \"%s\" for streamed changes", path); + + /* + * Create/open the buffiles under the logical streaming context so that we + * have those files until stream stop. + */ + oldcxt = MemoryContextSwitchTo(LogicalStreamingContext); + + /* + * If this is the first streamed segment, the file must not exist, so make + * sure we're the ones creating it. Otherwise just open the file for + * writing, in append mode. + */ + if (first_segment) + { + MemoryContext savectx; + SharedFileSet *fileset; + + /* + * We need to maintain shared fileset across multiple stream + * start/stop calls. So, need to allocate it in a persistent context. + */ + savectx = MemoryContextSwitchTo(ApplyContext); + fileset = palloc(sizeof(SharedFileSet)); + + SharedFileSetInit(fileset, NULL); + MemoryContextSwitchTo(savectx); + + stream_fd = BufFileCreateShared(fileset, path); + + /* Remember the fileset for the next stream of the same transaction */ + ent->xid = xid; + ent->stream_fileset = fileset; + ent->subxact_fileset = NULL; + } + else + { + /* + * Open the file and seek to the end of the file because we always + * append the changes file. + */ + stream_fd = BufFileOpenShared(ent->stream_fileset, path, O_RDWR); + BufFileSeek(stream_fd, 0, 0, SEEK_END); + } + + MemoryContextSwitchTo(oldcxt); +} + +/* + * stream_close_file + * Close the currently open file with streamed changes. + * + * This can only be called at the end of a streaming block, i.e. at stream_stop + * message from the upstream. + */ +static void +stream_close_file(void) +{ + Assert(in_streamed_transaction); + Assert(TransactionIdIsValid(stream_xid)); + Assert(stream_fd != NULL); + + BufFileClose(stream_fd); + + stream_xid = InvalidTransactionId; + stream_fd = NULL; +} + +/* + * stream_write_change + * Serialize a change to a file for the current toplevel transaction. + * + * The change is serialized in a simple format, with length (not including + * the length), action code (identifying the message type) and message + * contents (without the subxact TransactionId value). + */ +static void +stream_write_change(char action, StringInfo s) +{ + int len; + + Assert(in_streamed_transaction); + Assert(TransactionIdIsValid(stream_xid)); + Assert(stream_fd != NULL); + + /* total on-disk size, including the action type character */ + len = (s->len - s->cursor) + sizeof(char); + + /* first write the size */ + BufFileWrite(stream_fd, &len, sizeof(len)); + + /* then the action */ + BufFileWrite(stream_fd, &action, sizeof(action)); + + /* and finally the remaining part of the buffer (after the XID) */ + len = (s->len - s->cursor); + + BufFileWrite(stream_fd, &s->data[s->cursor], len); +} + +/* + * Cleanup the memory for subxacts and reset the related variables. + */ +static inline void +cleanup_subxact_info() +{ + if (subxact_data.subxacts) + pfree(subxact_data.subxacts); + + subxact_data.subxacts = NULL; + subxact_data.subxact_last = InvalidTransactionId; + subxact_data.nsubxacts = 0; + subxact_data.nsubxacts_max = 0; +} + /* Logical Replication Apply worker entry point */ void ApplyWorkerMain(Datum main_arg) @@ -2151,6 +3092,7 @@ ApplyWorkerMain(Datum main_arg) options.proto.logical.proto_version = LOGICALREP_PROTO_VERSION_NUM; options.proto.logical.publication_names = MySubscription->publications; options.proto.logical.binary = MySubscription->binary; + options.proto.logical.streaming = MySubscription->stream; /* Start normal logical streaming replication. */ walrcv_startstreaming(wrconn, &options); diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c index 81ef7dc4c1a3..c29c0888133a 100644 --- a/src/backend/replication/pgoutput/pgoutput.c +++ b/src/backend/replication/pgoutput/pgoutput.c @@ -47,17 +47,40 @@ static void pgoutput_truncate(LogicalDecodingContext *ctx, ReorderBufferChange *change); static bool pgoutput_origin_filter(LogicalDecodingContext *ctx, RepOriginId origin_id); +static void pgoutput_stream_start(struct LogicalDecodingContext *ctx, + ReorderBufferTXN *txn); +static void pgoutput_stream_stop(struct LogicalDecodingContext *ctx, + ReorderBufferTXN *txn); +static void pgoutput_stream_abort(struct LogicalDecodingContext *ctx, + ReorderBufferTXN *txn, + XLogRecPtr abort_lsn); +static void pgoutput_stream_commit(struct LogicalDecodingContext *ctx, + ReorderBufferTXN *txn, + XLogRecPtr commit_lsn); static bool publications_valid; +static bool in_streaming; static List *LoadPublications(List *pubnames); static void publication_invalidation_cb(Datum arg, int cacheid, uint32 hashvalue); -static void send_relation_and_attrs(Relation relation, LogicalDecodingContext *ctx); +static void send_relation_and_attrs(Relation relation, TransactionId xid, + LogicalDecodingContext *ctx); /* * Entry in the map used to remember which relation schemas we sent. * + * The schema_sent flag determines if the current schema record was already + * sent to the subscriber (in which case we don't need to send it again). + * + * The schema cache on downstream is however updated only at commit time, + * and with streamed transactions the commit order may be different from + * the order the transactions are sent in. Also, the (sub) transactions + * might get aborted so we need to send the schema for each (sub) transaction + * so that we don't loose the schema information on abort. For handling this, + * we maintain the list of xids (streamed_txns) for those we have already sent + * the schema. + * * For partitions, 'pubactions' considers not only the table's own * publications, but also those of all of its ancestors. */ @@ -70,6 +93,8 @@ typedef struct RelationSyncEntry * have been sent for this to be true. */ bool schema_sent; + List *streamed_txns; /* streamed toplevel transactions with this + * schema */ bool replicate_valid; PublicationActions pubactions; @@ -95,10 +120,15 @@ typedef struct RelationSyncEntry static HTAB *RelationSyncCache = NULL; static void init_rel_sync_cache(MemoryContext decoding_context); +static void cleanup_rel_sync_cache(TransactionId xid, bool is_commit); static RelationSyncEntry *get_rel_sync_entry(PGOutputData *data, Oid relid); static void rel_sync_cache_relation_cb(Datum arg, Oid relid); static void rel_sync_cache_publication_cb(Datum arg, int cacheid, uint32 hashvalue); +static void set_schema_sent_in_streamed_txn(RelationSyncEntry *entry, + TransactionId xid); +static bool get_schema_sent_in_streamed_txn(RelationSyncEntry *entry, + TransactionId xid); /* * Specify output plugin callbacks @@ -115,16 +145,26 @@ _PG_output_plugin_init(OutputPluginCallbacks *cb) cb->commit_cb = pgoutput_commit_txn; cb->filter_by_origin_cb = pgoutput_origin_filter; cb->shutdown_cb = pgoutput_shutdown; + + /* transaction streaming */ + cb->stream_start_cb = pgoutput_stream_start; + cb->stream_stop_cb = pgoutput_stream_stop; + cb->stream_abort_cb = pgoutput_stream_abort; + cb->stream_commit_cb = pgoutput_stream_commit; + cb->stream_change_cb = pgoutput_change; + cb->stream_truncate_cb = pgoutput_truncate; } static void parse_output_parameters(List *options, uint32 *protocol_version, - List **publication_names, bool *binary) + List **publication_names, bool *binary, + bool *enable_streaming) { ListCell *lc; bool protocol_version_given = false; bool publication_names_given = false; bool binary_option_given = false; + bool streaming_given = false; *binary = false; @@ -182,6 +222,16 @@ parse_output_parameters(List *options, uint32 *protocol_version, *binary = defGetBoolean(defel); } + else if (strcmp(defel->defname, "streaming") == 0) + { + if (streaming_given) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + streaming_given = true; + + *enable_streaming = defGetBoolean(defel); + } else elog(ERROR, "unrecognized pgoutput option: %s", defel->defname); } @@ -194,6 +244,7 @@ static void pgoutput_startup(LogicalDecodingContext *ctx, OutputPluginOptions *opt, bool is_init) { + bool enable_streaming = false; PGOutputData *data = palloc0(sizeof(PGOutputData)); /* Create our memory context for private allocations. */ @@ -217,7 +268,8 @@ pgoutput_startup(LogicalDecodingContext *ctx, OutputPluginOptions *opt, parse_output_parameters(ctx->output_plugin_options, &data->protocol_version, &data->publication_names, - &data->binary); + &data->binary, + &enable_streaming); /* Check if we support requested protocol */ if (data->protocol_version > LOGICALREP_PROTO_VERSION_NUM) @@ -237,6 +289,27 @@ pgoutput_startup(LogicalDecodingContext *ctx, OutputPluginOptions *opt, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("publication_names parameter missing"))); + /* + * Decide whether to enable streaming. It is disabled by default, in + * which case we just update the flag in decoding context. Otherwise + * we only allow it with sufficient version of the protocol, and when + * the output plugin supports it. + */ + if (!enable_streaming) + ctx->streaming = false; + else if (data->protocol_version < LOGICALREP_PROTO_STREAM_VERSION_NUM) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("requested proto_version=%d does not support streaming, need %d or higher", + data->protocol_version, LOGICALREP_PROTO_STREAM_VERSION_NUM))); + else if (!ctx->streaming) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("streaming requested, but not supported by output plugin"))); + + /* Also remember we're currently not streaming any transaction. */ + in_streaming = false; + /* Init publication state. */ data->publications = NIL; publications_valid = false; @@ -247,6 +320,11 @@ pgoutput_startup(LogicalDecodingContext *ctx, OutputPluginOptions *opt, /* Initialize relation schema cache. */ init_rel_sync_cache(CacheMemoryContext); } + else + { + /* Disable the streaming during the slot initialization mode. */ + ctx->streaming = false; + } } /* @@ -305,9 +383,47 @@ pgoutput_commit_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, */ static void maybe_send_schema(LogicalDecodingContext *ctx, + ReorderBufferTXN *txn, ReorderBufferChange *change, Relation relation, RelationSyncEntry *relentry) { - if (relentry->schema_sent) + bool schema_sent; + TransactionId xid = InvalidTransactionId; + TransactionId topxid = InvalidTransactionId; + + /* + * Remember XID of the (sub)transaction for the change. We don't care if + * it's top-level transaction or not (we have already sent that XID in + * start of the current streaming block). + * + * If we're not in a streaming block, just use InvalidTransactionId and + * the write methods will not include it. + */ + if (in_streaming) + xid = change->txn->xid; + + if (change->txn->toptxn) + topxid = change->txn->toptxn->xid; + else + topxid = xid; + + /* + * Do we need to send the schema? We do track streamed transactions + * separately, because those may be applied later (and the regular + * transactions won't see their effects until then) and in an order that + * we don't know at this point. + * + * XXX There is a scope of optimization here. Currently, we always send + * the schema first time in a streaming transaction but we can probably + * avoid that by checking 'relentry->schema_sent' flag. However, before + * doing that we need to study its impact on the case where we have a mix + * of streaming and non-streaming transactions. + */ + if (in_streaming) + schema_sent = get_schema_sent_in_streamed_txn(relentry, topxid); + else + schema_sent = relentry->schema_sent; + + if (schema_sent) return; /* If needed, send the ancestor's schema first. */ @@ -323,19 +439,24 @@ maybe_send_schema(LogicalDecodingContext *ctx, relentry->map = convert_tuples_by_name(CreateTupleDescCopy(indesc), CreateTupleDescCopy(outdesc)); MemoryContextSwitchTo(oldctx); - send_relation_and_attrs(ancestor, ctx); + send_relation_and_attrs(ancestor, xid, ctx); RelationClose(ancestor); } - send_relation_and_attrs(relation, ctx); - relentry->schema_sent = true; + send_relation_and_attrs(relation, xid, ctx); + + if (in_streaming) + set_schema_sent_in_streamed_txn(relentry, topxid); + else + relentry->schema_sent = true; } /* * Sends a relation */ static void -send_relation_and_attrs(Relation relation, LogicalDecodingContext *ctx) +send_relation_and_attrs(Relation relation, TransactionId xid, + LogicalDecodingContext *ctx) { TupleDesc desc = RelationGetDescr(relation); int i; @@ -359,17 +480,19 @@ send_relation_and_attrs(Relation relation, LogicalDecodingContext *ctx) continue; OutputPluginPrepareWrite(ctx, false); - logicalrep_write_typ(ctx->out, att->atttypid); + logicalrep_write_typ(ctx->out, xid, att->atttypid); OutputPluginWrite(ctx, false); } OutputPluginPrepareWrite(ctx, false); - logicalrep_write_rel(ctx->out, relation); + logicalrep_write_rel(ctx->out, xid, relation); OutputPluginWrite(ctx, false); } /* * Sends the decoded DML over wire. + * + * This is called both in streaming and non-streaming modes. */ static void pgoutput_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, @@ -378,10 +501,20 @@ pgoutput_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, PGOutputData *data = (PGOutputData *) ctx->output_plugin_private; MemoryContext old; RelationSyncEntry *relentry; + TransactionId xid = InvalidTransactionId; if (!is_publishable_relation(relation)) return; + /* + * Remember the xid for the change in streaming mode. We need to send xid + * with each change in the streaming mode so that subscriber can make + * their association and on aborts, it can discard the corresponding + * changes. + */ + if (in_streaming) + xid = change->txn->xid; + relentry = get_rel_sync_entry(data, RelationGetRelid(relation)); /* First check the table filter */ @@ -406,7 +539,7 @@ pgoutput_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, /* Avoid leaking memory by using and resetting our own context */ old = MemoryContextSwitchTo(data->context); - maybe_send_schema(ctx, relation, relentry); + maybe_send_schema(ctx, txn, change, relation, relentry); /* Send the data */ switch (change->action) @@ -426,7 +559,7 @@ pgoutput_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, } OutputPluginPrepareWrite(ctx, true); - logicalrep_write_insert(ctx->out, relation, tuple, + logicalrep_write_insert(ctx->out, xid, relation, tuple, data->binary); OutputPluginWrite(ctx, true); break; @@ -451,8 +584,8 @@ pgoutput_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, } OutputPluginPrepareWrite(ctx, true); - logicalrep_write_update(ctx->out, relation, oldtuple, newtuple, - data->binary); + logicalrep_write_update(ctx->out, xid, relation, oldtuple, + newtuple, data->binary); OutputPluginWrite(ctx, true); break; } @@ -472,7 +605,7 @@ pgoutput_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, } OutputPluginPrepareWrite(ctx, true); - logicalrep_write_delete(ctx->out, relation, oldtuple, + logicalrep_write_delete(ctx->out, xid, relation, oldtuple, data->binary); OutputPluginWrite(ctx, true); } @@ -498,6 +631,11 @@ pgoutput_truncate(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, int i; int nrelids; Oid *relids; + TransactionId xid = InvalidTransactionId; + + /* Remember the xid for the change in streaming mode. See pgoutput_change. */ + if (in_streaming) + xid = change->txn->xid; old = MemoryContextSwitchTo(data->context); @@ -526,13 +664,14 @@ pgoutput_truncate(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, continue; relids[nrelids++] = relid; - maybe_send_schema(ctx, relation, relentry); + maybe_send_schema(ctx, txn, change, relation, relentry); } if (nrelids > 0) { OutputPluginPrepareWrite(ctx, true); logicalrep_write_truncate(ctx->out, + xid, nrelids, relids, change->data.truncate.cascade, @@ -605,6 +744,118 @@ publication_invalidation_cb(Datum arg, int cacheid, uint32 hashvalue) rel_sync_cache_publication_cb(arg, cacheid, hashvalue); } +/* + * START STREAM callback + */ +static void +pgoutput_stream_start(struct LogicalDecodingContext *ctx, + ReorderBufferTXN *txn) +{ + bool send_replication_origin = txn->origin_id != InvalidRepOriginId; + + /* we can't nest streaming of transactions */ + Assert(!in_streaming); + + /* + * If we already sent the first stream for this transaction then don't + * send the origin id in the subsequent streams. + */ + if (rbtxn_is_streamed(txn)) + send_replication_origin = false; + + OutputPluginPrepareWrite(ctx, !send_replication_origin); + logicalrep_write_stream_start(ctx->out, txn->xid, !rbtxn_is_streamed(txn)); + + if (send_replication_origin) + { + char *origin; + + /* Message boundary */ + OutputPluginWrite(ctx, false); + OutputPluginPrepareWrite(ctx, true); + + if (replorigin_by_oid(txn->origin_id, true, &origin)) + logicalrep_write_origin(ctx->out, origin, InvalidXLogRecPtr); + } + + OutputPluginWrite(ctx, true); + + /* we're streaming a chunk of transaction now */ + in_streaming = true; +} + +/* + * STOP STREAM callback + */ +static void +pgoutput_stream_stop(struct LogicalDecodingContext *ctx, + ReorderBufferTXN *txn) +{ + /* we should be streaming a trasanction */ + Assert(in_streaming); + + OutputPluginPrepareWrite(ctx, true); + logicalrep_write_stream_stop(ctx->out); + OutputPluginWrite(ctx, true); + + /* we've stopped streaming a transaction */ + in_streaming = false; +} + +/* + * Notify downstream to discard the streamed transaction (along with all + * it's subtransactions, if it's a toplevel transaction). + */ +static void +pgoutput_stream_abort(struct LogicalDecodingContext *ctx, + ReorderBufferTXN *txn, + XLogRecPtr abort_lsn) +{ + ReorderBufferTXN *toptxn; + + /* + * The abort should happen outside streaming block, even for streamed + * transactions. The transaction has to be marked as streamed, though. + */ + Assert(!in_streaming); + + /* determine the toplevel transaction */ + toptxn = (txn->toptxn) ? txn->toptxn : txn; + + Assert(rbtxn_is_streamed(toptxn)); + + OutputPluginPrepareWrite(ctx, true); + logicalrep_write_stream_abort(ctx->out, toptxn->xid, txn->xid); + OutputPluginWrite(ctx, true); + + cleanup_rel_sync_cache(toptxn->xid, false); +} + +/* + * Notify downstream to apply the streamed transaction (along with all + * it's subtransactions). + */ +static void +pgoutput_stream_commit(struct LogicalDecodingContext *ctx, + ReorderBufferTXN *txn, + XLogRecPtr commit_lsn) +{ + /* + * The commit should happen outside streaming block, even for streamed + * transactions. The transaction has to be marked as streamed, though. + */ + Assert(!in_streaming); + Assert(rbtxn_is_streamed(txn)); + + OutputPluginUpdateProgress(ctx); + + OutputPluginPrepareWrite(ctx, true); + logicalrep_write_stream_commit(ctx->out, txn, commit_lsn); + OutputPluginWrite(ctx, true); + + cleanup_rel_sync_cache(txn->xid, true); +} + /* * Initialize the relation schema sync cache for a decoding session. * @@ -641,6 +892,39 @@ init_rel_sync_cache(MemoryContext cachectx) (Datum) 0); } +/* + * We expect relatively small number of streamed transactions. + */ +static bool +get_schema_sent_in_streamed_txn(RelationSyncEntry *entry, TransactionId xid) +{ + ListCell *lc; + + foreach(lc, entry->streamed_txns) + { + if (xid == (uint32) lfirst_int(lc)) + return true; + } + + return false; +} + +/* + * Add the xid in the rel sync entry for which we have already sent the schema + * of the relation. + */ +static void +set_schema_sent_in_streamed_txn(RelationSyncEntry *entry, TransactionId xid) +{ + MemoryContext oldctx; + + oldctx = MemoryContextSwitchTo(CacheMemoryContext); + + entry->streamed_txns = lappend_int(entry->streamed_txns, xid); + + MemoryContextSwitchTo(oldctx); +} + /* * Find or create entry in the relation schema cache. * @@ -771,11 +1055,58 @@ get_rel_sync_entry(PGOutputData *data, Oid relid) } if (!found) + { entry->schema_sent = false; + entry->streamed_txns = NULL; + } return entry; } +/* + * Cleanup list of streamed transactions and update the schema_sent flag. + * + * When a streamed transaction commits or aborts, we need to remove the + * toplevel XID from the schema cache. If the transaction aborted, the + * subscriber will simply throw away the schema records we streamed, so + * we don't need to do anything else. + * + * If the transaction is committed, the subscriber will update the relation + * cache - so tweak the schema_sent flag accordingly. + */ +static void +cleanup_rel_sync_cache(TransactionId xid, bool is_commit) +{ + HASH_SEQ_STATUS hash_seq; + RelationSyncEntry *entry; + ListCell *lc; + + Assert(RelationSyncCache != NULL); + + hash_seq_init(&hash_seq, RelationSyncCache); + while ((entry = hash_seq_search(&hash_seq)) != NULL) + { + /* + * We can set the schema_sent flag for an entry that has committed xid + * in the list as that ensures that the subscriber would have the + * corresponding schema and we don't need to send it unless there is + * any invalidation for that relation. + */ + foreach(lc, entry->streamed_txns) + { + if (xid == (uint32) lfirst_int(lc)) + { + if (is_commit) + entry->schema_sent = true; + + entry->streamed_txns = + foreach_delete_current(entry->streamed_txns, lc); + break; + } + } + } +} + /* * Relcache invalidation callback */ @@ -811,7 +1142,11 @@ rel_sync_cache_relation_cb(Datum arg, Oid relid) * Reset schema sent status as the relation definition may have changed. */ if (entry != NULL) + { entry->schema_sent = false; + list_free(entry->streamed_txns); + entry->streamed_txns = NULL; + } } /* diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 2cb3f9b083ec..d3ca54e4dc6a 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -4202,6 +4202,7 @@ getSubscriptions(Archive *fout) int i_oid; int i_subname; int i_rolname; + int i_substream; int i_subconninfo; int i_subslotname; int i_subsynccommit; @@ -4241,10 +4242,17 @@ getSubscriptions(Archive *fout) if (fout->remoteVersion >= 140000) appendPQExpBuffer(query, - " s.subbinary\n"); + " s.subbinary,\n"); else appendPQExpBuffer(query, - " false AS subbinary\n"); + " false AS subbinary,\n"); + + if (fout->remoteVersion >= 140000) + appendPQExpBuffer(query, + " s.substream\n"); + else + appendPQExpBuffer(query, + " false AS substream\n"); appendPQExpBuffer(query, "FROM pg_subscription s\n" @@ -4264,6 +4272,7 @@ getSubscriptions(Archive *fout) i_subsynccommit = PQfnumber(res, "subsynccommit"); i_subpublications = PQfnumber(res, "subpublications"); i_subbinary = PQfnumber(res, "subbinary"); + i_substream = PQfnumber(res, "substream"); subinfo = pg_malloc(ntups * sizeof(SubscriptionInfo)); @@ -4287,6 +4296,8 @@ getSubscriptions(Archive *fout) pg_strdup(PQgetvalue(res, i, i_subpublications)); subinfo[i].subbinary = pg_strdup(PQgetvalue(res, i, i_subbinary)); + subinfo[i].substream = + pg_strdup(PQgetvalue(res, i, i_substream)); if (strlen(subinfo[i].rolname) == 0) pg_log_warning("owner of subscription \"%s\" appears to be invalid", @@ -4358,6 +4369,9 @@ dumpSubscription(Archive *fout, SubscriptionInfo *subinfo) if (strcmp(subinfo->subbinary, "t") == 0) appendPQExpBuffer(query, ", binary = true"); + if (strcmp(subinfo->substream, "f") != 0) + appendPQExpBuffer(query, ", streaming = on"); + if (strcmp(subinfo->subsynccommit, "off") != 0) appendPQExpBuffer(query, ", synchronous_commit = %s", fmtId(subinfo->subsynccommit)); diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index 2f051b83d911..e0b42e83912f 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -626,6 +626,7 @@ typedef struct _SubscriptionInfo char *subconninfo; char *subslotname; char *subbinary; + char *substream; char *subsynccommit; char *subpublications; } SubscriptionInfo; diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 0266fc5fa858..0861d74a6fe0 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -5979,7 +5979,7 @@ describeSubscriptions(const char *pattern, bool verbose) PGresult *res; printQueryOpt myopt = pset.popt; static const bool translate_columns[] = {false, false, false, false, - false, false, false}; + false, false, false, false}; if (pset.sversion < 100000) { @@ -6005,11 +6005,13 @@ describeSubscriptions(const char *pattern, bool verbose) if (verbose) { - /* Binary mode is only supported in v14 and higher */ + /* Binary mode and streaming are only supported in v14 and higher */ if (pset.sversion >= 140000) appendPQExpBuffer(&buf, - ", subbinary AS \"%s\"\n", - gettext_noop("Binary")); + ", subbinary AS \"%s\"\n" + ", substream AS \"%s\"\n", + gettext_noop("Binary"), + gettext_noop("Streaming")); appendPQExpBuffer(&buf, ", subsynccommit AS \"%s\"\n" diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index c807f83baddd..0bbe0a122afd 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 202009021 +#define CATALOG_VERSION_NO 202009031 #endif diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h index 9795c35000d8..9ebec7bf0bff 100644 --- a/src/include/catalog/pg_subscription.h +++ b/src/include/catalog/pg_subscription.h @@ -51,6 +51,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW bool subbinary; /* True if the subscription wants the * publisher to send data in binary */ + bool substream; /* Stream in-progress transactions. */ + #ifdef CATALOG_VARLEN /* variable-length fields start here */ /* Connection string to the publisher */ text subconninfo BKI_FORCE_NOT_NULL; @@ -78,6 +80,7 @@ typedef struct Subscription bool enabled; /* Indicates if the subscription is enabled */ bool binary; /* Indicates if the subscription wants data in * binary format */ + bool stream; /* Allow streaming in-progress transactions. */ char *conninfo; /* Connection string to the publisher */ char *slotname; /* Name of the replication slot */ char *synccommit; /* Synchronous commit setting for worker */ diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 807a9c1edf6e..0dfbac46b4b0 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -982,7 +982,11 @@ typedef enum WAIT_EVENT_WAL_READ, WAIT_EVENT_WAL_SYNC, WAIT_EVENT_WAL_SYNC_METHOD_ASSIGN, - WAIT_EVENT_WAL_WRITE + WAIT_EVENT_WAL_WRITE, + WAIT_EVENT_LOGICAL_CHANGES_READ, + WAIT_EVENT_LOGICAL_CHANGES_WRITE, + WAIT_EVENT_LOGICAL_SUBXACT_READ, + WAIT_EVENT_LOGICAL_SUBXACT_WRITE } WaitEventIO; /* ---------- diff --git a/src/include/replication/logicalproto.h b/src/include/replication/logicalproto.h index 60a76bc85cf9..53905ee6080f 100644 --- a/src/include/replication/logicalproto.h +++ b/src/include/replication/logicalproto.h @@ -23,9 +23,13 @@ * we can support. LOGICALREP_PROTO_MIN_VERSION_NUM is the oldest version we * have backwards compatibility for. The client requests protocol version at * connect time. + * + * LOGICALREP_PROTO_STREAM_VERSION_NUM is the minimum protocol version with + * support for streaming large transactions. */ #define LOGICALREP_PROTO_MIN_VERSION_NUM 1 -#define LOGICALREP_PROTO_VERSION_NUM 1 +#define LOGICALREP_PROTO_STREAM_VERSION_NUM 2 +#define LOGICALREP_PROTO_VERSION_NUM 2 /* * This struct stores a tuple received via logical replication. @@ -98,25 +102,45 @@ extern void logicalrep_read_commit(StringInfo in, extern void logicalrep_write_origin(StringInfo out, const char *origin, XLogRecPtr origin_lsn); extern char *logicalrep_read_origin(StringInfo in, XLogRecPtr *origin_lsn); -extern void logicalrep_write_insert(StringInfo out, Relation rel, - HeapTuple newtuple, bool binary); +extern void logicalrep_write_insert(StringInfo out, TransactionId xid, + Relation rel, HeapTuple newtuple, + bool binary); extern LogicalRepRelId logicalrep_read_insert(StringInfo in, LogicalRepTupleData *newtup); -extern void logicalrep_write_update(StringInfo out, Relation rel, HeapTuple oldtuple, +extern void logicalrep_write_update(StringInfo out, TransactionId xid, + Relation rel, HeapTuple oldtuple, HeapTuple newtuple, bool binary); extern LogicalRepRelId logicalrep_read_update(StringInfo in, bool *has_oldtuple, LogicalRepTupleData *oldtup, LogicalRepTupleData *newtup); -extern void logicalrep_write_delete(StringInfo out, Relation rel, - HeapTuple oldtuple, bool binary); +extern void logicalrep_write_delete(StringInfo out, TransactionId xid, + Relation rel, HeapTuple oldtuple, + bool binary); extern LogicalRepRelId logicalrep_read_delete(StringInfo in, LogicalRepTupleData *oldtup); -extern void logicalrep_write_truncate(StringInfo out, int nrelids, Oid relids[], +extern void logicalrep_write_truncate(StringInfo out, TransactionId xid, + int nrelids, Oid relids[], bool cascade, bool restart_seqs); extern List *logicalrep_read_truncate(StringInfo in, bool *cascade, bool *restart_seqs); -extern void logicalrep_write_rel(StringInfo out, Relation rel); +extern void logicalrep_write_rel(StringInfo out, TransactionId xid, + Relation rel); extern LogicalRepRelation *logicalrep_read_rel(StringInfo in); -extern void logicalrep_write_typ(StringInfo out, Oid typoid); +extern void logicalrep_write_typ(StringInfo out, TransactionId xid, + Oid typoid); extern void logicalrep_read_typ(StringInfo out, LogicalRepTyp *ltyp); +extern void logicalrep_write_stream_start(StringInfo out, TransactionId xid, + bool first_segment); +extern TransactionId logicalrep_read_stream_start(StringInfo in, + bool *first_segment); +extern void logicalrep_write_stream_stop(StringInfo out); +extern TransactionId logicalrep_read_stream_stop(StringInfo in); +extern void logicalrep_write_stream_commit(StringInfo out, ReorderBufferTXN *txn, + XLogRecPtr commit_lsn); +extern TransactionId logicalrep_read_stream_commit(StringInfo out, + LogicalRepCommitData *commit_data); +extern void logicalrep_write_stream_abort(StringInfo out, TransactionId xid, + TransactionId subxid); +extern void logicalrep_read_stream_abort(StringInfo in, TransactionId *xid, + TransactionId *subxid); #endif /* LOGICAL_PROTO_H */ diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index c2d5dbee5491..1b05b39df4bd 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -178,6 +178,7 @@ typedef struct uint32 proto_version; /* Logical protocol version */ List *publication_names; /* String list of publications */ bool binary; /* Ask publisher to use binary */ + bool streaming; /* Streaming of large transactions */ } logical; } proto; } WalRcvStreamOptions; diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out index d71db0d52074..2fa9bce66a42 100644 --- a/src/test/regress/expected/subscription.out +++ b/src/test/regress/expected/subscription.out @@ -76,10 +76,10 @@ ALTER SUBSCRIPTION regress_testsub CONNECTION 'foobar'; ERROR: invalid connection string syntax: missing "=" after "foobar" in connection info string \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Synchronous commit | Conninfo ------------------+---------------------------+---------+-------------+--------+--------------------+----------------------------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | dbname=regress_doesnotexist + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Synchronous commit | Conninfo +-----------------+---------------------------+---------+-------------+--------+-----------+--------------------+----------------------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | f | off | dbname=regress_doesnotexist (1 row) ALTER SUBSCRIPTION regress_testsub SET PUBLICATION testpub2, testpub3 WITH (refresh = false); @@ -91,10 +91,10 @@ ERROR: subscription "regress_doesnotexist" does not exist ALTER SUBSCRIPTION regress_testsub SET (create_slot = false); ERROR: unrecognized subscription parameter: "create_slot" \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Synchronous commit | Conninfo ------------------+---------------------------+---------+---------------------+--------+--------------------+------------------------------ - regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | off | dbname=regress_doesnotexist2 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Synchronous commit | Conninfo +-----------------+---------------------------+---------+---------------------+--------+-----------+--------------------+------------------------------ + regress_testsub | regress_subscription_user | f | {testpub2,testpub3} | f | f | off | dbname=regress_doesnotexist2 (1 row) BEGIN; @@ -126,10 +126,10 @@ ALTER SUBSCRIPTION regress_testsub_foo SET (synchronous_commit = foobar); ERROR: invalid value for parameter "synchronous_commit": "foobar" HINT: Available values: local, remote_write, remote_apply, on, off. \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Synchronous commit | Conninfo ----------------------+---------------------------+---------+---------------------+--------+--------------------+------------------------------ - regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | local | dbname=regress_doesnotexist2 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Synchronous commit | Conninfo +---------------------+---------------------------+---------+---------------------+--------+-----------+--------------------+------------------------------ + regress_testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | f | f | local | dbname=regress_doesnotexist2 (1 row) -- rename back to keep the rest simple @@ -162,19 +162,42 @@ ERROR: binary requires a Boolean value CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, binary = true); WARNING: tables were not subscribed, you will have to run ALTER SUBSCRIPTION ... REFRESH PUBLICATION to subscribe the tables \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Synchronous commit | Conninfo ------------------+---------------------------+---------+-------------+--------+--------------------+----------------------------- - regress_testsub | regress_subscription_user | f | {testpub} | t | off | dbname=regress_doesnotexist + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Synchronous commit | Conninfo +-----------------+---------------------------+---------+-------------+--------+-----------+--------------------+----------------------------- + regress_testsub | regress_subscription_user | f | {testpub} | t | f | off | dbname=regress_doesnotexist (1 row) ALTER SUBSCRIPTION regress_testsub SET (binary = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Synchronous commit | Conninfo ------------------+---------------------------+---------+-------------+--------+--------------------+----------------------------- - regress_testsub | regress_subscription_user | f | {testpub} | f | off | dbname=regress_doesnotexist + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Synchronous commit | Conninfo +-----------------+---------------------------+---------+-------------+--------+-----------+--------------------+----------------------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | f | off | dbname=regress_doesnotexist +(1 row) + +DROP SUBSCRIPTION regress_testsub; +-- fail - streaming must be boolean +CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, streaming = foo); +ERROR: streaming requires a Boolean value +-- now it works +CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, streaming = true); +WARNING: tables were not subscribed, you will have to run ALTER SUBSCRIPTION ... REFRESH PUBLICATION to subscribe the tables +\dRs+ + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Synchronous commit | Conninfo +-----------------+---------------------------+---------+-------------+--------+-----------+--------------------+----------------------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | t | off | dbname=regress_doesnotexist +(1 row) + +ALTER SUBSCRIPTION regress_testsub SET (streaming = false); +ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); +\dRs+ + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Synchronous commit | Conninfo +-----------------+---------------------------+---------+-------------+--------+-----------+--------------------+----------------------------- + regress_testsub | regress_subscription_user | f | {testpub} | f | f | off | dbname=regress_doesnotexist (1 row) DROP SUBSCRIPTION regress_testsub; diff --git a/src/test/regress/sql/subscription.sql b/src/test/regress/sql/subscription.sql index eeb2ec06ebed..14fa0b247e1b 100644 --- a/src/test/regress/sql/subscription.sql +++ b/src/test/regress/sql/subscription.sql @@ -132,6 +132,21 @@ ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); DROP SUBSCRIPTION regress_testsub; +-- fail - streaming must be boolean +CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, streaming = foo); + +-- now it works +CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, streaming = true); + +\dRs+ + +ALTER SUBSCRIPTION regress_testsub SET (streaming = false); +ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); + +\dRs+ + +DROP SUBSCRIPTION regress_testsub; + RESET SESSION AUTHORIZATION; DROP ROLE regress_subscription_user; DROP ROLE regress_subscription_user2; diff --git a/src/test/subscription/t/015_stream.pl b/src/test/subscription/t/015_stream.pl new file mode 100644 index 000000000000..fffe00196507 --- /dev/null +++ b/src/test/subscription/t/015_stream.pl @@ -0,0 +1,98 @@ +# Test streaming of simple large transaction +use strict; +use warnings; +use PostgresNode; +use TestLib; +use Test::More tests => 4; + +# Create publisher node +my $node_publisher = get_new_node('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf('postgresql.conf', 'logical_decoding_work_mem = 64kB'); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = get_new_node('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->start; + +# Create some preexisting content on publisher +$node_publisher->safe_psql('postgres', + "CREATE TABLE test_tab (a int primary key, b varchar)"); +$node_publisher->safe_psql('postgres', + "INSERT INTO test_tab VALUES (1, 'foo'), (2, 'bar')"); + +# Setup structure on subscriber +$node_subscriber->safe_psql('postgres', "CREATE TABLE test_tab (a int primary key, b text, c timestamptz DEFAULT now(), d bigint DEFAULT 999)"); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +$node_publisher->safe_psql('postgres', "CREATE PUBLICATION tap_pub FOR TABLE test_tab"); + +my $appname = 'tap_sub'; +$node_subscriber->safe_psql('postgres', +"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)" +); + +$node_publisher->wait_for_catchup($appname); + +# Also wait for initial table sync to finish +my $synced_query = +"SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('r', 's');"; +$node_subscriber->poll_query_until('postgres', $synced_query) + or die "Timed out while waiting for subscriber to synchronize data"; + +my $result = + $node_subscriber->safe_psql('postgres', "SELECT count(*), count(c), count(d = 999) FROM test_tab"); +is($result, qq(2|2|2), 'check initial data was copied to subscriber'); + +# Insert, update and delete enough rows to exceed the 64kB limit. +$node_publisher->safe_psql('postgres', q{ +BEGIN; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3, 5000) s(i); +UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0; +DELETE FROM test_tab WHERE mod(a,3) = 0; +COMMIT; +}); + +$node_publisher->wait_for_catchup($appname); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*), count(c), count(d = 999) FROM test_tab"); +is($result, qq(3334|3334|3334), 'check extra columns contain local defaults'); + +# Test the streaming in binary mode +$node_subscriber->safe_psql('postgres', +"ALTER SUBSCRIPTION tap_sub SET (binary = on)" +); + +# Insert, update and delete enough rows to exceed the 64kB limit. +$node_publisher->safe_psql('postgres', q{ +BEGIN; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001, 10000) s(i); +UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0; +DELETE FROM test_tab WHERE mod(a,3) = 0; +COMMIT; +}); + +$node_publisher->wait_for_catchup($appname); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*), count(c), count(d = 999) FROM test_tab"); +is($result, qq(6667|6667|6667), 'check extra columns contain local defaults'); + +# Change the local values of the extra columns on the subscriber, +# update publisher, and check that subscriber retains the expected +# values. This is to ensure that non-streaming transactions behave +# properly after a streaming transaction. +$node_subscriber->safe_psql('postgres', "UPDATE test_tab SET c = 'epoch'::timestamptz + 987654321 * interval '1s'"); +$node_publisher->safe_psql('postgres', "UPDATE test_tab SET b = md5(a::text)"); + +$node_publisher->wait_for_catchup($appname); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*), count(extract(epoch from c) = 987654321), count(d = 999) FROM test_tab"); +is($result, qq(6667|6667|6667), 'check extra columns contain locally changed data'); + +$node_subscriber->stop; +$node_publisher->stop; diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 3d990463ce9c..500623e23019 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -111,6 +111,7 @@ Append AppendPath AppendRelInfo AppendState +ApplySubXactData Archive ArchiveEntryPtrType ArchiveFormat @@ -2370,6 +2371,7 @@ StopList StopWorkersData StrategyNumber StreamCtl +StreamXidHash StringInfo StringInfoData StripnullState @@ -2380,6 +2382,7 @@ SubPlanState SubTransactionId SubXactCallback SubXactCallbackItem +SubXactInfo SubXactEvent SubplanResultRelHashElem SubqueryScan From 4220e5721a27286aab726e87d3db5d1ee475fe23 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 3 Sep 2020 12:38:32 +0200 Subject: [PATCH 056/589] Fix XML id to match containing page This was apparently a typo when this part of the documentation was first added. --- doc/src/sgml/logical-replication.sgml | 2 +- doc/src/sgml/logicaldecoding.sgml | 2 +- doc/src/sgml/ref/alter_table.sgml | 2 +- doc/src/sgml/ref/psql-ref.sgml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index 3f69b7192682..c35415801f6f 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -133,7 +133,7 @@ fallback if no other solution is possible. If a replica identity other than full is set on the publisher side, a replica identity comprising the same or fewer columns must also be set on the subscriber - side. See for details on + side. See for details on how to set the replica identity. If a table without a replica identity is added to a publication that replicates UPDATE or DELETE operations then diff --git a/doc/src/sgml/logicaldecoding.sgml b/doc/src/sgml/logicaldecoding.sgml index 2765ed550eae..8d4fdf670064 100644 --- a/doc/src/sgml/logicaldecoding.sgml +++ b/doc/src/sgml/logicaldecoding.sgml @@ -24,7 +24,7 @@ by INSERT and the new row version created by UPDATE. Availability of old row versions for UPDATE and DELETE depends on - the configured replica identity (see ). + the configured replica identity (see ). diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index b2eb7097a957..c1576cc69628 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -811,7 +811,7 @@ WITH ( MODULUS numeric_literal, REM - + REPLICA IDENTITY diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml index e1e2236a71de..ef18fe27e03c 100644 --- a/doc/src/sgml/ref/psql-ref.sgml +++ b/doc/src/sgml/ref/psql-ref.sgml @@ -1182,7 +1182,7 @@ testdb=> more information is displayed: any comments associated with the columns of the table are shown, as is the presence of OIDs in the table, the view definition if the relation is a view, a non-default - replica + replica identity setting and the access method name if the relation has an access method. From cb6eb4a09e8adf3cc25e8eff9fdd43d8d9cd9c86 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 3 Sep 2020 13:15:53 +0200 Subject: [PATCH 057/589] doc: Add missing cross-links in system catalog documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes the first mention of a system catalog or view in each paragraph in the system system catalog and view documentation pages hyperlinks, for easier navigation. Also linkify the first mention of pg_hba.conf in pg_hba_file_rules, as that's more specific and easier to spot than the link to the client authentication chapter. Author: Dagfinn Ilmari Mannsåker Discussion: https://www.postgresql.org/message-id/flat/87mu5xqc11.fsf@wibble.ilmari.org --- doc/src/sgml/catalogs.sgml | 122 ++++++++++++++++++++----------------- 1 file changed, 66 insertions(+), 56 deletions(-) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 1d1b8ce8fb12..85084a7c2333 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -381,9 +381,10 @@ sum, count, and max. Each entry in pg_aggregate is an extension of an entry - in pg_proc. The pg_proc - entry carries the aggregate's name, input and output data types, and - other information that is similar to ordinary functions. + in pg_proc. + The pg_proc entry carries the aggregate's name, + input and output data types, and other information that is similar to + ordinary functions. @@ -902,7 +903,7 @@ catalog structure for performance reasons). Also, amoplefttype and amoprighttype must match the oprleft and oprright fields of the - referenced pg_operator entry. + referenced pg_operator entry. @@ -1099,7 +1100,8 @@ table columns. There will be exactly one pg_attribute row for every column in every table in the database. (There will also be attribute entries for - indexes, and indeed all objects that have pg_class + indexes, and indeed all objects that have + pg_class entries.) @@ -1270,7 +1272,7 @@ This column has a default expression or generation expression, in which case there will be a corresponding entry in the - pg_attrdef catalog that actually defines the + pg_attrdef catalog that actually defines the expression. (Check attgenerated to determine whether this is a default or a generation expression.) @@ -1402,7 +1404,7 @@ In a dropped column's pg_attribute entry, atttypid is reset to zero, but attlen and the other fields copied from - pg_type are still valid. This arrangement is needed + pg_type are still valid. This arrangement is needed to cope with the situation where the dropped column's data type was later dropped, and so there is no pg_type row anymore. attlen and the other fields can be used @@ -1836,13 +1838,15 @@ SCRAM-SHA-256$<iteration count>:&l The catalog pg_class catalogs tables and most everything else that has columns or is otherwise similar to a - table. This includes indexes (but see also - pg_index), sequences (but see also - pg_sequence), views, materialized - views, composite types, and TOAST tables; see relkind. - Below, when we mean all of these - kinds of objects we speak of relations. Not all - columns are meaningful for all relation types. + table. This includes indexes (but see also pg_index), + sequences (but see also pg_sequence), + views, materialized views, composite types, and TOAST tables; + see relkind. + Below, when we mean all of these kinds of objects we speak of + relations. Not all columns are meaningful for all relation + types.
@@ -2024,7 +2028,7 @@ SCRAM-SHA-256$<iteration count>:&l True if this table is shared across all databases in the cluster. Only - certain system catalogs (such as pg_database) + certain system catalogs (such as pg_database) are shared. @@ -2064,8 +2068,8 @@ SCRAM-SHA-256$<iteration count>:&l Number of user columns in the relation (system columns not counted). There must be this many corresponding entries in - pg_attribute. See also - pg_attribute.attnum. + pg_attribute. See also + pg_attribute.attnum. @@ -2409,7 +2413,8 @@ SCRAM-SHA-256$<iteration count>:&l key, unique, foreign key, and exclusion constraints on tables. (Column constraints are not treated specially. Every column constraint is equivalent to some table constraint.) - Not-null constraints are represented in the pg_attribute + Not-null constraints are represented in the + pg_attribute catalog, not here. @@ -2712,7 +2717,7 @@ SCRAM-SHA-256$<iteration count>:&l For other cases, a zero appears in conkey and the associated index must be consulted to discover the expression that is constrained. (conkey thus has the - same contents as pg_index.indkey for the + same contents as pg_index.indkey for the index.) @@ -2980,7 +2985,7 @@ SCRAM-SHA-256$<iteration count>:&l track whether the database needs to be vacuumed in order to prevent transaction ID wraparound or to allow pg_xact to be shrunk. It is the minimum of the per-table - pg_class.relfrozenxid values. + pg_class.relfrozenxid values. @@ -2994,7 +2999,7 @@ SCRAM-SHA-256$<iteration count>:&l track whether the database needs to be vacuumed in order to prevent multixact ID wraparound or to allow pg_multixact to be shrunk. It is the minimum of the per-table - pg_class.relminmxid values. + pg_class.relminmxid values. @@ -3006,7 +3011,7 @@ SCRAM-SHA-256$<iteration count>:&l The default tablespace for the database. Within this database, all tables for which - pg_class.reltablespace is zero + pg_class.reltablespace is zero will be stored in this tablespace; in particular, all the non-shared system catalogs will be there. @@ -3605,7 +3610,7 @@ SCRAM-SHA-256$<iteration count>:&l (references pg_type.oid) - The OID of the pg_type entry owning this enum value + The OID of the pg_type entry owning this enum value @@ -4100,8 +4105,9 @@ SCRAM-SHA-256$<iteration count>:&l The catalog pg_foreign_table contains auxiliary information about foreign tables. A foreign table is - primarily represented by a pg_class entry, - just like a regular table. Its pg_foreign_table + primarily represented by a + pg_class + entry, just like a regular table. Its pg_foreign_table entry contains the information that is pertinent only to foreign tables and not any other kind of relation. @@ -4127,7 +4133,7 @@ SCRAM-SHA-256$<iteration count>:&l (references pg_class.oid) - OID of the pg_class entry for this foreign table + The OID of the pg_class entry for this foreign table @@ -4165,7 +4171,7 @@ SCRAM-SHA-256$<iteration count>:&l The catalog pg_index contains part of the information about indexes. The rest is mostly in - pg_class. + pg_class.
@@ -4189,7 +4195,7 @@ SCRAM-SHA-256$<iteration count>:&l (references pg_class.oid) - The OID of the pg_class entry for this index + The OID of the pg_class entry for this index @@ -4199,7 +4205,7 @@ SCRAM-SHA-256$<iteration count>:&l (references pg_class.oid) - The OID of the pg_class entry for the table this index is for + The OID of the pg_class entry for the table this index is for @@ -5400,7 +5406,7 @@ SCRAM-SHA-256$<iteration count>:&l (references pg_class.oid) - The OID of the pg_class entry for this partitioned table + The OID of the pg_class entry for this partitioned table @@ -5429,7 +5435,7 @@ SCRAM-SHA-256$<iteration count>:&l (references pg_class.oid) - The OID of the pg_class entry for the default partition + The OID of the pg_class entry for the default partition of this partitioned table, or zero if this partitioned table does not have a default partition. @@ -5609,7 +5615,7 @@ SCRAM-SHA-256$<iteration count>:&l Policies stored in pg_policy are applied only when - pg_class.relrowsecurity is set for + pg_class.relrowsecurity is set for their table. @@ -5634,7 +5640,7 @@ SCRAM-SHA-256$<iteration count>:&l If prokind indicates that the entry is for an aggregate function, there should be a matching row in - pg_aggregate. + pg_aggregate.
@@ -6564,7 +6570,8 @@ SCRAM-SHA-256$<iteration count>:&l The catalog pg_sequence contains information about sequences. Some of the information about sequences, such as the name and - the schema, is in pg_class. + the schema, is in + pg_class
@@ -6588,7 +6595,7 @@ SCRAM-SHA-256$<iteration count>:&l (references pg_class.oid) - The OID of the pg_class entry for this sequence + The OID of the pg_class entry for this sequence @@ -7347,13 +7354,14 @@ SCRAM-SHA-256$<iteration count>:&l The catalog pg_statistic_ext_data - holds data for extended planner statistics defined in pg_statistic_ext. + holds data for extended planner statistics defined in + pg_statistic_ext. Each row in this catalog corresponds to a statistics object created with . - Like pg_statistic, + Like pg_statistic, pg_statistic_ext_data should not be readable by the public, since the contents might be considered sensitive. (Example: most common combinations of values in columns might be quite @@ -7361,7 +7369,7 @@ SCRAM-SHA-256$<iteration count>:&l pg_stats_ext is a publicly readable view on pg_statistic_ext_data (after joining - with pg_statistic_ext) that only exposes + with pg_statistic_ext) that only exposes information about those tables and columns that are readable by the current user. @@ -7960,7 +7968,7 @@ SCRAM-SHA-256$<iteration count>:&l (references pg_constraint.oid) - The pg_constraint entry associated with the trigger, if any + The pg_constraint entry associated with the trigger, if any @@ -8058,7 +8066,7 @@ SCRAM-SHA-256$<iteration count>:&l When tgconstraint is nonzero, tgconstrrelid, tgconstrindid, tgdeferrable, and tginitdeferred are - largely redundant with the referenced pg_constraint entry. + largely redundant with the referenced pg_constraint entry. However, it is possible for a non-deferrable trigger to be associated with a deferrable constraint: foreign key constraints can have some deferrable and some non-deferrable triggers. @@ -8205,7 +8213,7 @@ SCRAM-SHA-256$<iteration count>:&l (references pg_ts_config.oid) - The OID of the pg_ts_config entry owning this map entry + The OID of the pg_ts_config entry owning this map entry @@ -8719,11 +8727,11 @@ SCRAM-SHA-256$<iteration count>:&l If this is a composite type (see typtype), then this column points to - the pg_class entry that defines the + the pg_class entry that defines the corresponding table. (For a free-standing composite type, the - pg_class entry doesn't really represent + pg_class entry doesn't really represent a table, but it is needed anyway for the type's - pg_attribute entries to link to.) + pg_attribute entries to link to.) Zero for non-composite types. @@ -10112,8 +10120,9 @@ SCRAM-SHA-256$<iteration count>:&l The view pg_hba_file_rules provides a summary of - the contents of the client authentication configuration - file, pg_hba.conf. A row appears in this view for each + the contents of the client authentication configuration file, + pg_hba.conf. + A row appears in this view for each non-empty, non-comment line in the file, with annotations indicating whether the rule could be applied successfully. @@ -10354,8 +10363,8 @@ SCRAM-SHA-256$<iteration count>:&l individual tuples of relations, transaction IDs (both virtual and permanent IDs), and general database objects (identified by class OID and object OID, - in the same way as in pg_description or - pg_depend). Also, the right to extend a + in the same way as in pg_description or + pg_depend). Also, the right to extend a relation is represented as a separate lockable object, as is the right to update pg_database.datfrozenxid. Also, advisory locks can be taken on numbers that have @@ -10597,7 +10606,7 @@ SCRAM-SHA-256$<iteration count>:&l pg_locks provides a global view of all locks in the database cluster, not only those relevant to the current database. Although its relation column can be joined - against pg_class.oid to identify locked + against pg_class.oid to identify locked relations, this will only work correctly for relations in the current database (those for which the database column is either the current database's OID or zero). @@ -11112,8 +11121,9 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx The view pg_publication_tables provides information about the mapping between publications and the tables they - contain. Unlike the underlying - catalog pg_publication_rel, this view expands + contain. Unlike the underlying catalog + pg_publication_rel, + this view expands publications defined as FOR ALL TABLES, so for such publications there will be a row for each eligible table. @@ -11688,7 +11698,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx The pg_rules view excludes the ON SELECT rules of views and materialized views; those can be seen in - pg_views and pg_matviews. + pg_views and pg_matviews. @@ -12517,7 +12527,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx the information stored in the pg_statistic catalog. This view allows access only to rows of - pg_statistic that correspond to tables the + pg_statistic that correspond to tables the user has permission to read, and therefore it is safe to allow public read access to this view. @@ -12526,7 +12536,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx pg_stats is also designed to present the information in a more readable format than the underlying catalog — at the cost that its schema must be extended whenever new slot types - are defined for pg_statistic. + are defined for pg_statistic.
@@ -12728,7 +12738,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx linkend="catalog-pg-statistic-ext">pg_statistic_ext and pg_statistic_ext_data catalogs. This view allows access only to rows of - pg_statistic_ext and pg_statistic_ext_data + pg_statistic_ext and pg_statistic_ext_data that correspond to tables the user has permission to read, and therefore it is safe to allow public read access to this view. @@ -12737,7 +12747,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx pg_stats_ext is also designed to present the information in a more readable format than the underlying catalogs — at the cost that its schema must be extended whenever new types - of extended statistics are added to pg_statistic_ext. + of extended statistics are added to pg_statistic_ext.
From 5ccf32211832477a147b1428b190817baea08f2f Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 3 Sep 2020 13:15:53 +0200 Subject: [PATCH 058/589] doc: Make SQL command names in the catalog documentation links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In passing, fix the initdb references to be rather than , which is what we normally use. Author: Dagfinn Ilmari Mannsåker Discussion: https://www.postgresql.org/message-id/flat/87mu5xqc11.fsf@wibble.ilmari.org --- doc/src/sgml/catalogs.sgml | 110 +++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 85084a7c2333..508bea3bc644 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -1966,9 +1966,9 @@ SCRAM-SHA-256$<iteration count>:&l Size of the on-disk representation of this table in pages (of size BLCKSZ). This is only an estimate used by the - planner. It is updated by VACUUM, - ANALYZE, and a few DDL commands such as - CREATE INDEX. + planner. It is updated by , + , and a few DDL commands such as + . @@ -1978,9 +1978,9 @@ SCRAM-SHA-256$<iteration count>:&l Number of live rows in the table. This is only an estimate used by - the planner. It is updated by VACUUM, - ANALYZE, and a few DDL commands such as - CREATE INDEX. + the planner. It is updated by , + , and a few DDL commands such as + . If the table has never yet been vacuumed or analyzed, reltuples contains -1 indicating that the row count is @@ -1995,9 +1995,9 @@ SCRAM-SHA-256$<iteration count>:&l Number of pages that are marked all-visible in the table's visibility map. This is only an estimate used by the - planner. It is updated by VACUUM, - ANALYZE, and a few DDL commands such as - CREATE INDEX. + planner. It is updated by , + , and a few DDL commands such as + . @@ -2241,8 +2241,8 @@ SCRAM-SHA-256$<iteration count>:&l lazily: they are guaranteed to be true if that's the correct state, but may not be reset to false immediately when the condition is no longer true. For example, relhasindex is set by - CREATE INDEX, but it is never cleared by - DROP INDEX. Instead, VACUUM clears + , but it is never cleared by + . Instead, clears relhasindex if it finds the table has no indexes. This arrangement avoids race conditions and improves concurrency. @@ -2419,8 +2419,8 @@ SCRAM-SHA-256$<iteration count>:&l - User-defined constraint triggers (created with CREATE CONSTRAINT - TRIGGER) also give rise to an entry in this table. + User-defined constraint triggers (created with + CREATE CONSTRAINT TRIGGER) also give rise to an entry in this table. @@ -3425,7 +3425,7 @@ SCRAM-SHA-256$<iteration count>:&l the referenced object (see pg_extension). The dependent object can be dropped only via - DROP EXTENSION on the referenced object. + on the referenced object. Functionally this dependency type acts the same as an INTERNAL dependency, but it's kept separate for clarity and to simplify pg_dump. @@ -3456,7 +3456,7 @@ SCRAM-SHA-256$<iteration count>:&l There is no dependent object; this type of entry is a signal that the system itself depends on the referenced object, and so that object must never be deleted. Entries of this type are - created only by initdb. The columns for the + created only by initdb. The columns for the dependent object contain zeroes. @@ -4285,7 +4285,7 @@ SCRAM-SHA-256$<iteration count>:&l If true, the index is currently valid for queries. False means the index is possibly incomplete: it must still be modified by - INSERT/UPDATE operations, but it cannot safely + / operations, but it cannot safely be used for queries. If it is unique, the uniqueness property is not guaranteed true either. @@ -4309,7 +4309,7 @@ SCRAM-SHA-256$<iteration count>:&l If true, the index is currently ready for inserts. False means the - index must be ignored by INSERT/UPDATE + index must be ignored by / operations. @@ -4330,8 +4330,8 @@ SCRAM-SHA-256$<iteration count>:&l If true this index has been chosen as replica identity - using ALTER TABLE ... REPLICA IDENTITY USING INDEX - ... + using ALTER TABLE ... + REPLICA IDENTITY USING INDEX ... @@ -4504,11 +4504,11 @@ SCRAM-SHA-256$<iteration count>:&l Objects can have initial privileges either by having those privileges set when the system is initialized (by initdb) or when the - object is created during a CREATE EXTENSION and the - extension script sets initial privileges using the GRANT + object is created during a and the + extension script sets initial privileges using the system. Note that the system will automatically handle recording of the privileges during the extension script and that extension authors need - only use the GRANT and REVOKE + only use the and statements in their script to have the privileges recorded. The privtype column indicates if the initial privilege was set by initdb or during a @@ -5564,10 +5564,10 @@ SCRAM-SHA-256$<iteration count>:&l The command type to which the policy is applied: - r for SELECT, - a for INSERT, - w for UPDATE, - d for DELETE, + r for , + a for , + w for , + d for , or * for all @@ -6075,7 +6075,7 @@ SCRAM-SHA-256$<iteration count>:&l pubinsert bool - If true, INSERT operations are replicated for + If true, operations are replicated for tables in the publication. @@ -6085,7 +6085,7 @@ SCRAM-SHA-256$<iteration count>:&l pubupdate bool - If true, UPDATE operations are replicated for + If true, operations are replicated for tables in the publication. @@ -6095,7 +6095,7 @@ SCRAM-SHA-256$<iteration count>:&l pubdelete bool - If true, DELETE operations are replicated for + If true, operations are replicated for tables in the publication. @@ -6105,7 +6105,7 @@ SCRAM-SHA-256$<iteration count>:&l pubtruncate bool - If true, TRUNCATE operations are replicated for + If true, operations are replicated for tables in the publication. @@ -6408,9 +6408,9 @@ SCRAM-SHA-256$<iteration count>:&l ev_type char - Event type that the rule is for: 1 = SELECT, 2 = - UPDATE, 3 = INSERT, 4 = - DELETE + Event type that the rule is for: 1 = , 2 = + , 3 = , 4 = + @@ -6832,7 +6832,7 @@ SCRAM-SHA-256$<iteration count>:&l There is no dependent object; this type of entry is a signal that the system itself depends on the referenced object, and so that object must never be deleted. Entries of this type are - created only by initdb. The columns for the + created only by initdb. The columns for the dependent object contain zeroes. @@ -7336,9 +7336,9 @@ SCRAM-SHA-256$<iteration count>:&l The pg_statistic_ext entry is filled in - completely during CREATE STATISTICS, but the actual + completely during , but the actual statistical values are not computed then. - Subsequent ANALYZE commands compute the desired values + Subsequent commands compute the desired values and populate an entry in the pg_statistic_ext_data catalog. @@ -7590,8 +7590,9 @@ SCRAM-SHA-256$<iteration count>:&l This catalog only contains tables known to the subscription after running - either CREATE SUBSCRIPTION or - ALTER SUBSCRIPTION ... REFRESH PUBLICATION. + either or + ALTER SUBSCRIPTION ... REFRESH + PUBLICATION.
@@ -8837,7 +8838,7 @@ SCRAM-SHA-256$<iteration count>:&l (references pg_proc.oid) - Custom ANALYZE function, or 0 to use the standard function + Custom function, or 0 to use the standard function @@ -10324,7 +10325,7 @@ SCRAM-SHA-256$<iteration count>:&l indexdef text - Index definition (a reconstructed CREATE INDEX + Index definition (a reconstructed command) @@ -10779,7 +10780,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx definition text - Materialized view definition (a reconstructed SELECT query) + Materialized view definition (a reconstructed query) @@ -10949,7 +10950,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx The query string submitted by the client to create this prepared statement. For prepared statements created via SQL, - this is the PREPARE statement submitted by + this is the statement submitted by the client. For prepared statements created via the frontend/backend protocol, this is the text of the prepared statement itself. @@ -10983,7 +10984,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx true if the prepared statement was created - via the PREPARE SQL command; + via the SQL command; false if the statement was prepared via the frontend/backend protocol @@ -12114,7 +12115,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx reset_val text - Value that RESET would reset the parameter to + Value that would reset the parameter to in the current session @@ -12172,7 +12173,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx These settings cannot be changed directly; they reflect internally determined values. Some of them may be adjustable by rebuilding the server with different configuration options, or by changing options - supplied to initdb. + supplied to initdb. @@ -12247,7 +12248,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx These settings can be set from postgresql.conf, - or within a session via the SET command; but only superusers + or within a session via the command; but only superusers can change them via SET. Changes in postgresql.conf will affect existing sessions only if no session-local value has been established with SET. @@ -12260,7 +12261,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx These settings can be set from postgresql.conf, - or within a session via the SET command. Any user is + or within a session via the command. Any user is allowed to change their session-local value. Changes in postgresql.conf will affect existing sessions only if no session-local value has been established with SET. @@ -12276,7 +12277,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx The pg_settings view cannot be inserted into or - deleted from, but it can be updated. An UPDATE applied + deleted from, but it can be updated. An applied to a row of pg_settings is equivalent to executing the command on that named parameter. The change only affects the value used by the current @@ -12620,7 +12621,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx If greater than zero, the estimated number of distinct values in the column. If less than zero, the negative of the number of distinct values divided by the number of rows. (The negated form is used when - ANALYZE believes that the number of distinct values is + believes that the number of distinct values is likely to increase as the table grows; the positive form is used when the column seems to have a fixed number of possible values.) For example, -1 indicates a unique column in which the number of distinct @@ -12718,7 +12719,8 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx The maximum number of entries in the array fields can be controlled on a - column-by-column basis using the ALTER TABLE SET STATISTICS + column-by-column basis using the ALTER + TABLE SET STATISTICS command, or globally by setting the run-time parameter. @@ -12843,7 +12845,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx than zero, the estimated number of distinct values in the combination. If less than zero, the negative of the number of distinct values divided by the number of rows. - (The negated form is used when ANALYZE believes that + (The negated form is used when believes that the number of distinct values is likely to increase as the table grows; the positive form is used when the column seems to have a fixed number of possible values.) For example, -1 indicates a unique combination of @@ -12908,8 +12910,8 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx The maximum number of entries in the array fields can be controlled on a - column-by-column basis using the ALTER TABLE SET STATISTICS - command, or globally by setting the + column-by-column basis using the ALTER + TABLE SET STATISTICS command, or globally by setting the run-time parameter. @@ -13471,7 +13473,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx definition text - View definition (a reconstructed SELECT query) + View definition (a reconstructed query) From d2511d71328cb30b0d6b3f43af4809df83c4f0cc Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 3 Sep 2020 11:45:26 -0400 Subject: [PATCH 059/589] Doc: mention packager-supplied tools for server start/stop, initdb, etc. The majority of our audience is probably using a pre-packaged Postgres build rather than raw sources. For them, much of runtime.sgml is not too relevant, and they should be reading the packager's docs instead. Add some notes pointing that way in appropriate places. Text by me; thanks to Daniel Gustafsson for review and discussion, and to Laurenz Albe for an earlier version. Discussion: https://postgr.es/m/159430831443.16535.11360317280100947016@wrigleys.postgresql.org --- doc/src/sgml/runtime.sgml | 95 ++++++++++++++++++++++++++++++++------- 1 file changed, 79 insertions(+), 16 deletions(-) diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index 6cda39f3abfc..f5842319358b 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -4,10 +4,22 @@ Server Setup and Operation - This chapter discusses how to set up and run the database server + This chapter discusses how to set up and run the database server, and its interactions with the operating system. + + The directions in this chapter assume that you are working with + plain PostgreSQL without any additional + infrastructure, for example a copy that you built from source + according to the directions in the preceding chapters. + If you are working with a pre-packaged or vendor-supplied + version of PostgreSQL, it is likely that + the packager has made special provisions for installing and starting + the database server according to your system's conventions. + Consult the package-level documentation for details. + + The <productname>PostgreSQL</productname> User Account @@ -21,9 +33,15 @@ separate user account. This user account should only own the data that is managed by the server, and should not be shared with other daemons. (For example, using the user nobody is a bad - idea.) It is not advisable to install executables owned by this - user because compromised systems could then modify their own - binaries. + idea.) In particular, it is advisable that this user account not own + the PostgreSQL executable files, to ensure + that a compromised server process could not modify those executables. + + + + Pre-packaged versions of PostgreSQL will + typically create a suitable user account automatically during + package installation. @@ -71,11 +89,26 @@ completely up to you where you choose to store your data. There is no default, although locations such as /usr/local/pgsql/data or - /var/lib/pgsql/data are popular. To initialize a - database cluster, use the command ,initdb which is - installed with PostgreSQL. The desired - file system location of your database cluster is indicated by the + /var/lib/pgsql/data are popular. + The data directory must be initialized before being used, using the program + initdb + which is installed with PostgreSQL. + + + + If you are using a pre-packaged version + of PostgreSQL, it may well have a specific + convention for where to place the data directory, and it may also + provide a script for creating the data directory. In that case you + should use that script in preference to + running initdb directly. + Consult the package-level documentation for details. + + + + To initialize a database cluster manually, + run initdb and specify the desired + file system location of the database cluster with the option, for example: $ initdb -D /usr/local/pgsql/data @@ -309,10 +342,22 @@ postgres$ initdb -D /usr/local/pgsql/data Before anyone can access the database, you must start the database server. The database server program is called postgres.postgres - The postgres program must know where to - find the data it is supposed to use. This is done with the - option. Thus, the simplest way to start the - server is: + + + + If you are using a pre-packaged version + of PostgreSQL, it almost certainly includes + provisions for running the server as a background task according to the + conventions of your operating system. Using the package's + infrastructure to start the server will be much less work than figuring + out how to do this yourself. Consult the package-level documentation + for details. + + + + The bare-bones way to start the server manually is just to invoke + postgres directly, specifying the location of the + data directory with the option, for example: $ postgres -D /usr/local/pgsql/data @@ -364,7 +409,7 @@ pg_ctl start -l logfile starting the server during Autostart scripts are operating-system-specific. - There are a few distributed with + There are a few example scripts distributed with PostgreSQL in the contrib/start-scripts directory. Installing one will require root privileges. @@ -1481,9 +1526,23 @@ $ cat /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages - There are several ways to shut down the database server. You control - the type of shutdown by sending different signals to the supervisor + There are several ways to shut down the database server. + Under the hood, they all reduce to sending a signal to the supervisor postgres process. + + + + If you are using a pre-packaged version + of PostgreSQL, and you used its provisions + for starting the server, then you should also use its provisions for + stopping the server. Consult the package-level documentation for + details. + + + + When managing the server directly, you can control the type of shutdown + by sending different signals to the postgres + process: @@ -1620,6 +1679,10 @@ $ kill -INT `head -1 /usr/local/pgsql/data/postmaster.pid`. Replication methods are also available, as discussed below. + (If you are using a pre-packaged version + of PostgreSQL, it may provide scripts to + assist with major version upgrades. Consult the package-level + documentation for details.) From 8f8154a503c71a18ad72c64f4aefb9d847c45b86 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 3 Sep 2020 12:16:48 -0400 Subject: [PATCH 060/589] Allow records to span multiple lines in pg_hba.conf and pg_ident.conf. A backslash at the end of a line now causes the next line to be appended to the current one (effectively, the backslash and newline are discarded). This allows long HBA entries to be created without legibility problems. While we're here, get rid of the former hard-wired length limit on pg_hba.conf lines, by using an expansible StringInfo buffer instead of a fixed-size local variable. Since the same code is used to read the ident map file, these changes apply there as well. Fabien Coelho, reviewed by Justin Pryzby and David Zhang Discussion: https://postgr.es/m/alpine.DEB.2.21.2003251906140.15243@pseudo --- doc/src/sgml/client-auth.sgml | 6 +- src/backend/libpq/hba.c | 106 ++++++++++++++-------- src/test/authentication/t/001_password.pl | 3 +- 3 files changed, 76 insertions(+), 39 deletions(-) diff --git a/doc/src/sgml/client-auth.sgml b/doc/src/sgml/client-auth.sgml index 5cd88b462dbd..d62d1a061c9c 100644 --- a/doc/src/sgml/client-auth.sgml +++ b/doc/src/sgml/client-auth.sgml @@ -77,13 +77,15 @@ The general format of the pg_hba.conf file is a set of records, one per line. Blank lines are ignored, as is any text after the # comment character. - Records cannot be continued across lines. + A record can be continued onto the next line by ending the line with + a backslash. (Backslashes are not special except at the end of a line.) A record is made up of a number of fields which are separated by spaces and/or tabs. Fields can contain white space if the field value is double-quoted. Quoting one of the keywords in a database, user, or address field (e.g., all or replication) makes the word lose its special meaning, and just match a database, user, or host with that name. + Backslash line continuation applies even within quoted text or comments. @@ -821,7 +823,7 @@ local db1,db2,@demodbs all md5 map-name system-username database-username - Comments and whitespace are handled in the same way as in + Comments, whitespace and line continuations are handled in the same way as in pg_hba.conf. The map-name is an arbitrary name that will be used to refer to this mapping in pg_hba.conf. The other diff --git a/src/backend/libpq/hba.c b/src/backend/libpq/hba.c index 9d63830553e0..5991a21cf2db 100644 --- a/src/backend/libpq/hba.c +++ b/src/backend/libpq/hba.c @@ -29,6 +29,7 @@ #include "catalog/pg_collation.h" #include "catalog/pg_type.h" #include "common/ip.h" +#include "common/string.h" #include "funcapi.h" #include "libpq/ifaddr.h" #include "libpq/libpq.h" @@ -54,7 +55,6 @@ #define MAX_TOKEN 256 -#define MAX_LINE 8192 /* callback data for check_network_callback */ typedef struct check_network_data @@ -166,11 +166,19 @@ pg_isblank(const char c) /* * Grab one token out of the string pointed to by *lineptr. * - * Tokens are strings of non-blank - * characters bounded by blank characters, commas, beginning of line, and - * end of line. Blank means space or tab. Tokens can be delimited by - * double quotes (this allows the inclusion of blanks, but not newlines). - * Comments (started by an unquoted '#') are skipped. + * Tokens are strings of non-blank characters bounded by blank characters, + * commas, beginning of line, and end of line. Blank means space or tab. + * + * Tokens can be delimited by double quotes (this allows the inclusion of + * blanks or '#', but not newlines). As in SQL, write two double-quotes + * to represent a double quote. + * + * Comments (started by an unquoted '#') are skipped, i.e. the remainder + * of the line is ignored. + * + * (Note that line continuation processing happens before tokenization. + * Thus, if a continuation occurs within quoted text or a comment, the + * quoted text or comment is considered to continue to the next line.) * * The token, if any, is returned at *buf (a buffer of size bufsz), and * *lineptr is advanced past the token. @@ -470,6 +478,7 @@ static MemoryContext tokenize_file(const char *filename, FILE *file, List **tok_lines, int elevel) { int line_number = 1; + StringInfoData buf; MemoryContext linecxt; MemoryContext oldcxt; @@ -478,47 +487,72 @@ tokenize_file(const char *filename, FILE *file, List **tok_lines, int elevel) ALLOCSET_SMALL_SIZES); oldcxt = MemoryContextSwitchTo(linecxt); + initStringInfo(&buf); + *tok_lines = NIL; while (!feof(file) && !ferror(file)) { - char rawline[MAX_LINE]; char *lineptr; List *current_line = NIL; char *err_msg = NULL; + int last_backslash_buflen = 0; + int continuations = 0; - if (!fgets(rawline, sizeof(rawline), file)) - { - int save_errno = errno; + /* Collect the next input line, handling backslash continuations */ + resetStringInfo(&buf); - if (!ferror(file)) - break; /* normal EOF */ - /* I/O error! */ - ereport(elevel, - (errcode_for_file_access(), - errmsg("could not read file \"%s\": %m", filename))); - err_msg = psprintf("could not read file \"%s\": %s", - filename, strerror(save_errno)); - rawline[0] = '\0'; - } - if (strlen(rawline) == MAX_LINE - 1) + while (!feof(file) && !ferror(file)) { - /* Line too long! */ - ereport(elevel, - (errcode(ERRCODE_CONFIG_FILE_ERROR), - errmsg("authentication file line too long"), - errcontext("line %d of configuration file \"%s\"", - line_number, filename))); - err_msg = "authentication file line too long"; - } + /* Make sure there's a reasonable amount of room in the buffer */ + enlargeStringInfo(&buf, 128); + + /* Read some data, appending it to what we already have */ + if (fgets(buf.data + buf.len, buf.maxlen - buf.len, file) == NULL) + { + int save_errno = errno; + + if (!ferror(file)) + break; /* normal EOF */ + /* I/O error! */ + ereport(elevel, + (errcode_for_file_access(), + errmsg("could not read file \"%s\": %m", filename))); + err_msg = psprintf("could not read file \"%s\": %s", + filename, strerror(save_errno)); + resetStringInfo(&buf); + break; + } + buf.len += strlen(buf.data + buf.len); + + /* If we haven't got a whole line, loop to read more */ + if (!(buf.len > 0 && buf.data[buf.len - 1] == '\n')) + continue; + + /* Strip trailing newline, including \r in case we're on Windows */ + buf.len = pg_strip_crlf(buf.data); + + /* + * Check for backslash continuation. The backslash must be after + * the last place we found a continuation, else two backslashes + * followed by two \n's would behave surprisingly. + */ + if (buf.len > last_backslash_buflen && + buf.data[buf.len - 1] == '\\') + { + /* Continuation, so strip it and keep reading */ + buf.data[--buf.len] = '\0'; + last_backslash_buflen = buf.len; + continuations++; + continue; + } - /* Strip trailing linebreak from rawline */ - lineptr = rawline + strlen(rawline) - 1; - while (lineptr >= rawline && (*lineptr == '\n' || *lineptr == '\r')) - *lineptr-- = '\0'; + /* Nope, so we have the whole line */ + break; + } /* Parse fields */ - lineptr = rawline; + lineptr = buf.data; while (*lineptr && err_msg == NULL) { List *current_field; @@ -538,12 +572,12 @@ tokenize_file(const char *filename, FILE *file, List **tok_lines, int elevel) tok_line = (TokenizedLine *) palloc(sizeof(TokenizedLine)); tok_line->fields = current_line; tok_line->line_num = line_number; - tok_line->raw_line = pstrdup(rawline); + tok_line->raw_line = pstrdup(buf.data); tok_line->err_msg = err_msg; *tok_lines = lappend(*tok_lines, tok_line); } - line_number++; + line_number += continuations + 1; } MemoryContextSwitchTo(oldcxt); diff --git a/src/test/authentication/t/001_password.pl b/src/test/authentication/t/001_password.pl index 1b4323fe2a60..59b1b403c554 100644 --- a/src/test/authentication/t/001_password.pl +++ b/src/test/authentication/t/001_password.pl @@ -29,7 +29,8 @@ sub reset_pg_hba my $hba_method = shift; unlink($node->data_dir . '/pg_hba.conf'); - $node->append_conf('pg_hba.conf', "local all all $hba_method"); + # just for testing purposes, use a continuation line + $node->append_conf('pg_hba.conf', "local all all\\\n $hba_method"); $node->reload; return; } From be4b0c0077e6a1f7be0965f8d93696e0e0eadb52 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 3 Sep 2020 16:52:09 -0400 Subject: [PATCH 061/589] Avoid lockup of a parallel worker when reporting a long error message. Because sigsetjmp() will restore the initial state with signals blocked, the code path in bgworker.c for reporting an error and exiting would execute that way. Usually this is fairly harmless; but if a parallel worker had an error message exceeding the shared-memory communication buffer size (16K) it would lock up, because it would wait for a resume-sending signal from its parallel leader which it would never detect. To fix, just unblock signals at the appropriate point. This can be shown to fail back to 9.6. The lack of parallel query infrastructure makes it difficult to provide a simple test case for 9.5; but I'm pretty sure the issue exists in some form there as well, so apply the code change there too. Vignesh C, reviewed by Bharath Rupireddy, Robert Haas, and myself Discussion: https://postgr.es/m/CALDaNm1d1hHPZUg3xU4XjtWBOLCrA+-2cJcLpw-cePZ=GgDVfA@mail.gmail.com --- src/backend/postmaster/bgworker.c | 11 +++++++++-- src/test/regress/expected/select_parallel.out | 5 +++-- src/test/regress/sql/select_parallel.sql | 3 ++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index beb5e85434ce..d043ced6861a 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -743,7 +743,7 @@ StartBackgroundWorker(void) /* * If an exception is encountered, processing resumes here. * - * See notes in postgres.c about the design of this coding. + * We just need to clean up, report the error, and go away. */ if (sigsetjmp(local_sigjmp_buf, 1) != 0) { @@ -753,7 +753,14 @@ StartBackgroundWorker(void) /* Prevent interrupts while cleaning up */ HOLD_INTERRUPTS(); - /* Report the error to the server log */ + /* + * sigsetjmp will have blocked all signals, but we may need to accept + * signals while communicating with our parallel leader. Once we've + * done HOLD_INTERRUPTS() it should be safe to unblock signals. + */ + BackgroundWorkerUnblockSignals(); + + /* Report the error to the parallel leader and the server log */ EmitErrorReport(); /* diff --git a/src/test/regress/expected/select_parallel.out b/src/test/regress/expected/select_parallel.out index 96dfb7c8dd63..9b0c418db715 100644 --- a/src/test/regress/expected/select_parallel.out +++ b/src/test/regress/expected/select_parallel.out @@ -1056,10 +1056,11 @@ EXPLAIN (analyze, timing off, summary off, costs off) SELECT * FROM tenk1; ROLLBACK TO SAVEPOINT settings; -- provoke error in worker +-- (make the error message long enough to require multiple bufferloads) SAVEPOINT settings; SET LOCAL force_parallel_mode = 1; -select stringu1::int2 from tenk1 where unique1 = 1; -ERROR: invalid input syntax for type smallint: "BAAAAA" +select (stringu1 || repeat('abcd', 5000))::int2 from tenk1 where unique1 = 1; +ERROR: invalid input syntax for type smallint: "BAAAAAabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd" CONTEXT: parallel worker ROLLBACK TO SAVEPOINT settings; -- test interaction with set-returning functions diff --git a/src/test/regress/sql/select_parallel.sql b/src/test/regress/sql/select_parallel.sql index 11e773553324..5a01a98b2689 100644 --- a/src/test/regress/sql/select_parallel.sql +++ b/src/test/regress/sql/select_parallel.sql @@ -400,9 +400,10 @@ EXPLAIN (analyze, timing off, summary off, costs off) SELECT * FROM tenk1; ROLLBACK TO SAVEPOINT settings; -- provoke error in worker +-- (make the error message long enough to require multiple bufferloads) SAVEPOINT settings; SET LOCAL force_parallel_mode = 1; -select stringu1::int2 from tenk1 where unique1 = 1; +select (stringu1 || repeat('abcd', 5000))::int2 from tenk1 where unique1 = 1; ROLLBACK TO SAVEPOINT settings; -- test interaction with set-returning functions From 67a472d71c98c3d2fa322a1b4013080b20720b98 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 3 Sep 2020 20:09:18 -0400 Subject: [PATCH 062/589] Remove arbitrary restrictions on password length. This patch started out with the goal of harmonizing various arbitrary limits on password length, but after awhile a better idea emerged: let's just get rid of those fixed limits. recv_password_packet() has an arbitrary limit on the packet size, which we don't really need, so just drop it. (Note that this doesn't really affect anything for MD5 or SCRAM password verification, since those will hash the user's password to something shorter anyway. It does matter for auth methods that require a cleartext password.) Likewise remove the arbitrary error condition in pg_saslprep(). The remaining limits are mostly in client-side code that prompts for passwords. To improve those, refactor simple_prompt() so that it allocates its own result buffer that can be made as big as necessary. Actually, it proves best to make a separate routine pg_get_line() that has essentially the semantics of fgets(), except that it allocates a suitable result buffer and hence will never return a truncated line. (pg_get_line has a lot of potential applications to replace randomly-sized fgets buffers elsewhere, but I'll leave that for another patch.) I built pg_get_line() atop stringinfo.c, which requires moving that code to src/common/; but that seems fine since it was a poor fit for src/port/ anyway. This patch is mostly mine, but it owes a good deal to Nathan Bossart who pressed for a solution to the password length problem and created a predecessor patch. Also thanks to Peter Eisentraut and Stephen Frost for ideas and discussion. Discussion: https://postgr.es/m/09512C4F-8CB9-4021-B455-EF4C4F0D55A0@amazon.com --- contrib/oid2name/oid2name.c | 11 ++-- contrib/vacuumlo/vacuumlo.c | 18 +++---- src/backend/libpq/auth.c | 2 +- src/bin/initdb/initdb.c | 21 ++++---- src/bin/pg_basebackup/streamutil.c | 13 ++--- src/bin/pg_dump/pg_backup_db.c | 28 +++++----- src/bin/pg_dump/pg_dumpall.c | 18 +++---- src/bin/pgbench/pgbench.c | 11 ++-- src/bin/psql/command.c | 22 ++++---- src/bin/psql/startup.c | 14 +++-- src/bin/scripts/common.c | 34 +++++++----- src/bin/scripts/createuser.c | 18 +++---- src/bin/scripts/dropuser.c | 6 +-- src/common/Makefile | 4 +- src/common/pg_get_line.c | 85 ++++++++++++++++++++++++++++++ src/common/saslprep.c | 18 ------- src/{port => common}/sprompt.c | 45 ++++++---------- src/include/common/string.h | 7 +++ src/include/port.h | 4 -- src/port/Makefile | 3 +- src/tools/msvc/Mkvcbuild.pm | 6 +-- 21 files changed, 221 insertions(+), 167 deletions(-) create mode 100644 src/common/pg_get_line.c rename src/{port => common}/sprompt.c (81%) diff --git a/contrib/oid2name/oid2name.c b/contrib/oid2name/oid2name.c index 91b7958c48ef..5a884e29049f 100644 --- a/contrib/oid2name/oid2name.c +++ b/contrib/oid2name/oid2name.c @@ -12,6 +12,7 @@ #include "catalog/pg_class_d.h" #include "common/connect.h" #include "common/logging.h" +#include "common/string.h" #include "getopt_long.h" #include "libpq-fe.h" #include "pg_getopt.h" @@ -293,8 +294,7 @@ PGconn * sql_conn(struct options *my_opts) { PGconn *conn; - bool have_password = false; - char password[100]; + char *password = NULL; bool new_pass; PGresult *res; @@ -316,7 +316,7 @@ sql_conn(struct options *my_opts) keywords[2] = "user"; values[2] = my_opts->username; keywords[3] = "password"; - values[3] = have_password ? password : NULL; + values[3] = password; keywords[4] = "dbname"; values[4] = my_opts->dbname; keywords[5] = "fallback_application_name"; @@ -336,11 +336,10 @@ sql_conn(struct options *my_opts) if (PQstatus(conn) == CONNECTION_BAD && PQconnectionNeedsPassword(conn) && - !have_password) + !password) { PQfinish(conn); - simple_prompt("Password: ", password, sizeof(password), false); - have_password = true; + password = simple_prompt("Password: ", false); new_pass = true; } } while (new_pass); diff --git a/contrib/vacuumlo/vacuumlo.c b/contrib/vacuumlo/vacuumlo.c index e4019fafaa9e..532cc596c412 100644 --- a/contrib/vacuumlo/vacuumlo.c +++ b/contrib/vacuumlo/vacuumlo.c @@ -24,6 +24,7 @@ #include "catalog/pg_class_d.h" #include "common/connect.h" #include "common/logging.h" +#include "common/string.h" #include "getopt_long.h" #include "libpq-fe.h" #include "pg_getopt.h" @@ -69,15 +70,11 @@ vacuumlo(const char *database, const struct _param *param) int i; bool new_pass; bool success = true; - static bool have_password = false; - static char password[100]; + static char *password = NULL; /* Note: password can be carried over from a previous call */ - if (param->pg_prompt == TRI_YES && !have_password) - { - simple_prompt("Password: ", password, sizeof(password), false); - have_password = true; - } + if (param->pg_prompt == TRI_YES && !password) + password = simple_prompt("Password: ", false); /* * Start the connection. Loop until we have a password if requested by @@ -97,7 +94,7 @@ vacuumlo(const char *database, const struct _param *param) keywords[2] = "user"; values[2] = param->pg_user; keywords[3] = "password"; - values[3] = have_password ? password : NULL; + values[3] = password; keywords[4] = "dbname"; values[4] = database; keywords[5] = "fallback_application_name"; @@ -115,12 +112,11 @@ vacuumlo(const char *database, const struct _param *param) if (PQstatus(conn) == CONNECTION_BAD && PQconnectionNeedsPassword(conn) && - !have_password && + !password && param->pg_prompt != TRI_NO) { PQfinish(conn); - simple_prompt("Password: ", password, sizeof(password), false); - have_password = true; + password = simple_prompt("Password: ", false); new_pass = true; } } while (new_pass); diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c index 02b6c3f127c6..36565df4fc1e 100644 --- a/src/backend/libpq/auth.c +++ b/src/backend/libpq/auth.c @@ -698,7 +698,7 @@ recv_password_packet(Port *port) } initStringInfo(&buf); - if (pq_getmessage(&buf, 1000)) /* receive password */ + if (pq_getmessage(&buf, 0)) /* receive password */ { /* EOF - pq_getmessage already logged a suitable message */ pfree(buf.data); diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 786672b1b655..73ddf408654a 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -67,6 +67,7 @@ #include "common/file_utils.h" #include "common/logging.h" #include "common/restricted_token.h" +#include "common/string.h" #include "common/username.h" #include "fe_utils/string_utils.h" #include "getaddrinfo.h" @@ -1481,23 +1482,25 @@ setup_auth(FILE *cmdfd) static void get_su_pwd(void) { - char pwd1[100]; - char pwd2[100]; + char *pwd1; if (pwprompt) { /* * Read password from terminal */ + char *pwd2; + printf("\n"); fflush(stdout); - simple_prompt("Enter new superuser password: ", pwd1, sizeof(pwd1), false); - simple_prompt("Enter it again: ", pwd2, sizeof(pwd2), false); + pwd1 = simple_prompt("Enter new superuser password: ", false); + pwd2 = simple_prompt("Enter it again: ", false); if (strcmp(pwd1, pwd2) != 0) { fprintf(stderr, _("Passwords didn't match.\n")); exit(1); } + free(pwd2); } else { @@ -1510,7 +1513,6 @@ get_su_pwd(void) * for now. */ FILE *pwf = fopen(pwfilename, "r"); - int i; if (!pwf) { @@ -1518,7 +1520,8 @@ get_su_pwd(void) pwfilename); exit(1); } - if (!fgets(pwd1, sizeof(pwd1), pwf)) + pwd1 = pg_get_line(pwf); + if (!pwd1) { if (ferror(pwf)) pg_log_error("could not read password from file \"%s\": %m", @@ -1530,12 +1533,10 @@ get_su_pwd(void) } fclose(pwf); - i = strlen(pwd1); - while (i > 0 && (pwd1[i - 1] == '\r' || pwd1[i - 1] == '\n')) - pwd1[--i] = '\0'; + (void) pg_strip_crlf(pwd1); } - superuser_password = pg_strdup(pwd1); + superuser_password = pwd1; } /* diff --git a/src/bin/pg_basebackup/streamutil.c b/src/bin/pg_basebackup/streamutil.c index c08003e7f2c7..be653ebb2d94 100644 --- a/src/bin/pg_basebackup/streamutil.c +++ b/src/bin/pg_basebackup/streamutil.c @@ -22,6 +22,7 @@ #include "common/fe_memutils.h" #include "common/file_perm.h" #include "common/logging.h" +#include "common/string.h" #include "datatype/timestamp.h" #include "port/pg_bswap.h" #include "pqexpbuffer.h" @@ -49,8 +50,7 @@ char *dbuser = NULL; char *dbport = NULL; char *dbname = NULL; int dbgetpassword = 0; /* 0=auto, -1=never, 1=always */ -static bool have_password = false; -static char password[100]; +static char *password = NULL; PGconn *conn = NULL; /* @@ -150,20 +150,21 @@ GetConnection(void) } /* If -W was given, force prompt for password, but only the first time */ - need_password = (dbgetpassword == 1 && !have_password); + need_password = (dbgetpassword == 1 && !password); do { /* Get a new password if appropriate */ if (need_password) { - simple_prompt("Password: ", password, sizeof(password), false); - have_password = true; + if (password) + free(password); + password = simple_prompt("Password: ", false); need_password = false; } /* Use (or reuse, on a subsequent connection) password if we have it */ - if (have_password) + if (password) { keywords[i] = "password"; values[i] = password; diff --git a/src/bin/pg_dump/pg_backup_db.c b/src/bin/pg_dump/pg_backup_db.c index 94af11b80a39..12899e26e292 100644 --- a/src/bin/pg_dump/pg_backup_db.c +++ b/src/bin/pg_dump/pg_backup_db.c @@ -18,6 +18,7 @@ #endif #include "common/connect.h" +#include "common/string.h" #include "dumputils.h" #include "fe_utils/string_utils.h" #include "parallel.h" @@ -122,7 +123,6 @@ _connectDB(ArchiveHandle *AH, const char *reqdb, const char *requser) const char *newdb; const char *newuser; char *password; - char passbuf[100]; bool new_pass; if (!reqdb) @@ -141,10 +141,7 @@ _connectDB(ArchiveHandle *AH, const char *reqdb, const char *requser) password = AH->savedPassword; if (AH->promptPassword == TRI_YES && password == NULL) - { - simple_prompt("Password: ", passbuf, sizeof(passbuf), false); - password = passbuf; - } + password = simple_prompt("Password: ", false); initPQExpBuffer(&connstr); appendPQExpBufferStr(&connstr, "dbname="); @@ -191,8 +188,9 @@ _connectDB(ArchiveHandle *AH, const char *reqdb, const char *requser) if (AH->promptPassword != TRI_NO) { - simple_prompt("Password: ", passbuf, sizeof(passbuf), false); - password = passbuf; + if (password && password != AH->savedPassword) + free(password); + password = simple_prompt("Password: ", false); } else fatal("connection needs password"); @@ -201,6 +199,9 @@ _connectDB(ArchiveHandle *AH, const char *reqdb, const char *requser) } } while (new_pass); + if (password && password != AH->savedPassword) + free(password); + /* * We want to remember connection's actual password, whether or not we got * it by prompting. So we don't just store the password variable. @@ -242,7 +243,6 @@ ConnectDatabase(Archive *AHX, { ArchiveHandle *AH = (ArchiveHandle *) AHX; char *password; - char passbuf[100]; bool new_pass; if (AH->connection) @@ -251,10 +251,8 @@ ConnectDatabase(Archive *AHX, password = AH->savedPassword; if (prompt_password == TRI_YES && password == NULL) - { - simple_prompt("Password: ", passbuf, sizeof(passbuf), false); - password = passbuf; - } + password = simple_prompt("Password: ", false); + AH->promptPassword = prompt_password; /* @@ -293,8 +291,7 @@ ConnectDatabase(Archive *AHX, prompt_password != TRI_NO) { PQfinish(AH->connection); - simple_prompt("Password: ", passbuf, sizeof(passbuf), false); - password = passbuf; + password = simple_prompt("Password: ", false); new_pass = true; } } while (new_pass); @@ -309,6 +306,9 @@ ConnectDatabase(Archive *AHX, PQclear(ExecuteSqlQueryForSingleRow((Archive *) AH, ALWAYS_SECURE_SEARCH_PATH_SQL)); + if (password && password != AH->savedPassword) + free(password); + /* * We want to remember connection's actual password, whether or not we got * it by prompting. So we don't just store the password variable. diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 2c82b39af0d2..97d2b8dac1c6 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -21,6 +21,7 @@ #include "common/connect.h" #include "common/file_utils.h" #include "common/logging.h" +#include "common/string.h" #include "dumputils.h" #include "fe_utils/string_utils.h" #include "getopt_long.h" @@ -1643,14 +1644,10 @@ connectDatabase(const char *dbname, const char *connection_string, const char **keywords = NULL; const char **values = NULL; PQconninfoOption *conn_opts = NULL; - static bool have_password = false; - static char password[100]; + static char *password = NULL; - if (prompt_password == TRI_YES && !have_password) - { - simple_prompt("Password: ", password, sizeof(password), false); - have_password = true; - } + if (prompt_password == TRI_YES && !password) + password = simple_prompt("Password: ", false); /* * Start the connection. Loop until we have a password if requested by @@ -1730,7 +1727,7 @@ connectDatabase(const char *dbname, const char *connection_string, values[i] = pguser; i++; } - if (have_password) + if (password) { keywords[i] = "password"; values[i] = password; @@ -1757,12 +1754,11 @@ connectDatabase(const char *dbname, const char *connection_string, if (PQstatus(conn) == CONNECTION_BAD && PQconnectionNeedsPassword(conn) && - !have_password && + !password && prompt_password != TRI_NO) { PQfinish(conn); - simple_prompt("Password: ", password, sizeof(password), false); - have_password = true; + password = simple_prompt("Password: ", false); new_pass = true; } } while (new_pass); diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 08a5947a9e66..332eabf6379e 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -59,6 +59,7 @@ #include "common/int.h" #include "common/logging.h" +#include "common/string.h" #include "fe_utils/cancel.h" #include "fe_utils/conditional.h" #include "getopt_long.h" @@ -1174,8 +1175,7 @@ doConnect(void) { PGconn *conn; bool new_pass; - static bool have_password = false; - static char password[100]; + static char *password = NULL; /* * Start the connection. Loop until we have a password if requested by @@ -1195,7 +1195,7 @@ doConnect(void) keywords[2] = "user"; values[2] = login; keywords[3] = "password"; - values[3] = have_password ? password : NULL; + values[3] = password; keywords[4] = "dbname"; values[4] = dbName; keywords[5] = "fallback_application_name"; @@ -1215,11 +1215,10 @@ doConnect(void) if (PQstatus(conn) == CONNECTION_BAD && PQconnectionNeedsPassword(conn) && - !have_password) + !password) { PQfinish(conn); - simple_prompt("Password: ", password, sizeof(password), false); - have_password = true; + password = simple_prompt("Password: ", false); new_pass = true; } } while (new_pass); diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c index 9902a4a2ba8e..d4aa0976b5bf 100644 --- a/src/bin/psql/command.c +++ b/src/bin/psql/command.c @@ -26,6 +26,7 @@ #include "command.h" #include "common.h" #include "common/logging.h" +#include "common/string.h" #include "copy.h" #include "crosstabview.h" #include "describe.h" @@ -1964,11 +1965,11 @@ exec_command_password(PsqlScanState scan_state, bool active_branch) { char *opt0 = psql_scan_slash_option(scan_state, OT_SQLID, NULL, true); - char pw1[100]; - char pw2[100]; + char *pw1; + char *pw2; - simple_prompt("Enter new password: ", pw1, sizeof(pw1), false); - simple_prompt("Enter it again: ", pw2, sizeof(pw2), false); + pw1 = simple_prompt("Enter new password: ", false); + pw2 = simple_prompt("Enter it again: ", false); if (strcmp(pw1, pw2) != 0) { @@ -2013,6 +2014,8 @@ exec_command_password(PsqlScanState scan_state, bool active_branch) if (opt0) free(opt0); + free(pw1); + free(pw2); } else ignore_slash_options(scan_state); @@ -2058,8 +2061,7 @@ exec_command_prompt(PsqlScanState scan_state, bool active_branch, if (!pset.inputfile) { - result = (char *) pg_malloc(4096); - simple_prompt(prompt_text, result, 4096, true); + result = simple_prompt(prompt_text, true); } else { @@ -2982,19 +2984,19 @@ copy_previous_query(PQExpBuffer query_buf, PQExpBuffer previous_buf) static char * prompt_for_password(const char *username) { - char buf[100]; + char *result; if (username == NULL || username[0] == '\0') - simple_prompt("Password: ", buf, sizeof(buf), false); + result = simple_prompt("Password: ", false); else { char *prompt_text; prompt_text = psprintf(_("Password for user %s: "), username); - simple_prompt(prompt_text, buf, sizeof(buf), false); + result = simple_prompt(prompt_text, false); free(prompt_text); } - return pg_strdup(buf); + return result; } static bool diff --git a/src/bin/psql/startup.c b/src/bin/psql/startup.c index 3302bd4dd327..8232a0143bc9 100644 --- a/src/bin/psql/startup.c +++ b/src/bin/psql/startup.c @@ -17,6 +17,7 @@ #include "command.h" #include "common.h" #include "common/logging.h" +#include "common/string.h" #include "describe.h" #include "fe_utils/print.h" #include "getopt_long.h" @@ -119,8 +120,7 @@ main(int argc, char *argv[]) { struct adhoc_opts options; int successResult; - bool have_password = false; - char password[100]; + char *password = NULL; bool new_pass; pg_logging_init(argv[0]); @@ -233,8 +233,7 @@ main(int argc, char *argv[]) * offer a potentially wrong one. Typical uses of this option are * noninteractive anyway. */ - simple_prompt("Password: ", password, sizeof(password), false); - have_password = true; + password = simple_prompt("Password: ", false); } /* loop until we have a password if requested by backend */ @@ -251,7 +250,7 @@ main(int argc, char *argv[]) keywords[2] = "user"; values[2] = options.username; keywords[3] = "password"; - values[3] = have_password ? password : NULL; + values[3] = password; keywords[4] = "dbname"; /* see do_connect() */ values[4] = (options.list_dbs && options.dbname == NULL) ? "postgres" : options.dbname; @@ -269,7 +268,7 @@ main(int argc, char *argv[]) if (PQstatus(pset.db) == CONNECTION_BAD && PQconnectionNeedsPassword(pset.db) && - !have_password && + !password && pset.getPassword != TRI_NO) { /* @@ -287,9 +286,8 @@ main(int argc, char *argv[]) password_prompt = pg_strdup(_("Password: ")); PQfinish(pset.db); - simple_prompt(password_prompt, password, sizeof(password), false); + password = simple_prompt(password_prompt, false); free(password_prompt); - have_password = true; new_pass = true; } } while (new_pass); diff --git a/src/bin/scripts/common.c b/src/bin/scripts/common.c index 420d0d11a5a1..e987eef23434 100644 --- a/src/bin/scripts/common.c +++ b/src/bin/scripts/common.c @@ -20,6 +20,7 @@ #include "common.h" #include "common/connect.h" #include "common/logging.h" +#include "common/string.h" #include "fe_utils/cancel.h" #include "fe_utils/string_utils.h" @@ -68,18 +69,17 @@ connectDatabase(const char *dbname, const char *pghost, { PGconn *conn; bool new_pass; - static bool have_password = false; - static char password[100]; + static char *password = NULL; - if (!allow_password_reuse) - have_password = false; - - if (!have_password && prompt_password == TRI_YES) + if (!allow_password_reuse && password) { - simple_prompt("Password: ", password, sizeof(password), false); - have_password = true; + free(password); + password = NULL; } + if (!password && prompt_password == TRI_YES) + password = simple_prompt("Password: ", false); + /* * Start the connection. Loop until we have a password if requested by * backend. @@ -96,7 +96,7 @@ connectDatabase(const char *dbname, const char *pghost, keywords[2] = "user"; values[2] = pguser; keywords[3] = "password"; - values[3] = have_password ? password : NULL; + values[3] = password; keywords[4] = "dbname"; values[4] = dbname; keywords[5] = "fallback_application_name"; @@ -122,8 +122,9 @@ connectDatabase(const char *dbname, const char *pghost, prompt_password != TRI_NO) { PQfinish(conn); - simple_prompt("Password: ", password, sizeof(password), false); - have_password = true; + if (password) + free(password); + password = simple_prompt("Password: ", false); new_pass = true; } } while (new_pass); @@ -444,14 +445,21 @@ yesno_prompt(const char *question) for (;;) { - char resp[10]; + char *resp; - simple_prompt(prompt, resp, sizeof(resp), true); + resp = simple_prompt(prompt, true); if (strcmp(resp, _(PG_YESLETTER)) == 0) + { + free(resp); return true; + } if (strcmp(resp, _(PG_NOLETTER)) == 0) + { + free(resp); return false; + } + free(resp); printf(_("Please answer \"%s\" or \"%s\".\n"), _(PG_YESLETTER), _(PG_NOLETTER)); diff --git a/src/bin/scripts/createuser.c b/src/bin/scripts/createuser.c index 9ced079ac759..6179199563c4 100644 --- a/src/bin/scripts/createuser.c +++ b/src/bin/scripts/createuser.c @@ -13,6 +13,7 @@ #include "postgres_fe.h" #include "common.h" #include "common/logging.h" +#include "common/string.h" #include "fe_utils/simple_list.h" #include "fe_utils/string_utils.h" @@ -63,8 +64,6 @@ main(int argc, char *argv[]) int conn_limit = -2; /* less than minimum valid value */ bool pwprompt = false; char *newpassword = NULL; - char newuser_buf[128]; - char newpassword_buf[100]; /* Tri-valued variables. */ enum trivalue createdb = TRI_DEFAULT, @@ -191,9 +190,7 @@ main(int argc, char *argv[]) { if (interactive) { - simple_prompt("Enter name of role to add: ", - newuser_buf, sizeof(newuser_buf), true); - newuser = newuser_buf; + newuser = simple_prompt("Enter name of role to add: ", true); } else { @@ -206,17 +203,16 @@ main(int argc, char *argv[]) if (pwprompt) { - char pw2[100]; + char *pw2; - simple_prompt("Enter password for new role: ", - newpassword_buf, sizeof(newpassword_buf), false); - simple_prompt("Enter it again: ", pw2, sizeof(pw2), false); - if (strcmp(newpassword_buf, pw2) != 0) + newpassword = simple_prompt("Enter password for new role: ", false); + pw2 = simple_prompt("Enter it again: ", false); + if (strcmp(newpassword, pw2) != 0) { fprintf(stderr, _("Passwords didn't match.\n")); exit(1); } - newpassword = newpassword_buf; + free(pw2); } if (superuser == 0) diff --git a/src/bin/scripts/dropuser.c b/src/bin/scripts/dropuser.c index fee270d4f6d1..f7ddd1402db2 100644 --- a/src/bin/scripts/dropuser.c +++ b/src/bin/scripts/dropuser.c @@ -13,6 +13,7 @@ #include "postgres_fe.h" #include "common.h" #include "common/logging.h" +#include "common/string.h" #include "fe_utils/string_utils.h" @@ -47,7 +48,6 @@ main(int argc, char *argv[]) enum trivalue prompt_password = TRI_DEFAULT; bool echo = false; bool interactive = false; - char dropuser_buf[128]; PQExpBufferData sql; @@ -112,9 +112,7 @@ main(int argc, char *argv[]) { if (interactive) { - simple_prompt("Enter name of role to drop: ", - dropuser_buf, sizeof(dropuser_buf), true); - dropuser = dropuser_buf; + dropuser = simple_prompt("Enter name of role to drop: ", true); } else { diff --git a/src/common/Makefile b/src/common/Makefile index 16619e4ba883..ad8fd9e41c47 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -63,6 +63,7 @@ OBJS_COMMON = \ kwlookup.o \ link-canary.o \ md5.o \ + pg_get_line.o \ pg_lzcompress.o \ pgfnames.o \ psprintf.o \ @@ -92,7 +93,8 @@ OBJS_FRONTEND = \ fe_memutils.o \ file_utils.o \ logging.o \ - restricted_token.o + restricted_token.o \ + sprompt.o # foo.o, foo_shlib.o, and foo_srv.o are all built from foo.c OBJS_SHLIB = $(OBJS_FRONTEND:%.o=%_shlib.o) diff --git a/src/common/pg_get_line.c b/src/common/pg_get_line.c new file mode 100644 index 000000000000..38433675d433 --- /dev/null +++ b/src/common/pg_get_line.c @@ -0,0 +1,85 @@ +/*------------------------------------------------------------------------- + * + * pg_get_line.c + * fgets() with an expansible result buffer + * + * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/common/pg_get_line.c + * + *------------------------------------------------------------------------- + */ +#ifndef FRONTEND +#include "postgres.h" +#else +#include "postgres_fe.h" +#endif + +#include "common/string.h" +#include "lib/stringinfo.h" + + +/* + * pg_get_line() + * + * This is meant to be equivalent to fgets(), except that instead of + * reading into a caller-supplied, fixed-size buffer, it reads into + * a palloc'd (in frontend, really malloc'd) string, which is resized + * as needed to handle indefinitely long input lines. The caller is + * responsible for pfree'ing the result string when appropriate. + * + * As with fgets(), returns NULL if there is a read error or if no + * characters are available before EOF. The caller can distinguish + * these cases by checking ferror(stream). + * + * Since this is meant to be equivalent to fgets(), the trailing newline + * (if any) is not stripped. Callers may wish to apply pg_strip_crlf(). + * + * Note that while I/O errors are reflected back to the caller to be + * dealt with, an OOM condition for the palloc'd buffer will not be; + * there'll be an ereport(ERROR) or exit(1) inside stringinfo.c. + */ +char * +pg_get_line(FILE *stream) +{ + StringInfoData buf; + + initStringInfo(&buf); + + /* Read some data, appending it to whatever we already have */ + while (fgets(buf.data + buf.len, buf.maxlen - buf.len, stream) != NULL) + { + buf.len += strlen(buf.data + buf.len); + + /* Done if we have collected a newline */ + if (buf.len > 0 && buf.data[buf.len - 1] == '\n') + return buf.data; + + /* Make some more room in the buffer, and loop to read more data */ + enlargeStringInfo(&buf, 128); + } + + /* Did fgets() fail because of an I/O error? */ + if (ferror(stream)) + { + /* ensure that free() doesn't mess up errno */ + int save_errno = errno; + + pfree(buf.data); + errno = save_errno; + return NULL; + } + + /* If we read no data before reaching EOF, we should return NULL */ + if (buf.len == 0) + { + pfree(buf.data); + return NULL; + } + + /* No newline at EOF ... so return what we have */ + return buf.data; +} diff --git a/src/common/saslprep.c b/src/common/saslprep.c index 2dedf6b0fb68..d60452f75f28 100644 --- a/src/common/saslprep.c +++ b/src/common/saslprep.c @@ -29,12 +29,6 @@ #include "common/unicode_norm.h" #include "mb/pg_wchar.h" -/* - * Limit on how large password's we will try to process. A password - * larger than this will be treated the same as out-of-memory. - */ -#define MAX_PASSWORD_LENGTH 1024 - /* * In backend, we will use palloc/pfree. In frontend, use malloc, and * return SASLPREP_OOM on out-of-memory. @@ -1078,18 +1072,6 @@ pg_saslprep(const char *input, char **output) /* Ensure we return *output as NULL on failure */ *output = NULL; - /* Check that the password isn't stupendously long */ - if (strlen(input) > MAX_PASSWORD_LENGTH) - { -#ifndef FRONTEND - ereport(ERROR, - (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), - errmsg("password too long"))); -#else - return SASLPREP_OOM; -#endif - } - /* * Quick check if the input is pure ASCII. An ASCII string requires no * further processing. diff --git a/src/port/sprompt.c b/src/common/sprompt.c similarity index 81% rename from src/port/sprompt.c rename to src/common/sprompt.c index 6d8a8b2609ff..0ec75da5bfe6 100644 --- a/src/port/sprompt.c +++ b/src/common/sprompt.c @@ -8,12 +8,15 @@ * * * IDENTIFICATION - * src/port/sprompt.c + * src/common/sprompt.c * *------------------------------------------------------------------------- */ #include "c.h" +#include "common/fe_memutils.h" +#include "common/string.h" + #ifdef HAVE_TERMIOS_H #include #endif @@ -26,20 +29,17 @@ * passwords interactively. Reads from /dev/tty or stdin/stderr. * * prompt: The prompt to print, or NULL if none (automatically localized) - * destination: buffer in which to store result - * destlen: allocated length of destination * echo: Set to false if you want to hide what is entered (for passwords) * - * The input (without trailing newline) is returned in the destination buffer, - * with a '\0' appended. + * The input (without trailing newline) is returned as a malloc'd string. + * Caller is responsible for freeing it when done. */ -void -simple_prompt(const char *prompt, char *destination, size_t destlen, bool echo) +char * +simple_prompt(const char *prompt, bool echo) { - int length; + char *result; FILE *termin, *termout; - #if defined(HAVE_TERMIOS_H) struct termios t_orig, t; @@ -126,29 +126,14 @@ simple_prompt(const char *prompt, char *destination, size_t destlen, bool echo) fflush(termout); } - if (fgets(destination, destlen, termin) == NULL) - destination[0] = '\0'; + result = pg_get_line(termin); - length = strlen(destination); - if (length > 0 && destination[length - 1] != '\n') - { - /* eat rest of the line */ - char buf[128]; - int buflen; - - do - { - if (fgets(buf, sizeof(buf), termin) == NULL) - break; - buflen = strlen(buf); - } while (buflen > 0 && buf[buflen - 1] != '\n'); - } + /* If we failed to read anything, just return an empty string */ + if (result == NULL) + result = pg_strdup(""); /* strip trailing newline, including \r in case we're on Windows */ - while (length > 0 && - (destination[length - 1] == '\n' || - destination[length - 1] == '\r')) - destination[--length] = '\0'; + (void) pg_strip_crlf(result); if (!echo) { @@ -169,4 +154,6 @@ simple_prompt(const char *prompt, char *destination, size_t destlen, bool echo) fclose(termin); fclose(termout); } + + return result; } diff --git a/src/include/common/string.h b/src/include/common/string.h index 5113c04434c8..18aa1dc5aa5c 100644 --- a/src/include/common/string.h +++ b/src/include/common/string.h @@ -10,10 +10,17 @@ #ifndef COMMON_STRING_H #define COMMON_STRING_H +/* functions in src/common/string.c */ extern bool pg_str_endswith(const char *str, const char *end); extern int strtoint(const char *pg_restrict str, char **pg_restrict endptr, int base); extern void pg_clean_ascii(char *str); extern int pg_strip_crlf(char *str); +/* functions in src/common/pg_get_line.c */ +extern char *pg_get_line(FILE *stream); + +/* functions in src/common/sprompt.c */ +extern char *simple_prompt(const char *prompt, bool echo); + #endif /* COMMON_STRING_H */ diff --git a/src/include/port.h b/src/include/port.h index 271ff0d00bcc..84bf2c363f55 100644 --- a/src/include/port.h +++ b/src/include/port.h @@ -213,10 +213,6 @@ extern char *pg_strerror_r(int errnum, char *buf, size_t buflen); /* Wrap strsignal(), or provide our own version if necessary */ extern const char *pg_strsignal(int signum); -/* Portable prompt handling */ -extern void simple_prompt(const char *prompt, char *destination, size_t destlen, - bool echo); - extern int pclose_check(FILE *stream); /* Global variable holding time zone information. */ diff --git a/src/port/Makefile b/src/port/Makefile index 8defa1257bb8..e41b005c4f1b 100644 --- a/src/port/Makefile +++ b/src/port/Makefile @@ -35,6 +35,8 @@ include $(top_builddir)/src/Makefile.global override CPPFLAGS := -I$(top_builddir)/src/port -DFRONTEND $(CPPFLAGS) LIBS += $(PTHREAD_LIBS) +# If you add objects here, see also src/tools/msvc/Mkvcbuild.pm + OBJS = \ $(LIBOBJS) \ $(PG_CRC32C_OBJS) \ @@ -55,7 +57,6 @@ OBJS = \ qsort_arg.o \ quotes.o \ snprintf.o \ - sprompt.o \ strerror.o \ tar.o \ thread.o diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 20da7985c101..536df5a92e75 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -102,7 +102,7 @@ sub mkvcbuild pread.c pwrite.c pg_bitutils.c pg_strong_random.c pgcheckdir.c pgmkdirp.c pgsleep.c pgstrcasecmp.c pqsignal.c mkdtemp.c qsort.c qsort_arg.c quotes.c system.c - sprompt.c strerror.c tar.c thread.c + strerror.c tar.c thread.c win32env.c win32error.c win32security.c win32setlocale.c); push(@pgportfiles, 'strtof.c') if ($vsVersion < '14.00'); @@ -123,7 +123,7 @@ sub mkvcbuild config_info.c controldata_utils.c d2s.c encnames.c exec.c f2s.c file_perm.c hashfn.c ip.c jsonapi.c keywords.c kwlookup.c link-canary.c md5.c - pg_lzcompress.c pgfnames.c psprintf.c relpath.c rmtree.c + pg_get_line.c pg_lzcompress.c pgfnames.c psprintf.c relpath.c rmtree.c saslprep.c scram-common.c string.c stringinfo.c unicode_norm.c username.c wait_error.c wchar.c); @@ -139,7 +139,7 @@ sub mkvcbuild our @pgcommonfrontendfiles = ( @pgcommonallfiles, qw(fe_memutils.c file_utils.c - logging.c restricted_token.c)); + logging.c restricted_token.c sprompt.c)); our @pgcommonbkndfiles = @pgcommonallfiles; From 844c05abc3f1c1703bf17cf44ab66351ed9711d2 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Fri, 4 Sep 2020 10:36:35 +0900 Subject: [PATCH 063/589] Remove variable "concurrent" from ReindexStmt This node already handles multiple options using a bitmask, so having a separate boolean flag is not necessary. This simplifies the code a bit with less arguments to give to the reindex routines, by replacing the boolean with an equivalent bitmask value. Reviewed-by: Julien Rouhaud Discussion: https://postgr.es/m/20200902110326.GA14963@paquier.xyz --- src/backend/commands/indexcmds.c | 33 +++++++++++++++++++------------- src/backend/nodes/copyfuncs.c | 1 - src/backend/nodes/equalfuncs.c | 1 - src/backend/parser/gram.y | 12 ++++++++---- src/backend/tcop/utility.c | 8 ++++---- src/include/commands/defrem.h | 6 +++--- src/include/nodes/parsenodes.h | 2 +- 7 files changed, 36 insertions(+), 27 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b3a92381f95f..430e88b4c9fa 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -97,7 +97,7 @@ static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); */ struct ReindexIndexCallbackState { - bool concurrent; /* flag from statement */ + int options; /* options from statement */ Oid locked_table_oid; /* tracks previously locked table */ }; @@ -2420,7 +2420,7 @@ ChooseIndexColumnNames(List *indexElems) * Recreate a specific index. */ void -ReindexIndex(RangeVar *indexRelation, int options, bool concurrent) +ReindexIndex(RangeVar *indexRelation, int options) { struct ReindexIndexCallbackState state; Oid indOid; @@ -2437,10 +2437,11 @@ ReindexIndex(RangeVar *indexRelation, int options, bool concurrent) * upgrade the lock, but that's OK, because other sessions can't hold * locks on our temporary table. */ - state.concurrent = concurrent; + state.options = options; state.locked_table_oid = InvalidOid; indOid = RangeVarGetRelidExtended(indexRelation, - concurrent ? ShareUpdateExclusiveLock : AccessExclusiveLock, + (options & REINDEXOPT_CONCURRENTLY) != 0 ? + ShareUpdateExclusiveLock : AccessExclusiveLock, 0, RangeVarCallbackForReindexIndex, &state); @@ -2460,7 +2461,8 @@ ReindexIndex(RangeVar *indexRelation, int options, bool concurrent) persistence = irel->rd_rel->relpersistence; index_close(irel, NoLock); - if (concurrent && persistence != RELPERSISTENCE_TEMP) + if ((options & REINDEXOPT_CONCURRENTLY) != 0 && + persistence != RELPERSISTENCE_TEMP) ReindexRelationConcurrently(indOid, options); else reindex_index(indOid, false, persistence, @@ -2485,7 +2487,8 @@ RangeVarCallbackForReindexIndex(const RangeVar *relation, * non-concurrent case and table locks used by index_concurrently_*() for * concurrent case. */ - table_lockmode = state->concurrent ? ShareUpdateExclusiveLock : ShareLock; + table_lockmode = ((state->options & REINDEXOPT_CONCURRENTLY) != 0) ? + ShareUpdateExclusiveLock : ShareLock; /* * If we previously locked some other index's heap, and the name we're @@ -2542,7 +2545,7 @@ RangeVarCallbackForReindexIndex(const RangeVar *relation, * Recreate all indexes of a table (and of its toast table, if any) */ Oid -ReindexTable(RangeVar *relation, int options, bool concurrent) +ReindexTable(RangeVar *relation, int options) { Oid heapOid; bool result; @@ -2556,11 +2559,13 @@ ReindexTable(RangeVar *relation, int options, bool concurrent) * locks on our temporary table. */ heapOid = RangeVarGetRelidExtended(relation, - concurrent ? ShareUpdateExclusiveLock : ShareLock, + (options & REINDEXOPT_CONCURRENTLY) != 0 ? + ShareUpdateExclusiveLock : ShareLock, 0, RangeVarCallbackOwnsTable, NULL); - if (concurrent && get_rel_persistence(heapOid) != RELPERSISTENCE_TEMP) + if ((options & REINDEXOPT_CONCURRENTLY) != 0 && + get_rel_persistence(heapOid) != RELPERSISTENCE_TEMP) { result = ReindexRelationConcurrently(heapOid, options); @@ -2594,7 +2599,7 @@ ReindexTable(RangeVar *relation, int options, bool concurrent) */ void ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, - int options, bool concurrent) + int options) { Oid objectOid; Relation relationRelation; @@ -2613,7 +2618,8 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, objectKind == REINDEX_OBJECT_SYSTEM || objectKind == REINDEX_OBJECT_DATABASE); - if (objectKind == REINDEX_OBJECT_SYSTEM && concurrent) + if (objectKind == REINDEX_OBJECT_SYSTEM && + (options & REINDEXOPT_CONCURRENTLY) != 0) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot reindex system catalogs concurrently"))); @@ -2724,7 +2730,7 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, * Skip system tables, since index_create() would reject indexing them * concurrently (and it would likely fail if we tried). */ - if (concurrent && + if ((options & REINDEXOPT_CONCURRENTLY) != 0 && IsCatalogRelationOid(relid)) { if (!concurrent_warning) @@ -2774,7 +2780,8 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, continue; } - if (concurrent && get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + if ((options & REINDEXOPT_CONCURRENTLY) != 0 && + get_rel_persistence(relid) != RELPERSISTENCE_TEMP) { (void) ReindexRelationConcurrently(relid, options | diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 89c409de6640..0409a40b82a8 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -4450,7 +4450,6 @@ _copyReindexStmt(const ReindexStmt *from) COPY_NODE_FIELD(relation); COPY_STRING_FIELD(name); COPY_SCALAR_FIELD(options); - COPY_SCALAR_FIELD(concurrent); return newnode; } diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index e3f33c40be53..e2d1b987bf49 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -2135,7 +2135,6 @@ _equalReindexStmt(const ReindexStmt *a, const ReindexStmt *b) COMPARE_NODE_FIELD(relation); COMPARE_STRING_FIELD(name); COMPARE_SCALAR_FIELD(options); - COMPARE_SCALAR_FIELD(concurrent); return true; } diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index dbb47d498290..c5154b818cfb 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -8177,40 +8177,44 @@ ReindexStmt: { ReindexStmt *n = makeNode(ReindexStmt); n->kind = $2; - n->concurrent = $3; n->relation = $4; n->name = NULL; n->options = 0; + if ($3) + n->options |= REINDEXOPT_CONCURRENTLY; $$ = (Node *)n; } | REINDEX reindex_target_multitable opt_concurrently name { ReindexStmt *n = makeNode(ReindexStmt); n->kind = $2; - n->concurrent = $3; n->name = $4; n->relation = NULL; n->options = 0; + if ($3) + n->options |= REINDEXOPT_CONCURRENTLY; $$ = (Node *)n; } | REINDEX '(' reindex_option_list ')' reindex_target_type opt_concurrently qualified_name { ReindexStmt *n = makeNode(ReindexStmt); n->kind = $5; - n->concurrent = $6; n->relation = $7; n->name = NULL; n->options = $3; + if ($6) + n->options |= REINDEXOPT_CONCURRENTLY; $$ = (Node *)n; } | REINDEX '(' reindex_option_list ')' reindex_target_multitable opt_concurrently name { ReindexStmt *n = makeNode(ReindexStmt); n->kind = $5; - n->concurrent = $6; n->name = $7; n->relation = NULL; n->options = $3; + if ($6) + n->options |= REINDEXOPT_CONCURRENTLY; $$ = (Node *)n; } ; diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 6154d2c8c63b..b4cde5565ec1 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -919,17 +919,17 @@ standard_ProcessUtility(PlannedStmt *pstmt, { ReindexStmt *stmt = (ReindexStmt *) parsetree; - if (stmt->concurrent) + if ((stmt->options & REINDEXOPT_CONCURRENTLY) != 0) PreventInTransactionBlock(isTopLevel, "REINDEX CONCURRENTLY"); switch (stmt->kind) { case REINDEX_OBJECT_INDEX: - ReindexIndex(stmt->relation, stmt->options, stmt->concurrent); + ReindexIndex(stmt->relation, stmt->options); break; case REINDEX_OBJECT_TABLE: - ReindexTable(stmt->relation, stmt->options, stmt->concurrent); + ReindexTable(stmt->relation, stmt->options); break; case REINDEX_OBJECT_SCHEMA: case REINDEX_OBJECT_SYSTEM: @@ -945,7 +945,7 @@ standard_ProcessUtility(PlannedStmt *pstmt, (stmt->kind == REINDEX_OBJECT_SCHEMA) ? "REINDEX SCHEMA" : (stmt->kind == REINDEX_OBJECT_SYSTEM) ? "REINDEX SYSTEM" : "REINDEX DATABASE"); - ReindexMultipleTables(stmt->name, stmt->kind, stmt->options, stmt->concurrent); + ReindexMultipleTables(stmt->name, stmt->kind, stmt->options); break; default: elog(ERROR, "unrecognized object type: %d", diff --git a/src/include/commands/defrem.h b/src/include/commands/defrem.h index c26a102b175b..3129b684f633 100644 --- a/src/include/commands/defrem.h +++ b/src/include/commands/defrem.h @@ -34,10 +34,10 @@ extern ObjectAddress DefineIndex(Oid relationId, bool check_not_in_use, bool skip_build, bool quiet); -extern void ReindexIndex(RangeVar *indexRelation, int options, bool concurrent); -extern Oid ReindexTable(RangeVar *relation, int options, bool concurrent); +extern void ReindexIndex(RangeVar *indexRelation, int options); +extern Oid ReindexTable(RangeVar *relation, int options); extern void ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, - int options, bool concurrent); + int options); extern char *makeObjectName(const char *name1, const char *name2, const char *label); extern char *ChooseRelationName(const char *name1, const char *name2, diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index d52c563305ae..e83329fd6d10 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -3353,6 +3353,7 @@ typedef struct ConstraintsSetStmt #define REINDEXOPT_VERBOSE (1 << 0) /* print progress info */ #define REINDEXOPT_REPORT_PROGRESS (1 << 1) /* report pgstat progress */ #define REINDEXOPT_MISSING_OK (1 << 2) /* skip missing relations */ +#define REINDEXOPT_CONCURRENTLY (1 << 3) /* concurrent mode */ typedef enum ReindexObjectType { @@ -3371,7 +3372,6 @@ typedef struct ReindexStmt RangeVar *relation; /* Table or index to reindex */ const char *name; /* name of database to reindex */ int options; /* Reindex options flags */ - bool concurrent; /* reindex concurrently? */ } ReindexStmt; /* ---------------------- From e36e936e0ee664e07588732d1a8d105c5ec8e57d Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Thu, 3 Sep 2020 22:57:35 -0400 Subject: [PATCH 064/589] remove redundant initializations Reported-by: Ranier Vilela Discussion: https://postgr.es/m/CAEudQAo1+AcGppxDSg8k+zF4+Kv+eJyqzEDdbpDg58-=MQcerQ@mail.gmail.com Author: Ranier Vilela Backpatch-through: master --- src/backend/access/gist/gistxlog.c | 1 - src/backend/catalog/heap.c | 2 +- src/backend/storage/smgr/md.c | 6 ++++-- src/backend/utils/adt/json.c | 2 +- src/backend/utils/mmgr/mcxt.c | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/backend/access/gist/gistxlog.c b/src/backend/access/gist/gistxlog.c index dcd28f678b3d..91b3e111820d 100644 --- a/src/backend/access/gist/gistxlog.c +++ b/src/backend/access/gist/gistxlog.c @@ -405,7 +405,6 @@ gistRedoPageReuse(XLogReaderState *record) * logged value is very old, so that XID wrap-around already happened * on it, there can't be any snapshots that still see it. */ - nextXid = ReadNextFullTransactionId(); diff = U64FromFullTransactionId(nextXid) - U64FromFullTransactionId(latestRemovedFullXid); if (diff < MaxTransactionId / 2) diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index abd5bdb866b3..c151ad829d49 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -3436,7 +3436,7 @@ List * heap_truncate_find_FKs(List *relationIds) { List *result = NIL; - List *oids = list_copy(relationIds); + List *oids; List *parent_cons; ListCell *cell; ScanKeyData key; diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c index 0eacd461cd38..1d4aa482ccdd 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -734,9 +734,11 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, BlockNumber mdnblocks(SMgrRelation reln, ForkNumber forknum) { - MdfdVec *v = mdopenfork(reln, forknum, EXTENSION_FAIL); + MdfdVec *v; BlockNumber nblocks; - BlockNumber segno = 0; + BlockNumber segno; + + mdopenfork(reln, forknum, EXTENSION_FAIL); /* mdopen has opened the first segment */ Assert(reln->md_num_open_segs[forknum] > 0); diff --git a/src/backend/utils/adt/json.c b/src/backend/utils/adt/json.c index a7a91b72f69b..420d3cdcbb9d 100644 --- a/src/backend/utils/adt/json.c +++ b/src/backend/utils/adt/json.c @@ -990,7 +990,7 @@ catenate_stringinfo_string(StringInfo buffer, const char *addon) Datum json_build_object(PG_FUNCTION_ARGS) { - int nargs = PG_NARGS(); + int nargs; int i; const char *sep = ""; StringInfo result; diff --git a/src/backend/utils/mmgr/mcxt.c b/src/backend/utils/mmgr/mcxt.c index 88c76f290cea..dda70ef9f334 100644 --- a/src/backend/utils/mmgr/mcxt.c +++ b/src/backend/utils/mmgr/mcxt.c @@ -476,7 +476,7 @@ MemoryContextMemAllocated(MemoryContext context, bool recurse) if (recurse) { - MemoryContext child = context->firstchild; + MemoryContext child; for (child = context->firstchild; child != NULL; From ac15b499f7f92c26661835c327bfb0228a9b5e73 Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Fri, 4 Sep 2020 11:25:16 +0530 Subject: [PATCH 065/589] Fix inline marking introduced in commit 464824323e. Forgot to add inline marking in changes_filename() declaration. In the passing, add inline marking for a similar function subxact_filename(). Reported-By: Nathan Bossart Discussion: https://postgr.es/m/E98FBE8F-B878-480D-A728-A60C6EED3047@amazon.com --- src/backend/replication/logical/worker.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 812aca80112c..3c7ed80f9341 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -198,8 +198,8 @@ typedef struct ApplySubXactData static ApplySubXactData subxact_data = {0, 0, InvalidTransactionId, NULL}; -static void subxact_filename(char *path, Oid subid, TransactionId xid); -static void changes_filename(char *path, Oid subid, TransactionId xid); +static inline void subxact_filename(char *path, Oid subid, TransactionId xid); +static inline void changes_filename(char *path, Oid subid, TransactionId xid); /* * Information about subtransactions of a given toplevel transaction. @@ -2722,7 +2722,7 @@ subxact_info_add(TransactionId xid) } /* format filename for file containing the info about subxacts */ -static void +static inline void subxact_filename(char *path, Oid subid, TransactionId xid) { snprintf(path, MAXPGPATH, "%u-%u.subxacts", subid, xid); From 96cfcadd26e26e138ae60f4f3f94beddafba60bb Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 4 Sep 2020 08:02:58 +0200 Subject: [PATCH 066/589] Remove unused parameter unused since 93ee38eade1b2b4964354b95b01b09e17d6f098d Discussion: https://www.postgresql.org/message-id/flat/511bb100-f829-ba21-2f10-9f952ec06ead%402ndquadrant.com --- contrib/pageinspect/btreefuncs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/pageinspect/btreefuncs.c b/contrib/pageinspect/btreefuncs.c index e7a323044bf9..445605db58af 100644 --- a/contrib/pageinspect/btreefuncs.c +++ b/contrib/pageinspect/btreefuncs.c @@ -259,7 +259,7 @@ struct user_args * ------------------------------------------------------ */ static Datum -bt_page_print_tuples(FuncCallContext *fctx, struct user_args *uargs) +bt_page_print_tuples(struct user_args *uargs) { Page page = uargs->page; OffsetNumber offset = uargs->offset; @@ -498,7 +498,7 @@ bt_page_items(PG_FUNCTION_ARGS) if (fctx->call_cntr < fctx->max_calls) { - result = bt_page_print_tuples(fctx, uargs); + result = bt_page_print_tuples(uargs); uargs->offset++; SRF_RETURN_NEXT(fctx, result); } @@ -582,7 +582,7 @@ bt_page_items_bytea(PG_FUNCTION_ARGS) if (fctx->call_cntr < fctx->max_calls) { - result = bt_page_print_tuples(fctx, uargs); + result = bt_page_print_tuples(uargs); uargs->offset++; SRF_RETURN_NEXT(fctx, result); } From 6eee73e4e5b6d499f1a5529f209ce4af86ef5152 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 4 Sep 2020 08:19:20 +0200 Subject: [PATCH 067/589] doc: Use tags consistently in the tutorial chapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make more consistent use of and . Author: Jürgen Purtz Discussion: https://www.postgresql.org/message-id/flat/158996922318.7035.10603922579567326239@wrigleys.postgresql.org --- doc/src/sgml/query.sgml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/query.sgml b/doc/src/sgml/query.sgml index c0889743c4c4..bdb455f30082 100644 --- a/doc/src/sgml/query.sgml +++ b/doc/src/sgml/query.sgml @@ -564,14 +564,16 @@ SELECT * SELECT * FROM weather LEFT OUTER JOIN cities ON (weather.city = cities.name); + + city | temp_lo | temp_hi | prcp | date | name | location ---------------+---------+---------+------+------------+---------------+----------- Hayward | 37 | 54 | | 1994-11-29 | | San Francisco | 46 | 50 | 0.25 | 1994-11-27 | San Francisco | (-194,53) San Francisco | 43 | 57 | 0 | 1994-11-29 | San Francisco | (-194,53) (3 rows) - + This query is called a left outer join because the table mentioned on the left of the @@ -612,13 +614,15 @@ SELECT W1.city, W1.temp_lo AS low, W1.temp_hi AS high, FROM weather W1, weather W2 WHERE W1.temp_lo < W2.temp_lo AND W1.temp_hi > W2.temp_hi; + + city | low | high | city | low | high ---------------+-----+------+---------------+-----+------ San Francisco | 43 | 57 | San Francisco | 46 | 50 Hayward | 37 | 54 | San Francisco | 46 | 50 (2 rows) - + Here we have relabeled the weather table as W1 and W2 to be able to distinguish the left and right side From 79fd620b20b7721c2b591ef495be79b6347286c1 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 4 Sep 2020 08:39:01 +0200 Subject: [PATCH 068/589] doc: Fix whitespace issue in PDF MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move outside of to avoid whitespace issue in PDF output. Author: Jürgen Purtz Discussion: https://www.postgresql.org/message-id/flat/158996922318.7035.10603922579567326239@wrigleys.postgresql.org --- doc/src/sgml/query.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/query.sgml b/doc/src/sgml/query.sgml index bdb455f30082..497aae416b34 100644 --- a/doc/src/sgml/query.sgml +++ b/doc/src/sgml/query.sgml @@ -548,9 +548,9 @@ SELECT * it here to help you understand the following topics. - - joinouter + joinouter + Now we will figure out how we can get the Hayward records back in. What we want the query to do is to scan the weather table and for each row to find the From 49d7165117893405ae9b5b8d8e7877acff33c0e7 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 4 Sep 2020 08:45:57 +0200 Subject: [PATCH 069/589] doc: Change table alias names to lower case in tutorial chapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is needlessly different from our usual style otherwise. Author: Jürgen Purtz Discussion: https://www.postgresql.org/message-id/flat/158996922318.7035.10603922579567326239@wrigleys.postgresql.org --- doc/src/sgml/query.sgml | 14 +++++++------- src/tutorial/basics.source | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/doc/src/sgml/query.sgml b/doc/src/sgml/query.sgml index 497aae416b34..e73e805ec4f5 100644 --- a/doc/src/sgml/query.sgml +++ b/doc/src/sgml/query.sgml @@ -609,11 +609,11 @@ SELECT * following query: -SELECT W1.city, W1.temp_lo AS low, W1.temp_hi AS high, - W2.city, W2.temp_lo AS low, W2.temp_hi AS high - FROM weather W1, weather W2 - WHERE W1.temp_lo < W2.temp_lo - AND W1.temp_hi > W2.temp_hi; +SELECT w1.city, w1.temp_lo AS low, w1.temp_hi AS high, + w2.city, w2.temp_lo AS low, w2.temp_hi AS high + FROM weather w1, weather w2 + WHERE w1.temp_lo < w2.temp_lo + AND w1.temp_hi > w2.temp_hi; @@ -624,8 +624,8 @@ SELECT W1.city, W1.temp_lo AS low, W1.temp_hi AS high, (2 rows) - Here we have relabeled the weather table as W1 and - W2 to be able to distinguish the left and right side + Here we have relabeled the weather table as w1 and + w2 to be able to distinguish the left and right side of the join. You can also use these kinds of aliases in other queries to save some typing, e.g.: diff --git a/src/tutorial/basics.source b/src/tutorial/basics.source index 9dbd75eb154b..fe1cdfde2a87 100644 --- a/src/tutorial/basics.source +++ b/src/tutorial/basics.source @@ -126,13 +126,13 @@ SELECT * FROM weather LEFT OUTER JOIN cities ON (weather.city = cities.name); -- Suppose we want to find all the records that are in the temperature range --- of other records. W1 and W2 are aliases for weather. +-- of other records. w1 and w2 are aliases for weather. -SELECT W1.city, W1.temp_lo, W1.temp_hi, - W2.city, W2.temp_lo, W2.temp_hi -FROM weather W1, weather W2 -WHERE W1.temp_lo < W2.temp_lo - and W1.temp_hi > W2.temp_hi; +SELECT w1.city, w1.temp_lo, w1.temp_hi, + w2.city, w2.temp_lo, w2.temp_hi +FROM weather w1, weather w2 +WHERE w1.temp_lo < w2.temp_lo + and w1.temp_hi > w2.temp_hi; ----------------------------- From d54f99e41541de848a6ca53b3ec060f461e9ab71 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 4 Sep 2020 12:40:28 -0400 Subject: [PATCH 070/589] Fix rare deadlock failure in create_am regression test. The "DROP ACCESS METHOD gist2" test will require locking the index to be dropped and then its table; while most ordinary operations lock a table first then its index. While no concurrent test scripts should be touching fast_emp4000, autovacuum might chance to be processing that table when the DROP runs, resulting in a deadlock failure. This is pretty rare but we see it in the buildfarm from time to time. To fix, acquire a lock on fast_emp4000 before issuing the DROP. Since the point of the exercise is mostly to prevent buildfarm failures, back-patch to 9.6 where this test was introduced. Discussion: https://postgr.es/m/839004.1599185607@sss.pgh.pa.us --- src/test/regress/expected/create_am.out | 5 +++++ src/test/regress/sql/create_am.sql | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/test/regress/expected/create_am.out b/src/test/regress/expected/create_am.out index 84da403afc5d..b9dc82dd3c3a 100644 --- a/src/test/regress/expected/create_am.out +++ b/src/test/regress/expected/create_am.out @@ -102,8 +102,13 @@ ERROR: cannot drop access method gist2 because other objects depend on it DETAIL: index grect2ind2 depends on operator class box_ops for access method gist2 HINT: Use DROP ... CASCADE to drop the dependent objects too. -- Drop access method cascade +-- To prevent a (rare) deadlock against autovacuum, +-- we must lock the table that owns the index that will be dropped +BEGIN; +LOCK TABLE fast_emp4000; DROP ACCESS METHOD gist2 CASCADE; NOTICE: drop cascades to index grect2ind2 +COMMIT; -- -- Test table access methods -- diff --git a/src/test/regress/sql/create_am.sql b/src/test/regress/sql/create_am.sql index a7f6de7e9be4..97df244d172a 100644 --- a/src/test/regress/sql/create_am.sql +++ b/src/test/regress/sql/create_am.sql @@ -70,7 +70,12 @@ ROLLBACK; DROP ACCESS METHOD gist2; -- Drop access method cascade +-- To prevent a (rare) deadlock against autovacuum, +-- we must lock the table that owns the index that will be dropped +BEGIN; +LOCK TABLE fast_emp4000; DROP ACCESS METHOD gist2 CASCADE; +COMMIT; -- From 3b5af0e95ad5a3d9b478826336a11ad1d201c378 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Fri, 4 Sep 2020 13:27:52 -0400 Subject: [PATCH 071/589] C comment: correct use of 64-"byte" cache line size Reported-by: Kelly Min Discussion: https://postgr.es/m/CAPSbxatOiQO90LYpSC3+svAU9-sHgDfEP4oFhcEUt_X=DqFA9g@mail.gmail.com Backpatch-through: 9.5 --- src/include/storage/buf_internals.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h index e57f84ee9c8c..3377fa567680 100644 --- a/src/include/storage/buf_internals.h +++ b/src/include/storage/buf_internals.h @@ -203,7 +203,7 @@ typedef struct BufferDesc * Note that local buffer descriptors aren't forced to be aligned - as there's * no concurrent access to those it's unlikely to be beneficial. * - * We use 64bit as the cache line size here, because that's the most common + * We use a 64-byte cache line size here, because that's the most common * size. Making it bigger would be a waste of memory. Even if running on a * platform with either 32 or 128 byte line sizes, it's good to align to * boundaries and avoid false sharing. From 3eb3d3e7822d5eecfcaba871a90263c6025c5216 Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Fri, 4 Sep 2020 13:53:09 -0400 Subject: [PATCH 072/589] Collect attribute data on extension owned tables being dumped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If this data is not collected, pg_dump segfaults if asked for column inserts. Fix by Fabrízio de Royes Mello Backpatch to release 12 where the bug was introduced. --- src/bin/pg_dump/pg_dump.c | 4 ++++ src/test/modules/test_pg_dump/t/001_base.pl | 24 +++++++++++++++++-- .../test_pg_dump/test_pg_dump--1.0.sql | 5 ++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index d3ca54e4dc6a..784bceaec394 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -2095,6 +2095,8 @@ dumpTableData_insert(Archive *fout, void *dcontext) if (nfields == 0) continue; + Assert(tbinfo->attgenerated); + /* Emit a row heading */ if (rows_per_statement == 1) archputs(" (", fout); @@ -17913,6 +17915,8 @@ processExtensionTables(Archive *fout, ExtensionInfo extinfo[], configtbl->dataObj->filtercond = pg_strdup(extconditionarray[j]); } } + + configtbl->interesting = dumpobj; } } if (extconfigarray) diff --git a/src/test/modules/test_pg_dump/t/001_base.pl b/src/test/modules/test_pg_dump/t/001_base.pl index ae120a5ee366..78aa07ce511a 100644 --- a/src/test/modules/test_pg_dump/t/001_base.pl +++ b/src/test/modules/test_pg_dump/t/001_base.pl @@ -135,6 +135,12 @@ "$tempdir/defaults_tar_format.tar", ], }, + extension_schema => { + dump_cmd => [ + 'pg_dump', '--schema=public', '--inserts', + "--file=$tempdir/extension_schema.sql", 'postgres', + ], + }, pg_dumpall_globals => { dump_cmd => [ 'pg_dumpall', '--no-sync', @@ -301,8 +307,9 @@ \n/xm, like => { %full_runs, - data_only => 1, - section_data => 1, + data_only => 1, + section_data => 1, + extension_schema => 1, }, }, @@ -536,6 +543,7 @@ like => {%pgdump_runs}, unlike => { data_only => 1, + extension_schema => 1, pg_dumpall_globals => 1, section_data => 1, section_pre_data => 1, @@ -549,6 +557,7 @@ like => {%pgdump_runs}, unlike => { data_only => 1, + extension_schema => 1, pg_dumpall_globals => 1, section_data => 1, section_pre_data => 1, @@ -569,6 +578,17 @@ schema_only => 1, section_pre_data => 1, }, + }, + + # Dumpable object inside specific schema + 'INSERT INTO public.regress_table_dumpable VALUES (1);' => { + create_sql => 'INSERT INTO public.regress_table_dumpable VALUES (1);', + regexp => qr/^ + \QINSERT INTO public.regress_table_dumpable VALUES (1);\E + \n/xm, + like => { + extension_schema => 1, + }, },); ######################################### diff --git a/src/test/modules/test_pg_dump/test_pg_dump--1.0.sql b/src/test/modules/test_pg_dump/test_pg_dump--1.0.sql index 3ed007a7b1b4..90e461ed3573 100644 --- a/src/test/modules/test_pg_dump/test_pg_dump--1.0.sql +++ b/src/test/modules/test_pg_dump/test_pg_dump--1.0.sql @@ -13,6 +13,11 @@ CREATE SEQUENCE regress_pg_dump_seq; CREATE SEQUENCE regress_seq_dumpable; SELECT pg_catalog.pg_extension_config_dump('regress_seq_dumpable', ''); +CREATE TABLE regress_table_dumpable ( + col1 int +); +SELECT pg_catalog.pg_extension_config_dump('regress_table_dumpable', ''); + CREATE SCHEMA regress_pg_dump_schema; GRANT USAGE ON regress_pg_dump_seq TO regress_dump_test_role; From 38a2d703298c9a891dc9c24c0c087f417f555c70 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 4 Sep 2020 14:32:10 -0400 Subject: [PATCH 073/589] Remove some more useless assignments. Found with clang's scan-build tool. It also whines about a lot of other dead stores that we should *not* change IMO, either as a matter of style or future-proofing. But these places seem like clear oversights. Discussion: https://postgr.es/m/CAEudQAo1+AcGppxDSg8k+zF4+Kv+eJyqzEDdbpDg58-=MQcerQ@mail.gmail.com --- src/backend/access/brin/brin_tuple.c | 1 - src/backend/access/gin/ginbtree.c | 1 - src/backend/catalog/pg_depend.c | 6 +++--- src/backend/optimizer/plan/planner.c | 2 +- src/backend/parser/parse_utilcmd.c | 1 - src/backend/partitioning/partbounds.c | 4 ---- src/backend/statistics/extended_stats.c | 2 +- src/backend/tsearch/spell.c | 4 ++-- 8 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/backend/access/brin/brin_tuple.c b/src/backend/access/brin/brin_tuple.c index 6cb7c26b39f2..46e6b23c8742 100644 --- a/src/backend/access/brin/brin_tuple.c +++ b/src/backend/access/brin/brin_tuple.c @@ -243,7 +243,6 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple, *bitP |= bitmask; } - bitP = ((bits8 *) (rettuple + SizeOfBrinTuple)) - 1; } if (tuple->bt_placeholder) diff --git a/src/backend/access/gin/ginbtree.c b/src/backend/access/gin/ginbtree.c index 8d08b05f5156..82788a5c367a 100644 --- a/src/backend/access/gin/ginbtree.c +++ b/src/backend/access/gin/ginbtree.c @@ -241,7 +241,6 @@ ginFindParents(GinBtree btree, GinBtreeStack *stack) blkno = root->blkno; buffer = root->buffer; - offset = InvalidOffsetNumber; ptr = (GinBtreeStack *) palloc(sizeof(GinBtreeStack)); diff --git a/src/backend/catalog/pg_depend.c b/src/backend/catalog/pg_depend.c index 70baf03178f7..f263ff13e55c 100644 --- a/src/backend/catalog/pg_depend.c +++ b/src/backend/catalog/pg_depend.c @@ -478,7 +478,7 @@ changeDependenciesOf(Oid classId, Oid oldObjectId, while (HeapTupleIsValid((tup = systable_getnext(scan)))) { - Form_pg_depend depform = (Form_pg_depend) GETSTRUCT(tup); + Form_pg_depend depform; /* make a modifiable copy */ tup = heap_copytuple(tup); @@ -561,12 +561,12 @@ changeDependenciesOn(Oid refClassId, Oid oldRefObjectId, while (HeapTupleIsValid((tup = systable_getnext(scan)))) { - Form_pg_depend depform = (Form_pg_depend) GETSTRUCT(tup); - if (newIsPinned) CatalogTupleDelete(depRel, &tup->t_self); else { + Form_pg_depend depform; + /* make a modifiable copy */ tup = heap_copytuple(tup); depform = (Form_pg_depend) GETSTRUCT(tup); diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index b40a112c25b2..139c5e3dc245 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -5097,7 +5097,7 @@ create_ordered_paths(PlannerInfo *root, foreach(lc, input_rel->partial_pathlist) { Path *input_path = (Path *) lfirst(lc); - Path *sorted_path = input_path; + Path *sorted_path; bool is_sorted; int presorted_keys; double total_groups; diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index 6c49554defbc..ec944371dd36 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -1765,7 +1765,6 @@ generateClonedIndexStmt(RangeVar *heapRel, Relation source_idx, char *attname; attname = get_attname(indrelid, attnum, false); - keycoltype = get_atttype(indrelid, attnum); iparam->name = attname; iparam->expr = NULL; diff --git a/src/backend/partitioning/partbounds.c b/src/backend/partitioning/partbounds.c index 7553d5598773..419c8fe84516 100644 --- a/src/backend/partitioning/partbounds.c +++ b/src/backend/partitioning/partbounds.c @@ -4259,10 +4259,6 @@ get_qual_for_range(Relation parent, PartitionBoundSpec *spec, return result; } - lower_or_start_datum = list_head(spec->lowerdatums); - upper_or_start_datum = list_head(spec->upperdatums); - num_or_arms = key->partnatts; - /* * If it is the recursive call for default, we skip the get_range_nulltest * to avoid accumulating the NullTest on the same keys for each partition. diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c index ab6f1e1c9dc9..9336f9bc5e90 100644 --- a/src/backend/statistics/extended_stats.c +++ b/src/backend/statistics/extended_stats.c @@ -232,7 +232,7 @@ ComputeExtStatisticsRows(Relation onerel, foreach(lc, lstats) { StatExtEntry *stat = (StatExtEntry *) lfirst(lc); - int stattarget = stat->stattarget; + int stattarget; VacAttrStats **stats; int nattrs = bms_num_members(stat->columns); diff --git a/src/backend/tsearch/spell.c b/src/backend/tsearch/spell.c index 8aab96d3b066..49735bc06af8 100644 --- a/src/backend/tsearch/spell.c +++ b/src/backend/tsearch/spell.c @@ -2124,7 +2124,6 @@ CheckAffix(const char *word, size_t len, AFFIX *Affix, int flagflags, char *neww } else { - int err; pg_wchar *data; size_t data_len; int newword_len; @@ -2134,7 +2133,8 @@ CheckAffix(const char *word, size_t len, AFFIX *Affix, int flagflags, char *neww data = (pg_wchar *) palloc((newword_len + 1) * sizeof(pg_wchar)); data_len = pg_mb2wchar_with_len(newword, data, newword_len); - if (!(err = pg_regexec(&(Affix->reg.regex), data, data_len, 0, NULL, 0, NULL, 0))) + if (pg_regexec(&(Affix->reg.regex), data, data_len, + 0, NULL, 0, NULL, 0) == REG_OKAY) { pfree(data); return newword; From f43e295f68c3e04ef891627f62016a5b3d8ed4a8 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Fri, 4 Sep 2020 14:58:32 -0400 Subject: [PATCH 074/589] Report expected contrecord length on mismatch When reading a WAL record fails to find continuation record(s) of the proper length, report what it expects, for clarity. Reviewed-by: Tom Lane Discussion: https://postgr.es/m/20200903212152.GA15319@alvherre.pgsql --- src/backend/access/transam/xlogreader.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 67996018da27..a63ad8cfd0bf 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -464,8 +464,9 @@ XLogReadRecord(XLogReaderState *state, char **errormsg) total_len != (pageHeader->xlp_rem_len + gotlen)) { report_invalid_record(state, - "invalid contrecord length %u at %X/%X", + "invalid contrecord length %u (expected %lld) at %X/%X", pageHeader->xlp_rem_len, + ((long long) total_len) - gotlen, (uint32) (RecPtr >> 32), (uint32) RecPtr); goto err; } From 0852006a946aa9795b4913bccebb88d623942ca6 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Fri, 4 Sep 2020 12:01:58 -0700 Subject: [PATCH 075/589] Fix bogus MaxAllocSize check in logtape.c. Reported-by: Peter Geoghegan Discussion: https://postgr.es/m/CAH2-Wz=NZPZc3-fkdmvu=w2itx0PiB-G6QpxHXZOjuvFAzPdZw@mail.gmail.com Backpatch-through: 13 --- src/backend/utils/sort/logtape.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c index 788815cdab6c..bbb01f6d3373 100644 --- a/src/backend/utils/sort/logtape.c +++ b/src/backend/utils/sort/logtape.c @@ -491,7 +491,7 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum) * If the freelist becomes very large, just return and leak this free * block. */ - if (lts->freeBlocksLen * 2 > MaxAllocSize) + if (lts->freeBlocksLen * 2 * sizeof(long) > MaxAllocSize) return; lts->freeBlocksLen *= 2; From 9a851039aac6fc7b78a80e60699e7bd8924b6f12 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 4 Sep 2020 18:17:47 -0400 Subject: [PATCH 076/589] Remove still more useless assignments. Fix some more things scan-build pointed to as dead stores. In some of these cases, rearranging the code a little leads to more readable code IMO. It's all cosmetic, though. Discussion: https://postgr.es/m/CAEudQAo1+AcGppxDSg8k+zF4+Kv+eJyqzEDdbpDg58-=MQcerQ@mail.gmail.com --- src/backend/replication/slotfuncs.c | 3 --- src/backend/utils/adt/formatting.c | 27 ++++++++++++--------------- src/backend/utils/adt/jsonfuncs.c | 14 ++++++-------- src/bin/scripts/vacuumdb.c | 19 ++++++------------- 4 files changed, 24 insertions(+), 39 deletions(-) diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c index f88694672fba..1725ad0736fd 100644 --- a/src/backend/replication/slotfuncs.c +++ b/src/backend/replication/slotfuncs.c @@ -518,9 +518,6 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto) */ XLogBeginRead(ctx->reader, MyReplicationSlot->data.restart_lsn); - /* Initialize our return value in case we don't do anything */ - retlsn = MyReplicationSlot->data.confirmed_flush; - /* invalidate non-timetravel entries */ InvalidateSystemCaches(); diff --git a/src/backend/utils/adt/formatting.c b/src/backend/utils/adt/formatting.c index 9de63686ecb5..bf9643ffb4a1 100644 --- a/src/backend/utils/adt/formatting.c +++ b/src/backend/utils/adt/formatting.c @@ -6434,13 +6434,12 @@ float4_to_char(PG_FUNCTION_ARGS) int out_pre_spaces = 0, sign = 0; char *numstr, - *orgnum, *p; NUM_TOCHAR_prepare; if (IS_ROMAN(&Num)) - numstr = orgnum = int_to_roman((int) rint(value)); + numstr = int_to_roman((int) rint(value)); else if (IS_EEEE(&Num)) { if (isnan(value) || isinf(value)) @@ -6456,20 +6455,19 @@ float4_to_char(PG_FUNCTION_ARGS) } else { - numstr = orgnum = psprintf("%+.*e", Num.post, value); + numstr = psprintf("%+.*e", Num.post, value); /* * Swap a leading positive sign for a space. */ - if (*orgnum == '+') - *orgnum = ' '; - - numstr = orgnum; + if (*numstr == '+') + *numstr = ' '; } } else { float4 val = value; + char *orgnum; int numstr_pre_len; if (IS_MULTI(&Num)) @@ -6480,7 +6478,7 @@ float4_to_char(PG_FUNCTION_ARGS) Num.pre += Num.multi; } - orgnum = (char *) psprintf("%.0f", fabs(val)); + orgnum = psprintf("%.0f", fabs(val)); numstr_pre_len = strlen(orgnum); /* adjust post digits to fit max float digits */ @@ -6538,13 +6536,12 @@ float8_to_char(PG_FUNCTION_ARGS) int out_pre_spaces = 0, sign = 0; char *numstr, - *orgnum, *p; NUM_TOCHAR_prepare; if (IS_ROMAN(&Num)) - numstr = orgnum = int_to_roman((int) rint(value)); + numstr = int_to_roman((int) rint(value)); else if (IS_EEEE(&Num)) { if (isnan(value) || isinf(value)) @@ -6560,20 +6557,19 @@ float8_to_char(PG_FUNCTION_ARGS) } else { - numstr = orgnum = (char *) psprintf("%+.*e", Num.post, value); + numstr = psprintf("%+.*e", Num.post, value); /* * Swap a leading positive sign for a space. */ - if (*orgnum == '+') - *orgnum = ' '; - - numstr = orgnum; + if (*numstr == '+') + *numstr = ' '; } } else { float8 val = value; + char *orgnum; int numstr_pre_len; if (IS_MULTI(&Num)) @@ -6583,6 +6579,7 @@ float8_to_char(PG_FUNCTION_ARGS) val = value * multi; Num.pre += Num.multi; } + orgnum = psprintf("%.0f", fabs(val)); numstr_pre_len = strlen(orgnum); diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c index 5a09d65fdcec..d370348a1c51 100644 --- a/src/backend/utils/adt/jsonfuncs.c +++ b/src/backend/utils/adt/jsonfuncs.c @@ -4687,8 +4687,8 @@ IteratorConcat(JsonbIterator **it1, JsonbIterator **it2, rk1, rk2; - r1 = rk1 = JsonbIteratorNext(it1, &v1, false); - r2 = rk2 = JsonbIteratorNext(it2, &v2, false); + rk1 = JsonbIteratorNext(it1, &v1, false); + rk2 = JsonbIteratorNext(it2, &v2, false); /* * Both elements are objects. @@ -4696,15 +4696,15 @@ IteratorConcat(JsonbIterator **it1, JsonbIterator **it2, if (rk1 == WJB_BEGIN_OBJECT && rk2 == WJB_BEGIN_OBJECT) { /* - * Append the all tokens from v1 to res, except last WJB_END_OBJECT + * Append all the tokens from v1 to res, except last WJB_END_OBJECT * (because res will not be finished yet). */ - pushJsonbValue(state, r1, NULL); + pushJsonbValue(state, rk1, NULL); while ((r1 = JsonbIteratorNext(it1, &v1, true)) != WJB_END_OBJECT) pushJsonbValue(state, r1, &v1); /* - * Append the all tokens from v2 to res, include last WJB_END_OBJECT + * Append all the tokens from v2 to res, include last WJB_END_OBJECT * (the concatenation will be completed). */ while ((r2 = JsonbIteratorNext(it2, &v2, true)) != WJB_DONE) @@ -4716,7 +4716,7 @@ IteratorConcat(JsonbIterator **it1, JsonbIterator **it2, */ else if (rk1 == WJB_BEGIN_ARRAY && rk2 == WJB_BEGIN_ARRAY) { - pushJsonbValue(state, r1, NULL); + pushJsonbValue(state, rk1, NULL); while ((r1 = JsonbIteratorNext(it1, &v1, true)) != WJB_END_ARRAY) { @@ -4736,10 +4736,8 @@ IteratorConcat(JsonbIterator **it1, JsonbIterator **it2, else if (((rk1 == WJB_BEGIN_ARRAY && !(*it1)->isScalar) && rk2 == WJB_BEGIN_OBJECT) || (rk1 == WJB_BEGIN_OBJECT && (rk2 == WJB_BEGIN_ARRAY && !(*it2)->isScalar))) { - JsonbIterator **it_array = rk1 == WJB_BEGIN_ARRAY ? it1 : it2; JsonbIterator **it_object = rk1 == WJB_BEGIN_OBJECT ? it1 : it2; - bool prepend = (rk1 == WJB_BEGIN_OBJECT); pushJsonbValue(state, WJB_BEGIN_ARRAY, NULL); diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c index 125ed2ff5a46..a8bc65421966 100644 --- a/src/bin/scripts/vacuumdb.c +++ b/src/bin/scripts/vacuumdb.c @@ -408,7 +408,6 @@ vacuum_one_database(const char *dbname, vacuumingOptions *vacopts, int i; int ntups; bool failed = false; - bool parallel = concurrentCons > 1; bool tables_listed = false; bool has_where = false; const char *stage_commands[] = { @@ -651,25 +650,19 @@ vacuum_one_database(const char *dbname, vacuumingOptions *vacopts, PQclear(res); /* - * If there are more connections than vacuumable relations, we don't need - * to use them all. + * Ensure concurrentCons is sane. If there are more connections than + * vacuumable relations, we don't need to use them all. */ - if (parallel) - { - if (concurrentCons > ntups) - concurrentCons = ntups; - if (concurrentCons <= 1) - parallel = false; - } + if (concurrentCons > ntups) + concurrentCons = ntups; + if (concurrentCons <= 0) + concurrentCons = 1; /* * Setup the database connections. We reuse the connection we already have * for the first slot. If not in parallel mode, the first slot in the * array contains the connection. */ - if (concurrentCons <= 0) - concurrentCons = 1; - slots = ParallelSlotsSetup(dbname, host, port, username, prompt_password, progname, echo, conn, concurrentCons); From c8746f999ea2feba22832b485fba37b5301b54a3 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 4 Sep 2020 20:20:05 -0400 Subject: [PATCH 077/589] Fix over-eager ping'ing in logical replication receiver. Commit 3f60f690f only partially fixed the broken-status-tracking issue in LogicalRepApplyLoop: we need ping_sent to have the same lifetime as last_recv_timestamp. The effects are much less serious than what that commit fixed, though. AFAICS this would just lead to extra ping requests being sent, once per second until the sender responds. Still, it's a bug, so backpatch to v10 as before. Discussion: https://postgr.es/m/959627.1599248476@sss.pgh.pa.us --- src/backend/replication/logical/worker.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 3c7ed80f9341..c37aafed0d29 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -2060,6 +2060,7 @@ static void LogicalRepApplyLoop(XLogRecPtr last_received) { TimestampTz last_recv_timestamp = GetCurrentTimestamp(); + bool ping_sent = false; /* * Init the ApplyMessageContext which we clean up after each replication @@ -2080,6 +2081,7 @@ LogicalRepApplyLoop(XLogRecPtr last_received) /* mark as idle, before starting to loop */ pgstat_report_activity(STATE_IDLE, NULL); + /* This outer loop iterates once per wait. */ for (;;) { pgsocket fd = PGINVALID_SOCKET; @@ -2087,7 +2089,6 @@ LogicalRepApplyLoop(XLogRecPtr last_received) int len; char *buf = NULL; bool endofstream = false; - bool ping_sent = false; long wait_time; CHECK_FOR_INTERRUPTS(); @@ -2098,7 +2099,7 @@ LogicalRepApplyLoop(XLogRecPtr last_received) if (len != 0) { - /* Process the data */ + /* Loop to process all available data (without blocking). */ for (;;) { CHECK_FOR_INTERRUPTS(); @@ -2267,10 +2268,7 @@ LogicalRepApplyLoop(XLogRecPtr last_received) ereport(ERROR, (errmsg("terminating logical replication worker due to timeout"))); - /* - * We didn't receive anything new, for half of receiver - * replication timeout. Ping the server. - */ + /* Check to see if it's time for a ping. */ if (!ping_sent) { timeout = TimestampTzPlusMilliseconds(last_recv_timestamp, From 4d41823c5267184fb021ac456caf95ef734cc8b0 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 4 Sep 2020 21:01:58 -0400 Subject: [PATCH 078/589] Make new authentication test case more robust. I happened to notice that the new test case I added in b55b4dad9 falls over if one runs "make check" repeatedly; though not in branches after v10. That's because it was assuming that tmp_check/pgpass wouldn't exist already. However, it's only been since v11 that the Makefiles forcibly remove all of tmp_check/ before starting a TAP run. This fix to unlink the file is therefore strictly necessary only in v10 ... but it seems wisest to do it across the board, rather than let the test rely on external logic to get the conditions right. --- src/test/authentication/t/001_password.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/authentication/t/001_password.pl b/src/test/authentication/t/001_password.pl index 59b1b403c554..36a616d7c7a7 100644 --- a/src/test/authentication/t/001_password.pl +++ b/src/test/authentication/t/001_password.pl @@ -107,6 +107,7 @@ sub test_role delete $ENV{"PGCHANNELBINDING"}; $ENV{"PGPASSFILE"} = $pgpassfile; +unlink($pgpassfile); append_to_file($pgpassfile, qq! # This very long comment is just here to exercise handling of long lines in the file. This very long comment is just here to exercise handling of long lines in the file. This very long comment is just here to exercise handling of long lines in the file. This very long comment is just here to exercise handling of long lines in the file. This very long comment is just here to exercise handling of long lines in the file. *:*:postgres:scram_role:pass:this is not part of the password. From 63110c6264a5363863209c8fbdd0498c03535507 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Sat, 5 Sep 2020 13:52:47 +0900 Subject: [PATCH 079/589] Use multi-inserts for pg_depend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a follow-up of the work done in e3931d01. This case is a bit different than pg_attribute and pg_shdepend: the maximum number of items to insert is known in advance, but there is no need to handle pinned dependencies. Hence, the base allocation for slots is done based on the number of items and the maximum allowed with a cap at 64kB. Slots are initialized once used to minimize the overhead of the operation. The insertions can be done for dependencies of the same type. More could be done by grouping the insertion of multiple dependency types in a single batch. This is left as future work. Some of the multi-insert logic is also simplified for pg_shdepend, as per the feedback discussed for this specific patch. This also moves to indexing.h the variable capping the maximum amount of data that can be used at once for a multi-insert, instead of having separate definitions for pg_attribute, pg_depend and pg_shdepend. Author: Daniel Gustafsson, Michael Paquier Reviewed-by: Andres Freund, Álvaro Herrera Discussion: https://postgr.es/m/20200807061619.GA23955@paquier.xyz --- src/backend/catalog/heap.c | 8 +-- src/backend/catalog/pg_depend.c | 90 ++++++++++++++++++++++--------- src/backend/catalog/pg_shdepend.c | 66 +++++++++++------------ src/include/catalog/indexing.h | 6 +++ 4 files changed, 105 insertions(+), 65 deletions(-) diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index c151ad829d49..9d2d357233b3 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -709,12 +709,6 @@ CheckAttributeType(const char *attname, } } -/* - * Cap the maximum amount of bytes allocated for InsertPgAttributeTuples() - * slots. - */ -#define MAX_PGATTRIBUTE_INSERT_BYTES 65535 - /* * InsertPgAttributeTuples * Construct and insert a set of tuples in pg_attribute. @@ -750,7 +744,7 @@ InsertPgAttributeTuples(Relation pg_attribute_rel, /* Initialize the number of slots to use */ nslots = Min(tupdesc->natts, - (MAX_PGATTRIBUTE_INSERT_BYTES / sizeof(FormData_pg_attribute))); + (MAX_CATALOG_MULTI_INSERT_BYTES / sizeof(FormData_pg_attribute))); slot = palloc(sizeof(TupleTableSlot *) * nslots); for (int i = 0; i < nslots; i++) slot[i] = MakeSingleTupleTableSlot(td, &TTSOpsHeapTuple); diff --git a/src/backend/catalog/pg_depend.c b/src/backend/catalog/pg_depend.c index f263ff13e55c..454e569fa98d 100644 --- a/src/backend/catalog/pg_depend.c +++ b/src/backend/catalog/pg_depend.c @@ -59,10 +59,11 @@ recordMultipleDependencies(const ObjectAddress *depender, { Relation dependDesc; CatalogIndexState indstate; - HeapTuple tup; - int i; - bool nulls[Natts_pg_depend]; - Datum values[Natts_pg_depend]; + TupleTableSlot **slot; + int i, + max_slots, + slot_init_count, + slot_stored_count; if (nreferenced <= 0) return; /* nothing to do */ @@ -76,11 +77,21 @@ recordMultipleDependencies(const ObjectAddress *depender, dependDesc = table_open(DependRelationId, RowExclusiveLock); + /* + * Allocate the slots to use, but delay costly initialization until we + * know that they will be used. + */ + max_slots = Min(nreferenced, + MAX_CATALOG_MULTI_INSERT_BYTES / sizeof(FormData_pg_depend)); + slot = palloc(sizeof(TupleTableSlot *) * max_slots); + /* Don't open indexes unless we need to make an update */ indstate = NULL; - memset(nulls, false, sizeof(nulls)); - + /* number of slots currently storing tuples */ + slot_stored_count = 0; + /* number of slots currently initialized */ + slot_init_count = 0; for (i = 0; i < nreferenced; i++, referenced++) { /* @@ -88,38 +99,69 @@ recordMultipleDependencies(const ObjectAddress *depender, * need to record dependencies on it. This saves lots of space in * pg_depend, so it's worth the time taken to check. */ - if (!isObjectPinned(referenced, dependDesc)) - { - /* - * Record the Dependency. Note we don't bother to check for - * duplicate dependencies; there's no harm in them. - */ - values[Anum_pg_depend_classid - 1] = ObjectIdGetDatum(depender->classId); - values[Anum_pg_depend_objid - 1] = ObjectIdGetDatum(depender->objectId); - values[Anum_pg_depend_objsubid - 1] = Int32GetDatum(depender->objectSubId); - - values[Anum_pg_depend_refclassid - 1] = ObjectIdGetDatum(referenced->classId); - values[Anum_pg_depend_refobjid - 1] = ObjectIdGetDatum(referenced->objectId); - values[Anum_pg_depend_refobjsubid - 1] = Int32GetDatum(referenced->objectSubId); + if (isObjectPinned(referenced, dependDesc)) + continue; - values[Anum_pg_depend_deptype - 1] = CharGetDatum((char) behavior); + if (slot_init_count < max_slots) + { + slot[slot_stored_count] = MakeSingleTupleTableSlot(RelationGetDescr(dependDesc), + &TTSOpsHeapTuple); + slot_init_count++; + } - tup = heap_form_tuple(dependDesc->rd_att, values, nulls); + ExecClearTuple(slot[slot_stored_count]); + /* + * Record the dependency. Note we don't bother to check for duplicate + * dependencies; there's no harm in them. + */ + slot[slot_stored_count]->tts_values[Anum_pg_depend_refclassid - 1] = ObjectIdGetDatum(referenced->classId); + slot[slot_stored_count]->tts_values[Anum_pg_depend_refobjid - 1] = ObjectIdGetDatum(referenced->objectId); + slot[slot_stored_count]->tts_values[Anum_pg_depend_refobjsubid - 1] = Int32GetDatum(referenced->objectSubId); + slot[slot_stored_count]->tts_values[Anum_pg_depend_deptype - 1] = CharGetDatum((char) behavior); + slot[slot_stored_count]->tts_values[Anum_pg_depend_classid - 1] = ObjectIdGetDatum(depender->classId); + slot[slot_stored_count]->tts_values[Anum_pg_depend_objid - 1] = ObjectIdGetDatum(depender->objectId); + slot[slot_stored_count]->tts_values[Anum_pg_depend_objsubid - 1] = Int32GetDatum(depender->objectSubId); + + memset(slot[slot_stored_count]->tts_isnull, false, + slot[slot_stored_count]->tts_tupleDescriptor->natts * sizeof(bool)); + + ExecStoreVirtualTuple(slot[slot_stored_count]); + slot_stored_count++; + + /* If slots are full, insert a batch of tuples */ + if (slot_stored_count == max_slots) + { /* fetch index info only when we know we need it */ if (indstate == NULL) indstate = CatalogOpenIndexes(dependDesc); - CatalogTupleInsertWithInfo(dependDesc, tup, indstate); - - heap_freetuple(tup); + CatalogTuplesMultiInsertWithInfo(dependDesc, slot, slot_stored_count, + indstate); + slot_stored_count = 0; } } + /* Insert any tuples left in the buffer */ + if (slot_stored_count > 0) + { + /* fetch index info only when we know we need it */ + if (indstate == NULL) + indstate = CatalogOpenIndexes(dependDesc); + + CatalogTuplesMultiInsertWithInfo(dependDesc, slot, slot_stored_count, + indstate); + } + if (indstate != NULL) CatalogCloseIndexes(indstate); table_close(dependDesc, RowExclusiveLock); + + /* Drop only the number of slots used */ + for (i = 0; i < slot_init_count; i++) + ExecDropSingleTupleTableSlot(slot[i]); + pfree(slot); } /* diff --git a/src/backend/catalog/pg_shdepend.c b/src/backend/catalog/pg_shdepend.c index 30b234e90e12..3dd7afd34339 100644 --- a/src/backend/catalog/pg_shdepend.c +++ b/src/backend/catalog/pg_shdepend.c @@ -786,12 +786,6 @@ checkSharedDependencies(Oid classId, Oid objectId, } -/* - * Cap the maximum amount of bytes allocated for copyTemplateDependencies() - * slots. - */ -#define MAX_PGSHDEPEND_INSERT_BYTES 65535 - /* * copyTemplateDependencies * @@ -806,21 +800,20 @@ copyTemplateDependencies(Oid templateDbId, Oid newDbId) ScanKeyData key[1]; SysScanDesc scan; HeapTuple tup; - int slotCount; CatalogIndexState indstate; TupleTableSlot **slot; - int nslots, - max_slots; - bool slot_init = true; + int max_slots, + slot_init_count, + slot_stored_count; sdepRel = table_open(SharedDependRelationId, RowExclusiveLock); sdepDesc = RelationGetDescr(sdepRel); /* - * Allocate the slots to use, but delay initialization until we know that - * they will be used. + * Allocate the slots to use, but delay costly initialization until we + * know that they will be used. */ - max_slots = MAX_PGSHDEPEND_INSERT_BYTES / sizeof(FormData_pg_shdepend); + max_slots = MAX_CATALOG_MULTI_INSERT_BYTES / sizeof(FormData_pg_shdepend); slot = palloc(sizeof(TupleTableSlot *) * max_slots); indstate = CatalogOpenIndexes(sdepRel); @@ -834,6 +827,11 @@ copyTemplateDependencies(Oid templateDbId, Oid newDbId) scan = systable_beginscan(sdepRel, SharedDependDependerIndexId, true, NULL, 1, key); + /* number of slots currently storing tuples */ + slot_stored_count = 0; + /* number of slots currently initialized */ + slot_init_count = 0; + /* * Copy the entries of the original database, changing the database Id to * that of the new database. Note that because we are not copying rows @@ -841,41 +839,42 @@ copyTemplateDependencies(Oid templateDbId, Oid newDbId) * copy the ownership dependency of the template database itself; this is * what we want. */ - slotCount = 0; while (HeapTupleIsValid(tup = systable_getnext(scan))) { Form_pg_shdepend shdep; - if (slot_init) - slot[slotCount] = MakeSingleTupleTableSlot(sdepDesc, &TTSOpsHeapTuple); + if (slot_init_count < max_slots) + { + slot[slot_stored_count] = MakeSingleTupleTableSlot(sdepDesc, &TTSOpsHeapTuple); + slot_init_count++; + } - ExecClearTuple(slot[slotCount]); + ExecClearTuple(slot[slot_stored_count]); shdep = (Form_pg_shdepend) GETSTRUCT(tup); - slot[slotCount]->tts_values[Anum_pg_shdepend_dbid] = ObjectIdGetDatum(newDbId); - slot[slotCount]->tts_values[Anum_pg_shdepend_classid] = shdep->classid; - slot[slotCount]->tts_values[Anum_pg_shdepend_objid] = shdep->objid; - slot[slotCount]->tts_values[Anum_pg_shdepend_objsubid] = shdep->objsubid; - slot[slotCount]->tts_values[Anum_pg_shdepend_refclassid] = shdep->refclassid; - slot[slotCount]->tts_values[Anum_pg_shdepend_refobjid] = shdep->refobjid; - slot[slotCount]->tts_values[Anum_pg_shdepend_deptype] = shdep->deptype; + slot[slot_stored_count]->tts_values[Anum_pg_shdepend_dbid] = ObjectIdGetDatum(newDbId); + slot[slot_stored_count]->tts_values[Anum_pg_shdepend_classid] = shdep->classid; + slot[slot_stored_count]->tts_values[Anum_pg_shdepend_objid] = shdep->objid; + slot[slot_stored_count]->tts_values[Anum_pg_shdepend_objsubid] = shdep->objsubid; + slot[slot_stored_count]->tts_values[Anum_pg_shdepend_refclassid] = shdep->refclassid; + slot[slot_stored_count]->tts_values[Anum_pg_shdepend_refobjid] = shdep->refobjid; + slot[slot_stored_count]->tts_values[Anum_pg_shdepend_deptype] = shdep->deptype; - ExecStoreVirtualTuple(slot[slotCount]); - slotCount++; + ExecStoreVirtualTuple(slot[slot_stored_count]); + slot_stored_count++; /* If slots are full, insert a batch of tuples */ - if (slotCount == max_slots) + if (slot_stored_count == max_slots) { - CatalogTuplesMultiInsertWithInfo(sdepRel, slot, slotCount, indstate); - slotCount = 0; - slot_init = false; + CatalogTuplesMultiInsertWithInfo(sdepRel, slot, slot_stored_count, indstate); + slot_stored_count = 0; } } /* Insert any tuples left in the buffer */ - if (slotCount > 0) - CatalogTuplesMultiInsertWithInfo(sdepRel, slot, slotCount, indstate); + if (slot_stored_count > 0) + CatalogTuplesMultiInsertWithInfo(sdepRel, slot, slot_stored_count, indstate); systable_endscan(scan); @@ -883,8 +882,7 @@ copyTemplateDependencies(Oid templateDbId, Oid newDbId) table_close(sdepRel, RowExclusiveLock); /* Drop only the number of slots used */ - nslots = slot_init ? slotCount : max_slots; - for (int i = 0; i < nslots; i++) + for (int i = 0; i < slot_init_count; i++) ExecDropSingleTupleTableSlot(slot[i]); pfree(slot); } diff --git a/src/include/catalog/indexing.h b/src/include/catalog/indexing.h index a7e2a9b26b46..d86729dc6ce0 100644 --- a/src/include/catalog/indexing.h +++ b/src/include/catalog/indexing.h @@ -29,6 +29,12 @@ */ typedef struct ResultRelInfo *CatalogIndexState; +/* + * Cap the maximum amount of bytes allocated for multi-inserts with system + * catalogs, limiting the number of slots used. + */ +#define MAX_CATALOG_MULTI_INSERT_BYTES 65535 + /* * indexing.c prototypes */ From 556cbdfce4ffe01410b89dbf01b27315aa201bbf Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sat, 5 Sep 2020 11:32:20 +0200 Subject: [PATCH 080/589] Fix typo in comment --- src/backend/libpq/auth-scram.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/libpq/auth-scram.c b/src/backend/libpq/auth-scram.c index 5214d328656f..0f79b28bb5ab 100644 --- a/src/backend/libpq/auth-scram.c +++ b/src/backend/libpq/auth-scram.c @@ -1400,8 +1400,8 @@ scram_mock_salt(const char *username) /* * Generate salt using a SHA256 hash of the username and the cluster's * mock authentication nonce. (This works as long as the salt length is - * not larger the SHA256 digest length. If the salt is smaller, the caller - * will just ignore the extra data.) + * not larger than the SHA256 digest length. If the salt is smaller, the + * caller will just ignore the extra data.) */ StaticAssertStmt(PG_SHA256_DIGEST_LENGTH >= SCRAM_DEFAULT_SALT_LEN, "salt length greater than SHA256 digest length"); From 11b80d900fe4297e8e4bc231f6a41b53d604ed9e Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sat, 5 Sep 2020 13:28:05 +0200 Subject: [PATCH 081/589] Extend SQL function tests lightly The basic tests that defined SQL functions didn't actually run the functions to see if they worked. Add that, and also fix a minor mistake in a function that was revealed by this. (This is not a question of test coverage, since there are other places where SQL functions are run, but it is a bit of a silly test design.) Discussion: https://www.postgresql.org/message-id/flat/1c11f1eb-f00c-43b7-799d-2d44132c02d7@2ndquadrant.com --- .../regress/expected/create_function_3.out | 20 ++++++++++++++++++- src/test/regress/sql/create_function_3.sql | 6 +++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/test/regress/expected/create_function_3.out b/src/test/regress/expected/create_function_3.out index ba260df99607..ce508ae1dcc9 100644 --- a/src/test/regress/expected/create_function_3.out +++ b/src/test/regress/expected/create_function_3.out @@ -17,7 +17,7 @@ SET search_path TO temp_func_test, public; CREATE FUNCTION functest_A_1(text, date) RETURNS bool LANGUAGE 'sql' AS 'SELECT $1 = ''abcd'' AND $2 > ''2001-01-01'''; CREATE FUNCTION functest_A_2(text[]) RETURNS int LANGUAGE 'sql' - AS 'SELECT $1[0]::int'; + AS 'SELECT $1[1]::int'; CREATE FUNCTION functest_A_3() RETURNS bool LANGUAGE 'sql' AS 'SELECT false'; SELECT proname, prorettype::regtype, proargtypes::regtype[] FROM pg_proc @@ -31,6 +31,24 @@ SELECT proname, prorettype::regtype, proargtypes::regtype[] FROM pg_proc functest_a_3 | boolean | {} (3 rows) +SELECT functest_A_1('abcd', '2020-01-01'); + functest_a_1 +-------------- + t +(1 row) + +SELECT functest_A_2(ARRAY['1', '2', '3']); + functest_a_2 +-------------- + 1 +(1 row) + +SELECT functest_A_3(); + functest_a_3 +-------------- + f +(1 row) + -- -- IMMUTABLE | STABLE | VOLATILE -- diff --git a/src/test/regress/sql/create_function_3.sql b/src/test/regress/sql/create_function_3.sql index 7a2df0ea8a1b..bd108a918fbf 100644 --- a/src/test/regress/sql/create_function_3.sql +++ b/src/test/regress/sql/create_function_3.sql @@ -23,7 +23,7 @@ SET search_path TO temp_func_test, public; CREATE FUNCTION functest_A_1(text, date) RETURNS bool LANGUAGE 'sql' AS 'SELECT $1 = ''abcd'' AND $2 > ''2001-01-01'''; CREATE FUNCTION functest_A_2(text[]) RETURNS int LANGUAGE 'sql' - AS 'SELECT $1[0]::int'; + AS 'SELECT $1[1]::int'; CREATE FUNCTION functest_A_3() RETURNS bool LANGUAGE 'sql' AS 'SELECT false'; SELECT proname, prorettype::regtype, proargtypes::regtype[] FROM pg_proc @@ -31,6 +31,10 @@ SELECT proname, prorettype::regtype, proargtypes::regtype[] FROM pg_proc 'functest_A_2'::regproc, 'functest_A_3'::regproc) ORDER BY proname; +SELECT functest_A_1('abcd', '2020-01-01'); +SELECT functest_A_2(ARRAY['1', '2', '3']); +SELECT functest_A_3(); + -- -- IMMUTABLE | STABLE | VOLATILE -- From 8febfd1855450f50f17419def41c2ea9bcf994d5 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Sat, 5 Sep 2020 21:33:53 +0900 Subject: [PATCH 082/589] Switch to multi-inserts when registering dependencies for many code paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit improves the dependency registrations by taking advantage of the preliminary work done in 63110c62, to group together the insertion of dependencies of the same type to pg_depend. With the current layer of routines available, and as only dependencies of the same type can be grouped, there are code paths still doing more than one multi-insert when it is necessary to register dependencies of multiple types (constraint and index creation are two cases doing that). While on it, this refactors some of the code to use ObjectAddressSet() when manipulating object addresses. Author: Daniel Gustafsson, Michael Paquier Reviewed-by: Andres Freund, Álvaro Herrera Discussion: https://postgr.es/m/20200807061619.GA23955@paquier.xyz --- src/backend/catalog/heap.c | 59 +++++++------- src/backend/catalog/index.c | 21 ++++- src/backend/catalog/pg_aggregate.c | 23 +++--- src/backend/catalog/pg_cast.c | 28 ++++--- src/backend/catalog/pg_constraint.c | 34 +++++--- src/backend/catalog/pg_operator.c | 20 +++-- src/backend/catalog/pg_proc.c | 30 +++++--- src/backend/catalog/pg_range.c | 39 ++++------ src/backend/catalog/pg_type.c | 64 ++++++++------- src/backend/commands/functioncmds.c | 34 ++++---- src/backend/commands/proclang.c | 24 +++--- src/backend/commands/tsearchcmds.c | 90 +++++++++++----------- src/test/regress/expected/create_index.out | 6 +- 13 files changed, 252 insertions(+), 220 deletions(-) diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 9d2d357233b3..67144aa3c944 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -1428,15 +1428,9 @@ heap_create_with_catalog(const char *relname, { ObjectAddress myself, referenced; + ObjectAddresses *addrs; - myself.classId = RelationRelationId; - myself.objectId = relid; - myself.objectSubId = 0; - - referenced.classId = NamespaceRelationId; - referenced.objectId = relnamespace; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(myself, RelationRelationId, relid); recordDependencyOnOwner(RelationRelationId, relid, ownerid); @@ -1444,12 +1438,15 @@ heap_create_with_catalog(const char *relname, recordDependencyOnCurrentExtension(&myself, false); + addrs = new_object_addresses(); + + ObjectAddressSet(referenced, NamespaceRelationId, relnamespace); + add_exact_object_address(&referenced, addrs); + if (reloftypeid) { - referenced.classId = TypeRelationId; - referenced.objectId = reloftypeid; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(referenced, TypeRelationId, reloftypeid); + add_exact_object_address(&referenced, addrs); } /* @@ -1462,11 +1459,12 @@ heap_create_with_catalog(const char *relname, if (relkind == RELKIND_RELATION || relkind == RELKIND_MATVIEW) { - referenced.classId = AccessMethodRelationId; - referenced.objectId = accessmtd; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(referenced, AccessMethodRelationId, accessmtd); + add_exact_object_address(&referenced, addrs); } + + record_object_address_dependencies(&myself, addrs, DEPENDENCY_NORMAL); + free_object_addresses(addrs); } /* Post creation hook for new relation */ @@ -3574,6 +3572,7 @@ StorePartitionKey(Relation rel, bool nulls[Natts_pg_partitioned_table]; ObjectAddress myself; ObjectAddress referenced; + ObjectAddresses *addrs; Assert(rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); @@ -3617,31 +3616,27 @@ StorePartitionKey(Relation rel, table_close(pg_partitioned_table, RowExclusiveLock); /* Mark this relation as dependent on a few things as follows */ - myself.classId = RelationRelationId; - myself.objectId = RelationGetRelid(rel); - myself.objectSubId = 0; + addrs = new_object_addresses(); + ObjectAddressSet(myself, RelationRelationId, RelationGetRelid(rel)); /* Operator class and collation per key column */ for (i = 0; i < partnatts; i++) { - referenced.classId = OperatorClassRelationId; - referenced.objectId = partopclass[i]; - referenced.objectSubId = 0; - - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(referenced, OperatorClassRelationId, partopclass[i]); + add_exact_object_address(&referenced, addrs); /* The default collation is pinned, so don't bother recording it */ if (OidIsValid(partcollation[i]) && partcollation[i] != DEFAULT_COLLATION_OID) { - referenced.classId = CollationRelationId; - referenced.objectId = partcollation[i]; - referenced.objectSubId = 0; - - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(referenced, CollationRelationId, partcollation[i]); + add_exact_object_address(&referenced, addrs); } } + record_object_address_dependencies(&myself, addrs, DEPENDENCY_NORMAL); + free_object_addresses(addrs); + /* * The partitioning columns are made internally dependent on the table, * because we cannot drop any of them without dropping the whole table. @@ -3653,10 +3648,8 @@ StorePartitionKey(Relation rel, if (partattrs[i] == 0) continue; /* ignore expressions here */ - referenced.classId = RelationRelationId; - referenced.objectId = RelationGetRelid(rel); - referenced.objectSubId = partattrs[i]; - + ObjectAddressSubSet(referenced, RelationRelationId, + RelationGetRelid(rel), partattrs[i]); recordDependencyOn(&referenced, &myself, DEPENDENCY_INTERNAL); } diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 1d662e9af433..b2c8cb320efc 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -1018,6 +1018,7 @@ index_create(Relation heapRelation, { ObjectAddress myself, referenced; + ObjectAddresses *addrs; ObjectAddressSet(myself, RelationRelationId, indexRelationId); @@ -1054,6 +1055,8 @@ index_create(Relation heapRelation, { bool have_simple_col = false; + addrs = new_object_addresses(); + /* Create auto dependencies on simply-referenced columns */ for (i = 0; i < indexInfo->ii_NumIndexAttrs; i++) { @@ -1062,7 +1065,7 @@ index_create(Relation heapRelation, ObjectAddressSubSet(referenced, RelationRelationId, heapRelationId, indexInfo->ii_IndexAttrNumbers[i]); - recordDependencyOn(&myself, &referenced, DEPENDENCY_AUTO); + add_exact_object_address(&referenced, addrs); have_simple_col = true; } } @@ -1077,8 +1080,11 @@ index_create(Relation heapRelation, { ObjectAddressSet(referenced, RelationRelationId, heapRelationId); - recordDependencyOn(&myself, &referenced, DEPENDENCY_AUTO); + add_exact_object_address(&referenced, addrs); } + + record_object_address_dependencies(&myself, addrs, DEPENDENCY_AUTO); + free_object_addresses(addrs); } /* @@ -1096,7 +1102,11 @@ index_create(Relation heapRelation, recordDependencyOn(&myself, &referenced, DEPENDENCY_PARTITION_SEC); } + /* placeholder for normal dependencies */ + addrs = new_object_addresses(); + /* Store dependency on collations */ + /* The default collation is pinned, so don't bother recording it */ for (i = 0; i < indexInfo->ii_NumIndexKeyAttrs; i++) { @@ -1105,7 +1115,7 @@ index_create(Relation heapRelation, { ObjectAddressSet(referenced, CollationRelationId, collationObjectId[i]); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } } @@ -1113,9 +1123,12 @@ index_create(Relation heapRelation, for (i = 0; i < indexInfo->ii_NumIndexKeyAttrs; i++) { ObjectAddressSet(referenced, OperatorClassRelationId, classObjectId[i]); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } + record_object_address_dependencies(&myself, addrs, DEPENDENCY_NORMAL); + free_object_addresses(addrs); + /* Store dependencies on anything mentioned in index expressions */ if (indexInfo->ii_Expressions) { diff --git a/src/backend/catalog/pg_aggregate.c b/src/backend/catalog/pg_aggregate.c index 89007ad1ed7f..a0554f0d7973 100644 --- a/src/backend/catalog/pg_aggregate.c +++ b/src/backend/catalog/pg_aggregate.c @@ -105,6 +105,7 @@ AggregateCreate(const char *aggName, int i; ObjectAddress myself, referenced; + ObjectAddresses *addrs; AclResult aclresult; /* sanity checks (caller should have caught these) */ @@ -741,66 +742,70 @@ AggregateCreate(const char *aggName, * way. */ + addrs = new_object_addresses(); + /* Depends on transition function */ ObjectAddressSet(referenced, ProcedureRelationId, transfn); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); /* Depends on final function, if any */ if (OidIsValid(finalfn)) { ObjectAddressSet(referenced, ProcedureRelationId, finalfn); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } /* Depends on combine function, if any */ if (OidIsValid(combinefn)) { ObjectAddressSet(referenced, ProcedureRelationId, combinefn); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } /* Depends on serialization function, if any */ if (OidIsValid(serialfn)) { ObjectAddressSet(referenced, ProcedureRelationId, serialfn); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } /* Depends on deserialization function, if any */ if (OidIsValid(deserialfn)) { ObjectAddressSet(referenced, ProcedureRelationId, deserialfn); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } /* Depends on forward transition function, if any */ if (OidIsValid(mtransfn)) { ObjectAddressSet(referenced, ProcedureRelationId, mtransfn); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } /* Depends on inverse transition function, if any */ if (OidIsValid(minvtransfn)) { ObjectAddressSet(referenced, ProcedureRelationId, minvtransfn); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } /* Depends on final function, if any */ if (OidIsValid(mfinalfn)) { ObjectAddressSet(referenced, ProcedureRelationId, mfinalfn); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } /* Depends on sort operator, if any */ if (OidIsValid(sortop)) { ObjectAddressSet(referenced, OperatorRelationId, sortop); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } + record_object_address_dependencies(&myself, addrs, DEPENDENCY_NORMAL); + free_object_addresses(addrs); return myself; } diff --git a/src/backend/catalog/pg_cast.c b/src/backend/catalog/pg_cast.c index 5ea2b82b0834..c03e82d74fef 100644 --- a/src/backend/catalog/pg_cast.c +++ b/src/backend/catalog/pg_cast.c @@ -50,6 +50,7 @@ CastCreate(Oid sourcetypeid, Oid targettypeid, Oid funcid, char castcontext, bool nulls[Natts_pg_cast]; ObjectAddress myself, referenced; + ObjectAddresses *addrs; relation = table_open(CastRelationId, RowExclusiveLock); @@ -83,32 +84,29 @@ CastCreate(Oid sourcetypeid, Oid targettypeid, Oid funcid, char castcontext, CatalogTupleInsert(relation, tuple); + addrs = new_object_addresses(); + /* make dependency entries */ - myself.classId = CastRelationId; - myself.objectId = castid; - myself.objectSubId = 0; + ObjectAddressSet(myself, CastRelationId, castid); /* dependency on source type */ - referenced.classId = TypeRelationId; - referenced.objectId = sourcetypeid; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, behavior); + ObjectAddressSet(referenced, TypeRelationId, sourcetypeid); + add_exact_object_address(&referenced, addrs); /* dependency on target type */ - referenced.classId = TypeRelationId; - referenced.objectId = targettypeid; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, behavior); + ObjectAddressSet(referenced, TypeRelationId, targettypeid); + add_exact_object_address(&referenced, addrs); /* dependency on function */ if (OidIsValid(funcid)) { - referenced.classId = ProcedureRelationId; - referenced.objectId = funcid; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, behavior); + ObjectAddressSet(referenced, ProcedureRelationId, funcid); + add_exact_object_address(&referenced, addrs); } + record_object_address_dependencies(&myself, addrs, behavior); + free_object_addresses(addrs); + /* dependency on extension */ recordDependencyOnCurrentExtension(&myself, false); diff --git a/src/backend/catalog/pg_constraint.c b/src/backend/catalog/pg_constraint.c index 6a6b2cb8c0c8..0d70cb0c3c98 100644 --- a/src/backend/catalog/pg_constraint.c +++ b/src/backend/catalog/pg_constraint.c @@ -91,6 +91,8 @@ CreateConstraintEntry(const char *constraintName, NameData cname; int i; ObjectAddress conobject; + ObjectAddresses *addrs_auto; + ObjectAddresses *addrs_normal; conDesc = table_open(ConstraintRelationId, RowExclusiveLock); @@ -227,6 +229,9 @@ CreateConstraintEntry(const char *constraintName, table_close(conDesc, RowExclusiveLock); + /* Handle set of auto dependencies */ + addrs_auto = new_object_addresses(); + if (OidIsValid(relId)) { /* @@ -241,13 +246,13 @@ CreateConstraintEntry(const char *constraintName, { ObjectAddressSubSet(relobject, RelationRelationId, relId, constraintKey[i]); - recordDependencyOn(&conobject, &relobject, DEPENDENCY_AUTO); + add_exact_object_address(&relobject, addrs_auto); } } else { ObjectAddressSet(relobject, RelationRelationId, relId); - recordDependencyOn(&conobject, &relobject, DEPENDENCY_AUTO); + add_exact_object_address(&relobject, addrs_auto); } } @@ -259,9 +264,16 @@ CreateConstraintEntry(const char *constraintName, ObjectAddress domobject; ObjectAddressSet(domobject, TypeRelationId, domainId); - recordDependencyOn(&conobject, &domobject, DEPENDENCY_AUTO); + add_exact_object_address(&domobject, addrs_auto); } + record_object_address_dependencies(&conobject, addrs_auto, + DEPENDENCY_AUTO); + free_object_addresses(addrs_auto); + + /* Handle set of normal dependencies */ + addrs_normal = new_object_addresses(); + if (OidIsValid(foreignRelId)) { /* @@ -276,13 +288,13 @@ CreateConstraintEntry(const char *constraintName, { ObjectAddressSubSet(relobject, RelationRelationId, foreignRelId, foreignKey[i]); - recordDependencyOn(&conobject, &relobject, DEPENDENCY_NORMAL); + add_exact_object_address(&relobject, addrs_normal); } } else { ObjectAddressSet(relobject, RelationRelationId, foreignRelId); - recordDependencyOn(&conobject, &relobject, DEPENDENCY_NORMAL); + add_exact_object_address(&relobject, addrs_normal); } } @@ -297,7 +309,7 @@ CreateConstraintEntry(const char *constraintName, ObjectAddress relobject; ObjectAddressSet(relobject, RelationRelationId, indexRelId); - recordDependencyOn(&conobject, &relobject, DEPENDENCY_NORMAL); + add_exact_object_address(&relobject, addrs_normal); } if (foreignNKeys > 0) @@ -316,20 +328,24 @@ CreateConstraintEntry(const char *constraintName, for (i = 0; i < foreignNKeys; i++) { oprobject.objectId = pfEqOp[i]; - recordDependencyOn(&conobject, &oprobject, DEPENDENCY_NORMAL); + add_exact_object_address(&oprobject, addrs_normal); if (ppEqOp[i] != pfEqOp[i]) { oprobject.objectId = ppEqOp[i]; - recordDependencyOn(&conobject, &oprobject, DEPENDENCY_NORMAL); + add_exact_object_address(&oprobject, addrs_normal); } if (ffEqOp[i] != pfEqOp[i]) { oprobject.objectId = ffEqOp[i]; - recordDependencyOn(&conobject, &oprobject, DEPENDENCY_NORMAL); + add_exact_object_address(&oprobject, addrs_normal); } } } + record_object_address_dependencies(&conobject, addrs_normal, + DEPENDENCY_NORMAL); + free_object_addresses(addrs_normal); + /* * We don't bother to register dependencies on the exclusion operators of * an exclusion constraint. We assume they are members of the opclass diff --git a/src/backend/catalog/pg_operator.c b/src/backend/catalog/pg_operator.c index 65a36be5ee61..f7c07c9b5b89 100644 --- a/src/backend/catalog/pg_operator.c +++ b/src/backend/catalog/pg_operator.c @@ -775,6 +775,7 @@ makeOperatorDependencies(HeapTuple tuple, bool isUpdate) Form_pg_operator oper = (Form_pg_operator) GETSTRUCT(tuple); ObjectAddress myself, referenced; + ObjectAddresses *addrs; ObjectAddressSet(myself, OperatorRelationId, oper->oid); @@ -788,32 +789,34 @@ makeOperatorDependencies(HeapTuple tuple, bool isUpdate) deleteSharedDependencyRecordsFor(myself.classId, myself.objectId, 0); } + addrs = new_object_addresses(); + /* Dependency on namespace */ if (OidIsValid(oper->oprnamespace)) { ObjectAddressSet(referenced, NamespaceRelationId, oper->oprnamespace); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } /* Dependency on left type */ if (OidIsValid(oper->oprleft)) { ObjectAddressSet(referenced, TypeRelationId, oper->oprleft); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } /* Dependency on right type */ if (OidIsValid(oper->oprright)) { ObjectAddressSet(referenced, TypeRelationId, oper->oprright); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } /* Dependency on result type */ if (OidIsValid(oper->oprresult)) { ObjectAddressSet(referenced, TypeRelationId, oper->oprresult); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } /* @@ -829,23 +832,26 @@ makeOperatorDependencies(HeapTuple tuple, bool isUpdate) if (OidIsValid(oper->oprcode)) { ObjectAddressSet(referenced, ProcedureRelationId, oper->oprcode); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } /* Dependency on restriction selectivity function */ if (OidIsValid(oper->oprrest)) { ObjectAddressSet(referenced, ProcedureRelationId, oper->oprrest); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } /* Dependency on join selectivity function */ if (OidIsValid(oper->oprjoin)) { ObjectAddressSet(referenced, ProcedureRelationId, oper->oprjoin); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } + record_object_address_dependencies(&myself, addrs, DEPENDENCY_NORMAL); + free_object_addresses(addrs); + /* Dependency on owner */ recordDependencyOnOwner(OperatorRelationId, oper->oid, oper->oprowner); diff --git a/src/backend/catalog/pg_proc.c b/src/backend/catalog/pg_proc.c index a28ab74d6086..40d65dc6bab1 100644 --- a/src/backend/catalog/pg_proc.c +++ b/src/backend/catalog/pg_proc.c @@ -114,6 +114,7 @@ ProcedureCreate(const char *procedureName, char *detailmsg; int i; Oid trfid; + ObjectAddresses *addrs; /* * sanity checks @@ -585,53 +586,58 @@ ProcedureCreate(const char *procedureName, if (is_update) deleteDependencyRecordsFor(ProcedureRelationId, retval, true); + addrs = new_object_addresses(); + ObjectAddressSet(myself, ProcedureRelationId, retval); /* dependency on namespace */ ObjectAddressSet(referenced, NamespaceRelationId, procNamespace); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); /* dependency on implementation language */ ObjectAddressSet(referenced, LanguageRelationId, languageObjectId); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); /* dependency on return type */ ObjectAddressSet(referenced, TypeRelationId, returnType); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); /* dependency on transform used by return type, if any */ if ((trfid = get_transform_oid(returnType, languageObjectId, true))) { ObjectAddressSet(referenced, TransformRelationId, trfid); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } /* dependency on parameter types */ for (i = 0; i < allParamCount; i++) { ObjectAddressSet(referenced, TypeRelationId, allParams[i]); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); /* dependency on transform used by parameter type, if any */ if ((trfid = get_transform_oid(allParams[i], languageObjectId, true))) { ObjectAddressSet(referenced, TransformRelationId, trfid); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } } - /* dependency on parameter default expressions */ - if (parameterDefaults) - recordDependencyOnExpr(&myself, (Node *) parameterDefaults, - NIL, DEPENDENCY_NORMAL); - /* dependency on support function, if any */ if (OidIsValid(prosupport)) { ObjectAddressSet(referenced, ProcedureRelationId, prosupport); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } + record_object_address_dependencies(&myself, addrs, DEPENDENCY_NORMAL); + free_object_addresses(addrs); + + /* dependency on parameter default expressions */ + if (parameterDefaults) + recordDependencyOnExpr(&myself, (Node *) parameterDefaults, + NIL, DEPENDENCY_NORMAL); + /* dependency on owner */ if (!is_update) recordDependencyOnOwner(ProcedureRelationId, retval, proowner); diff --git a/src/backend/catalog/pg_range.c b/src/backend/catalog/pg_range.c index b5bc36c2bd6b..a606d8c3adb4 100644 --- a/src/backend/catalog/pg_range.c +++ b/src/backend/catalog/pg_range.c @@ -43,6 +43,7 @@ RangeCreate(Oid rangeTypeOid, Oid rangeSubType, Oid rangeCollation, HeapTuple tup; ObjectAddress myself; ObjectAddress referenced; + ObjectAddresses *addrs; pg_range = table_open(RangeRelationId, RowExclusiveLock); @@ -61,45 +62,37 @@ RangeCreate(Oid rangeTypeOid, Oid rangeSubType, Oid rangeCollation, heap_freetuple(tup); /* record type's dependencies on range-related items */ + addrs = new_object_addresses(); - myself.classId = TypeRelationId; - myself.objectId = rangeTypeOid; - myself.objectSubId = 0; + ObjectAddressSet(myself, TypeRelationId, rangeTypeOid); - referenced.classId = TypeRelationId; - referenced.objectId = rangeSubType; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(referenced, TypeRelationId, rangeSubType); + add_exact_object_address(&referenced, addrs); - referenced.classId = OperatorClassRelationId; - referenced.objectId = rangeSubOpclass; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(referenced, OperatorClassRelationId, rangeSubOpclass); + add_exact_object_address(&referenced, addrs); if (OidIsValid(rangeCollation)) { - referenced.classId = CollationRelationId; - referenced.objectId = rangeCollation; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(referenced, CollationRelationId, rangeCollation); + add_exact_object_address(&referenced, addrs); } if (OidIsValid(rangeCanonical)) { - referenced.classId = ProcedureRelationId; - referenced.objectId = rangeCanonical; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(referenced, ProcedureRelationId, rangeCanonical); + add_exact_object_address(&referenced, addrs); } if (OidIsValid(rangeSubDiff)) { - referenced.classId = ProcedureRelationId; - referenced.objectId = rangeSubDiff; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(referenced, ProcedureRelationId, rangeSubDiff); + add_exact_object_address(&referenced, addrs); } + record_object_address_dependencies(&myself, addrs, DEPENDENCY_NORMAL); + free_object_addresses(addrs); + table_close(pg_range, RowExclusiveLock); } diff --git a/src/backend/catalog/pg_type.c b/src/backend/catalog/pg_type.c index 79ffe317dde4..0b04dff7731c 100644 --- a/src/backend/catalog/pg_type.c +++ b/src/backend/catalog/pg_type.c @@ -554,6 +554,7 @@ GenerateTypeDependencies(HeapTuple typeTuple, bool isNull; ObjectAddress myself, referenced; + ObjectAddresses *addrs_normal; /* Extract defaultExpr if caller didn't pass it */ if (defaultExpr == NULL) @@ -587,6 +588,10 @@ GenerateTypeDependencies(HeapTuple typeTuple, * Skip these for a dependent type, since it will have such dependencies * indirectly through its depended-on type or relation. */ + + /* placeholder for all normal dependencies */ + addrs_normal = new_object_addresses(); + if (!isDependentType) { ObjectAddressSet(referenced, NamespaceRelationId, @@ -606,45 +611,70 @@ GenerateTypeDependencies(HeapTuple typeTuple, if (OidIsValid(typeForm->typinput)) { ObjectAddressSet(referenced, ProcedureRelationId, typeForm->typinput); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs_normal); } if (OidIsValid(typeForm->typoutput)) { ObjectAddressSet(referenced, ProcedureRelationId, typeForm->typoutput); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs_normal); } if (OidIsValid(typeForm->typreceive)) { ObjectAddressSet(referenced, ProcedureRelationId, typeForm->typreceive); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs_normal); } if (OidIsValid(typeForm->typsend)) { ObjectAddressSet(referenced, ProcedureRelationId, typeForm->typsend); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs_normal); } if (OidIsValid(typeForm->typmodin)) { ObjectAddressSet(referenced, ProcedureRelationId, typeForm->typmodin); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs_normal); } if (OidIsValid(typeForm->typmodout)) { ObjectAddressSet(referenced, ProcedureRelationId, typeForm->typmodout); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs_normal); } if (OidIsValid(typeForm->typanalyze)) { ObjectAddressSet(referenced, ProcedureRelationId, typeForm->typanalyze); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs_normal); + } + + /* Normal dependency from a domain to its base type. */ + if (OidIsValid(typeForm->typbasetype)) + { + ObjectAddressSet(referenced, TypeRelationId, typeForm->typbasetype); + add_exact_object_address(&referenced, addrs_normal); + } + + /* + * Normal dependency from a domain to its collation. We know the default + * collation is pinned, so don't bother recording it. + */ + if (OidIsValid(typeForm->typcollation) && + typeForm->typcollation != DEFAULT_COLLATION_OID) + { + ObjectAddressSet(referenced, CollationRelationId, typeForm->typcollation); + add_exact_object_address(&referenced, addrs_normal); } + record_object_address_dependencies(&myself, addrs_normal, DEPENDENCY_NORMAL); + free_object_addresses(addrs_normal); + + /* Normal dependency on the default expression. */ + if (defaultExpr) + recordDependencyOnExpr(&myself, defaultExpr, NIL, DEPENDENCY_NORMAL); + /* * If the type is a rowtype for a relation, mark it as internally * dependent on the relation, *unless* it is a stand-alone composite type @@ -675,26 +705,6 @@ GenerateTypeDependencies(HeapTuple typeTuple, recordDependencyOn(&myself, &referenced, isImplicitArray ? DEPENDENCY_INTERNAL : DEPENDENCY_NORMAL); } - - /* Normal dependency from a domain to its base type. */ - if (OidIsValid(typeForm->typbasetype)) - { - ObjectAddressSet(referenced, TypeRelationId, typeForm->typbasetype); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); - } - - /* Normal dependency from a domain to its collation. */ - /* We know the default collation is pinned, so don't bother recording it */ - if (OidIsValid(typeForm->typcollation) && - typeForm->typcollation != DEFAULT_COLLATION_OID) - { - ObjectAddressSet(referenced, CollationRelationId, typeForm->typcollation); - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); - } - - /* Normal dependency on the default expression. */ - if (defaultExpr) - recordDependencyOnExpr(&myself, defaultExpr, NIL, DEPENDENCY_NORMAL); } /* diff --git a/src/backend/commands/functioncmds.c b/src/backend/commands/functioncmds.c index 1b5bdcec8b87..e236581a8e06 100644 --- a/src/backend/commands/functioncmds.c +++ b/src/backend/commands/functioncmds.c @@ -1696,6 +1696,7 @@ CreateTransform(CreateTransformStmt *stmt) Relation relation; ObjectAddress myself, referenced; + ObjectAddresses *addrs; bool is_replace; /* @@ -1836,39 +1837,34 @@ CreateTransform(CreateTransformStmt *stmt) if (is_replace) deleteDependencyRecordsFor(TransformRelationId, transformid, true); + addrs = new_object_addresses(); + /* make dependency entries */ - myself.classId = TransformRelationId; - myself.objectId = transformid; - myself.objectSubId = 0; + ObjectAddressSet(myself, TransformRelationId, transformid); /* dependency on language */ - referenced.classId = LanguageRelationId; - referenced.objectId = langid; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(referenced, LanguageRelationId, langid); + add_exact_object_address(&referenced, addrs); /* dependency on type */ - referenced.classId = TypeRelationId; - referenced.objectId = typeid; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(referenced, TypeRelationId, typeid); + add_exact_object_address(&referenced, addrs); /* dependencies on functions */ if (OidIsValid(fromsqlfuncid)) { - referenced.classId = ProcedureRelationId; - referenced.objectId = fromsqlfuncid; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(referenced, ProcedureRelationId, fromsqlfuncid); + add_exact_object_address(&referenced, addrs); } if (OidIsValid(tosqlfuncid)) { - referenced.classId = ProcedureRelationId; - referenced.objectId = tosqlfuncid; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(referenced, ProcedureRelationId, tosqlfuncid); + add_exact_object_address(&referenced, addrs); } + record_object_address_dependencies(&myself, addrs, DEPENDENCY_NORMAL); + free_object_addresses(addrs); + /* dependency on extension */ recordDependencyOnCurrentExtension(&myself, is_replace); diff --git a/src/backend/commands/proclang.c b/src/backend/commands/proclang.c index 14153426bdda..8ef60374f590 100644 --- a/src/backend/commands/proclang.c +++ b/src/backend/commands/proclang.c @@ -57,6 +57,7 @@ CreateProceduralLanguage(CreatePLangStmt *stmt) bool is_update; ObjectAddress myself, referenced; + ObjectAddresses *addrs; /* * Check permission @@ -186,30 +187,29 @@ CreateProceduralLanguage(CreatePLangStmt *stmt) /* dependency on extension */ recordDependencyOnCurrentExtension(&myself, is_update); + addrs = new_object_addresses(); + /* dependency on the PL handler function */ - referenced.classId = ProcedureRelationId; - referenced.objectId = handlerOid; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(referenced, ProcedureRelationId, handlerOid); + add_exact_object_address(&referenced, addrs); /* dependency on the inline handler function, if any */ if (OidIsValid(inlineOid)) { - referenced.classId = ProcedureRelationId; - referenced.objectId = inlineOid; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(referenced, ProcedureRelationId, inlineOid); + add_exact_object_address(&referenced, addrs); } /* dependency on the validator function, if any */ if (OidIsValid(valOid)) { - referenced.classId = ProcedureRelationId; - referenced.objectId = valOid; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(referenced, ProcedureRelationId, valOid); + add_exact_object_address(&referenced, addrs); } + record_object_address_dependencies(&myself, addrs, DEPENDENCY_NORMAL); + free_object_addresses(addrs); + /* Post creation hook for new procedural language */ InvokeObjectPostCreateHook(LanguageRelationId, myself.objectId, 0); diff --git a/src/backend/commands/tsearchcmds.c b/src/backend/commands/tsearchcmds.c index 319a62012ed0..f5d1d137b814 100644 --- a/src/backend/commands/tsearchcmds.c +++ b/src/backend/commands/tsearchcmds.c @@ -133,42 +133,41 @@ makeParserDependencies(HeapTuple tuple) Form_pg_ts_parser prs = (Form_pg_ts_parser) GETSTRUCT(tuple); ObjectAddress myself, referenced; + ObjectAddresses *addrs; - myself.classId = TSParserRelationId; - myself.objectId = prs->oid; - myself.objectSubId = 0; - - /* dependency on namespace */ - referenced.classId = NamespaceRelationId; - referenced.objectId = prs->prsnamespace; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(myself, TSParserRelationId, prs->oid); /* dependency on extension */ recordDependencyOnCurrentExtension(&myself, false); - /* dependencies on functions */ - referenced.classId = ProcedureRelationId; - referenced.objectSubId = 0; + addrs = new_object_addresses(); - referenced.objectId = prs->prsstart; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + /* dependency on namespace */ + ObjectAddressSet(referenced, NamespaceRelationId, prs->prsnamespace); + add_exact_object_address(&referenced, addrs); + + /* dependencies on functions */ + ObjectAddressSet(referenced, ProcedureRelationId, prs->prsstart); + add_exact_object_address(&referenced, addrs); referenced.objectId = prs->prstoken; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); referenced.objectId = prs->prsend; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); referenced.objectId = prs->prslextype; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); if (OidIsValid(prs->prsheadline)) { referenced.objectId = prs->prsheadline; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } + record_object_address_dependencies(&myself, addrs, DEPENDENCY_NORMAL); + free_object_addresses(addrs); + return myself; } @@ -304,16 +303,9 @@ makeDictionaryDependencies(HeapTuple tuple) Form_pg_ts_dict dict = (Form_pg_ts_dict) GETSTRUCT(tuple); ObjectAddress myself, referenced; + ObjectAddresses *addrs; - myself.classId = TSDictionaryRelationId; - myself.objectId = dict->oid; - myself.objectSubId = 0; - - /* dependency on namespace */ - referenced.classId = NamespaceRelationId; - referenced.objectId = dict->dictnamespace; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(myself, TSDictionaryRelationId, dict->oid); /* dependency on owner */ recordDependencyOnOwner(myself.classId, myself.objectId, dict->dictowner); @@ -321,11 +313,18 @@ makeDictionaryDependencies(HeapTuple tuple) /* dependency on extension */ recordDependencyOnCurrentExtension(&myself, false); + addrs = new_object_addresses(); + + /* dependency on namespace */ + ObjectAddressSet(referenced, NamespaceRelationId, dict->dictnamespace); + add_exact_object_address(&referenced, addrs); + /* dependency on template */ - referenced.classId = TSTemplateRelationId; - referenced.objectId = dict->dicttemplate; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(referenced, TSTemplateRelationId, dict->dicttemplate); + add_exact_object_address(&referenced, addrs); + + record_object_address_dependencies(&myself, addrs, DEPENDENCY_NORMAL); + free_object_addresses(addrs); return myself; } @@ -649,33 +648,32 @@ makeTSTemplateDependencies(HeapTuple tuple) Form_pg_ts_template tmpl = (Form_pg_ts_template) GETSTRUCT(tuple); ObjectAddress myself, referenced; + ObjectAddresses *addrs; - myself.classId = TSTemplateRelationId; - myself.objectId = tmpl->oid; - myself.objectSubId = 0; - - /* dependency on namespace */ - referenced.classId = NamespaceRelationId; - referenced.objectId = tmpl->tmplnamespace; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + ObjectAddressSet(myself, TSTemplateRelationId, tmpl->oid); /* dependency on extension */ recordDependencyOnCurrentExtension(&myself, false); - /* dependencies on functions */ - referenced.classId = ProcedureRelationId; - referenced.objectSubId = 0; + addrs = new_object_addresses(); - referenced.objectId = tmpl->tmpllexize; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + /* dependency on namespace */ + ObjectAddressSet(referenced, NamespaceRelationId, tmpl->tmplnamespace); + add_exact_object_address(&referenced, addrs); + + /* dependencies on functions */ + ObjectAddressSet(referenced, ProcedureRelationId, tmpl->tmpllexize); + add_exact_object_address(&referenced, addrs); if (OidIsValid(tmpl->tmplinit)) { referenced.objectId = tmpl->tmplinit; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + add_exact_object_address(&referenced, addrs); } + record_object_address_dependencies(&myself, addrs, DEPENDENCY_NORMAL); + free_object_addresses(addrs); + return myself; } diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index e3e6634d7e1c..814416d936b0 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2069,11 +2069,10 @@ WHERE classid = 'pg_class'::regclass AND index concur_reindex_ind3 | column c1 of table concur_reindex_tab | a index concur_reindex_ind3 | table concur_reindex_tab | a index concur_reindex_ind4 | column c1 of table concur_reindex_tab | a - index concur_reindex_ind4 | column c1 of table concur_reindex_tab | a index concur_reindex_ind4 | column c2 of table concur_reindex_tab | a materialized view concur_reindex_matview | schema public | n table concur_reindex_tab | schema public | n -(9 rows) +(8 rows) REINDEX INDEX CONCURRENTLY concur_reindex_ind1; REINDEX TABLE CONCURRENTLY concur_reindex_tab; @@ -2097,11 +2096,10 @@ WHERE classid = 'pg_class'::regclass AND index concur_reindex_ind3 | column c1 of table concur_reindex_tab | a index concur_reindex_ind3 | table concur_reindex_tab | a index concur_reindex_ind4 | column c1 of table concur_reindex_tab | a - index concur_reindex_ind4 | column c1 of table concur_reindex_tab | a index concur_reindex_ind4 | column c2 of table concur_reindex_tab | a materialized view concur_reindex_matview | schema public | n table concur_reindex_tab | schema public | n -(9 rows) +(8 rows) -- Check that comments are preserved CREATE TABLE testcomment (i int); From a5cc4dab6d1d694f113912a2aca7012a95262f0b Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 5 Sep 2020 13:17:32 -0400 Subject: [PATCH 083/589] Yet more elimination of dead stores and useless initializations. I'm not sure what tool Ranier was using, but the ones I contributed were found by using a newer version of scan-build than I tried before. Ranier Vilela and Tom Lane Discussion: https://postgr.es/m/CAEudQAo1+AcGppxDSg8k+zF4+Kv+eJyqzEDdbpDg58-=MQcerQ@mail.gmail.com --- src/backend/access/common/heaptuple.c | 2 +- src/backend/access/gist/gistutil.c | 2 +- src/backend/access/nbtree/nbtsearch.c | 6 +++--- src/backend/catalog/storage.c | 3 +-- src/backend/commands/async.c | 3 +-- src/backend/storage/ipc/procarray.c | 2 +- src/backend/tsearch/spell.c | 2 +- src/backend/utils/adt/formatting.c | 19 +++++++++---------- src/backend/utils/adt/tsrank.c | 4 ++-- src/bin/pg_dump/pg_backup_custom.c | 7 ++----- 10 files changed, 22 insertions(+), 28 deletions(-) diff --git a/src/backend/access/common/heaptuple.c b/src/backend/access/common/heaptuple.c index f89769f37937..f08221eed3c9 100644 --- a/src/backend/access/common/heaptuple.c +++ b/src/backend/access/common/heaptuple.c @@ -736,7 +736,7 @@ expand_tuple(HeapTuple *targetHeapTuple, { AttrMissing *attrmiss = NULL; int attnum; - int firstmissingnum = 0; + int firstmissingnum; bool hasNulls = HeapTupleHasNulls(sourceTuple); HeapTupleHeader targetTHeader; HeapTupleHeader sourceTHeader = sourceTuple->t_data; diff --git a/src/backend/access/gist/gistutil.c b/src/backend/access/gist/gistutil.c index bfda7fbe3d58..0516059e3ddc 100644 --- a/src/backend/access/gist/gistutil.c +++ b/src/backend/access/gist/gistutil.c @@ -32,7 +32,6 @@ void gistfillbuffer(Page page, IndexTuple *itup, int len, OffsetNumber off) { - OffsetNumber l = InvalidOffsetNumber; int i; if (off == InvalidOffsetNumber) @@ -42,6 +41,7 @@ gistfillbuffer(Page page, IndexTuple *itup, int len, OffsetNumber off) for (i = 0; i < len; i++) { Size sz = IndexTupleSize(itup[i]); + OffsetNumber l; l = PageAddItem(page, (Item) itup[i], sz, off, false, false); if (l == InvalidOffsetNumber) diff --git a/src/backend/access/nbtree/nbtsearch.c b/src/backend/access/nbtree/nbtsearch.c index 28dc196b55e3..1e628a33d77e 100644 --- a/src/backend/access/nbtree/nbtsearch.c +++ b/src/backend/access/nbtree/nbtsearch.c @@ -860,7 +860,7 @@ _bt_first(IndexScanDesc scan, ScanDirection dir) ScanKeyData notnullkeys[INDEX_MAX_KEYS]; int keysCount = 0; int i; - bool status = true; + bool status; StrategyNumber strat_total; BTScanPosItem *currItem; BlockNumber blkno; @@ -1858,7 +1858,7 @@ _bt_steppage(IndexScanDesc scan, ScanDirection dir) { BTScanOpaque so = (BTScanOpaque) scan->opaque; BlockNumber blkno = InvalidBlockNumber; - bool status = true; + bool status; Assert(BTScanPosIsValid(so->currPos)); @@ -1967,7 +1967,7 @@ _bt_readnextpage(IndexScanDesc scan, BlockNumber blkno, ScanDirection dir) Relation rel; Page page; BTPageOpaque opaque; - bool status = true; + bool status; rel = scan->indexRelation; diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c index 9e6e6c42d3c8..dbbd3aa31fe1 100644 --- a/src/backend/catalog/storage.c +++ b/src/backend/catalog/storage.c @@ -599,7 +599,6 @@ smgrDoPendingDeletes(bool isCommit) PendingRelDelete *prev; PendingRelDelete *next; int nrels = 0, - i = 0, maxrels = 0; SMgrRelation *srels = NULL; @@ -650,7 +649,7 @@ smgrDoPendingDeletes(bool isCommit) { smgrdounlinkall(srels, nrels, false); - for (i = 0; i < nrels; i++) + for (int i = 0; i < nrels; i++) smgrclose(srels[i]); pfree(srels); diff --git a/src/backend/commands/async.c b/src/backend/commands/async.c index 774b26fd2c4d..cb341365df46 100644 --- a/src/backend/commands/async.c +++ b/src/backend/commands/async.c @@ -1918,7 +1918,6 @@ static void asyncQueueReadAllNotifications(void) { volatile QueuePosition pos; - QueuePosition oldpos; QueuePosition head; Snapshot snapshot; @@ -1933,7 +1932,7 @@ asyncQueueReadAllNotifications(void) LWLockAcquire(NotifyQueueLock, LW_SHARED); /* Assert checks that we have a valid state entry */ Assert(MyProcPid == QUEUE_BACKEND_PID(MyBackendId)); - pos = oldpos = QUEUE_BACKEND_POS(MyBackendId); + pos = QUEUE_BACKEND_POS(MyBackendId); head = QUEUE_HEAD; LWLockRelease(NotifyQueueLock); diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c index a023090fbbd3..1c0cd6b2487b 100644 --- a/src/backend/storage/ipc/procarray.c +++ b/src/backend/storage/ipc/procarray.c @@ -3433,7 +3433,6 @@ CancelDBBackends(Oid databaseid, ProcSignalReason sigmode, bool conflictPending) { ProcArrayStruct *arrayP = procArray; int index; - pid_t pid = 0; /* tell all backends to die */ LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); @@ -3446,6 +3445,7 @@ CancelDBBackends(Oid databaseid, ProcSignalReason sigmode, bool conflictPending) if (databaseid == InvalidOid || proc->databaseId == databaseid) { VirtualTransactionId procvxid; + pid_t pid; GET_VXID_FROM_PGPROC(procvxid, *proc); diff --git a/src/backend/tsearch/spell.c b/src/backend/tsearch/spell.c index 49735bc06af8..05d08cfc0102 100644 --- a/src/backend/tsearch/spell.c +++ b/src/backend/tsearch/spell.c @@ -1710,7 +1710,7 @@ void NISortDictionary(IspellDict *Conf) { int i; - int naffix = 0; + int naffix; int curaffix; /* compress affixes */ diff --git a/src/backend/utils/adt/formatting.c b/src/backend/utils/adt/formatting.c index bf9643ffb4a1..7d09537d82b9 100644 --- a/src/backend/utils/adt/formatting.c +++ b/src/backend/utils/adt/formatting.c @@ -1510,8 +1510,7 @@ static const char * get_th(char *num, int type) { int len = strlen(num), - last, - seclast; + last; last = *(num + (len - 1)); if (!isdigit((unsigned char) last)) @@ -1523,7 +1522,7 @@ get_th(char *num, int type) * All "teens" (1[0-9]) get 'TH/th', while [02-9][123] still get * 'ST/st', 'ND/nd', 'RD/rd', respectively */ - if ((len > 1) && ((seclast = num[len - 2]) == '1')) + if ((len > 1) && (num[len - 2] == '1')) last = 0; switch (last) @@ -4932,9 +4931,9 @@ NUM_cache(int len, NUMDesc *Num, text *pars_str, bool *shouldFree) static char * int_to_roman(int number) { - int len = 0, - num = 0; - char *p = NULL, + int len, + num; + char *p, *result, numstr[12]; @@ -4950,7 +4949,7 @@ int_to_roman(int number) for (p = numstr; *p != '\0'; p++, --len) { - num = *p - 49; /* 48 ascii + 1 */ + num = *p - ('0' + 1); if (num < 0) continue; @@ -6118,7 +6117,7 @@ numeric_to_char(PG_FUNCTION_ARGS) x = DatumGetNumeric(DirectFunctionCall2(numeric_round, NumericGetDatum(value), Int32GetDatum(0))); - numstr = orgnum = + numstr = int_to_roman(DatumGetInt32(DirectFunctionCall1(numeric_int4, NumericGetDatum(x)))); } @@ -6239,7 +6238,7 @@ int4_to_char(PG_FUNCTION_ARGS) * On DateType depend part (int32) */ if (IS_ROMAN(&Num)) - numstr = orgnum = int_to_roman(value); + numstr = int_to_roman(value); else if (IS_EEEE(&Num)) { /* we can do it easily because float8 won't lose any precision */ @@ -6335,7 +6334,7 @@ int8_to_char(PG_FUNCTION_ARGS) if (IS_ROMAN(&Num)) { /* Currently don't support int8 conversion to roman... */ - numstr = orgnum = int_to_roman(DatumGetInt32(DirectFunctionCall1(int84, Int64GetDatum(value)))); + numstr = int_to_roman(DatumGetInt32(DirectFunctionCall1(int84, Int64GetDatum(value)))); } else if (IS_EEEE(&Num)) { diff --git a/src/backend/utils/adt/tsrank.c b/src/backend/utils/adt/tsrank.c index c88ebfc7d411..38b413f6ffff 100644 --- a/src/backend/utils/adt/tsrank.c +++ b/src/backend/utils/adt/tsrank.c @@ -857,8 +857,7 @@ calc_rank_cd(const float4 *arrdata, TSVector txt, TSQuery query, int method) double Wdoc = 0.0; double invws[lengthof(weights)]; double SumDist = 0.0, - PrevExtPos = 0.0, - CurExtPos = 0.0; + PrevExtPos = 0.0; int NExtent = 0; QueryRepresentation qr; @@ -889,6 +888,7 @@ calc_rank_cd(const float4 *arrdata, TSVector txt, TSQuery query, int method) { double Cpos = 0.0; double InvSum = 0.0; + double CurExtPos; int nNoise; DocRepresentation *ptr = ext.begin; diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 971e6adf4875..77d402c323e9 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -619,7 +619,6 @@ _skipData(ArchiveHandle *AH) size_t blkLen; char *buf = NULL; int buflen = 0; - size_t cnt; blkLen = ReadInt(AH); while (blkLen != 0) @@ -638,7 +637,7 @@ _skipData(ArchiveHandle *AH) buf = (char *) pg_malloc(blkLen); buflen = blkLen; } - if ((cnt = fread(buf, 1, blkLen, AH->FH)) != blkLen) + if (fread(buf, 1, blkLen, AH->FH) != blkLen) { if (feof(AH->FH)) fatal("could not read from input file: end of file"); @@ -664,9 +663,7 @@ _skipData(ArchiveHandle *AH) static int _WriteByte(ArchiveHandle *AH, const int i) { - int res; - - if ((res = fputc(i, AH->FH)) == EOF) + if (fputc(i, AH->FH) == EOF) WRITE_ERROR_EXIT; return 1; From e0f05cd5ba76a75e2ce3b85ba050e48e857dca00 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 5 Sep 2020 16:20:04 -0400 Subject: [PATCH 084/589] Improve some ancient, crufty code in bootstrap + initdb. At some point back in the last century, somebody felt that reading all of pg_type twice was cheaper, or at least easier, than using repalloc() to resize the Typ[] array dynamically. That seems like an entirely wacko proposition, so rewrite the code to do it the other way. (To add insult to injury, there were two not-quite-identical copies of said code.) initdb.c's readfile() function had the same disease of preferring to do double the I/O to avoid resizing its output array. Here, we can make things easier by using the just-invented pg_get_line() function to handle reading individual lines without a predetermined notion of how long they are. On my machine, it's difficult to detect any net change in the overall runtime of initdb from these changes; but they should help on slower buildfarm machines (especially since a buildfarm cycle involves a lot of initdb's these days). My attention was drawn to these places by scan-build complaints, but on inspection they needed a lot more work than just suppressing dead stores :-( --- src/backend/bootstrap/bootstrap.c | 119 +++++++++++++++--------------- src/bin/initdb/initdb.c | 46 ++++-------- 2 files changed, 74 insertions(+), 91 deletions(-) diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index 45b7efbe4659..76b2f5066f68 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -53,14 +53,12 @@ uint32 bootstrap_data_checksum_version = 0; /* No checksum */ -#define ALLOC(t, c) \ - ((t *) MemoryContextAllocZero(TopMemoryContext, (unsigned)(c) * sizeof(t))) - static void CheckerModeMain(void); static void BootstrapModeMain(void); static void bootstrap_signals(void); static void ShutdownAuxiliaryProcess(int code, Datum arg); static Form_pg_attribute AllocateAttribute(void); +static void populate_typ_array(void); static Oid gettype(char *type); static void cleanup(void); @@ -583,46 +581,24 @@ ShutdownAuxiliaryProcess(int code, Datum arg) /* ---------------- * boot_openrel + * + * Execute BKI OPEN command. * ---------------- */ void boot_openrel(char *relname) { int i; - struct typmap **app; - Relation rel; - TableScanDesc scan; - HeapTuple tup; if (strlen(relname) >= NAMEDATALEN) relname[NAMEDATALEN - 1] = '\0'; + /* + * pg_type must be filled before any OPEN command is executed, hence we + * can now populate the Typ array if we haven't yet. + */ if (Typ == NULL) - { - /* We can now load the pg_type data */ - rel = table_open(TypeRelationId, NoLock); - scan = table_beginscan_catalog(rel, 0, NULL); - i = 0; - while ((tup = heap_getnext(scan, ForwardScanDirection)) != NULL) - ++i; - table_endscan(scan); - app = Typ = ALLOC(struct typmap *, i + 1); - while (i-- > 0) - *app++ = ALLOC(struct typmap, 1); - *app = NULL; - scan = table_beginscan_catalog(rel, 0, NULL); - app = Typ; - while ((tup = heap_getnext(scan, ForwardScanDirection)) != NULL) - { - (*app)->am_oid = ((Form_pg_type) GETSTRUCT(tup))->oid; - memcpy((char *) &(*app)->am_typ, - (char *) GETSTRUCT(tup), - sizeof((*app)->am_typ)); - app++; - } - table_endscan(scan); - table_close(rel, NoLock); - } + populate_typ_array(); if (boot_reldesc != NULL) closerel(NULL); @@ -889,6 +865,52 @@ cleanup(void) closerel(NULL); } +/* ---------------- + * populate_typ_array + * + * Load the Typ array by reading pg_type. + * ---------------- + */ +static void +populate_typ_array(void) +{ + Relation rel; + TableScanDesc scan; + HeapTuple tup; + int nalloc; + int i; + + Assert(Typ == NULL); + + nalloc = 512; + Typ = (struct typmap **) + MemoryContextAlloc(TopMemoryContext, nalloc * sizeof(struct typmap *)); + + rel = table_open(TypeRelationId, NoLock); + scan = table_beginscan_catalog(rel, 0, NULL); + i = 0; + while ((tup = heap_getnext(scan, ForwardScanDirection)) != NULL) + { + Form_pg_type typForm = (Form_pg_type) GETSTRUCT(tup); + + /* make sure there will be room for a trailing NULL pointer */ + if (i >= nalloc - 1) + { + nalloc *= 2; + Typ = (struct typmap **) + repalloc(Typ, nalloc * sizeof(struct typmap *)); + } + Typ[i] = (struct typmap *) + MemoryContextAlloc(TopMemoryContext, sizeof(struct typmap)); + Typ[i]->am_oid = typForm->oid; + memcpy(&(Typ[i]->am_typ), typForm, sizeof(Typ[i]->am_typ)); + i++; + } + Typ[i] = NULL; /* Fill trailing NULL pointer */ + table_endscan(scan); + table_close(rel, NoLock); +} + /* ---------------- * gettype * @@ -903,14 +925,10 @@ cleanup(void) static Oid gettype(char *type) { - int i; - Relation rel; - TableScanDesc scan; - HeapTuple tup; - struct typmap **app; - if (Typ != NULL) { + struct typmap **app; + for (app = Typ; *app != NULL; app++) { if (strncmp(NameStr((*app)->am_typ.typname), type, NAMEDATALEN) == 0) @@ -922,33 +940,16 @@ gettype(char *type) } else { + int i; + for (i = 0; i < n_types; i++) { if (strncmp(type, TypInfo[i].name, NAMEDATALEN) == 0) return i; } + /* Not in TypInfo, so we'd better be able to read pg_type now */ elog(DEBUG4, "external type: %s", type); - rel = table_open(TypeRelationId, NoLock); - scan = table_beginscan_catalog(rel, 0, NULL); - i = 0; - while ((tup = heap_getnext(scan, ForwardScanDirection)) != NULL) - ++i; - table_endscan(scan); - app = Typ = ALLOC(struct typmap *, i + 1); - while (i-- > 0) - *app++ = ALLOC(struct typmap, 1); - *app = NULL; - scan = table_beginscan_catalog(rel, 0, NULL); - app = Typ; - while ((tup = heap_getnext(scan, ForwardScanDirection)) != NULL) - { - (*app)->am_oid = ((Form_pg_type) GETSTRUCT(tup))->oid; - memmove((char *) &(*app++)->am_typ, - (char *) GETSTRUCT(tup), - sizeof((*app)->am_typ)); - } - table_endscan(scan); - table_close(rel, NoLock); + populate_typ_array(); return gettype(type); } elog(ERROR, "unrecognized type \"%s\"", type); diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 73ddf408654a..861b8817b931 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -468,14 +468,11 @@ filter_lines_with_token(char **lines, const char *token) static char ** readfile(const char *path) { + char **result; FILE *infile; - int maxlength = 1, - linelen = 0; - int nlines = 0; + int maxlines; int n; - char **result; - char *buffer; - int c; + char *ln; if ((infile = fopen(path, "r")) == NULL) { @@ -483,39 +480,24 @@ readfile(const char *path) exit(1); } - /* pass over the file twice - the first time to size the result */ + maxlines = 1024; + result = (char **) pg_malloc(maxlines * sizeof(char *)); - while ((c = fgetc(infile)) != EOF) + n = 0; + while ((ln = pg_get_line(infile)) != NULL) { - linelen++; - if (c == '\n') + /* make sure there will be room for a trailing NULL pointer */ + if (n >= maxlines - 1) { - nlines++; - if (linelen > maxlength) - maxlength = linelen; - linelen = 0; + maxlines *= 2; + result = (char **) pg_realloc(result, maxlines * sizeof(char *)); } - } - - /* handle last line without a terminating newline (yuck) */ - if (linelen) - nlines++; - if (linelen > maxlength) - maxlength = linelen; - /* set up the result and the line buffer */ - result = (char **) pg_malloc((nlines + 1) * sizeof(char *)); - buffer = (char *) pg_malloc(maxlength + 1); - - /* now reprocess the file and store the lines */ - rewind(infile); - n = 0; - while (fgets(buffer, maxlength + 1, infile) != NULL && n < nlines) - result[n++] = pg_strdup(buffer); + result[n++] = ln; + } + result[n] = NULL; fclose(infile); - free(buffer); - result[n] = NULL; return result; } From 76af9744db168cfe96f45769b8ec68eb222d8fcf Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sun, 6 Sep 2020 09:32:16 +0200 Subject: [PATCH 085/589] Remove unused parameter unused since 84d723b6cefcf25b8c800f8aa6cf3c9538a546b4 Discussion: https://www.postgresql.org/message-id/flat/511bb100-f829-ba21-2f10-9f952ec06ead%402ndquadrant.com --- contrib/tablefunc/tablefunc.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/contrib/tablefunc/tablefunc.c b/contrib/tablefunc/tablefunc.c index 3802ae905e8f..02f02eab5742 100644 --- a/contrib/tablefunc/tablefunc.c +++ b/contrib/tablefunc/tablefunc.c @@ -49,7 +49,6 @@ static HTAB *load_categories_hash(char *cats_sql, MemoryContext per_query_ctx); static Tuplestorestate *get_crosstab_tuplestore(char *sql, HTAB *crosstab_hash, TupleDesc tupdesc, - MemoryContext per_query_ctx, bool randomAccess); static void validateConnectbyTupleDesc(TupleDesc tupdesc, bool show_branch, bool show_serial); static bool compatCrosstabTupleDescs(TupleDesc tupdesc1, TupleDesc tupdesc2); @@ -680,7 +679,6 @@ crosstab_hash(PG_FUNCTION_ARGS) rsinfo->setResult = get_crosstab_tuplestore(sql, crosstab_hash, tupdesc, - per_query_ctx, rsinfo->allowedModes & SFRM_Materialize_Random); /* @@ -793,7 +791,6 @@ static Tuplestorestate * get_crosstab_tuplestore(char *sql, HTAB *crosstab_hash, TupleDesc tupdesc, - MemoryContext per_query_ctx, bool randomAccess) { Tuplestorestate *tupstore; From cd153b54eb6542d0e0b4addf1f10a1b34a328e17 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sun, 6 Sep 2020 16:46:13 +0200 Subject: [PATCH 086/589] doc: Don't hide the "Up" link when it is the same as "Home" The original stylesheets seemed to think this was a good idea, but our users find it confusing and unhelpful, so undo that logic. Reported-by: Fabien COELHO Discussion: https://www.postgresql.org/message-id/flat/alpine.DEB.2.22.394.2006210914370.859381%40pseudo --- doc/src/sgml/stylesheet.xsl | 163 +++++++++++++++++++++++++++++++++++- 1 file changed, 159 insertions(+), 4 deletions(-) diff --git a/doc/src/sgml/stylesheet.xsl b/doc/src/sgml/stylesheet.xsl index aeaa1e7c104b..69d7dccb00de 100644 --- a/doc/src/sgml/stylesheet.xsl +++ b/doc/src/sgml/stylesheet.xsl @@ -57,7 +57,6 @@ Customization of header @@ -95,8 +94,7 @@ Customization of header
- + @@ -117,7 +115,6 @@ Customization of header @@ -176,4 +173,162 @@ Customization of header + + + + + + + + + + + + + + + +
+ +
+ + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + +   + + + + + + + + + + + + + + + + + +   + + +   + + + + + + + + + + + + + + + +
+ + + +   + + + + + + + + + + + + + + + + + +  |  + + +   + + + + + + + + + + -toc + + + + + + + + +   + + + +
+ +
+ + + From 666e9a90f09a1ab89959774273597764e6e0f9ad Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 6 Sep 2020 11:50:40 -0400 Subject: [PATCH 087/589] Remove useless lstat() call in pg_rewind. This is duplicative of an lstat that was just done by the calling function (traverse_datadir), besides which we weren't really doing anything with the results. There's not much point in checking to see if someone removed the file since the previous lstat, since the FILE_ACTION_REMOVE code would have to deal with missing-file cases anyway. Moreover, the "exists = false" assignment was a dead store; nothing was done with that value later. A syscall saved is a syscall earned, so back-patch to 9.5 where this code was introduced. Discussion: https://postgr.es/m/1221796.1599329320@sss.pgh.pa.us --- src/bin/pg_rewind/filemap.c | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index 1879257b66a3..1abc257177ef 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -349,8 +349,6 @@ process_target_file(const char *path, file_type_t type, size_t oldsize, const char *link_target) { bool exists; - char localpath[MAXPGPATH]; - struct stat statbuf; file_entry_t key; file_entry_t *key_ptr; filemap_t *map = filemap; @@ -362,16 +360,6 @@ process_target_file(const char *path, file_type_t type, size_t oldsize, * the source data folder when processing the source files. */ - snprintf(localpath, sizeof(localpath), "%s/%s", datadir_target, path); - if (lstat(localpath, &statbuf) < 0) - { - if (errno != ENOENT) - pg_fatal("could not stat file \"%s\": %m", - localpath); - - exists = false; - } - if (map->array == NULL) { /* on first call, initialize lookup array */ From 19ad7e1d7b8b54ea3aa405e60057ab4baefcedbf Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 6 Sep 2020 12:55:13 -0400 Subject: [PATCH 088/589] Fix misleading error message about inconsistent moving-aggregate types. We reported the wrong types when complaining that an aggregate's moving-aggregate implementation is inconsistent with its regular implementation. This was wrong since the feature was introduced, so back-patch to all supported branches. Jeff Janes Discussion: https://postgr.es/m/CAMkU=1x808LH=LPhZp9mNSP0Xd1xDqEd+XeGcvEe48dfE6xV=A@mail.gmail.com --- src/backend/catalog/pg_aggregate.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/catalog/pg_aggregate.c b/src/backend/catalog/pg_aggregate.c index a0554f0d7973..0cf1da6ebba5 100644 --- a/src/backend/catalog/pg_aggregate.c +++ b/src/backend/catalog/pg_aggregate.c @@ -565,8 +565,8 @@ AggregateCreate(const char *aggName, ereport(ERROR, (errcode(ERRCODE_INVALID_FUNCTION_DEFINITION), errmsg("moving-aggregate implementation returns type %s, but plain implementation returns type %s", - format_type_be(aggmTransType), - format_type_be(aggTransType)))); + format_type_be(rettype), + format_type_be(finaltype)))); } /* handle sortop, if supplied */ From 2a093355aa629468eab82242ced83a1c7fff76a8 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Sun, 6 Sep 2020 19:26:55 +0200 Subject: [PATCH 089/589] Fix typo in comment Author: Hou, Zhijie --- src/backend/storage/ipc/procsignal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c index 4fa385b0ece4..ffe67acea1ce 100644 --- a/src/backend/storage/ipc/procsignal.c +++ b/src/backend/storage/ipc/procsignal.c @@ -430,7 +430,7 @@ WaitForProcSignalBarrier(uint64 generation) * cannot safely access the barrier generation inside the signal handler as * 64bit atomics might use spinlock based emulation, even for reads. As this * routine only gets called when PROCSIG_BARRIER is sent that won't cause a - * lot fo unnecessary work. + * lot of unnecessary work. */ static void HandleProcSignalBarrierInterrupt(void) From 68b603e1a934dcd82e259b7776565ec1776e7a29 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Sun, 6 Sep 2020 19:28:32 +0200 Subject: [PATCH 090/589] Change path in example of file_fdw for logs It's better to use a relative path into the data directory, than to a hardcoded home directory of user 'josh'. Discussion: https://postgr.es/m/CABUevEyuf67Yu_r9gpDMs5MKifK7+-+pe=ZjKzya4JEn9kUk1w@mail.gmail.com --- doc/src/sgml/file-fdw.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/file-fdw.sgml b/doc/src/sgml/file-fdw.sgml index 599952ed3d7d..29d79832a33c 100644 --- a/doc/src/sgml/file-fdw.sgml +++ b/doc/src/sgml/file-fdw.sgml @@ -267,7 +267,7 @@ CREATE FOREIGN TABLE pglog ( application_name text, backend_type text ) SERVER pglog -OPTIONS ( filename '/home/josh/data/log/pglog.csv', format 'csv' ); +OPTIONS ( filename log/pglog.csv', format 'csv' ); From 8e3c58e6e459b285d37edb6129e412ed25cd90c1 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 6 Sep 2020 13:57:10 -0400 Subject: [PATCH 091/589] Refactor pg_get_line() to expose an alternative StringInfo-based API. Letting the caller provide a StringInfo to read into is helpful when the caller needs to merge lines or otherwise modify the data after it's been read. Notably, now the code added by commit 8f8154a50 can use pg_get_line_append() instead of having its own copy of that logic. A follow-on commit will also make use of this. Also, since StringInfo buffers are a minimum of 1KB long, blindly using pg_get_line() in a loop can eat a lot more memory than one would expect. I discovered for instance that commit e0f05cd5b caused initdb to consume circa 10MB to read postgres.bki, even though that's under 1MB worth of data. A less memory-hungry alternative is to re-use the same StringInfo for all lines and pg_strdup the results. Discussion: https://postgr.es/m/1315832.1599345736@sss.pgh.pa.us --- src/backend/libpq/hba.c | 40 ++++++++------------- src/bin/initdb/initdb.c | 12 +++++-- src/common/pg_get_line.c | 70 ++++++++++++++++++++++++++----------- src/include/common/string.h | 3 ++ 4 files changed, 75 insertions(+), 50 deletions(-) diff --git a/src/backend/libpq/hba.c b/src/backend/libpq/hba.c index 5991a21cf2db..9f106653f3f8 100644 --- a/src/backend/libpq/hba.c +++ b/src/backend/libpq/hba.c @@ -502,33 +502,8 @@ tokenize_file(const char *filename, FILE *file, List **tok_lines, int elevel) /* Collect the next input line, handling backslash continuations */ resetStringInfo(&buf); - while (!feof(file) && !ferror(file)) + while (pg_get_line_append(file, &buf)) { - /* Make sure there's a reasonable amount of room in the buffer */ - enlargeStringInfo(&buf, 128); - - /* Read some data, appending it to what we already have */ - if (fgets(buf.data + buf.len, buf.maxlen - buf.len, file) == NULL) - { - int save_errno = errno; - - if (!ferror(file)) - break; /* normal EOF */ - /* I/O error! */ - ereport(elevel, - (errcode_for_file_access(), - errmsg("could not read file \"%s\": %m", filename))); - err_msg = psprintf("could not read file \"%s\": %s", - filename, strerror(save_errno)); - resetStringInfo(&buf); - break; - } - buf.len += strlen(buf.data + buf.len); - - /* If we haven't got a whole line, loop to read more */ - if (!(buf.len > 0 && buf.data[buf.len - 1] == '\n')) - continue; - /* Strip trailing newline, including \r in case we're on Windows */ buf.len = pg_strip_crlf(buf.data); @@ -551,6 +526,19 @@ tokenize_file(const char *filename, FILE *file, List **tok_lines, int elevel) break; } + if (ferror(file)) + { + /* I/O error! */ + int save_errno = errno; + + ereport(elevel, + (errcode_for_file_access(), + errmsg("could not read file \"%s\": %m", filename))); + err_msg = psprintf("could not read file \"%s\": %s", + filename, strerror(save_errno)); + break; + } + /* Parse fields */ lineptr = buf.data; while (*lineptr && err_msg == NULL) diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 861b8817b931..37e0d7ceab93 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -470,9 +470,9 @@ readfile(const char *path) { char **result; FILE *infile; + StringInfoData line; int maxlines; int n; - char *ln; if ((infile = fopen(path, "r")) == NULL) { @@ -480,11 +480,13 @@ readfile(const char *path) exit(1); } + initStringInfo(&line); + maxlines = 1024; result = (char **) pg_malloc(maxlines * sizeof(char *)); n = 0; - while ((ln = pg_get_line(infile)) != NULL) + while (pg_get_line_append(infile, &line)) { /* make sure there will be room for a trailing NULL pointer */ if (n >= maxlines - 1) @@ -493,10 +495,14 @@ readfile(const char *path) result = (char **) pg_realloc(result, maxlines * sizeof(char *)); } - result[n++] = ln; + result[n++] = pg_strdup(line.data); + + resetStringInfo(&line); } result[n] = NULL; + pfree(line.data); + fclose(infile); return result; diff --git a/src/common/pg_get_line.c b/src/common/pg_get_line.c index 38433675d433..2fb8e198933d 100644 --- a/src/common/pg_get_line.c +++ b/src/common/pg_get_line.c @@ -41,6 +41,11 @@ * Note that while I/O errors are reflected back to the caller to be * dealt with, an OOM condition for the palloc'd buffer will not be; * there'll be an ereport(ERROR) or exit(1) inside stringinfo.c. + * + * Also note that the palloc'd buffer is usually a lot longer than + * strictly necessary, so it may be inadvisable to use this function + * to collect lots of long-lived data. A less memory-hungry option + * is to use pg_get_line_append() in a loop, then pstrdup() each line. */ char * pg_get_line(FILE *stream) @@ -49,21 +54,7 @@ pg_get_line(FILE *stream) initStringInfo(&buf); - /* Read some data, appending it to whatever we already have */ - while (fgets(buf.data + buf.len, buf.maxlen - buf.len, stream) != NULL) - { - buf.len += strlen(buf.data + buf.len); - - /* Done if we have collected a newline */ - if (buf.len > 0 && buf.data[buf.len - 1] == '\n') - return buf.data; - - /* Make some more room in the buffer, and loop to read more data */ - enlargeStringInfo(&buf, 128); - } - - /* Did fgets() fail because of an I/O error? */ - if (ferror(stream)) + if (!pg_get_line_append(stream, &buf)) { /* ensure that free() doesn't mess up errno */ int save_errno = errno; @@ -73,13 +64,50 @@ pg_get_line(FILE *stream) return NULL; } - /* If we read no data before reaching EOF, we should return NULL */ - if (buf.len == 0) + return buf.data; +} + +/* + * pg_get_line_append() + * + * This has similar behavior to pg_get_line(), and thence to fgets(), + * except that the collected data is appended to whatever is in *buf. + * + * Returns true if a line was successfully collected (including the + * case of a non-newline-terminated line at EOF). Returns false if + * there was an I/O error or no data was available before EOF. + * (Check ferror(stream) to distinguish these cases.) + * + * In the false-result case, the contents of *buf are logically unmodified, + * though it's possible that the buffer has been resized. + */ +bool +pg_get_line_append(FILE *stream, StringInfo buf) +{ + int orig_len = buf->len; + + /* Read some data, appending it to whatever we already have */ + while (fgets(buf->data + buf->len, buf->maxlen - buf->len, stream) != NULL) + { + buf->len += strlen(buf->data + buf->len); + + /* Done if we have collected a newline */ + if (buf->len > orig_len && buf->data[buf->len - 1] == '\n') + return true; + + /* Make some more room in the buffer, and loop to read more data */ + enlargeStringInfo(buf, 128); + } + + /* Check for I/O errors and EOF */ + if (ferror(stream) || buf->len == orig_len) { - pfree(buf.data); - return NULL; + /* Discard any data we collected before detecting error */ + buf->len = orig_len; + buf->data[orig_len] = '\0'; + return false; } - /* No newline at EOF ... so return what we have */ - return buf.data; + /* No newline at EOF, but we did collect some data */ + return true; } diff --git a/src/include/common/string.h b/src/include/common/string.h index 18aa1dc5aa5c..50c241a811b6 100644 --- a/src/include/common/string.h +++ b/src/include/common/string.h @@ -10,6 +10,8 @@ #ifndef COMMON_STRING_H #define COMMON_STRING_H +struct StringInfoData; /* avoid including stringinfo.h here */ + /* functions in src/common/string.c */ extern bool pg_str_endswith(const char *str, const char *end); extern int strtoint(const char *pg_restrict str, char **pg_restrict endptr, @@ -19,6 +21,7 @@ extern int pg_strip_crlf(char *str); /* functions in src/common/pg_get_line.c */ extern char *pg_get_line(FILE *stream); +extern bool pg_get_line_append(FILE *stream, struct StringInfoData *buf); /* functions in src/common/sprompt.c */ extern char *simple_prompt(const char *prompt, bool echo); From 784b1ba1a2b9306697544bedb2ef9425185dd206 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 6 Sep 2020 14:13:02 -0400 Subject: [PATCH 092/589] Remove arbitrary line length limits in pg_regress (plain and ECPG). Refactor replace_string() to use a StringInfo for the modifiable string argument. This allows the string to be of indefinite size initially and/or grow substantially during replacement. The previous logic in convert_sourcefiles_in() had a hard-wired limit of 1024 bytes on any line in input/*.sql or output/*.out files. While we've not had reports of trouble yet, it'd surely have bit us someday. This also fixes replace_string() so it won't get into an infinite loop if the string-to-be-replaced is a substring of the replacement. That's unlikely to happen in current usage, but the function surely shouldn't depend on it. Also fix ecpg_filter() to use a StringInfo and thereby remove its hard limit of 300 bytes on the length of an ecpg source line. Asim Rama Praveen and Georgios Kokolatos, reviewed by Alvaro Herrera and myself Discussion: https://postgr.es/m/y9Dlk2QhiZ39DhaB1QE9mgZ95HcOQKZCNtGwN7XCRKMdBRBnX_0woaRUtTjloEp4PKA6ERmcUcfq3lPGfKPOJ5xX2TV-5WoRYyySeNHRzdw=@protonmail.com --- src/interfaces/ecpg/test/pg_regress_ecpg.c | 76 +++++++++++----------- src/test/regress/pg_regress.c | 52 ++++++++++----- src/test/regress/pg_regress.h | 5 +- 3 files changed, 77 insertions(+), 56 deletions(-) diff --git a/src/interfaces/ecpg/test/pg_regress_ecpg.c b/src/interfaces/ecpg/test/pg_regress_ecpg.c index 46b9e78fe59d..a2d7b70d9a30 100644 --- a/src/interfaces/ecpg/test/pg_regress_ecpg.c +++ b/src/interfaces/ecpg/test/pg_regress_ecpg.c @@ -19,8 +19,9 @@ #include "postgres_fe.h" #include "pg_regress.h" +#include "common/string.h" +#include "lib/stringinfo.h" -#define LINEBUFSIZE 300 static void ecpg_filter(const char *sourcefile, const char *outfile) @@ -31,7 +32,7 @@ ecpg_filter(const char *sourcefile, const char *outfile) */ FILE *s, *t; - char linebuf[LINEBUFSIZE]; + StringInfoData linebuf; s = fopen(sourcefile, "r"); if (!s) @@ -46,13 +47,14 @@ ecpg_filter(const char *sourcefile, const char *outfile) exit(2); } - while (fgets(linebuf, LINEBUFSIZE, s)) + initStringInfo(&linebuf); + + while (pg_get_line_append(s, &linebuf)) { /* check for "#line " in the beginning */ - if (strstr(linebuf, "#line ") == linebuf) + if (strstr(linebuf.data, "#line ") == linebuf.data) { - char *p = strchr(linebuf, '"'); - char *n; + char *p = strchr(linebuf.data, '"'); int plen = 1; while (*p && (*(p + plen) == '.' || strchr(p + plen, '/') != NULL)) @@ -62,13 +64,15 @@ ecpg_filter(const char *sourcefile, const char *outfile) /* plen is one more than the number of . and / characters */ if (plen > 1) { - n = (char *) malloc(plen); - strlcpy(n, p + 1, plen); - replace_string(linebuf, n, ""); + memmove(p + 1, p + plen, strlen(p + plen) + 1); + /* we don't bother to fix up linebuf.len */ } } - fputs(linebuf, t); + fputs(linebuf.data, t); + resetStringInfo(&linebuf); } + + pfree(linebuf.data); fclose(s); fclose(t); } @@ -87,40 +91,42 @@ ecpg_start_test(const char *testname, PID_TYPE pid; char inprg[MAXPGPATH]; char insource[MAXPGPATH]; - char *outfile_stdout, + StringInfoData testname_dash; + char outfile_stdout[MAXPGPATH], expectfile_stdout[MAXPGPATH]; - char *outfile_stderr, + char outfile_stderr[MAXPGPATH], expectfile_stderr[MAXPGPATH]; - char *outfile_source, + char outfile_source[MAXPGPATH], expectfile_source[MAXPGPATH]; char cmd[MAXPGPATH * 3]; - char *testname_dash; char *appnameenv; snprintf(inprg, sizeof(inprg), "%s/%s", inputdir, testname); + snprintf(insource, sizeof(insource), "%s.c", testname); + + initStringInfo(&testname_dash); + appendStringInfoString(&testname_dash, testname); + replace_string(&testname_dash, "/", "-"); - testname_dash = strdup(testname); - replace_string(testname_dash, "/", "-"); snprintf(expectfile_stdout, sizeof(expectfile_stdout), "%s/expected/%s.stdout", - outputdir, testname_dash); + outputdir, testname_dash.data); snprintf(expectfile_stderr, sizeof(expectfile_stderr), "%s/expected/%s.stderr", - outputdir, testname_dash); + outputdir, testname_dash.data); snprintf(expectfile_source, sizeof(expectfile_source), "%s/expected/%s.c", - outputdir, testname_dash); - - /* - * We can use replace_string() here because the replacement string does - * not occupy more space than the replaced one. - */ - outfile_stdout = strdup(expectfile_stdout); - replace_string(outfile_stdout, "/expected/", "/results/"); - outfile_stderr = strdup(expectfile_stderr); - replace_string(outfile_stderr, "/expected/", "/results/"); - outfile_source = strdup(expectfile_source); - replace_string(outfile_source, "/expected/", "/results/"); + outputdir, testname_dash.data); + + snprintf(outfile_stdout, sizeof(outfile_stdout), + "%s/results/%s.stdout", + outputdir, testname_dash.data); + snprintf(outfile_stderr, sizeof(outfile_stderr), + "%s/results/%s.stderr", + outputdir, testname_dash.data); + snprintf(outfile_source, sizeof(outfile_source), + "%s/results/%s.c", + outputdir, testname_dash.data); add_stringlist_item(resultfiles, outfile_stdout); add_stringlist_item(expectfiles, expectfile_stdout); @@ -134,18 +140,15 @@ ecpg_start_test(const char *testname, add_stringlist_item(expectfiles, expectfile_source); add_stringlist_item(tags, "source"); - snprintf(insource, sizeof(insource), "%s.c", testname); ecpg_filter(insource, outfile_source); - snprintf(inprg, sizeof(inprg), "%s/%s", inputdir, testname); - snprintf(cmd, sizeof(cmd), "\"%s\" >\"%s\" 2>\"%s\"", inprg, outfile_stdout, outfile_stderr); - appnameenv = psprintf("PGAPPNAME=ecpg/%s", testname_dash); + appnameenv = psprintf("PGAPPNAME=ecpg/%s", testname_dash.data); putenv(appnameenv); pid = spawn_process(cmd); @@ -160,10 +163,7 @@ ecpg_start_test(const char *testname, unsetenv("PGAPPNAME"); free(appnameenv); - free(testname_dash); - free(outfile_stdout); - free(outfile_stderr); - free(outfile_source); + free(testname_dash.data); return pid; } diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c index d82e0189dcfb..74fd026856ea 100644 --- a/src/test/regress/pg_regress.c +++ b/src/test/regress/pg_regress.c @@ -31,8 +31,10 @@ #include "common/logging.h" #include "common/restricted_token.h" +#include "common/string.h" #include "common/username.h" #include "getopt_long.h" +#include "lib/stringinfo.h" #include "libpq/pqcomm.h" /* needed for UNIXSOCK_PATH() */ #include "pg_config_paths.h" #include "pg_regress.h" @@ -435,22 +437,32 @@ string_matches_pattern(const char *str, const char *pattern) } /* - * Replace all occurrences of a string in a string with a different string. - * NOTE: Assumes there is enough room in the target buffer! + * Replace all occurrences of "replace" in "string" with "replacement". + * The StringInfo will be suitably enlarged if necessary. + * + * Note: this is optimized on the assumption that most calls will find + * no more than one occurrence of "replace", and quite likely none. */ void -replace_string(char *string, const char *replace, const char *replacement) +replace_string(StringInfo string, const char *replace, const char *replacement) { + int pos = 0; char *ptr; - while ((ptr = strstr(string, replace)) != NULL) + while ((ptr = strstr(string->data + pos, replace)) != NULL) { - char *dup = pg_strdup(string); + /* Must copy the remainder of the string out of the StringInfo */ + char *suffix = pg_strdup(ptr + strlen(replace)); - strlcpy(string, dup, ptr - string + 1); - strcat(string, replacement); - strcat(string, dup + (ptr - string) + strlen(replace)); - free(dup); + /* Truncate StringInfo at start of found string ... */ + string->len = ptr - string->data; + /* ... and append the replacement (this restores the trailing '\0') */ + appendStringInfoString(string, replacement); + /* Next search should start after the replacement */ + pos = string->len; + /* Put back the remainder of the string */ + appendStringInfoString(string, suffix); + free(suffix); } } @@ -521,7 +533,7 @@ convert_sourcefiles_in(const char *source_subdir, const char *dest_dir, const ch char prefix[MAXPGPATH]; FILE *infile, *outfile; - char line[1024]; + StringInfoData line; /* reject filenames not finishing in ".source" */ if (strlen(*name) < 8) @@ -551,15 +563,21 @@ convert_sourcefiles_in(const char *source_subdir, const char *dest_dir, const ch progname, destfile, strerror(errno)); exit(2); } - while (fgets(line, sizeof(line), infile)) + + initStringInfo(&line); + + while (pg_get_line_append(infile, &line)) { - replace_string(line, "@abs_srcdir@", inputdir); - replace_string(line, "@abs_builddir@", outputdir); - replace_string(line, "@testtablespace@", testtablespace); - replace_string(line, "@libdir@", dlpath); - replace_string(line, "@DLSUFFIX@", DLSUFFIX); - fputs(line, outfile); + replace_string(&line, "@abs_srcdir@", inputdir); + replace_string(&line, "@abs_builddir@", outputdir); + replace_string(&line, "@testtablespace@", testtablespace); + replace_string(&line, "@libdir@", dlpath); + replace_string(&line, "@DLSUFFIX@", DLSUFFIX); + fputs(line.data, outfile); + resetStringInfo(&line); } + + pfree(line.data); fclose(infile); fclose(outfile); } diff --git a/src/test/regress/pg_regress.h b/src/test/regress/pg_regress.h index ee6e0d42f401..726f9c904898 100644 --- a/src/test/regress/pg_regress.h +++ b/src/test/regress/pg_regress.h @@ -18,6 +18,8 @@ #define INVALID_PID INVALID_HANDLE_VALUE #endif +struct StringInfoData; /* avoid including stringinfo.h here */ + /* simple list of strings */ typedef struct _stringlist { @@ -49,5 +51,6 @@ int regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc); void add_stringlist_item(_stringlist **listhead, const char *str); PID_TYPE spawn_process(const char *cmdline); -void replace_string(char *string, const char *replace, const char *replacement); +void replace_string(struct StringInfoData *string, + const char *replace, const char *replacement); bool file_exists(const char *file); From 695de5d1eda6382b20fadb0a2b2d286b57a6a61c Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 6 Sep 2020 21:28:16 -0400 Subject: [PATCH 093/589] Split Makefile symbol CFLAGS_VECTOR into two symbols. Replace CFLAGS_VECTOR with CFLAGS_UNROLL_LOOPS and CFLAGS_VECTORIZE, allowing us to distinguish whether we want to apply -funroll-loops, -ftree-vectorize, or both to a particular source file. Up to now the only consumer of the symbol has been checksum.c which wants both, so that there was no need to distinguish; but that's about to change. Amit Khandekar, reviewed and edited a little by me Discussion: https://postgr.es/m/CAJ3gD9evtA_vBo+WMYMyT-u=keHX7-r8p2w7OSRfXf42LTwCZQ@mail.gmail.com --- configure | 32 ++++++++++++++++++------------- configure.ac | 17 ++++++++++------ src/Makefile.global.in | 3 ++- src/backend/storage/page/Makefile | 4 ++-- 4 files changed, 34 insertions(+), 22 deletions(-) diff --git a/configure b/configure index cb8fbe105114..19a3cd09a0a3 100755 --- a/configure +++ b/configure @@ -734,7 +734,8 @@ CPP CFLAGS_SL BITCODE_CXXFLAGS BITCODE_CFLAGS -CFLAGS_VECTOR +CFLAGS_VECTORIZE +CFLAGS_UNROLL_LOOPS PERMIT_DECLARATION_AFTER_STATEMENT LLVM_BINPATH LLVM_CXXFLAGS @@ -5266,9 +5267,12 @@ BITCODE_CFLAGS="" user_BITCODE_CXXFLAGS=$BITCODE_CXXFLAGS BITCODE_CXXFLAGS="" -# set CFLAGS_VECTOR from the environment, if available -if test "$ac_env_CFLAGS_VECTOR_set" = set; then - CFLAGS_VECTOR=$ac_env_CFLAGS_VECTOR_value +# set CFLAGS_UNROLL_LOOPS and CFLAGS_VECTORIZE from the environment, if present +if test "$ac_env_CFLAGS_UNROLL_LOOPS_set" = set; then + CFLAGS_UNROLL_LOOPS=$ac_env_CFLAGS_UNROLL_LOOPS_value +fi +if test "$ac_env_CFLAGS_VECTORIZE_set" = set; then + CFLAGS_VECTORIZE=$ac_env_CFLAGS_VECTORIZE_value fi # Some versions of GCC support some additional useful warning flags. @@ -6102,16 +6106,16 @@ if test x"$pgac_cv_prog_CXX_cxxflags__fexcess_precision_standard" = x"yes"; then fi - # Optimization flags for specific files that benefit from vectorization - { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ${CC} supports -funroll-loops, for CFLAGS_VECTOR" >&5 -$as_echo_n "checking whether ${CC} supports -funroll-loops, for CFLAGS_VECTOR... " >&6; } + # Optimization flags for specific files that benefit from loop unrolling + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ${CC} supports -funroll-loops, for CFLAGS_UNROLL_LOOPS" >&5 +$as_echo_n "checking whether ${CC} supports -funroll-loops, for CFLAGS_UNROLL_LOOPS... " >&6; } if ${pgac_cv_prog_CC_cflags__funroll_loops+:} false; then : $as_echo_n "(cached) " >&6 else pgac_save_CFLAGS=$CFLAGS pgac_save_CC=$CC CC=${CC} -CFLAGS="${CFLAGS_VECTOR} -funroll-loops" +CFLAGS="${CFLAGS_UNROLL_LOOPS} -funroll-loops" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -6138,19 +6142,20 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_prog_CC_cflags__funroll_loops" >&5 $as_echo "$pgac_cv_prog_CC_cflags__funroll_loops" >&6; } if test x"$pgac_cv_prog_CC_cflags__funroll_loops" = x"yes"; then - CFLAGS_VECTOR="${CFLAGS_VECTOR} -funroll-loops" + CFLAGS_UNROLL_LOOPS="${CFLAGS_UNROLL_LOOPS} -funroll-loops" fi - { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ${CC} supports -ftree-vectorize, for CFLAGS_VECTOR" >&5 -$as_echo_n "checking whether ${CC} supports -ftree-vectorize, for CFLAGS_VECTOR... " >&6; } + # Optimization flags for specific files that benefit from vectorization + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ${CC} supports -ftree-vectorize, for CFLAGS_VECTORIZE" >&5 +$as_echo_n "checking whether ${CC} supports -ftree-vectorize, for CFLAGS_VECTORIZE... " >&6; } if ${pgac_cv_prog_CC_cflags__ftree_vectorize+:} false; then : $as_echo_n "(cached) " >&6 else pgac_save_CFLAGS=$CFLAGS pgac_save_CC=$CC CC=${CC} -CFLAGS="${CFLAGS_VECTOR} -ftree-vectorize" +CFLAGS="${CFLAGS_VECTORIZE} -ftree-vectorize" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -6177,7 +6182,7 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_prog_CC_cflags__ftree_vectorize" >&5 $as_echo "$pgac_cv_prog_CC_cflags__ftree_vectorize" >&6; } if test x"$pgac_cv_prog_CC_cflags__ftree_vectorize" = x"yes"; then - CFLAGS_VECTOR="${CFLAGS_VECTOR} -ftree-vectorize" + CFLAGS_VECTORIZE="${CFLAGS_VECTORIZE} -ftree-vectorize" fi @@ -6782,6 +6787,7 @@ fi + # Determine flags used to emit bitcode for JIT inlining. Need to test # for behaviour changing compiler flags, to keep compatibility with # compiler used for normal postgres code. diff --git a/configure.ac b/configure.ac index eb2c731b58fb..6b9d0487a8db 100644 --- a/configure.ac +++ b/configure.ac @@ -466,9 +466,12 @@ BITCODE_CFLAGS="" user_BITCODE_CXXFLAGS=$BITCODE_CXXFLAGS BITCODE_CXXFLAGS="" -# set CFLAGS_VECTOR from the environment, if available -if test "$ac_env_CFLAGS_VECTOR_set" = set; then - CFLAGS_VECTOR=$ac_env_CFLAGS_VECTOR_value +# set CFLAGS_UNROLL_LOOPS and CFLAGS_VECTORIZE from the environment, if present +if test "$ac_env_CFLAGS_UNROLL_LOOPS_set" = set; then + CFLAGS_UNROLL_LOOPS=$ac_env_CFLAGS_UNROLL_LOOPS_value +fi +if test "$ac_env_CFLAGS_VECTORIZE_set" = set; then + CFLAGS_VECTORIZE=$ac_env_CFLAGS_VECTORIZE_value fi # Some versions of GCC support some additional useful warning flags. @@ -512,9 +515,10 @@ if test "$GCC" = yes -a "$ICC" = no; then # Disable FP optimizations that cause various errors on gcc 4.5+ or maybe 4.6+ PGAC_PROG_CC_CFLAGS_OPT([-fexcess-precision=standard]) PGAC_PROG_CXX_CFLAGS_OPT([-fexcess-precision=standard]) + # Optimization flags for specific files that benefit from loop unrolling + PGAC_PROG_CC_VAR_OPT(CFLAGS_UNROLL_LOOPS, [-funroll-loops]) # Optimization flags for specific files that benefit from vectorization - PGAC_PROG_CC_VAR_OPT(CFLAGS_VECTOR, [-funroll-loops]) - PGAC_PROG_CC_VAR_OPT(CFLAGS_VECTOR, [-ftree-vectorize]) + PGAC_PROG_CC_VAR_OPT(CFLAGS_VECTORIZE, [-ftree-vectorize]) # We want to suppress clang's unhelpful unused-command-line-argument warnings # but gcc won't complain about unrecognized -Wno-foo switches, so we have to # test for the positive form and if that works, add the negative form @@ -555,7 +559,8 @@ elif test "$PORTNAME" = "hpux"; then PGAC_PROG_CXX_CFLAGS_OPT([+Olibmerrno]) fi -AC_SUBST(CFLAGS_VECTOR) +AC_SUBST(CFLAGS_UNROLL_LOOPS) +AC_SUBST(CFLAGS_VECTORIZE) # Determine flags used to emit bitcode for JIT inlining. Need to test # for behaviour changing compiler flags, to keep compatibility with diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 9a6265b3a0bc..7ca1e9aac594 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -259,7 +259,8 @@ SUN_STUDIO_CC = @SUN_STUDIO_CC@ CXX = @CXX@ CFLAGS = @CFLAGS@ CFLAGS_SL = @CFLAGS_SL@ -CFLAGS_VECTOR = @CFLAGS_VECTOR@ +CFLAGS_UNROLL_LOOPS = @CFLAGS_UNROLL_LOOPS@ +CFLAGS_VECTORIZE = @CFLAGS_VECTORIZE@ CFLAGS_SSE42 = @CFLAGS_SSE42@ CFLAGS_ARMV8_CRC32C = @CFLAGS_ARMV8_CRC32C@ PERMIT_DECLARATION_AFTER_STATEMENT = @PERMIT_DECLARATION_AFTER_STATEMENT@ diff --git a/src/backend/storage/page/Makefile b/src/backend/storage/page/Makefile index 10021e2bb31e..da539b113a69 100644 --- a/src/backend/storage/page/Makefile +++ b/src/backend/storage/page/Makefile @@ -19,5 +19,5 @@ OBJS = \ include $(top_srcdir)/src/backend/common.mk -# important optimizations flags for checksum.c -checksum.o: CFLAGS += ${CFLAGS_VECTOR} +# Provide special optimization flags for checksum.c +checksum.o: CFLAGS += ${CFLAGS_UNROLL_LOOPS} ${CFLAGS_VECTORIZE} From 88709176236caf3cb9655acda6bad2df0323ac8f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 6 Sep 2020 21:40:39 -0400 Subject: [PATCH 094/589] Apply auto-vectorization to the inner loop of numeric multiplication. Compile numeric.c with -ftree-vectorize where available, and adjust the innermost loop of mul_var() so that it is amenable to being auto-vectorized. (Mainly, that involves making it process the arrays left-to-right not right-to-left.) Applying -ftree-vectorize actually makes numeric.o smaller, at least with my compiler (gcc 8.3.1 on x86_64), and it's a little faster too. Independently of that, fixing the inner loop to be vectorizable also makes things a bit faster. But doing both is a huge win for multiplications with lots of digits. For me, the numeric regression test is the same speed to within measurement noise, but numeric_big is a full 45% faster. We also looked into applying -funroll-loops, but that makes numeric.o bloat quite a bit, and the additional speed improvement is very marginal. Amit Khandekar, reviewed and edited a little by me Discussion: https://postgr.es/m/CAJ3gD9evtA_vBo+WMYMyT-u=keHX7-r8p2w7OSRfXf42LTwCZQ@mail.gmail.com --- src/backend/utils/adt/Makefile | 3 +++ src/backend/utils/adt/numeric.c | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/backend/utils/adt/Makefile b/src/backend/utils/adt/Makefile index 54d5c3794726..b4d55e849b3d 100644 --- a/src/backend/utils/adt/Makefile +++ b/src/backend/utils/adt/Makefile @@ -125,6 +125,9 @@ clean distclean maintainer-clean: like.o: like.c like_match.c +# Some code in numeric.c benefits from auto-vectorization +numeric.o: CFLAGS += ${CFLAGS_VECTORIZE} + varlena.o: varlena.c levenshtein.c include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c index ed825a1fddf9..d2a42b811dac 100644 --- a/src/backend/utils/adt/numeric.c +++ b/src/backend/utils/adt/numeric.c @@ -8191,6 +8191,7 @@ mul_var(const NumericVar *var1, const NumericVar *var2, NumericVar *result, int res_weight; int maxdigits; int *dig; + int *dig_i1_2; int carry; int maxdig; int newdig; @@ -8327,10 +8328,18 @@ mul_var(const NumericVar *var1, const NumericVar *var2, NumericVar *result, * * As above, digits of var2 can be ignored if they don't contribute, * so we only include digits for which i1+i2+2 <= res_ndigits - 1. + * + * This inner loop is the performance bottleneck for multiplication, + * so we want to keep it simple enough so that it can be + * auto-vectorized. Accordingly, process the digits left-to-right + * even though schoolbook multiplication would suggest right-to-left. + * Since we aren't propagating carries in this loop, the order does + * not matter. */ - for (i2 = Min(var2ndigits - 1, res_ndigits - i1 - 3), i = i1 + i2 + 2; - i2 >= 0; i2--) - dig[i--] += var1digit * var2digits[i2]; + i = Min(var2ndigits - 1, res_ndigits - i1 - 3); + dig_i1_2 = &dig[i1 + 2]; + for (i2 = 0; i2 <= i; i2++) + dig_i1_2[i2] += var1digit * var2digits[i2]; } /* From 58b5ae9d62bdd44879c6f9a570849903ccd6cd66 Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Mon, 7 Sep 2020 08:08:58 +0530 Subject: [PATCH 095/589] Add additional tests to test streaming of in-progress transactions. This covers the functionality tests for streaming in-progress subtransactions, streaming transactions containing rollback to savepoints, and streaming transactions having DDLs. Author: Tomas Vondra, Amit Kapila and Dilip Kumar Reviewed-by: Dilip Kumar Discussion: https://postgr.es/m/688b0b7f-2f6c-d827-c27b-216a8e3ea700@2ndquadrant.com --- src/test/subscription/t/016_stream_subxact.pl | 81 ++++++++++++ src/test/subscription/t/017_stream_ddl.pl | 110 ++++++++++++++++ .../t/018_stream_subxact_abort.pl | 117 ++++++++++++++++++ .../t/019_stream_subxact_ddl_abort.pl | 76 ++++++++++++ 4 files changed, 384 insertions(+) create mode 100644 src/test/subscription/t/016_stream_subxact.pl create mode 100644 src/test/subscription/t/017_stream_ddl.pl create mode 100644 src/test/subscription/t/018_stream_subxact_abort.pl create mode 100644 src/test/subscription/t/019_stream_subxact_ddl_abort.pl diff --git a/src/test/subscription/t/016_stream_subxact.pl b/src/test/subscription/t/016_stream_subxact.pl new file mode 100644 index 000000000000..b6a2d10e91ef --- /dev/null +++ b/src/test/subscription/t/016_stream_subxact.pl @@ -0,0 +1,81 @@ +# Test streaming of large transaction containing large subtransactions +use strict; +use warnings; +use PostgresNode; +use TestLib; +use Test::More tests => 2; + +# Create publisher node +my $node_publisher = get_new_node('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf('postgresql.conf', 'logical_decoding_work_mem = 64kB'); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = get_new_node('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->start; + +# Create some preexisting content on publisher +$node_publisher->safe_psql('postgres', + "CREATE TABLE test_tab (a int primary key, b varchar)"); +$node_publisher->safe_psql('postgres', + "INSERT INTO test_tab VALUES (1, 'foo'), (2, 'bar')"); + +# Setup structure on subscriber +$node_subscriber->safe_psql('postgres', "CREATE TABLE test_tab (a int primary key, b text, c timestamptz DEFAULT now(), d bigint DEFAULT 999)"); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +$node_publisher->safe_psql('postgres', "CREATE PUBLICATION tap_pub FOR TABLE test_tab"); + +my $appname = 'tap_sub'; +$node_subscriber->safe_psql('postgres', +"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)" +); + +$node_publisher->wait_for_catchup($appname); + +# Also wait for initial table sync to finish +my $synced_query = +"SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('r', 's');"; +$node_subscriber->poll_query_until('postgres', $synced_query) + or die "Timed out while waiting for subscriber to synchronize data"; + +my $result = + $node_subscriber->safe_psql('postgres', "SELECT count(*), count(c), count(d = 999) FROM test_tab"); +is($result, qq(2|2|2), 'check initial data was copied to subscriber'); + +# Insert, update and delete enough rows to exceed 64kB limit. +$node_publisher->safe_psql('postgres', q{ +BEGIN; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series( 3, 500) s(i); +UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0; +DELETE FROM test_tab WHERE mod(a,3) = 0; +SAVEPOINT s1; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(501, 1000) s(i); +UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0; +DELETE FROM test_tab WHERE mod(a,3) = 0; +SAVEPOINT s2; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1001, 1500) s(i); +UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0; +DELETE FROM test_tab WHERE mod(a,3) = 0; +SAVEPOINT s3; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1501, 2000) s(i); +UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0; +DELETE FROM test_tab WHERE mod(a,3) = 0; +SAVEPOINT s4; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2001, 2500) s(i); +UPDATE test_tab SET b = md5(b) WHERE mod(a,2) = 0; +DELETE FROM test_tab WHERE mod(a,3) = 0; +COMMIT; +}); + +$node_publisher->wait_for_catchup($appname); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*), count(c), count(d = 999) FROM test_tab"); +is($result, qq(1667|1667|1667), 'check data was copied to subscriber in streaming mode and extra columns contain local defaults'); + +$node_subscriber->stop; +$node_publisher->stop; diff --git a/src/test/subscription/t/017_stream_ddl.pl b/src/test/subscription/t/017_stream_ddl.pl new file mode 100644 index 000000000000..be7d7d74e3fb --- /dev/null +++ b/src/test/subscription/t/017_stream_ddl.pl @@ -0,0 +1,110 @@ +# Test streaming of large transaction with DDL and subtransactions +use strict; +use warnings; +use PostgresNode; +use TestLib; +use Test::More tests => 3; + +# Create publisher node +my $node_publisher = get_new_node('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf('postgresql.conf', 'logical_decoding_work_mem = 64kB'); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = get_new_node('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->start; + +# Create some preexisting content on publisher +$node_publisher->safe_psql('postgres', + "CREATE TABLE test_tab (a int primary key, b varchar)"); +$node_publisher->safe_psql('postgres', + "INSERT INTO test_tab VALUES (1, 'foo'), (2, 'bar')"); + +# Setup structure on subscriber +$node_subscriber->safe_psql('postgres', "CREATE TABLE test_tab (a int primary key, b text, c INT, d INT, e INT, f INT)"); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +$node_publisher->safe_psql('postgres', "CREATE PUBLICATION tap_pub FOR TABLE test_tab"); + +my $appname = 'tap_sub'; +$node_subscriber->safe_psql('postgres', +"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)" +); + +$node_publisher->wait_for_catchup($appname); + +# Also wait for initial table sync to finish +my $synced_query = +"SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('r', 's');"; +$node_subscriber->poll_query_until('postgres', $synced_query) + or die "Timed out while waiting for subscriber to synchronize data"; + +my $result = + $node_subscriber->safe_psql('postgres', "SELECT count(*), count(c), count(d = 999) FROM test_tab"); +is($result, qq(2|0|0), 'check initial data was copied to subscriber'); + +# a small (non-streamed) transaction with DDL and DML +$node_publisher->safe_psql('postgres', q{ +BEGIN; +INSERT INTO test_tab VALUES (3, md5(3::text)); +ALTER TABLE test_tab ADD COLUMN c INT; +SAVEPOINT s1; +INSERT INTO test_tab VALUES (4, md5(4::text), -4); +COMMIT; +}); + +# large (streamed) transaction with DDL and DML +$node_publisher->safe_psql('postgres', q{ +BEGIN; +INSERT INTO test_tab SELECT i, md5(i::text), -i FROM generate_series(5, 1000) s(i); +ALTER TABLE test_tab ADD COLUMN d INT; +SAVEPOINT s1; +INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i FROM generate_series(1001, 2000) s(i); +COMMIT; +}); + +# a small (non-streamed) transaction with DDL and DML +$node_publisher->safe_psql('postgres', q{ +BEGIN; +INSERT INTO test_tab VALUES (2001, md5(2001::text), -2001, 2*2001); +ALTER TABLE test_tab ADD COLUMN e INT; +SAVEPOINT s1; +INSERT INTO test_tab VALUES (2002, md5(2002::text), -2002, 2*2002, -3*2002); +COMMIT; +}); + +$node_publisher->wait_for_catchup($appname); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*), count(c), count(d), count(e) FROM test_tab"); +is($result, qq(2002|1999|1002|1), 'check data was copied to subscriber in streaming mode and extra columns contain local defaults'); + +# A large (streamed) transaction with DDL and DML. One of the DDL is performed +# after DML to ensure that we invalidate the schema sent for test_tab so that +# the next transaction has to send the schema again. +$node_publisher->safe_psql('postgres', q{ +BEGIN; +INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i FROM generate_series(2003,5000) s(i); +ALTER TABLE test_tab ADD COLUMN f INT; +COMMIT; +}); + +# A small transaction that won't get streamed. This is just to ensure that we +# send the schema again to reflect the last column added in the previous test. +$node_publisher->safe_psql('postgres', q{ +BEGIN; +INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i, 4*i FROM generate_series(5001,5005) s(i); +COMMIT; +}); + +$node_publisher->wait_for_catchup($appname); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*), count(c), count(d), count(e), count(f) FROM test_tab"); +is($result, qq(5005|5002|4005|3004|5), 'check data was copied to subscriber for both streaming and non-streaming transactions'); + +$node_subscriber->stop; +$node_publisher->stop; diff --git a/src/test/subscription/t/018_stream_subxact_abort.pl b/src/test/subscription/t/018_stream_subxact_abort.pl new file mode 100644 index 000000000000..ddf0621558a5 --- /dev/null +++ b/src/test/subscription/t/018_stream_subxact_abort.pl @@ -0,0 +1,117 @@ +# Test streaming of large transaction containing multiple subtransactions and rollbacks +use strict; +use warnings; +use PostgresNode; +use TestLib; +use Test::More tests => 4; + +# Create publisher node +my $node_publisher = get_new_node('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf('postgresql.conf', 'logical_decoding_work_mem = 64kB'); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = get_new_node('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->start; + +# Create some preexisting content on publisher +$node_publisher->safe_psql('postgres', + "CREATE TABLE test_tab (a int primary key, b varchar)"); +$node_publisher->safe_psql('postgres', + "INSERT INTO test_tab VALUES (1, 'foo'), (2, 'bar')"); + +# Setup structure on subscriber +$node_subscriber->safe_psql('postgres', "CREATE TABLE test_tab (a int primary key, b text, c INT, d INT, e INT)"); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +$node_publisher->safe_psql('postgres', "CREATE PUBLICATION tap_pub FOR TABLE test_tab"); + +my $appname = 'tap_sub'; +$node_subscriber->safe_psql('postgres', +"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)" +); + +$node_publisher->wait_for_catchup($appname); + +# Also wait for initial table sync to finish +my $synced_query = +"SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('r', 's');"; +$node_subscriber->poll_query_until('postgres', $synced_query) + or die "Timed out while waiting for subscriber to synchronize data"; + +my $result = + $node_subscriber->safe_psql('postgres', "SELECT count(*), count(c) FROM test_tab"); +is($result, qq(2|0), 'check initial data was copied to subscriber'); + +# large (streamed) transaction with DDL, DML and ROLLBACKs +$node_publisher->safe_psql('postgres', q{ +BEGIN; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3,500) s(i); +SAVEPOINT s1; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(501,1000) s(i); +SAVEPOINT s2; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1001,1500) s(i); +SAVEPOINT s3; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(1501,2000) s(i); +ROLLBACK TO s2; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2001,2500) s(i); +ROLLBACK TO s1; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(2501,3000) s(i); +SAVEPOINT s4; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3001,3500) s(i); +SAVEPOINT s5; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3501,4000) s(i); +COMMIT; +}); + +$node_publisher->wait_for_catchup($appname); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*), count(c) FROM test_tab"); +is($result, qq(2000|0), 'check rollback to savepoint was reflected on subscriber and extra columns contain local defaults'); + +# large (streamed) transaction with subscriber receiving out of order +# subtransaction ROLLBACKs +$node_publisher->safe_psql('postgres', q{ +BEGIN; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(4001,4500) s(i); +SAVEPOINT s1; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(5001,5500) s(i); +SAVEPOINT s2; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(6001,6500) s(i); +SAVEPOINT s3; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(7001,7500) s(i); +RELEASE s2; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(8001,8500) s(i); +ROLLBACK TO s1; +COMMIT; +}); + +$node_publisher->wait_for_catchup($appname); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*), count(c) FROM test_tab"); +is($result, qq(2500|0), 'check rollback to savepoint was reflected on subscriber'); + +# large (streamed) transaction with subscriber receiving rollback +$node_publisher->safe_psql('postgres', q{ +BEGIN; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(8501,9000) s(i); +SAVEPOINT s1; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(9001,9500) s(i); +SAVEPOINT s2; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(9501,10000) s(i); +ROLLBACK; +}); + +$node_publisher->wait_for_catchup($appname); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*), count(c) FROM test_tab"); +is($result, qq(2500|0), 'check rollback was reflected on subscriber'); + +$node_subscriber->stop; +$node_publisher->stop; diff --git a/src/test/subscription/t/019_stream_subxact_ddl_abort.pl b/src/test/subscription/t/019_stream_subxact_ddl_abort.pl new file mode 100644 index 000000000000..33e42edfef26 --- /dev/null +++ b/src/test/subscription/t/019_stream_subxact_ddl_abort.pl @@ -0,0 +1,76 @@ +# Test streaming of large transaction with subtransactions, DDLs, DMLs, and +# rollbacks +use strict; +use warnings; +use PostgresNode; +use TestLib; +use Test::More tests => 2; + +# Create publisher node +my $node_publisher = get_new_node('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf('postgresql.conf', 'logical_decoding_work_mem = 64kB'); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = get_new_node('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->start; + +# Create some preexisting content on publisher +$node_publisher->safe_psql('postgres', + "CREATE TABLE test_tab (a int primary key, b varchar)"); +$node_publisher->safe_psql('postgres', + "INSERT INTO test_tab VALUES (1, 'foo'), (2, 'bar')"); + +# Setup structure on subscriber +$node_subscriber->safe_psql('postgres', "CREATE TABLE test_tab (a int primary key, b text, c INT, d INT, e INT)"); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +$node_publisher->safe_psql('postgres', "CREATE PUBLICATION tap_pub FOR TABLE test_tab"); + +my $appname = 'tap_sub'; +$node_subscriber->safe_psql('postgres', +"CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = on)" +); + +$node_publisher->wait_for_catchup($appname); + +# Also wait for initial table sync to finish +my $synced_query = +"SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('r', 's');"; +$node_subscriber->poll_query_until('postgres', $synced_query) + or die "Timed out while waiting for subscriber to synchronize data"; + +my $result = + $node_subscriber->safe_psql('postgres', "SELECT count(*), count(c) FROM test_tab"); +is($result, qq(2|0), 'check initial data was copied to subscriber'); + +# large (streamed) transaction with DDL, DML and ROLLBACKs +$node_publisher->safe_psql('postgres', q{ +BEGIN; +INSERT INTO test_tab SELECT i, md5(i::text) FROM generate_series(3,500) s(i); +ALTER TABLE test_tab ADD COLUMN c INT; +SAVEPOINT s1; +INSERT INTO test_tab SELECT i, md5(i::text), -i FROM generate_series(501,1000) s(i); +ALTER TABLE test_tab ADD COLUMN d INT; +SAVEPOINT s2; +INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i FROM generate_series(1001,1500) s(i); +ALTER TABLE test_tab ADD COLUMN e INT; +SAVEPOINT s3; +INSERT INTO test_tab SELECT i, md5(i::text), -i, 2*i, -3*i FROM generate_series(1501,2000) s(i); +ALTER TABLE test_tab DROP COLUMN c; +ROLLBACK TO s1; +INSERT INTO test_tab SELECT i, md5(i::text), i FROM generate_series(501,1000) s(i); +COMMIT; +}); + +$node_publisher->wait_for_catchup($appname); + +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*), count(c) FROM test_tab"); +is($result, qq(1000|500), 'check rollback to savepoint was reflected on subscriber and extra columns contain local defaults'); + +$node_subscriber->stop; +$node_publisher->stop; From f0942b1327e8fa32b38a02eaff627c16b517c3f9 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 7 Sep 2020 14:34:59 +0900 Subject: [PATCH 096/589] doc: Tweak sentence for pg_checksums when enabling checksums The previous version of the docs mentioned that files are rewritten, implying that a second copy of each file gets created, but each file is updated in-place. Author: Michael Banck Reviewed-by: Daniel Gustafsson, Michael Paquier Discussion: https://postgr.es/m/858086b6a42fb7d17995b6175856f7e7ec44d0a2.camel@credativ.de Backpatch-through: 12 --- doc/src/sgml/ref/pg_checksums.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/ref/pg_checksums.sgml b/doc/src/sgml/ref/pg_checksums.sgml index 8e7807f86bd9..1dd4e54ff11b 100644 --- a/doc/src/sgml/ref/pg_checksums.sgml +++ b/doc/src/sgml/ref/pg_checksums.sgml @@ -47,8 +47,8 @@ PostgreSQL documentation When verifying checksums, every file in the cluster is scanned. When - enabling checksums, every file in the cluster is rewritten. Disabling - checksums only updates the file pg_control. + enabling checksums, every file in the cluster is rewritten in-place. + Disabling checksums only updates the file pg_control. From 861c6e7c8e4dfdd842442dde47cc653764baff4f Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Mon, 7 Sep 2020 18:11:46 +1200 Subject: [PATCH 097/589] Skip unnecessary stat() calls in walkdir(). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some kernels can tell us the type of a "dirent", so we can avoid a call to stat() or lstat() in many cases. Define a new function get_dirent_type() to contain that logic, for use by the backend and frontend versions of walkdir(), and perhaps other callers in future. Reviewed-by: Tom Lane Reviewed-by: Juan José Santamaría Flecha Discussion: https://postgr.es/m/CA%2BhUKG%2BFzxupGGN4GpUdbzZN%2Btn6FQPHo8w0Q%2BAPH5Wz8RG%2Bww%40mail.gmail.com --- src/backend/storage/file/fd.c | 33 +++++----- src/common/Makefile | 2 +- src/common/file_utils.c | 109 ++++++++++++++++++++++++++----- src/include/common/file_utils.h | 20 +++++- src/tools/msvc/Mkvcbuild.pm | 4 +- src/tools/pgindent/typedefs.list | 1 + 6 files changed, 129 insertions(+), 40 deletions(-) diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c index f376a97ed677..bd72a87ee307 100644 --- a/src/backend/storage/file/fd.c +++ b/src/backend/storage/file/fd.c @@ -89,6 +89,7 @@ #include "access/xlog.h" #include "catalog/pg_tablespace.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "miscadmin.h" #include "pgstat.h" #include "portability/mem.h" @@ -3340,8 +3341,6 @@ walkdir(const char *path, while ((de = ReadDirExtended(dir, path, elevel)) != NULL) { char subpath[MAXPGPATH * 2]; - struct stat fst; - int sret; CHECK_FOR_INTERRUPTS(); @@ -3351,23 +3350,23 @@ walkdir(const char *path, snprintf(subpath, sizeof(subpath), "%s/%s", path, de->d_name); - if (process_symlinks) - sret = stat(subpath, &fst); - else - sret = lstat(subpath, &fst); - - if (sret < 0) + switch (get_dirent_type(subpath, de, process_symlinks, elevel)) { - ereport(elevel, - (errcode_for_file_access(), - errmsg("could not stat file \"%s\": %m", subpath))); - continue; - } + case PGFILETYPE_REG: + (*action) (subpath, false, elevel); + break; + case PGFILETYPE_DIR: + walkdir(subpath, action, false, elevel); + break; + default: - if (S_ISREG(fst.st_mode)) - (*action) (subpath, false, elevel); - else if (S_ISDIR(fst.st_mode)) - walkdir(subpath, action, false, elevel); + /* + * Errors are already reported directly by get_dirent_type(), + * and any remaining symlinks and unknown file types are + * ignored. + */ + break; + } } FreeDir(dir); /* we ignore any error here */ diff --git a/src/common/Makefile b/src/common/Makefile index ad8fd9e41c47..f2817628851e 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -56,6 +56,7 @@ OBJS_COMMON = \ exec.o \ f2s.o \ file_perm.o \ + file_utils.o \ hashfn.o \ ip.o \ jsonapi.o \ @@ -91,7 +92,6 @@ endif OBJS_FRONTEND = \ $(OBJS_COMMON) \ fe_memutils.o \ - file_utils.o \ logging.o \ restricted_token.o \ sprompt.o diff --git a/src/common/file_utils.c b/src/common/file_utils.c index a2faafdf13a3..12474730905c 100644 --- a/src/common/file_utils.c +++ b/src/common/file_utils.c @@ -14,10 +14,10 @@ */ #ifndef FRONTEND -#error "This file is not expected to be compiled for backend code" -#endif - +#include "postgres.h" +#else #include "postgres_fe.h" +#endif #include #include @@ -25,8 +25,11 @@ #include #include "common/file_utils.h" +#ifdef FRONTEND #include "common/logging.h" +#endif +#ifdef FRONTEND /* Define PG_FLUSH_DATA_WORKS if we have an implementation for pg_flush_data */ #if defined(HAVE_SYNC_FILE_RANGE) @@ -167,8 +170,6 @@ walkdir(const char *path, while (errno = 0, (de = readdir(dir)) != NULL) { char subpath[MAXPGPATH * 2]; - struct stat fst; - int sret; if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) @@ -176,21 +177,23 @@ walkdir(const char *path, snprintf(subpath, sizeof(subpath), "%s/%s", path, de->d_name); - if (process_symlinks) - sret = stat(subpath, &fst); - else - sret = lstat(subpath, &fst); - - if (sret < 0) + switch (get_dirent_type(subpath, de, process_symlinks, PG_LOG_ERROR)) { - pg_log_error("could not stat file \"%s\": %m", subpath); - continue; + case PGFILETYPE_REG: + (*action) (subpath, false); + break; + case PGFILETYPE_DIR: + walkdir(subpath, action, false); + break; + default: + + /* + * Errors are already reported directly by get_dirent_type(), + * and any remaining symlinks and unknown file types are + * ignored. + */ + break; } - - if (S_ISREG(fst.st_mode)) - (*action) (subpath, false); - else if (S_ISDIR(fst.st_mode)) - walkdir(subpath, action, false); } if (errno) @@ -394,3 +397,73 @@ durable_rename(const char *oldfile, const char *newfile) return 0; } + +#endif /* FRONTEND */ + +/* + * Return the type of a directory entry. + * + * In frontend code, elevel should be a level from logging.h; in backend code + * it should be a level from elog.h. + */ +PGFileType +get_dirent_type(const char *path, + const struct dirent *de, + bool look_through_symlinks, + int elevel) +{ + PGFileType result; + + /* + * Some systems tell us the type directly in the dirent struct, but that's + * a BSD and Linux extension not required by POSIX. Even when the + * interface is present, sometimes the type is unknown, depending on the + * filesystem. + */ +#if defined(DT_REG) && defined(DT_DIR) && defined(DT_LNK) + if (de->d_type == DT_REG) + result = PGFILETYPE_REG; + else if (de->d_type == DT_DIR) + result = PGFILETYPE_DIR; + else if (de->d_type == DT_LNK && !look_through_symlinks) + result = PGFILETYPE_LNK; + else + result = PGFILETYPE_UNKNOWN; +#else + result = PGFILETYPE_UNKNOWN; +#endif + + if (result == PGFILETYPE_UNKNOWN) + { + struct stat fst; + int sret; + + + if (look_through_symlinks) + sret = stat(path, &fst); + else + sret = lstat(path, &fst); + + if (sret < 0) + { + result = PGFILETYPE_ERROR; +#ifdef FRONTEND + pg_log_generic(elevel, "could not stat file \"%s\": %m", path); +#else + ereport(elevel, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", path))); +#endif + } + else if (S_ISREG(fst.st_mode)) + result = PGFILETYPE_REG; + else if (S_ISDIR(fst.st_mode)) + result = PGFILETYPE_DIR; +#ifdef S_ISLNK + else if (S_ISLNK(fst.st_mode)) + result = PGFILETYPE_LNK; +#endif + } + + return result; +} diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h index a7add75efa1d..fef846485f83 100644 --- a/src/include/common/file_utils.h +++ b/src/include/common/file_utils.h @@ -1,6 +1,4 @@ /*------------------------------------------------------------------------- - * - * File-processing utility routines for frontend code * * Assorted utility functions to work on files. * @@ -15,10 +13,28 @@ #ifndef FILE_UTILS_H #define FILE_UTILS_H +#include + +typedef enum PGFileType +{ + PGFILETYPE_ERROR, + PGFILETYPE_UNKNOWN, + PGFILETYPE_REG, + PGFILETYPE_DIR, + PGFILETYPE_LNK +} PGFileType; + +#ifdef FRONTEND extern int fsync_fname(const char *fname, bool isdir); extern void fsync_pgdata(const char *pg_data, int serverVersion); extern void fsync_dir_recurse(const char *dir); extern int durable_rename(const char *oldfile, const char *newfile); extern int fsync_parent_path(const char *fname); +#endif + +extern PGFileType get_dirent_type(const char *path, + const struct dirent *de, + bool look_through_symlinks, + int elevel); #endif /* FILE_UTILS_H */ diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 536df5a92e75..89e1b3903656 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -121,7 +121,7 @@ sub mkvcbuild our @pgcommonallfiles = qw( archive.c base64.c checksum_helper.c config_info.c controldata_utils.c d2s.c encnames.c exec.c - f2s.c file_perm.c hashfn.c ip.c jsonapi.c + f2s.c file_perm.c file_utils.c hashfn.c ip.c jsonapi.c keywords.c kwlookup.c link-canary.c md5.c pg_get_line.c pg_lzcompress.c pgfnames.c psprintf.c relpath.c rmtree.c saslprep.c scram-common.c string.c stringinfo.c unicode_norm.c username.c @@ -138,7 +138,7 @@ sub mkvcbuild } our @pgcommonfrontendfiles = ( - @pgcommonallfiles, qw(fe_memutils.c file_utils.c + @pgcommonallfiles, qw(fe_memutils.c logging.c restricted_token.c sprompt.c)); our @pgcommonbkndfiles = @pgcommonallfiles; diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 500623e23019..391c204f7219 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1515,6 +1515,7 @@ PGEventResultCopy PGEventResultCreate PGEventResultDestroy PGFInfoFunction +PGFileType PGFunction PGLZ_HistEntry PGLZ_Strategy From 74ff28197c14de0aa709497ce707e31bb9af7253 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 7 Sep 2020 09:59:50 +0200 Subject: [PATCH 098/589] doc: Fix table cell overflow Fix one instance of a table cell overflow by adding a zero-width space. The visual impact of this is minimal, but since this is currently the only such case reported by FOP ("contents of ... exceed the available area"), it seems worth getting rid of. --- doc/src/sgml/high-availability.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index ab9e6d87ce7a..beb309e668e2 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -305,7 +305,7 @@ protocol to make nodes agree on a serializable transactional order. File System Repl. Write-Ahead Log Shipping Logical Repl. - Trigger-Based Repl. + Trigger-&zwsp;Based Repl. SQL Repl. Middle-ware Async. MM Repl. Sync. MM Repl. From 87e6ed7c8c6abb6dd62119259d2fd89169a04ac0 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Mon, 7 Sep 2020 23:18:22 +1200 Subject: [PATCH 099/589] Add d_type to our Windows dirent emulation. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows us to skip some stat calls, by extending commit 861c6e7c to cover Windows systems. Author: Juan José Santamaría Flecha Reviewed-by: Alvaro Herrera Reviewed-by: Andres Freund Reviewed-by: Magnus Hagander Reviewed-by: Thomas Munro Discussion: https://postgr.es/m/CA%2BhUKG%2BFzxupGGN4GpUdbzZN%2Btn6FQPHo8w0Q%2BAPH5Wz8RG%2Bww%40mail.gmail.com --- src/include/port/win32_msvc/dirent.h | 11 +++++++++++ src/port/dirent.c | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/include/port/win32_msvc/dirent.h b/src/include/port/win32_msvc/dirent.h index 9fabdf332b2c..62799db00141 100644 --- a/src/include/port/win32_msvc/dirent.h +++ b/src/include/port/win32_msvc/dirent.h @@ -10,6 +10,7 @@ struct dirent { long d_ino; unsigned short d_reclen; + unsigned char d_type; unsigned short d_namlen; char d_name[MAX_PATH]; }; @@ -20,4 +21,14 @@ DIR *opendir(const char *); struct dirent *readdir(DIR *); int closedir(DIR *); +/* File types for 'd_type'. */ +#define DT_UNKNOWN 0 +#define DT_FIFO 1 +#define DT_CHR 2 +#define DT_DIR 4 +#define DT_BLK 6 +#define DT_REG 8 +#define DT_LNK 10 +#define DT_SOCK 12 +#define DT_WHT 14 #endif diff --git a/src/port/dirent.c b/src/port/dirent.c index b264484fca24..70a444f9a53f 100644 --- a/src/port/dirent.c +++ b/src/port/dirent.c @@ -69,6 +69,7 @@ opendir(const char *dirname) d->handle = INVALID_HANDLE_VALUE; d->ret.d_ino = 0; /* no inodes on win32 */ d->ret.d_reclen = 0; /* not used on win32 */ + d->ret.d_type = DT_UNKNOWN; return d; } @@ -105,6 +106,15 @@ readdir(DIR *d) } strcpy(d->ret.d_name, fd.cFileName); /* Both strings are MAX_PATH long */ d->ret.d_namlen = strlen(d->ret.d_name); + /* The only identified types are: directory, regular file or symbolic link */ + if ((fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) + d->ret.d_type = DT_DIR; + /* For reparse points dwReserved0 field will contain the ReparseTag */ + else if ((fd.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0 && + (fd.dwReserved0 == IO_REPARSE_TAG_MOUNT_POINT)) + d->ret.d_type = DT_LNK; + else + d->ret.d_type = DT_REG; return &d->ret; } From 9c79e646c6f0f8df06d966c536d0c6aa33bf1b06 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 7 Sep 2020 12:03:04 -0400 Subject: [PATCH 100/589] Frob numeric.c loop so that clang will auto-vectorize it too. Experimentation shows that clang will auto-vectorize the critical multiplication loop if the termination condition is written "i2 < limit" rather than "i2 <= limit". This seems unbelievably stupid, but I've reproduced it on both clang 9.0.1 (RHEL8) and 11.0.3 (macOS Catalina). gcc doesn't care, so tweak the code to do it that way. Discussion: https://postgr.es/m/CAJ3gD9evtA_vBo+WMYMyT-u=keHX7-r8p2w7OSRfXf42LTwCZQ@mail.gmail.com --- src/backend/utils/adt/numeric.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c index d2a42b811dac..dfd455fc7488 100644 --- a/src/backend/utils/adt/numeric.c +++ b/src/backend/utils/adt/numeric.c @@ -8191,7 +8191,6 @@ mul_var(const NumericVar *var1, const NumericVar *var2, NumericVar *result, int res_weight; int maxdigits; int *dig; - int *dig_i1_2; int carry; int maxdig; int newdig; @@ -8327,7 +8326,7 @@ mul_var(const NumericVar *var1, const NumericVar *var2, NumericVar *result, * Add the appropriate multiple of var2 into the accumulator. * * As above, digits of var2 can be ignored if they don't contribute, - * so we only include digits for which i1+i2+2 <= res_ndigits - 1. + * so we only include digits for which i1+i2+2 < res_ndigits. * * This inner loop is the performance bottleneck for multiplication, * so we want to keep it simple enough so that it can be @@ -8336,10 +8335,13 @@ mul_var(const NumericVar *var1, const NumericVar *var2, NumericVar *result, * Since we aren't propagating carries in this loop, the order does * not matter. */ - i = Min(var2ndigits - 1, res_ndigits - i1 - 3); - dig_i1_2 = &dig[i1 + 2]; - for (i2 = 0; i2 <= i; i2++) - dig_i1_2[i2] += var1digit * var2digits[i2]; + { + int i2limit = Min(var2ndigits, res_ndigits - i1 - 2); + int *dig_i1_2 = &dig[i1 + 2]; + + for (i2 = 0; i2 < i2limit; i2++) + dig_i1_2[i2] += var1digit * var2digits[i2]; + } } /* From 53367e6c62e3111eccdd578a3fa607c48c3e1fb6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 7 Sep 2020 14:52:33 -0400 Subject: [PATCH 101/589] Clarify comments in enforce_generic_type_consistency(). Some of the pre-existing comments were vague about whether they referred to all polymorphic types or only the old-style ones. Also be more consistent about using the "family 1" vs "family 2" terminology. Himanshu Upadhyaya and Tom Lane Discussion: https://postgr.es/m/CAPF61jBUg9XoMPNuLpoZ+h6UZ2VxKdNt3rQL1xw1GOBwjWzAXQ@mail.gmail.com --- src/backend/parser/parse_coerce.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/backend/parser/parse_coerce.c b/src/backend/parser/parse_coerce.c index 1b11cf731ced..2ffe47026b9b 100644 --- a/src/backend/parser/parse_coerce.c +++ b/src/backend/parser/parse_coerce.c @@ -2155,8 +2155,8 @@ enforce_generic_type_consistency(const Oid *actual_arg_types, else { /* - * Only way to get here is if all the polymorphic args have - * UNKNOWN inputs + * Only way to get here is if all the family-1 polymorphic + * arguments have UNKNOWN inputs. */ ereport(ERROR, (errcode(ERRCODE_DATATYPE_MISMATCH), @@ -2254,10 +2254,10 @@ enforce_generic_type_consistency(const Oid *actual_arg_types, else { /* - * Only way to get here is if all the ANYCOMPATIBLE args have - * UNKNOWN inputs. Resolve to TEXT as select_common_type() - * would do. That doesn't license us to use TEXTRANGE, - * though. + * Only way to get here is if all the family-2 polymorphic + * arguments have UNKNOWN inputs. Resolve to TEXT as + * select_common_type() would do. That doesn't license us to + * use TEXTRANGE, though. */ anycompatible_typeid = TEXTOID; anycompatible_array_typeid = TEXTARRAYOID; @@ -2269,7 +2269,7 @@ enforce_generic_type_consistency(const Oid *actual_arg_types, } } - /* replace polymorphic types by selected types */ + /* replace family-2 polymorphic types by selected types */ for (int j = 0; j < nargs; j++) { Oid decl_type = declared_arg_types[j]; @@ -2285,11 +2285,11 @@ enforce_generic_type_consistency(const Oid *actual_arg_types, } /* - * If we had any UNKNOWN inputs for polymorphic arguments, re-scan to - * assign correct types to them. + * If we had any UNKNOWN inputs for family-1 polymorphic arguments, + * re-scan to assign correct types to them. * * Note: we don't have to consider unknown inputs that were matched to - * ANYCOMPATIBLE-family arguments, because we forcibly updated their + * family-2 polymorphic arguments, because we forcibly updated their * declared_arg_types[] positions just above. */ if (have_poly_unknowns) From a547e6867527ca16628a3fb1cf3ef6f785210a31 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Mon, 7 Sep 2020 13:31:59 -0700 Subject: [PATCH 102/589] Adjust cost model for HashAgg that spills to disk. Tomas Vondra observed that the IO behavior for HashAgg tends to be worse than for Sort. Penalize HashAgg IO costs accordingly. Also, account for the CPU effort of spilling the tuples and reading them back. Discussion: https://postgr.es/m/20200906212112.nzoy5ytrzjjodpfh@development Reviewed-by: Tomas Vondra Backpatch-through: 13 --- src/backend/optimizer/path/costsize.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index fda4b2c6e875..cd3716d494f0 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -2416,6 +2416,7 @@ cost_agg(Path *path, PlannerInfo *root, double pages; double pages_written = 0.0; double pages_read = 0.0; + double spill_cost; double hashentrysize; double nbatches; Size mem_limit; @@ -2453,9 +2454,21 @@ cost_agg(Path *path, PlannerInfo *root, pages = relation_byte_size(input_tuples, input_width) / BLCKSZ; pages_written = pages_read = pages * depth; + /* + * HashAgg has somewhat worse IO behavior than Sort on typical + * hardware/OS combinations. Account for this with a generic penalty. + */ + pages_read *= 2.0; + pages_written *= 2.0; + startup_cost += pages_written * random_page_cost; total_cost += pages_written * random_page_cost; total_cost += pages_read * seq_page_cost; + + /* account for CPU cost of spilling a tuple and reading it back */ + spill_cost = depth * input_tuples * 2.0 * cpu_tuple_cost; + startup_cost += spill_cost; + total_cost += spill_cost; } /* From a6642b3ae060976b42830b7dc8f29ec190ab05e4 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 8 Sep 2020 10:09:22 +0900 Subject: [PATCH 103/589] Add support for partitioned tables and indexes in REINDEX Until now, REINDEX was not able to work with partitioned tables and indexes, forcing users to reindex partitions one by one. This extends REINDEX INDEX and REINDEX TABLE so as they can accept a partitioned index and table in input, respectively, to reindex all the partitions assigned to them with physical storage (foreign tables, partitioned tables and indexes are then discarded). This shares some logic with schema and database REINDEX as each partition gets processed in its own transaction after building a list of relations to work on. This choice has the advantage to minimize the number of invalid indexes to one partition with REINDEX CONCURRENTLY in the event a cancellation or failure in-flight, as the only indexes handled at once in a single REINDEX CONCURRENTLY loop are the ones from the partition being working on. Isolation tests are added to emulate some cases I bumped into while developing this feature, particularly with the concurrent drop of a leaf partition reindexed. However, this is rather limited as LOCK would cause REINDEX to block in the first transaction building the list of partitions. Per its multi-transaction nature, this new flavor cannot run in a transaction block, similarly to REINDEX SCHEMA, SYSTEM and DATABASE. Author: Justin Pryzby, Michael Paquier Reviewed-by: Anastasia Lubennikova Discussion: https://postgr.es/m/db12e897-73ff-467e-94cb-4af03705435f.adger.lj@alibaba-inc.com --- .../postgres_fdw/expected/postgres_fdw.out | 21 ++ contrib/postgres_fdw/sql/postgres_fdw.sql | 20 ++ doc/src/sgml/ref/reindex.sgml | 13 +- src/backend/catalog/index.c | 25 +- src/backend/commands/indexcmds.c | 245 ++++++++++++++---- src/backend/tcop/utility.c | 6 +- src/include/commands/defrem.h | 4 +- .../isolation/expected/reindex-partitions.out | 107 ++++++++ src/test/isolation/isolation_schedule | 1 + .../isolation/specs/reindex-partitions.spec | 59 +++++ src/test/regress/expected/create_index.out | 158 ++++++++++- src/test/regress/sql/create_index.sql | 88 ++++++- src/tools/pgindent/typedefs.list | 1 + 13 files changed, 656 insertions(+), 92 deletions(-) create mode 100644 src/test/isolation/expected/reindex-partitions.out create mode 100644 src/test/isolation/specs/reindex-partitions.spec diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 90db550b9214..84bc0ee38171 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -4044,6 +4044,27 @@ SELECT f_test(100); DROP FUNCTION f_test(int); -- =================================================================== +-- REINDEX +-- =================================================================== +-- remote table is not created here +CREATE FOREIGN TABLE reindex_foreign (c1 int, c2 int) + SERVER loopback2 OPTIONS (table_name 'reindex_local'); +REINDEX TABLE reindex_foreign; -- error +ERROR: "reindex_foreign" is not a table or materialized view +REINDEX TABLE CONCURRENTLY reindex_foreign; -- error +ERROR: "reindex_foreign" is not a table or materialized view +DROP FOREIGN TABLE reindex_foreign; +-- partitions and foreign tables +CREATE TABLE reind_fdw_parent (c1 int) PARTITION BY RANGE (c1); +CREATE TABLE reind_fdw_0_10 PARTITION OF reind_fdw_parent + FOR VALUES FROM (0) TO (10); +CREATE FOREIGN TABLE reind_fdw_10_20 PARTITION OF reind_fdw_parent + FOR VALUES FROM (10) TO (20) + SERVER loopback OPTIONS (table_name 'reind_local_10_20'); +REINDEX TABLE reind_fdw_parent; -- ok +REINDEX TABLE CONCURRENTLY reind_fdw_parent; -- ok +DROP TABLE reind_fdw_parent; +-- =================================================================== -- conversion error -- =================================================================== ALTER FOREIGN TABLE ft1 ALTER COLUMN c8 TYPE int; diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index 83971665e351..d452d063430a 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -1081,6 +1081,26 @@ $$ LANGUAGE plpgsql; SELECT f_test(100); DROP FUNCTION f_test(int); +-- =================================================================== +-- REINDEX +-- =================================================================== +-- remote table is not created here +CREATE FOREIGN TABLE reindex_foreign (c1 int, c2 int) + SERVER loopback2 OPTIONS (table_name 'reindex_local'); +REINDEX TABLE reindex_foreign; -- error +REINDEX TABLE CONCURRENTLY reindex_foreign; -- error +DROP FOREIGN TABLE reindex_foreign; +-- partitions and foreign tables +CREATE TABLE reind_fdw_parent (c1 int) PARTITION BY RANGE (c1); +CREATE TABLE reind_fdw_0_10 PARTITION OF reind_fdw_parent + FOR VALUES FROM (0) TO (10); +CREATE FOREIGN TABLE reind_fdw_10_20 PARTITION OF reind_fdw_parent + FOR VALUES FROM (10) TO (20) + SERVER loopback OPTIONS (table_name 'reind_local_10_20'); +REINDEX TABLE reind_fdw_parent; -- ok +REINDEX TABLE CONCURRENTLY reind_fdw_parent; -- ok +DROP TABLE reind_fdw_parent; + -- =================================================================== -- conversion error -- =================================================================== diff --git a/doc/src/sgml/ref/reindex.sgml b/doc/src/sgml/ref/reindex.sgml index c16f223e4edb..33af4ae02a13 100644 --- a/doc/src/sgml/ref/reindex.sgml +++ b/doc/src/sgml/ref/reindex.sgml @@ -88,7 +88,9 @@ REINDEX [ ( option [, ...] ) ] { IN INDEX - Recreate the specified index. + Recreate the specified index. This form of REINDEX + cannot be executed inside a transaction block when used with a + partitioned index. @@ -99,6 +101,8 @@ REINDEX [ ( option [, ...] ) ] { IN Recreate all indexes of the specified table. If the table has a secondary TOAST table, that is reindexed as well. + This form of REINDEX cannot be executed inside a + transaction block when used with a partitioned table. @@ -259,8 +263,11 @@ REINDEX [ ( option [, ...] ) ] { IN - Reindexing partitioned tables or partitioned indexes is not supported. - Each individual partition can be reindexed separately instead. + Reindexing partitioned indexes or partitioned tables is supported + with REINDEX INDEX or REINDEX TABLE, + respectively. Each partition of the specified partitioned relation is + reindexed in a separate transaction. Those commands cannot be used inside + a transaction block when working on a partitioned table or index. diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index b2c8cb320efc..117e3fdef7dc 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -77,6 +77,7 @@ #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/pg_rusage.h" +#include "utils/rel.h" #include "utils/snapmgr.h" #include "utils/syscache.h" #include "utils/tuplesort.h" @@ -3486,11 +3487,12 @@ reindex_index(Oid indexId, bool skip_constraint_checks, char persistence, iRel->rd_rel->relam); /* - * The case of reindexing partitioned tables and indexes is handled - * differently by upper layers, so this case shouldn't arise. + * Partitioned indexes should never get processed here, as they have no + * physical storage. */ if (iRel->rd_rel->relkind == RELKIND_PARTITIONED_INDEX) - elog(ERROR, "unsupported relation kind for index \"%s\"", + elog(ERROR, "cannot reindex partitioned index \"%s.%s\"", + get_namespace_name(RelationGetNamespace(iRel)), RelationGetRelationName(iRel)); /* @@ -3707,20 +3709,13 @@ reindex_relation(Oid relid, int flags, int options) return false; /* - * This may be useful when implemented someday; but that day is not today. - * For now, avoid erroring out when called in a multi-table context - * (REINDEX SCHEMA) and happen to come across a partitioned table. The - * partitions may be reindexed on their own anyway. + * Partitioned tables should never get processed here, as they have no + * physical storage. */ if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) - { - ereport(WARNING, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("REINDEX of partitioned tables is not yet implemented, skipping \"%s\"", - RelationGetRelationName(rel)))); - table_close(rel, ShareLock); - return false; - } + elog(ERROR, "cannot reindex partitioned table \"%s.%s\"", + get_namespace_name(RelationGetNamespace(rel)), + RelationGetRelationName(rel)); toast_relid = rel->rd_rel->reltoastrelid; diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 430e88b4c9fa..f1b5f87e6a8c 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -88,7 +88,10 @@ static List *ChooseIndexColumnNames(List *indexElems); static void RangeVarCallbackForReindexIndex(const RangeVar *relation, Oid relId, Oid oldRelId, void *arg); static bool ReindexRelationConcurrently(Oid relationOid, int options); -static void ReindexPartitionedIndex(Relation parentIdx); + +static void ReindexPartitions(Oid relid, int options, bool isTopLevel); +static void ReindexMultipleInternal(List *relids, int options); +static void reindex_error_callback(void *args); static void update_relispartition(Oid relationId, bool newval); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); @@ -101,6 +104,16 @@ struct ReindexIndexCallbackState Oid locked_table_oid; /* tracks previously locked table */ }; +/* + * callback arguments for reindex_error_callback() + */ +typedef struct ReindexErrorInfo +{ + char *relname; + char *relnamespace; + char relkind; +} ReindexErrorInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2420,12 +2433,12 @@ ChooseIndexColumnNames(List *indexElems) * Recreate a specific index. */ void -ReindexIndex(RangeVar *indexRelation, int options) +ReindexIndex(RangeVar *indexRelation, int options, bool isTopLevel) { struct ReindexIndexCallbackState state; Oid indOid; - Relation irel; char persistence; + char relkind; /* * Find and lock index, and check permissions on table; use callback to @@ -2447,22 +2460,16 @@ ReindexIndex(RangeVar *indexRelation, int options) &state); /* - * Obtain the current persistence of the existing index. We already hold - * lock on the index. + * Obtain the current persistence and kind of the existing index. We + * already hold a lock on the index. */ - irel = index_open(indOid, NoLock); - - if (irel->rd_rel->relkind == RELKIND_PARTITIONED_INDEX) - { - ReindexPartitionedIndex(irel); - return; - } + persistence = get_rel_persistence(indOid); + relkind = get_rel_relkind(indOid); - persistence = irel->rd_rel->relpersistence; - index_close(irel, NoLock); - - if ((options & REINDEXOPT_CONCURRENTLY) != 0 && - persistence != RELPERSISTENCE_TEMP) + if (relkind == RELKIND_PARTITIONED_INDEX) + ReindexPartitions(indOid, options, isTopLevel); + else if ((options & REINDEXOPT_CONCURRENTLY) != 0 && + persistence != RELPERSISTENCE_TEMP) ReindexRelationConcurrently(indOid, options); else reindex_index(indOid, false, persistence, @@ -2545,7 +2552,7 @@ RangeVarCallbackForReindexIndex(const RangeVar *relation, * Recreate all indexes of a table (and of its toast table, if any) */ Oid -ReindexTable(RangeVar *relation, int options) +ReindexTable(RangeVar *relation, int options, bool isTopLevel) { Oid heapOid; bool result; @@ -2564,8 +2571,10 @@ ReindexTable(RangeVar *relation, int options) 0, RangeVarCallbackOwnsTable, NULL); - if ((options & REINDEXOPT_CONCURRENTLY) != 0 && - get_rel_persistence(heapOid) != RELPERSISTENCE_TEMP) + if (get_rel_relkind(heapOid) == RELKIND_PARTITIONED_TABLE) + ReindexPartitions(heapOid, options, isTopLevel); + else if ((options & REINDEXOPT_CONCURRENTLY) != 0 && + get_rel_persistence(heapOid) != RELPERSISTENCE_TEMP) { result = ReindexRelationConcurrently(heapOid, options); @@ -2609,7 +2618,6 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, MemoryContext private_context; MemoryContext old; List *relids = NIL; - ListCell *l; int num_keys; bool concurrent_warning = false; @@ -2694,11 +2702,8 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, * Only regular tables and matviews can have indexes, so ignore any * other kind of relation. * - * It is tempting to also consider partitioned tables here, but that - * has the problem that if the children are in the same schema, they - * would be processed twice. Maybe we could have a separate list of - * partitioned tables, and expand that afterwards into relids, - * ignoring any duplicates. + * Partitioned tables/indexes are skipped but matching leaf partitions + * are processed. */ if (classtuple->relkind != RELKIND_RELATION && classtuple->relkind != RELKIND_MATVIEW) @@ -2761,14 +2766,151 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, table_endscan(scan); table_close(relationRelation, AccessShareLock); - /* Now reindex each rel in a separate transaction */ + /* + * Process each relation listed in a separate transaction. Note that this + * commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(relids, options); + + MemoryContextDelete(private_context); +} + +/* + * Error callback specific to ReindexPartitions(). + */ +static void +reindex_error_callback(void *arg) +{ + ReindexErrorInfo *errinfo = (ReindexErrorInfo *) arg; + + Assert(errinfo->relkind == RELKIND_PARTITIONED_INDEX || + errinfo->relkind == RELKIND_PARTITIONED_TABLE); + + if (errinfo->relkind == RELKIND_PARTITIONED_TABLE) + errcontext("while reindexing partitioned table \"%s.%s\"", + errinfo->relnamespace, errinfo->relname); + else if (errinfo->relkind == RELKIND_PARTITIONED_INDEX) + errcontext("while reindexing partitioned index \"%s.%s\"", + errinfo->relnamespace, errinfo->relname); +} + +/* + * ReindexPartitions + * + * Reindex a set of partitions, per the partitioned index or table given + * by the caller. + */ +static void +ReindexPartitions(Oid relid, int options, bool isTopLevel) +{ + List *partitions = NIL; + char relkind = get_rel_relkind(relid); + char *relname = get_rel_name(relid); + char *relnamespace = get_namespace_name(get_rel_namespace(relid)); + MemoryContext reindex_context; + List *inhoids; + ListCell *lc; + ErrorContextCallback errcallback; + ReindexErrorInfo errinfo; + + Assert(relkind == RELKIND_PARTITIONED_INDEX || + relkind == RELKIND_PARTITIONED_TABLE); + + /* + * Check if this runs in a transaction block, with an error callback to + * provide more context under which a problem happens. + */ + errinfo.relname = pstrdup(relname); + errinfo.relnamespace = pstrdup(relnamespace); + errinfo.relkind = relkind; + errcallback.callback = reindex_error_callback; + errcallback.arg = (void *) &errinfo; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + PreventInTransactionBlock(isTopLevel, + relkind == RELKIND_PARTITIONED_TABLE ? + "REINDEX TABLE" : "REINDEX INDEX"); + + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + + /* + * Create special memory context for cross-transaction storage. + * + * Since it is a child of PortalContext, it will go away eventually even + * if we suffer an error so there is no need for special abort cleanup + * logic. + */ + reindex_context = AllocSetContextCreate(PortalContext, "Reindex", + ALLOCSET_DEFAULT_SIZES); + + /* ShareLock is enough to prevent schema modifications */ + inhoids = find_all_inheritors(relid, ShareLock, NULL); + + /* + * The list of relations to reindex are the physical partitions of the + * tree so discard any partitioned table or index. + */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + char partkind = get_rel_relkind(partoid); + MemoryContext old_context; + + /* + * This discards partitioned tables, partitioned indexes and foreign + * tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX || + partkind == RELKIND_RELATION); + + /* Save partition OID */ + old_context = MemoryContextSwitchTo(reindex_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(old_context); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, options); + + /* + * Clean up working storage --- note we must do this after + * StartTransactionCommand, else we might be trying to delete the active + * context! + */ + MemoryContextDelete(reindex_context); +} + +/* + * ReindexMultipleInternal + * + * Reindex a list of relations, each one being processed in its own + * transaction. This commits the existing transaction immediately, + * and starts a new transaction when finished. + */ +static void +ReindexMultipleInternal(List *relids, int options) +{ + ListCell *l; + PopActiveSnapshot(); CommitTransactionCommand(); + foreach(l, relids) { Oid relid = lfirst_oid(l); + char relkind; + char relpersistence; StartTransactionCommand(); + /* functions in indexes may want a snapshot set */ PushActiveSnapshot(GetTransactionSnapshot()); @@ -2780,14 +2922,33 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, continue; } + relkind = get_rel_relkind(relid); + relpersistence = get_rel_persistence(relid); + + /* + * Partitioned tables and indexes can never be processed directly, and + * a list of their leaves should be built first. + */ + Assert(relkind != RELKIND_PARTITIONED_INDEX && + relkind != RELKIND_PARTITIONED_TABLE); + if ((options & REINDEXOPT_CONCURRENTLY) != 0 && - get_rel_persistence(relid) != RELPERSISTENCE_TEMP) + relpersistence != RELPERSISTENCE_TEMP) { (void) ReindexRelationConcurrently(relid, options | REINDEXOPT_MISSING_OK); /* ReindexRelationConcurrently() does the verbose output */ } + else if (relkind == RELKIND_INDEX) + { + reindex_index(relid, false, relpersistence, + options | + REINDEXOPT_REPORT_PROGRESS | + REINDEXOPT_MISSING_OK); + PopActiveSnapshot(); + /* reindex_index() does the verbose output */ + } else { bool result; @@ -2810,9 +2971,8 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, CommitTransactionCommand(); } - StartTransactionCommand(); - MemoryContextDelete(private_context); + StartTransactionCommand(); } @@ -2824,8 +2984,7 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, * view. For tables and materialized views, all its indexes will be rebuilt, * excluding invalid indexes and any indexes used in exclusion constraints, * but including its associated toast table indexes. For indexes, the index - * itself will be rebuilt. If 'relationOid' belongs to a partitioned table - * then we issue a warning to mention these are not yet supported. + * itself will be rebuilt. * * The locks taken on parent tables and involved indexes are kept until the * transaction is committed, at which point a session lock is taken on each @@ -3064,13 +3223,9 @@ ReindexRelationConcurrently(Oid relationOid, int options) MemoryContextSwitchTo(oldcontext); break; } + case RELKIND_PARTITIONED_TABLE: - /* see reindex_relation() */ - ereport(WARNING, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("REINDEX of partitioned tables is not yet implemented, skipping \"%s\"", - get_rel_name(relationOid)))); - return false; + case RELKIND_PARTITIONED_INDEX: default: /* Return error if type of relation is not supported */ ereport(ERROR, @@ -3537,20 +3692,6 @@ ReindexRelationConcurrently(Oid relationOid, int options) return true; } -/* - * ReindexPartitionedIndex - * Reindex each child of the given partitioned index. - * - * Not yet implemented. - */ -static void -ReindexPartitionedIndex(Relation parentIdx) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("REINDEX is not yet implemented for partitioned indexes"))); -} - /* * Insert or delete an appropriate pg_inherits tuple to make the given index * be a partition of the indicated parent index. diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index b4cde5565ec1..9713a7ac41a4 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -926,10 +926,12 @@ standard_ProcessUtility(PlannedStmt *pstmt, switch (stmt->kind) { case REINDEX_OBJECT_INDEX: - ReindexIndex(stmt->relation, stmt->options); + ReindexIndex(stmt->relation, stmt->options, + isTopLevel); break; case REINDEX_OBJECT_TABLE: - ReindexTable(stmt->relation, stmt->options); + ReindexTable(stmt->relation, stmt->options, + isTopLevel); break; case REINDEX_OBJECT_SCHEMA: case REINDEX_OBJECT_SYSTEM: diff --git a/src/include/commands/defrem.h b/src/include/commands/defrem.h index 3129b684f633..7a079ef07f06 100644 --- a/src/include/commands/defrem.h +++ b/src/include/commands/defrem.h @@ -34,8 +34,8 @@ extern ObjectAddress DefineIndex(Oid relationId, bool check_not_in_use, bool skip_build, bool quiet); -extern void ReindexIndex(RangeVar *indexRelation, int options); -extern Oid ReindexTable(RangeVar *relation, int options); +extern void ReindexIndex(RangeVar *indexRelation, int options, bool isTopLevel); +extern Oid ReindexTable(RangeVar *relation, int options, bool isTopLevel); extern void ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, int options); extern char *makeObjectName(const char *name1, const char *name2, diff --git a/src/test/isolation/expected/reindex-partitions.out b/src/test/isolation/expected/reindex-partitions.out new file mode 100644 index 000000000000..5373abde259b --- /dev/null +++ b/src/test/isolation/expected/reindex-partitions.out @@ -0,0 +1,107 @@ +Parsed test spec with 3 sessions + +starting permutation: begin1 lockexcl1 reindex2 drop3 end1 +step begin1: BEGIN; +step lockexcl1: LOCK reind_conc_parent IN ACCESS EXCLUSIVE MODE; +step reindex2: REINDEX TABLE reind_conc_parent; +step drop3: DROP TABLE reind_conc_10_20; +step end1: COMMIT; +step reindex2: <... completed> +step drop3: <... completed> + +starting permutation: begin1 lockexcl1 reindex_conc2 drop3 end1 +step begin1: BEGIN; +step lockexcl1: LOCK reind_conc_parent IN ACCESS EXCLUSIVE MODE; +step reindex_conc2: REINDEX TABLE CONCURRENTLY reind_conc_parent; +step drop3: DROP TABLE reind_conc_10_20; +step end1: COMMIT; +step reindex_conc2: <... completed> +step drop3: <... completed> + +starting permutation: begin1 lockshare1 reindex2 drop3 end1 +step begin1: BEGIN; +step lockshare1: LOCK reind_conc_parent IN SHARE MODE; +step reindex2: REINDEX TABLE reind_conc_parent; +step drop3: DROP TABLE reind_conc_10_20; +step end1: COMMIT; +step drop3: <... completed> + +starting permutation: begin1 lockshare1 reindex_conc2 drop3 end1 +step begin1: BEGIN; +step lockshare1: LOCK reind_conc_parent IN SHARE MODE; +step reindex_conc2: REINDEX TABLE CONCURRENTLY reind_conc_parent; +step drop3: DROP TABLE reind_conc_10_20; +step end1: COMMIT; +step reindex_conc2: <... completed> +step drop3: <... completed> + +starting permutation: begin1 lockupdate1 reindex2 drop3 end1 +step begin1: BEGIN; +step lockupdate1: LOCK reind_conc_parent IN SHARE UPDATE EXCLUSIVE MODE; +step reindex2: REINDEX TABLE reind_conc_parent; +step drop3: DROP TABLE reind_conc_10_20; +step end1: COMMIT; +step reindex2: <... completed> +step drop3: <... completed> + +starting permutation: begin1 lockupdate1 reindex_conc2 drop3 end1 +step begin1: BEGIN; +step lockupdate1: LOCK reind_conc_parent IN SHARE UPDATE EXCLUSIVE MODE; +step reindex_conc2: REINDEX TABLE CONCURRENTLY reind_conc_parent; +step drop3: DROP TABLE reind_conc_10_20; +step end1: COMMIT; +step reindex_conc2: <... completed> +step drop3: <... completed> + +starting permutation: begin1 lockpartexcl1 reindex2 drop3 end1 +step begin1: BEGIN; +step lockpartexcl1: LOCK reind_conc_10_20 IN ACCESS EXCLUSIVE MODE; +step reindex2: REINDEX TABLE reind_conc_parent; +step drop3: DROP TABLE reind_conc_10_20; +step end1: COMMIT; +step reindex2: <... completed> +step drop3: <... completed> + +starting permutation: begin1 lockpartexcl1 reindex_conc2 drop3 end1 +step begin1: BEGIN; +step lockpartexcl1: LOCK reind_conc_10_20 IN ACCESS EXCLUSIVE MODE; +step reindex_conc2: REINDEX TABLE CONCURRENTLY reind_conc_parent; +step drop3: DROP TABLE reind_conc_10_20; +step end1: COMMIT; +step reindex_conc2: <... completed> +step drop3: <... completed> + +starting permutation: begin1 lockpartshare1 reindex2 drop3 end1 +step begin1: BEGIN; +step lockpartshare1: LOCK reind_conc_10_20 IN SHARE MODE; +step reindex2: REINDEX TABLE reind_conc_parent; +step drop3: DROP TABLE reind_conc_10_20; +step end1: COMMIT; +step drop3: <... completed> + +starting permutation: begin1 lockpartshare1 reindex_conc2 drop3 end1 +step begin1: BEGIN; +step lockpartshare1: LOCK reind_conc_10_20 IN SHARE MODE; +step reindex_conc2: REINDEX TABLE CONCURRENTLY reind_conc_parent; +step drop3: DROP TABLE reind_conc_10_20; +step end1: COMMIT; +step reindex_conc2: <... completed> +step drop3: <... completed> + +starting permutation: begin1 lockpartupdate1 reindex2 drop3 end1 +step begin1: BEGIN; +step lockpartupdate1: LOCK reind_conc_10_20 IN SHARE UPDATE EXCLUSIVE MODE; +step reindex2: REINDEX TABLE reind_conc_parent; +step drop3: DROP TABLE reind_conc_10_20; +step end1: COMMIT; +step reindex2: <... completed> +step drop3: <... completed> + +starting permutation: begin1 lockpartupdate1 reindex_conc2 drop3 end1 +step begin1: BEGIN; +step lockpartupdate1: LOCK reind_conc_10_20 IN SHARE UPDATE EXCLUSIVE MODE; +step reindex_conc2: REINDEX TABLE CONCURRENTLY reind_conc_parent; +step drop3: DROP TABLE reind_conc_10_20; +step end1: COMMIT; +step reindex_conc2: <... completed> +step drop3: <... completed> diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index 6acbb695ecea..65d1443ac684 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -50,6 +50,7 @@ test: lock-committed-keyupdate test: update-locked-tuple test: reindex-concurrently test: reindex-schema +test: reindex-partitions test: propagate-lock-delete test: tuplelock-conflict test: tuplelock-update diff --git a/src/test/isolation/specs/reindex-partitions.spec b/src/test/isolation/specs/reindex-partitions.spec new file mode 100644 index 000000000000..1a4dd1bf2874 --- /dev/null +++ b/src/test/isolation/specs/reindex-partitions.spec @@ -0,0 +1,59 @@ +# REINDEX with partitioned tables +# +# Ensure that concurrent and non-concurrent operations work correctly when +# a REINDEX is performed on a partitioned table or index. + +# In the cases dealt with here, partition leaves are dropped in parallel of +# a REINDEX. DROP TABLE gets blocked by the first transaction of REINDEX +# building the list of partitions, so it will finish executing once REINDEX +# is done. + +setup +{ + CREATE TABLE reind_conc_parent (id int) PARTITION BY RANGE (id); + CREATE TABLE reind_conc_0_10 PARTITION OF reind_conc_parent + FOR VALUES FROM (0) TO (10); + CREATE TABLE reind_conc_10_20 PARTITION OF reind_conc_parent + FOR VALUES FROM (10) TO (20); + INSERT INTO reind_conc_parent VALUES (generate_series(0, 19)); +} + +teardown +{ + DROP TABLE reind_conc_parent; +} + +session "s1" +step "begin1" { BEGIN; } +step "lockexcl1" { LOCK reind_conc_parent IN ACCESS EXCLUSIVE MODE; } +step "lockshare1" { LOCK reind_conc_parent IN SHARE MODE; } +step "lockupdate1" { LOCK reind_conc_parent IN SHARE UPDATE EXCLUSIVE MODE; } +step "lockpartexcl1" { LOCK reind_conc_10_20 IN ACCESS EXCLUSIVE MODE; } +step "lockpartshare1" { LOCK reind_conc_10_20 IN SHARE MODE; } +step "lockpartupdate1" { LOCK reind_conc_10_20 IN SHARE UPDATE EXCLUSIVE MODE; } +step "end1" { COMMIT; } + +session "s2" +step "reindex2" { REINDEX TABLE reind_conc_parent; } +step "reindex_conc2" { REINDEX TABLE CONCURRENTLY reind_conc_parent; } + +session "s3" +step "drop3" { DROP TABLE reind_conc_10_20; } + +# An existing partition leaf is dropped after reindex is done when the +# parent is locked. +permutation "begin1" "lockexcl1" "reindex2" "drop3" "end1" +permutation "begin1" "lockexcl1" "reindex_conc2" "drop3" "end1" +permutation "begin1" "lockshare1" "reindex2" "drop3" "end1" +permutation "begin1" "lockshare1" "reindex_conc2" "drop3" "end1" +permutation "begin1" "lockupdate1" "reindex2" "drop3" "end1" +permutation "begin1" "lockupdate1" "reindex_conc2" "drop3" "end1" + +# An existing partition leaf is dropped after reindex is done when this +# leaf is locked. +permutation "begin1" "lockpartexcl1" "reindex2" "drop3" "end1" +permutation "begin1" "lockpartexcl1" "reindex_conc2" "drop3" "end1" +permutation "begin1" "lockpartshare1" "reindex2" "drop3" "end1" +permutation "begin1" "lockpartshare1" "reindex_conc2" "drop3" "end1" +permutation "begin1" "lockpartupdate1" "reindex2" "drop3" "end1" +permutation "begin1" "lockpartupdate1" "reindex_conc2" "drop3" "end1" diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 814416d936b0..64c0c668593e 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2194,18 +2194,6 @@ SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_ind concur_reindex_part_index_0_2 | concur_reindex_part_index_0 | 2 (5 rows) --- REINDEX fails for partitioned indexes -REINDEX INDEX concur_reindex_part_index_10; -ERROR: REINDEX is not yet implemented for partitioned indexes -REINDEX INDEX CONCURRENTLY concur_reindex_part_index_10; -ERROR: REINDEX is not yet implemented for partitioned indexes --- REINDEX is a no-op for partitioned tables -REINDEX TABLE concur_reindex_part_10; -WARNING: REINDEX of partitioned tables is not yet implemented, skipping "concur_reindex_part_10" -NOTICE: table "concur_reindex_part_10" has no indexes to reindex -REINDEX TABLE CONCURRENTLY concur_reindex_part_10; -WARNING: REINDEX of partitioned tables is not yet implemented, skipping "concur_reindex_part_10" -NOTICE: table "concur_reindex_part_10" has no indexes that can be reindexed concurrently SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_index') ORDER BY relid, level; relid | parentrelid | level @@ -2318,6 +2306,152 @@ SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_ind concur_reindex_part_index_0_2 | concur_reindex_part_index_0 | 2 (5 rows) +-- REINDEX for partitioned indexes +-- REINDEX TABLE fails for partitioned indexes +-- Top-most parent index +REINDEX TABLE concur_reindex_part_index; -- error +ERROR: "concur_reindex_part_index" is not a table or materialized view +REINDEX TABLE CONCURRENTLY concur_reindex_part_index; -- error +ERROR: "concur_reindex_part_index" is not a table or materialized view +-- Partitioned index with no leaves +REINDEX TABLE concur_reindex_part_index_10; -- error +ERROR: "concur_reindex_part_index_10" is not a table or materialized view +REINDEX TABLE CONCURRENTLY concur_reindex_part_index_10; -- error +ERROR: "concur_reindex_part_index_10" is not a table or materialized view +-- Cannot run in a transaction block +BEGIN; +REINDEX INDEX concur_reindex_part_index; +ERROR: REINDEX INDEX cannot run inside a transaction block +CONTEXT: while reindexing partitioned index "public.concur_reindex_part_index" +ROLLBACK; +-- Helper functions to track changes of relfilenodes in a partition tree. +-- Create a table tracking the relfilenode state. +CREATE OR REPLACE FUNCTION create_relfilenode_part(relname text, indname text) + RETURNS VOID AS + $func$ + BEGIN + EXECUTE format(' + CREATE TABLE %I AS + SELECT oid, relname, relfilenode, relkind, reltoastrelid + FROM pg_class + WHERE oid IN + (SELECT relid FROM pg_partition_tree(''%I''));', + relname, indname); + END + $func$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION compare_relfilenode_part(tabname text) + RETURNS TABLE (relname name, relkind "char", state text) AS + $func$ + BEGIN + RETURN QUERY EXECUTE + format( + 'SELECT b.relname, + b.relkind, + CASE WHEN a.relfilenode = b.relfilenode THEN ''relfilenode is unchanged'' + ELSE ''relfilenode has changed'' END + -- Do not join with OID here as CONCURRENTLY changes it. + FROM %I b JOIN pg_class a ON b.relname = a.relname + ORDER BY 1;', tabname); + END + $func$ LANGUAGE plpgsql; +-- Check that expected relfilenodes are changed, non-concurrent case. +SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); + create_relfilenode_part +------------------------- + +(1 row) + +REINDEX INDEX concur_reindex_part_index; +SELECT * FROM compare_relfilenode_part('reindex_index_status'); + relname | relkind | state +-------------------------------+---------+-------------------------- + concur_reindex_part_index | I | relfilenode is unchanged + concur_reindex_part_index_0 | I | relfilenode is unchanged + concur_reindex_part_index_0_1 | i | relfilenode has changed + concur_reindex_part_index_0_2 | i | relfilenode has changed + concur_reindex_part_index_10 | I | relfilenode is unchanged +(5 rows) + +DROP TABLE reindex_index_status; +-- concurrent case. +SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); + create_relfilenode_part +------------------------- + +(1 row) + +REINDEX INDEX CONCURRENTLY concur_reindex_part_index; +SELECT * FROM compare_relfilenode_part('reindex_index_status'); + relname | relkind | state +-------------------------------+---------+-------------------------- + concur_reindex_part_index | I | relfilenode is unchanged + concur_reindex_part_index_0 | I | relfilenode is unchanged + concur_reindex_part_index_0_1 | i | relfilenode has changed + concur_reindex_part_index_0_2 | i | relfilenode has changed + concur_reindex_part_index_10 | I | relfilenode is unchanged +(5 rows) + +DROP TABLE reindex_index_status; +-- REINDEX for partitioned tables +-- REINDEX INDEX fails for partitioned tables +-- Top-most parent +REINDEX INDEX concur_reindex_part; -- error +ERROR: "concur_reindex_part" is not an index +REINDEX INDEX CONCURRENTLY concur_reindex_part; -- error +ERROR: "concur_reindex_part" is not an index +-- Partitioned with no leaves +REINDEX INDEX concur_reindex_part_10; -- error +ERROR: "concur_reindex_part_10" is not an index +REINDEX INDEX CONCURRENTLY concur_reindex_part_10; -- error +ERROR: "concur_reindex_part_10" is not an index +-- Cannot run in a transaction block +BEGIN; +REINDEX TABLE concur_reindex_part; +ERROR: REINDEX TABLE cannot run inside a transaction block +CONTEXT: while reindexing partitioned table "public.concur_reindex_part" +ROLLBACK; +-- Check that expected relfilenodes are changed, non-concurrent case. +-- Note that the partition tree changes of the *indexes* need to be checked. +SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); + create_relfilenode_part +------------------------- + +(1 row) + +REINDEX TABLE concur_reindex_part; +SELECT * FROM compare_relfilenode_part('reindex_index_status'); + relname | relkind | state +-------------------------------+---------+-------------------------- + concur_reindex_part_index | I | relfilenode is unchanged + concur_reindex_part_index_0 | I | relfilenode is unchanged + concur_reindex_part_index_0_1 | i | relfilenode has changed + concur_reindex_part_index_0_2 | i | relfilenode has changed + concur_reindex_part_index_10 | I | relfilenode is unchanged +(5 rows) + +DROP TABLE reindex_index_status; +-- concurrent case. +SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); + create_relfilenode_part +------------------------- + +(1 row) + +REINDEX TABLE CONCURRENTLY concur_reindex_part; +SELECT * FROM compare_relfilenode_part('reindex_index_status'); + relname | relkind | state +-------------------------------+---------+-------------------------- + concur_reindex_part_index | I | relfilenode is unchanged + concur_reindex_part_index_0 | I | relfilenode is unchanged + concur_reindex_part_index_0_1 | i | relfilenode has changed + concur_reindex_part_index_0_2 | i | relfilenode has changed + concur_reindex_part_index_10 | I | relfilenode is unchanged +(5 rows) + +DROP TABLE reindex_index_status; +DROP FUNCTION create_relfilenode_part; +DROP FUNCTION compare_relfilenode_part; +-- Cleanup of partition tree used for REINDEX test. DROP TABLE concur_reindex_part; -- Check errors -- Cannot run inside a transaction block diff --git a/src/test/regress/sql/create_index.sql b/src/test/regress/sql/create_index.sql index f3667bacdc92..37f7259da9e6 100644 --- a/src/test/regress/sql/create_index.sql +++ b/src/test/regress/sql/create_index.sql @@ -903,12 +903,6 @@ CREATE INDEX concur_reindex_part_index_0_2 ON ONLY concur_reindex_part_0_2 (c1); ALTER INDEX concur_reindex_part_index_0 ATTACH PARTITION concur_reindex_part_index_0_2; SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_index') ORDER BY relid, level; --- REINDEX fails for partitioned indexes -REINDEX INDEX concur_reindex_part_index_10; -REINDEX INDEX CONCURRENTLY concur_reindex_part_index_10; --- REINDEX is a no-op for partitioned tables -REINDEX TABLE concur_reindex_part_10; -REINDEX TABLE CONCURRENTLY concur_reindex_part_10; SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_index') ORDER BY relid, level; -- REINDEX should preserve dependencies of partition tree. @@ -948,6 +942,88 @@ WHERE classid = 'pg_class'::regclass AND ORDER BY 1, 2; SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_index') ORDER BY relid, level; + +-- REINDEX for partitioned indexes +-- REINDEX TABLE fails for partitioned indexes +-- Top-most parent index +REINDEX TABLE concur_reindex_part_index; -- error +REINDEX TABLE CONCURRENTLY concur_reindex_part_index; -- error +-- Partitioned index with no leaves +REINDEX TABLE concur_reindex_part_index_10; -- error +REINDEX TABLE CONCURRENTLY concur_reindex_part_index_10; -- error +-- Cannot run in a transaction block +BEGIN; +REINDEX INDEX concur_reindex_part_index; +ROLLBACK; +-- Helper functions to track changes of relfilenodes in a partition tree. +-- Create a table tracking the relfilenode state. +CREATE OR REPLACE FUNCTION create_relfilenode_part(relname text, indname text) + RETURNS VOID AS + $func$ + BEGIN + EXECUTE format(' + CREATE TABLE %I AS + SELECT oid, relname, relfilenode, relkind, reltoastrelid + FROM pg_class + WHERE oid IN + (SELECT relid FROM pg_partition_tree(''%I''));', + relname, indname); + END + $func$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION compare_relfilenode_part(tabname text) + RETURNS TABLE (relname name, relkind "char", state text) AS + $func$ + BEGIN + RETURN QUERY EXECUTE + format( + 'SELECT b.relname, + b.relkind, + CASE WHEN a.relfilenode = b.relfilenode THEN ''relfilenode is unchanged'' + ELSE ''relfilenode has changed'' END + -- Do not join with OID here as CONCURRENTLY changes it. + FROM %I b JOIN pg_class a ON b.relname = a.relname + ORDER BY 1;', tabname); + END + $func$ LANGUAGE plpgsql; +-- Check that expected relfilenodes are changed, non-concurrent case. +SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); +REINDEX INDEX concur_reindex_part_index; +SELECT * FROM compare_relfilenode_part('reindex_index_status'); +DROP TABLE reindex_index_status; +-- concurrent case. +SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); +REINDEX INDEX CONCURRENTLY concur_reindex_part_index; +SELECT * FROM compare_relfilenode_part('reindex_index_status'); +DROP TABLE reindex_index_status; + +-- REINDEX for partitioned tables +-- REINDEX INDEX fails for partitioned tables +-- Top-most parent +REINDEX INDEX concur_reindex_part; -- error +REINDEX INDEX CONCURRENTLY concur_reindex_part; -- error +-- Partitioned with no leaves +REINDEX INDEX concur_reindex_part_10; -- error +REINDEX INDEX CONCURRENTLY concur_reindex_part_10; -- error +-- Cannot run in a transaction block +BEGIN; +REINDEX TABLE concur_reindex_part; +ROLLBACK; +-- Check that expected relfilenodes are changed, non-concurrent case. +-- Note that the partition tree changes of the *indexes* need to be checked. +SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); +REINDEX TABLE concur_reindex_part; +SELECT * FROM compare_relfilenode_part('reindex_index_status'); +DROP TABLE reindex_index_status; +-- concurrent case. +SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); +REINDEX TABLE CONCURRENTLY concur_reindex_part; +SELECT * FROM compare_relfilenode_part('reindex_index_status'); +DROP TABLE reindex_index_status; + +DROP FUNCTION create_relfilenode_part; +DROP FUNCTION compare_relfilenode_part; + +-- Cleanup of partition tree used for REINDEX test. DROP TABLE concur_reindex_part; -- Check errors diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 391c204f7219..f151e13d7f1c 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2038,6 +2038,7 @@ RegProcedure Regis RegisNode RegisteredBgWorker +ReindexErrorInfo ReindexObjectType ReindexStmt ReindexType From 6785974287afa27221a49ae7202ea3d0695c0a84 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 8 Sep 2020 11:15:21 +0900 Subject: [PATCH 104/589] Remove isolation test reindex-partitions The isolation test added by a6642b3 is proving to be unstable, as once the first transaction holding a lock on the top-most partitioned table or on a partition commits, the commit order of the follow-up DROP TABLE and REINDEX could become reversed depending on the timing. The only part of the test that could be entirely reliable is the one using a SHARE lock, allowing REINDEX to commit first, but it is the least interesting of the set. Per buildfarm members rorqual and mylodon. Discussion: https://postgr.es/m/E1kFSBj-00062c-Mu@gemulon.postgresql.org --- .../isolation/expected/reindex-partitions.out | 107 ------------------ src/test/isolation/isolation_schedule | 1 - .../isolation/specs/reindex-partitions.spec | 59 ---------- 3 files changed, 167 deletions(-) delete mode 100644 src/test/isolation/expected/reindex-partitions.out delete mode 100644 src/test/isolation/specs/reindex-partitions.spec diff --git a/src/test/isolation/expected/reindex-partitions.out b/src/test/isolation/expected/reindex-partitions.out deleted file mode 100644 index 5373abde259b..000000000000 --- a/src/test/isolation/expected/reindex-partitions.out +++ /dev/null @@ -1,107 +0,0 @@ -Parsed test spec with 3 sessions - -starting permutation: begin1 lockexcl1 reindex2 drop3 end1 -step begin1: BEGIN; -step lockexcl1: LOCK reind_conc_parent IN ACCESS EXCLUSIVE MODE; -step reindex2: REINDEX TABLE reind_conc_parent; -step drop3: DROP TABLE reind_conc_10_20; -step end1: COMMIT; -step reindex2: <... completed> -step drop3: <... completed> - -starting permutation: begin1 lockexcl1 reindex_conc2 drop3 end1 -step begin1: BEGIN; -step lockexcl1: LOCK reind_conc_parent IN ACCESS EXCLUSIVE MODE; -step reindex_conc2: REINDEX TABLE CONCURRENTLY reind_conc_parent; -step drop3: DROP TABLE reind_conc_10_20; -step end1: COMMIT; -step reindex_conc2: <... completed> -step drop3: <... completed> - -starting permutation: begin1 lockshare1 reindex2 drop3 end1 -step begin1: BEGIN; -step lockshare1: LOCK reind_conc_parent IN SHARE MODE; -step reindex2: REINDEX TABLE reind_conc_parent; -step drop3: DROP TABLE reind_conc_10_20; -step end1: COMMIT; -step drop3: <... completed> - -starting permutation: begin1 lockshare1 reindex_conc2 drop3 end1 -step begin1: BEGIN; -step lockshare1: LOCK reind_conc_parent IN SHARE MODE; -step reindex_conc2: REINDEX TABLE CONCURRENTLY reind_conc_parent; -step drop3: DROP TABLE reind_conc_10_20; -step end1: COMMIT; -step reindex_conc2: <... completed> -step drop3: <... completed> - -starting permutation: begin1 lockupdate1 reindex2 drop3 end1 -step begin1: BEGIN; -step lockupdate1: LOCK reind_conc_parent IN SHARE UPDATE EXCLUSIVE MODE; -step reindex2: REINDEX TABLE reind_conc_parent; -step drop3: DROP TABLE reind_conc_10_20; -step end1: COMMIT; -step reindex2: <... completed> -step drop3: <... completed> - -starting permutation: begin1 lockupdate1 reindex_conc2 drop3 end1 -step begin1: BEGIN; -step lockupdate1: LOCK reind_conc_parent IN SHARE UPDATE EXCLUSIVE MODE; -step reindex_conc2: REINDEX TABLE CONCURRENTLY reind_conc_parent; -step drop3: DROP TABLE reind_conc_10_20; -step end1: COMMIT; -step reindex_conc2: <... completed> -step drop3: <... completed> - -starting permutation: begin1 lockpartexcl1 reindex2 drop3 end1 -step begin1: BEGIN; -step lockpartexcl1: LOCK reind_conc_10_20 IN ACCESS EXCLUSIVE MODE; -step reindex2: REINDEX TABLE reind_conc_parent; -step drop3: DROP TABLE reind_conc_10_20; -step end1: COMMIT; -step reindex2: <... completed> -step drop3: <... completed> - -starting permutation: begin1 lockpartexcl1 reindex_conc2 drop3 end1 -step begin1: BEGIN; -step lockpartexcl1: LOCK reind_conc_10_20 IN ACCESS EXCLUSIVE MODE; -step reindex_conc2: REINDEX TABLE CONCURRENTLY reind_conc_parent; -step drop3: DROP TABLE reind_conc_10_20; -step end1: COMMIT; -step reindex_conc2: <... completed> -step drop3: <... completed> - -starting permutation: begin1 lockpartshare1 reindex2 drop3 end1 -step begin1: BEGIN; -step lockpartshare1: LOCK reind_conc_10_20 IN SHARE MODE; -step reindex2: REINDEX TABLE reind_conc_parent; -step drop3: DROP TABLE reind_conc_10_20; -step end1: COMMIT; -step drop3: <... completed> - -starting permutation: begin1 lockpartshare1 reindex_conc2 drop3 end1 -step begin1: BEGIN; -step lockpartshare1: LOCK reind_conc_10_20 IN SHARE MODE; -step reindex_conc2: REINDEX TABLE CONCURRENTLY reind_conc_parent; -step drop3: DROP TABLE reind_conc_10_20; -step end1: COMMIT; -step reindex_conc2: <... completed> -step drop3: <... completed> - -starting permutation: begin1 lockpartupdate1 reindex2 drop3 end1 -step begin1: BEGIN; -step lockpartupdate1: LOCK reind_conc_10_20 IN SHARE UPDATE EXCLUSIVE MODE; -step reindex2: REINDEX TABLE reind_conc_parent; -step drop3: DROP TABLE reind_conc_10_20; -step end1: COMMIT; -step reindex2: <... completed> -step drop3: <... completed> - -starting permutation: begin1 lockpartupdate1 reindex_conc2 drop3 end1 -step begin1: BEGIN; -step lockpartupdate1: LOCK reind_conc_10_20 IN SHARE UPDATE EXCLUSIVE MODE; -step reindex_conc2: REINDEX TABLE CONCURRENTLY reind_conc_parent; -step drop3: DROP TABLE reind_conc_10_20; -step end1: COMMIT; -step reindex_conc2: <... completed> -step drop3: <... completed> diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index 65d1443ac684..6acbb695ecea 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -50,7 +50,6 @@ test: lock-committed-keyupdate test: update-locked-tuple test: reindex-concurrently test: reindex-schema -test: reindex-partitions test: propagate-lock-delete test: tuplelock-conflict test: tuplelock-update diff --git a/src/test/isolation/specs/reindex-partitions.spec b/src/test/isolation/specs/reindex-partitions.spec deleted file mode 100644 index 1a4dd1bf2874..000000000000 --- a/src/test/isolation/specs/reindex-partitions.spec +++ /dev/null @@ -1,59 +0,0 @@ -# REINDEX with partitioned tables -# -# Ensure that concurrent and non-concurrent operations work correctly when -# a REINDEX is performed on a partitioned table or index. - -# In the cases dealt with here, partition leaves are dropped in parallel of -# a REINDEX. DROP TABLE gets blocked by the first transaction of REINDEX -# building the list of partitions, so it will finish executing once REINDEX -# is done. - -setup -{ - CREATE TABLE reind_conc_parent (id int) PARTITION BY RANGE (id); - CREATE TABLE reind_conc_0_10 PARTITION OF reind_conc_parent - FOR VALUES FROM (0) TO (10); - CREATE TABLE reind_conc_10_20 PARTITION OF reind_conc_parent - FOR VALUES FROM (10) TO (20); - INSERT INTO reind_conc_parent VALUES (generate_series(0, 19)); -} - -teardown -{ - DROP TABLE reind_conc_parent; -} - -session "s1" -step "begin1" { BEGIN; } -step "lockexcl1" { LOCK reind_conc_parent IN ACCESS EXCLUSIVE MODE; } -step "lockshare1" { LOCK reind_conc_parent IN SHARE MODE; } -step "lockupdate1" { LOCK reind_conc_parent IN SHARE UPDATE EXCLUSIVE MODE; } -step "lockpartexcl1" { LOCK reind_conc_10_20 IN ACCESS EXCLUSIVE MODE; } -step "lockpartshare1" { LOCK reind_conc_10_20 IN SHARE MODE; } -step "lockpartupdate1" { LOCK reind_conc_10_20 IN SHARE UPDATE EXCLUSIVE MODE; } -step "end1" { COMMIT; } - -session "s2" -step "reindex2" { REINDEX TABLE reind_conc_parent; } -step "reindex_conc2" { REINDEX TABLE CONCURRENTLY reind_conc_parent; } - -session "s3" -step "drop3" { DROP TABLE reind_conc_10_20; } - -# An existing partition leaf is dropped after reindex is done when the -# parent is locked. -permutation "begin1" "lockexcl1" "reindex2" "drop3" "end1" -permutation "begin1" "lockexcl1" "reindex_conc2" "drop3" "end1" -permutation "begin1" "lockshare1" "reindex2" "drop3" "end1" -permutation "begin1" "lockshare1" "reindex_conc2" "drop3" "end1" -permutation "begin1" "lockupdate1" "reindex2" "drop3" "end1" -permutation "begin1" "lockupdate1" "reindex_conc2" "drop3" "end1" - -# An existing partition leaf is dropped after reindex is done when this -# leaf is locked. -permutation "begin1" "lockpartexcl1" "reindex2" "drop3" "end1" -permutation "begin1" "lockpartexcl1" "reindex_conc2" "drop3" "end1" -permutation "begin1" "lockpartshare1" "reindex2" "drop3" "end1" -permutation "begin1" "lockpartshare1" "reindex_conc2" "drop3" "end1" -permutation "begin1" "lockpartupdate1" "reindex2" "drop3" "end1" -permutation "begin1" "lockpartupdate1" "reindex_conc2" "drop3" "end1" From 1dec091d5b0bfad1a9b061cac820236e98cbd72c Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 8 Sep 2020 10:08:46 +0200 Subject: [PATCH 105/589] Remove unused parameter unused since f0d6f20278b7c5c412ce40a9b86c6b31dc2fbfdd Discussion: https://www.postgresql.org/message-id/flat/511bb100-f829-ba21-2f10-9f952ec06ead%402ndquadrant.com --- contrib/pg_stat_statements/pg_stat_statements.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index 6b91c62c31a8..1eac9edaee72 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -376,7 +376,7 @@ static void JumbleRowMarks(pgssJumbleState *jstate, List *rowMarks); static void JumbleExpr(pgssJumbleState *jstate, Node *node); static void RecordConstLocation(pgssJumbleState *jstate, int location); static char *generate_normalized_query(pgssJumbleState *jstate, const char *query, - int query_loc, int *query_len_p, int encoding); + int query_loc, int *query_len_p); static void fill_in_constant_lengths(pgssJumbleState *jstate, const char *query, int query_loc); static int comp_location(const void *a, const void *b); @@ -1336,8 +1336,7 @@ pgss_store(const char *query, uint64 queryId, LWLockRelease(pgss->lock); norm_query = generate_normalized_query(jstate, query, query_location, - &query_len, - encoding); + &query_len); LWLockAcquire(pgss->lock, LW_SHARED); } @@ -3235,7 +3234,7 @@ RecordConstLocation(pgssJumbleState *jstate, int location) */ static char * generate_normalized_query(pgssJumbleState *jstate, const char *query, - int query_loc, int *query_len_p, int encoding) + int query_loc, int *query_len_p) { char *norm_query; int query_len = *query_len_p; From 728d4bc16b79dea0dd6d540aa5c55a84ac722a09 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 8 Sep 2020 17:11:16 +0200 Subject: [PATCH 106/589] Use for name of unnamed portal's memory context Otherwise just printing an empty string makes the memory context debug output slightly confusing. Discussion: https://www.postgresql.org/message-id/flat/ccb353ef-89ff-09b3-8046-1d2514624b9c%402ndquadrant.com --- src/backend/utils/mmgr/portalmem.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/utils/mmgr/portalmem.c b/src/backend/utils/mmgr/portalmem.c index 7072ce48a3ec..ec6f80ee99b2 100644 --- a/src/backend/utils/mmgr/portalmem.c +++ b/src/backend/utils/mmgr/portalmem.c @@ -220,8 +220,8 @@ CreatePortal(const char *name, bool allowDup, bool dupSilent) /* put portal in table (sets portal->name) */ PortalHashTableInsert(portal, name); - /* reuse portal->name copy */ - MemoryContextSetIdentifier(portal->portalContext, portal->name); + /* for named portals reuse portal->name copy */ + MemoryContextSetIdentifier(portal->portalContext, portal->name[0] ? portal->name : ""); return portal; } From 3438c988fd757c74ea4ddd80cdbb7405f4a1bc39 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 8 Sep 2020 11:47:37 -0400 Subject: [PATCH 107/589] Use plain memset() in numeric.c, not MemSet and friends. This essentially reverts a micro-optimization I made years ago, as part of the much larger commit d72f6c750. It's doubtful that there was any hard evidence for it being helpful even then, and the case is even more dubious now that modern compilers are so much smarter about inlining memset(). The proximate reason for undoing it is to get rid of the type punning inherent in MemSet, for fear that that may cause problems now that we're applying additional optimization switches to numeric.c. At the very least this'll silence some warnings from a few old buildfarm animals. (It's probably past time for another look at whether MemSet is still worth anything at all, but I do not propose to tackle that question right now.) Discussion: https://postgr.es/m/CAJ3gD9evtA_vBo+WMYMyT-u=keHX7-r8p2w7OSRfXf42LTwCZQ@mail.gmail.com --- src/backend/utils/adt/numeric.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c index dfd455fc7488..69d313dd52b1 100644 --- a/src/backend/utils/adt/numeric.c +++ b/src/backend/utils/adt/numeric.c @@ -492,7 +492,7 @@ static void dump_var(const char *str, NumericVar *var); pfree(buf); \ } while (0) -#define init_var(v) MemSetAligned(v, 0, sizeof(NumericVar)) +#define init_var(v) memset(v, 0, sizeof(NumericVar)) #define NUMERIC_DIGITS(num) (NUMERIC_HEADER_IS_SHORT(num) ? \ (num)->choice.n_short.n_data : (num)->choice.n_long.n_data) From 5871f09c98588acd486a31d7bb7bd084a6553a39 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Tue, 8 Sep 2020 11:25:34 -0700 Subject: [PATCH 108/589] Fix autovacuum cancellation. The problem is caused by me (Andres) having ProcSleep() look at the wrong PGPROC entry in 5788e258bb2. Unfortunately it seems hard to write a reliable test for autovacuum cancellations. Perhaps somebody will come up with a good approach, but it seems worth fixing the issue even without a test. Reported-By: Jeff Janes Author: Jeff Janes Discussion: https://postgr.es/m/CAMkU=1wH2aUy+wDRDz+5RZALdcUnEofV1t9PzXS_gBJO9vZZ0Q@mail.gmail.com --- src/backend/storage/lmgr/proc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c index aa9fbd80545b..19a9f9394921 100644 --- a/src/backend/storage/lmgr/proc.c +++ b/src/backend/storage/lmgr/proc.c @@ -1318,7 +1318,7 @@ ProcSleep(LOCALLOCK *locallock, LockMethod lockMethodTable) * Only do it if the worker is not working to protect against Xid * wraparound. */ - vacuumFlags = ProcGlobal->vacuumFlags[proc->pgxactoff]; + vacuumFlags = ProcGlobal->vacuumFlags[autovac->pgxactoff]; if ((vacuumFlags & PROC_IS_AUTOVACUUM) && !(vacuumFlags & PROC_VACUUM_FOR_WRAPAROUND)) { From c9ae5cbb8834dd440cdfa7058c03443f98436284 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 8 Sep 2020 15:54:25 -0400 Subject: [PATCH 109/589] Install an error check into cancel_before_shmem_exit(). Historically, cancel_before_shmem_exit() just silently did nothing if the specified callback wasn't the top-of-stack. The folly of ignoring this case was exposed by the bugs fixed in 303640199 and bab150045, so let's make it throw elog(ERROR) instead. There is a decent argument to be made that PG_ENSURE_ERROR_CLEANUP should use some separate infrastructure, so it wouldn't break if something inside the guarded code decides to register a new before_shmem_exit callback. However, a survey of the surviving uses of before_shmem_exit() and PG_ENSURE_ERROR_CLEANUP doesn't show any plausible conflicts of that sort today, so for now we'll forgo the extra complexity. (It will almost certainly become necessary if anyone ever wants to wrap PG_ENSURE_ERROR_CLEANUP around arbitrary user-defined actions, though.) No backpatch, since this is developer support not a production issue. Bharath Rupireddy, per advice from Andres Freund, Robert Haas, and myself Discussion: https://postgr.es/m/CALj2ACWk7j4F2v2fxxYfrroOF=AdFNPr1WsV+AGtHAFQOqm_pw@mail.gmail.com --- src/backend/storage/ipc/ipc.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/backend/storage/ipc/ipc.c b/src/backend/storage/ipc/ipc.c index bdbc2c3ac4bc..11c3f132a100 100644 --- a/src/backend/storage/ipc/ipc.c +++ b/src/backend/storage/ipc/ipc.c @@ -381,9 +381,9 @@ on_shmem_exit(pg_on_exit_callback function, Datum arg) * cancel_before_shmem_exit * * this function removes a previously-registered before_shmem_exit - * callback. For simplicity, only the latest entry can be - * removed. (We could work harder but there is no need for - * current uses.) + * callback. We only look at the latest entry for removal, as we + * expect callers to add and remove temporary before_shmem_exit + * callbacks in strict LIFO order. * ---------------------------------------------------------------- */ void @@ -394,6 +394,9 @@ cancel_before_shmem_exit(pg_on_exit_callback function, Datum arg) == function && before_shmem_exit_list[before_shmem_exit_index - 1].arg == arg) --before_shmem_exit_index; + else + elog(ERROR, "before_shmem_exit callback (%p,0x%llx) is not the latest entry", + function, (long long) arg); } /* ---------------------------------------------------------------- From f481d2823297e692e9d44c9edfd18e626e768b0c Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Tue, 8 Sep 2020 19:35:15 -0300 Subject: [PATCH 110/589] Check default partitions constraints while descending MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Partitioning tuple route code assumes that the partition chosen while descending the partition hierarchy is always the correct one. This is true except when the partition is the default partition and another partition has been added concurrently: the partition constraint changes and we don't recheck it. This can lead to tuples mistakenly being added to the default partition that should have been rejected. Fix by rechecking the default partition constraint while descending the hierarchy. An isolation test based on the reproduction steps described by Hao Wu (with tweaks for extra coverage) is included. Backpatch to 12, where this bug came in with 898e5e3290a7. Reported by: Hao Wu Author: Amit Langote Author: Álvaro Herrera Discussion: https://postgr.es/m/CA+HiwqFqBmcSSap4sFnCBUEL_VfOMmEKaQ3gwUhyfa4c7J_-nA@mail.gmail.com Discussion: https://postgr.es/m/DM5PR0501MB3910E97A9EDFB4C775CF3D75A42F0@DM5PR0501MB3910.namprd05.prod.outlook.com --- src/backend/executor/execPartition.c | 127 ++++++++++++++---- .../expected/partition-concurrent-attach.out | 49 +++++++ src/test/isolation/isolation_schedule | 1 + .../specs/partition-concurrent-attach.spec | 43 ++++++ 4 files changed, 195 insertions(+), 25 deletions(-) create mode 100644 src/test/isolation/expected/partition-concurrent-attach.out create mode 100644 src/test/isolation/specs/partition-concurrent-attach.spec diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 79fcbd6b0665..bd2ea2580475 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -51,6 +51,11 @@ * PartitionDispatchData->indexes for details on how this array is * indexed. * + * nonleaf_partitions + * Array of 'max_dispatch' elements containing pointers to fake + * ResultRelInfo objects for nonleaf partitions, useful for checking + * the partition constraint. + * * num_dispatch * The current number of items stored in the 'partition_dispatch_info' * array. Also serves as the index of the next free array element for @@ -89,6 +94,7 @@ struct PartitionTupleRouting { Relation partition_root; PartitionDispatch *partition_dispatch_info; + ResultRelInfo **nonleaf_partitions; int num_dispatch; int max_dispatch; ResultRelInfo **partitions; @@ -280,9 +286,11 @@ ExecFindPartition(ModifyTableState *mtstate, PartitionDispatch dispatch; PartitionDesc partdesc; ExprContext *ecxt = GetPerTupleExprContext(estate); - TupleTableSlot *ecxt_scantuple_old = ecxt->ecxt_scantuple; + TupleTableSlot *ecxt_scantuple_saved = ecxt->ecxt_scantuple; + TupleTableSlot *rootslot = slot; TupleTableSlot *myslot = NULL; MemoryContext oldcxt; + ResultRelInfo *rri = NULL; /* use per-tuple context here to avoid leaking memory */ oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate)); @@ -296,9 +304,8 @@ ExecFindPartition(ModifyTableState *mtstate, /* start with the root partitioned table */ dispatch = pd[0]; - while (true) + while (dispatch != NULL) { - AttrMap *map = dispatch->tupmap; int partidx = -1; CHECK_FOR_INTERRUPTS(); @@ -306,17 +313,6 @@ ExecFindPartition(ModifyTableState *mtstate, rel = dispatch->reldesc; partdesc = dispatch->partdesc; - /* - * Convert the tuple to this parent's layout, if different from the - * current relation. - */ - myslot = dispatch->tupslot; - if (myslot != NULL) - { - Assert(map != NULL); - slot = execute_attr_map_slot(map, slot, myslot); - } - /* * Extract partition key from tuple. Expression evaluation machinery * that FormPartitionKeyDatum() invokes expects ecxt_scantuple to @@ -352,11 +348,9 @@ ExecFindPartition(ModifyTableState *mtstate, if (partdesc->is_leaf[partidx]) { - ResultRelInfo *rri; - /* - * Look to see if we've already got a ResultRelInfo for this - * partition. + * We've reached the leaf -- hurray, we're done. Look to see if + * we've already got a ResultRelInfo for this partition. */ if (likely(dispatch->indexes[partidx] >= 0)) { @@ -400,14 +394,10 @@ ExecFindPartition(ModifyTableState *mtstate, dispatch, rootResultRelInfo, partidx); } + Assert(rri != NULL); - /* Release the tuple in the lowest parent's dedicated slot. */ - if (slot == myslot) - ExecClearTuple(myslot); - - MemoryContextSwitchTo(oldcxt); - ecxt->ecxt_scantuple = ecxt_scantuple_old; - return rri; + /* Signal to terminate the loop */ + dispatch = NULL; } else { @@ -419,6 +409,8 @@ ExecFindPartition(ModifyTableState *mtstate, /* Already built. */ Assert(dispatch->indexes[partidx] < proute->num_dispatch); + rri = proute->nonleaf_partitions[dispatch->indexes[partidx]]; + /* * Move down to the next partition level and search again * until we find a leaf partition that matches this tuple @@ -440,10 +432,75 @@ ExecFindPartition(ModifyTableState *mtstate, dispatch, partidx); Assert(dispatch->indexes[partidx] >= 0 && dispatch->indexes[partidx] < proute->num_dispatch); + + rri = proute->nonleaf_partitions[dispatch->indexes[partidx]]; dispatch = subdispatch; } + + /* + * Convert the tuple to the new parent's layout, if different from + * the previous parent. + */ + if (dispatch->tupslot) + { + AttrMap *map = dispatch->tupmap; + TupleTableSlot *tempslot = myslot; + + myslot = dispatch->tupslot; + slot = execute_attr_map_slot(map, slot, myslot); + + if (tempslot != NULL) + ExecClearTuple(tempslot); + } + } + + /* + * If this partition is the default one, we must check its partition + * constraint now, which may have changed concurrently due to + * partitions being added to the parent. + * + * (We do this here, and do not rely on ExecInsert doing it, because + * we don't want to miss doing it for non-leaf partitions.) + */ + if (partidx == partdesc->boundinfo->default_index) + { + PartitionRoutingInfo *partrouteinfo = rri->ri_PartitionInfo; + + /* + * The tuple must match the partition's layout for the constraint + * expression to be evaluated successfully. If the partition is + * sub-partitioned, that would already be the case due to the code + * above, but for a leaf partition the tuple still matches the + * parent's layout. + * + * Note that we have a map to convert from root to current + * partition, but not from immediate parent to current partition. + * So if we have to convert, do it from the root slot; if not, use + * the root slot as-is. + */ + if (partrouteinfo) + { + TupleConversionMap *map = partrouteinfo->pi_RootToPartitionMap; + + if (map) + slot = execute_attr_map_slot(map->attrMap, rootslot, + partrouteinfo->pi_PartitionTupleSlot); + else + slot = rootslot; + } + + ExecPartitionCheck(rri, slot, estate, true); } } + + /* Release the tuple in the lowest parent's dedicated slot. */ + if (myslot != NULL) + ExecClearTuple(myslot); + /* and restore ecxt's scantuple */ + ecxt->ecxt_scantuple = ecxt_scantuple_saved; + MemoryContextSwitchTo(oldcxt); + + return rri; } /* @@ -1060,6 +1117,8 @@ ExecInitPartitionDispatchInfo(EState *estate, proute->max_dispatch = 4; proute->partition_dispatch_info = (PartitionDispatch *) palloc(sizeof(PartitionDispatch) * proute->max_dispatch); + proute->nonleaf_partitions = (ResultRelInfo **) + palloc(sizeof(ResultRelInfo *) * proute->max_dispatch); } else { @@ -1067,10 +1126,28 @@ ExecInitPartitionDispatchInfo(EState *estate, proute->partition_dispatch_info = (PartitionDispatch *) repalloc(proute->partition_dispatch_info, sizeof(PartitionDispatch) * proute->max_dispatch); + proute->nonleaf_partitions = (ResultRelInfo **) + repalloc(proute->nonleaf_partitions, + sizeof(ResultRelInfo *) * proute->max_dispatch); } } proute->partition_dispatch_info[dispatchidx] = pd; + /* + * If setting up a PartitionDispatch for a sub-partitioned table, we may + * also need a minimally valid ResultRelInfo for checking the partition + * constraint later; set that up now. + */ + if (parent_pd) + { + ResultRelInfo *rri = makeNode(ResultRelInfo); + + InitResultRelInfo(rri, rel, 1, proute->partition_root, 0); + proute->nonleaf_partitions[dispatchidx] = rri; + } + else + proute->nonleaf_partitions[dispatchidx] = NULL; + /* * Finally, if setting up a PartitionDispatch for a sub-partitioned table, * install a downlink in the parent to allow quick descent. diff --git a/src/test/isolation/expected/partition-concurrent-attach.out b/src/test/isolation/expected/partition-concurrent-attach.out new file mode 100644 index 000000000000..17fac3998982 --- /dev/null +++ b/src/test/isolation/expected/partition-concurrent-attach.out @@ -0,0 +1,49 @@ +Parsed test spec with 2 sessions + +starting permutation: s1b s1a s2b s2i s1c s2c s2s +step s1b: begin; +step s1a: alter table tpart attach partition tpart_2 for values from (100) to (200); +step s2b: begin; +step s2i: insert into tpart values (110,'xxx'), (120, 'yyy'), (150, 'zzz'); +step s1c: commit; +step s2i: <... completed> +error in steps s1c s2i: ERROR: new row for relation "tpart_default" violates partition constraint +step s2c: commit; +step s2s: select tableoid::regclass, * from tpart; +tableoid i j + +tpart_2 110 xxx +tpart_2 120 yyy +tpart_2 150 zzz + +starting permutation: s1b s1a s2b s2i2 s1c s2c s2s +step s1b: begin; +step s1a: alter table tpart attach partition tpart_2 for values from (100) to (200); +step s2b: begin; +step s2i2: insert into tpart_default (i, j) values (110, 'xxx'), (120, 'yyy'), (150, 'zzz'); +step s1c: commit; +step s2i2: <... completed> +error in steps s1c s2i2: ERROR: new row for relation "tpart_default" violates partition constraint +step s2c: commit; +step s2s: select tableoid::regclass, * from tpart; +tableoid i j + +tpart_2 110 xxx +tpart_2 120 yyy +tpart_2 150 zzz + +starting permutation: s1b s2b s2i s1a s2c s1c s2s +step s1b: begin; +step s2b: begin; +step s2i: insert into tpart values (110,'xxx'), (120, 'yyy'), (150, 'zzz'); +step s1a: alter table tpart attach partition tpart_2 for values from (100) to (200); +step s2c: commit; +step s1a: <... completed> +error in steps s2c s1a: ERROR: updated partition constraint for default partition "tpart_default_default" would be violated by some row +step s1c: commit; +step s2s: select tableoid::regclass, * from tpart; +tableoid i j + +tpart_default_default110 xxx +tpart_default_default120 yyy +tpart_default_default150 zzz diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index 6acbb695ecea..aa386ab1a25b 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -81,6 +81,7 @@ test: vacuum-skip-locked test: predicate-hash test: predicate-gist test: predicate-gin +test: partition-concurrent-attach test: partition-key-update-1 test: partition-key-update-2 test: partition-key-update-3 diff --git a/src/test/isolation/specs/partition-concurrent-attach.spec b/src/test/isolation/specs/partition-concurrent-attach.spec new file mode 100644 index 000000000000..48c3f83e0c8b --- /dev/null +++ b/src/test/isolation/specs/partition-concurrent-attach.spec @@ -0,0 +1,43 @@ +# Verify that default partition constraint is enforced correctly +# in light of partitions being added concurrently to its parent +setup { + drop table if exists tpart; + create table tpart(i int, j text) partition by range(i); + create table tpart_1(like tpart); + create table tpart_2(like tpart); + create table tpart_default (a int, j text, i int) partition by list (j); + create table tpart_default_default (a int, i int, b int, j text); + alter table tpart_default_default drop b; + alter table tpart_default attach partition tpart_default_default default; + alter table tpart_default drop a; + alter table tpart attach partition tpart_default default; + alter table tpart attach partition tpart_1 for values from(0) to (100); + insert into tpart_2 values (110,'xxx'), (120, 'yyy'), (150, 'zzz'); +} + +session "s1" +step "s1b" { begin; } +step "s1a" { alter table tpart attach partition tpart_2 for values from (100) to (200); } +step "s1c" { commit; } + +session "s2" +step "s2b" { begin; } +step "s2i" { insert into tpart values (110,'xxx'), (120, 'yyy'), (150, 'zzz'); } +step "s2i2" { insert into tpart_default (i, j) values (110, 'xxx'), (120, 'yyy'), (150, 'zzz'); } +step "s2c" { commit; } +step "s2s" { select tableoid::regclass, * from tpart; } + +teardown { drop table tpart; } + +# insert into tpart by s2 which routes to tpart_default due to not seeing +# concurrently added tpart_2 should fail, because the partition constraint +# of tpart_default would have changed due to tpart_2 having been added +permutation "s1b" "s1a" "s2b" "s2i" "s1c" "s2c" "s2s" + +# similar to above, but now insert into sub-partitioned tpart_default +permutation "s1b" "s1a" "s2b" "s2i2" "s1c" "s2c" "s2s" + +# reverse: now the insert into tpart_default by s2 occurs first followed by +# attach in s1, which should fail when it scans the leaf default partition +# find the violating rows +permutation "s1b" "s2b" "s2i" "s1a" "s2c" "s1c" "s2s" From fe2bf8f81096ea1b7b2fc58501309fc5e41ccd47 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 9 Sep 2020 09:58:12 +0200 Subject: [PATCH 111/589] Add some more numeric test coverage max(numeric) wasn't tested at all, min(numeric) was only used by some unrelated tests. Add explicit tests with the other numeric aggregate functions. --- src/test/regress/expected/numeric.out | 12 ++++++++++++ src/test/regress/sql/numeric.sql | 2 ++ 2 files changed, 14 insertions(+) diff --git a/src/test/regress/expected/numeric.out b/src/test/regress/expected/numeric.out index 8546ce901fa7..86940ec68386 100644 --- a/src/test/regress/expected/numeric.out +++ b/src/test/regress/expected/numeric.out @@ -1096,6 +1096,18 @@ SELECT AVG(val) FROM num_data; -13430913.592242320700 (1 row) +SELECT MAX(val) FROM num_data; + max +-------------------- + 7799461.4119000000 +(1 row) + +SELECT MIN(val) FROM num_data; + min +---------------------- + -83028485.0000000000 +(1 row) + SELECT STDDEV(val) FROM num_data; stddev ------------------------------- diff --git a/src/test/regress/sql/numeric.sql b/src/test/regress/sql/numeric.sql index 416c16722a9c..febb096af23b 100644 --- a/src/test/regress/sql/numeric.sql +++ b/src/test/regress/sql/numeric.sql @@ -752,6 +752,8 @@ SELECT power('-inf'::numeric, '-inf'); -- ****************************** -- numeric AVG used to fail on some platforms SELECT AVG(val) FROM num_data; +SELECT MAX(val) FROM num_data; +SELECT MIN(val) FROM num_data; SELECT STDDEV(val) FROM num_data; SELECT VARIANCE(val) FROM num_data; From 60df530c571a8a5fbae82dc82e4931bb6325ddc0 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Wed, 9 Sep 2020 12:20:53 +0200 Subject: [PATCH 112/589] Add missing quote in docs Mistake in commit 68b603e1a9. Reported-by: Ian Barwick --- doc/src/sgml/file-fdw.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/file-fdw.sgml b/doc/src/sgml/file-fdw.sgml index 29d79832a33c..eefc6e7e5b19 100644 --- a/doc/src/sgml/file-fdw.sgml +++ b/doc/src/sgml/file-fdw.sgml @@ -267,7 +267,7 @@ CREATE FOREIGN TABLE pglog ( application_name text, backend_type text ) SERVER pglog -OPTIONS ( filename log/pglog.csv', format 'csv' ); +OPTIONS ( filename 'log/pglog.csv', format 'csv' ); From f3e1e6619655bf219131d93ec85348cad75c7f12 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 9 Sep 2020 11:53:39 -0400 Subject: [PATCH 113/589] Minor fixes in docs and error messages. Alexander Lakhin Discussion: https://postgr.es/m/ce7debdd-c943-d7a7-9b41-687107b27831@gmail.com --- doc/src/sgml/bgworker.sgml | 20 ++++++++++++-------- doc/src/sgml/btree.sgml | 2 +- doc/src/sgml/intarray.sgml | 2 +- src/backend/replication/backup_manifest.c | 2 +- src/backend/utils/misc/guc.c | 2 +- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/doc/src/sgml/bgworker.sgml b/doc/src/sgml/bgworker.sgml index 6e1cf121de08..7fd673ab54ee 100644 --- a/doc/src/sgml/bgworker.sgml +++ b/doc/src/sgml/bgworker.sgml @@ -34,14 +34,18 @@ PostgreSQL is started by including the module name in shared_preload_libraries. A module wishing to run a background worker can register it by calling - RegisterBackgroundWorker(BackgroundWorker *worker) - from its _PG_init(). Background workers can also be started - after the system is up and running by calling the function - RegisterDynamicBackgroundWorker(BackgroundWorker - *worker, BackgroundWorkerHandle **handle). Unlike - RegisterBackgroundWorker, which can only be called from within - the postmaster, RegisterDynamicBackgroundWorker must be - called from a regular backend or another background worker. + RegisterBackgroundWorker(BackgroundWorker + *worker) + from its _PG_init() function. + Background workers can also be started + after the system is up and running by calling + RegisterDynamicBackgroundWorker(BackgroundWorker + *worker, BackgroundWorkerHandle + **handle). Unlike + RegisterBackgroundWorker, which can only be called from + within the postmaster process, + RegisterDynamicBackgroundWorker must be called + from a regular backend or another background worker. diff --git a/doc/src/sgml/btree.sgml b/doc/src/sgml/btree.sgml index 435b7cb24da9..20cabe921fb3 100644 --- a/doc/src/sgml/btree.sgml +++ b/doc/src/sgml/btree.sgml @@ -263,7 +263,7 @@ - inrange + in_range in_range support functions diff --git a/doc/src/sgml/intarray.sgml b/doc/src/sgml/intarray.sgml index c8db87e97df9..af44c7b2142d 100644 --- a/doc/src/sgml/intarray.sgml +++ b/doc/src/sgml/intarray.sgml @@ -453,7 +453,7 @@ -- a message can be in one or more sections CREATE TABLE message (mid INT PRIMARY KEY, sections INT[], ...); --- create specialized index with sigature length of 32 bytes +-- create specialized index with signature length of 32 bytes CREATE INDEX message_rdtree_idx ON message USING GIST (sections gist__int_ops(siglen=32)); -- select messages in section 1 OR 2 - OVERLAP operator diff --git a/src/backend/replication/backup_manifest.c b/src/backend/replication/backup_manifest.c index b6260049271b..a43c793e2892 100644 --- a/src/backend/replication/backup_manifest.c +++ b/src/backend/replication/backup_manifest.c @@ -272,7 +272,7 @@ AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr, */ if (!found_start_timeline) ereport(ERROR, - errmsg("start timeline %u not found history of timeline %u", + errmsg("start timeline %u not found in history of timeline %u", starttli, endtli)); /* Terminate the list of WAL ranges. */ diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index de87ad6ef702..73518d986621 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -3140,7 +3140,7 @@ static struct config_int ConfigureNamesInt[] = }, { {"autovacuum_vacuum_insert_threshold", PGC_SIGHUP, AUTOVACUUM, - gettext_noop("Minimum number of tuple inserts prior to vacuum, or -1 to disable insert vacuums"), + gettext_noop("Minimum number of tuple inserts prior to vacuum, or -1 to disable insert vacuums."), NULL }, &autovacuum_vac_ins_thresh, From a273dcc6fd34c8b6aa290fafa45e516ccd8d907d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 9 Sep 2020 12:00:49 -0400 Subject: [PATCH 114/589] Doc: adjust documentation related to index support functions. Commit 15cb2bd27 neglected to make the running text match the tables, leaving the reader with the strong impression that we cannot count. Also, don't drop an unrelated para between a table and the para describing it. --- doc/src/sgml/xindex.sgml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/doc/src/sgml/xindex.sgml b/doc/src/sgml/xindex.sgml index 2cfd71b5b77a..609fa35d4cad 100644 --- a/doc/src/sgml/xindex.sgml +++ b/doc/src/sgml/xindex.sgml @@ -400,22 +400,22 @@ specified by the index method. + + Additionally, some opclasses allow users to specify parameters which + control their behavior. Each builtin index access method has an optional + options support function, which defines a set of + opclass-specific parameters. + + B-trees require a comparison support function, - and allow three additional support functions to be + and allow four additional support functions to be supplied at the operator class author's option, as shown in . The requirements for these support functions are explained further in . - - Additionally, some opclasses allow users to specify parameters which - control their behavior. Each builtin index access method has an optional - options support function, which defines a set of - opclass-specific parameters. - - B-Tree Support Functions @@ -469,8 +469,8 @@
- Hash indexes require one support function, and allow a second one to be - supplied at the operator class author's option, as shown in . @@ -511,7 +511,7 @@
- GiST indexes have nine support functions, two of which are optional, + GiST indexes have ten support functions, three of which are optional, as shown in . (For more information see .) @@ -594,7 +594,7 @@ - SP-GiST indexes require five support functions, as + SP-GiST indexes have six support functions, one of which is optional, as shown in . (For more information see .) @@ -653,7 +653,7 @@ - GIN indexes have six support functions, three of which are optional, + GIN indexes have seven support functions, four of which are optional, as shown in . (For more information see .) @@ -730,9 +730,9 @@ - BRIN indexes have four basic support functions, as shown in - ; those basic functions - may require additional support functions to be provided. + BRIN indexes have five basic support functions, one of which is optional, + as shown in . Some versions of + the basic functions require additional support functions to be provided. (For more information see .) From 0aa8f764088ea0f36620ae2955fa6c54ec736c46 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 9 Sep 2020 20:16:28 +0200 Subject: [PATCH 115/589] Expose internal function for converting int64 to numeric Existing callers had to take complicated detours via DirectFunctionCall1(). This simplifies a lot of code. Reviewed-by: Tom Lane Discussion: https://www.postgresql.org/message-id/flat/42b73d2d-da12-ba9f-570a-420e0cce19d9@phystech.edu --- contrib/btree_gist/btree_numeric.c | 2 +- contrib/jsonb_plperl/jsonb_plperl.c | 4 +- src/backend/utils/adt/cash.c | 7 +- src/backend/utils/adt/dbsize.c | 21 ++--- src/backend/utils/adt/formatting.c | 19 ++--- src/backend/utils/adt/jsonpath_exec.c | 11 +-- src/backend/utils/adt/numeric.c | 116 +++++++------------------- src/include/utils/numeric.h | 2 + 8 files changed, 50 insertions(+), 132 deletions(-) diff --git a/contrib/btree_gist/btree_numeric.c b/contrib/btree_gist/btree_numeric.c index d66901680e33..35e466cdd942 100644 --- a/contrib/btree_gist/btree_numeric.c +++ b/contrib/btree_gist/btree_numeric.c @@ -195,7 +195,7 @@ gbt_numeric_penalty(PG_FUNCTION_ARGS) } else { - Numeric nul = DatumGetNumeric(DirectFunctionCall1(int4_numeric, Int32GetDatum(0))); + Numeric nul = int64_to_numeric(0); *result = 0.0; diff --git a/contrib/jsonb_plperl/jsonb_plperl.c b/contrib/jsonb_plperl/jsonb_plperl.c index b81ba54b809d..22e90afe1b6e 100644 --- a/contrib/jsonb_plperl/jsonb_plperl.c +++ b/contrib/jsonb_plperl/jsonb_plperl.c @@ -216,9 +216,7 @@ SV_to_JsonbValue(SV *in, JsonbParseState **jsonb_state, bool is_elem) IV ival = SvIV(in); out.type = jbvNumeric; - out.val.numeric = - DatumGetNumeric(DirectFunctionCall1(int8_numeric, - Int64GetDatum((int64) ival))); + out.val.numeric = int64_to_numeric(ival); } else if (SvNOK(in)) { diff --git a/src/backend/utils/adt/cash.c b/src/backend/utils/adt/cash.c index 6515fc8ec695..d093ce80386f 100644 --- a/src/backend/utils/adt/cash.c +++ b/src/backend/utils/adt/cash.c @@ -1042,7 +1042,7 @@ cash_numeric(PG_FUNCTION_ARGS) fpoint = 2; /* convert the integral money value to numeric */ - result = DirectFunctionCall1(int8_numeric, Int64GetDatum(money)); + result = NumericGetDatum(int64_to_numeric(money)); /* scale appropriately, if needed */ if (fpoint > 0) @@ -1056,8 +1056,7 @@ cash_numeric(PG_FUNCTION_ARGS) scale = 1; for (i = 0; i < fpoint; i++) scale *= 10; - numeric_scale = DirectFunctionCall1(int8_numeric, - Int64GetDatum(scale)); + numeric_scale = NumericGetDatum(int64_to_numeric(scale)); /* * Given integral inputs approaching INT64_MAX, select_div_scale() @@ -1107,7 +1106,7 @@ numeric_cash(PG_FUNCTION_ARGS) scale *= 10; /* multiply the input amount by scale factor */ - numeric_scale = DirectFunctionCall1(int8_numeric, Int64GetDatum(scale)); + numeric_scale = NumericGetDatum(int64_to_numeric(scale)); amount = DirectFunctionCall2(numeric_mul, amount, numeric_scale); /* note that numeric_int8 will round to nearest integer for us */ diff --git a/src/backend/utils/adt/dbsize.c b/src/backend/utils/adt/dbsize.c index 2320c06a9bc7..7def7392b955 100644 --- a/src/backend/utils/adt/dbsize.c +++ b/src/backend/utils/adt/dbsize.c @@ -579,14 +579,6 @@ numeric_to_cstring(Numeric n) return DatumGetCString(DirectFunctionCall1(numeric_out, d)); } -static Numeric -int64_to_numeric(int64 v) -{ - Datum d = Int64GetDatum(v); - - return DatumGetNumeric(DirectFunctionCall1(int8_numeric, d)); -} - static bool numeric_is_less(Numeric a, Numeric b) { @@ -615,9 +607,9 @@ numeric_half_rounded(Numeric n) Datum two; Datum result; - zero = DirectFunctionCall1(int8_numeric, Int64GetDatum(0)); - one = DirectFunctionCall1(int8_numeric, Int64GetDatum(1)); - two = DirectFunctionCall1(int8_numeric, Int64GetDatum(2)); + zero = NumericGetDatum(int64_to_numeric(0)); + one = NumericGetDatum(int64_to_numeric(1)); + two = NumericGetDatum(int64_to_numeric(2)); if (DatumGetBool(DirectFunctionCall2(numeric_ge, d, zero))) d = DirectFunctionCall2(numeric_add, d, one); @@ -632,12 +624,10 @@ static Numeric numeric_shift_right(Numeric n, unsigned count) { Datum d = NumericGetDatum(n); - Datum divisor_int64; Datum divisor_numeric; Datum result; - divisor_int64 = Int64GetDatum((int64) (1 << count)); - divisor_numeric = DirectFunctionCall1(int8_numeric, divisor_int64); + divisor_numeric = NumericGetDatum(int64_to_numeric(1 << count)); result = DirectFunctionCall2(numeric_div_trunc, d, divisor_numeric); return DatumGetNumeric(result); } @@ -832,8 +822,7 @@ pg_size_bytes(PG_FUNCTION_ARGS) { Numeric mul_num; - mul_num = DatumGetNumeric(DirectFunctionCall1(int8_numeric, - Int64GetDatum(multiplier))); + mul_num = int64_to_numeric(multiplier); num = DatumGetNumeric(DirectFunctionCall2(numeric_mul, NumericGetDatum(mul_num), diff --git a/src/backend/utils/adt/formatting.c b/src/backend/utils/adt/formatting.c index 7d09537d82b9..f9aa968f0985 100644 --- a/src/backend/utils/adt/formatting.c +++ b/src/backend/utils/adt/formatting.c @@ -6070,10 +6070,8 @@ numeric_to_number(PG_FUNCTION_ARGS) if (IS_MULTI(&Num)) { Numeric x; - Numeric a = DatumGetNumeric(DirectFunctionCall1(int4_numeric, - Int32GetDatum(10))); - Numeric b = DatumGetNumeric(DirectFunctionCall1(int4_numeric, - Int32GetDatum(-Num.multi))); + Numeric a = int64_to_numeric(10); + Numeric b = int64_to_numeric(-Num.multi); x = DatumGetNumeric(DirectFunctionCall2(numeric_power, NumericGetDatum(a), @@ -6162,10 +6160,8 @@ numeric_to_char(PG_FUNCTION_ARGS) if (IS_MULTI(&Num)) { - Numeric a = DatumGetNumeric(DirectFunctionCall1(int4_numeric, - Int32GetDatum(10))); - Numeric b = DatumGetNumeric(DirectFunctionCall1(int4_numeric, - Int32GetDatum(Num.multi))); + Numeric a = int64_to_numeric(10); + Numeric b = int64_to_numeric(Num.multi); x = DatumGetNumeric(DirectFunctionCall2(numeric_power, NumericGetDatum(a), @@ -6339,11 +6335,8 @@ int8_to_char(PG_FUNCTION_ARGS) else if (IS_EEEE(&Num)) { /* to avoid loss of precision, must go via numeric not float8 */ - Numeric val; - - val = DatumGetNumeric(DirectFunctionCall1(int8_numeric, - Int64GetDatum(value))); - orgnum = numeric_out_sci(val, Num.post); + orgnum = numeric_out_sci(int64_to_numeric(value), + Num.post); /* * numeric_out_sci() does not emit a sign for positive numbers. We diff --git a/src/backend/utils/adt/jsonpath_exec.c b/src/backend/utils/adt/jsonpath_exec.c index f146767bfc3a..7403c760b486 100644 --- a/src/backend/utils/adt/jsonpath_exec.c +++ b/src/backend/utils/adt/jsonpath_exec.c @@ -842,9 +842,7 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, lastjbv = hasNext ? &tmpjbv : palloc(sizeof(*lastjbv)); lastjbv->type = jbvNumeric; - lastjbv->val.numeric = - DatumGetNumeric(DirectFunctionCall1(int4_numeric, - Int32GetDatum(last))); + lastjbv->val.numeric = int64_to_numeric(last); res = executeNextItem(cxt, jsp, &elem, lastjbv, found, hasNext); @@ -1012,9 +1010,7 @@ executeItemOptUnwrapTarget(JsonPathExecContext *cxt, JsonPathItem *jsp, jb = palloc(sizeof(*jb)); jb->type = jbvNumeric; - jb->val.numeric = - DatumGetNumeric(DirectFunctionCall1(int4_numeric, - Int32GetDatum(size))); + jb->val.numeric = int64_to_numeric(size); res = executeNextItem(cxt, jsp, NULL, jb, found, false); } @@ -1979,8 +1975,7 @@ executeKeyValueMethod(JsonPathExecContext *cxt, JsonPathItem *jsp, id += (int64) cxt->baseObject.id * INT64CONST(10000000000); idval.type = jbvNumeric; - idval.val.numeric = DatumGetNumeric(DirectFunctionCall1(int8_numeric, - Int64GetDatum(id))); + idval.val.numeric = int64_to_numeric(id); it = JsonbIteratorInit(jbc); diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c index 69d313dd52b1..27d65557dfa9 100644 --- a/src/backend/utils/adt/numeric.c +++ b/src/backend/utils/adt/numeric.c @@ -4073,23 +4073,29 @@ numeric_trim_scale(PG_FUNCTION_ARGS) * ---------------------------------------------------------------------- */ - -Datum -int4_numeric(PG_FUNCTION_ARGS) +Numeric +int64_to_numeric(int64 val) { - int32 val = PG_GETARG_INT32(0); Numeric res; NumericVar result; init_var(&result); - int64_to_numericvar((int64) val, &result); + int64_to_numericvar(val, &result); res = make_result(&result); free_var(&result); - PG_RETURN_NUMERIC(res); + return res; +} + +Datum +int4_numeric(PG_FUNCTION_ARGS) +{ + int32 val = PG_GETARG_INT32(0); + + PG_RETURN_NUMERIC(int64_to_numeric(val)); } int32 @@ -4174,18 +4180,8 @@ Datum int8_numeric(PG_FUNCTION_ARGS) { int64 val = PG_GETARG_INT64(0); - Numeric res; - NumericVar result; - init_var(&result); - - int64_to_numericvar(val, &result); - - res = make_result(&result); - - free_var(&result); - - PG_RETURN_NUMERIC(res); + PG_RETURN_NUMERIC(int64_to_numeric(val)); } @@ -4224,18 +4220,8 @@ Datum int2_numeric(PG_FUNCTION_ARGS) { int16 val = PG_GETARG_INT16(0); - Numeric res; - NumericVar result; - - init_var(&result); - - int64_to_numericvar((int64) val, &result); - - res = make_result(&result); - free_var(&result); - - PG_RETURN_NUMERIC(res); + PG_RETURN_NUMERIC(int64_to_numeric(val)); } @@ -5290,11 +5276,7 @@ int2_accum(PG_FUNCTION_ARGS) #ifdef HAVE_INT128 do_int128_accum(state, (int128) PG_GETARG_INT16(1)); #else - Numeric newval; - - newval = DatumGetNumeric(DirectFunctionCall1(int2_numeric, - PG_GETARG_DATUM(1))); - do_numeric_accum(state, newval); + do_numeric_accum(state, int64_to_numeric(PG_GETARG_INT16(1))); #endif } @@ -5317,11 +5299,7 @@ int4_accum(PG_FUNCTION_ARGS) #ifdef HAVE_INT128 do_int128_accum(state, (int128) PG_GETARG_INT32(1)); #else - Numeric newval; - - newval = DatumGetNumeric(DirectFunctionCall1(int4_numeric, - PG_GETARG_DATUM(1))); - do_numeric_accum(state, newval); + do_numeric_accum(state, int64_to_numeric(PG_GETARG_INT32(1))); #endif } @@ -5340,13 +5318,7 @@ int8_accum(PG_FUNCTION_ARGS) state = makeNumericAggState(fcinfo, true); if (!PG_ARGISNULL(1)) - { - Numeric newval; - - newval = DatumGetNumeric(DirectFunctionCall1(int8_numeric, - PG_GETARG_DATUM(1))); - do_numeric_accum(state, newval); - } + do_numeric_accum(state, int64_to_numeric(PG_GETARG_INT64(1))); PG_RETURN_POINTER(state); } @@ -5570,11 +5542,7 @@ int8_avg_accum(PG_FUNCTION_ARGS) #ifdef HAVE_INT128 do_int128_accum(state, (int128) PG_GETARG_INT64(1)); #else - Numeric newval; - - newval = DatumGetNumeric(DirectFunctionCall1(int8_numeric, - PG_GETARG_DATUM(1))); - do_numeric_accum(state, newval); + do_numeric_accum(state, int64_to_numeric(PG_GETARG_INT64(1))); #endif } @@ -5767,13 +5735,8 @@ int2_accum_inv(PG_FUNCTION_ARGS) #ifdef HAVE_INT128 do_int128_discard(state, (int128) PG_GETARG_INT16(1)); #else - Numeric newval; - - newval = DatumGetNumeric(DirectFunctionCall1(int2_numeric, - PG_GETARG_DATUM(1))); - /* Should never fail, all inputs have dscale 0 */ - if (!do_numeric_discard(state, newval)) + if (!do_numeric_discard(state, int64_to_numeric(PG_GETARG_INT16(1)))) elog(ERROR, "do_numeric_discard failed unexpectedly"); #endif } @@ -5797,13 +5760,8 @@ int4_accum_inv(PG_FUNCTION_ARGS) #ifdef HAVE_INT128 do_int128_discard(state, (int128) PG_GETARG_INT32(1)); #else - Numeric newval; - - newval = DatumGetNumeric(DirectFunctionCall1(int4_numeric, - PG_GETARG_DATUM(1))); - /* Should never fail, all inputs have dscale 0 */ - if (!do_numeric_discard(state, newval)) + if (!do_numeric_discard(state, int64_to_numeric(PG_GETARG_INT32(1)))) elog(ERROR, "do_numeric_discard failed unexpectedly"); #endif } @@ -5824,13 +5782,8 @@ int8_accum_inv(PG_FUNCTION_ARGS) if (!PG_ARGISNULL(1)) { - Numeric newval; - - newval = DatumGetNumeric(DirectFunctionCall1(int8_numeric, - PG_GETARG_DATUM(1))); - /* Should never fail, all inputs have dscale 0 */ - if (!do_numeric_discard(state, newval)) + if (!do_numeric_discard(state, int64_to_numeric(PG_GETARG_INT64(1)))) elog(ERROR, "do_numeric_discard failed unexpectedly"); } @@ -5853,13 +5806,8 @@ int8_avg_accum_inv(PG_FUNCTION_ARGS) #ifdef HAVE_INT128 do_int128_discard(state, (int128) PG_GETARG_INT64(1)); #else - Numeric newval; - - newval = DatumGetNumeric(DirectFunctionCall1(int8_numeric, - PG_GETARG_DATUM(1))); - /* Should never fail, all inputs have dscale 0 */ - if (!do_numeric_discard(state, newval)) + if (!do_numeric_discard(state, int64_to_numeric(PG_GETARG_INT64(1)))) elog(ERROR, "do_numeric_discard failed unexpectedly"); #endif } @@ -5914,8 +5862,7 @@ numeric_poly_avg(PG_FUNCTION_ARGS) int128_to_numericvar(state->sumX, &result); - countd = DirectFunctionCall1(int8_numeric, - Int64GetDatumFast(state->N)); + countd = NumericGetDatum(int64_to_numeric(state->N)); sumd = NumericGetDatum(make_result(&result)); free_var(&result); @@ -5951,7 +5898,7 @@ numeric_avg(PG_FUNCTION_ARGS) if (state->nInfcount > 0) PG_RETURN_NUMERIC(make_result(&const_ninf)); - N_datum = DirectFunctionCall1(int8_numeric, Int64GetDatum(state->N)); + N_datum = NumericGetDatum(int64_to_numeric(state->N)); init_var(&sumX_var); accum_sum_final(&state->sumX, &sumX_var); @@ -6411,7 +6358,6 @@ Datum int8_sum(PG_FUNCTION_ARGS) { Numeric oldsum; - Datum newval; if (PG_ARGISNULL(0)) { @@ -6419,8 +6365,7 @@ int8_sum(PG_FUNCTION_ARGS) if (PG_ARGISNULL(1)) PG_RETURN_NULL(); /* still no non-null */ /* This is the first non-null input. */ - newval = DirectFunctionCall1(int8_numeric, PG_GETARG_DATUM(1)); - PG_RETURN_DATUM(newval); + PG_RETURN_NUMERIC(int64_to_numeric(PG_GETARG_INT64(1))); } /* @@ -6436,10 +6381,9 @@ int8_sum(PG_FUNCTION_ARGS) PG_RETURN_NUMERIC(oldsum); /* OK to do the addition. */ - newval = DirectFunctionCall1(int8_numeric, PG_GETARG_DATUM(1)); - PG_RETURN_DATUM(DirectFunctionCall2(numeric_add, - NumericGetDatum(oldsum), newval)); + NumericGetDatum(oldsum), + NumericGetDatum(int64_to_numeric(PG_GETARG_INT64(1))))); } @@ -6618,10 +6562,8 @@ int8_avg(PG_FUNCTION_ARGS) if (transdata->count == 0) PG_RETURN_NULL(); - countd = DirectFunctionCall1(int8_numeric, - Int64GetDatumFast(transdata->count)); - sumd = DirectFunctionCall1(int8_numeric, - Int64GetDatumFast(transdata->sum)); + countd = NumericGetDatum(int64_to_numeric(transdata->count)); + sumd = NumericGetDatum(int64_to_numeric(transdata->sum)); PG_RETURN_DATUM(DirectFunctionCall2(numeric_div, sumd, countd)); } diff --git a/src/include/utils/numeric.h b/src/include/utils/numeric.h index 0b7d4ba3c4bc..2a768b9a04ad 100644 --- a/src/include/utils/numeric.h +++ b/src/include/utils/numeric.h @@ -62,6 +62,8 @@ int32 numeric_maximum_size(int32 typmod); extern char *numeric_out_sci(Numeric num, int scale); extern char *numeric_normalize(Numeric num); +extern Numeric int64_to_numeric(int64 val); + extern Numeric numeric_add_opt_error(Numeric num1, Numeric num2, bool *have_error); extern Numeric numeric_sub_opt_error(Numeric num1, Numeric num2, From bedadc73220f7b09f29a4741dccd143a21a08dda Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 9 Sep 2020 15:32:34 -0400 Subject: [PATCH 116/589] Make archiver's SIGQUIT handler exit via _exit(). Commit 8e19a8264 changed the SIGQUIT handlers of almost all server processes not to run atexit callbacks. The archiver process was skipped, perhaps because it's not connected to shared memory; but it's just as true here that running atexit callbacks in a signal handler is unsafe. So let's make it work like the rest. In HEAD and v13, we can use the common SignalHandlerForCrashExit handler. Before that, just tweak pgarch_exit to use _exit(2) explicitly. Like the previous commit, back-patch to all supported branches. Kyotaro Horiguchi, back-patching by me Discussion: https://postgr.es/m/1850884.1599601164@sss.pgh.pa.us --- src/backend/postmaster/pgarch.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/backend/postmaster/pgarch.c b/src/backend/postmaster/pgarch.c index 01ffd6513c7c..37be0e2bbbe7 100644 --- a/src/backend/postmaster/pgarch.c +++ b/src/backend/postmaster/pgarch.c @@ -96,7 +96,6 @@ static pid_t pgarch_forkexec(void); #endif NON_EXEC_STATIC void PgArchiverMain(int argc, char *argv[]) pg_attribute_noreturn(); -static void pgarch_exit(SIGNAL_ARGS); static void pgarch_waken(SIGNAL_ARGS); static void pgarch_waken_stop(SIGNAL_ARGS); static void pgarch_MainLoop(void); @@ -229,7 +228,7 @@ PgArchiverMain(int argc, char *argv[]) pqsignal(SIGHUP, SignalHandlerForConfigReload); pqsignal(SIGINT, SIG_IGN); pqsignal(SIGTERM, SignalHandlerForShutdownRequest); - pqsignal(SIGQUIT, pgarch_exit); + pqsignal(SIGQUIT, SignalHandlerForCrashExit); pqsignal(SIGALRM, SIG_IGN); pqsignal(SIGPIPE, SIG_IGN); pqsignal(SIGUSR1, pgarch_waken); @@ -246,14 +245,6 @@ PgArchiverMain(int argc, char *argv[]) exit(0); } -/* SIGQUIT signal handler for archiver process */ -static void -pgarch_exit(SIGNAL_ARGS) -{ - /* SIGQUIT means curl up and die ... */ - exit(1); -} - /* SIGUSR1 signal handler for archiver process */ static void pgarch_waken(SIGNAL_ARGS) From fe4d022c8e171ba3a9165bd55fa6b2ca3a40fa4e Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Wed, 9 Sep 2020 18:50:24 -0700 Subject: [PATCH 117/589] Fix rd_firstRelfilenodeSubid for nailed relations, in parallel workers. Move applicable code out of RelationBuildDesc(), which nailed relations bypass. Non-assert builds experienced no known problems. Back-patch to v13, where commit c6b92041d38512a4176ed76ad06f713d2e6c01a8 introduced rd_firstRelfilenodeSubid. Kyotaro Horiguchi. Reported by Justin Pryzby. Discussion: https://postgr.es/m/20200907023737.GA7158@telsasoft.com --- src/backend/utils/cache/relcache.c | 23 ++++++++++++------- src/test/regress/expected/reindex_catalog.out | 10 ++++++++ src/test/regress/sql/reindex_catalog.sql | 11 +++++++++ 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 96ecad02ddb1..9061af81a3e3 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -1243,14 +1243,6 @@ RelationBuildDesc(Oid targetRelId, bool insertIt) if (insertIt) RelationCacheInsert(relation, true); - /* - * For RelationNeedsWAL() to answer correctly on parallel workers, restore - * rd_firstRelfilenodeSubid. No subtransactions start or end while in - * parallel mode, so the specific SubTransactionId does not matter. - */ - if (IsParallelWorker() && RelFileNodeSkippingWAL(relation->rd_node)) - relation->rd_firstRelfilenodeSubid = TopSubTransactionId; - /* It's fully valid */ relation->rd_isvalid = true; @@ -1273,6 +1265,8 @@ RelationBuildDesc(Oid targetRelId, bool insertIt) static void RelationInitPhysicalAddr(Relation relation) { + Oid oldnode = relation->rd_node.relNode; + /* these relations kinds never have storage */ if (!RELKIND_HAS_STORAGE(relation->rd_rel->relkind)) return; @@ -1330,6 +1324,19 @@ RelationInitPhysicalAddr(Relation relation) elog(ERROR, "could not find relation mapping for relation \"%s\", OID %u", RelationGetRelationName(relation), relation->rd_id); } + + /* + * For RelationNeedsWAL() to answer correctly on parallel workers, restore + * rd_firstRelfilenodeSubid. No subtransactions start or end while in + * parallel mode, so the specific SubTransactionId does not matter. + */ + if (IsParallelWorker() && oldnode != relation->rd_node.relNode) + { + if (RelFileNodeSkippingWAL(relation->rd_node)) + relation->rd_firstRelfilenodeSubid = TopSubTransactionId; + else + relation->rd_firstRelfilenodeSubid = InvalidSubTransactionId; + } } /* diff --git a/src/test/regress/expected/reindex_catalog.out b/src/test/regress/expected/reindex_catalog.out index 4b5fba494939..204f056c9a56 100644 --- a/src/test/regress/expected/reindex_catalog.out +++ b/src/test/regress/expected/reindex_catalog.out @@ -36,3 +36,13 @@ REINDEX INDEX pg_index_indexrelid_index; -- non-mapped, non-shared, critical REINDEX INDEX pg_index_indrelid_index; -- non-mapped, non-shared, non-critical REINDEX INDEX pg_database_oid_index; -- mapped, shared, critical REINDEX INDEX pg_shdescription_o_c_index; -- mapped, shared, non-critical +-- Check the same REINDEX INDEX statements under parallelism. +BEGIN; +SET min_parallel_table_scan_size = 0; +REINDEX INDEX pg_class_oid_index; -- mapped, non-shared, critical +REINDEX INDEX pg_class_relname_nsp_index; -- mapped, non-shared, non-critical +REINDEX INDEX pg_index_indexrelid_index; -- non-mapped, non-shared, critical +REINDEX INDEX pg_index_indrelid_index; -- non-mapped, non-shared, non-critical +REINDEX INDEX pg_database_oid_index; -- mapped, shared, critical +REINDEX INDEX pg_shdescription_o_c_index; -- mapped, shared, non-critical +ROLLBACK; diff --git a/src/test/regress/sql/reindex_catalog.sql b/src/test/regress/sql/reindex_catalog.sql index 87ecf52244f1..8203641cf9d2 100644 --- a/src/test/regress/sql/reindex_catalog.sql +++ b/src/test/regress/sql/reindex_catalog.sql @@ -39,3 +39,14 @@ REINDEX INDEX pg_index_indexrelid_index; -- non-mapped, non-shared, critical REINDEX INDEX pg_index_indrelid_index; -- non-mapped, non-shared, non-critical REINDEX INDEX pg_database_oid_index; -- mapped, shared, critical REINDEX INDEX pg_shdescription_o_c_index; -- mapped, shared, non-critical + +-- Check the same REINDEX INDEX statements under parallelism. +BEGIN; +SET min_parallel_table_scan_size = 0; +REINDEX INDEX pg_class_oid_index; -- mapped, non-shared, critical +REINDEX INDEX pg_class_relname_nsp_index; -- mapped, non-shared, non-critical +REINDEX INDEX pg_index_indexrelid_index; -- non-mapped, non-shared, critical +REINDEX INDEX pg_index_indrelid_index; -- non-mapped, non-shared, non-critical +REINDEX INDEX pg_database_oid_index; -- mapped, shared, critical +REINDEX INDEX pg_shdescription_o_c_index; -- mapped, shared, non-critical +ROLLBACK; From aad546bd0a51059aafe44e8c4a783afa1557d1d9 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 10 Sep 2020 15:50:19 +0900 Subject: [PATCH 118/589] doc: Fix some grammar and inconsistencies Some comments are fixed while on it. Author: Justin Pryzby Discussion: https://postgr.es/m/20200818171702.GK17022@telsasoft.com Backpatch-through: 9.6 --- doc/src/sgml/logicaldecoding.sgml | 2 +- doc/src/sgml/ref/alter_table.sgml | 2 +- doc/src/sgml/ref/create_subscription.sgml | 2 +- doc/src/sgml/ref/pg_verifybackup.sgml | 7 ++++--- doc/src/sgml/sources.sgml | 2 +- src/backend/replication/logical/relation.c | 4 ++-- src/backend/storage/lmgr/proc.c | 2 +- 7 files changed, 11 insertions(+), 10 deletions(-) diff --git a/doc/src/sgml/logicaldecoding.sgml b/doc/src/sgml/logicaldecoding.sgml index 8d4fdf670064..813a037facec 100644 --- a/doc/src/sgml/logicaldecoding.sgml +++ b/doc/src/sgml/logicaldecoding.sgml @@ -223,7 +223,7 @@ $ pg_recvlogical -d postgres --slot=test --drop-slot A logical slot will emit each change just once in normal operation. The current position of each slot is persisted only at checkpoint, so in the case of a crash the slot may return to an earlier LSN, which will - then cause recent changes to be resent when the server restarts. + then cause recent changes to be sent again when the server restarts. Logical decoding clients are responsible for avoiding ill effects from handling the same message more than once. Clients may wish to record the last LSN they saw when decoding and skip over any repeated data or diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index c1576cc69628..e9c6a8a6c15c 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -889,7 +889,7 @@ WITH ( MODULUS numeric_literal, REM from the parent table will be created in the partition, if they don't already exist. If any of the CHECK constraints of the table being - attached is marked NO INHERIT, the command will fail; + attached are marked NO INHERIT, the command will fail; such constraints must be recreated without the NO INHERIT clause. diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index b7d7457d004e..e812beee3738 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -160,7 +160,7 @@ CREATE SUBSCRIPTION subscription_name It is safe to use off for logical replication: If the subscriber loses transactions because of missing - synchronization, the data will be resent from the publisher. + synchronization, the data will be sent again from the publisher. diff --git a/doc/src/sgml/ref/pg_verifybackup.sgml b/doc/src/sgml/ref/pg_verifybackup.sgml index c160992e6d7d..a0989d3cd165 100644 --- a/doc/src/sgml/ref/pg_verifybackup.sgml +++ b/doc/src/sgml/ref/pg_verifybackup.sgml @@ -82,8 +82,8 @@ PostgreSQL documentation for any files for which the computed checksum does not match the checksum stored in the manifest. This step is not performed for any files which produced errors in the previous step, since they are already known - to have problems. Also, files which were ignored in the previous step are - also ignored in this step. + to have problems. Files which were ignored in the previous step are also + ignored in this step. @@ -121,7 +121,8 @@ PostgreSQL documentation Options - The following command-line options control the behavior. + pg_verifybackup accepts the following + command-line arguments: diff --git a/doc/src/sgml/sources.sgml b/doc/src/sgml/sources.sgml index 998e7d5fba19..d4f73a03c3a7 100644 --- a/doc/src/sgml/sources.sgml +++ b/doc/src/sgml/sources.sgml @@ -373,7 +373,7 @@ ereport(ERROR, specify suppression of the CONTEXT: portion of a message in the postmaster log. This should only be used for verbose debugging messages where the repeated inclusion of context would bloat the log - volume too much. + too much. diff --git a/src/backend/replication/logical/relation.c b/src/backend/replication/logical/relation.c index a60c73d74d5b..3d2d56295b0c 100644 --- a/src/backend/replication/logical/relation.c +++ b/src/backend/replication/logical/relation.c @@ -631,8 +631,8 @@ logicalrep_partition_open(LogicalRepRelMapEntry *root, /* * If the partition's attributes don't match the root relation's, we'll * need to make a new attrmap which maps partition attribute numbers to - * remoterel's, instead the original which maps root relation's attribute - * numbers to remoterel's. + * remoterel's, instead of the original which maps root relation's + * attribute numbers to remoterel's. * * Note that 'map' which comes from the tuple routing data structure * contains 1-based attribute numbers (of the parent relation). However, diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c index 19a9f9394921..88566bd9fab0 100644 --- a/src/backend/storage/lmgr/proc.c +++ b/src/backend/storage/lmgr/proc.c @@ -1369,7 +1369,7 @@ ProcSleep(LOCALLOCK *locallock, LockMethod lockMethodTable) else LWLockRelease(ProcArrayLock); - /* prevent signal from being resent more than once */ + /* prevent signal from being sent again more than once */ allow_autovacuum_cancel = false; } From 3857f98f143166f96c6f6359810ab4241ab4590f Mon Sep 17 00:00:00 2001 From: Etsuro Fujita Date: Thu, 10 Sep 2020 18:00:00 +0900 Subject: [PATCH 119/589] Clean up some code and comments in partbounds.c. Do some minor cleanup for commit c8434d64c: 1) remove a useless assignment (in normal builds) and 2) improve comments a little. Back-patch to v13 where the aforementioned commit went in. Author: Etsuro Fujita Reviewed-by: Alvaro Herrera Discussion: https://postgr.es/m/CAPmGK16yCd2R4=bQ4g8N2dT9TtA5ZU+qNmJ3LPc_nypbNy4_2A@mail.gmail.com --- src/backend/partitioning/partbounds.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/backend/partitioning/partbounds.c b/src/backend/partitioning/partbounds.c index 419c8fe84516..58f9b46289a0 100644 --- a/src/backend/partitioning/partbounds.c +++ b/src/backend/partitioning/partbounds.c @@ -1020,8 +1020,6 @@ partition_bounds_merge(int partnatts, JoinType jointype, List **outer_parts, List **inner_parts) { - PartitionBoundInfo outer_binfo = outer_rel->boundinfo; - /* * Currently, this function is called only from try_partitionwise_join(), * so the join type should be INNER, LEFT, FULL, SEMI, or ANTI. @@ -1031,10 +1029,10 @@ partition_bounds_merge(int partnatts, jointype == JOIN_ANTI); /* The partitioning strategies should be the same. */ - Assert(outer_binfo->strategy == inner_rel->boundinfo->strategy); + Assert(outer_rel->boundinfo->strategy == inner_rel->boundinfo->strategy); *outer_parts = *inner_parts = NIL; - switch (outer_binfo->strategy) + switch (outer_rel->boundinfo->strategy) { case PARTITION_STRATEGY_HASH: @@ -1075,7 +1073,7 @@ partition_bounds_merge(int partnatts, default: elog(ERROR, "unexpected partition strategy: %d", - (int) outer_binfo->strategy); + (int) outer_rel->boundinfo->strategy); return NULL; /* keep compiler quiet */ } } @@ -1528,7 +1526,7 @@ merge_range_bounds(int partnatts, FmgrInfo *partsupfuncs, &next_index); Assert(merged_index >= 0); - /* Get the range of the merged partition. */ + /* Get the range bounds of the merged partition. */ get_merged_range_bounds(partnatts, partsupfuncs, partcollations, jointype, &outer_lb, &outer_ub, @@ -1833,7 +1831,7 @@ merge_matching_partitions(PartitionMap *outer_map, PartitionMap *inner_map, /* * If neither of them has been merged, merge them. Otherwise, if one has - * been merged with a dummy relation on the other side (and the other + * been merged with a dummy partition on the other side (and the other * hasn't yet been merged with anything), re-merge them. Otherwise, they * can't be merged, so return -1. */ From 994a58407c89724917b05a1564dcf1f2f7f3ea9a Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Thu, 10 Sep 2020 14:15:26 +0200 Subject: [PATCH 120/589] Fix title in reference section Reported-by: Robert Kahlert Author: Daniel Gustafsson --- doc/src/sgml/biblio.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/biblio.sgml b/doc/src/sgml/biblio.sgml index 128072ded9be..73a21b6add1b 100644 --- a/doc/src/sgml/biblio.sgml +++ b/doc/src/sgml/biblio.sgml @@ -121,8 +121,8 @@ - Principles of Database and Knowledge - Base Systems + Principles of Database and Knowledge-Base Systems + Classical Database Systems Jeffrey D. From 540612fa469eaae3345ede7a160b146dd903e7ee Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 10 Sep 2020 14:52:36 +0200 Subject: [PATCH 121/589] Add more tests for EXTRACT of date type EXTRACT of date type is implemented as a wrapper around EXTRACT of timestamp, so the code is already tested there. But the externally visible behavior of EXTRACT on date is not recorded anywhere. Since there is some discussion about reimplementing or refactoring some of this, add some more explicit tests of EXTRACT on date, similar in structure to existing EXTRACT tests on other data types. Discussion: https://www.postgresql.org/message-id/flat/42b73d2d-da12-ba9f-570a-420e0cce19d9@phystech.edu --- src/test/regress/expected/date.out | 192 ++++++++++++++++++++-- src/test/regress/expected/expressions.out | 4 +- src/test/regress/sql/date.sql | 49 +++++- 3 files changed, 228 insertions(+), 17 deletions(-) diff --git a/src/test/regress/expected/date.out b/src/test/regress/expected/date.out index 4cdf1635f2a0..d035fe1f1e0a 100644 --- a/src/test/regress/expected/date.out +++ b/src/test/regress/expected/date.out @@ -21,9 +21,10 @@ INSERT INTO DATE_TBL VALUES ('2000-04-03'); INSERT INTO DATE_TBL VALUES ('2038-04-08'); INSERT INTO DATE_TBL VALUES ('2039-04-09'); INSERT INTO DATE_TBL VALUES ('2040-04-10'); -SELECT f1 AS "Fifteen" FROM DATE_TBL; - Fifteen ------------- +INSERT INTO DATE_TBL VALUES ('2040-04-10 BC'); +SELECT f1 FROM DATE_TBL; + f1 +--------------- 04-09-1957 06-13-1957 02-28-1996 @@ -39,11 +40,12 @@ SELECT f1 AS "Fifteen" FROM DATE_TBL; 04-08-2038 04-09-2039 04-10-2040 -(15 rows) + 04-10-2040 BC +(16 rows) -SELECT f1 AS "Nine" FROM DATE_TBL WHERE f1 < '2000-01-01'; - Nine ------------- +SELECT f1 FROM DATE_TBL WHERE f1 < '2000-01-01'; + f1 +--------------- 04-09-1957 06-13-1957 02-28-1996 @@ -53,11 +55,12 @@ SELECT f1 AS "Nine" FROM DATE_TBL WHERE f1 < '2000-01-01'; 02-28-1997 03-01-1997 03-02-1997 -(9 rows) + 04-10-2040 BC +(10 rows) -SELECT f1 AS "Three" FROM DATE_TBL +SELECT f1 FROM DATE_TBL WHERE f1 BETWEEN '2000-01-01' AND '2001-01-01'; - Three + f1 ------------ 04-01-2000 04-02-2000 @@ -860,7 +863,8 @@ SELECT f1 - date '2000-01-01' AS "Days From 2K" FROM DATE_TBL; 13977 14343 14710 -(15 rows) + -1475115 +(16 rows) SELECT f1 - date 'epoch' AS "Days From Epoch" FROM DATE_TBL; Days From Epoch @@ -880,7 +884,8 @@ SELECT f1 - date 'epoch' AS "Days From Epoch" FROM DATE_TBL; 24934 25300 25667 -(15 rows) + -1464158 +(16 rows) SELECT date 'yesterday' - date 'today' AS "One day"; One day @@ -920,6 +925,43 @@ SELECT date 'tomorrow' - date 'yesterday' AS "Two days"; -- -- test extract! +-- +SELECT f1 as "date", + date_part('year', f1) AS year, + date_part('month', f1) AS month, + date_part('day', f1) AS day, + date_part('quarter', f1) AS quarter, + date_part('decade', f1) AS decade, + date_part('century', f1) AS century, + date_part('millennium', f1) AS millennium, + date_part('isoyear', f1) AS isoyear, + date_part('week', f1) AS week, + date_part('dow', f1) AS dow, + date_part('isodow', f1) AS isodow, + date_part('doy', f1) AS doy, + date_part('julian', f1) AS julian, + date_part('epoch', f1) AS epoch + FROM date_tbl; + date | year | month | day | quarter | decade | century | millennium | isoyear | week | dow | isodow | doy | julian | epoch +---------------+-------+-------+-----+---------+--------+---------+------------+---------+------+-----+--------+-----+---------+--------------- + 04-09-1957 | 1957 | 4 | 9 | 2 | 195 | 20 | 2 | 1957 | 15 | 2 | 2 | 99 | 2435938 | -401760000 + 06-13-1957 | 1957 | 6 | 13 | 2 | 195 | 20 | 2 | 1957 | 24 | 4 | 4 | 164 | 2436003 | -396144000 + 02-28-1996 | 1996 | 2 | 28 | 1 | 199 | 20 | 2 | 1996 | 9 | 3 | 3 | 59 | 2450142 | 825465600 + 02-29-1996 | 1996 | 2 | 29 | 1 | 199 | 20 | 2 | 1996 | 9 | 4 | 4 | 60 | 2450143 | 825552000 + 03-01-1996 | 1996 | 3 | 1 | 1 | 199 | 20 | 2 | 1996 | 9 | 5 | 5 | 61 | 2450144 | 825638400 + 03-02-1996 | 1996 | 3 | 2 | 1 | 199 | 20 | 2 | 1996 | 9 | 6 | 6 | 62 | 2450145 | 825724800 + 02-28-1997 | 1997 | 2 | 28 | 1 | 199 | 20 | 2 | 1997 | 9 | 5 | 5 | 59 | 2450508 | 857088000 + 03-01-1997 | 1997 | 3 | 1 | 1 | 199 | 20 | 2 | 1997 | 9 | 6 | 6 | 60 | 2450509 | 857174400 + 03-02-1997 | 1997 | 3 | 2 | 1 | 199 | 20 | 2 | 1997 | 9 | 0 | 7 | 61 | 2450510 | 857260800 + 04-01-2000 | 2000 | 4 | 1 | 2 | 200 | 20 | 2 | 2000 | 13 | 6 | 6 | 92 | 2451636 | 954547200 + 04-02-2000 | 2000 | 4 | 2 | 2 | 200 | 20 | 2 | 2000 | 13 | 0 | 7 | 93 | 2451637 | 954633600 + 04-03-2000 | 2000 | 4 | 3 | 2 | 200 | 20 | 2 | 2000 | 14 | 1 | 1 | 94 | 2451638 | 954720000 + 04-08-2038 | 2038 | 4 | 8 | 2 | 203 | 21 | 3 | 2038 | 14 | 4 | 4 | 98 | 2465522 | 2154297600 + 04-09-2039 | 2039 | 4 | 9 | 2 | 203 | 21 | 3 | 2039 | 14 | 6 | 6 | 99 | 2465888 | 2185920000 + 04-10-2040 | 2040 | 4 | 10 | 2 | 204 | 21 | 3 | 2040 | 15 | 2 | 2 | 101 | 2466255 | 2217628800 + 04-10-2040 BC | -2040 | 4 | 10 | 2 | -204 | -21 | -3 | -2040 | 15 | 1 | 1 | 100 | 976430 | -126503251200 +(16 rows) + -- -- epoch -- @@ -1111,6 +1153,132 @@ SELECT EXTRACT(CENTURY FROM TIMESTAMP '1970-03-20 04:30:00.00000'); -- 20 20 (1 row) +-- +-- all possible fields +-- +SELECT EXTRACT(MICROSECONDS FROM DATE '2020-08-11'); + date_part +----------- + 0 +(1 row) + +SELECT EXTRACT(MILLISECONDS FROM DATE '2020-08-11'); + date_part +----------- + 0 +(1 row) + +SELECT EXTRACT(SECOND FROM DATE '2020-08-11'); + date_part +----------- + 0 +(1 row) + +SELECT EXTRACT(MINUTE FROM DATE '2020-08-11'); + date_part +----------- + 0 +(1 row) + +SELECT EXTRACT(HOUR FROM DATE '2020-08-11'); + date_part +----------- + 0 +(1 row) + +SELECT EXTRACT(DAY FROM DATE '2020-08-11'); + date_part +----------- + 11 +(1 row) + +SELECT EXTRACT(MONTH FROM DATE '2020-08-11'); + date_part +----------- + 8 +(1 row) + +SELECT EXTRACT(YEAR FROM DATE '2020-08-11'); + date_part +----------- + 2020 +(1 row) + +SELECT EXTRACT(DECADE FROM DATE '2020-08-11'); + date_part +----------- + 202 +(1 row) + +SELECT EXTRACT(CENTURY FROM DATE '2020-08-11'); + date_part +----------- + 21 +(1 row) + +SELECT EXTRACT(MILLENNIUM FROM DATE '2020-08-11'); + date_part +----------- + 3 +(1 row) + +SELECT EXTRACT(ISOYEAR FROM DATE '2020-08-11'); + date_part +----------- + 2020 +(1 row) + +SELECT EXTRACT(QUARTER FROM DATE '2020-08-11'); + date_part +----------- + 3 +(1 row) + +SELECT EXTRACT(WEEK FROM DATE '2020-08-11'); + date_part +----------- + 33 +(1 row) + +SELECT EXTRACT(DOW FROM DATE '2020-08-11'); + date_part +----------- + 2 +(1 row) + +SELECT EXTRACT(ISODOW FROM DATE '2020-08-11'); + date_part +----------- + 2 +(1 row) + +SELECT EXTRACT(DOY FROM DATE '2020-08-11'); + date_part +----------- + 224 +(1 row) + +SELECT EXTRACT(TIMEZONE FROM DATE '2020-08-11'); +ERROR: timestamp units "timezone" not supported +CONTEXT: SQL function "date_part" statement 1 +SELECT EXTRACT(TIMEZONE_M FROM DATE '2020-08-11'); +ERROR: timestamp units "timezone_m" not supported +CONTEXT: SQL function "date_part" statement 1 +SELECT EXTRACT(TIMEZONE_H FROM DATE '2020-08-11'); +ERROR: timestamp units "timezone_h" not supported +CONTEXT: SQL function "date_part" statement 1 +SELECT EXTRACT(EPOCH FROM DATE '2020-08-11'); + date_part +------------ + 1597104000 +(1 row) + +SELECT EXTRACT(JULIAN FROM DATE '2020-08-11'); + date_part +----------- + 2459073 +(1 row) + -- -- test trunc function! -- diff --git a/src/test/regress/expected/expressions.out b/src/test/regress/expected/expressions.out index 4f4deaec2231..05a6eb07b2e7 100644 --- a/src/test/regress/expected/expressions.out +++ b/src/test/regress/expected/expressions.out @@ -121,7 +121,7 @@ select count(*) from date_tbl where f1 not between '1997-01-01' and '1998-01-01'; count ------- - 12 + 13 (1 row) explain (costs off) @@ -155,6 +155,6 @@ select count(*) from date_tbl where f1 not between symmetric '1997-01-01' and '1998-01-01'; count ------- - 12 + 13 (1 row) diff --git a/src/test/regress/sql/date.sql b/src/test/regress/sql/date.sql index 1c3adf70ced7..488f5faa076f 100644 --- a/src/test/regress/sql/date.sql +++ b/src/test/regress/sql/date.sql @@ -20,12 +20,13 @@ INSERT INTO DATE_TBL VALUES ('2000-04-03'); INSERT INTO DATE_TBL VALUES ('2038-04-08'); INSERT INTO DATE_TBL VALUES ('2039-04-09'); INSERT INTO DATE_TBL VALUES ('2040-04-10'); +INSERT INTO DATE_TBL VALUES ('2040-04-10 BC'); -SELECT f1 AS "Fifteen" FROM DATE_TBL; +SELECT f1 FROM DATE_TBL; -SELECT f1 AS "Nine" FROM DATE_TBL WHERE f1 < '2000-01-01'; +SELECT f1 FROM DATE_TBL WHERE f1 < '2000-01-01'; -SELECT f1 AS "Three" FROM DATE_TBL +SELECT f1 FROM DATE_TBL WHERE f1 BETWEEN '2000-01-01' AND '2001-01-01'; -- @@ -218,6 +219,23 @@ SELECT date 'tomorrow' - date 'yesterday' AS "Two days"; -- -- test extract! -- +SELECT f1 as "date", + date_part('year', f1) AS year, + date_part('month', f1) AS month, + date_part('day', f1) AS day, + date_part('quarter', f1) AS quarter, + date_part('decade', f1) AS decade, + date_part('century', f1) AS century, + date_part('millennium', f1) AS millennium, + date_part('isoyear', f1) AS isoyear, + date_part('week', f1) AS week, + date_part('dow', f1) AS dow, + date_part('isodow', f1) AS isodow, + date_part('doy', f1) AS doy, + date_part('julian', f1) AS julian, + date_part('epoch', f1) AS epoch + FROM date_tbl; +-- -- epoch -- SELECT EXTRACT(EPOCH FROM DATE '1970-01-01'); -- 0 @@ -264,6 +282,31 @@ SELECT EXTRACT(DECADE FROM DATE '0012-12-31 BC'); -- -2 SELECT EXTRACT(CENTURY FROM NOW())>=21 AS True; -- true SELECT EXTRACT(CENTURY FROM TIMESTAMP '1970-03-20 04:30:00.00000'); -- 20 -- +-- all possible fields +-- +SELECT EXTRACT(MICROSECONDS FROM DATE '2020-08-11'); +SELECT EXTRACT(MILLISECONDS FROM DATE '2020-08-11'); +SELECT EXTRACT(SECOND FROM DATE '2020-08-11'); +SELECT EXTRACT(MINUTE FROM DATE '2020-08-11'); +SELECT EXTRACT(HOUR FROM DATE '2020-08-11'); +SELECT EXTRACT(DAY FROM DATE '2020-08-11'); +SELECT EXTRACT(MONTH FROM DATE '2020-08-11'); +SELECT EXTRACT(YEAR FROM DATE '2020-08-11'); +SELECT EXTRACT(DECADE FROM DATE '2020-08-11'); +SELECT EXTRACT(CENTURY FROM DATE '2020-08-11'); +SELECT EXTRACT(MILLENNIUM FROM DATE '2020-08-11'); +SELECT EXTRACT(ISOYEAR FROM DATE '2020-08-11'); +SELECT EXTRACT(QUARTER FROM DATE '2020-08-11'); +SELECT EXTRACT(WEEK FROM DATE '2020-08-11'); +SELECT EXTRACT(DOW FROM DATE '2020-08-11'); +SELECT EXTRACT(ISODOW FROM DATE '2020-08-11'); +SELECT EXTRACT(DOY FROM DATE '2020-08-11'); +SELECT EXTRACT(TIMEZONE FROM DATE '2020-08-11'); +SELECT EXTRACT(TIMEZONE_M FROM DATE '2020-08-11'); +SELECT EXTRACT(TIMEZONE_H FROM DATE '2020-08-11'); +SELECT EXTRACT(EPOCH FROM DATE '2020-08-11'); +SELECT EXTRACT(JULIAN FROM DATE '2020-08-11'); +-- -- test trunc function! -- SELECT DATE_TRUNC('MILLENNIUM', TIMESTAMP '1970-03-20 04:30:00.00000'); -- 1001 From 4fff515e9efff0053d50fc0b811612a4844b1c65 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 10 Sep 2020 15:31:09 +0200 Subject: [PATCH 122/589] doc: Remove buggy ICU collation from documentation We have had multiple reports that point to the '@colReorder=latn-digit' collation customization being buggy. We have reported this to ICU and are waiting for a fix. In the meantime, remove references to this from the documentation and replace it by another reordering example. Apparently, many users have been picking up this example specifically from the documentation. Author: Jehan-Guillaume de Rorthais Discussion: https://www.postgresql.org/message-id/flat/153201618542.1404.3611626898935613264%40wrigleys.postgresql.org --- doc/src/sgml/charset.sgml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/src/sgml/charset.sgml b/doc/src/sgml/charset.sgml index 4b4563c5b9d8..2745b4441765 100644 --- a/doc/src/sgml/charset.sgml +++ b/doc/src/sgml/charset.sgml @@ -791,11 +791,11 @@ CREATE COLLATION german (provider = libc, locale = 'de_DE'); - CREATE COLLATION digitslast (provider = icu, locale = 'en-u-kr-latn-digit'); - CREATE COLLATION digitslast (provider = icu, locale = 'en@colReorder=latn-digit'); + CREATE COLLATION latinlast (provider = icu, locale = 'en-u-kr-grek-latn'); + CREATE COLLATION latinlast (provider = icu, locale = 'en@colReorder=grek-latn'); - Sort digits after Latin letters. (The default is digits before letters.) + Sort Greek letters before Latin ones. (The default is Latin before Greek.) @@ -811,9 +811,9 @@ CREATE COLLATION german (provider = libc, locale = 'de_DE'); - - CREATE COLLATION special (provider = icu, locale = 'en-u-kf-upper-kr-latn-digit'); - CREATE COLLATION special (provider = icu, locale = 'en@colCaseFirst=upper;colReorder=latn-digit'); + + CREATE COLLATION special (provider = icu, locale = 'en-u-kf-upper-kr-grek-latn'); + CREATE COLLATION special (provider = icu, locale = 'en@colCaseFirst=upper;colReorder=grek-latn'); Combines both of the above options. From beff361bc1edc24ee5f8b2073a1e5e4c92ea66eb Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 10 Sep 2020 15:55:31 +0200 Subject: [PATCH 123/589] Add libpq's openssl dependencies to pkg-config file Add libssl and libcrypto to libpq.pc's Requires.private. This allows static linking to work if those libssl or libcrypto themselves have dependencies in their *.private fields, such as -lz in some cases. Reported-by: Sandro Mani Discussion: https://www.postgresql.org/message-id/flat/837d1dcf-2fca-ee6e-0d7e-6bce1a1bac75@gmail.com --- src/interfaces/libpq/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index d4919970f888..4ac5f4b340f5 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -89,6 +89,8 @@ SHLIB_PREREQS = submake-libpgport SHLIB_EXPORTS = exports.txt +PKG_CONFIG_REQUIRES_PRIVATE = libssl libcrypto + all: all-lib # Shared library stuff From c02767d2415d17329e2998a9a182fc240a90b554 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 10 Sep 2020 16:13:19 +0200 Subject: [PATCH 124/589] Remove unused parameter Apparently, this was never used when introduced (3dad73e71f08abd86564d5090a58ca71740e07e0). Discussion: https://www.postgresql.org/message-id/flat/511bb100-f829-ba21-2f10-9f952ec06ead%402ndquadrant.com --- src/bin/pg_basebackup/receivelog.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/bin/pg_basebackup/receivelog.c b/src/bin/pg_basebackup/receivelog.c index d3f99d89c5c8..dc97c7e89c4d 100644 --- a/src/bin/pg_basebackup/receivelog.c +++ b/src/bin/pg_basebackup/receivelog.c @@ -46,8 +46,7 @@ static bool ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, i XLogRecPtr *blockpos); static PGresult *HandleEndOfCopyStream(PGconn *conn, StreamCtl *stream, char *copybuf, XLogRecPtr blockpos, XLogRecPtr *stoppos); -static bool CheckCopyStreamStop(PGconn *conn, StreamCtl *stream, XLogRecPtr blockpos, - XLogRecPtr *stoppos); +static bool CheckCopyStreamStop(PGconn *conn, StreamCtl *stream, XLogRecPtr blockpos); static long CalculateCopyStreamSleeptime(TimestampTz now, int standby_message_timeout, TimestampTz last_status); @@ -747,7 +746,7 @@ HandleCopyStream(PGconn *conn, StreamCtl *stream, /* * Check if we should continue streaming, or abort at this point. */ - if (!CheckCopyStreamStop(conn, stream, blockpos, stoppos)) + if (!CheckCopyStreamStop(conn, stream, blockpos)) goto error; now = feGetCurrentTimestamp(); @@ -825,7 +824,7 @@ HandleCopyStream(PGconn *conn, StreamCtl *stream, * Check if we should continue streaming, or abort at this * point. */ - if (!CheckCopyStreamStop(conn, stream, blockpos, stoppos)) + if (!CheckCopyStreamStop(conn, stream, blockpos)) goto error; } else @@ -1203,8 +1202,7 @@ HandleEndOfCopyStream(PGconn *conn, StreamCtl *stream, char *copybuf, * Check if we should continue streaming, or abort at this point. */ static bool -CheckCopyStreamStop(PGconn *conn, StreamCtl *stream, XLogRecPtr blockpos, - XLogRecPtr *stoppos) +CheckCopyStreamStop(PGconn *conn, StreamCtl *stream, XLogRecPtr blockpos) { if (still_sending && stream->stream_stop(blockpos, stream->timeline, false)) { From 34a947ca13e52389eb3e81f1ceb38e592ad53110 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 10 Sep 2020 11:10:55 -0400 Subject: [PATCH 125/589] New contrib module, pg_surgery, with heap surgery functions. Sometimes it happens that the visibility information for a tuple becomes corrupted, either due to bugs in the database software or external factors. Provide a function heap_force_kill() that can be used to truncate such dead tuples to dead line pointers, and a function heap_force_freeze() that can be used to overwrite the visibility information in such a way that the tuple becomes all-visible. These functions are unsafe, in that you can easily use them to corrupt a database that was not previously corrupted, and you can use them to further corrupt an already-corrupted database or to destroy data. The documentation accordingly cautions against casual use. However, in some cases they permit recovery of data that would otherwise be very difficult to recover, or to allow a system to continue to function when it would otherwise be difficult to do so. Because we may want to add other functions for performing other kinds of surgery in the future, the new contrib module is called pg_surgery rather than something specific to these functions. I proposed back-patching this so that it could be more easily used by people running existing releases who are facing these kinds of problems, but that proposal did not attract enough support, so no back-patch for now. Ashutosh Sharma, reviewed and tested by Andrey M. Borodin, M. Beena Emerson, Masahiko Sawada, Rajkumar Raghuwanshi, Asim Praveen, and Mark Dilger, and somewhat revised by me. Discussion: http://postgr.es/m/CA+TgmoZW1fsU-QUNCRUQMGUygBDPVeOTLCqRdQZch=EYZnctSA@mail.gmail.com --- contrib/Makefile | 1 + contrib/pg_surgery/.gitignore | 4 + contrib/pg_surgery/Makefile | 23 + contrib/pg_surgery/expected/heap_surgery.out | 180 ++++++++ contrib/pg_surgery/heap_surgery.c | 428 +++++++++++++++++++ contrib/pg_surgery/pg_surgery--1.0.sql | 18 + contrib/pg_surgery/pg_surgery.control | 5 + contrib/pg_surgery/sql/heap_surgery.sql | 91 ++++ doc/src/sgml/contrib.sgml | 1 + doc/src/sgml/filelist.sgml | 1 + doc/src/sgml/pgsurgery.sgml | 107 +++++ src/tools/pgindent/typedefs.list | 1 + 12 files changed, 860 insertions(+) create mode 100644 contrib/pg_surgery/.gitignore create mode 100644 contrib/pg_surgery/Makefile create mode 100644 contrib/pg_surgery/expected/heap_surgery.out create mode 100644 contrib/pg_surgery/heap_surgery.c create mode 100644 contrib/pg_surgery/pg_surgery--1.0.sql create mode 100644 contrib/pg_surgery/pg_surgery.control create mode 100644 contrib/pg_surgery/sql/heap_surgery.sql create mode 100644 doc/src/sgml/pgsurgery.sgml diff --git a/contrib/Makefile b/contrib/Makefile index 1846d415b6fe..c8d2a1627358 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -34,6 +34,7 @@ SUBDIRS = \ pg_prewarm \ pg_standby \ pg_stat_statements \ + pg_surgery \ pg_trgm \ pgcrypto \ pgrowlocks \ diff --git a/contrib/pg_surgery/.gitignore b/contrib/pg_surgery/.gitignore new file mode 100644 index 000000000000..5dcb3ff97235 --- /dev/null +++ b/contrib/pg_surgery/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/contrib/pg_surgery/Makefile b/contrib/pg_surgery/Makefile new file mode 100644 index 000000000000..a66776c4c413 --- /dev/null +++ b/contrib/pg_surgery/Makefile @@ -0,0 +1,23 @@ +# contrib/pg_surgery/Makefile + +MODULE_big = pg_surgery +OBJS = \ + $(WIN32RES) \ + heap_surgery.o + +EXTENSION = pg_surgery +DATA = pg_surgery--1.0.sql +PGFILEDESC = "pg_surgery - perform surgery on a damaged relation" + +REGRESS = heap_surgery + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/pg_surgery +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/pg_surgery/expected/heap_surgery.out b/contrib/pg_surgery/expected/heap_surgery.out new file mode 100644 index 000000000000..9451c5747bc0 --- /dev/null +++ b/contrib/pg_surgery/expected/heap_surgery.out @@ -0,0 +1,180 @@ +create extension pg_surgery; +-- create a normal heap table and insert some rows. +-- note that we don't commit the transaction, so autovacuum can't interfere. +begin; +create table htab(a int); +insert into htab values (100), (200), (300), (400), (500); +-- test empty TID array +select heap_force_freeze('htab'::regclass, ARRAY[]::tid[]); + heap_force_freeze +------------------- + +(1 row) + +-- nothing should be frozen yet +select * from htab where xmin = 2; + a +--- +(0 rows) + +-- freeze forcibly +select heap_force_freeze('htab'::regclass, ARRAY['(0, 4)']::tid[]); + heap_force_freeze +------------------- + +(1 row) + +-- now we should have one frozen tuple +select ctid, xmax from htab where xmin = 2; + ctid | xmax +-------+------ + (0,4) | 0 +(1 row) + +-- kill forcibly +select heap_force_kill('htab'::regclass, ARRAY['(0, 4)']::tid[]); + heap_force_kill +----------------- + +(1 row) + +-- should be gone now +select * from htab where ctid = '(0, 4)'; + a +--- +(0 rows) + +-- should now be skipped because it's already dead +select heap_force_kill('htab'::regclass, ARRAY['(0, 4)']::tid[]); +NOTICE: skipping tid (0, 4) for relation "htab" because it is marked dead + heap_force_kill +----------------- + +(1 row) + +select heap_force_freeze('htab'::regclass, ARRAY['(0, 4)']::tid[]); +NOTICE: skipping tid (0, 4) for relation "htab" because it is marked dead + heap_force_freeze +------------------- + +(1 row) + +-- freeze two TIDs at once while skipping an out-of-range block number +select heap_force_freeze('htab'::regclass, + ARRAY['(0, 1)', '(0, 3)', '(1, 1)']::tid[]); +NOTICE: skipping block 1 for relation "htab" because the block number is out of range + heap_force_freeze +------------------- + +(1 row) + +-- we should now have two frozen tuples +select ctid, xmax from htab where xmin = 2; + ctid | xmax +-------+------ + (0,1) | 0 + (0,3) | 0 +(2 rows) + +-- out-of-range TIDs should be skipped +select heap_force_freeze('htab'::regclass, ARRAY['(0, 0)', '(0, 6)']::tid[]); +NOTICE: skipping tid (0, 0) for relation "htab" because the item number is out of range +NOTICE: skipping tid (0, 6) for relation "htab" because the item number is out of range + heap_force_freeze +------------------- + +(1 row) + +rollback; +-- set up a new table with a redirected line pointer +create table htab2(a int) with (autovacuum_enabled = off); +insert into htab2 values (100); +update htab2 set a = 200; +vacuum htab2; +-- redirected TIDs should be skipped +select heap_force_kill('htab2'::regclass, ARRAY['(0, 1)']::tid[]); +NOTICE: skipping tid (0, 1) for relation "htab2" because it redirects to item 2 + heap_force_kill +----------------- + +(1 row) + +-- now create an unused line pointer +select ctid from htab2; + ctid +------- + (0,2) +(1 row) + +update htab2 set a = 300; +select ctid from htab2; + ctid +------- + (0,3) +(1 row) + +vacuum freeze htab2; +-- unused TIDs should be skipped +select heap_force_kill('htab2'::regclass, ARRAY['(0, 2)']::tid[]); +NOTICE: skipping tid (0, 2) for relation "htab2" because it is marked unused + heap_force_kill +----------------- + +(1 row) + +-- multidimensional TID array should be rejected +select heap_force_kill('htab2'::regclass, ARRAY[['(0, 2)']]::tid[]); +ERROR: argument must be empty or one-dimensional array +-- TID array with nulls should be rejected +select heap_force_kill('htab2'::regclass, ARRAY[NULL]::tid[]); +ERROR: array must not contain nulls +-- but we should be able to kill the one tuple we have +select heap_force_kill('htab2'::regclass, ARRAY['(0, 3)']::tid[]); + heap_force_kill +----------------- + +(1 row) + +-- materialized view. +-- note that we don't commit the transaction, so autovacuum can't interfere. +begin; +create materialized view mvw as select a from generate_series(1, 3) a; +select * from mvw where xmin = 2; + a +--- +(0 rows) + +select heap_force_freeze('mvw'::regclass, ARRAY['(0, 3)']::tid[]); + heap_force_freeze +------------------- + +(1 row) + +select * from mvw where xmin = 2; + a +--- + 3 +(1 row) + +select heap_force_kill('mvw'::regclass, ARRAY['(0, 3)']::tid[]); + heap_force_kill +----------------- + +(1 row) + +select * from mvw where ctid = '(0, 3)'; + a +--- +(0 rows) + +rollback; +-- check that it fails on an unsupported relkind +create view vw as select 1; +select heap_force_kill('vw'::regclass, ARRAY['(0, 1)']::tid[]); +ERROR: "vw" is not a table, materialized view, or TOAST table +select heap_force_freeze('vw'::regclass, ARRAY['(0, 1)']::tid[]); +ERROR: "vw" is not a table, materialized view, or TOAST table +-- cleanup. +drop table htab2; +drop view vw; +drop extension pg_surgery; diff --git a/contrib/pg_surgery/heap_surgery.c b/contrib/pg_surgery/heap_surgery.c new file mode 100644 index 000000000000..eb96b4bb36d8 --- /dev/null +++ b/contrib/pg_surgery/heap_surgery.c @@ -0,0 +1,428 @@ +/*------------------------------------------------------------------------- + * + * heap_surgery.c + * Functions to perform surgery on the damaged heap table. + * + * Copyright (c) 2020, PostgreSQL Global Development Group + * + * IDENTIFICATION + * contrib/pg_surgery/heap_surgery.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/heapam.h" +#include "access/visibilitymap.h" +#include "catalog/pg_am_d.h" +#include "catalog/pg_proc_d.h" +#include "miscadmin.h" +#include "storage/bufmgr.h" +#include "utils/acl.h" +#include "utils/rel.h" + +PG_MODULE_MAGIC; + +/* Options to forcefully change the state of a heap tuple. */ +typedef enum HeapTupleForceOption +{ + HEAP_FORCE_KILL, + HEAP_FORCE_FREEZE +} HeapTupleForceOption; + +PG_FUNCTION_INFO_V1(heap_force_kill); +PG_FUNCTION_INFO_V1(heap_force_freeze); + +static int32 tidcmp(const void *a, const void *b); +static Datum heap_force_common(FunctionCallInfo fcinfo, + HeapTupleForceOption heap_force_opt); +static void sanity_check_tid_array(ArrayType *ta, int *ntids); +static void sanity_check_relation(Relation rel); +static BlockNumber find_tids_one_page(ItemPointer tids, int ntids, + OffsetNumber *next_start_ptr); + +/*------------------------------------------------------------------------- + * heap_force_kill() + * + * Force kill the tuple(s) pointed to by the item pointer(s) stored in the + * given TID array. + * + * Usage: SELECT heap_force_kill(regclass, tid[]); + *------------------------------------------------------------------------- + */ +Datum +heap_force_kill(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(heap_force_common(fcinfo, HEAP_FORCE_KILL)); +} + +/*------------------------------------------------------------------------- + * heap_force_freeze() + * + * Force freeze the tuple(s) pointed to by the item pointer(s) stored in the + * given TID array. + * + * Usage: SELECT heap_force_freeze(regclass, tid[]); + *------------------------------------------------------------------------- + */ +Datum +heap_force_freeze(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(heap_force_common(fcinfo, HEAP_FORCE_FREEZE)); +} + +/*------------------------------------------------------------------------- + * heap_force_common() + * + * Common code for heap_force_kill and heap_force_freeze + *------------------------------------------------------------------------- + */ +static Datum +heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt) +{ + Oid relid = PG_GETARG_OID(0); + ArrayType *ta = PG_GETARG_ARRAYTYPE_P_COPY(1); + ItemPointer tids; + int ntids, + nblocks; + Relation rel; + OffsetNumber curr_start_ptr, + next_start_ptr; + bool include_this_tid[MaxHeapTuplesPerPage]; + + if (RecoveryInProgress()) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("recovery is in progress"), + errhint("heap surgery functions cannot be executed during recovery."))); + + /* Check inputs. */ + sanity_check_tid_array(ta, &ntids); + + rel = relation_open(relid, RowExclusiveLock); + + /* Check target relation. */ + sanity_check_relation(rel); + + tids = ((ItemPointer) ARR_DATA_PTR(ta)); + + /* + * If there is more than one TID in the array, sort them so that we can + * easily fetch all the TIDs belonging to one particular page from the + * array. + */ + if (ntids > 1) + qsort((void *) tids, ntids, sizeof(ItemPointerData), tidcmp); + + curr_start_ptr = next_start_ptr = 0; + nblocks = RelationGetNumberOfBlocks(rel); + + /* + * Loop, performing the necessary actions for each block. + */ + while (next_start_ptr != ntids) + { + Buffer buf; + Buffer vmbuf = InvalidBuffer; + Page page; + BlockNumber blkno; + OffsetNumber curoff; + OffsetNumber maxoffset; + int i; + bool did_modify_page = false; + bool did_modify_vm = false; + + CHECK_FOR_INTERRUPTS(); + + /* + * Find all the TIDs belonging to one particular page starting from + * next_start_ptr and process them one by one. + */ + blkno = find_tids_one_page(tids, ntids, &next_start_ptr); + + /* Check whether the block number is valid. */ + if (blkno >= nblocks) + { + /* Update the current_start_ptr before moving to the next page. */ + curr_start_ptr = next_start_ptr; + + ereport(NOTICE, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("skipping block %u for relation \"%s\" because the block number is out of range", + blkno, RelationGetRelationName(rel)))); + continue; + } + + buf = ReadBuffer(rel, blkno); + LockBufferForCleanup(buf); + + page = BufferGetPage(buf); + + maxoffset = PageGetMaxOffsetNumber(page); + + /* + * Figure out which TIDs we are going to process and which ones we are + * going to skip. + */ + memset(include_this_tid, 0, sizeof(include_this_tid)); + for (i = curr_start_ptr; i < next_start_ptr; i++) + { + OffsetNumber offno = ItemPointerGetOffsetNumberNoCheck(&tids[i]); + ItemId itemid; + + /* Check whether the offset number is valid. */ + if (offno == InvalidOffsetNumber || offno > maxoffset) + { + ereport(NOTICE, + errmsg("skipping tid (%u, %u) for relation \"%s\" because the item number is out of range", + blkno, offno, RelationGetRelationName(rel))); + continue; + } + + itemid = PageGetItemId(page, offno); + + /* Only accept an item ID that is used. */ + if (ItemIdIsRedirected(itemid)) + { + ereport(NOTICE, + errmsg("skipping tid (%u, %u) for relation \"%s\" because it redirects to item %u", + blkno, offno, RelationGetRelationName(rel), + ItemIdGetRedirect(itemid))); + continue; + } + else if (ItemIdIsDead(itemid)) + { + ereport(NOTICE, + (errmsg("skipping tid (%u, %u) for relation \"%s\" because it is marked dead", + blkno, offno, RelationGetRelationName(rel)))); + continue; + } + else if (!ItemIdIsUsed(itemid)) + { + ereport(NOTICE, + (errmsg("skipping tid (%u, %u) for relation \"%s\" because it is marked unused", + blkno, offno, RelationGetRelationName(rel)))); + continue; + } + + /* Mark it for processing. */ + Assert(offno < MaxHeapTuplesPerPage); + include_this_tid[offno] = true; + } + + /* + * Before entering the critical section, pin the visibility map page + * if it appears to be necessary. + */ + if (heap_force_opt == HEAP_FORCE_KILL && PageIsAllVisible(page)) + visibilitymap_pin(rel, blkno, &vmbuf); + + /* No ereport(ERROR) from here until all the changes are logged. */ + START_CRIT_SECTION(); + + for (curoff = FirstOffsetNumber; curoff <= maxoffset; + curoff = OffsetNumberNext(curoff)) + { + ItemId itemid; + + if (!include_this_tid[curoff]) + continue; + + itemid = PageGetItemId(page, curoff); + Assert(ItemIdIsNormal(itemid)); + + did_modify_page = true; + + if (heap_force_opt == HEAP_FORCE_KILL) + { + ItemIdSetDead(itemid); + + /* + * If the page is marked all-visible, we must clear + * PD_ALL_VISIBLE flag on the page header and an all-visible + * bit on the visibility map corresponding to the page. + */ + if (PageIsAllVisible(page)) + { + PageClearAllVisible(page); + visibilitymap_clear(rel, blkno, vmbuf, + VISIBILITYMAP_VALID_BITS); + did_modify_vm = true; + } + } + else + { + HeapTupleHeader htup; + + Assert(heap_force_opt == HEAP_FORCE_FREEZE); + + htup = (HeapTupleHeader) PageGetItem(page, itemid); + + /* + * Reset all visibility-related fields of the tuple. This + * logic should mimic heap_execute_freeze_tuple(), but we + * choose to reset xmin and ctid just to be sure that no + * potentially-garbled data is left behind. + */ + ItemPointerSet(&htup->t_ctid, blkno, curoff); + HeapTupleHeaderSetXmin(htup, FrozenTransactionId); + HeapTupleHeaderSetXmax(htup, InvalidTransactionId); + if (htup->t_infomask & HEAP_MOVED) + { + if (htup->t_infomask & HEAP_MOVED_OFF) + HeapTupleHeaderSetXvac(htup, InvalidTransactionId); + else + HeapTupleHeaderSetXvac(htup, FrozenTransactionId); + } + + /* + * Clear all the visibility-related bits of this tuple and + * mark it as frozen. Also, get rid of HOT_UPDATED and + * KEYS_UPDATES bits. + */ + htup->t_infomask &= ~HEAP_XACT_MASK; + htup->t_infomask |= (HEAP_XMIN_FROZEN | HEAP_XMAX_INVALID); + htup->t_infomask2 &= ~HEAP_HOT_UPDATED; + htup->t_infomask2 &= ~HEAP_KEYS_UPDATED; + } + } + + /* + * If the page was modified, only then, we mark the buffer dirty or do + * the WAL logging. + */ + if (did_modify_page) + { + /* Mark buffer dirty before we write WAL. */ + MarkBufferDirty(buf); + + /* XLOG stuff */ + if (RelationNeedsWAL(rel)) + log_newpage_buffer(buf, true); + } + + /* WAL log the VM page if it was modified. */ + if (did_modify_vm && RelationNeedsWAL(rel)) + log_newpage_buffer(vmbuf, false); + + END_CRIT_SECTION(); + + UnlockReleaseBuffer(buf); + + if (vmbuf != InvalidBuffer) + ReleaseBuffer(vmbuf); + + /* Update the current_start_ptr before moving to the next page. */ + curr_start_ptr = next_start_ptr; + } + + relation_close(rel, RowExclusiveLock); + + pfree(ta); + + PG_RETURN_VOID(); +} + +/*------------------------------------------------------------------------- + * tidcmp() + * + * Compare two item pointers, return -1, 0, or +1. + * + * See ItemPointerCompare for details. + * ------------------------------------------------------------------------ + */ +static int32 +tidcmp(const void *a, const void *b) +{ + ItemPointer iptr1 = ((const ItemPointer) a); + ItemPointer iptr2 = ((const ItemPointer) b); + + return ItemPointerCompare(iptr1, iptr2); +} + +/*------------------------------------------------------------------------- + * sanity_check_tid_array() + * + * Perform sanity checks on the given tid array, and set *ntids to the + * number of items in the array. + * ------------------------------------------------------------------------ + */ +static void +sanity_check_tid_array(ArrayType *ta, int *ntids) +{ + if (ARR_HASNULL(ta) && array_contains_nulls(ta)) + ereport(ERROR, + (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("array must not contain nulls"))); + + if (ARR_NDIM(ta) > 1) + ereport(ERROR, + (errcode(ERRCODE_DATA_EXCEPTION), + errmsg("argument must be empty or one-dimensional array"))); + + *ntids = ArrayGetNItems(ARR_NDIM(ta), ARR_DIMS(ta)); +} + +/*------------------------------------------------------------------------- + * sanity_check_relation() + * + * Perform sanity checks on the given relation. + * ------------------------------------------------------------------------ + */ +static void +sanity_check_relation(Relation rel) +{ + if (rel->rd_rel->relkind != RELKIND_RELATION && + rel->rd_rel->relkind != RELKIND_MATVIEW && + rel->rd_rel->relkind != RELKIND_TOASTVALUE) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("\"%s\" is not a table, materialized view, or TOAST table", + RelationGetRelationName(rel)))); + + if (rel->rd_rel->relam != HEAP_TABLE_AM_OID) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("only heap AM is supported"))); + + /* Must be owner of the table or superuser. */ + if (!pg_class_ownercheck(RelationGetRelid(rel), GetUserId())) + aclcheck_error(ACLCHECK_NOT_OWNER, + get_relkind_objtype(rel->rd_rel->relkind), + RelationGetRelationName(rel)); +} + +/*------------------------------------------------------------------------- + * find_tids_one_page() + * + * Find all the tids residing in the same page as tids[next_start_ptr], and + * update next_start_ptr so that it points to the first tid in the next page. + * + * NOTE: The input tids[] array must be sorted. + * ------------------------------------------------------------------------ + */ +static BlockNumber +find_tids_one_page(ItemPointer tids, int ntids, OffsetNumber *next_start_ptr) +{ + int i; + BlockNumber prev_blkno, + blkno; + + prev_blkno = blkno = InvalidBlockNumber; + + for (i = *next_start_ptr; i < ntids; i++) + { + ItemPointerData tid = tids[i]; + + blkno = ItemPointerGetBlockNumberNoCheck(&tid); + + if (i == *next_start_ptr) + prev_blkno = blkno; + + if (prev_blkno != blkno) + break; + } + + *next_start_ptr = i; + return prev_blkno; +} diff --git a/contrib/pg_surgery/pg_surgery--1.0.sql b/contrib/pg_surgery/pg_surgery--1.0.sql new file mode 100644 index 000000000000..2ae7f228c74b --- /dev/null +++ b/contrib/pg_surgery/pg_surgery--1.0.sql @@ -0,0 +1,18 @@ +/* contrib/pg_surgery/pg_surgery--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION pg_surgery" to load this file. \quit + +CREATE FUNCTION heap_force_kill(reloid regclass, tids tid[]) +RETURNS VOID +AS 'MODULE_PATHNAME', 'heap_force_kill' +LANGUAGE C STRICT; + +REVOKE EXECUTE ON FUNCTION heap_force_kill(regclass, tid[]) FROM PUBLIC; + +CREATE FUNCTION heap_force_freeze(reloid regclass, tids tid[]) +RETURNS VOID +AS 'MODULE_PATHNAME', 'heap_force_freeze' +LANGUAGE C STRICT; + +REVOKE EXECUTE ON FUNCTION heap_force_freeze(regclass, tid[]) FROM PUBLIC; \ No newline at end of file diff --git a/contrib/pg_surgery/pg_surgery.control b/contrib/pg_surgery/pg_surgery.control new file mode 100644 index 000000000000..2bcdad1e3f7f --- /dev/null +++ b/contrib/pg_surgery/pg_surgery.control @@ -0,0 +1,5 @@ +# pg_surgery extension +comment = 'extension to perform surgery on a damaged relation' +default_version = '1.0' +module_pathname = '$libdir/pg_surgery' +relocatable = true diff --git a/contrib/pg_surgery/sql/heap_surgery.sql b/contrib/pg_surgery/sql/heap_surgery.sql new file mode 100644 index 000000000000..8a27214e9cd1 --- /dev/null +++ b/contrib/pg_surgery/sql/heap_surgery.sql @@ -0,0 +1,91 @@ +create extension pg_surgery; + +-- create a normal heap table and insert some rows. +-- note that we don't commit the transaction, so autovacuum can't interfere. +begin; +create table htab(a int); +insert into htab values (100), (200), (300), (400), (500); + +-- test empty TID array +select heap_force_freeze('htab'::regclass, ARRAY[]::tid[]); + +-- nothing should be frozen yet +select * from htab where xmin = 2; + +-- freeze forcibly +select heap_force_freeze('htab'::regclass, ARRAY['(0, 4)']::tid[]); + +-- now we should have one frozen tuple +select ctid, xmax from htab where xmin = 2; + +-- kill forcibly +select heap_force_kill('htab'::regclass, ARRAY['(0, 4)']::tid[]); + +-- should be gone now +select * from htab where ctid = '(0, 4)'; + +-- should now be skipped because it's already dead +select heap_force_kill('htab'::regclass, ARRAY['(0, 4)']::tid[]); +select heap_force_freeze('htab'::regclass, ARRAY['(0, 4)']::tid[]); + +-- freeze two TIDs at once while skipping an out-of-range block number +select heap_force_freeze('htab'::regclass, + ARRAY['(0, 1)', '(0, 3)', '(1, 1)']::tid[]); + +-- we should now have two frozen tuples +select ctid, xmax from htab where xmin = 2; + +-- out-of-range TIDs should be skipped +select heap_force_freeze('htab'::regclass, ARRAY['(0, 0)', '(0, 6)']::tid[]); + +rollback; + +-- set up a new table with a redirected line pointer +create table htab2(a int) with (autovacuum_enabled = off); +insert into htab2 values (100); +update htab2 set a = 200; +vacuum htab2; + +-- redirected TIDs should be skipped +select heap_force_kill('htab2'::regclass, ARRAY['(0, 1)']::tid[]); + +-- now create an unused line pointer +select ctid from htab2; +update htab2 set a = 300; +select ctid from htab2; +vacuum freeze htab2; + +-- unused TIDs should be skipped +select heap_force_kill('htab2'::regclass, ARRAY['(0, 2)']::tid[]); + +-- multidimensional TID array should be rejected +select heap_force_kill('htab2'::regclass, ARRAY[['(0, 2)']]::tid[]); + +-- TID array with nulls should be rejected +select heap_force_kill('htab2'::regclass, ARRAY[NULL]::tid[]); + +-- but we should be able to kill the one tuple we have +select heap_force_kill('htab2'::regclass, ARRAY['(0, 3)']::tid[]); + +-- materialized view. +-- note that we don't commit the transaction, so autovacuum can't interfere. +begin; +create materialized view mvw as select a from generate_series(1, 3) a; + +select * from mvw where xmin = 2; +select heap_force_freeze('mvw'::regclass, ARRAY['(0, 3)']::tid[]); +select * from mvw where xmin = 2; + +select heap_force_kill('mvw'::regclass, ARRAY['(0, 3)']::tid[]); +select * from mvw where ctid = '(0, 3)'; +rollback; + +-- check that it fails on an unsupported relkind +create view vw as select 1; +select heap_force_kill('vw'::regclass, ARRAY['(0, 1)']::tid[]); +select heap_force_freeze('vw'::regclass, ARRAY['(0, 1)']::tid[]); + +-- cleanup. +drop table htab2; +drop view vw; +drop extension pg_surgery; diff --git a/doc/src/sgml/contrib.sgml b/doc/src/sgml/contrib.sgml index 261a559e81c3..c82dde272639 100644 --- a/doc/src/sgml/contrib.sgml +++ b/doc/src/sgml/contrib.sgml @@ -125,6 +125,7 @@ CREATE EXTENSION module_name; &pgrowlocks; &pgstatstatements; &pgstattuple; + &pgsurgery; &pgtrgm; &pgvisibility; &postgres-fdw; diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 64b5da0070c5..828396d4a9ee 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -139,6 +139,7 @@ + diff --git a/doc/src/sgml/pgsurgery.sgml b/doc/src/sgml/pgsurgery.sgml new file mode 100644 index 000000000000..134be9bebde0 --- /dev/null +++ b/doc/src/sgml/pgsurgery.sgml @@ -0,0 +1,107 @@ + + + + pg_surgery + + + pg_surgery + + + + The pg_surgery module provides various functions to + perform surgery on a damaged relation. These functions are unsafe by design + and using them may corrupt (or further corrupt) your database. For example, + these functions can easily be used to make a table inconsistent with its + own indexes, to cause UNIQUE or + FOREIGN KEY constraint violations, or even to make + tuples visible which, when read, will cause a database server crash. + They should be used with great caution and only as a last resort. + + + + Functions + + + + + heap_force_kill(regclass, tid[]) returns void + + + + + heap_force_kill marks used line + pointers as dead without examining the tuples. The + intended use of this function is to forcibly remove tuples that are not + otherwise accessible. For example: + +test=> select * from t1 where ctid = '(0, 1)'; +ERROR: could not access status of transaction 4007513275 +DETAIL: Could not open file "pg_xact/0EED": No such file or directory. + +test=# select heap_force_kill('t1'::regclass, ARRAY['(0, 1)']::tid[]); + heap_force_kill +----------------- + +(1 row) + +test=# select * from t1 where ctid = '(0, 1)'; +(0 rows) + + + + + + + + + heap_force_freeze(regclass, tid[]) returns void + + + + + heap_force_freeze marks tuples as frozen without + examining the tuple data. The intended use of this function is to + make accessible tuples which are inaccessible due to corrupted + visibility information, or which prevent the table from being + successfully vacuumed due to corrupted visibility information. + For example: + +test=> vacuum t1; +ERROR: found xmin 507 from before relfrozenxid 515 +CONTEXT: while scanning block 0 of relation "public.t1" + +test=# select ctid from t1 where xmin = 507; + ctid +------- + (0,3) +(1 row) + +test=# select heap_force_freeze('t1'::regclass, ARRAY['(0, 3)']::tid[]); + heap_force_freeze +------------------- + +(1 row) + +test=# select ctid from t1 where xmin = 2; + ctid +------- + (0,3) +(1 row) + + + + + + + + + + + Authors + + + Ashutosh Sharma ashu.coek88@gmail.com + + + + diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f151e13d7f1c..b1afb345c368 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -3578,3 +3578,4 @@ yyscan_t z_stream z_streamp zic_t +HeapTupleForceOption From 58c6feccfae1321ea4ff16c535c6f3fb90a07d69 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 10 Sep 2020 12:06:26 -0400 Subject: [PATCH 126/589] Use _exit(2) for SIGQUIT during ProcessStartupPacket, too. Bring the signal handling for startup-packet collection into line with the policy established in commits bedadc732 and 8e19a8264, namely don't risk running atexit callbacks when handling SIGQUIT. Ideally, we'd not do so for SIGTERM or timeout interrupts either, but that change seems a bit too risky for the back branches. For now, just improve the comments in this area to describe the risk. Also relocate where BackendInitialize re-disables these interrupts, to minimize the code span where they're active. This doesn't buy a whole lot of safety, but it can't hurt. In passing, rename startup_die() to remove confusion about whether it is for the startup process. Like the previous commits, back-patch to all supported branches. Discussion: https://postgr.es/m/1850884.1599601164@sss.pgh.pa.us --- src/backend/postmaster/postmaster.c | 83 ++++++++++++++++++----------- 1 file changed, 51 insertions(+), 32 deletions(-) diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 42223c0f61e2..15ad675bc136 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -112,6 +112,7 @@ #include "postmaster/autovacuum.h" #include "postmaster/bgworker_internals.h" #include "postmaster/fork_process.h" +#include "postmaster/interrupt.h" #include "postmaster/pgarch.h" #include "postmaster/postmaster.h" #include "postmaster/syslogger.h" @@ -405,7 +406,7 @@ static void SIGHUP_handler(SIGNAL_ARGS); static void pmdie(SIGNAL_ARGS); static void reaper(SIGNAL_ARGS); static void sigusr1_handler(SIGNAL_ARGS); -static void startup_die(SIGNAL_ARGS); +static void process_startup_packet_die(SIGNAL_ARGS); static void dummy_handler(SIGNAL_ARGS); static void StartupPacketTimeoutHandler(void); static void CleanupBackend(int pid, int exitstatus); @@ -4340,22 +4341,30 @@ BackendInitialize(Port *port) whereToSendOutput = DestRemote; /* now safe to ereport to client */ /* - * We arrange for a simple exit(1) if we receive SIGTERM or SIGQUIT or - * timeout while trying to collect the startup packet. Otherwise the - * postmaster cannot shutdown the database FAST or IMMED cleanly if a - * buggy client fails to send the packet promptly. XXX it follows that - * the remainder of this function must tolerate losing control at any - * instant. Likewise, any pg_on_exit_callback registered before or during - * this function must be prepared to execute at any instant between here - * and the end of this function. Furthermore, affected callbacks execute - * partially or not at all when a second exit-inducing signal arrives - * after proc_exit_prepare() decrements on_proc_exit_index. (Thanks to - * that mechanic, callbacks need not anticipate more than one call.) This - * is fragile; it ought to instead follow the norm of handling interrupts - * at selected, safe opportunities. - */ - pqsignal(SIGTERM, startup_die); - pqsignal(SIGQUIT, startup_die); + * We arrange to do proc_exit(1) if we receive SIGTERM or timeout while + * trying to collect the startup packet; while SIGQUIT results in + * _exit(2). Otherwise the postmaster cannot shutdown the database FAST + * or IMMED cleanly if a buggy client fails to send the packet promptly. + * + * XXX this is pretty dangerous; signal handlers should not call anything + * as complex as proc_exit() directly. We minimize the hazard by not + * keeping these handlers active for longer than we must. However, it + * seems necessary to be able to escape out of DNS lookups as well as the + * startup packet reception proper, so we can't narrow the scope further + * than is done here. + * + * XXX it follows that the remainder of this function must tolerate losing + * control at any instant. Likewise, any pg_on_exit_callback registered + * before or during this function must be prepared to execute at any + * instant between here and the end of this function. Furthermore, + * affected callbacks execute partially or not at all when a second + * exit-inducing signal arrives after proc_exit_prepare() decrements + * on_proc_exit_index. (Thanks to that mechanic, callbacks need not + * anticipate more than one call.) This is fragile; it ought to instead + * follow the norm of handling interrupts at selected, safe opportunities. + */ + pqsignal(SIGTERM, process_startup_packet_die); + pqsignal(SIGQUIT, SignalHandlerForCrashExit); InitializeTimeouts(); /* establishes SIGALRM handler */ PG_SETMASK(&StartupBlockSig); @@ -4411,8 +4420,8 @@ BackendInitialize(Port *port) port->remote_hostname = strdup(remote_host); /* - * Ready to begin client interaction. We will give up and exit(1) after a - * time delay, so that a broken client can't hog a connection + * Ready to begin client interaction. We will give up and proc_exit(1) + * after a time delay, so that a broken client can't hog a connection * indefinitely. PreAuthDelay and any DNS interactions above don't count * against the time limit. * @@ -4434,6 +4443,12 @@ BackendInitialize(Port *port) */ status = ProcessStartupPacket(port, false, false); + /* + * Disable the timeout, and prevent SIGTERM/SIGQUIT again. + */ + disable_timeout(STARTUP_PACKET_TIMEOUT, false); + PG_SETMASK(&BlockSig); + /* * Stop here if it was bad or a cancel packet. ProcessStartupPacket * already did any appropriate error reporting. @@ -4459,12 +4474,6 @@ BackendInitialize(Port *port) pfree(ps_data.data); set_ps_display("initializing"); - - /* - * Disable the timeout, and prevent SIGTERM/SIGQUIT again. - */ - disable_timeout(STARTUP_PACKET_TIMEOUT, false); - PG_SETMASK(&BlockSig); } @@ -5359,16 +5368,22 @@ sigusr1_handler(SIGNAL_ARGS) } /* - * SIGTERM or SIGQUIT while processing startup packet. + * SIGTERM while processing startup packet. * Clean up and exit(1). * - * XXX: possible future improvement: try to send a message indicating - * why we are disconnecting. Problem is to be sure we don't block while - * doing so, nor mess up SSL initialization. In practice, if the client - * has wedged here, it probably couldn't do anything with the message anyway. + * Running proc_exit() from a signal handler is pretty unsafe, since we + * can't know what code we've interrupted. But the alternative of using + * _exit(2) is also unpalatable, since it'd mean that a "fast shutdown" + * would cause a database crash cycle (forcing WAL replay at restart) + * if any sessions are in authentication. So we live with it for now. + * + * One might be tempted to try to send a message indicating why we are + * disconnecting. However, that would make this even more unsafe. Also, + * it seems undesirable to provide clues about the database's state to + * a client that has not yet completed authentication. */ static void -startup_die(SIGNAL_ARGS) +process_startup_packet_die(SIGNAL_ARGS) { proc_exit(1); } @@ -5389,7 +5404,11 @@ dummy_handler(SIGNAL_ARGS) /* * Timeout while processing startup packet. - * As for startup_die(), we clean up and exit(1). + * As for process_startup_packet_die(), we clean up and exit(1). + * + * This is theoretically just as hazardous as in process_startup_packet_die(), + * although in practice we're almost certainly waiting for client input, + * which greatly reduces the risk. */ static void StartupPacketTimeoutHandler(void) From 9f1cf97bb5387a6243c8a6c9725616ef7447962e Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Thu, 10 Sep 2020 19:37:02 -0300 Subject: [PATCH 127/589] Print WAL logical message contents in pg_waldump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This helps debuggability when looking at WAL streams containing logical messages. Author: Ashutosh Bapat Reviewed-by: Álvaro Herrera Discussion: https://postgr.es/m/CAExHW5sWx49rKmXbg5H1Xc1t+nRv9PaYKQmgw82HPt6vWDVmDg@mail.gmail.com --- src/backend/access/rmgrdesc/logicalmsgdesc.c | 17 ++++++++++++++--- src/backend/replication/logical/message.c | 1 + src/include/replication/message.h | 5 ++--- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/backend/access/rmgrdesc/logicalmsgdesc.c b/src/backend/access/rmgrdesc/logicalmsgdesc.c index bff298c9287f..83ab93a24be9 100644 --- a/src/backend/access/rmgrdesc/logicalmsgdesc.c +++ b/src/backend/access/rmgrdesc/logicalmsgdesc.c @@ -24,10 +24,21 @@ logicalmsg_desc(StringInfo buf, XLogReaderState *record) if (info == XLOG_LOGICAL_MESSAGE) { xl_logical_message *xlrec = (xl_logical_message *) rec; + char *prefix = xlrec->message; + char *message = xlrec->message + xlrec->prefix_size; + char *sep = ""; - appendStringInfo(buf, "%s message size %zu bytes", - xlrec->transactional ? "transactional" : "nontransactional", - xlrec->message_size); + Assert(prefix[xlrec->prefix_size] != '\0'); + + appendStringInfo(buf, "%s, prefix \"%s\"; payload (%zu bytes): ", + xlrec->transactional ? "transactional" : "non-transactional", + prefix, xlrec->message_size); + /* Write message payload as a series of hex bytes */ + for (int cnt = 0; cnt < xlrec->message_size; cnt++) + { + appendStringInfo(buf, "%s%02X", sep, (unsigned char) message[cnt]); + sep = " "; + } } } diff --git a/src/backend/replication/logical/message.c b/src/backend/replication/logical/message.c index db33cbe5a7a2..bd4b08543e66 100644 --- a/src/backend/replication/logical/message.c +++ b/src/backend/replication/logical/message.c @@ -59,6 +59,7 @@ LogLogicalMessage(const char *prefix, const char *message, size_t size, xlrec.dbId = MyDatabaseId; xlrec.transactional = transactional; + /* trailing zero is critical; see logicalmsg_desc */ xlrec.prefix_size = strlen(prefix) + 1; xlrec.message_size = size; diff --git a/src/include/replication/message.h b/src/include/replication/message.h index 937addde4858..e97891ebcafa 100644 --- a/src/include/replication/message.h +++ b/src/include/replication/message.h @@ -23,9 +23,8 @@ typedef struct xl_logical_message bool transactional; /* is message transactional? */ Size prefix_size; /* length of prefix */ Size message_size; /* size of the message */ - char message[FLEXIBLE_ARRAY_MEMBER]; /* message including the null - * terminated prefix of length - * prefix_size */ + /* payload, including null-terminated prefix of length prefix_size */ + char message[FLEXIBLE_ARRAY_MEMBER]; } xl_logical_message; #define SizeOfLogicalMessage (offsetof(xl_logical_message, message)) From 0ba5181c00eb0216bdfd9afbf3d680fee67d34b3 Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Fri, 11 Sep 2020 10:00:01 +0530 Subject: [PATCH 128/589] Skip empty transaction stream in test_decoding. We were decoding empty transactions via streaming APIs added in commit 45fdc9738b even when the user used the option 'skip-empty-xacts'. The APIs makes no effort to skip empty xacts under the assumption that we will never try to stream such transactions. However, that is not true because we can pick to stream a transaction that has change messages for REORDER_BUFFER_CHANGE_INTERNAL_SNAPSHOT and we don't send such messages to downstream rather they are just to update the internal state. So, we need to skip such xacts when plugin uses the option 'skip-empty-xacts'. Diagnosed-By: Amit Kapila Author: Dilip Kumar Reviewed-by: Amit Kapila Discussion: https://postgr.es/m/CAA4eK1+OqgFNZkf7=ETe_y5ntjgDk3T0wcdkd4Sot_u1hySGfw@mail.gmail.com --- contrib/test_decoding/Makefile | 2 +- .../expected/concurrent_stream.out | 19 +++++++ contrib/test_decoding/expected/stream.out | 5 +- .../specs/concurrent_stream.spec | 37 +++++++++++++ contrib/test_decoding/test_decoding.c | 55 +++++++++++++------ 5 files changed, 95 insertions(+), 23 deletions(-) create mode 100644 contrib/test_decoding/expected/concurrent_stream.out create mode 100644 contrib/test_decoding/specs/concurrent_stream.spec diff --git a/contrib/test_decoding/Makefile b/contrib/test_decoding/Makefile index ed9a3d6c0ede..f23f15b04d4a 100644 --- a/contrib/test_decoding/Makefile +++ b/contrib/test_decoding/Makefile @@ -7,7 +7,7 @@ REGRESS = ddl xact rewrite toast permissions decoding_in_xact \ decoding_into_rel binary prepared replorigin time messages \ spill slot truncate stream ISOLATION = mxact delayed_startup ondisk_startup concurrent_ddl_dml \ - oldest_xmin snapshot_transfer subxact_without_top + oldest_xmin snapshot_transfer subxact_without_top concurrent_stream REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/test_decoding/logical.conf ISOLATION_OPTS = --temp-config $(top_srcdir)/contrib/test_decoding/logical.conf diff --git a/contrib/test_decoding/expected/concurrent_stream.out b/contrib/test_decoding/expected/concurrent_stream.out new file mode 100644 index 000000000000..e731d13d8fa9 --- /dev/null +++ b/contrib/test_decoding/expected/concurrent_stream.out @@ -0,0 +1,19 @@ +Parsed test spec with 2 sessions + +starting permutation: s0_begin s0_ddl s1_ddl s1_begin s1_toast_insert s1_commit s1_get_stream_changes +step s0_begin: BEGIN; +step s0_ddl: CREATE TABLE stream_test1(data text); +step s1_ddl: CREATE TABLE stream_test(data text); +step s1_begin: BEGIN; +step s1_toast_insert: INSERT INTO stream_test SELECT large_val(); +step s1_commit: COMMIT; +step s1_get_stream_changes: SELECT data FROM pg_logical_slot_get_changes('isolation_slot', NULL,NULL, 'include-xids', '0', 'skip-empty-xacts', '1', 'stream-changes', '1'); +data + +opening a streamed block for transaction +streaming change for transaction +closing a streamed block for transaction +committing streamed transaction +?column? + +stop diff --git a/contrib/test_decoding/expected/stream.out b/contrib/test_decoding/expected/stream.out index d7e32f818546..e1c3bc838d5e 100644 --- a/contrib/test_decoding/expected/stream.out +++ b/contrib/test_decoding/expected/stream.out @@ -29,10 +29,7 @@ COMMIT; SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL, 'include-xids', '0', 'skip-empty-xacts', '1', 'stream-changes', '1'); data ---------------------------------------------------------- - opening a streamed block for transaction streaming message: transactional: 1 prefix: test, sz: 50 - closing a streamed block for transaction - aborting streamed (sub)transaction opening a streamed block for transaction streaming change for transaction streaming change for transaction @@ -56,7 +53,7 @@ SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL, 'incl streaming change for transaction closing a streamed block for transaction committing streamed transaction -(27 rows) +(24 rows) -- streaming test for toast changes ALTER TABLE stream_test ALTER COLUMN data set storage external; diff --git a/contrib/test_decoding/specs/concurrent_stream.spec b/contrib/test_decoding/specs/concurrent_stream.spec new file mode 100644 index 000000000000..ad9fde9c2844 --- /dev/null +++ b/contrib/test_decoding/specs/concurrent_stream.spec @@ -0,0 +1,37 @@ +# Test decoding of in-progress transaction containing dml and a concurrent +# transaction with ddl operation. The transaction containing ddl operation +# should not get streamed as it doesn't have any changes. + +setup +{ + SELECT 'init' FROM pg_create_logical_replication_slot('isolation_slot', 'test_decoding'); + + -- consume DDL + SELECT data FROM pg_logical_slot_get_changes('isolation_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); + CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS 'select array_agg(md5(g::text))::text from generate_series(1, 80000) g'; +} + +teardown +{ + DROP TABLE IF EXISTS stream_test; + DROP TABLE IF EXISTS stream_test1; + SELECT 'stop' FROM pg_drop_replication_slot('isolation_slot'); +} + +session "s0" +setup { SET synchronous_commit=on; } +step "s0_begin" { BEGIN; } +step "s0_ddl" {CREATE TABLE stream_test1(data text);} + +# The transaction commit for s1_ddl will add the INTERNAL_SNAPSHOT change to +# the currently running s0_ddl and we want to test that s0_ddl should not get +# streamed when user asked to skip-empty-xacts. +session "s1" +setup { SET synchronous_commit=on; } +step "s1_ddl" { CREATE TABLE stream_test(data text); } +step "s1_begin" { BEGIN; } +step "s1_toast_insert" {INSERT INTO stream_test SELECT large_val();} +step "s1_commit" { COMMIT; } +step "s1_get_stream_changes" { SELECT data FROM pg_logical_slot_get_changes('isolation_slot', NULL,NULL, 'include-xids', '0', 'skip-empty-xacts', '1', 'stream-changes', '1');} + +permutation "s0_begin" "s0_ddl" "s1_ddl" "s1_begin" "s1_toast_insert" "s1_commit" "s1_get_stream_changes" diff --git a/contrib/test_decoding/test_decoding.c b/contrib/test_decoding/test_decoding.c index 34745150e9ba..e60ab34a5a7c 100644 --- a/contrib/test_decoding/test_decoding.c +++ b/contrib/test_decoding/test_decoding.c @@ -64,6 +64,10 @@ static void pg_decode_message(LogicalDecodingContext *ctx, Size sz, const char *message); static void pg_decode_stream_start(LogicalDecodingContext *ctx, ReorderBufferTXN *txn); +static void pg_output_stream_start(LogicalDecodingContext *ctx, + TestDecodingData *data, + ReorderBufferTXN *txn, + bool last_write); static void pg_decode_stream_stop(LogicalDecodingContext *ctx, ReorderBufferTXN *txn); static void pg_decode_stream_abort(LogicalDecodingContext *ctx, @@ -583,34 +587,38 @@ pg_decode_message(LogicalDecodingContext *ctx, OutputPluginWrite(ctx, true); } -/* - * We never try to stream any empty xact so we don't need any special handling - * for skip_empty_xacts in streaming mode APIs. - */ static void pg_decode_stream_start(LogicalDecodingContext *ctx, ReorderBufferTXN *txn) { TestDecodingData *data = ctx->output_plugin_private; - OutputPluginPrepareWrite(ctx, true); + data->xact_wrote_changes = false; + if (data->skip_empty_xacts) + return; + pg_output_stream_start(ctx, data, txn, true); +} + +static void +pg_output_stream_start(LogicalDecodingContext *ctx, TestDecodingData *data, ReorderBufferTXN *txn, bool last_write) +{ + OutputPluginPrepareWrite(ctx, last_write); if (data->include_xids) appendStringInfo(ctx->out, "opening a streamed block for transaction TXN %u", txn->xid); else appendStringInfo(ctx->out, "opening a streamed block for transaction"); - OutputPluginWrite(ctx, true); + OutputPluginWrite(ctx, last_write); } -/* - * We never try to stream any empty xact so we don't need any special handling - * for skip_empty_xacts in streaming mode APIs. - */ static void pg_decode_stream_stop(LogicalDecodingContext *ctx, ReorderBufferTXN *txn) { TestDecodingData *data = ctx->output_plugin_private; + if (data->skip_empty_xacts && !data->xact_wrote_changes) + return; + OutputPluginPrepareWrite(ctx, true); if (data->include_xids) appendStringInfo(ctx->out, "closing a streamed block for transaction TXN %u", txn->xid); @@ -619,10 +627,6 @@ pg_decode_stream_stop(LogicalDecodingContext *ctx, OutputPluginWrite(ctx, true); } -/* - * We never try to stream any empty xact so we don't need any special handling - * for skip_empty_xacts in streaming mode APIs. - */ static void pg_decode_stream_abort(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, @@ -630,6 +634,9 @@ pg_decode_stream_abort(LogicalDecodingContext *ctx, { TestDecodingData *data = ctx->output_plugin_private; + if (data->skip_empty_xacts && !data->xact_wrote_changes) + return; + OutputPluginPrepareWrite(ctx, true); if (data->include_xids) appendStringInfo(ctx->out, "aborting streamed (sub)transaction TXN %u", txn->xid); @@ -638,10 +645,6 @@ pg_decode_stream_abort(LogicalDecodingContext *ctx, OutputPluginWrite(ctx, true); } -/* - * We never try to stream any empty xact so we don't need any special handling - * for skip_empty_xacts in streaming mode APIs. - */ static void pg_decode_stream_commit(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, @@ -649,6 +652,9 @@ pg_decode_stream_commit(LogicalDecodingContext *ctx, { TestDecodingData *data = ctx->output_plugin_private; + if (data->skip_empty_xacts && !data->xact_wrote_changes) + return; + OutputPluginPrepareWrite(ctx, true); if (data->include_xids) @@ -676,6 +682,13 @@ pg_decode_stream_change(LogicalDecodingContext *ctx, { TestDecodingData *data = ctx->output_plugin_private; + /* output stream start if we haven't yet */ + if (data->skip_empty_xacts && !data->xact_wrote_changes) + { + pg_output_stream_start(ctx, data, txn, false); + } + data->xact_wrote_changes = true; + OutputPluginPrepareWrite(ctx, true); if (data->include_xids) appendStringInfo(ctx->out, "streaming change for TXN %u", txn->xid); @@ -722,6 +735,12 @@ pg_decode_stream_truncate(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, { TestDecodingData *data = ctx->output_plugin_private; + if (data->skip_empty_xacts && !data->xact_wrote_changes) + { + pg_output_stream_start(ctx, data, txn, false); + } + data->xact_wrote_changes = true; + OutputPluginPrepareWrite(ctx, true); if (data->include_xids) appendStringInfo(ctx->out, "streaming truncate for TXN %u", txn->xid); From 6a68a233cef86d81b35fb4c1c21a094768875af2 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Fri, 11 Sep 2020 12:53:25 -0300 Subject: [PATCH 129/589] Update copyright year Thinko in 40b3e2c201af. Reported-by: "Wang, Shenhao" Discussion: https://postgr.es/m/ed98706b82694b57a8c0d339a10732aa@G08CNEXMBPEKD06.g08.fujitsu.local --- src/backend/catalog/pg_cast.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/catalog/pg_cast.c b/src/backend/catalog/pg_cast.c index c03e82d74fef..d3f2db41863b 100644 --- a/src/backend/catalog/pg_cast.c +++ b/src/backend/catalog/pg_cast.c @@ -3,7 +3,7 @@ * pg_cast.c * routines to support manipulation of the pg_cast relation * - * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * * From 6693a96b329ec46f1df916f2a28d640cc9a9977d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 11 Sep 2020 12:20:16 -0400 Subject: [PATCH 130/589] Don't run atexit callbacks during signal exits from ProcessStartupPacket. Although 58c6feccf fixed the case for SIGQUIT, we were still calling proc_exit() from signal handlers for SIGTERM and timeout failures in ProcessStartupPacket. Fortunately, at the point where that code runs, we haven't yet connected to shared memory in any meaningful way, so there is nothing we need to undo in shared memory. This means it should be safe to use _exit(1) here, ie, not run any atexit handlers but also inform the postmaster that it's not a crash exit. To make sure nobody breaks the "nothing to undo" expectation, add a cross-check that no on-shmem-exit or before-shmem-exit handlers have been registered yet when we finish using these signal handlers. This change is simple enough that maybe it could be back-patched, but I won't risk that right now. Discussion: https://postgr.es/m/1850884.1599601164@sss.pgh.pa.us --- src/backend/postmaster/postmaster.c | 72 +++++++++++++---------------- src/backend/storage/ipc/ipc.c | 17 +++++++ src/include/storage/ipc.h | 1 + 3 files changed, 51 insertions(+), 39 deletions(-) diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 15ad675bc136..081022a20658 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -4298,6 +4298,8 @@ report_fork_failure_to_client(Port *port, int errnum) * returns: nothing. Will not return at all if there's any failure. * * Note: this code does not depend on having any access to shared memory. + * Indeed, our approach to SIGTERM/timeout handling *requires* that + * shared memory not have been touched yet; see comments within. * In the EXEC_BACKEND case, we are physically attached to shared memory * but have not yet set up most of our local pointers to shmem structures. */ @@ -4341,27 +4343,14 @@ BackendInitialize(Port *port) whereToSendOutput = DestRemote; /* now safe to ereport to client */ /* - * We arrange to do proc_exit(1) if we receive SIGTERM or timeout while - * trying to collect the startup packet; while SIGQUIT results in - * _exit(2). Otherwise the postmaster cannot shutdown the database FAST - * or IMMED cleanly if a buggy client fails to send the packet promptly. + * We arrange to do _exit(1) if we receive SIGTERM or timeout while trying + * to collect the startup packet; while SIGQUIT results in _exit(2). + * Otherwise the postmaster cannot shutdown the database FAST or IMMED + * cleanly if a buggy client fails to send the packet promptly. * - * XXX this is pretty dangerous; signal handlers should not call anything - * as complex as proc_exit() directly. We minimize the hazard by not - * keeping these handlers active for longer than we must. However, it - * seems necessary to be able to escape out of DNS lookups as well as the - * startup packet reception proper, so we can't narrow the scope further - * than is done here. - * - * XXX it follows that the remainder of this function must tolerate losing - * control at any instant. Likewise, any pg_on_exit_callback registered - * before or during this function must be prepared to execute at any - * instant between here and the end of this function. Furthermore, - * affected callbacks execute partially or not at all when a second - * exit-inducing signal arrives after proc_exit_prepare() decrements - * on_proc_exit_index. (Thanks to that mechanic, callbacks need not - * anticipate more than one call.) This is fragile; it ought to instead - * follow the norm of handling interrupts at selected, safe opportunities. + * Exiting with _exit(1) is only possible because we have not yet touched + * shared memory; therefore no outside-the-process state needs to get + * cleaned up. */ pqsignal(SIGTERM, process_startup_packet_die); pqsignal(SIGQUIT, SignalHandlerForCrashExit); @@ -4420,8 +4409,8 @@ BackendInitialize(Port *port) port->remote_hostname = strdup(remote_host); /* - * Ready to begin client interaction. We will give up and proc_exit(1) - * after a time delay, so that a broken client can't hog a connection + * Ready to begin client interaction. We will give up and _exit(1) after + * a time delay, so that a broken client can't hog a connection * indefinitely. PreAuthDelay and any DNS interactions above don't count * against the time limit. * @@ -4449,6 +4438,17 @@ BackendInitialize(Port *port) disable_timeout(STARTUP_PACKET_TIMEOUT, false); PG_SETMASK(&BlockSig); + /* + * As a safety check that nothing in startup has yet performed + * shared-memory modifications that would need to be undone if we had + * exited through SIGTERM or timeout above, check that no on_shmem_exit + * handlers have been registered yet. (This isn't terribly bulletproof, + * since someone might misuse an on_proc_exit handler for shmem cleanup, + * but it's a cheap and helpful check. We cannot disallow on_proc_exit + * handlers unfortunately, since pq_init() already registered one.) + */ + check_on_shmem_exit_lists_are_empty(); + /* * Stop here if it was bad or a cancel packet. ProcessStartupPacket * already did any appropriate error reporting. @@ -5369,23 +5369,21 @@ sigusr1_handler(SIGNAL_ARGS) /* * SIGTERM while processing startup packet. - * Clean up and exit(1). * - * Running proc_exit() from a signal handler is pretty unsafe, since we - * can't know what code we've interrupted. But the alternative of using - * _exit(2) is also unpalatable, since it'd mean that a "fast shutdown" - * would cause a database crash cycle (forcing WAL replay at restart) - * if any sessions are in authentication. So we live with it for now. + * Running proc_exit() from a signal handler would be quite unsafe. + * However, since we have not yet touched shared memory, we can just + * pull the plug and exit without running any atexit handlers. * - * One might be tempted to try to send a message indicating why we are - * disconnecting. However, that would make this even more unsafe. Also, - * it seems undesirable to provide clues about the database's state to - * a client that has not yet completed authentication. + * One might be tempted to try to send a message, or log one, indicating + * why we are disconnecting. However, that would be quite unsafe in itself. + * Also, it seems undesirable to provide clues about the database's state + * to a client that has not yet completed authentication, or even sent us + * a startup packet. */ static void process_startup_packet_die(SIGNAL_ARGS) { - proc_exit(1); + _exit(1); } /* @@ -5404,16 +5402,12 @@ dummy_handler(SIGNAL_ARGS) /* * Timeout while processing startup packet. - * As for process_startup_packet_die(), we clean up and exit(1). - * - * This is theoretically just as hazardous as in process_startup_packet_die(), - * although in practice we're almost certainly waiting for client input, - * which greatly reduces the risk. + * As for process_startup_packet_die(), we exit via _exit(1). */ static void StartupPacketTimeoutHandler(void) { - proc_exit(1); + _exit(1); } diff --git a/src/backend/storage/ipc/ipc.c b/src/backend/storage/ipc/ipc.c index 11c3f132a100..36a067c9244c 100644 --- a/src/backend/storage/ipc/ipc.c +++ b/src/backend/storage/ipc/ipc.c @@ -416,3 +416,20 @@ on_exit_reset(void) on_proc_exit_index = 0; reset_on_dsm_detach(); } + +/* ---------------------------------------------------------------- + * check_on_shmem_exit_lists_are_empty + * + * Debugging check that no shmem cleanup handlers have been registered + * prematurely in the current process. + * ---------------------------------------------------------------- + */ +void +check_on_shmem_exit_lists_are_empty(void) +{ + if (before_shmem_exit_index) + elog(FATAL, "before_shmem_exit has been called prematurely"); + if (on_shmem_exit_index) + elog(FATAL, "on_shmem_exit has been called prematurely"); + /* Checking DSM detach state seems unnecessary given the above */ +} diff --git a/src/include/storage/ipc.h b/src/include/storage/ipc.h index 462fe463417c..88994fdc260b 100644 --- a/src/include/storage/ipc.h +++ b/src/include/storage/ipc.h @@ -72,6 +72,7 @@ extern void on_shmem_exit(pg_on_exit_callback function, Datum arg); extern void before_shmem_exit(pg_on_exit_callback function, Datum arg); extern void cancel_before_shmem_exit(pg_on_exit_callback function, Datum arg); extern void on_exit_reset(void); +extern void check_on_shmem_exit_lists_are_empty(void); /* ipci.c */ extern PGDLLIMPORT shmem_startup_hook_type shmem_startup_hook; From 10095ca634fb39d78cfae8000489a19f4f4e27ef Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 11 Sep 2020 12:24:46 -0400 Subject: [PATCH 131/589] Log a message when resorting to SIGKILL during shutdown/crash recovery. Currently, no useful trace is left in the logs when the postmaster is forced to use SIGKILL to shut down children that failed to respond to SIGQUIT. Some questions were raised about how often that scenario happens in the buildfarm, so let's add a LOG-level message showing that it happened. Discussion: https://postgr.es/m/1850884.1599601164@sss.pgh.pa.us --- src/backend/postmaster/postmaster.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 081022a20658..3cd6fa30eb0a 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -1850,6 +1850,8 @@ ServerLoop(void) (now - AbortStartTime) >= SIGKILL_CHILDREN_AFTER_SECS) { /* We were gentle with them before. Not anymore */ + ereport(LOG, + (errmsg("issuing SIGKILL to recalcitrant children"))); TerminateChildren(SIGKILL); /* reset flag so we don't SIGKILL again */ AbortStartTime = 0; From 3c99230b4f0d10c9eac5f4efdd2394eccb2af3a0 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Fri, 11 Sep 2020 16:15:47 -0300 Subject: [PATCH 132/589] psql: Display stats target of extended statistics The stats target can be set since commit d06215d03, but wasn't shown by psql. Author: Justin Pryzby Discussion: https://postgr.es/m/20200831050047.GG5450@telsasoft.com Reviewed-by: Georgios Kokolatos Reviewed-by: Tatsuro Yamada --- src/bin/psql/describe.c | 14 ++++++++++++-- src/test/regress/expected/stats_ext.out | 18 ++++++++++++++++++ src/test/regress/sql/stats_ext.sql | 2 ++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 0861d74a6fe0..f22d907b1f9f 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -2683,8 +2683,13 @@ describeOneTableDetails(const char *schemaname, " a.attnum = s.attnum AND NOT attisdropped)) AS columns,\n" " 'd' = any(stxkind) AS ndist_enabled,\n" " 'f' = any(stxkind) AS deps_enabled,\n" - " 'm' = any(stxkind) AS mcv_enabled\n" - "FROM pg_catalog.pg_statistic_ext stat " + " 'm' = any(stxkind) AS mcv_enabled,\n"); + + if (pset.sversion >= 130000) + appendPQExpBufferStr(&buf, " stxstattarget\n"); + else + appendPQExpBufferStr(&buf, " -1 AS stxstattarget\n"); + appendPQExpBuffer(&buf, "FROM pg_catalog.pg_statistic_ext stat\n" "WHERE stxrelid = '%s'\n" "ORDER BY 1;", oid); @@ -2732,6 +2737,11 @@ describeOneTableDetails(const char *schemaname, PQgetvalue(result, i, 4), PQgetvalue(result, i, 1)); + /* Show the stats target if it's not default */ + if (strcmp(PQgetvalue(result, i, 8), "-1") != 0) + appendPQExpBuffer(&buf, "; STATISTICS %s", + PQgetvalue(result, i, 8)); + printTableAddFooter(&cont, buf.data); } } diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out index 8c667d786a21..4c3edd213fb8 100644 --- a/src/test/regress/expected/stats_ext.out +++ b/src/test/regress/expected/stats_ext.out @@ -102,6 +102,15 @@ WARNING: statistics object "public.ab1_a_b_stats" could not be computed for rel ALTER TABLE ab1 ALTER a SET STATISTICS -1; -- setting statistics target 0 skips the statistics, without printing any message, so check catalog ALTER STATISTICS ab1_a_b_stats SET STATISTICS 0; +\d ab1 + Table "public.ab1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | +Statistics objects: + "public"."ab1_a_b_stats" (ndistinct, dependencies, mcv) ON a, b FROM ab1; STATISTICS 0 + ANALYZE ab1; SELECT stxname, stxdndistinct, stxddependencies, stxdmcv FROM pg_statistic_ext s, pg_statistic_ext_data d @@ -113,6 +122,15 @@ SELECT stxname, stxdndistinct, stxddependencies, stxdmcv (1 row) ALTER STATISTICS ab1_a_b_stats SET STATISTICS -1; +\d+ ab1 + Table "public.ab1" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------+---------+-----------+----------+---------+---------+--------------+------------- + a | integer | | | | plain | | + b | integer | | | | plain | | +Statistics objects: + "public"."ab1_a_b_stats" (ndistinct, dependencies, mcv) ON a, b FROM ab1 + -- partial analyze doesn't build stats either ANALYZE ab1 (a); WARNING: statistics object "public.ab1_a_b_stats" could not be computed for relation "public.ab1" diff --git a/src/test/regress/sql/stats_ext.sql b/src/test/regress/sql/stats_ext.sql index f8d947af9e80..9781e590a30c 100644 --- a/src/test/regress/sql/stats_ext.sql +++ b/src/test/regress/sql/stats_ext.sql @@ -72,12 +72,14 @@ ANALYZE ab1; ALTER TABLE ab1 ALTER a SET STATISTICS -1; -- setting statistics target 0 skips the statistics, without printing any message, so check catalog ALTER STATISTICS ab1_a_b_stats SET STATISTICS 0; +\d ab1 ANALYZE ab1; SELECT stxname, stxdndistinct, stxddependencies, stxdmcv FROM pg_statistic_ext s, pg_statistic_ext_data d WHERE s.stxname = 'ab1_a_b_stats' AND d.stxoid = s.oid; ALTER STATISTICS ab1_a_b_stats SET STATISTICS -1; +\d+ ab1 -- partial analyze doesn't build stats either ANALYZE ab1 (a); ANALYZE ab1; From 7634bd4f6d38bdef1fe442df5c2e0da73f1f90f4 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 11 Sep 2020 16:01:28 -0400 Subject: [PATCH 133/589] Accept SIGQUIT during error recovery in auxiliary processes. The bgwriter, checkpointer, walwriter, and walreceiver processes claimed to allow SIGQUIT "at all times". In reality SIGQUIT would get re-blocked during error recovery, because we didn't update the actual signal mask immediately, so sigsetjmp() would save and reinstate a mask that includes SIGQUIT. This appears to be simply a coding oversight. There's never a good reason to hold off SIGQUIT in these processes, because it's going to just call _exit(2) which should be safe enough, especially since the postmaster is going to tear down shared memory afterwards. Hence, stick in PG_SETMASK() calls to install the modified BlockSig mask immediately. Also try to improve the comments around sigsetjmp blocks. Most of them were just referencing postgres.c, which is misleading because actually postgres.c manages the signals differently. No back-patch, since there's no evidence that this is causing any problems in the field. Discussion: https://postgr.es/m/CALDaNm1d1hHPZUg3xU4XjtWBOLCrA+-2cJcLpw-cePZ=GgDVfA@mail.gmail.com --- src/backend/postmaster/autovacuum.c | 16 +++++++++++++++- src/backend/postmaster/bgwriter.c | 18 ++++++++++++++++-- src/backend/postmaster/checkpointer.c | 18 ++++++++++++++++-- src/backend/postmaster/walwriter.c | 18 ++++++++++++++++-- src/backend/replication/walreceiver.c | 3 ++- 5 files changed, 65 insertions(+), 8 deletions(-) diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index 1b8cd7bacd43..19ba26b914e9 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -495,6 +495,12 @@ AutoVacLauncherMain(int argc, char *argv[]) * If an exception is encountered, processing resumes here. * * This code is a stripped down version of PostgresMain error recovery. + * + * Note that we use sigsetjmp(..., 1), so that the prevailing signal mask + * (to wit, BlockSig) will be restored when longjmp'ing to here. Thus, + * signals will be blocked until we complete error recovery. It might + * seem that this policy makes the HOLD_INTERRUPTS() call redundant, but + * it is not since InterruptPending might be set already. */ if (sigsetjmp(local_sigjmp_buf, 1) != 0) { @@ -1550,7 +1556,15 @@ AutoVacWorkerMain(int argc, char *argv[]) /* * If an exception is encountered, processing resumes here. * - * See notes in postgres.c about the design of this coding. + * Unlike most auxiliary processes, we don't attempt to continue + * processing after an error; we just clean up and exit. The autovac + * launcher is responsible for spawning another worker later. + * + * Note that we use sigsetjmp(..., 1), so that the prevailing signal mask + * (to wit, BlockSig) will be restored when longjmp'ing to here. Thus, + * signals will be blocked until we exit. It might seem that this policy + * makes the HOLD_INTERRUPTS() call redundant, but it is not since + * InterruptPending might be set already. */ if (sigsetjmp(local_sigjmp_buf, 1) != 0) { diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c index 069e27e427fe..c96568149fe0 100644 --- a/src/backend/postmaster/bgwriter.c +++ b/src/backend/postmaster/bgwriter.c @@ -115,8 +115,9 @@ BackgroundWriterMain(void) */ pqsignal(SIGCHLD, SIG_DFL); - /* We allow SIGQUIT (quickdie) at all times */ + /* We allow SIGQUIT (SignalHandlerForCrashExit) at all times */ sigdelset(&BlockSig, SIGQUIT); + PG_SETMASK(&BlockSig); /* * We just started, assume there has been either a shutdown or @@ -140,7 +141,20 @@ BackgroundWriterMain(void) /* * If an exception is encountered, processing resumes here. * - * See notes in postgres.c about the design of this coding. + * You might wonder why this isn't coded as an infinite loop around a + * PG_TRY construct. The reason is that this is the bottom of the + * exception stack, and so with PG_TRY there would be no exception handler + * in force at all during the CATCH part. By leaving the outermost setjmp + * always active, we have at least some chance of recovering from an error + * during error recovery. (If we get into an infinite loop thereby, it + * will soon be stopped by overflow of elog.c's internal state stack.) + * + * Note that we use sigsetjmp(..., 1), so that the prevailing signal mask + * (to wit, BlockSig) will be restored when longjmp'ing to here. Thus, + * signals other than SIGQUIT will be blocked until we complete error + * recovery. It might seem that this policy makes the HOLD_INTERRUPTS() + * call redundant, but it is not since InterruptPending might be set + * already. */ if (sigsetjmp(local_sigjmp_buf, 1) != 0) { diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c index 624a3238b804..45f5deca72ee 100644 --- a/src/backend/postmaster/checkpointer.c +++ b/src/backend/postmaster/checkpointer.c @@ -209,8 +209,9 @@ CheckpointerMain(void) */ pqsignal(SIGCHLD, SIG_DFL); - /* We allow SIGQUIT (quickdie) at all times */ + /* We allow SIGQUIT (SignalHandlerForCrashExit) at all times */ sigdelset(&BlockSig, SIGQUIT); + PG_SETMASK(&BlockSig); /* * Initialize so that first time-driven event happens at the correct time. @@ -231,7 +232,20 @@ CheckpointerMain(void) /* * If an exception is encountered, processing resumes here. * - * See notes in postgres.c about the design of this coding. + * You might wonder why this isn't coded as an infinite loop around a + * PG_TRY construct. The reason is that this is the bottom of the + * exception stack, and so with PG_TRY there would be no exception handler + * in force at all during the CATCH part. By leaving the outermost setjmp + * always active, we have at least some chance of recovering from an error + * during error recovery. (If we get into an infinite loop thereby, it + * will soon be stopped by overflow of elog.c's internal state stack.) + * + * Note that we use sigsetjmp(..., 1), so that the prevailing signal mask + * (to wit, BlockSig) will be restored when longjmp'ing to here. Thus, + * signals other than SIGQUIT will be blocked until we complete error + * recovery. It might seem that this policy makes the HOLD_INTERRUPTS() + * call redundant, but it is not since InterruptPending might be set + * already. */ if (sigsetjmp(local_sigjmp_buf, 1) != 0) { diff --git a/src/backend/postmaster/walwriter.c b/src/backend/postmaster/walwriter.c index 45a2757969be..358c0916ac23 100644 --- a/src/backend/postmaster/walwriter.c +++ b/src/backend/postmaster/walwriter.c @@ -112,8 +112,9 @@ WalWriterMain(void) */ pqsignal(SIGCHLD, SIG_DFL); - /* We allow SIGQUIT (quickdie) at all times */ + /* We allow SIGQUIT (SignalHandlerForCrashExit) at all times */ sigdelset(&BlockSig, SIGQUIT); + PG_SETMASK(&BlockSig); /* * Create a memory context that we will do all our work in. We do this so @@ -129,7 +130,20 @@ WalWriterMain(void) /* * If an exception is encountered, processing resumes here. * - * This code is heavily based on bgwriter.c, q.v. + * You might wonder why this isn't coded as an infinite loop around a + * PG_TRY construct. The reason is that this is the bottom of the + * exception stack, and so with PG_TRY there would be no exception handler + * in force at all during the CATCH part. By leaving the outermost setjmp + * always active, we have at least some chance of recovering from an error + * during error recovery. (If we get into an infinite loop thereby, it + * will soon be stopped by overflow of elog.c's internal state stack.) + * + * Note that we use sigsetjmp(..., 1), so that the prevailing signal mask + * (to wit, BlockSig) will be restored when longjmp'ing to here. Thus, + * signals other than SIGQUIT will be blocked until we complete error + * recovery. It might seem that this policy makes the HOLD_INTERRUPTS() + * call redundant, but it is not since InterruptPending might be set + * already. */ if (sigsetjmp(local_sigjmp_buf, 1) != 0) { diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 7c11e1ab44cb..b180598507f1 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -279,8 +279,9 @@ WalReceiverMain(void) /* Reset some signals that are accepted by postmaster but not here */ pqsignal(SIGCHLD, SIG_DFL); - /* We allow SIGQUIT (quickdie) at all times */ + /* We allow SIGQUIT (SignalHandlerForCrashExit) at all times */ sigdelset(&BlockSig, SIGQUIT); + PG_SETMASK(&BlockSig); /* Load the libpq-specific functions */ load_file("libpqwalreceiver", false); From 07589649639410032df281e98469db88a0b86271 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Fri, 11 Sep 2020 17:10:02 -0700 Subject: [PATCH 134/589] logtape.c: do not preallocate for tapes when sorting The preallocation logic is only useful for HashAgg, so disable it when sorting. Also, adjust an out-of-date comment. Reviewed-by: Peter Geoghegan Discussion: https://postgr.es/m/CAH2-Wzn_o7tE2+hRVvwSFghRb75AJ5g-nqGzDUqLYMexjOAe=g@mail.gmail.com Backpatch-through: 13 --- src/backend/executor/nodeAgg.c | 2 +- src/backend/utils/sort/logtape.c | 38 ++++++++++++++++++++---------- src/backend/utils/sort/tuplesort.c | 7 +++--- src/include/utils/logtape.h | 3 ++- 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c index 9776263ae75a..f74d4841f170 100644 --- a/src/backend/executor/nodeAgg.c +++ b/src/backend/executor/nodeAgg.c @@ -2882,7 +2882,7 @@ hashagg_tapeinfo_init(AggState *aggstate) HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo)); int init_tapes = 16; /* expanded dynamically */ - tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, NULL, NULL, -1); + tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1); tapeinfo->ntapes = init_tapes; tapeinfo->nfreetapes = init_tapes; tapeinfo->freetapes_alloc = init_tapes; diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c index bbb01f6d3373..d6d1e1911ea4 100644 --- a/src/backend/utils/sort/logtape.c +++ b/src/backend/utils/sort/logtape.c @@ -212,6 +212,7 @@ struct LogicalTapeSet long *freeBlocks; /* resizable array holding minheap */ long nFreeBlocks; /* # of currently free blocks */ Size freeBlocksLen; /* current allocated length of freeBlocks[] */ + bool enable_prealloc; /* preallocate write blocks? */ /* The array of logical tapes. */ int nTapes; /* # of logical tapes in set */ @@ -220,6 +221,7 @@ struct LogicalTapeSet static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer); static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer); +static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt); static long ltsGetFreeBlock(LogicalTapeSet *lts); static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt); static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum); @@ -242,12 +244,8 @@ ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer) * that's past the current end of file, fill the space between the current * end of file and the target block with zeros. * - * This should happen rarely, otherwise you are not writing very - * sequentially. In current use, this only happens when the sort ends - * writing a run, and switches to another tape. The last block of the - * previous tape isn't flushed to disk until the end of the sort, so you - * get one-block hole, where the last block of the previous tape will - * later go. + * This can happen either when tapes preallocate blocks; or for the last + * block of a tape which might not have been flushed. * * Note that BufFile concatenation can leave "holes" in BufFile between * worker-owned block ranges. These are tracked for reporting purposes @@ -373,8 +371,20 @@ parent_offset(unsigned long i) } /* - * Select the lowest currently unused block by taking the first element from - * the freelist min heap. + * Get the next block for writing. + */ +static long +ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt) +{ + if (lts->enable_prealloc) + return ltsGetPreallocBlock(lts, lt); + else + return ltsGetFreeBlock(lts); +} + +/* + * Select the lowest currently unused block from the tape set's global free + * list min heap. */ static long ltsGetFreeBlock(LogicalTapeSet *lts) @@ -430,7 +440,8 @@ ltsGetFreeBlock(LogicalTapeSet *lts) /* * Return the lowest free block number from the tape's preallocation list. - * Refill the preallocation list if necessary. + * Refill the preallocation list with blocks from the tape set's free list if + * necessary. */ static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt) @@ -671,8 +682,8 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt) * infrastructure that may be lifted in the future. */ LogicalTapeSet * -LogicalTapeSetCreate(int ntapes, TapeShare *shared, SharedFileSet *fileset, - int worker) +LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared, + SharedFileSet *fileset, int worker) { LogicalTapeSet *lts; int i; @@ -689,6 +700,7 @@ LogicalTapeSetCreate(int ntapes, TapeShare *shared, SharedFileSet *fileset, lts->freeBlocksLen = 32; /* reasonable initial guess */ lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long)); lts->nFreeBlocks = 0; + lts->enable_prealloc = preallocate; lts->nTapes = ntapes; lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape)); @@ -782,7 +794,7 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum, Assert(lt->firstBlockNumber == -1); Assert(lt->pos == 0); - lt->curBlockNumber = ltsGetPreallocBlock(lts, lt); + lt->curBlockNumber = ltsGetBlock(lts, lt); lt->firstBlockNumber = lt->curBlockNumber; TapeBlockGetTrailer(lt->buffer)->prev = -1L; @@ -806,7 +818,7 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum, * First allocate the next block, so that we can store it in the * 'next' pointer of this block. */ - nextBlockNumber = ltsGetPreallocBlock(lts, lt); + nextBlockNumber = ltsGetBlock(lts, lt); /* set the next-pointer and dump the current block. */ TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber; diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c index 3c49476483b1..cbda911f4652 100644 --- a/src/backend/utils/sort/tuplesort.c +++ b/src/backend/utils/sort/tuplesort.c @@ -2591,7 +2591,7 @@ inittapes(Tuplesortstate *state, bool mergeruns) /* Create the tape set and allocate the per-tape data arrays */ inittapestate(state, maxTapes); state->tapeset = - LogicalTapeSetCreate(maxTapes, NULL, + LogicalTapeSetCreate(maxTapes, false, NULL, state->shared ? &state->shared->fileset : NULL, state->worker); @@ -4657,8 +4657,9 @@ leader_takeover_tapes(Tuplesortstate *state) * randomAccess is disallowed for parallel sorts. */ inittapestate(state, nParticipants + 1); - state->tapeset = LogicalTapeSetCreate(nParticipants + 1, shared->tapes, - &shared->fileset, state->worker); + state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false, + shared->tapes, &shared->fileset, + state->worker); /* mergeruns() relies on currentRun for # of runs (in one-pass cases) */ state->currentRun = nParticipants; diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h index 39a99174afe3..da5159e4c6c7 100644 --- a/src/include/utils/logtape.h +++ b/src/include/utils/logtape.h @@ -54,7 +54,8 @@ typedef struct TapeShare * prototypes for functions in logtape.c */ -extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, TapeShare *shared, +extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate, + TapeShare *shared, SharedFileSet *fileset, int worker); extern void LogicalTapeSetClose(LogicalTapeSet *lts); extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts); From ddd5f6d2609b25f252d81dc43746347e0113bfbe Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Sat, 12 Sep 2020 07:47:53 +0530 Subject: [PATCH 135/589] Remove unused function declaration in logicalproto.h. In the passing, fix a typo in pgoutput.c. Reported-by: Tomas Vondra Author: Tomas Vondra Reviewed-by: Dilip Kumar Discussion: https://postgr.es/m/20200909084353.pncuclpbwlr7vylh@development --- src/backend/replication/pgoutput/pgoutput.c | 2 +- src/include/replication/logicalproto.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c index c29c0888133a..343f03129fe7 100644 --- a/src/backend/replication/pgoutput/pgoutput.c +++ b/src/backend/replication/pgoutput/pgoutput.c @@ -77,7 +77,7 @@ static void send_relation_and_attrs(Relation relation, TransactionId xid, * and with streamed transactions the commit order may be different from * the order the transactions are sent in. Also, the (sub) transactions * might get aborted so we need to send the schema for each (sub) transaction - * so that we don't loose the schema information on abort. For handling this, + * so that we don't lose the schema information on abort. For handling this, * we maintain the list of xids (streamed_txns) for those we have already sent * the schema. * diff --git a/src/include/replication/logicalproto.h b/src/include/replication/logicalproto.h index 53905ee6080f..607a728508bb 100644 --- a/src/include/replication/logicalproto.h +++ b/src/include/replication/logicalproto.h @@ -133,7 +133,6 @@ extern void logicalrep_write_stream_start(StringInfo out, TransactionId xid, extern TransactionId logicalrep_read_stream_start(StringInfo in, bool *first_segment); extern void logicalrep_write_stream_stop(StringInfo out); -extern TransactionId logicalrep_read_stream_stop(StringInfo in); extern void logicalrep_write_stream_commit(StringInfo out, ReorderBufferTXN *txn, XLogRecPtr commit_lsn); extern TransactionId logicalrep_read_stream_commit(StringInfo out, From 03c7f1f37a1f3bb69e348f915296af2289257e89 Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Sat, 12 Sep 2020 08:02:54 +0530 Subject: [PATCH 136/589] Fix inconsistency in determining the timestamp of the db statfile. We use the timestamp of the global statfile if we are not able to determine it for a particular database in case the entry for that database doesn't exist. However, we were using it even when the statfile is corrupt. As there is no user reported issue and it is not clear if there is any impact of this on actual application so decided not to backpatch. Reported-by: Amit Kapila Author: Amit Kapila Reviewed-by: Sawada Masahiko, Magnus Hagander and Alvaro Herrera Discussion: https://postgr.es/m/CAA4eK1J3oTJKyVq6v7K4d3jD+vtnruG9fHRib6UuWWsrwAR6Aw@mail.gmail.com --- src/backend/postmaster/pgstat.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 5f4b168fd16b..e6be2b7836a4 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -5557,7 +5557,8 @@ pgstat_read_db_statsfile(Oid databaseid, HTAB *tabhash, HTAB *funchash, * pgstat_read_db_statsfile_timestamp() - * * Attempt to determine the timestamp of the last db statfile write. - * Returns true if successful; the timestamp is stored in *ts. + * Returns true if successful; the timestamp is stored in *ts. The caller must + * rely on timestamp stored in *ts iff the function returns true. * * This needs to be careful about handling databases for which no stats file * exists, such as databases without a stat entry or those not yet written: @@ -5665,7 +5666,8 @@ pgstat_read_db_statsfile_timestamp(Oid databaseid, bool permanent, ereport(pgStatRunningInCollector ? LOG : WARNING, (errmsg("corrupted statistics file \"%s\"", statfile))); - goto done; + FreeFile(fpin); + return false; } /* @@ -5684,10 +5686,13 @@ pgstat_read_db_statsfile_timestamp(Oid databaseid, bool permanent, goto done; default: - ereport(pgStatRunningInCollector ? LOG : WARNING, - (errmsg("corrupted statistics file \"%s\"", - statfile))); - goto done; + { + ereport(pgStatRunningInCollector ? LOG : WARNING, + (errmsg("corrupted statistics file \"%s\"", + statfile))); + FreeFile(fpin); + return false; + } } } From 19f5a37b9fc48a12c77edafb732543875da2f4a3 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 13 Sep 2020 12:51:21 -0400 Subject: [PATCH 137/589] Use the properly transformed RangeVar for expandTableLikeClause(). transformCreateStmt() adjusts the transformed statement's RangeVar to specify the target schema explicitly, for the express reason of making sure that auxiliary statements derived by parse transformation operate on the right table. But the refactoring I did in commit 502898192 got this wrong and passed the untransformed RangeVar to expandTableLikeClause(). This could lead to assertion failures or weird misbehavior if the wrong table was accessed. Per report from Alexander Lakhin. Like the previous patch, back-patch to all supported branches. Discussion: https://postgr.es/m/05051f9d-b32b-cb35-6735-0e9f2ab86b5f@gmail.com --- src/backend/tcop/utility.c | 23 ++++++++++++++----- .../regress/expected/create_table_like.out | 18 +++++++++++++++ src/test/regress/sql/create_table_like.sql | 5 ++++ 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 9713a7ac41a4..9a35147b26af 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -1139,6 +1139,7 @@ ProcessUtilitySlow(ParseState *pstate, { List *stmts; ListCell *l; + RangeVar *table_rv = NULL; /* Run parse analysis ... */ stmts = transformCreateStmt((CreateStmt *) parsetree, @@ -1151,11 +1152,15 @@ ProcessUtilitySlow(ParseState *pstate, if (IsA(stmt, CreateStmt)) { + CreateStmt *cstmt = (CreateStmt *) stmt; Datum toast_options; static char *validnsps[] = HEAP_RELOPT_NAMESPACES; + /* Remember transformed RangeVar for LIKE */ + table_rv = cstmt->relation; + /* Create the table itself */ - address = DefineRelation((CreateStmt *) stmt, + address = DefineRelation(cstmt, RELKIND_RELATION, InvalidOid, NULL, queryString); @@ -1174,7 +1179,7 @@ ProcessUtilitySlow(ParseState *pstate, * table */ toast_options = transformRelOptions((Datum) 0, - ((CreateStmt *) stmt)->options, + cstmt->options, "toast", validnsps, true, @@ -1188,12 +1193,17 @@ ProcessUtilitySlow(ParseState *pstate, } else if (IsA(stmt, CreateForeignTableStmt)) { + CreateForeignTableStmt *cstmt = (CreateForeignTableStmt *) stmt; + + /* Remember transformed RangeVar for LIKE */ + table_rv = cstmt->base.relation; + /* Create the table itself */ - address = DefineRelation((CreateStmt *) stmt, + address = DefineRelation(&cstmt->base, RELKIND_FOREIGN_TABLE, InvalidOid, NULL, queryString); - CreateForeignTable((CreateForeignTableStmt *) stmt, + CreateForeignTable(cstmt, address.objectId); EventTriggerCollectSimpleCommand(address, secondaryObject, @@ -1208,10 +1218,11 @@ ProcessUtilitySlow(ParseState *pstate, * to-do list. */ TableLikeClause *like = (TableLikeClause *) stmt; - RangeVar *rv = ((CreateStmt *) parsetree)->relation; List *morestmts; - morestmts = expandTableLikeClause(rv, like); + Assert(table_rv != NULL); + + morestmts = expandTableLikeClause(table_rv, like); stmts = list_concat(stmts, morestmts); /* diff --git a/src/test/regress/expected/create_table_like.out b/src/test/regress/expected/create_table_like.out index e3edbd8b511c..912c73d351ef 100644 --- a/src/test/regress/expected/create_table_like.out +++ b/src/test/regress/expected/create_table_like.out @@ -421,6 +421,24 @@ CREATE TABLE inh_error2 (LIKE ctlt4 INCLUDING STORAGE) INHERITS (ctlt1); NOTICE: merging column "a" with inherited definition ERROR: column "a" has a storage parameter conflict DETAIL: MAIN versus EXTENDED +-- Check that LIKE isn't confused by a system catalog of the same name +CREATE TABLE pg_attrdef (LIKE ctlt1 INCLUDING ALL); +\d+ public.pg_attrdef + Table "public.pg_attrdef" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------+------+-----------+----------+---------+----------+--------------+------------- + a | text | | not null | | main | | A + b | text | | | | extended | | B +Indexes: + "pg_attrdef_pkey" PRIMARY KEY, btree (a) + "pg_attrdef_b_idx" btree (b) + "pg_attrdef_expr_idx" btree ((a || b)) +Check constraints: + "ctlt1_a_check" CHECK (length(a) > 2) +Statistics objects: + "public"."pg_attrdef_a_b_stat" (ndistinct, dependencies, mcv) ON a, b FROM public.pg_attrdef + +DROP TABLE public.pg_attrdef; DROP TABLE ctlt1, ctlt2, ctlt3, ctlt4, ctlt12_storage, ctlt12_comments, ctlt1_inh, ctlt13_inh, ctlt13_like, ctlt_all, ctla, ctlb CASCADE; NOTICE: drop cascades to table inhe -- LIKE must respect NO INHERIT property of constraints diff --git a/src/test/regress/sql/create_table_like.sql b/src/test/regress/sql/create_table_like.sql index f0a8a56b76fa..e484bac0a461 100644 --- a/src/test/regress/sql/create_table_like.sql +++ b/src/test/regress/sql/create_table_like.sql @@ -163,6 +163,11 @@ SELECT s.stxname, objsubid, description FROM pg_description, pg_statistic_ext s CREATE TABLE inh_error1 () INHERITS (ctlt1, ctlt4); CREATE TABLE inh_error2 (LIKE ctlt4 INCLUDING STORAGE) INHERITS (ctlt1); +-- Check that LIKE isn't confused by a system catalog of the same name +CREATE TABLE pg_attrdef (LIKE ctlt1 INCLUDING ALL); +\d+ public.pg_attrdef +DROP TABLE public.pg_attrdef; + DROP TABLE ctlt1, ctlt2, ctlt3, ctlt4, ctlt12_storage, ctlt12_comments, ctlt1_inh, ctlt13_inh, ctlt13_like, ctlt_all, ctla, ctlb CASCADE; -- LIKE must respect NO INHERIT property of constraints From ac673a1aaff197f3e01f7bac69da0dd700854e13 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 14 Sep 2020 10:44:23 +0900 Subject: [PATCH 138/589] Avoid useless allocations for information of dumpable objects in pg_dump/ If there are no objects of a certain type, there is no need to do an allocation for a set of DumpableObject items. The previous coding did an allocation of 1 byte instead as per the fallback of pg_malloc() in the event of an allocation size of zero. This assigns NULL instead for a set of dumpable objects. A similar rule already applied to findObjectByOid(), so this makes the code more defensive as we would just fail with a pointer dereference instead of attempting to use some incorrect data if a non-existing, positive, OID is given by a caller of this function. Author: Daniel Gustafsson Reviewed-by: Julien Rouhaud, Ranier Vilela Discussion: https://postgr.es/m/26C43E58-BDD0-4F1A-97CC-4A07B52E32C5@yesql.se --- src/bin/pg_dump/common.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/bin/pg_dump/common.c b/src/bin/pg_dump/common.c index 08239dde4f92..634ca86cfb78 100644 --- a/src/bin/pg_dump/common.c +++ b/src/bin/pg_dump/common.c @@ -719,6 +719,9 @@ buildIndexArray(void *objArray, int numObjs, Size objSize) DumpableObject **ptrs; int i; + if (numObjs <= 0) + return NULL; + ptrs = (DumpableObject **) pg_malloc(numObjs * sizeof(DumpableObject *)); for (i = 0; i < numObjs; i++) ptrs[i] = (DumpableObject *) ((char *) objArray + i * objSize); From 3e0242b24c3c059870890644b69d6c4491a45651 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 14 Sep 2020 06:42:07 +0200 Subject: [PATCH 139/589] Message fixes and style improvements --- src/backend/access/heap/vacuumlazy.c | 12 ++-- src/backend/access/transam/xlog.c | 2 +- src/backend/commands/opclasscmds.c | 8 +-- src/backend/commands/tablecmds.c | 2 +- src/backend/libpq/hba.c | 4 +- src/backend/nodes/params.c | 4 +- src/backend/parser/gram.y | 2 +- src/backend/parser/parse_clause.c | 2 +- src/backend/replication/basebackup.c | 5 +- src/backend/storage/ipc/procarray.c | 2 +- src/backend/utils/adt/jsonpath_exec.c | 4 +- src/backend/utils/fmgr/fmgr.c | 2 +- src/backend/utils/misc/guc.c | 4 +- src/backend/utils/sort/sharedtuplestore.c | 2 +- src/bin/pg_verifybackup/parse_manifest.c | 14 ++--- src/bin/pg_verifybackup/pg_verifybackup.c | 2 +- src/bin/pg_verifybackup/t/005_bad_manifest.pl | 16 ++--- src/bin/pgbench/t/001_pgbench_with_server.pl | 10 ++-- src/bin/scripts/vacuumdb.c | 2 +- src/fe_utils/archive.c | 4 +- .../test_misc/t/001_constraint_validation.pl | 8 +-- src/test/regress/expected/alter_generic.out | 6 +- src/test/regress/expected/jsonb_jsonpath.out | 60 +++++++++---------- src/test/regress/expected/limit.out | 4 +- 24 files changed, 92 insertions(+), 89 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 92389e6666bc..4f2f38168dc2 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -677,11 +677,10 @@ heap_vacuum_rel(Relation onerel, VacuumParams *params, read_rate, write_rate); appendStringInfo(&buf, _("system usage: %s\n"), pg_rusage_show(&ru0)); appendStringInfo(&buf, - _("WAL usage: %ld records, %ld full page images, " - UINT64_FORMAT " bytes"), + _("WAL usage: %ld records, %ld full page images, %llu bytes"), walusage.wal_records, walusage.wal_fpi, - walusage.wal_bytes); + (unsigned long long) walusage.wal_bytes); ereport(LOG, (errmsg_internal("%s", buf.data))); @@ -3523,9 +3522,10 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) false); elevel = lvshared->elevel; - ereport(DEBUG1, - (errmsg("starting parallel vacuum worker for %s", - lvshared->for_cleanup ? "cleanup" : "bulk delete"))); + if (lvshared->for_cleanup) + elog(DEBUG1, "starting parallel vacuum worker for cleanup"); + else + elog(DEBUG1, "starting parallel vacuum worker for bulk delete"); /* Set debug_query_string for individual workers */ sharedquery = shm_toc_lookup(toc, PARALLEL_VACUUM_KEY_QUERY_TEXT, false); diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 09c01ed4ae48..a38371a64f96 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -12504,7 +12504,7 @@ StartupRequestWalReceiverRestart(void) if (currentSource == XLOG_FROM_STREAM && WalRcvRunning()) { ereport(LOG, - (errmsg("wal receiver process shutdown requested"))); + (errmsg("WAL receiver process shutdown requested"))); pendingWalRcvRestart = true; } diff --git a/src/backend/commands/opclasscmds.c b/src/backend/commands/opclasscmds.c index 28395d5946f3..c46db7d11cb4 100644 --- a/src/backend/commands/opclasscmds.c +++ b/src/backend/commands/opclasscmds.c @@ -1212,14 +1212,14 @@ assignProcTypes(OpFamilyMember *member, Oid amoid, Oid typeoid, (OidIsValid(member->righttype) && member->righttype != typeoid)) ereport(ERROR, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), - errmsg("associated data types for opclass options parsing functions must match opclass input type"))); + errmsg("associated data types for operator class options parsing functions must match opclass input type"))); } else { if (member->lefttype != member->righttype) ereport(ERROR, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), - errmsg("left and right associated data types for opclass options parsing functions must match"))); + errmsg("left and right associated data types for operator class options parsing functions must match"))); } if (procform->prorettype != VOIDOID || @@ -1227,8 +1227,8 @@ assignProcTypes(OpFamilyMember *member, Oid amoid, Oid typeoid, procform->proargtypes.values[0] != INTERNALOID) ereport(ERROR, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), - errmsg("invalid opclass options parsing function"), - errhint("Valid signature of opclass options parsing function is '%s'.", + errmsg("invalid operator class options parsing function"), + errhint("Valid signature of operator class options parsing function is %s.", "(internal) RETURNS void"))); } diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 3e57c7f9e1db..c21a309f04fd 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -6855,7 +6855,7 @@ NotNullImpliedByRelConstraints(Relation rel, Form_pg_attribute attr) if (ConstraintImpliedByRelConstraint(rel, list_make1(nnulltest), NIL)) { ereport(DEBUG1, - (errmsg("existing constraints on column \"%s\".\"%s\" are sufficient to prove that it does not contain nulls", + (errmsg("existing constraints on column \"%s.%s\" are sufficient to prove that it does not contain nulls", RelationGetRelationName(rel), NameStr(attr->attname)))); return true; } diff --git a/src/backend/libpq/hba.c b/src/backend/libpq/hba.c index 9f106653f3f8..7b54ffc31ea1 100644 --- a/src/backend/libpq/hba.c +++ b/src/backend/libpq/hba.c @@ -1746,10 +1746,10 @@ parse_hba_auth_opt(char *name, char *val, HbaLine *hbaline, { ereport(elevel, (errcode(ERRCODE_CONFIG_FILE_ERROR), - errmsg("clientcert can not be set to \"no-verify\" when using \"cert\" authentication"), + errmsg("clientcert cannot be set to \"no-verify\" when using \"cert\" authentication"), errcontext("line %d of configuration file \"%s\"", line_num, HbaFileName))); - *err_msg = "clientcert can not be set to \"no-verify\" when using \"cert\" authentication"; + *err_msg = "clientcert cannot be set to \"no-verify\" when using \"cert\" authentication"; return false; } hbaline->clientcert = clientCertOff; diff --git a/src/backend/nodes/params.c b/src/backend/nodes/params.c index bce0c7e72b2c..c05f04a259c1 100644 --- a/src/backend/nodes/params.c +++ b/src/backend/nodes/params.c @@ -414,9 +414,9 @@ ParamsErrorCallback(void *arg) return; if (data->portalName && data->portalName[0] != '\0') - errcontext("extended query \"%s\" with parameters: %s", + errcontext("portal \"%s\" with parameters: %s", data->portalName, data->params->paramValuesStr); else - errcontext("extended query with parameters: %s", + errcontext("unnamed portal with parameters: %s", data->params->paramValuesStr); } diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index c5154b818cfb..9f47745ee246 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -15919,7 +15919,7 @@ insertSelectOptions(SelectStmt *stmt, if (!stmt->sortClause && limitClause->limitOption == LIMIT_OPTION_WITH_TIES) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("WITH TIES options can not be specified without ORDER BY clause"))); + errmsg("WITH TIES cannot be specified without ORDER BY clause"))); stmt->limitOption = limitClause->limitOption; } if (withClause) diff --git a/src/backend/parser/parse_clause.c b/src/backend/parser/parse_clause.c index 6fff13479e47..edcaf276c0ad 100644 --- a/src/backend/parser/parse_clause.c +++ b/src/backend/parser/parse_clause.c @@ -1770,7 +1770,7 @@ transformLimitClause(ParseState *pstate, Node *clause, IsA(clause, A_Const) && ((A_Const *) clause)->val.type == T_Null) ereport(ERROR, (errcode(ERRCODE_INVALID_ROW_COUNT_IN_LIMIT_CLAUSE), - errmsg("row count cannot be NULL in FETCH FIRST ... WITH TIES clause"))); + errmsg("row count cannot be null in FETCH FIRST ... WITH TIES clause"))); return qual; } diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 6064384e32a4..b89df01fa76f 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -719,7 +719,10 @@ perform_base_backup(basebackup_options *opt) { if (total_checksum_failures > 1) ereport(WARNING, - (errmsg("%lld total checksum verification failures", total_checksum_failures))); + (errmsg_plural("%lld total checksum verification failure", + "%lld total checksum verification failures", + total_checksum_failures, + total_checksum_failures))); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c index 1c0cd6b2487b..4262b3cec825 100644 --- a/src/backend/storage/ipc/procarray.c +++ b/src/backend/storage/ipc/procarray.c @@ -3627,7 +3627,7 @@ TerminateOtherDBBackends(Oid databaseId) if (nprepared > 0) ereport(ERROR, (errcode(ERRCODE_OBJECT_IN_USE), - errmsg("database \"%s\" is being used by prepared transaction", + errmsg("database \"%s\" is being used by prepared transactions", get_database_name(databaseId)), errdetail_plural("There is %d prepared transaction using the database.", "There are %d prepared transactions using the database.", diff --git a/src/backend/utils/adt/jsonpath_exec.c b/src/backend/utils/adt/jsonpath_exec.c index 7403c760b486..2c0b362502f3 100644 --- a/src/backend/utils/adt/jsonpath_exec.c +++ b/src/backend/utils/adt/jsonpath_exec.c @@ -2582,9 +2582,9 @@ checkTimezoneIsUsedForCast(bool useTz, const char *type1, const char *type2) if (!useTz) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot convert value from %s to %s without timezone usage", + errmsg("cannot convert value from %s to %s without time zone usage", type1, type2), - errhint("Use *_tz() function for timezone support."))); + errhint("Use *_tz() function for time zone support."))); } /* Convert time datum to timetz datum */ diff --git a/src/backend/utils/fmgr/fmgr.c b/src/backend/utils/fmgr/fmgr.c index 03c614b234a3..2681b7fbc603 100644 --- a/src/backend/utils/fmgr/fmgr.c +++ b/src/backend/utils/fmgr/fmgr.c @@ -2000,7 +2000,7 @@ get_fn_opclass_options(FmgrInfo *flinfo) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("opclass options info is absent in function call context"))); + errmsg("operator class options info is absent in function call context"))); return NULL; } diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 73518d986621..596bcb7b842e 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -2837,7 +2837,7 @@ static struct config_int ConfigureNamesInt[] = gettext_noop("Sets the minimum execution time above which " "a sample of statements will be logged." " Sampling is determined by log_statement_sample_rate."), - gettext_noop("Zero log a sample of all queries. -1 turns this feature off."), + gettext_noop("Zero logs a sample of all queries. -1 turns this feature off."), GUC_UNIT_MS }, &log_min_duration_sample, @@ -3700,7 +3700,7 @@ static struct config_string ConfigureNamesString[] = { {"restore_command", PGC_POSTMASTER, WAL_ARCHIVE_RECOVERY, - gettext_noop("Sets the shell command that will retrieve an archived WAL file."), + gettext_noop("Sets the shell command that will be called to retrieve an archived WAL file."), NULL }, &recoveryRestoreCommand, diff --git a/src/backend/utils/sort/sharedtuplestore.c b/src/backend/utils/sort/sharedtuplestore.c index b83fb50dac8f..fe298ce92ed5 100644 --- a/src/backend/utils/sort/sharedtuplestore.c +++ b/src/backend/utils/sort/sharedtuplestore.c @@ -566,7 +566,7 @@ sts_parallel_scan_next(SharedTuplestoreAccessor *accessor, void *meta_data) if (BufFileSeekBlock(accessor->read_file, read_page) != 0) ereport(ERROR, (errcode_for_file_access(), - errmsg("could not seek block %u in shared tuplestore temporary file", + errmsg("could not seek to block %u in shared tuplestore temporary file", read_page))); nread = BufFileRead(accessor->read_file, &chunk_header, STS_CHUNK_HEADER_SIZE); diff --git a/src/bin/pg_verifybackup/parse_manifest.c b/src/bin/pg_verifybackup/parse_manifest.c index faee423c7ece..608e23538bad 100644 --- a/src/bin/pg_verifybackup/parse_manifest.c +++ b/src/bin/pg_verifybackup/parse_manifest.c @@ -325,7 +325,7 @@ json_manifest_object_field_start(void *state, char *fname, bool isnull) /* It's not a field we recognize. */ json_manifest_parse_failure(parse->context, - "unknown toplevel field"); + "unrecognized top-level field"); break; case JM_EXPECT_THIS_FILE_FIELD: @@ -358,7 +358,7 @@ json_manifest_object_field_start(void *state, char *fname, bool isnull) parse->wal_range_field = JMWRF_END_LSN; else json_manifest_parse_failure(parse->context, - "unexpected wal range field"); + "unexpected WAL range field"); parse->state = JM_EXPECT_THIS_WAL_RANGE_VALUE; break; @@ -469,10 +469,10 @@ json_manifest_finalize_file(JsonManifestParseState *parse) /* Pathname and size are required. */ if (parse->pathname == NULL && parse->encoded_pathname == NULL) - json_manifest_parse_failure(parse->context, "missing pathname"); + json_manifest_parse_failure(parse->context, "missing path name"); if (parse->pathname != NULL && parse->encoded_pathname != NULL) json_manifest_parse_failure(parse->context, - "both pathname and encoded pathname"); + "both path name and encoded path name"); if (parse->size == NULL) json_manifest_parse_failure(parse->context, "missing size"); if (parse->algorithm == NULL && parse->checksum != NULL) @@ -491,7 +491,7 @@ json_manifest_finalize_file(JsonManifestParseState *parse) parse->encoded_pathname, raw_length)) json_manifest_parse_failure(parse->context, - "unable to decode filename"); + "could not decode file name"); parse->pathname[raw_length] = '\0'; pfree(parse->encoded_pathname); parse->encoded_pathname = NULL; @@ -582,10 +582,10 @@ json_manifest_finalize_wal_range(JsonManifestParseState *parse) "timeline is not an integer"); if (!parse_xlogrecptr(&start_lsn, parse->start_lsn)) json_manifest_parse_failure(parse->context, - "unable to parse start LSN"); + "could not parse start LSN"); if (!parse_xlogrecptr(&end_lsn, parse->end_lsn)) json_manifest_parse_failure(parse->context, - "unable to parse end LSN"); + "could not parse end LSN"); /* Invoke the callback with the details we've gathered. */ context->perwalrange_cb(context, tli, start_lsn, end_lsn); diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c index 70b6ffdec00b..20140aa02748 100644 --- a/src/bin/pg_verifybackup/pg_verifybackup.c +++ b/src/bin/pg_verifybackup/pg_verifybackup.c @@ -471,7 +471,7 @@ record_manifest_details_for_file(JsonManifestParseContext *context, /* Make a new entry in the hash table for this file. */ m = manifest_files_insert(ht, pathname, &found); if (found) - report_fatal_error("duplicate pathname in backup manifest: \"%s\"", + report_fatal_error("duplicate path name in backup manifest: \"%s\"", pathname); /* Initialize the entry. */ diff --git a/src/bin/pg_verifybackup/t/005_bad_manifest.pl b/src/bin/pg_verifybackup/t/005_bad_manifest.pl index afd64d1a96b0..5bd5556038c3 100644 --- a/src/bin/pg_verifybackup/t/005_bad_manifest.pl +++ b/src/bin/pg_verifybackup/t/005_bad_manifest.pl @@ -38,7 +38,7 @@ {"PostgreSQL-Backup-Manifest-Version": 1, "Files": true} EOM -test_parse_error('unknown toplevel field', <= "10.03.2017".datetime("dd.mm.yyyy"))'); -ERROR: cannot convert value from date to timestamptz without timezone usage -HINT: Use *_tz() function for timezone support. +ERROR: cannot convert value from date to timestamptz without time zone usage +HINT: Use *_tz() function for time zone support. select jsonb_path_query( '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03 +04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03 +04", "2017-03-10 03:00:00 +03"]', '$[*].datetime() ? (@ < "10.03.2017".datetime("dd.mm.yyyy"))'); -ERROR: cannot convert value from date to timestamptz without timezone usage -HINT: Use *_tz() function for timezone support. +ERROR: cannot convert value from date to timestamptz without time zone usage +HINT: Use *_tz() function for time zone support. select jsonb_path_query_tz( '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03 +04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03 +04", "2017-03-10 03:00:00 +03"]', '$[*].datetime() ? (@ == "10.03.2017".datetime("dd.mm.yyyy"))'); @@ -1989,18 +1989,18 @@ select jsonb_path_query_tz( select jsonb_path_query( '["12:34:00", "12:35:00", "12:36:00", "12:35:00 +00", "12:35:00 +01", "13:35:00 +01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +01"]', '$[*].datetime() ? (@ == "12:35".datetime("HH24:MI"))'); -ERROR: cannot convert value from time to timetz without timezone usage -HINT: Use *_tz() function for timezone support. +ERROR: cannot convert value from time to timetz without time zone usage +HINT: Use *_tz() function for time zone support. select jsonb_path_query( '["12:34:00", "12:35:00", "12:36:00", "12:35:00 +00", "12:35:00 +01", "13:35:00 +01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +01"]', '$[*].datetime() ? (@ >= "12:35".datetime("HH24:MI"))'); -ERROR: cannot convert value from time to timetz without timezone usage -HINT: Use *_tz() function for timezone support. +ERROR: cannot convert value from time to timetz without time zone usage +HINT: Use *_tz() function for time zone support. select jsonb_path_query( '["12:34:00", "12:35:00", "12:36:00", "12:35:00 +00", "12:35:00 +01", "13:35:00 +01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +01"]', '$[*].datetime() ? (@ < "12:35".datetime("HH24:MI"))'); -ERROR: cannot convert value from time to timetz without timezone usage -HINT: Use *_tz() function for timezone support. +ERROR: cannot convert value from time to timetz without time zone usage +HINT: Use *_tz() function for time zone support. select jsonb_path_query_tz( '["12:34:00", "12:35:00", "12:36:00", "12:35:00 +00", "12:35:00 +01", "13:35:00 +01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +01"]', '$[*].datetime() ? (@ == "12:35".datetime("HH24:MI"))'); @@ -2034,18 +2034,18 @@ select jsonb_path_query_tz( select jsonb_path_query( '["12:34:00 +01", "12:35:00 +01", "12:36:00 +01", "12:35:00 +02", "12:35:00 -02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', '$[*].datetime() ? (@ == "12:35 +1".datetime("HH24:MI TZH"))'); -ERROR: cannot convert value from time to timetz without timezone usage -HINT: Use *_tz() function for timezone support. +ERROR: cannot convert value from time to timetz without time zone usage +HINT: Use *_tz() function for time zone support. select jsonb_path_query( '["12:34:00 +01", "12:35:00 +01", "12:36:00 +01", "12:35:00 +02", "12:35:00 -02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', '$[*].datetime() ? (@ >= "12:35 +1".datetime("HH24:MI TZH"))'); -ERROR: cannot convert value from time to timetz without timezone usage -HINT: Use *_tz() function for timezone support. +ERROR: cannot convert value from time to timetz without time zone usage +HINT: Use *_tz() function for time zone support. select jsonb_path_query( '["12:34:00 +01", "12:35:00 +01", "12:36:00 +01", "12:35:00 +02", "12:35:00 -02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', '$[*].datetime() ? (@ < "12:35 +1".datetime("HH24:MI TZH"))'); -ERROR: cannot convert value from time to timetz without timezone usage -HINT: Use *_tz() function for timezone support. +ERROR: cannot convert value from time to timetz without time zone usage +HINT: Use *_tz() function for time zone support. select jsonb_path_query_tz( '["12:34:00 +01", "12:35:00 +01", "12:36:00 +01", "12:35:00 +02", "12:35:00 -02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', '$[*].datetime() ? (@ == "12:35 +1".datetime("HH24:MI TZH"))'); @@ -2080,18 +2080,18 @@ select jsonb_path_query_tz( select jsonb_path_query( '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00 +01", "2017-03-10 13:35:00 +01", "2017-03-10 12:35:00 -01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', '$[*].datetime() ? (@ == "10.03.2017 12:35".datetime("dd.mm.yyyy HH24:MI"))'); -ERROR: cannot convert value from timestamp to timestamptz without timezone usage -HINT: Use *_tz() function for timezone support. +ERROR: cannot convert value from timestamp to timestamptz without time zone usage +HINT: Use *_tz() function for time zone support. select jsonb_path_query( '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00 +01", "2017-03-10 13:35:00 +01", "2017-03-10 12:35:00 -01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', '$[*].datetime() ? (@ >= "10.03.2017 12:35".datetime("dd.mm.yyyy HH24:MI"))'); -ERROR: cannot convert value from timestamp to timestamptz without timezone usage -HINT: Use *_tz() function for timezone support. +ERROR: cannot convert value from timestamp to timestamptz without time zone usage +HINT: Use *_tz() function for time zone support. select jsonb_path_query( '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00 +01", "2017-03-10 13:35:00 +01", "2017-03-10 12:35:00 -01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', '$[*].datetime() ? (@ < "10.03.2017 12:35".datetime("dd.mm.yyyy HH24:MI"))'); -ERROR: cannot convert value from timestamp to timestamptz without timezone usage -HINT: Use *_tz() function for timezone support. +ERROR: cannot convert value from timestamp to timestamptz without time zone usage +HINT: Use *_tz() function for time zone support. select jsonb_path_query_tz( '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00 +01", "2017-03-10 13:35:00 +01", "2017-03-10 12:35:00 -01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', '$[*].datetime() ? (@ == "10.03.2017 12:35".datetime("dd.mm.yyyy HH24:MI"))'); @@ -2127,18 +2127,18 @@ select jsonb_path_query_tz( select jsonb_path_query( '["2017-03-10 12:34:00 +01", "2017-03-10 12:35:00 +01", "2017-03-10 12:36:00 +01", "2017-03-10 12:35:00 +02", "2017-03-10 12:35:00 -02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', '$[*].datetime() ? (@ == "10.03.2017 12:35 +1".datetime("dd.mm.yyyy HH24:MI TZH"))'); -ERROR: cannot convert value from timestamp to timestamptz without timezone usage -HINT: Use *_tz() function for timezone support. +ERROR: cannot convert value from timestamp to timestamptz without time zone usage +HINT: Use *_tz() function for time zone support. select jsonb_path_query( '["2017-03-10 12:34:00 +01", "2017-03-10 12:35:00 +01", "2017-03-10 12:36:00 +01", "2017-03-10 12:35:00 +02", "2017-03-10 12:35:00 -02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', '$[*].datetime() ? (@ >= "10.03.2017 12:35 +1".datetime("dd.mm.yyyy HH24:MI TZH"))'); -ERROR: cannot convert value from timestamp to timestamptz without timezone usage -HINT: Use *_tz() function for timezone support. +ERROR: cannot convert value from timestamp to timestamptz without time zone usage +HINT: Use *_tz() function for time zone support. select jsonb_path_query( '["2017-03-10 12:34:00 +01", "2017-03-10 12:35:00 +01", "2017-03-10 12:36:00 +01", "2017-03-10 12:35:00 +02", "2017-03-10 12:35:00 -02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', '$[*].datetime() ? (@ < "10.03.2017 12:35 +1".datetime("dd.mm.yyyy HH24:MI TZH"))'); -ERROR: cannot convert value from timestamp to timestamptz without timezone usage -HINT: Use *_tz() function for timezone support. +ERROR: cannot convert value from timestamp to timestamptz without time zone usage +HINT: Use *_tz() function for time zone support. select jsonb_path_query_tz( '["2017-03-10 12:34:00 +01", "2017-03-10 12:35:00 +01", "2017-03-10 12:36:00 +01", "2017-03-10 12:35:00 +02", "2017-03-10 12:35:00 -02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', '$[*].datetime() ? (@ == "10.03.2017 12:35 +1".datetime("dd.mm.yyyy HH24:MI TZH"))'); diff --git a/src/test/regress/expected/limit.out b/src/test/regress/expected/limit.out index e6f6809fbee8..b75afcc01a39 100644 --- a/src/test/regress/expected/limit.out +++ b/src/test/regress/expected/limit.out @@ -623,7 +623,7 @@ SELECT thousand SELECT ''::text AS two, unique1, unique2, stringu1 FROM onek WHERE unique1 > 50 FETCH FIRST 2 ROW WITH TIES; -ERROR: WITH TIES options can not be specified without ORDER BY clause +ERROR: WITH TIES cannot be specified without ORDER BY clause -- test ruleutils CREATE VIEW limit_thousand_v_1 AS SELECT thousand FROM onek WHERE thousand < 995 ORDER BY thousand FETCH FIRST 5 ROWS WITH TIES OFFSET 10; @@ -657,7 +657,7 @@ View definition: CREATE VIEW limit_thousand_v_3 AS SELECT thousand FROM onek WHERE thousand < 995 ORDER BY thousand FETCH FIRST NULL ROWS WITH TIES; -- fails -ERROR: row count cannot be NULL in FETCH FIRST ... WITH TIES clause +ERROR: row count cannot be null in FETCH FIRST ... WITH TIES clause CREATE VIEW limit_thousand_v_3 AS SELECT thousand FROM onek WHERE thousand < 995 ORDER BY thousand FETCH FIRST (NULL+1) ROWS WITH TIES; \d+ limit_thousand_v_3 From 83158f74d3ab8dd48aed92e84b6207562992d13d Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 14 Sep 2020 13:56:41 +0900 Subject: [PATCH 140/589] Make index_set_state_flags() transactional 3c84046 is the original commit that introduced index_set_state_flags(), where the presence of SnapshotNow made necessary the use of an in-place update. SnapshotNow has been removed in 813fb03, so there is no actual reasons to not make this operation transactional. Note that while making the operation more robust, using a transactional operation in this routine was not strictly necessary as there was no use case for it yet. However, some future features are going to need a transactional behavior, like support for CREATE/DROP INDEX CONCURRENTLY with partitioned tables, where indexes in a partition tree need to have all their pg_index.indis* flags updated in the same transaction to make the operation stable to the end-user by keeping partition trees consistent, even with a failure mid-flight. REINDEX CONCURRENTLY uses already transactional updates when swapping the old and new indexes, making this change more consistent with the index-swapping logic. Author: Michael Paquier Reviewed-by: Anastasia Lubennikova Discussion: https://postgr.es/m/20200903080440.GA8559@paquier.xyz --- src/backend/catalog/index.c | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 117e3fdef7dc..0974f3e23a23 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -3311,18 +3311,10 @@ validate_index_callback(ItemPointer itemptr, void *opaque) * index_set_state_flags - adjust pg_index state flags * * This is used during CREATE/DROP INDEX CONCURRENTLY to adjust the pg_index - * flags that denote the index's state. Because the update is not - * transactional and will not roll back on error, this must only be used as - * the last step in a transaction that has not made any transactional catalog - * updates! + * flags that denote the index's state. * - * Note that heap_inplace_update does send a cache inval message for the + * Note that CatalogTupleUpdate() sends a cache invalidation message for the * tuple, so other sessions will hear about the update as soon as we commit. - * - * NB: In releases prior to PostgreSQL 9.4, the use of a non-transactional - * update here would have been unsafe; now that MVCC rules apply even for - * system catalog scans, we could potentially use a transactional update here - * instead. */ void index_set_state_flags(Oid indexId, IndexStateFlagsAction action) @@ -3331,9 +3323,6 @@ index_set_state_flags(Oid indexId, IndexStateFlagsAction action) HeapTuple indexTuple; Form_pg_index indexForm; - /* Assert that current xact hasn't done any transactional updates */ - Assert(GetTopTransactionIdIfAny() == InvalidTransactionId); - /* Open pg_index and fetch a writable copy of the index's tuple */ pg_index = table_open(IndexRelationId, RowExclusiveLock); @@ -3397,8 +3386,8 @@ index_set_state_flags(Oid indexId, IndexStateFlagsAction action) break; } - /* ... and write it back in-place */ - heap_inplace_update(pg_index, indexTuple); + /* ... and update it */ + CatalogTupleUpdate(pg_index, &indexTuple->t_self, indexTuple); table_close(pg_index, RowExclusiveLock); } From 95233011a08fcfa872f4505b295bc501b2844a08 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Mon, 14 Sep 2020 14:16:07 +0900 Subject: [PATCH 141/589] Fix typos. Author: Naoki Nakamichi Discussion: https://postgr.es/m/b6919d145af00295a8e86ce4d034b7cd@oss.nttdata.com --- src/backend/storage/ipc/procarray.c | 2 +- src/backend/utils/time/snapmgr.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c index 4262b3cec825..802b119c4904 100644 --- a/src/backend/storage/ipc/procarray.c +++ b/src/backend/storage/ipc/procarray.c @@ -4106,7 +4106,7 @@ GlobalVisCheckRemovableXid(Relation rel, TransactionId xid) * * Be very careful about when to use this function. It can only safely be used * when there is a guarantee that xid is within MaxTransactionId / 2 xids of - * rel. That e.g. can be guaranteed if the the caller assures a snapshot is + * rel. That e.g. can be guaranteed if the caller assures a snapshot is * held by the backend and xid is from a table (where vacuum/freezing ensures * the xid has to be within that range), or if xid is from the procarray and * prevents xid wraparound that way. diff --git a/src/backend/utils/time/snapmgr.c b/src/backend/utils/time/snapmgr.c index 22cf3ebaf472..ed7f5239a059 100644 --- a/src/backend/utils/time/snapmgr.c +++ b/src/backend/utils/time/snapmgr.c @@ -1855,7 +1855,7 @@ TransactionIdLimitedForOldSnapshots(TransactionId recentXmin, if (ts == threshold_timestamp) { /* - * Current timestamp is in same bucket as the the last limit that + * Current timestamp is in same bucket as the last limit that * was applied. Reuse. */ xlimit = threshold_xid; From 47a3a1c3d4981c85f1819bb495ec4ff70f1b9456 Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Sun, 13 Sep 2020 23:29:51 -0700 Subject: [PATCH 142/589] Fix interpolation in test name. A pre-commit review had reported the problem, but the fix reached only v10 and earlier. Back-patch to v11. Discussion: https://postgr.es/m/20200423.140546.1055476118690602079.horikyota.ntt@gmail.com --- src/test/recovery/t/020_archive_status.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/recovery/t/020_archive_status.pl b/src/test/recovery/t/020_archive_status.pl index c726453417b1..eb5c04c41158 100644 --- a/src/test/recovery/t/020_archive_status.pl +++ b/src/test/recovery/t/020_archive_status.pl @@ -64,7 +64,7 @@ FROM pg_stat_archiver }), "0|$segment_name_1", - 'pg_stat_archiver failed to archive $segment_name_1'); + "pg_stat_archiver failed to archive $segment_name_1"); # Crash the cluster for the next test in charge of checking that non-archived # WAL segments are not removed. From f560209c6e99e000f3f6c972f34f1d9dc3857f25 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 14 Sep 2020 12:35:00 -0400 Subject: [PATCH 143/589] Make walsenders show their replication commands in pg_stat_activity. A walsender process that has executed a SQL command left the text of that command in pg_stat_activity.query indefinitely, which is quite confusing if it's in RUNNING state but not doing that query. An easy and useful fix is to treat replication commands as if they were SQL queries, and show them in pg_stat_activity according to the same rules as for regular queries. While we're at it, it seems also sensible to set debug_query_string, allowing error logging and debugging to see the replication command. While here, clean up assorted silliness in exec_replication_command: * The SQLCmd path failed to restore CurrentMemoryContext to the caller's value, and failed to delete the temp context created in this routine. It's only through great good fortune that these oversights did not result in long-term memory leaks or other problems. It seems cleaner to code SQLCmd as a separate early-exit path, so do it like that. * Remove useless duplicate call of SnapBuildClearExportedSnapshot(). * replication_scanner_finish() was never called. None of those things are significant enough to merit a backpatch, so this is for HEAD only. Discussion: https://postgr.es/m/880181.1600026471@sss.pgh.pa.us --- src/backend/replication/walsender.c | 62 ++++++++++++++++------------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 3f756b470af1..67093383e66b 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1545,6 +1545,9 @@ exec_replication_command(const char *cmd_string) CHECK_FOR_INTERRUPTS(); + /* + * Parse the command. + */ cmd_context = AllocSetContextCreate(CurrentMemoryContext, "Replication command context", ALLOCSET_DEFAULT_SIZES); @@ -1557,31 +1560,47 @@ exec_replication_command(const char *cmd_string) (errcode(ERRCODE_SYNTAX_ERROR), errmsg_internal("replication command parser returned %d", parse_rc))); + replication_scanner_finish(); cmd_node = replication_parse_result; /* - * Log replication command if log_replication_commands is enabled. Even - * when it's disabled, log the command with DEBUG1 level for backward - * compatibility. Note that SQL commands are not logged here, and will be - * logged later if log_statement is enabled. + * If it's a SQL command, just clean up our mess and return false; the + * caller will take care of executing it. */ - if (cmd_node->type != T_SQLCmd) - ereport(log_replication_commands ? LOG : DEBUG1, - (errmsg("received replication command: %s", cmd_string))); + if (IsA(cmd_node, SQLCmd)) + { + if (MyDatabaseId == InvalidOid) + ereport(ERROR, + (errmsg("cannot execute SQL commands in WAL sender for physical replication"))); + + MemoryContextSwitchTo(old_context); + MemoryContextDelete(cmd_context); + + /* Tell the caller that this wasn't a WalSender command. */ + return false; + } /* - * CREATE_REPLICATION_SLOT ... LOGICAL exports a snapshot. If it was - * called outside of transaction the snapshot should be cleared here. + * Report query to various monitoring facilities. For this purpose, we + * report replication commands just like SQL commands. */ - if (!IsTransactionBlock()) - SnapBuildClearExportedSnapshot(); + debug_query_string = cmd_string; + + pgstat_report_activity(STATE_RUNNING, cmd_string); /* - * For aborted transactions, don't allow anything except pure SQL, the - * exec_simple_query() will handle it correctly. + * Log replication command if log_replication_commands is enabled. Even + * when it's disabled, log the command with DEBUG1 level for backward + * compatibility. */ - if (IsAbortedTransactionBlockState() && !IsA(cmd_node, SQLCmd)) + ereport(log_replication_commands ? LOG : DEBUG1, + (errmsg("received replication command: %s", cmd_string))); + + /* + * Disallow replication commands in aborted transaction blocks. + */ + if (IsAbortedTransactionBlockState()) ereport(ERROR, (errcode(ERRCODE_IN_FAILED_SQL_TRANSACTION), errmsg("current transaction is aborted, " @@ -1597,9 +1616,6 @@ exec_replication_command(const char *cmd_string) initStringInfo(&reply_message); initStringInfo(&tmpbuf); - /* Report to pgstat that this process is running */ - pgstat_report_activity(STATE_RUNNING, NULL); - switch (cmd_node->type) { case T_IdentifySystemCmd: @@ -1651,17 +1667,6 @@ exec_replication_command(const char *cmd_string) } break; - case T_SQLCmd: - if (MyDatabaseId == InvalidOid) - ereport(ERROR, - (errmsg("cannot execute SQL commands in WAL sender for physical replication"))); - - /* Report to pgstat that this process is now idle */ - pgstat_report_activity(STATE_IDLE, NULL); - - /* Tell the caller that this wasn't a WalSender command. */ - return false; - default: elog(ERROR, "unrecognized replication command node tag: %u", cmd_node->type); @@ -1677,6 +1682,7 @@ exec_replication_command(const char *cmd_string) /* Report to pgstat that this process is now idle */ pgstat_report_activity(STATE_IDLE, NULL); + debug_query_string = NULL; return true; } From fe4f36bcde182d57dee5dba898076aba5d826515 Mon Sep 17 00:00:00 2001 From: David Rowley Date: Tue, 15 Sep 2020 15:07:57 +1200 Subject: [PATCH 144/589] Fix compiler warning Introduced in 0aa8f7640. MSVC warned about performing 32-bit bit shifting when it appeared like we might like a 64-bit result. We did, but it just so happened that none of the calls to this function could have caused the 32-bit shift to overflow. Here we just cast the constant to int64 to make the compiler happy. Discussion: https://postgr.es/m/CAApHDvofA_vsrpC13mq_hZyuye5B-ssKEaer04OouXYCO5-uXQ@mail.gmail.com --- src/backend/utils/adt/dbsize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/utils/adt/dbsize.c b/src/backend/utils/adt/dbsize.c index 7def7392b955..3319e9761e4c 100644 --- a/src/backend/utils/adt/dbsize.c +++ b/src/backend/utils/adt/dbsize.c @@ -627,7 +627,7 @@ numeric_shift_right(Numeric n, unsigned count) Datum divisor_numeric; Datum result; - divisor_numeric = NumericGetDatum(int64_to_numeric(1 << count)); + divisor_numeric = NumericGetDatum(int64_to_numeric(((int64) 1) << count)); result = DirectFunctionCall2(numeric_div_trunc, d, divisor_numeric); return DatumGetNumeric(result); } From 62e221e1c01e3985d2b8e4b68c364f8486c327ab Mon Sep 17 00:00:00 2001 From: David Rowley Date: Tue, 15 Sep 2020 23:44:45 +1200 Subject: [PATCH 145/589] Allow incremental sorts for windowing functions This expands on the work done in d2d8a229b and allows incremental sort to be considered during create_window_paths(). Author: David Rowley Reviewed-by: Daniel Gustafsson, Tomas Vondra Discussion: https://postgr.es/m/CAApHDvoOHobiA2x13NtWnWLcTXYj9ddpCkv9PnAJQBMegYf_xw%40mail.gmail.com --- src/backend/optimizer/plan/planner.c | 41 +++++++++++++++++++++----- src/test/regress/expected/window.out | 44 ++++++++++++++++++++++++++++ src/test/regress/sql/window.sql | 22 ++++++++++++++ 3 files changed, 100 insertions(+), 7 deletions(-) diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 139c5e3dc245..8007e205ed77 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -4582,14 +4582,17 @@ create_window_paths(PlannerInfo *root, /* * Consider computing window functions starting from the existing * cheapest-total path (which will likely require a sort) as well as any - * existing paths that satisfy root->window_pathkeys (which won't). + * existing paths that satisfy or partially satisfy root->window_pathkeys. */ foreach(lc, input_rel->pathlist) { Path *path = (Path *) lfirst(lc); + int presorted_keys; if (path == input_rel->cheapest_total_path || - pathkeys_contained_in(root->window_pathkeys, path->pathkeys)) + pathkeys_count_contained_in(root->window_pathkeys, path->pathkeys, + &presorted_keys) || + presorted_keys > 0) create_one_window_path(root, window_rel, path, @@ -4664,18 +4667,42 @@ create_one_window_path(PlannerInfo *root, { WindowClause *wc = lfirst_node(WindowClause, l); List *window_pathkeys; + int presorted_keys; + bool is_sorted; window_pathkeys = make_pathkeys_for_window(root, wc, root->processed_tlist); + is_sorted = pathkeys_count_contained_in(window_pathkeys, + path->pathkeys, + &presorted_keys); + /* Sort if necessary */ - if (!pathkeys_contained_in(window_pathkeys, path->pathkeys)) + if (!is_sorted) { - path = (Path *) create_sort_path(root, window_rel, - path, - window_pathkeys, - -1.0); + /* + * No presorted keys or incremental sort disabled, just perform a + * complete sort. + */ + if (presorted_keys == 0 || !enable_incremental_sort) + path = (Path *) create_sort_path(root, window_rel, + path, + window_pathkeys, + -1.0); + else + { + /* + * Since we have presorted keys and incremental sort is + * enabled, just use incremental sort. + */ + path = (Path *) create_incremental_sort_path(root, + window_rel, + path, + window_pathkeys, + presorted_keys, + -1.0); + } } if (lnext(activeWindows, l)) diff --git a/src/test/regress/expected/window.out b/src/test/regress/expected/window.out index 13c91c9916fa..21c6cac491f1 100644 --- a/src/test/regress/expected/window.out +++ b/src/test/regress/expected/window.out @@ -3200,6 +3200,50 @@ FROM empsalary; -> Seq Scan on empsalary (5 rows) +-- Test incremental sorting +EXPLAIN (COSTS OFF) +SELECT * FROM + (SELECT depname, + empno, + salary, + enroll_date, + row_number() OVER (PARTITION BY depname ORDER BY enroll_date) AS first_emp, + row_number() OVER (PARTITION BY depname ORDER BY enroll_date DESC) AS last_emp + FROM empsalary) emp +WHERE first_emp = 1 OR last_emp = 1; + QUERY PLAN +----------------------------------------------------------------------------------- + Subquery Scan on emp + Filter: ((emp.first_emp = 1) OR (emp.last_emp = 1)) + -> WindowAgg + -> Incremental Sort + Sort Key: empsalary.depname, empsalary.enroll_date + Presorted Key: empsalary.depname + -> WindowAgg + -> Sort + Sort Key: empsalary.depname, empsalary.enroll_date DESC + -> Seq Scan on empsalary +(10 rows) + +SELECT * FROM + (SELECT depname, + empno, + salary, + enroll_date, + row_number() OVER (PARTITION BY depname ORDER BY enroll_date) AS first_emp, + row_number() OVER (PARTITION BY depname ORDER BY enroll_date DESC) AS last_emp + FROM empsalary) emp +WHERE first_emp = 1 OR last_emp = 1; + depname | empno | salary | enroll_date | first_emp | last_emp +-----------+-------+--------+-------------+-----------+---------- + develop | 8 | 6000 | 10-01-2006 | 1 | 5 + develop | 7 | 4200 | 01-01-2008 | 5 | 1 + personnel | 2 | 3900 | 12-23-2006 | 1 | 2 + personnel | 5 | 3500 | 12-10-2007 | 2 | 1 + sales | 1 | 5000 | 10-01-2006 | 1 | 3 + sales | 4 | 4800 | 08-08-2007 | 3 | 1 +(6 rows) + -- cleanup DROP TABLE empsalary; -- test user-defined window function with named args and default args diff --git a/src/test/regress/sql/window.sql b/src/test/regress/sql/window.sql index af206ca4664e..9485aebce85c 100644 --- a/src/test/regress/sql/window.sql +++ b/src/test/regress/sql/window.sql @@ -936,6 +936,28 @@ SELECT lag(1) OVER (PARTITION BY depname ORDER BY salary,enroll_date,empno) FROM empsalary; +-- Test incremental sorting +EXPLAIN (COSTS OFF) +SELECT * FROM + (SELECT depname, + empno, + salary, + enroll_date, + row_number() OVER (PARTITION BY depname ORDER BY enroll_date) AS first_emp, + row_number() OVER (PARTITION BY depname ORDER BY enroll_date DESC) AS last_emp + FROM empsalary) emp +WHERE first_emp = 1 OR last_emp = 1; + +SELECT * FROM + (SELECT depname, + empno, + salary, + enroll_date, + row_number() OVER (PARTITION BY depname ORDER BY enroll_date) AS first_emp, + row_number() OVER (PARTITION BY depname ORDER BY enroll_date DESC) AS last_emp + FROM empsalary) emp +WHERE first_emp = 1 OR last_emp = 1; + -- cleanup DROP TABLE empsalary; From 10a5b35a0061d9747ac7594c5e6faa0513276593 Mon Sep 17 00:00:00 2001 From: David Rowley Date: Wed, 16 Sep 2020 11:25:46 +1200 Subject: [PATCH 146/589] Report resource usage at the end of recovery Reporting this has been rather useful in some recent recovery speedup work. It also seems like something that will be useful to the average DBA too. Author: David Rowley Reviewed-by: Thomas Munro Discussion: https://postgr.es/m/CAApHDvqYVORiZxq2xPvP6_ndmmsTkvr6jSYv4UTNaFa5i1kd%3DQ%40mail.gmail.com --- src/backend/access/transam/xlog.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index a38371a64f96..61754312e269 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -76,6 +76,7 @@ #include "utils/memutils.h" #include "utils/ps_status.h" #include "utils/relmapper.h" +#include "utils/pg_rusage.h" #include "utils/snapmgr.h" #include "utils/timestamp.h" @@ -7169,6 +7170,9 @@ StartupXLOG(void) { ErrorContextCallback errcallback; TimestampTz xtime; + PGRUsage ru0; + + pg_rusage_init(&ru0); InRedo = true; @@ -7435,8 +7439,9 @@ StartupXLOG(void) } ereport(LOG, - (errmsg("redo done at %X/%X", - (uint32) (ReadRecPtr >> 32), (uint32) ReadRecPtr))); + (errmsg("redo done at %X/%X system usage: %s", + (uint32) (ReadRecPtr >> 32), (uint32) ReadRecPtr, + pg_rusage_show(&ru0)))); xtime = GetLatestXTime(); if (xtime) ereport(LOG, From ced138e8cbac7f5a840de8679e9882665478c680 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Tue, 15 Sep 2020 21:03:14 -0300 Subject: [PATCH 147/589] Fix use-after-free bug with event triggers in an extension script ALTER TABLE commands in an extension script are added to an event trigger command list; but starting with commit b5810de3f4 they do so in a memory context that's too short-lived, so when execution ends and time comes to use the entries, they've already been freed. (This would also be a problem with ALTER TABLE commands in a multi-command query string, but these serendipitously end in PortalContext -- which probably explains why it took so long for this to be reported.) Fix by using the memory context specifically set for that, instead. Backpatch to 13, where the aforementioned commit appeared. Reported-by: Philippe Beaudoin Author: Jehan-Guillaume de Rorthais Discussion: https://postgr.es/m/20200902193715.6e0269d4@firost --- src/backend/commands/event_trigger.c | 6 ++++++ src/test/modules/test_extensions/Makefile | 6 ++++-- .../test_extensions/expected/test_extensions.out | 5 +++++ .../test_extensions/sql/test_extensions.sql | 6 ++++++ .../test_ext_evttrig--1.0--2.0.sql | 7 +++++++ .../test_extensions/test_ext_evttrig--1.0.sql | 16 ++++++++++++++++ .../test_extensions/test_ext_evttrig.control | 3 +++ 7 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 src/test/modules/test_extensions/test_ext_evttrig--1.0--2.0.sql create mode 100644 src/test/modules/test_extensions/test_ext_evttrig--1.0.sql create mode 100644 src/test/modules/test_extensions/test_ext_evttrig.control diff --git a/src/backend/commands/event_trigger.c b/src/backend/commands/event_trigger.c index 7844880170ae..8bb17c34f5b0 100644 --- a/src/backend/commands/event_trigger.c +++ b/src/backend/commands/event_trigger.c @@ -1646,9 +1646,15 @@ EventTriggerAlterTableEnd(void) /* If no subcommands, don't collect */ if (list_length(currentEventTriggerState->currentCommand->d.alterTable.subcmds) != 0) { + MemoryContext oldcxt; + + oldcxt = MemoryContextSwitchTo(currentEventTriggerState->cxt); + currentEventTriggerState->commandList = lappend(currentEventTriggerState->commandList, currentEventTriggerState->currentCommand); + + MemoryContextSwitchTo(oldcxt); } else pfree(currentEventTriggerState->currentCommand); diff --git a/src/test/modules/test_extensions/Makefile b/src/test/modules/test_extensions/Makefile index d18108e4e5ab..77ee4d5d9ecd 100644 --- a/src/test/modules/test_extensions/Makefile +++ b/src/test/modules/test_extensions/Makefile @@ -4,11 +4,13 @@ MODULE = test_extensions PGFILEDESC = "test_extensions - regression testing for EXTENSION support" EXTENSION = test_ext1 test_ext2 test_ext3 test_ext4 test_ext5 test_ext6 \ - test_ext7 test_ext8 test_ext_cyclic1 test_ext_cyclic2 + test_ext7 test_ext8 test_ext_cyclic1 test_ext_cyclic2 \ + test_ext_evttrig DATA = test_ext1--1.0.sql test_ext2--1.0.sql test_ext3--1.0.sql \ test_ext4--1.0.sql test_ext5--1.0.sql test_ext6--1.0.sql \ test_ext7--1.0.sql test_ext7--1.0--2.0.sql test_ext8--1.0.sql \ - test_ext_cyclic1--1.0.sql test_ext_cyclic2--1.0.sql + test_ext_cyclic1--1.0.sql test_ext_cyclic2--1.0.sql \ + test_ext_evttrig--1.0.sql test_ext_evttrig--1.0--2.0.sql REGRESS = test_extensions test_extdepend diff --git a/src/test/modules/test_extensions/expected/test_extensions.out b/src/test/modules/test_extensions/expected/test_extensions.out index b5cbdfcad4f3..30ae621d05ef 100644 --- a/src/test/modules/test_extensions/expected/test_extensions.out +++ b/src/test/modules/test_extensions/expected/test_extensions.out @@ -154,3 +154,8 @@ DROP TABLE test_ext4_tab; DROP FUNCTION create_extension_with_temp_schema(); RESET client_min_messages; \unset SHOW_CONTEXT +-- Test case of an event trigger run in an extension upgrade script. +-- See: https://postgr.es/m/20200902193715.6e0269d4@firost +CREATE EXTENSION test_ext_evttrig; +ALTER EXTENSION test_ext_evttrig UPDATE TO '2.0'; +DROP EXTENSION test_ext_evttrig; diff --git a/src/test/modules/test_extensions/sql/test_extensions.sql b/src/test/modules/test_extensions/sql/test_extensions.sql index f505466ab4eb..c16fd36da896 100644 --- a/src/test/modules/test_extensions/sql/test_extensions.sql +++ b/src/test/modules/test_extensions/sql/test_extensions.sql @@ -93,3 +93,9 @@ DROP TABLE test_ext4_tab; DROP FUNCTION create_extension_with_temp_schema(); RESET client_min_messages; \unset SHOW_CONTEXT + +-- Test case of an event trigger run in an extension upgrade script. +-- See: https://postgr.es/m/20200902193715.6e0269d4@firost +CREATE EXTENSION test_ext_evttrig; +ALTER EXTENSION test_ext_evttrig UPDATE TO '2.0'; +DROP EXTENSION test_ext_evttrig; diff --git a/src/test/modules/test_extensions/test_ext_evttrig--1.0--2.0.sql b/src/test/modules/test_extensions/test_ext_evttrig--1.0--2.0.sql new file mode 100644 index 000000000000..fdd2f3542e6c --- /dev/null +++ b/src/test/modules/test_extensions/test_ext_evttrig--1.0--2.0.sql @@ -0,0 +1,7 @@ +/* src/test/modules/test_extensions/test_event_trigger--1.0--2.0.sql */ +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION test_event_trigger UPDATE TO '2.0'" to load this file. \quit + +-- Test extension upgrade with event trigger. +ALTER EVENT TRIGGER table_rewrite_trg DISABLE; +ALTER TABLE t DROP COLUMN id; diff --git a/src/test/modules/test_extensions/test_ext_evttrig--1.0.sql b/src/test/modules/test_extensions/test_ext_evttrig--1.0.sql new file mode 100644 index 000000000000..0071712cb88b --- /dev/null +++ b/src/test/modules/test_extensions/test_ext_evttrig--1.0.sql @@ -0,0 +1,16 @@ +/* src/test/modules/test_extensions/test_event_trigger--1.0.sql */ +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_event_trigger" to load this file. \quit + +-- Base table with event trigger, used in a regression test involving +-- extension upgrades. +CREATE TABLE t (id text); +CREATE OR REPLACE FUNCTION _evt_table_rewrite_fnct() +RETURNS EVENT_TRIGGER LANGUAGE plpgsql AS +$$ + BEGIN + END; +$$; +CREATE EVENT TRIGGER table_rewrite_trg + ON table_rewrite + EXECUTE PROCEDURE _evt_table_rewrite_fnct(); diff --git a/src/test/modules/test_extensions/test_ext_evttrig.control b/src/test/modules/test_extensions/test_ext_evttrig.control new file mode 100644 index 000000000000..915fae61666a --- /dev/null +++ b/src/test/modules/test_extensions/test_ext_evttrig.control @@ -0,0 +1,3 @@ +comment = 'Test extension - event trigger' +default_version = '1.0' +relocatable = true From 19c60ad69a91f346edf66996b2cf726f594d3d2b Mon Sep 17 00:00:00 2001 From: David Rowley Date: Wed, 16 Sep 2020 13:22:20 +1200 Subject: [PATCH 148/589] Optimize compactify_tuples function This function could often be seen in profiles of vacuum and could often be a significant bottleneck during recovery. The problem was that a qsort was performed in order to sort an array of item pointers in reverse offset order so that we could use that to safely move tuples up to the end of the page without overwriting the memory of yet-to-be-moved tuples. i.e. we used to compact the page starting at the back of the page and move towards the front. The qsort that this required could be expensive for pages with a large number of tuples. In this commit, we take another approach to tuple compactification. Now, instead of sorting the remaining item pointers array we first check if the array is presorted and only memmove() the tuples that need to be moved. This presorted check can be done very cheaply in the calling functions when the array is being populated. This presorted case is very fast. When the item pointer array is not presorted we must copy tuples that need to be moved into a temp buffer before copying them back into the page again. This differs from what we used to do here as we're now copying the tuples back into the page in reverse line pointer order. Previously we left the existing order alone. Reordering the tuples results in an increased likelihood of hitting the pre-sorted case the next time around. Any newly added tuple which consumes a new line pointer will also maintain the correct sort order of tuples in the page which will also result in the presorted case being hit the next time. Only consuming an unused line pointer can cause the order of tuples to go out again, but that will be corrected next time the function is called for the page. Benchmarks have shown that the non-presorted case is at least equally as fast as the original qsort method even when the page just has a few tuples. As the number of tuples becomes larger the new method maintains its performance whereas the original qsort method became much slower when the number of tuples on the page became large. Author: David Rowley Reviewed-by: Thomas Munro Tested-by: Jakub Wartak Discussion: https://postgr.es/m/CA+hUKGKMQFVpjr106gRhwk6R-nXv0qOcTreZuQzxgpHESAL6dw@mail.gmail.com --- src/backend/storage/page/bufpage.c | 284 +++++++++++++++++++++++++---- 1 file changed, 252 insertions(+), 32 deletions(-) diff --git a/src/backend/storage/page/bufpage.c b/src/backend/storage/page/bufpage.c index d708117a4067..4bc2bf955dfd 100644 --- a/src/backend/storage/page/bufpage.c +++ b/src/backend/storage/page/bufpage.c @@ -411,51 +411,250 @@ PageRestoreTempPage(Page tempPage, Page oldPage) } /* - * sorting support for PageRepairFragmentation and PageIndexMultiDelete + * Tuple defrag support for PageRepairFragmentation and PageIndexMultiDelete */ -typedef struct itemIdSortData +typedef struct itemIdCompactData { uint16 offsetindex; /* linp array index */ int16 itemoff; /* page offset of item data */ uint16 alignedlen; /* MAXALIGN(item data len) */ -} itemIdSortData; -typedef itemIdSortData *itemIdSort; - -static int -itemoffcompare(const void *itemidp1, const void *itemidp2) -{ - /* Sort in decreasing itemoff order */ - return ((itemIdSort) itemidp2)->itemoff - - ((itemIdSort) itemidp1)->itemoff; -} +} itemIdCompactData; +typedef itemIdCompactData *itemIdCompact; /* * After removing or marking some line pointers unused, move the tuples to - * remove the gaps caused by the removed items. + * remove the gaps caused by the removed items and reorder them back into + * reverse line pointer order in the page. + * + * This function can often be fairly hot, so it pays to take some measures to + * make it as optimal as possible. + * + * Callers may pass 'presorted' as true if the 'itemidbase' array is sorted in + * descending order of itemoff. When this is true we can just memmove() + * tuples towards the end of the page. This is quite a common case as it's + * the order that tuples are initially inserted into pages. When we call this + * function to defragment the tuples in the page then any new line pointers + * added to the page will keep that presorted order, so hitting this case is + * still very common for tables that are commonly updated. + * + * When the 'itemidbase' array is not presorted then we're unable to just + * memmove() tuples around freely. Doing so could cause us to overwrite the + * memory belonging to a tuple we've not moved yet. In this case, we copy all + * the tuples that need to be moved into a temporary buffer. We can then + * simply memcpy() out of that temp buffer back into the page at the correct + * location. Tuples are copied back into the page in the same order as the + * 'itemidbase' array, so we end up reordering the tuples back into reverse + * line pointer order. This will increase the chances of hitting the + * presorted case the next time around. + * + * Callers must ensure that nitems is > 0 */ static void -compactify_tuples(itemIdSort itemidbase, int nitems, Page page) +compactify_tuples(itemIdCompact itemidbase, int nitems, Page page, bool presorted) { PageHeader phdr = (PageHeader) page; Offset upper; + Offset copy_tail; + Offset copy_head; + itemIdCompact itemidptr; int i; - /* sort itemIdSortData array into decreasing itemoff order */ - qsort((char *) itemidbase, nitems, sizeof(itemIdSortData), - itemoffcompare); + /* Code within will not work correctly if nitems == 0 */ + Assert(nitems > 0); - upper = phdr->pd_special; - for (i = 0; i < nitems; i++) + if (presorted) { - itemIdSort itemidptr = &itemidbase[i]; - ItemId lp; - lp = PageGetItemId(page, itemidptr->offsetindex + 1); - upper -= itemidptr->alignedlen; +#ifdef USE_ASSERT_CHECKING + { + /* + * Verify we've not gotten any new callers that are incorrectly + * passing a true presorted value. + */ + Offset lastoff = phdr->pd_special; + + for (i = 0; i < nitems; i++) + { + itemidptr = &itemidbase[i]; + + Assert(lastoff > itemidptr->itemoff); + + lastoff = itemidptr->itemoff; + } + } +#endif /* USE_ASSERT_CHECKING */ + + /* + * 'itemidbase' is already in the optimal order, i.e, lower item + * pointers have a higher offset. This allows us to memmove() the + * tuples up to the end of the page without having to worry about + * overwriting other tuples that have not been moved yet. + * + * There's a good chance that there are tuples already right at the + * end of the page that we can simply skip over because they're + * already in the correct location within the page. We'll do that + * first... + */ + upper = phdr->pd_special; + i = 0; + do + { + itemidptr = &itemidbase[i]; + if (upper != itemidptr->itemoff + itemidptr->alignedlen) + break; + upper -= itemidptr->alignedlen; + + i++; + } while (i < nitems); + + /* + * Now that we've found the first tuple that needs to be moved, we can + * do the tuple compactification. We try and make the least number of + * memmove() calls and only call memmove() when there's a gap. When + * we see a gap we just move all tuples after the gap up until the + * point of the last move operation. + */ + copy_tail = copy_head = itemidptr->itemoff + itemidptr->alignedlen; + for (; i < nitems; i++) + { + ItemId lp; + + itemidptr = &itemidbase[i]; + lp = PageGetItemId(page, itemidptr->offsetindex + 1); + + if (copy_head != itemidptr->itemoff + itemidptr->alignedlen) + { + memmove((char *) page + upper, + page + copy_head, + copy_tail - copy_head); + + /* + * We've now moved all tuples already seen, but not the + * current tuple, so we set the copy_tail to the end of this + * tuple so it can be moved in another iteration of the loop. + */ + copy_tail = itemidptr->itemoff + itemidptr->alignedlen; + } + /* shift the target offset down by the length of this tuple */ + upper -= itemidptr->alignedlen; + /* point the copy_head to the start of this tuple */ + copy_head = itemidptr->itemoff; + + /* update the line pointer to reference the new offset */ + lp->lp_off = upper; + + } + + /* move the remaining tuples. */ memmove((char *) page + upper, - (char *) page + itemidptr->itemoff, - itemidptr->alignedlen); - lp->lp_off = upper; + page + copy_head, + copy_tail - copy_head); + } + else + { + PGAlignedBlock scratch; + char *scratchptr = scratch.data; + + /* + * Non-presorted case: The tuples in the itemidbase array may be in + * any order. So, in order to move these to the end of the page we + * must make a temp copy of each tuple that needs to be moved before + * we copy them back into the page at the new offset. + * + * If a large percentage of tuples have been pruned (>75%) then we'll + * copy these into the temp buffer tuple-by-tuple, otherwise, we'll + * just do a single memcpy() for all tuples that need to be moved. + * When so many tuples have been removed there's likely to be a lot of + * gaps and it's unlikely that many non-movable tuples remain at the + * end of the page. + */ + if (nitems < PageGetMaxOffsetNumber(page) / 4) + { + i = 0; + do + { + itemidptr = &itemidbase[i]; + memcpy(scratchptr + itemidptr->itemoff, page + itemidptr->itemoff, + itemidptr->alignedlen); + i++; + } while (i < nitems); + + /* Set things up for the compactification code below */ + i = 0; + itemidptr = &itemidbase[0]; + upper = phdr->pd_special; + } + else + { + upper = phdr->pd_special; + + /* + * Many tuples are likely to already be in the correct location. + * There's no need to copy these into the temp buffer. Instead + * we'll just skip forward in the itemidbase array to the position + * that we do need to move tuples from so that the code below just + * leaves these ones alone. + */ + i = 0; + do + { + itemidptr = &itemidbase[i]; + if (upper != itemidptr->itemoff + itemidptr->alignedlen) + break; + upper -= itemidptr->alignedlen; + + i++; + } while (i < nitems); + + /* Copy all tuples that need to be moved into the temp buffer */ + memcpy(scratchptr + phdr->pd_upper, + page + phdr->pd_upper, + upper - phdr->pd_upper); + } + + /* + * Do the tuple compactification. itemidptr is already pointing to + * the first tuple that we're going to move. Here we collapse the + * memcpy calls for adjacent tuples into a single call. This is done + * by delaying the memcpy call until we find a gap that needs to be + * closed. + */ + copy_tail = copy_head = itemidptr->itemoff + itemidptr->alignedlen; + for (; i < nitems; i++) + { + ItemId lp; + + itemidptr = &itemidbase[i]; + lp = PageGetItemId(page, itemidptr->offsetindex + 1); + + /* copy pending tuples when we detect a gap */ + if (copy_head != itemidptr->itemoff + itemidptr->alignedlen) + { + memcpy((char *) page + upper, + scratchptr + copy_head, + copy_tail - copy_head); + + /* + * We've now copied all tuples already seen, but not the + * current tuple, so we set the copy_tail to the end of this + * tuple. + */ + copy_tail = itemidptr->itemoff + itemidptr->alignedlen; + } + /* shift the target offset down by the length of this tuple */ + upper -= itemidptr->alignedlen; + /* point the copy_head to the start of this tuple */ + copy_head = itemidptr->itemoff; + + /* update the line pointer to reference the new offset */ + lp->lp_off = upper; + + } + + /* Copy the remaining chunk */ + memcpy((char *) page + upper, + scratchptr + copy_head, + copy_tail - copy_head); } phdr->pd_upper = upper; @@ -477,14 +676,16 @@ PageRepairFragmentation(Page page) Offset pd_lower = ((PageHeader) page)->pd_lower; Offset pd_upper = ((PageHeader) page)->pd_upper; Offset pd_special = ((PageHeader) page)->pd_special; - itemIdSortData itemidbase[MaxHeapTuplesPerPage]; - itemIdSort itemidptr; + Offset last_offset; + itemIdCompactData itemidbase[MaxHeapTuplesPerPage]; + itemIdCompact itemidptr; ItemId lp; int nline, nstorage, nunused; int i; Size totallen; + bool presorted = true; /* For now */ /* * It's worth the trouble to be more paranoid here than in most places, @@ -509,6 +710,7 @@ PageRepairFragmentation(Page page) nline = PageGetMaxOffsetNumber(page); itemidptr = itemidbase; nunused = totallen = 0; + last_offset = pd_special; for (i = FirstOffsetNumber; i <= nline; i++) { lp = PageGetItemId(page, i); @@ -518,6 +720,12 @@ PageRepairFragmentation(Page page) { itemidptr->offsetindex = i - 1; itemidptr->itemoff = ItemIdGetOffset(lp); + + if (last_offset > itemidptr->itemoff) + last_offset = itemidptr->itemoff; + else + presorted = false; + if (unlikely(itemidptr->itemoff < (int) pd_upper || itemidptr->itemoff >= (int) pd_special)) ereport(ERROR, @@ -552,7 +760,7 @@ PageRepairFragmentation(Page page) errmsg("corrupted item lengths: total %u, available space %u", (unsigned int) totallen, pd_special - pd_lower))); - compactify_tuples(itemidbase, nstorage, page); + compactify_tuples(itemidbase, nstorage, page, presorted); } /* Set hint bit for PageAddItem */ @@ -831,9 +1039,10 @@ PageIndexMultiDelete(Page page, OffsetNumber *itemnos, int nitems) Offset pd_lower = phdr->pd_lower; Offset pd_upper = phdr->pd_upper; Offset pd_special = phdr->pd_special; - itemIdSortData itemidbase[MaxIndexTuplesPerPage]; + Offset last_offset; + itemIdCompactData itemidbase[MaxIndexTuplesPerPage]; ItemIdData newitemids[MaxIndexTuplesPerPage]; - itemIdSort itemidptr; + itemIdCompact itemidptr; ItemId lp; int nline, nused; @@ -842,6 +1051,7 @@ PageIndexMultiDelete(Page page, OffsetNumber *itemnos, int nitems) unsigned offset; int nextitm; OffsetNumber offnum; + bool presorted = true; /* For now */ Assert(nitems <= MaxIndexTuplesPerPage); @@ -883,6 +1093,7 @@ PageIndexMultiDelete(Page page, OffsetNumber *itemnos, int nitems) totallen = 0; nused = 0; nextitm = 0; + last_offset = pd_special; for (offnum = FirstOffsetNumber; offnum <= nline; offnum = OffsetNumberNext(offnum)) { lp = PageGetItemId(page, offnum); @@ -906,6 +1117,12 @@ PageIndexMultiDelete(Page page, OffsetNumber *itemnos, int nitems) { itemidptr->offsetindex = nused; /* where it will go */ itemidptr->itemoff = offset; + + if (last_offset > itemidptr->itemoff) + last_offset = itemidptr->itemoff; + else + presorted = false; + itemidptr->alignedlen = MAXALIGN(size); totallen += itemidptr->alignedlen; newitemids[nused] = *lp; @@ -932,7 +1149,10 @@ PageIndexMultiDelete(Page page, OffsetNumber *itemnos, int nitems) phdr->pd_lower = SizeOfPageHeaderData + nused * sizeof(ItemIdData); /* and compactify the tuple data */ - compactify_tuples(itemidbase, nused, page); + if (nused > 0) + compactify_tuples(itemidbase, nused, page, presorted); + else + phdr->pd_upper = pd_special; } From 69bd60672af63eaa8b19cdcea175df5ff172e80e Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Wed, 16 Sep 2020 07:45:44 +0530 Subject: [PATCH 149/589] Fix initialization of RelationSyncEntry for streaming transactions. In commit 464824323e, for each RelationSyncEntry we maintained the list of xids (streamed_txns) for which we have already sent the schema. This helps us to track when to send the schema to the downstream node for replication of streaming transactions. Before this list got initialized, we were processing invalidation messages which access this list and led to an assertion failure. In passing, clean up the nearby code: * Initialize the list of xids with NIL instead of NULL which is our usual coding practice. * Remove the MemoryContext switch for creating a RelationSyncEntry in dynahash. Diagnosed-by: Amit Kapila and Tom Lane Author: Amit Kapila Reviewed-by: Tom Lane and Dilip Kumar Discussion: https://postgr.es/m/904373.1600033123@sss.pgh.pa.us --- src/backend/replication/pgoutput/pgoutput.c | 29 +++++++++++---------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c index 343f03129fe7..eb1f23004e77 100644 --- a/src/backend/replication/pgoutput/pgoutput.c +++ b/src/backend/replication/pgoutput/pgoutput.c @@ -945,16 +945,26 @@ get_rel_sync_entry(PGOutputData *data, Oid relid) Assert(RelationSyncCache != NULL); - /* Find cached function info, creating if not found */ - oldctx = MemoryContextSwitchTo(CacheMemoryContext); + /* Find cached relation info, creating if not found */ entry = (RelationSyncEntry *) hash_search(RelationSyncCache, (void *) &relid, HASH_ENTER, &found); - MemoryContextSwitchTo(oldctx); Assert(entry != NULL); /* Not found means schema wasn't sent */ - if (!found || !entry->replicate_valid) + if (!found) + { + /* immediately make a new entry valid enough to satisfy callbacks */ + entry->schema_sent = false; + entry->streamed_txns = NIL; + entry->replicate_valid = false; + entry->pubactions.pubinsert = entry->pubactions.pubupdate = + entry->pubactions.pubdelete = entry->pubactions.pubtruncate = false; + entry->publish_as_relid = InvalidOid; + } + + /* Validate the entry */ + if (!entry->replicate_valid) { List *pubids = GetRelationPublications(relid); ListCell *lc; @@ -977,9 +987,6 @@ get_rel_sync_entry(PGOutputData *data, Oid relid) * relcache considers all publications given relation is in, but here * we only need to consider ones that the subscriber requested. */ - entry->pubactions.pubinsert = entry->pubactions.pubupdate = - entry->pubactions.pubdelete = entry->pubactions.pubtruncate = false; - foreach(lc, data->publications) { Publication *pub = lfirst(lc); @@ -1054,12 +1061,6 @@ get_rel_sync_entry(PGOutputData *data, Oid relid) entry->replicate_valid = true; } - if (!found) - { - entry->schema_sent = false; - entry->streamed_txns = NULL; - } - return entry; } @@ -1145,7 +1146,7 @@ rel_sync_cache_relation_cb(Datum arg, Oid relid) { entry->schema_sent = false; list_free(entry->streamed_txns); - entry->streamed_txns = NULL; + entry->streamed_txns = NIL; } } From 3bd35d4f516adfc492360b20e72911949c961e47 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Tue, 15 Sep 2020 21:16:31 -0700 Subject: [PATCH 150/589] HashAgg: release write buffers sooner by rewinding tape. This was an oversight. The purpose of 7fdd919ae7 was to avoid keeping tape buffers around unnecessisarily, but HashAgg didn't rewind early enough. Reviewed-by: Peter Geoghegan Discussion: https://postgr.es/m/1fb1151c2cddf8747d14e0532da283c3f97e2685.camel@j-davis.com Backpatch-through: 13 --- src/backend/executor/nodeAgg.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c index f74d4841f170..28802e6588db 100644 --- a/src/backend/executor/nodeAgg.c +++ b/src/backend/executor/nodeAgg.c @@ -2639,8 +2639,6 @@ agg_refill_hash_table(AggState *aggstate) */ hashagg_recompile_expressions(aggstate, true, true); - LogicalTapeRewindForRead(tapeinfo->tapeset, batch->input_tapenum, - HASHAGG_READ_BUFFER_SIZE); for (;;) { TupleTableSlot *spillslot = aggstate->hash_spill_rslot; @@ -2923,6 +2921,7 @@ hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions, static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum) { + /* rewinding frees the buffer while not in use */ LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum); if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes) { @@ -3152,6 +3151,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno) for (i = 0; i < spill->npartitions; i++) { + LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset; int tapenum = spill->partitions[i]; HashAggBatch *new_batch; double cardinality; @@ -3163,9 +3163,13 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno) cardinality = estimateHyperLogLog(&spill->hll_card[i]); freeHyperLogLog(&spill->hll_card[i]); - new_batch = hashagg_batch_new(aggstate->hash_tapeinfo->tapeset, - tapenum, setno, spill->ntuples[i], - cardinality, used_bits); + /* rewinding frees the buffer while not in use */ + LogicalTapeRewindForRead(tapeset, tapenum, + HASHAGG_READ_BUFFER_SIZE); + + new_batch = hashagg_batch_new(tapeset, tapenum, setno, + spill->ntuples[i], cardinality, + used_bits); aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches); aggstate->hash_batches_used++; } From c8aeaf3ab31edeedf1791e37c74bcedf61a916ed Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Tue, 15 Sep 2020 21:34:05 -0700 Subject: [PATCH 151/589] Change LogicalTapeSetBlocks() to use nBlocksWritten. Previously, it was based on nBlocksAllocated to account for tapes with open write buffers that may not have made it to the BufFile yet. That was unnecessary, because callers do not need to get the number of blocks while a tape has an open write buffer; and it also conflicted with the preallocation logic added for HashAgg. Reviewed-by: Peter Geoghegan Discussion: https://postgr.es/m/ce5af05900fdbd0e9185747825a7423c48501964.camel@j-davis.com Backpatch-through: 13 --- src/backend/executor/nodeAgg.c | 2 +- src/backend/utils/sort/logtape.c | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c index 28802e6588db..75e5bbf209d5 100644 --- a/src/backend/executor/nodeAgg.c +++ b/src/backend/executor/nodeAgg.c @@ -2704,8 +2704,8 @@ agg_refill_hash_table(AggState *aggstate) if (spill_initialized) { - hash_agg_update_metrics(aggstate, true, spill.npartitions); hashagg_spill_finish(aggstate, &spill, batch->setno); + hash_agg_update_metrics(aggstate, true, spill.npartitions); } else hash_agg_update_metrics(aggstate, true, 0); diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c index d6d1e1911ea4..28905124f965 100644 --- a/src/backend/utils/sort/logtape.c +++ b/src/backend/utils/sort/logtape.c @@ -1264,9 +1264,19 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum, /* * Obtain total disk space currently used by a LogicalTapeSet, in blocks. + * + * This should not be called while there are open write buffers; otherwise it + * may not account for buffered data. */ long LogicalTapeSetBlocks(LogicalTapeSet *lts) { - return lts->nBlocksAllocated - lts->nHoleBlocks; +#ifdef USE_ASSERT_CHECKING + for (int i = 0; i < lts->nTapes; i++) + { + LogicalTape *lt = <s->tapes[i]; + Assert(!lt->writing || lt->buffer == NULL); + } +#endif + return lts->nBlocksWritten - lts->nHoleBlocks; } From 5423853feebd30772b7ff9b306885dcb02b79e76 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 16 Sep 2020 16:26:50 +0900 Subject: [PATCH 152/589] Avoid retrieval of CHECK constraints and DEFAULT exprs in data-only dump Those extra queries are not necessary when doing a data-only dump. With this change, this means that the dependencies between CHECK/DEFAULT and the parent table are not tracked anymore for a data-only dump. However, these dependencies are only used for the schema generation and we have never guaranteed that a dump can be reloaded if a CHECK constraint uses a custom function whose behavior changes when loading the data, like when using cross-table references in the CHECK function. Author: Julien Rouhaud Reviewed-by: Daniel Gustafsson, Michael Paquier Discussion: https://postgr.es/m/20200712054850.GA92357@nol --- src/bin/pg_dump/pg_dump.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 784bceaec394..340638887217 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -8644,9 +8644,10 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) PQclear(res); /* - * Get info about column defaults + * Get info about column defaults. This is skipped for a data-only + * dump, as it is only needed for table schemas. */ - if (hasdefaults) + if (!dopt->dataOnly && hasdefaults) { AttrDefInfo *attrdefs; int numDefaults; @@ -8731,9 +8732,10 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) } /* - * Get info about table CHECK constraints + * Get info about table CHECK constraints. This is skipped for a + * data-only dump, as it is only needed for table schemas. */ - if (tbinfo->ncheck > 0) + if (!dopt->dataOnly && tbinfo->ncheck > 0) { ConstraintInfo *constrs; int numConstrs; From e568ed0eb07239b7e53d948565ebaeb6f379630f Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Wed, 16 Sep 2020 18:47:39 +0900 Subject: [PATCH 153/589] Add leader_pid field into the example of file_fdw for csvlog. Commit b8fdee7d0c added leader_pid field into csvlog, but forgot to update the example of file_fdw for csvlog. Author: Yuta Katsuragi --- doc/src/sgml/file-fdw.sgml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/file-fdw.sgml b/doc/src/sgml/file-fdw.sgml index eefc6e7e5b19..8831f5911f12 100644 --- a/doc/src/sgml/file-fdw.sgml +++ b/doc/src/sgml/file-fdw.sgml @@ -265,7 +265,8 @@ CREATE FOREIGN TABLE pglog ( query_pos integer, location text, application_name text, - backend_type text + backend_type text, + leader_pid integer ) SERVER pglog OPTIONS ( filename 'log/pglog.csv', format 'csv' ); From 3d65b0593c5578014f62e09d4008006f1783f64d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 16 Sep 2020 12:07:31 -0400 Subject: [PATCH 154/589] Fix bogus cache-invalidation logic in logical replication worker. The code recorded cache invalidation events by zeroing the "localreloid" field of affected cache entries. However, it's possible for an inval event to occur even while we have the entry open and locked. So an ill-timed inval could result in "cache lookup failed for relation 0" errors, if the worker's code tried to use the cleared field. We can fix that by creating a separate bool field to record whether the entry needs to be revalidated. (In the back branches, cram the bool into what had been padding space, to avoid an ABI break in the somewhat unlikely event that any extension is looking at this struct.) Also, rearrange the logic in logicalrep_rel_open so that it does the right thing in cases where table_open would fail. We should retry the lookup by name in that case, but we didn't. The real-world impact of this is probably small. In the first place, the error conditions are very low probability, and in the second place, the worker would just exit and get restarted. We only noticed because in a CLOBBER_CACHE_ALWAYS build, the failure can occur repeatedly, preventing the worker from making progress. Nonetheless, it's clearly a bug, and it impedes a useful type of testing; so back-patch to v10 where this code was introduced. Discussion: https://postgr.es/m/1032727.1600096803@sss.pgh.pa.us --- src/backend/replication/logical/relation.c | 71 ++++++++++++++-------- src/include/replication/logicalrelation.h | 11 +++- 2 files changed, 53 insertions(+), 29 deletions(-) diff --git a/src/backend/replication/logical/relation.c b/src/backend/replication/logical/relation.c index 3d2d56295b0c..9ee70a2563e4 100644 --- a/src/backend/replication/logical/relation.c +++ b/src/backend/replication/logical/relation.c @@ -77,7 +77,7 @@ logicalrep_relmap_invalidate_cb(Datum arg, Oid reloid) { if (entry->localreloid == reloid) { - entry->localreloid = InvalidOid; + entry->localrelvalid = false; hash_seq_term(&status); break; } @@ -91,7 +91,7 @@ logicalrep_relmap_invalidate_cb(Datum arg, Oid reloid) hash_seq_init(&status, LogicalRepRelMap); while ((entry = (LogicalRepRelMapEntry *) hash_seq_search(&status)) != NULL) - entry->localreloid = InvalidOid; + entry->localrelvalid = false; } } @@ -230,15 +230,13 @@ logicalrep_rel_att_by_name(LogicalRepRelation *remoterel, const char *attname) /* * Open the local relation associated with the remote one. * - * Optionally rebuilds the Relcache mapping if it was invalidated - * by local DDL. + * Rebuilds the Relcache mapping if it was invalidated by local DDL. */ LogicalRepRelMapEntry * logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode) { LogicalRepRelMapEntry *entry; bool found; - Oid relid = InvalidOid; LogicalRepRelation *remoterel; if (LogicalRepRelMap == NULL) @@ -254,14 +252,45 @@ logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode) remoterel = &entry->remoterel; + /* Ensure we don't leak a relcache refcount. */ + if (entry->localrel) + elog(ERROR, "remote relation ID %u is already open", remoteid); + /* * When opening and locking a relation, pending invalidation messages are - * processed which can invalidate the relation. We need to update the - * local cache both when we are first time accessing the relation and when - * the relation is invalidated (aka entry->localreloid is set InvalidOid). + * processed which can invalidate the relation. Hence, if the entry is + * currently considered valid, try to open the local relation by OID and + * see if invalidation ensues. + */ + if (entry->localrelvalid) + { + entry->localrel = try_table_open(entry->localreloid, lockmode); + if (!entry->localrel) + { + /* Table was renamed or dropped. */ + entry->localrelvalid = false; + } + else if (!entry->localrelvalid) + { + /* Note we release the no-longer-useful lock here. */ + table_close(entry->localrel, lockmode); + entry->localrel = NULL; + } + } + + /* + * If the entry has been marked invalid since we last had lock on it, + * re-open the local relation by name and rebuild all derived data. */ - if (!OidIsValid(entry->localreloid)) + if (!entry->localrelvalid) { + Oid relid; + int found; + Bitmapset *idkey; + TupleDesc desc; + MemoryContext oldctx; + int i; + /* Try to find and lock the relation by name. */ relid = RangeVarGetRelid(makeRangeVar(remoterel->nspname, remoterel->relname, -1), @@ -272,21 +301,7 @@ logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode) errmsg("logical replication target relation \"%s.%s\" does not exist", remoterel->nspname, remoterel->relname))); entry->localrel = table_open(relid, NoLock); - - } - else - { - relid = entry->localreloid; - entry->localrel = table_open(entry->localreloid, lockmode); - } - - if (!OidIsValid(entry->localreloid)) - { - int found; - Bitmapset *idkey; - TupleDesc desc; - MemoryContext oldctx; - int i; + entry->localreloid = relid; /* Check for supported relkind. */ CheckSubscriptionRelkind(entry->localrel->rd_rel->relkind, @@ -380,7 +395,7 @@ logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode) } } - entry->localreloid = relid; + entry->localrelvalid = true; } if (entry->state != SUBREL_STATE_READY) @@ -523,7 +538,7 @@ logicalrep_partmap_invalidate_cb(Datum arg, Oid reloid) { if (entry->localreloid == reloid) { - entry->localreloid = InvalidOid; + entry->localrelvalid = false; hash_seq_term(&status); break; } @@ -537,7 +552,7 @@ logicalrep_partmap_invalidate_cb(Datum arg, Oid reloid) hash_seq_init(&status, LogicalRepPartMap); while ((entry = (LogicalRepRelMapEntry *) hash_seq_search(&status)) != NULL) - entry->localreloid = InvalidOid; + entry->localrelvalid = false; } } @@ -656,6 +671,8 @@ logicalrep_partition_open(LogicalRepRelMapEntry *root, entry->updatable = root->updatable; + entry->localrelvalid = true; + /* state and statelsn are left set to 0. */ MemoryContextSwitchTo(oldctx); diff --git a/src/include/replication/logicalrelation.h b/src/include/replication/logicalrelation.h index a6b44b12bd1f..62ddd3c7a2ae 100644 --- a/src/include/replication/logicalrelation.h +++ b/src/include/replication/logicalrelation.h @@ -19,9 +19,16 @@ typedef struct LogicalRepRelMapEntry { LogicalRepRelation remoterel; /* key is remoterel.remoteid */ - /* Mapping to local relation, filled as needed. */ + /* + * Validity flag -- when false, revalidate all derived info at next + * logicalrep_rel_open. (While the localrel is open, we assume our lock + * on that rel ensures the info remains good.) + */ + bool localrelvalid; + + /* Mapping to local relation. */ Oid localreloid; /* local relation id */ - Relation localrel; /* relcache entry */ + Relation localrel; /* relcache entry (NULL when closed) */ AttrMap *attrmap; /* map of local attributes to remote ones */ bool updatable; /* Can apply updates/deletes? */ From e5fac1cb1941e4adbcb88206f914e2035e5cccf2 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 16 Sep 2020 13:38:26 -0400 Subject: [PATCH 155/589] Avoid unnecessary recursion to child tables in ALTER TABLE SET NOT NULL. If a partitioned table's column is already marked NOT NULL, there is no need to examine its partitions, because we can rely on previous DDL to have enforced that the child columns are NOT NULL as well. (Unfortunately, the same cannot be said for traditional inheritance, so for now we have to restrict the optimization to partitioned tables.) Hence, we may skip recursing to child tables in this situation. The reason this case is worth worrying about is that when pg_dump dumps a partitioned table having a primary key, it will include the requisite NOT NULL markings in the CREATE TABLE commands, and then add the primary key as a separate step. The primary key addition generates a SET NOT NULL as a subcommand, just to be sure. So the situation where a SET NOT NULL is redundant does arise in the real world. Skipping the recursion does more than just save a few cycles: it means that a command such as "ALTER TABLE ONLY partition_parent ADD PRIMARY KEY" will take locks only on the partition parent table, not on the partitions. It turns out that parallel pg_restore is effectively assuming that that's true, and has little choice but to do so because the dependencies listed for such a TOC entry don't include the partitions. pg_restore could thus issue this ALTER while data restores on the partitions are still in progress. Taking unnecessary locks on the partitions not only hurts concurrency, but can lead to actual deadlock failures, as reported by Domagoj Smoljanovic. (A contributing factor in the deadlock is that TRUNCATE on a child partition wants a non-exclusive lock on the parent. This seems likewise unnecessary, but the fix for it is more invasive so we won't consider back-patching it. Fortunately, getting rid of one of these two poor behaviors is enough to remove the deadlock.) Although support for partitioned primary keys came in with v11, this patch is dependent on the SET NOT NULL refactoring done by commit f4a3fdfbd, so we can only patch back to v12. Patch by me; thanks to Alvaro Herrera and Amit Langote for review. Discussion: https://postgr.es/m/VI1PR03MB31670CA1BD9625C3A8C5DD05EB230@VI1PR03MB3167.eurprd03.prod.outlook.com --- src/backend/commands/tablecmds.c | 45 +++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index c21a309f04fd..eab570a8675c 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -5681,14 +5681,10 @@ ATSimpleRecursion(List **wqueue, Relation rel, AlterTableUtilityContext *context) { /* - * Propagate to children if desired. Only plain tables, foreign tables - * and partitioned tables have children, so no need to search for other - * relkinds. + * Propagate to children, if desired and if there are (or might be) any + * children. */ - if (recurse && - (rel->rd_rel->relkind == RELKIND_RELATION || - rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE || - rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)) + if (recurse && rel->rd_rel->relhassubclass) { Oid relid = RelationGetRelid(rel); ListCell *child; @@ -6698,6 +6694,41 @@ ATPrepSetNotNull(List **wqueue, Relation rel, if (recursing) return; + /* + * If the target column is already marked NOT NULL, we can skip recursing + * to children, because their columns should already be marked NOT NULL as + * well. But there's no point in checking here unless the relation has + * some children; else we can just wait till execution to check. (If it + * does have children, however, this can save taking per-child locks + * unnecessarily. This greatly improves concurrency in some parallel + * restore scenarios.) + * + * Unfortunately, we can only apply this optimization to partitioned + * tables, because traditional inheritance doesn't enforce that child + * columns be NOT NULL when their parent is. (That's a bug that should + * get fixed someday.) + */ + if (rel->rd_rel->relhassubclass && + rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + { + HeapTuple tuple; + bool attnotnull; + + tuple = SearchSysCacheAttName(RelationGetRelid(rel), cmd->name); + + /* Might as well throw the error now, if name is bad */ + if (!HeapTupleIsValid(tuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("column \"%s\" of relation \"%s\" does not exist", + cmd->name, RelationGetRelationName(rel)))); + + attnotnull = ((Form_pg_attribute) GETSTRUCT(tuple))->attnotnull; + ReleaseSysCache(tuple); + if (attnotnull) + return; + } + /* * If we have ALTER TABLE ONLY ... SET NOT NULL on a partitioned table, * apply ALTER TABLE ... CHECK NOT NULL to every child. Otherwise, use From aac80bfcddf7df3fbd1eb73e8a386115d922c714 Mon Sep 17 00:00:00 2001 From: Peter Geoghegan Date: Wed, 16 Sep 2020 10:42:30 -0700 Subject: [PATCH 156/589] Fix amcheck child check pg_upgrade bug. Commit d114cc53 overlooked the fact that pg_upgrade'd B-Tree indexes have leaf page high keys whose offset numbers do not match the one from the copy of the tuple one level up (the copy stored with a downlink for leaf page's right sibling page). This led to false positive reports of corruption from bt_index_parent_check() when it was called to verify a pg_upgrade'd index. To fix, skip comparing the offset number on pg_upgrade'd B-Tree indexes. Author: Anastasia Lubennikova Author: Peter Geoghegan Reported-By: Andrew Bille Diagnosed-By: Anastasia Lubennikova Bug: #16619 Discussion: https://postgr.es/m/16619-aaba10f83fdc1c3c@postgresql.org Backpatch: 13-, where child check was enhanced. --- contrib/amcheck/verify_nbtree.c | 34 +++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/contrib/amcheck/verify_nbtree.c b/contrib/amcheck/verify_nbtree.c index 5f3de3c0b7f6..6d86e3ccdacf 100644 --- a/contrib/amcheck/verify_nbtree.c +++ b/contrib/amcheck/verify_nbtree.c @@ -1752,14 +1752,36 @@ bt_right_page_check_scankey(BtreeCheckState *state) * this function is capable to compare pivot keys on different levels. */ static bool -bt_pivot_tuple_identical(IndexTuple itup1, IndexTuple itup2) +bt_pivot_tuple_identical(bool heapkeyspace, IndexTuple itup1, IndexTuple itup2) { if (IndexTupleSize(itup1) != IndexTupleSize(itup2)) return false; - if (memcmp(&itup1->t_tid.ip_posid, &itup2->t_tid.ip_posid, - IndexTupleSize(itup1) - offsetof(ItemPointerData, ip_posid)) != 0) - return false; + if (heapkeyspace) + { + /* + * Offset number will contain important information in heapkeyspace + * indexes: the number of attributes left in the pivot tuple following + * suffix truncation. Don't skip over it (compare it too). + */ + if (memcmp(&itup1->t_tid.ip_posid, &itup2->t_tid.ip_posid, + IndexTupleSize(itup1) - + offsetof(ItemPointerData, ip_posid)) != 0) + return false; + } + else + { + /* + * Cannot rely on offset number field having consistent value across + * levels on pg_upgrade'd !heapkeyspace indexes. Compare contents of + * tuple starting from just after item pointer (i.e. after block + * number and offset number). + */ + if (memcmp(&itup1->t_info, &itup2->t_info, + IndexTupleSize(itup1) - + offsetof(IndexTupleData, t_info)) != 0) + return false; + } return true; } @@ -1913,7 +1935,7 @@ bt_child_highkey_check(BtreeCheckState *state, rightsplit = P_INCOMPLETE_SPLIT(opaque); /* - * If we visit page with high key, check that it is be equal to the + * If we visit page with high key, check that it is equal to the * target key next to corresponding downlink. */ if (!rightsplit && !P_RIGHTMOST(opaque)) @@ -2007,7 +2029,7 @@ bt_child_highkey_check(BtreeCheckState *state, itup = state->lowkey; } - if (!bt_pivot_tuple_identical(highkey, itup)) + if (!bt_pivot_tuple_identical(state->heapkeyspace, highkey, itup)) { ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), From 2000b6c10aa6777929f1a8b613f30426bb90f849 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 16 Sep 2020 14:28:11 -0400 Subject: [PATCH 157/589] Don't fetch partition check expression during InitResultRelInfo. Since there is only one place that actually needs the partition check expression, namely ExecPartitionCheck, it's better to fetch it from the relcache there. In this way we will never fetch it at all if the query never has use for it, and we still fetch it just once when we do need it. The reason for taking an interest in this is that if the relcache doesn't already have the check expression cached, fetching it requires obtaining AccessShareLock on the partition root. That means that operations that look like they should only touch the partition itself will also take a lock on the root. In particular we observed that TRUNCATE on a partition may take a lock on the partition's root, contributing to a deadlock situation in parallel pg_restore. As written, this patch does have a small cost, which is that we are microscopically reducing efficiency for the case where a partition has an empty check expression. ExecPartitionCheck will be called, and will go through the motions of setting up and checking an empty qual, where before it would not have been called at all. We could avoid that by adding a separate boolean flag to track whether there is a partition expression to test. However, this case only arises for a default partition with no siblings, which surely is not an interesting case in practice. Hence adding complexity for it does not seem like a good trade-off. Amit Langote, per a suggestion by me Discussion: https://postgr.es/m/VI1PR03MB31670CA1BD9625C3A8C5DD05EB230@VI1PR03MB3167.eurprd03.prod.outlook.com --- src/backend/commands/copy.c | 2 +- src/backend/executor/execMain.c | 41 +++++++++--------------- src/backend/executor/execPartition.c | 2 +- src/backend/executor/execReplication.c | 4 +-- src/backend/executor/nodeModifyTable.c | 4 +-- src/backend/replication/logical/worker.c | 2 +- src/include/nodes/execnodes.h | 11 +++---- 7 files changed, 26 insertions(+), 40 deletions(-) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index db7d24a511e3..2047557e5200 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -3236,7 +3236,7 @@ CopyFrom(CopyState cstate) * we don't need to if there's no BR trigger defined on the * partition. */ - if (resultRelInfo->ri_PartitionCheck && + if (resultRelInfo->ri_RelationDesc->rd_rel->relispartition && (proute == NULL || has_before_insert_row_trig)) ExecPartitionCheck(resultRelInfo, myslot, estate, true); diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index 4fdffad6f35d..2e27e26ba446 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -1280,8 +1280,6 @@ InitResultRelInfo(ResultRelInfo *resultRelInfo, Relation partition_root, int instrument_options) { - List *partition_check = NIL; - MemSet(resultRelInfo, 0, sizeof(ResultRelInfo)); resultRelInfo->type = T_ResultRelInfo; resultRelInfo->ri_RangeTableIndex = resultRelationIndex; @@ -1325,23 +1323,6 @@ InitResultRelInfo(ResultRelInfo *resultRelInfo, resultRelInfo->ri_ReturningSlot = NULL; resultRelInfo->ri_TrigOldSlot = NULL; resultRelInfo->ri_TrigNewSlot = NULL; - - /* - * Partition constraint, which also includes the partition constraint of - * all the ancestors that are partitions. Note that it will be checked - * even in the case of tuple-routing where this table is the target leaf - * partition, if there any BR triggers defined on the table. Although - * tuple-routing implicitly preserves the partition constraint of the - * target partition for a given row, the BR triggers may change the row - * such that the constraint is no longer satisfied, which we must fail for - * by checking it explicitly. - * - * If this is a partitioned table, the partition constraint (if any) of a - * given row will be checked just before performing tuple-routing. - */ - partition_check = RelationGetPartitionQual(resultRelationDesc); - - resultRelInfo->ri_PartitionCheck = partition_check; resultRelInfo->ri_PartitionRoot = partition_root; resultRelInfo->ri_PartitionInfo = NULL; /* may be set later */ resultRelInfo->ri_CopyMultiInsertBuffer = NULL; @@ -1776,7 +1757,7 @@ ExecRelCheck(ResultRelInfo *resultRelInfo, * ExecPartitionCheck --- check that tuple meets the partition constraint. * * Returns true if it meets the partition constraint. If the constraint - * fails and we're asked to emit to error, do so and don't return; otherwise + * fails and we're asked to emit an error, do so and don't return; otherwise * return false. */ bool @@ -1788,14 +1769,22 @@ ExecPartitionCheck(ResultRelInfo *resultRelInfo, TupleTableSlot *slot, /* * If first time through, build expression state tree for the partition - * check expression. Keep it in the per-query memory context so they'll - * survive throughout the query. + * check expression. (In the corner case where the partition check + * expression is empty, ie there's a default partition and nothing else, + * we'll be fooled into executing this code each time through. But it's + * pretty darn cheap in that case, so we don't worry about it.) */ if (resultRelInfo->ri_PartitionCheckExpr == NULL) { - List *qual = resultRelInfo->ri_PartitionCheck; + /* + * Ensure that the qual tree and prepared expression are in the + * query-lifespan context. + */ + MemoryContext oldcxt = MemoryContextSwitchTo(estate->es_query_cxt); + List *qual = RelationGetPartitionQual(resultRelInfo->ri_RelationDesc); resultRelInfo->ri_PartitionCheckExpr = ExecPrepareCheck(qual, estate); + MemoryContextSwitchTo(oldcxt); } /* @@ -1904,9 +1893,9 @@ ExecConstraints(ResultRelInfo *resultRelInfo, Bitmapset *insertedCols; Bitmapset *updatedCols; - Assert(constr || resultRelInfo->ri_PartitionCheck); + Assert(constr); /* we should not be called otherwise */ - if (constr && constr->has_not_null) + if (constr->has_not_null) { int natts = tupdesc->natts; int attrChk; @@ -1967,7 +1956,7 @@ ExecConstraints(ResultRelInfo *resultRelInfo, } } - if (constr && constr->num_check > 0) + if (constr->num_check > 0) { const char *failed; diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index bd2ea2580475..33d2c6f63de4 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -299,7 +299,7 @@ ExecFindPartition(ModifyTableState *mtstate, * First check the root table's partition constraint, if any. No point in * routing the tuple if it doesn't belong in the root table itself. */ - if (rootResultRelInfo->ri_PartitionCheck) + if (rootResultRelInfo->ri_RelationDesc->rd_rel->relispartition) ExecPartitionCheck(rootResultRelInfo, slot, estate, true); /* start with the root partitioned table */ diff --git a/src/backend/executor/execReplication.c b/src/backend/executor/execReplication.c index 8f474faed066..b29db7bf4f95 100644 --- a/src/backend/executor/execReplication.c +++ b/src/backend/executor/execReplication.c @@ -435,7 +435,7 @@ ExecSimpleRelationInsert(EState *estate, TupleTableSlot *slot) /* Check the constraints of the tuple */ if (rel->rd_att->constr) ExecConstraints(resultRelInfo, slot, estate); - if (resultRelInfo->ri_PartitionCheck) + if (rel->rd_rel->relispartition) ExecPartitionCheck(resultRelInfo, slot, estate, true); /* OK, store the tuple and create index entries for it */ @@ -501,7 +501,7 @@ ExecSimpleRelationUpdate(EState *estate, EPQState *epqstate, /* Check the constraints of the tuple */ if (rel->rd_att->constr) ExecConstraints(resultRelInfo, slot, estate); - if (resultRelInfo->ri_PartitionCheck) + if (rel->rd_rel->relispartition) ExecPartitionCheck(resultRelInfo, slot, estate, true); simple_table_tuple_update(rel, tid, slot, estate->es_snapshot, diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index 20a4c474cc47..98120891619b 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -491,7 +491,7 @@ ExecInsert(ModifyTableState *mtstate, * one; except that if we got here via tuple-routing, we don't need to * if there's no BR trigger defined on the partition. */ - if (resultRelInfo->ri_PartitionCheck && + if (resultRelationDesc->rd_rel->relispartition && (resultRelInfo->ri_PartitionRoot == NULL || (resultRelInfo->ri_TrigDesc && resultRelInfo->ri_TrigDesc->trig_insert_before_row))) @@ -1181,7 +1181,7 @@ lreplace:; * row. So skip the WCO checks if the partition constraint fails. */ partition_constraint_failed = - resultRelInfo->ri_PartitionCheck && + resultRelationDesc->rd_rel->relispartition && !ExecPartitionCheck(resultRelInfo, slot, estate, false); if (!partition_constraint_failed && diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index c37aafed0d29..d239d28c094b 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -1676,7 +1676,7 @@ apply_handle_tuple_routing(ResultRelInfo *relinfo, * Does the updated tuple still satisfy the current * partition's constraint? */ - if (partrelinfo->ri_PartitionCheck == NULL || + if (!partrel->rd_rel->relispartition || ExecPartitionCheck(partrelinfo, remoteslot_part, estate, false)) { diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 0b42dd6f9441..a5ab1aed14d5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -477,19 +477,16 @@ typedef struct ResultRelInfo /* ON CONFLICT evaluation state */ OnConflictSetState *ri_onConflict; - /* partition check expression */ - List *ri_PartitionCheck; - - /* partition check expression state */ + /* partition check expression state (NULL if not set up yet) */ ExprState *ri_PartitionCheckExpr; - /* relation descriptor for root partitioned table */ + /* relation descriptor for partitioned table's root, if any */ Relation ri_PartitionRoot; - /* Additional information specific to partition tuple routing */ + /* info for partition tuple routing (NULL if not set up yet) */ struct PartitionRoutingInfo *ri_PartitionInfo; - /* For use by copy.c when performing multi-inserts */ + /* for use by copy.c when performing multi-inserts */ struct CopyMultiInsertBuffer *ri_CopyMultiInsertBuffer; } ResultRelInfo; From 44fc6e259b799f9924de206eeddc1b1fcbcd172f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 16 Sep 2020 16:04:36 -0400 Subject: [PATCH 158/589] Centralize setup of SIGQUIT handling for postmaster child processes. We decided that the policy established in commit 7634bd4f6 for the bgwriter, checkpointer, walwriter, and walreceiver processes, namely that they should accept SIGQUIT at all times, really ought to apply uniformly to all postmaster children. Therefore, get rid of the duplicative and inconsistent per-process code for establishing that signal handler and removing SIGQUIT from BlockSig. Instead, make InitPostmasterChild do it. The handler set up by InitPostmasterChild is SignalHandlerForCrashExit, which just summarily does _exit(2). In interactive backends, we almost immediately replace that with quickdie, since we would prefer to try to tell the client that we're dying. However, this patch is changing the behavior of autovacuum (both launcher and workers), as well as walsenders. Those processes formerly also used quickdie, but AFAICS that was just mindless copy-and-paste: they don't have any interactive client that's likely to benefit from being told this. The stats collector continues to be an outlier, in that it thinks SIGQUIT means normal exit. That should probably be changed for consistency, but there's another patch set where that's being dealt with, so I didn't do so here. Discussion: https://postgr.es/m/644875.1599933441@sss.pgh.pa.us --- src/backend/postmaster/autovacuum.c | 18 ++++++++++-------- src/backend/postmaster/bgworker.c | 2 +- src/backend/postmaster/bgwriter.c | 6 +----- src/backend/postmaster/checkpointer.c | 6 +----- src/backend/postmaster/pgarch.c | 2 +- src/backend/postmaster/postmaster.c | 8 ++------ src/backend/postmaster/startup.c | 2 +- src/backend/postmaster/walwriter.c | 6 +----- src/backend/replication/walreceiver.c | 6 +----- src/backend/replication/walsender.c | 2 +- src/backend/tcop/postgres.c | 16 +++++----------- src/backend/utils/init/miscinit.c | 26 ++++++++++++++++++++++++++ 12 files changed, 51 insertions(+), 49 deletions(-) diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index 19ba26b914e9..2cef56f115f4 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -454,8 +454,8 @@ AutoVacLauncherMain(int argc, char *argv[]) pqsignal(SIGHUP, SignalHandlerForConfigReload); pqsignal(SIGINT, StatementCancelHandler); pqsignal(SIGTERM, SignalHandlerForShutdownRequest); + /* SIGQUIT handler was already set up by InitPostmasterChild */ - pqsignal(SIGQUIT, quickdie); InitializeTimeouts(); /* establishes SIGALRM handler */ pqsignal(SIGPIPE, SIG_IGN); @@ -498,9 +498,10 @@ AutoVacLauncherMain(int argc, char *argv[]) * * Note that we use sigsetjmp(..., 1), so that the prevailing signal mask * (to wit, BlockSig) will be restored when longjmp'ing to here. Thus, - * signals will be blocked until we complete error recovery. It might - * seem that this policy makes the HOLD_INTERRUPTS() call redundant, but - * it is not since InterruptPending might be set already. + * signals other than SIGQUIT will be blocked until we complete error + * recovery. It might seem that this policy makes the HOLD_INTERRUPTS() + * call redundant, but it is not since InterruptPending might be set + * already. */ if (sigsetjmp(local_sigjmp_buf, 1) != 0) { @@ -1531,7 +1532,8 @@ AutoVacWorkerMain(int argc, char *argv[]) */ pqsignal(SIGINT, StatementCancelHandler); pqsignal(SIGTERM, die); - pqsignal(SIGQUIT, quickdie); + /* SIGQUIT handler was already set up by InitPostmasterChild */ + InitializeTimeouts(); /* establishes SIGALRM handler */ pqsignal(SIGPIPE, SIG_IGN); @@ -1562,9 +1564,9 @@ AutoVacWorkerMain(int argc, char *argv[]) * * Note that we use sigsetjmp(..., 1), so that the prevailing signal mask * (to wit, BlockSig) will be restored when longjmp'ing to here. Thus, - * signals will be blocked until we exit. It might seem that this policy - * makes the HOLD_INTERRUPTS() call redundant, but it is not since - * InterruptPending might be set already. + * signals other than SIGQUIT will be blocked until we exit. It might + * seem that this policy makes the HOLD_INTERRUPTS() call redundant, but + * it is not since InterruptPending might be set already. */ if (sigsetjmp(local_sigjmp_buf, 1) != 0) { diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index d043ced6861a..5a9a0e343537 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -731,9 +731,9 @@ StartBackgroundWorker(void) pqsignal(SIGFPE, SIG_IGN); } pqsignal(SIGTERM, bgworker_die); + /* SIGQUIT handler was already set up by InitPostmasterChild */ pqsignal(SIGHUP, SIG_IGN); - pqsignal(SIGQUIT, SignalHandlerForCrashExit); InitializeTimeouts(); /* establishes SIGALRM handler */ pqsignal(SIGPIPE, SIG_IGN); diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c index c96568149fe0..a7afa758b618 100644 --- a/src/backend/postmaster/bgwriter.c +++ b/src/backend/postmaster/bgwriter.c @@ -104,7 +104,7 @@ BackgroundWriterMain(void) pqsignal(SIGHUP, SignalHandlerForConfigReload); pqsignal(SIGINT, SIG_IGN); pqsignal(SIGTERM, SignalHandlerForShutdownRequest); - pqsignal(SIGQUIT, SignalHandlerForCrashExit); + /* SIGQUIT handler was already set up by InitPostmasterChild */ pqsignal(SIGALRM, SIG_IGN); pqsignal(SIGPIPE, SIG_IGN); pqsignal(SIGUSR1, procsignal_sigusr1_handler); @@ -115,10 +115,6 @@ BackgroundWriterMain(void) */ pqsignal(SIGCHLD, SIG_DFL); - /* We allow SIGQUIT (SignalHandlerForCrashExit) at all times */ - sigdelset(&BlockSig, SIGQUIT); - PG_SETMASK(&BlockSig); - /* * We just started, assume there has been either a shutdown or * end-of-recovery snapshot. diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c index 45f5deca72ee..3e7dcd4f764d 100644 --- a/src/backend/postmaster/checkpointer.c +++ b/src/backend/postmaster/checkpointer.c @@ -198,7 +198,7 @@ CheckpointerMain(void) pqsignal(SIGHUP, SignalHandlerForConfigReload); pqsignal(SIGINT, ReqCheckpointHandler); /* request checkpoint */ pqsignal(SIGTERM, SIG_IGN); /* ignore SIGTERM */ - pqsignal(SIGQUIT, SignalHandlerForCrashExit); + /* SIGQUIT handler was already set up by InitPostmasterChild */ pqsignal(SIGALRM, SIG_IGN); pqsignal(SIGPIPE, SIG_IGN); pqsignal(SIGUSR1, procsignal_sigusr1_handler); @@ -209,10 +209,6 @@ CheckpointerMain(void) */ pqsignal(SIGCHLD, SIG_DFL); - /* We allow SIGQUIT (SignalHandlerForCrashExit) at all times */ - sigdelset(&BlockSig, SIGQUIT); - PG_SETMASK(&BlockSig); - /* * Initialize so that first time-driven event happens at the correct time. */ diff --git a/src/backend/postmaster/pgarch.c b/src/backend/postmaster/pgarch.c index 37be0e2bbbe7..ed1b65358df8 100644 --- a/src/backend/postmaster/pgarch.c +++ b/src/backend/postmaster/pgarch.c @@ -228,7 +228,7 @@ PgArchiverMain(int argc, char *argv[]) pqsignal(SIGHUP, SignalHandlerForConfigReload); pqsignal(SIGINT, SIG_IGN); pqsignal(SIGTERM, SignalHandlerForShutdownRequest); - pqsignal(SIGQUIT, SignalHandlerForCrashExit); + /* SIGQUIT handler was already set up by InitPostmasterChild */ pqsignal(SIGALRM, SIG_IGN); pqsignal(SIGPIPE, SIG_IGN); pqsignal(SIGUSR1, pgarch_waken); diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 3cd6fa30eb0a..959e3b887381 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -4355,7 +4355,7 @@ BackendInitialize(Port *port) * cleaned up. */ pqsignal(SIGTERM, process_startup_packet_die); - pqsignal(SIGQUIT, SignalHandlerForCrashExit); + /* SIGQUIT handler was already set up by InitPostmasterChild */ InitializeTimeouts(); /* establishes SIGALRM handler */ PG_SETMASK(&StartupBlockSig); @@ -4435,7 +4435,7 @@ BackendInitialize(Port *port) status = ProcessStartupPacket(port, false, false); /* - * Disable the timeout, and prevent SIGTERM/SIGQUIT again. + * Disable the timeout, and prevent SIGTERM again. */ disable_timeout(STARTUP_PACKET_TIMEOUT, false); PG_SETMASK(&BlockSig); @@ -4983,10 +4983,6 @@ SubPostmasterMain(int argc, char *argv[]) if (strcmp(argv[1], "--forkavworker") == 0) AutovacuumWorkerIAm(); - /* In EXEC_BACKEND case we will not have inherited these settings */ - pqinitmask(); - PG_SETMASK(&BlockSig); - /* Read in remaining GUC variables */ read_nondefault_variables(); diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c index fd9ac35dac1f..64af7b8707cc 100644 --- a/src/backend/postmaster/startup.c +++ b/src/backend/postmaster/startup.c @@ -175,7 +175,7 @@ StartupProcessMain(void) pqsignal(SIGHUP, StartupProcSigHupHandler); /* reload config file */ pqsignal(SIGINT, SIG_IGN); /* ignore query cancel */ pqsignal(SIGTERM, StartupProcShutdownHandler); /* request shutdown */ - pqsignal(SIGQUIT, SignalHandlerForCrashExit); + /* SIGQUIT handler was already set up by InitPostmasterChild */ InitializeTimeouts(); /* establishes SIGALRM handler */ pqsignal(SIGPIPE, SIG_IGN); pqsignal(SIGUSR1, procsignal_sigusr1_handler); diff --git a/src/backend/postmaster/walwriter.c b/src/backend/postmaster/walwriter.c index 358c0916ac23..a52832fe900a 100644 --- a/src/backend/postmaster/walwriter.c +++ b/src/backend/postmaster/walwriter.c @@ -101,7 +101,7 @@ WalWriterMain(void) pqsignal(SIGHUP, SignalHandlerForConfigReload); pqsignal(SIGINT, SignalHandlerForShutdownRequest); pqsignal(SIGTERM, SignalHandlerForShutdownRequest); - pqsignal(SIGQUIT, SignalHandlerForCrashExit); + /* SIGQUIT handler was already set up by InitPostmasterChild */ pqsignal(SIGALRM, SIG_IGN); pqsignal(SIGPIPE, SIG_IGN); pqsignal(SIGUSR1, procsignal_sigusr1_handler); @@ -112,10 +112,6 @@ WalWriterMain(void) */ pqsignal(SIGCHLD, SIG_DFL); - /* We allow SIGQUIT (SignalHandlerForCrashExit) at all times */ - sigdelset(&BlockSig, SIGQUIT); - PG_SETMASK(&BlockSig); - /* * Create a memory context that we will do all our work in. We do this so * that we can reset the context during error recovery and thereby avoid diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index b180598507f1..17f1a49f8711 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -270,7 +270,7 @@ WalReceiverMain(void) pqsignal(SIGHUP, WalRcvSigHupHandler); /* set flag to read config file */ pqsignal(SIGINT, SIG_IGN); pqsignal(SIGTERM, WalRcvShutdownHandler); /* request shutdown */ - pqsignal(SIGQUIT, SignalHandlerForCrashExit); + /* SIGQUIT handler was already set up by InitPostmasterChild */ pqsignal(SIGALRM, SIG_IGN); pqsignal(SIGPIPE, SIG_IGN); pqsignal(SIGUSR1, procsignal_sigusr1_handler); @@ -279,10 +279,6 @@ WalReceiverMain(void) /* Reset some signals that are accepted by postmaster but not here */ pqsignal(SIGCHLD, SIG_DFL); - /* We allow SIGQUIT (SignalHandlerForCrashExit) at all times */ - sigdelset(&BlockSig, SIGQUIT); - PG_SETMASK(&BlockSig); - /* Load the libpq-specific functions */ load_file("libpqwalreceiver", false); if (WalReceiverFunctions == NULL) diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 67093383e66b..4dbffea240a3 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -3041,7 +3041,7 @@ WalSndSignals(void) pqsignal(SIGHUP, SignalHandlerForConfigReload); pqsignal(SIGINT, StatementCancelHandler); /* query cancel */ pqsignal(SIGTERM, die); /* request shutdown */ - pqsignal(SIGQUIT, quickdie); /* hard crash time */ + /* SIGQUIT handler was already set up by InitPostmasterChild */ InitializeTimeouts(); /* establishes SIGALRM handler */ pqsignal(SIGPIPE, SIG_IGN); pqsignal(SIGUSR1, procsignal_sigusr1_handler); diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index c9424f167c8d..411cfadbff35 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -3820,7 +3820,8 @@ PostgresMain(int argc, char *argv[], } /* - * Set up signal handlers and masks. + * Set up signal handlers. (InitPostmasterChild or InitStandaloneProcess + * has already set up BlockSig and made that the active signal mask.) * * Note that postmaster blocked all signals before forking child process, * so there is no race condition whereby we might receive a signal before @@ -3842,6 +3843,9 @@ PostgresMain(int argc, char *argv[], pqsignal(SIGTERM, die); /* cancel current query and exit */ /* + * In a postmaster child backend, replace SignalHandlerForCrashExit + * with quickdie, so we can tell the client we're dying. + * * In a standalone backend, SIGQUIT can be generated from the keyboard * easily, while SIGTERM cannot, so we make both signals do die() * rather than quickdie(). @@ -3871,16 +3875,6 @@ PostgresMain(int argc, char *argv[], * platforms */ } - pqinitmask(); - - if (IsUnderPostmaster) - { - /* We allow SIGQUIT (quickdie) at all times */ - sigdelset(&BlockSig, SIGQUIT); - } - - PG_SETMASK(&BlockSig); /* block everything except SIGQUIT */ - if (!IsUnderPostmaster) { /* diff --git a/src/backend/utils/init/miscinit.c b/src/backend/utils/init/miscinit.c index cf8f9579c345..ed2ab4b5b29a 100644 --- a/src/backend/utils/init/miscinit.c +++ b/src/backend/utils/init/miscinit.c @@ -32,10 +32,12 @@ #include "catalog/pg_authid.h" #include "common/file_perm.h" #include "libpq/libpq.h" +#include "libpq/pqsignal.h" #include "mb/pg_wchar.h" #include "miscadmin.h" #include "pgstat.h" #include "postmaster/autovacuum.h" +#include "postmaster/interrupt.h" #include "postmaster/postmaster.h" #include "storage/fd.h" #include "storage/ipc.h" @@ -133,6 +135,23 @@ InitPostmasterChild(void) elog(FATAL, "setsid() failed: %m"); #endif + /* In EXEC_BACKEND case we will not have inherited BlockSig etc values */ +#ifdef EXEC_BACKEND + pqinitmask(); +#endif + + /* + * Every postmaster child process is expected to respond promptly to + * SIGQUIT at all times. Therefore we centrally remove SIGQUIT from + * BlockSig and install a suitable signal handler. (Client-facing + * processes may choose to replace this default choice of handler with + * quickdie().) All other blockable signals remain blocked for now. + */ + pqsignal(SIGQUIT, SignalHandlerForCrashExit); + + sigdelset(&BlockSig, SIGQUIT); + PG_SETMASK(&BlockSig); + /* Request a signal if the postmaster dies, if possible. */ PostmasterDeathSignalInit(); } @@ -155,6 +174,13 @@ InitStandaloneProcess(const char *argv0) InitLatch(MyLatch); InitializeLatchWaitSet(); + /* + * For consistency with InitPostmasterChild, initialize signal mask here. + * But we don't unblock SIGQUIT or provide a default handler for it. + */ + pqinitmask(); + PG_SETMASK(&BlockSig); + /* Compute paths, no postmaster to inherit from */ if (my_exec_path[0] == '\0') { From 07082b08cc5d3c378d22c105c65841ec0952e3ed Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Wed, 16 Sep 2020 13:04:38 -0300 Subject: [PATCH 159/589] Fix bogus completion tag usage in walsender Since commit fd5942c18f97 (2012, 9.3-era), walsender has been sending completion tags for certain replication commands twice -- and they're not even consistent. Apparently neither libpq nor JDBC have a problem with it, but it's not kosher. Fix by remove the EndCommand() call in the common code path for them all, and inserting specific calls to EndReplicationCommand() specifically in those places where it's needed. EndReplicationCommand() is a new simple function to send the completion tag for replication commands. Do this instead of sending a generic SELECT completion tag for them all, which was also pretty bogus (if innocuous). While at it, change StartReplication() to use EndReplicationCommand() instead of pg_puttextmessage(). In commit 2f9661311b83, I failed to realize that replication commands are not close-enough kin of regular SQL commands, so the DROP_REPLICATION_SLOT tag I added is undeserved and a type pun. Take it out. Backpatch to 13, where the latter commit appeared. The duplicate tag has been sent since 9.3, but since nothing is broken, it doesn't seem worth fixing. Per complaints from Tom Lane. Discussion: https://postgr.es/m/1347966.1600195735@sss.pgh.pa.us --- src/backend/replication/walsender.c | 34 ++++++++++++++++++----------- src/backend/tcop/dest.c | 12 ++++++++++ src/include/tcop/cmdtaglist.h | 1 - src/include/tcop/dest.h | 1 + 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 4dbffea240a3..c1b5ad35deb5 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -799,7 +799,7 @@ StartReplication(StartReplicationCmd *cmd) } /* Send CommandComplete message */ - pq_puttextmessage('C', "START_STREAMING"); + EndReplicationCommand("START_STREAMING"); } /* @@ -1122,11 +1122,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) static void DropReplicationSlot(DropReplicationSlotCmd *cmd) { - QueryCompletion qc; - ReplicationSlotDrop(cmd->slotname, !cmd->wait); - SetQueryCompletion(&qc, CMDTAG_DROP_REPLICATION_SLOT, 0); - EndCommand(&qc, DestRemote, false); } /* @@ -1517,9 +1513,9 @@ exec_replication_command(const char *cmd_string) { int parse_rc; Node *cmd_node; + const char *cmdtag; MemoryContext cmd_context; MemoryContext old_context; - QueryCompletion qc; /* * If WAL sender has been told that shutdown is getting close, switch its @@ -1619,40 +1615,53 @@ exec_replication_command(const char *cmd_string) switch (cmd_node->type) { case T_IdentifySystemCmd: + cmdtag = "IDENTIFY_SYSTEM"; IdentifySystem(); + EndReplicationCommand(cmdtag); break; case T_BaseBackupCmd: - PreventInTransactionBlock(true, "BASE_BACKUP"); + cmdtag = "BASE_BACKUP"; + PreventInTransactionBlock(true, cmdtag); SendBaseBackup((BaseBackupCmd *) cmd_node); + EndReplicationCommand(cmdtag); break; case T_CreateReplicationSlotCmd: + cmdtag = "CREATE_REPLICATION_SLOT"; CreateReplicationSlot((CreateReplicationSlotCmd *) cmd_node); + EndReplicationCommand(cmdtag); break; case T_DropReplicationSlotCmd: + cmdtag = "DROP_REPLICATION_SLOT"; DropReplicationSlot((DropReplicationSlotCmd *) cmd_node); + EndReplicationCommand(cmdtag); break; case T_StartReplicationCmd: { StartReplicationCmd *cmd = (StartReplicationCmd *) cmd_node; - PreventInTransactionBlock(true, "START_REPLICATION"); + cmdtag = "START_REPLICATION"; + PreventInTransactionBlock(true, cmdtag); if (cmd->kind == REPLICATION_KIND_PHYSICAL) StartReplication(cmd); else StartLogicalReplication(cmd); + /* callees already sent their own completion message */ + Assert(xlogreader != NULL); break; } case T_TimeLineHistoryCmd: - PreventInTransactionBlock(true, "TIMELINE_HISTORY"); + cmdtag = "TIMELINE_HISTORY"; + PreventInTransactionBlock(true, cmdtag); SendTimeLineHistory((TimeLineHistoryCmd *) cmd_node); + EndReplicationCommand(cmdtag); break; case T_VariableShowStmt: @@ -1660,10 +1669,13 @@ exec_replication_command(const char *cmd_string) DestReceiver *dest = CreateDestReceiver(DestRemoteSimple); VariableShowStmt *n = (VariableShowStmt *) cmd_node; + cmdtag = "SHOW"; + /* syscache access needs a transaction environment */ StartTransactionCommand(); GetPGVariable(n->name, dest); CommitTransactionCommand(); + EndReplicationCommand(cmdtag); } break; @@ -1676,10 +1688,6 @@ exec_replication_command(const char *cmd_string) MemoryContextSwitchTo(old_context); MemoryContextDelete(cmd_context); - /* Send CommandComplete message */ - SetQueryCompletion(&qc, CMDTAG_SELECT, 0); - EndCommand(&qc, DestRemote, true); - /* Report to pgstat that this process is now idle */ pgstat_report_activity(STATE_IDLE, NULL); debug_query_string = NULL; diff --git a/src/backend/tcop/dest.c b/src/backend/tcop/dest.c index 7208751ec781..96789f88ef93 100644 --- a/src/backend/tcop/dest.c +++ b/src/backend/tcop/dest.c @@ -211,6 +211,18 @@ EndCommand(const QueryCompletion *qc, CommandDest dest, bool force_undecorated_o } } +/* ---------------- + * EndReplicationCommand - stripped down version of EndCommand + * + * For use by replication commands. + * ---------------- + */ +void +EndReplicationCommand(const char *commandTag) +{ + pq_putmessage('C', commandTag, strlen(commandTag) + 1); +} + /* ---------------- * NullCommand - tell dest that an empty query string was recognized * diff --git a/src/include/tcop/cmdtaglist.h b/src/include/tcop/cmdtaglist.h index 8ef0f55e7487..be94852bbd30 100644 --- a/src/include/tcop/cmdtaglist.h +++ b/src/include/tcop/cmdtaglist.h @@ -157,7 +157,6 @@ PG_CMDTAG(CMDTAG_DROP_OWNED, "DROP OWNED", true, false, false) PG_CMDTAG(CMDTAG_DROP_POLICY, "DROP POLICY", true, false, false) PG_CMDTAG(CMDTAG_DROP_PROCEDURE, "DROP PROCEDURE", true, false, false) PG_CMDTAG(CMDTAG_DROP_PUBLICATION, "DROP PUBLICATION", true, false, false) -PG_CMDTAG(CMDTAG_DROP_REPLICATION_SLOT, "DROP REPLICATION SLOT", false, false, false) PG_CMDTAG(CMDTAG_DROP_ROLE, "DROP ROLE", false, false, false) PG_CMDTAG(CMDTAG_DROP_ROUTINE, "DROP ROUTINE", true, false, false) PG_CMDTAG(CMDTAG_DROP_RULE, "DROP RULE", true, false, false) diff --git a/src/include/tcop/dest.h b/src/include/tcop/dest.h index 662ce8a56f80..2e07f1516d16 100644 --- a/src/include/tcop/dest.h +++ b/src/include/tcop/dest.h @@ -139,6 +139,7 @@ extern void BeginCommand(CommandTag commandTag, CommandDest dest); extern DestReceiver *CreateDestReceiver(CommandDest dest); extern void EndCommand(const QueryCompletion *qc, CommandDest dest, bool force_undecorated_output); +extern void EndReplicationCommand(const char *commandTag); /* Additional functions that go with destination management, more or less. */ From add105840b673ab3949abc2568da0d4f2fd496c5 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 16 Sep 2020 20:31:19 -0400 Subject: [PATCH 160/589] Improve formatting of create_help.pl and plperl_opmask.pl output. Adjust the whitespace in the emitted files so that it matches what pgindent would do. This makes the generated files look like they match project style, and avoids confusion if someone does run pgindent on the generated files. Also, add probes.h to pgindent's exclusion list, because it can confuse pgindent, plus there's not much point in processing it. Daniel Gustafsson, additional fixes by me Discussion: https://postgr.es/m/79ed5348-be7a-b647-dd40-742207186a22@2ndquadrant.com --- src/bin/psql/create_help.pl | 25 ++++++++++++------------ src/pl/plperl/plperl_opmask.pl | 2 +- src/tools/pgindent/README | 8 ++++++++ src/tools/pgindent/exclude_file_patterns | 1 + 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/bin/psql/create_help.pl b/src/bin/psql/create_help.pl index ee82e645832e..60e093bad490 100644 --- a/src/bin/psql/create_help.pl +++ b/src/bin/psql/create_help.pl @@ -63,11 +63,12 @@ struct _helpStruct { - const char *cmd; /* the command name */ - const char *help; /* the help associated with it */ - const char *docbook_id; /* DocBook XML id (for generating URL) */ - void (*syntaxfunc)(PQExpBuffer); /* function that prints the syntax associated with it */ - int nl_count; /* number of newlines in syntax (for pager) */ + const char *cmd; /* the command name */ + const char *help; /* the help associated with it */ + const char *docbook_id; /* DocBook XML id (for generating URL) */ + void (*syntaxfunc) (PQExpBuffer); /* function that prints the + * syntax associated with it */ + int nl_count; /* number of newlines in syntax (for pager) */ }; extern const struct _helpStruct QL_HELP[]; @@ -190,17 +191,17 @@ { my $id = $_; $id =~ s/ /_/g; - print $cfile_handle " { \"$_\", - N_(\"$entries{$_}{cmddesc}\"), - \"$entries{$_}{cmdid}\", - sql_help_$id, - $entries{$_}{nl_count} }, + print $cfile_handle "\t{\"$_\", +\t\tN_(\"$entries{$_}{cmddesc}\"), +\t\t\"$entries{$_}{cmdid}\", +\t\tsql_help_$id, +\t$entries{$_}{nl_count}}, "; } print $cfile_handle " - { NULL, NULL, NULL } /* End of list marker */ +\t{NULL, NULL, NULL}\t\t\t/* End of list marker */ }; "; @@ -210,7 +211,7 @@ #define QL_MAX_CMD_LEN $maxlen /* largest strlen(cmd) */ -#endif /* $define */ +#endif /* $define */ "; close $cfile_handle; diff --git a/src/pl/plperl/plperl_opmask.pl b/src/pl/plperl/plperl_opmask.pl index 3b33112ff945..ee18e915289b 100644 --- a/src/pl/plperl/plperl_opmask.pl +++ b/src/pl/plperl/plperl_opmask.pl @@ -52,7 +52,7 @@ printf $fh qq{ opmask[OP_%-12s] = 0;\t/* %s */ \\\n}, uc($opname), opdesc($opname); } -printf $fh " /* end */ \n"; +printf $fh " /* end */\n"; close $fh or die "Error closing $plperl_opmask_tmp: $!"; diff --git a/src/tools/pgindent/README b/src/tools/pgindent/README index 8eb15fafb996..9b16b0ed504a 100644 --- a/src/tools/pgindent/README +++ b/src/tools/pgindent/README @@ -157,6 +157,14 @@ are excluded because those files are imported from an external project, not maintained locally, and are machine-generated anyway. Likewise for plperl/ppport.h. +src/include/jit/llvmjit.h is excluded because it contains C++ constructs +that confuse pgindent. + +src/backend/utils/probes.h and its alias src/include/utils/probes.h +are excluded because that file is machine-generated by code not under +our control, and some versions of dtrace build files that confuse +pgindent. + The perltidy run processes all *.pl and *.pm files, plus a few executable Perl scripts that are not named that way. See the "find" diff --git a/src/tools/pgindent/exclude_file_patterns b/src/tools/pgindent/exclude_file_patterns index c8efc9a91310..a8f1a92f4b3b 100644 --- a/src/tools/pgindent/exclude_file_patterns +++ b/src/tools/pgindent/exclude_file_patterns @@ -6,5 +6,6 @@ /snowball/libstemmer/ /pl/plperl/ppport\.h$ /jit/llvmjit\.h$ +/utils/probes\.h$ /tmp_check/ /tmp_install/ From babef40c9a999949abe0ae8e82240cac3f154237 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 16 Sep 2020 21:06:50 -0400 Subject: [PATCH 161/589] Teach walsender to update its process title for replication commands. Because the code path taken for SQL commands executed in a walsender will update the process title, we pretty much have to update the title for replication commands as well. Otherwise, the title shows "idle" for the rest of a logical walsender's lifetime once it's executed any SQL command. Playing with this, I confirm that a walsender now typically spends most of its life reporting walsender postgres [local] START_REPLICATION Considering this in isolation, it might be better to have it say walsender postgres [local] sending replication data However, consistency with the other cases seems to be a stronger argument. In passing, remove duplicative pgstat_report_activity call. Discussion: https://postgr.es/m/880181.1600026471@sss.pgh.pa.us --- src/backend/replication/walsender.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index c1b5ad35deb5..7c9d1b67dfbd 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1616,12 +1616,14 @@ exec_replication_command(const char *cmd_string) { case T_IdentifySystemCmd: cmdtag = "IDENTIFY_SYSTEM"; + set_ps_display(cmdtag); IdentifySystem(); EndReplicationCommand(cmdtag); break; case T_BaseBackupCmd: cmdtag = "BASE_BACKUP"; + set_ps_display(cmdtag); PreventInTransactionBlock(true, cmdtag); SendBaseBackup((BaseBackupCmd *) cmd_node); EndReplicationCommand(cmdtag); @@ -1629,12 +1631,14 @@ exec_replication_command(const char *cmd_string) case T_CreateReplicationSlotCmd: cmdtag = "CREATE_REPLICATION_SLOT"; + set_ps_display(cmdtag); CreateReplicationSlot((CreateReplicationSlotCmd *) cmd_node); EndReplicationCommand(cmdtag); break; case T_DropReplicationSlotCmd: cmdtag = "DROP_REPLICATION_SLOT"; + set_ps_display(cmdtag); DropReplicationSlot((DropReplicationSlotCmd *) cmd_node); EndReplicationCommand(cmdtag); break; @@ -1644,6 +1648,7 @@ exec_replication_command(const char *cmd_string) StartReplicationCmd *cmd = (StartReplicationCmd *) cmd_node; cmdtag = "START_REPLICATION"; + set_ps_display(cmdtag); PreventInTransactionBlock(true, cmdtag); if (cmd->kind == REPLICATION_KIND_PHYSICAL) @@ -1659,6 +1664,7 @@ exec_replication_command(const char *cmd_string) case T_TimeLineHistoryCmd: cmdtag = "TIMELINE_HISTORY"; + set_ps_display(cmdtag); PreventInTransactionBlock(true, cmdtag); SendTimeLineHistory((TimeLineHistoryCmd *) cmd_node); EndReplicationCommand(cmdtag); @@ -1670,6 +1676,7 @@ exec_replication_command(const char *cmd_string) VariableShowStmt *n = (VariableShowStmt *) cmd_node; cmdtag = "SHOW"; + set_ps_display(cmdtag); /* syscache access needs a transaction environment */ StartTransactionCommand(); @@ -1688,8 +1695,11 @@ exec_replication_command(const char *cmd_string) MemoryContextSwitchTo(old_context); MemoryContextDelete(cmd_context); - /* Report to pgstat that this process is now idle */ - pgstat_report_activity(STATE_IDLE, NULL); + /* + * We need not update ps display or pg_stat_activity, because PostgresMain + * will reset those to "idle". But we must reset debug_query_string to + * ensure it doesn't become a dangling pointer. + */ debug_query_string = NULL; return true; From 7307df16a05984396649db33df0faf6d840cd223 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 17 Sep 2020 11:49:29 +0900 Subject: [PATCH 162/589] Improve tab completion of IMPORT FOREIGN SCHEMA in psql It is not possible to get a list of foreign schemas as the server is not known, so this provides instead a list of local schemas, which is more useful than nothing if using a loopback server or having schema names matching in the local and remote servers. Author: Jeff Janes Reviewed-by: Tom Lane, Michael Paquier Discussion: https://postgr.es/m/CAMkU=1wr7Roj41q-XiJs=Uyc2xCmHhcGGy7J-peJQK-e+w=ghw@mail.gmail.com --- src/bin/psql/tab-complete.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index f41785f11c12..9c6f5ecb6a8c 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -3293,6 +3293,17 @@ psql_completion(const char *text, int start, int end) COMPLETE_WITH("FOREIGN SCHEMA"); else if (Matches("IMPORT", "FOREIGN")) COMPLETE_WITH("SCHEMA"); + else if (Matches("IMPORT", "FOREIGN", "SCHEMA", MatchAny)) + COMPLETE_WITH("EXCEPT (", "FROM SERVER", "LIMIT TO ("); + else if (TailMatches("LIMIT", "TO", "(*)") || + TailMatches("EXCEPT", "(*)")) + COMPLETE_WITH("FROM SERVER"); + else if (TailMatches("FROM", "SERVER", MatchAny)) + COMPLETE_WITH("INTO"); + else if (TailMatches("FROM", "SERVER", MatchAny, "INTO")) + COMPLETE_WITH_QUERY(Query_for_list_of_schemas); + else if (TailMatches("FROM", "SERVER", MatchAny, "INTO", MatchAny)) + COMPLETE_WITH("OPTIONS ("); /* INSERT --- can be inside EXPLAIN, RULE, etc */ /* Complete INSERT with "INTO" */ From 089da3c4778fdc1931f721a265caa0c6fca38584 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 17 Sep 2020 16:33:22 +0900 Subject: [PATCH 163/589] doc: Apply more consistently markup for OpenSSL OpenSSL was quoted in inconsistent ways in many places of the docs, sometimes with , or just nothing. Author: Daniel Gustafsson Discussion: https://postgr.es/m/DA91E5F0-5F9D-41A7-A7A6-B91CDE0F1D63@yesql.se --- doc/src/sgml/config.sgml | 16 ++++++----- doc/src/sgml/installation.sgml | 2 +- doc/src/sgml/libpq.sgml | 51 +++++++++++++++++++--------------- doc/src/sgml/pgcrypto.sgml | 14 ++++++---- doc/src/sgml/sslinfo.sgml | 4 +-- 5 files changed, 49 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index c4ba49ffaf58..2c75876e3224 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -1261,10 +1261,11 @@ include_dir 'conf.d' +3DES - The OpenSSL default order for HIGH is problematic - because it orders 3DES higher than AES128. This is wrong because - 3DES offers less security than AES128, and it is also much - slower. +3DES reorders it after all other + The OpenSSL default order for + HIGH is problematic because it orders 3DES + higher than AES128. This is wrong because 3DES offers less + security than AES128, and it is also much slower. + +3DES reorders it after all other HIGH and MEDIUM ciphers. @@ -1284,8 +1285,8 @@ include_dir 'conf.d' - Available cipher suite details will vary across OpenSSL versions. Use - the command + Available cipher suite details will vary across + OpenSSL versions. Use the command openssl ciphers -v 'HIGH:MEDIUM:+3DES:!aNULL' to see actual details for the currently installed OpenSSL version. Note that this list is filtered at run time based on the @@ -1337,7 +1338,8 @@ include_dir 'conf.d' - OpenSSL names for the most common curves are: + OpenSSL names for the most common curves + are: prime256v1 (NIST P-256), secp384r1 (NIST P-384), secp521r1 (NIST P-521). diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 552303e21142..b585f224085c 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -2293,7 +2293,7 @@ ERROR: could not load library "/opt/dbs/pgsql/lib/plperl.so": Bad address - OpenSSL is not supported. + OpenSSL is not supported. diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index a397073526f4..b50391caee05 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -812,7 +812,8 @@ int callback_fn(char *buf, int size, PGconn *conn); its path will be in conn->sslkey when the callback is invoked. This will be empty if the default key path is being used. For keys that are engine specifiers, it is up to engine implementations - whether they use the OpenSSL password callback or define their own handling. + whether they use the OpenSSL password + callback or define their own handling. @@ -1672,13 +1673,15 @@ postgresql://%2Fvar%2Flib%2Fpostgresql/dbname Specifying this parameter with any non-empty value suppresses the Enter PEM pass phrase: - prompt that OpenSSL will emit by default when an encrypted client - certificate key is provided to libpq. + prompt that OpenSSL will emit by default + when an encrypted client certificate key is provided to + libpq. - If the key is not encrypted this parameter is ignored. The parameter has no - effect on keys specified by OpenSSL engines unless the engine uses the - OpenSSL password callback mechanism for prompts. + If the key is not encrypted this parameter is ignored. The parameter + has no effect on keys specified by OpenSSL + engines unless the engine uses the OpenSSL + password callback mechanism for prompts. There is no environment variable equivalent to this option, and no @@ -2471,8 +2474,9 @@ void *PQsslStruct(const PGconn *conn, const char *struct_name); The struct(s) available depend on the SSL implementation in use. - For OpenSSL, there is one struct, available under the name "OpenSSL", - and it returns a pointer to the OpenSSL SSL struct. + For OpenSSL, there is one struct, + available under the name "OpenSSL", and it returns a pointer to the + OpenSSL SSL struct. To use this function, code along the following lines could be used: @@ -2516,8 +2520,9 @@ void *PQgetssl(const PGconn *conn); This function is equivalent to PQsslStruct(conn, "OpenSSL"). It should not be used in new applications, because the returned struct is - specific to OpenSSL and will not be available if another SSL - implementation is used. To check if a connection uses SSL, call + specific to OpenSSL and will not be + available if another SSL implementation is used. + To check if a connection uses SSL, call instead, and for more details about the connection, use . @@ -7665,15 +7670,17 @@ ldap://ldap.acme.com/cn=dbserver,cn=hosts?pgconnectinfo?base?(objectclass=*) The key may be - stored in cleartext or encrypted with a passphrase using any algorithm supported - by OpenSSL, like AES-128. If the key is stored encrypted, then the passphrase - may be provided in the connection - option. If an encrypted key is supplied and the sslpassword - option is absent or blank, a password will be prompted for interactively by - OpenSSL with a Enter PEM pass phrase: - prompt if a TTY is available. Applications can override the client certificate - prompt and the handling of the sslpassword parameter by supplying - their own key password callback; see + stored in cleartext or encrypted with a passphrase using any algorithm + supported by OpenSSL, like AES-128. If the key + is stored encrypted, then the passphrase may be provided in the + connection option. If an + encrypted key is supplied and the sslpassword option + is absent or blank, a password will be prompted for interactively by + OpenSSL with a + Enter PEM pass phrase: prompt if a TTY is available. + Applications can override the client certificate prompt and the handling + of the sslpassword parameter by supplying their own + key password callback; see . @@ -7936,7 +7943,7 @@ void PQinitOpenSSL(int do_ssl, int do_crypto); When do_ssl is non-zero, libpq - will initialize the OpenSSL library before first + will initialize the OpenSSL library before first opening a database connection. When do_crypto is non-zero, the libcrypto library will be initialized. By default (if is not called), both libraries @@ -7945,7 +7952,7 @@ void PQinitOpenSSL(int do_ssl, int do_crypto); - If your application uses and initializes either OpenSSL + If your application uses and initializes either OpenSSL or its underlying libcrypto library, you must call this function with zeroes for the appropriate parameter(s) before first opening a database connection. Also be sure that you @@ -7967,7 +7974,7 @@ void PQinitSSL(int do_ssl); This function is equivalent to PQinitOpenSSL(do_ssl, do_ssl). It is sufficient for applications that initialize both or neither - of OpenSSL and libcrypto. + of OpenSSL and libcrypto. diff --git a/doc/src/sgml/pgcrypto.sgml b/doc/src/sgml/pgcrypto.sgml index 6fd645aa70ad..8748c64e2da6 100644 --- a/doc/src/sgml/pgcrypto.sgml +++ b/doc/src/sgml/pgcrypto.sgml @@ -45,8 +45,8 @@ digest(data bytea, type text) returns bytea sha224, sha256, sha384 and sha512. If pgcrypto was built with - OpenSSL, more algorithms are available, as detailed in - . + OpenSSL, more algorithms are available, as + detailed in . @@ -1162,9 +1162,10 @@ gen_random_uuid() returns uuid - When compiled with OpenSSL, there will be more algorithms available. - Also public-key encryption functions will be faster as OpenSSL - has more optimized BIGNUM functions. + When compiled with OpenSSL, there will be + more algorithms available. Also public-key encryption functions will + be faster as OpenSSL has more optimized + BIGNUM functions. @@ -1239,7 +1240,8 @@ gen_random_uuid() returns uuid - Any digest algorithm OpenSSL supports is automatically picked up. + Any digest algorithm OpenSSL supports + is automatically picked up. This is not possible with ciphers, which need to be supported explicitly. diff --git a/doc/src/sgml/sslinfo.sgml b/doc/src/sgml/sslinfo.sgml index 7d3fcb716707..e16f61b41d72 100644 --- a/doc/src/sgml/sslinfo.sgml +++ b/doc/src/sgml/sslinfo.sgml @@ -173,8 +173,8 @@ This function returns the value of the specified field in the certificate subject, or NULL if the field is not present. - Field names are string constants that are - converted into ASN1 object identifiers using the OpenSSL object + Field names are string constants that are converted into ASN1 object + identifiers using the OpenSSL object database. The following values are acceptable: From 16fa9b2b30a357b4aea982bd878ec2e5e002dbcc Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 17 Sep 2020 11:33:40 +0300 Subject: [PATCH 164/589] Add support for building GiST index by sorting. This adds a new optional support function to the GiST access method: sortsupport. If it is defined, the GiST index is built by sorting all data to the order defined by the sortsupport's comparator function, and packing the tuples in that order to GiST pages. This is similar to how B-tree index build works, and is much faster than inserting the tuples one by one. The resulting index is smaller too, because the pages are packed more tightly, upto 'fillfactor'. The normal build method works by splitting pages, which tends to lead to more wasted space. The quality of the resulting index depends on how good the opclass-defined sort order is. A good order preserves locality of the input data. As the first user of this facility, add 'sortsupport' function to the point_ops opclass. It sorts the points in Z-order (aka Morton Code), by interleaving the bits of the X and Y coordinates. Author: Andrey Borodin Reviewed-by: Pavel Borisov, Thomas Munro Discussion: https://www.postgresql.org/message-id/1A36620E-CAD8-4267-9067-FB31385E7C0D%40yandex-team.ru --- doc/src/sgml/gist.sgml | 70 +++ src/backend/access/gist/gistbuild.c | 510 +++++++++++++++++---- src/backend/access/gist/gistproc.c | 229 +++++++++ src/backend/access/gist/gistutil.c | 53 ++- src/backend/access/gist/gistvalidate.c | 6 +- src/backend/access/transam/xloginsert.c | 57 +++ src/backend/utils/sort/sortsupport.c | 34 ++ src/backend/utils/sort/tuplesort.c | 57 +++ src/include/access/gist.h | 3 +- src/include/access/gist_private.h | 3 + src/include/access/xloginsert.h | 2 + src/include/catalog/catversion.h | 1 + src/include/catalog/pg_amproc.dat | 2 + src/include/catalog/pg_proc.dat | 3 + src/include/utils/sortsupport.h | 1 + src/include/utils/tuplesort.h | 4 + src/test/regress/expected/create_index.out | 6 +- 17 files changed, 935 insertions(+), 106 deletions(-) diff --git a/doc/src/sgml/gist.sgml b/doc/src/sgml/gist.sgml index f9226e7a35cb..192338be8810 100644 --- a/doc/src/sgml/gist.sgml +++ b/doc/src/sgml/gist.sgml @@ -259,6 +259,8 @@ CREATE INDEX ON my_table USING GIST (my_inet_column inet_ops); compress method is omitted. The optional tenth method options is needed if the operator class provides the user-specified parameters. + The sortsupport method is also optional and is used to + speed up building a GiST index. @@ -1065,6 +1067,74 @@ my_compress(PG_FUNCTION_ARGS) + + + sortsupport + + + Returns a comparator function to sort data in a way that preserves + locality. It is used by CREATE INDEX and + REINDEX commands. The quality of the created index + depends on how well the sort order determined by the comparator function + preserves locality of the inputs. + + + The sortsupport method is optional. If it is not + provided, CREATE INDEX builds the index by inserting + each tuple to the tree using the penalty and + picksplit functions, which is much slower. + + + + The SQL declaration of the function must look like + this: + + +CREATE OR REPLACE FUNCTION my_sortsupport(internal) +RETURNS void +AS 'MODULE_PATHNAME' +LANGUAGE C STRICT; + + + The argument is a pointer to a SortSupport + struct. At a minimum, the function must fill in its comparator field. + The comparator takes three arguments: two Datums to compare, and + a pointer to the SortSupport struct. The + Datums are the two indexed values in the format that they are stored + in the index; that is, in the format returned by the + compress method. The full API is defined in + src/include/utils/sortsupport.h. + + + + The matching code in the C module could then follow this skeleton: + + +PG_FUNCTION_INFO_V1(my_sortsupport); + +static int +my_fastcmp(Datum x, Datum y, SortSupport ssup) +{ + /* establish order between x and y by computing some sorting value z */ + + int z1 = ComputeSpatialCode(x); + int z2 = ComputeSpatialCode(y); + + return z1 == z2 ? 0 : z1 > z2 ? 1 : -1; +} + +Datum +my_sortsupport(PG_FUNCTION_ARGS) +{ + SortSupport ssup = (SortSupport) PG_GETARG_POINTER(0); + + ssup->comparator = my_fastcmp; + PG_RETURN_VOID(); +} + + + + diff --git a/src/backend/access/gist/gistbuild.c b/src/backend/access/gist/gistbuild.c index 671b5e9186ff..230625cf1e2c 100644 --- a/src/backend/access/gist/gistbuild.c +++ b/src/backend/access/gist/gistbuild.c @@ -3,6 +3,24 @@ * gistbuild.c * build algorithm for GiST indexes implementation. * + * There are two different strategies: + * + * 1. Sort all input tuples, pack them into GiST leaf pages in the sorted + * order, and create downlinks and internal pages as we go. This builds + * the index from the bottom up, similar to how B-tree index build + * works. + * + * 2. Start with an empty index, and insert all tuples one by one. + * + * The sorted method is used if the operator classes for all columns have + * a 'sortsupport' defined. Otherwise, we resort to the second strategy. + * + * The second strategy can optionally use buffers at different levels of + * the tree to reduce I/O, see "Buffering build algorithm" in the README + * for a more detailed explanation. It initially calls insert over and + * over, but switches to the buffered algorithm after a certain number of + * tuples (unless buffering mode is disabled). + * * * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California @@ -28,6 +46,7 @@ #include "storage/smgr.h" #include "utils/memutils.h" #include "utils/rel.h" +#include "utils/tuplesort.h" /* Step of index tuples for check whether to switch to buffering build mode */ #define BUFFERING_MODE_SWITCH_CHECK_STEP 256 @@ -40,8 +59,14 @@ */ #define BUFFERING_MODE_TUPLE_SIZE_STATS_TARGET 4096 +/* + * Strategy used to build the index. It can change between the + * GIST_BUFFERING_* modes on the fly, but if the Sorted method is used, + * that needs to be decided up-front and cannot be changed afterwards. + */ typedef enum { + GIST_SORTED_BUILD, /* bottom-up build by sorting */ GIST_BUFFERING_DISABLED, /* in regular build mode and aren't going to * switch */ GIST_BUFFERING_AUTO, /* in regular build mode, but will switch to @@ -51,7 +76,7 @@ typedef enum * before switching to the buffering build * mode */ GIST_BUFFERING_ACTIVE /* in buffering build mode */ -} GistBufferingMode; +} GistBuildMode; /* Working state for gistbuild and its callback */ typedef struct @@ -60,23 +85,58 @@ typedef struct Relation heaprel; GISTSTATE *giststate; - int64 indtuples; /* number of tuples indexed */ - int64 indtuplesSize; /* total size of all indexed tuples */ - Size freespace; /* amount of free space to leave on pages */ + GistBuildMode buildMode; + + int64 indtuples; /* number of tuples indexed */ + /* * Extra data structures used during a buffering build. 'gfbb' contains * information related to managing the build buffers. 'parentMap' is a * lookup table of the parent of each internal page. */ + int64 indtuplesSize; /* total size of all indexed tuples */ GISTBuildBuffers *gfbb; HTAB *parentMap; - GistBufferingMode bufferingMode; + /* + * Extra data structures used during a sorting build. + */ + Tuplesortstate *sortstate; /* state data for tuplesort.c */ + + BlockNumber pages_allocated; + BlockNumber pages_written; + + int ready_num_pages; + BlockNumber ready_blknos[XLR_MAX_BLOCK_ID]; + Page ready_pages[XLR_MAX_BLOCK_ID]; } GISTBuildState; +/* + * In sorted build, we use a stack of these structs, one for each level, + * to hold an in-memory buffer of the righmost page at the level. When the + * page fills up, it is written out and a new page is allocated. + */ +typedef struct GistSortedBuildPageState +{ + Page page; + struct GistSortedBuildPageState *parent; /* Upper level, if any */ +} GistSortedBuildPageState; + /* prototypes for private functions */ + +static void gistSortedBuildCallback(Relation index, ItemPointer tid, + Datum *values, bool *isnull, + bool tupleIsAlive, void *state); +static void gist_indexsortbuild(GISTBuildState *state); +static void gist_indexsortbuild_pagestate_add(GISTBuildState *state, + GistSortedBuildPageState *pagestate, + IndexTuple itup); +static void gist_indexsortbuild_pagestate_flush(GISTBuildState *state, + GistSortedBuildPageState *pagestate); +static void gist_indexsortbuild_flush_ready_pages(GISTBuildState *state); + static void gistInitBuffering(GISTBuildState *buildstate); static int calculatePagesPerBuffer(GISTBuildState *buildstate, int levelStep); static void gistBuildCallback(Relation index, @@ -107,10 +167,9 @@ static void gistMemorizeParent(GISTBuildState *buildstate, BlockNumber child, static void gistMemorizeAllDownlinks(GISTBuildState *buildstate, Buffer parent); static BlockNumber gistGetParent(GISTBuildState *buildstate, BlockNumber child); + /* - * Main entry point to GiST index build. Initially calls insert over and over, - * but switches to more efficient buffering build algorithm after a certain - * number of tuples (unless buffering mode is disabled). + * Main entry point to GiST index build. */ IndexBuildResult * gistbuild(Relation heap, Relation index, IndexInfo *indexInfo) @@ -118,124 +177,407 @@ gistbuild(Relation heap, Relation index, IndexInfo *indexInfo) IndexBuildResult *result; double reltuples; GISTBuildState buildstate; - Buffer buffer; - Page page; MemoryContext oldcxt = CurrentMemoryContext; int fillfactor; + Oid SortSupportFnOids[INDEX_MAX_KEYS]; + bool hasallsortsupports; + int keyscount = IndexRelationGetNumberOfKeyAttributes(index); + GiSTOptions *options = NULL; + + /* + * We expect to be called exactly once for any index relation. If that's + * not the case, big trouble's what we have. + */ + if (RelationGetNumberOfBlocks(index) != 0) + elog(ERROR, "index \"%s\" already contains data", + RelationGetRelationName(index)); + + if (index->rd_options) + options = (GiSTOptions *) index->rd_options; buildstate.indexrel = index; buildstate.heaprel = heap; + buildstate.sortstate = NULL; + buildstate.giststate = initGISTstate(index); - if (index->rd_options) + /* + * Create a temporary memory context that is reset once for each tuple + * processed. (Note: we don't bother to make this a child of the + * giststate's scanCxt, so we have to delete it separately at the end.) + */ + buildstate.giststate->tempCxt = createTempGistContext(); + + /* + * Choose build strategy. If all keys support sorting, do that. Otherwise + * the default strategy is switch to buffering mode when the index grows + * too large to fit in cache. + */ + hasallsortsupports = true; + for (int i = 0; i < keyscount; i++) { - /* Get buffering mode from the options string */ - GiSTOptions *options = (GiSTOptions *) index->rd_options; + SortSupportFnOids[i] = index_getprocid(index, i + 1, + GIST_SORTSUPPORT_PROC); + if (!OidIsValid(SortSupportFnOids[i])) + { + hasallsortsupports = false; + break; + } + } + if (hasallsortsupports) + { + buildstate.buildMode = GIST_SORTED_BUILD; + } + else if (options) + { if (options->buffering_mode == GIST_OPTION_BUFFERING_ON) - buildstate.bufferingMode = GIST_BUFFERING_STATS; + buildstate.buildMode = GIST_BUFFERING_STATS; else if (options->buffering_mode == GIST_OPTION_BUFFERING_OFF) - buildstate.bufferingMode = GIST_BUFFERING_DISABLED; + buildstate.buildMode = GIST_BUFFERING_DISABLED; else - buildstate.bufferingMode = GIST_BUFFERING_AUTO; - - fillfactor = options->fillfactor; + buildstate.buildMode = GIST_BUFFERING_AUTO; } else { - /* - * By default, switch to buffering mode when the index grows too large - * to fit in cache. - */ - buildstate.bufferingMode = GIST_BUFFERING_AUTO; - fillfactor = GIST_DEFAULT_FILLFACTOR; + buildstate.buildMode = GIST_BUFFERING_AUTO; } - /* Calculate target amount of free space to leave on pages */ + + /* + * Calculate target amount of free space to leave on pages. + */ + fillfactor = options ? options->fillfactor : GIST_DEFAULT_FILLFACTOR; buildstate.freespace = BLCKSZ * (100 - fillfactor) / 100; /* - * We expect to be called exactly once for any index relation. If that's - * not the case, big trouble's what we have. + * Build the index using the chosen strategy. */ - if (RelationGetNumberOfBlocks(index) != 0) - elog(ERROR, "index \"%s\" already contains data", - RelationGetRelationName(index)); + buildstate.indtuples = 0; + buildstate.indtuplesSize = 0; - /* no locking is needed */ - buildstate.giststate = initGISTstate(index); + if (buildstate.buildMode == GIST_SORTED_BUILD) + { + /* + * Sort all data, build the index from bottom up. + */ + buildstate.sortstate = tuplesort_begin_index_gist(heap, + index, + maintenance_work_mem, + NULL, + false); + + /* Scan the table, adding all tuples to the tuplesort */ + reltuples = table_index_build_scan(heap, index, indexInfo, true, true, + gistSortedBuildCallback, + (void *) &buildstate, NULL); + + /* + * Perform the sort and build index pages. + */ + tuplesort_performsort(buildstate.sortstate); + + gist_indexsortbuild(&buildstate); + + tuplesort_end(buildstate.sortstate); + } + else + { + /* + * Initialize an empty index and insert all tuples, possibly using + * buffers on intermediate levels. + */ + Buffer buffer; + Page page; + + /* initialize the root page */ + buffer = gistNewBuffer(index); + Assert(BufferGetBlockNumber(buffer) == GIST_ROOT_BLKNO); + page = BufferGetPage(buffer); + + START_CRIT_SECTION(); + + GISTInitBuffer(buffer, F_LEAF); + + MarkBufferDirty(buffer); + PageSetLSN(page, GistBuildLSN); + + UnlockReleaseBuffer(buffer); + + END_CRIT_SECTION(); + + /* Scan the table, inserting all the tuples to the index. */ + reltuples = table_index_build_scan(heap, index, indexInfo, true, true, + gistBuildCallback, + (void *) &buildstate, NULL); + + /* + * If buffering was used, flush out all the tuples that are still in + * the buffers. + */ + if (buildstate.buildMode == GIST_BUFFERING_ACTIVE) + { + elog(DEBUG1, "all tuples processed, emptying buffers"); + gistEmptyAllBuffers(&buildstate); + gistFreeBuildBuffers(buildstate.gfbb); + } + + /* + * We didn't write WAL records as we built the index, so if + * WAL-logging is required, write all pages to the WAL now. + */ + if (RelationNeedsWAL(index)) + { + log_newpage_range(index, MAIN_FORKNUM, + 0, RelationGetNumberOfBlocks(index), + true); + } + } + + /* okay, all heap tuples are indexed */ + MemoryContextSwitchTo(oldcxt); + MemoryContextDelete(buildstate.giststate->tempCxt); + + freeGISTstate(buildstate.giststate); /* - * Create a temporary memory context that is reset once for each tuple - * processed. (Note: we don't bother to make this a child of the - * giststate's scanCxt, so we have to delete it separately at the end.) + * Return statistics */ - buildstate.giststate->tempCxt = createTempGistContext(); + result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult)); - /* initialize the root page */ - buffer = gistNewBuffer(index); - Assert(BufferGetBlockNumber(buffer) == GIST_ROOT_BLKNO); - page = BufferGetPage(buffer); + result->heap_tuples = reltuples; + result->index_tuples = (double) buildstate.indtuples; - START_CRIT_SECTION(); + return result; +} - GISTInitBuffer(buffer, F_LEAF); +/*------------------------------------------------------------------------- + * Routines for sorted build + *------------------------------------------------------------------------- + */ - MarkBufferDirty(buffer); - PageSetLSN(page, GistBuildLSN); +/* + * Per-tuple callback for table_index_build_scan. + */ +static void +gistSortedBuildCallback(Relation index, + ItemPointer tid, + Datum *values, + bool *isnull, + bool tupleIsAlive, + void *state) +{ + GISTBuildState *buildstate = (GISTBuildState *) state; + MemoryContext oldCtx; + Datum compressed_values[INDEX_MAX_KEYS]; - UnlockReleaseBuffer(buffer); + oldCtx = MemoryContextSwitchTo(buildstate->giststate->tempCxt); - END_CRIT_SECTION(); + /* Form an index tuple and point it at the heap tuple */ + gistCompressValues(buildstate->giststate, index, + values, isnull, + true, compressed_values); - /* build the index */ - buildstate.indtuples = 0; - buildstate.indtuplesSize = 0; + tuplesort_putindextuplevalues(buildstate->sortstate, + buildstate->indexrel, + tid, + compressed_values, isnull); + + MemoryContextSwitchTo(oldCtx); + MemoryContextReset(buildstate->giststate->tempCxt); + + /* Update tuple count. */ + buildstate->indtuples += 1; +} + +/* + * Build GiST index from bottom up from pre-sorted tuples. + */ +static void +gist_indexsortbuild(GISTBuildState *state) +{ + IndexTuple itup; + GistSortedBuildPageState *leafstate; + GistSortedBuildPageState *pagestate; + Page page; + + state->pages_allocated = 0; + state->pages_written = 0; + state->ready_num_pages = 0; /* - * Do the heap scan. + * Write an empty page as a placeholder for the root page. It will be + * replaced with the real root page at the end. */ - reltuples = table_index_build_scan(heap, index, indexInfo, true, true, - gistBuildCallback, - (void *) &buildstate, NULL); + page = palloc0(BLCKSZ); + smgrextend(state->indexrel->rd_smgr, MAIN_FORKNUM, GIST_ROOT_BLKNO, + page, true); + state->pages_allocated++; + state->pages_written++; + + /* Allocate a temporary buffer for the first leaf page. */ + leafstate = palloc(sizeof(GistSortedBuildPageState)); + leafstate->page = page; + leafstate->parent = NULL; + gistinitpage(page, F_LEAF); /* - * If buffering was used, flush out all the tuples that are still in the - * buffers. + * Fill index pages with tuples in the sorted order. */ - if (buildstate.bufferingMode == GIST_BUFFERING_ACTIVE) + while ((itup = tuplesort_getindextuple(state->sortstate, true)) != NULL) { - elog(DEBUG1, "all tuples processed, emptying buffers"); - gistEmptyAllBuffers(&buildstate); - gistFreeBuildBuffers(buildstate.gfbb); + gist_indexsortbuild_pagestate_add(state, leafstate, itup); + MemoryContextReset(state->giststate->tempCxt); } - /* okay, all heap tuples are indexed */ - MemoryContextSwitchTo(oldcxt); - MemoryContextDelete(buildstate.giststate->tempCxt); - - freeGISTstate(buildstate.giststate); - /* - * We didn't write WAL records as we built the index, so if WAL-logging is - * required, write all pages to the WAL now. + * Write out the partially full non-root pages. + * + * Keep in mind that flush can build a new root. */ - if (RelationNeedsWAL(index)) + pagestate = leafstate; + while (pagestate->parent != NULL) { - log_newpage_range(index, MAIN_FORKNUM, - 0, RelationGetNumberOfBlocks(index), - true); + GistSortedBuildPageState *parent; + + gist_indexsortbuild_pagestate_flush(state, pagestate); + parent = pagestate->parent; + pfree(pagestate->page); + pfree(pagestate); + pagestate = parent; } + gist_indexsortbuild_flush_ready_pages(state); + + /* Write out the root */ + PageSetLSN(pagestate->page, GistBuildLSN); + smgrwrite(state->indexrel->rd_smgr, MAIN_FORKNUM, GIST_ROOT_BLKNO, + pagestate->page, true); + if (RelationNeedsWAL(state->indexrel)) + log_newpage(&state->indexrel->rd_node, MAIN_FORKNUM, GIST_ROOT_BLKNO, + pagestate->page, true); + + pfree(pagestate->page); + pfree(pagestate); +} + +/* + * Add tuple to a page. If the pages is full, write it out and re-initialize + * a new page first. + */ +static void +gist_indexsortbuild_pagestate_add(GISTBuildState *state, + GistSortedBuildPageState *pagestate, + IndexTuple itup) +{ + Size sizeNeeded; + + /* Does the tuple fit? If not, flush */ + sizeNeeded = IndexTupleSize(itup) + sizeof(ItemIdData) + state->freespace; + if (PageGetFreeSpace(pagestate->page) < sizeNeeded) + gist_indexsortbuild_pagestate_flush(state, pagestate); + + gistfillbuffer(pagestate->page, &itup, 1, InvalidOffsetNumber); +} + +static void +gist_indexsortbuild_pagestate_flush(GISTBuildState *state, + GistSortedBuildPageState *pagestate) +{ + GistSortedBuildPageState *parent; + IndexTuple *itvec; + IndexTuple union_tuple; + int vect_len; + bool isleaf; + BlockNumber blkno; + MemoryContext oldCtx; + + /* check once per page */ + CHECK_FOR_INTERRUPTS(); + + if (state->ready_num_pages == XLR_MAX_BLOCK_ID) + gist_indexsortbuild_flush_ready_pages(state); + /* - * Return statistics + * The page is now complete. Assign a block number to it, and add it to + * the list of finished pages. (We don't write it out immediately, because + * we want to WAL-log the pages in batches.) */ - result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult)); + blkno = state->pages_allocated++; + state->ready_blknos[state->ready_num_pages] = blkno; + state->ready_pages[state->ready_num_pages] = pagestate->page; + state->ready_num_pages++; - result->heap_tuples = reltuples; - result->index_tuples = (double) buildstate.indtuples; + isleaf = GistPageIsLeaf(pagestate->page); - return result; + /* + * Form a downlink tuple to represent all the tuples on the page. + */ + oldCtx = MemoryContextSwitchTo(state->giststate->tempCxt); + itvec = gistextractpage(pagestate->page, &vect_len); + union_tuple = gistunion(state->indexrel, itvec, vect_len, + state->giststate); + ItemPointerSetBlockNumber(&(union_tuple->t_tid), blkno); + MemoryContextSwitchTo(oldCtx); + + /* + * Insert the downlink to the parent page. If this was the root, create a + * new page as the parent, which becomes the new root. + */ + parent = pagestate->parent; + if (parent == NULL) + { + parent = palloc(sizeof(GistSortedBuildPageState)); + parent->page = (Page) palloc(BLCKSZ); + parent->parent = NULL; + gistinitpage(parent->page, 0); + + pagestate->parent = parent; + } + gist_indexsortbuild_pagestate_add(state, parent, union_tuple); + + /* Re-initialize the page buffer for next page on this level. */ + pagestate->page = palloc(BLCKSZ); + gistinitpage(pagestate->page, isleaf ? F_LEAF : 0); +} + +static void +gist_indexsortbuild_flush_ready_pages(GISTBuildState *state) +{ + if (state->ready_num_pages == 0) + return; + + for (int i = 0; i < state->ready_num_pages; i++) + { + Page page = state->ready_pages[i]; + + /* Currently, the blocks must be buffered in order. */ + if (state->ready_blknos[i] != state->pages_written) + elog(ERROR, "unexpected block number to flush GiST sorting build"); + + PageSetLSN(page, GistBuildLSN); + + smgrextend(state->indexrel->rd_smgr, + MAIN_FORKNUM, + state->pages_written++, + page, + true); + } + + if (RelationNeedsWAL(state->indexrel)) + log_newpages(&state->indexrel->rd_node, MAIN_FORKNUM, state->ready_num_pages, + state->ready_blknos, state->ready_pages, true); + + for (int i = 0; i < state->ready_num_pages; i++) + pfree(state->ready_pages[i]); + + state->ready_num_pages = 0; } + +/*------------------------------------------------------------------------- + * Routines for non-sorted build + *------------------------------------------------------------------------- + */ + /* * Attempt to switch to buffering mode. * @@ -375,7 +717,7 @@ gistInitBuffering(GISTBuildState *buildstate) if (levelStep <= 0) { elog(DEBUG1, "failed to switch to buffered GiST build"); - buildstate->bufferingMode = GIST_BUFFERING_DISABLED; + buildstate->buildMode = GIST_BUFFERING_DISABLED; return; } @@ -392,7 +734,7 @@ gistInitBuffering(GISTBuildState *buildstate) gistInitParentMap(buildstate); - buildstate->bufferingMode = GIST_BUFFERING_ACTIVE; + buildstate->buildMode = GIST_BUFFERING_ACTIVE; elog(DEBUG1, "switched to buffered GiST build; level step = %d, pagesPerBuffer = %d", levelStep, pagesPerBuffer); @@ -453,10 +795,12 @@ gistBuildCallback(Relation index, oldCtx = MemoryContextSwitchTo(buildstate->giststate->tempCxt); /* form an index tuple and point it at the heap tuple */ - itup = gistFormTuple(buildstate->giststate, index, values, isnull, true); + itup = gistFormTuple(buildstate->giststate, index, + values, isnull, + true); itup->t_tid = *tid; - if (buildstate->bufferingMode == GIST_BUFFERING_ACTIVE) + if (buildstate->buildMode == GIST_BUFFERING_ACTIVE) { /* We have buffers, so use them. */ gistBufferingBuildInsert(buildstate, itup); @@ -478,7 +822,7 @@ gistBuildCallback(Relation index, MemoryContextSwitchTo(oldCtx); MemoryContextReset(buildstate->giststate->tempCxt); - if (buildstate->bufferingMode == GIST_BUFFERING_ACTIVE && + if (buildstate->buildMode == GIST_BUFFERING_ACTIVE && buildstate->indtuples % BUFFERING_MODE_TUPLE_SIZE_STATS_TARGET == 0) { /* Adjust the target buffer size now */ @@ -493,10 +837,10 @@ gistBuildCallback(Relation index, * To avoid excessive calls to smgrnblocks(), only check this every * BUFFERING_MODE_SWITCH_CHECK_STEP index tuples */ - if ((buildstate->bufferingMode == GIST_BUFFERING_AUTO && + if ((buildstate->buildMode == GIST_BUFFERING_AUTO && buildstate->indtuples % BUFFERING_MODE_SWITCH_CHECK_STEP == 0 && effective_cache_size < smgrnblocks(index->rd_smgr, MAIN_FORKNUM)) || - (buildstate->bufferingMode == GIST_BUFFERING_STATS && + (buildstate->buildMode == GIST_BUFFERING_STATS && buildstate->indtuples >= BUFFERING_MODE_TUPLE_SIZE_STATS_TARGET)) { /* diff --git a/src/backend/access/gist/gistproc.c b/src/backend/access/gist/gistproc.c index 9ace64c3c4a9..27d9c0f77c30 100644 --- a/src/backend/access/gist/gistproc.c +++ b/src/backend/access/gist/gistproc.c @@ -24,6 +24,7 @@ #include "utils/builtins.h" #include "utils/float.h" #include "utils/geo_decls.h" +#include "utils/sortsupport.h" static bool gist_box_leaf_consistent(BOX *key, BOX *query, @@ -31,6 +32,15 @@ static bool gist_box_leaf_consistent(BOX *key, BOX *query, static bool rtree_internal_consistent(BOX *key, BOX *query, StrategyNumber strategy); +static uint64 point_zorder_internal(float4 x, float4 y); +static uint64 part_bits32_by2(uint32 x); +static uint32 ieee_float32_to_uint32(float f); +static int gist_bbox_zorder_cmp(Datum a, Datum b, SortSupport ssup); +static Datum gist_bbox_zorder_abbrev_convert(Datum original, SortSupport ssup); +static int gist_bbox_zorder_cmp_abbrev(Datum z1, Datum z2, SortSupport ssup); +static bool gist_bbox_zorder_abbrev_abort(int memtupcount, SortSupport ssup); + + /* Minimum accepted ratio of split */ #define LIMIT_RATIO 0.3 @@ -1540,3 +1550,222 @@ gist_poly_distance(PG_FUNCTION_ARGS) PG_RETURN_FLOAT8(distance); } + +/* + * Z-order routines for fast index build + */ + +/* + * Compute Z-value of a point + * + * Z-order (also known as Morton Code) maps a two-dimensional point to a + * single integer, in a way that preserves locality. Points that are close in + * the two-dimensional space are mapped to integer that are not far from each + * other. We do that by interleaving the bits in the X and Y components. + * + * Morton Code is normally defined only for integers, but the X and Y values + * of a point are floating point. We expect floats to be in IEEE format. + */ +static uint64 +point_zorder_internal(float4 x, float4 y) +{ + uint32 ix = ieee_float32_to_uint32(x); + uint32 iy = ieee_float32_to_uint32(y); + + /* Interleave the bits */ + return part_bits32_by2(ix) | (part_bits32_by2(iy) << 1); +} + +/* Interleave 32 bits with zeroes */ +static uint64 +part_bits32_by2(uint32 x) +{ + uint64 n = x; + + n = (n | (n << 16)) & UINT64CONST(0x0000FFFF0000FFFF); + n = (n | (n << 8)) & UINT64CONST(0x00FF00FF00FF00FF); + n = (n | (n << 4)) & UINT64CONST(0x0F0F0F0F0F0F0F0F); + n = (n | (n << 2)) & UINT64CONST(0x3333333333333333); + n = (n | (n << 1)) & UINT64CONST(0x5555555555555555); + + return n; +} + +/* + * Convert a 32-bit IEEE float to uint32 in a way that preserves the ordering + */ +static uint32 +ieee_float32_to_uint32(float f) +{ + /*---- + * + * IEEE 754 floating point format + * ------------------------------ + * + * IEEE 754 floating point numbers have this format: + * + * exponent (8 bits) + * | + * s eeeeeeee mmmmmmmmmmmmmmmmmmmmmmm + * | | + * sign mantissa (23 bits) + * + * Infinity has all bits in the exponent set and the mantissa is all + * zeros. Negative infinity is the same but with the sign bit set. + * + * NaNs are represented with all bits in the exponent set, and the least + * significant bit in the mantissa also set. The rest of the mantissa bits + * can be used to distinguish different kinds of NaNs. + * + * The IEEE format has the nice property that when you take the bit + * representation and interpret it as an integer, the order is preserved, + * except for the sign. That holds for the +-Infinity values too. + * + * Mapping to uint32 + * ----------------- + * + * In order to have a smooth transition from negative to positive numbers, + * we map floats to unsigned integers like this: + * + * x < 0 to range 0-7FFFFFFF + * x = 0 to value 8000000 (both positive and negative zero) + * x > 0 to range 8000001-FFFFFFFF + * + * We don't care to distinguish different kind of NaNs, so they are all + * mapped to the same arbitrary value, FFFFFFFF. Because of the IEEE bit + * representation of NaNs, there aren't any non-NaN values that would be + * mapped to FFFFFFFF. In fact, there is a range of unused values on both + * ends of the uint32 space. + */ + if (isnan(f)) + return 0xFFFFFFFF; + else + { + union + { + float f; + uint32 i; + } u; + + u.f = f; + + /* Check the sign bit */ + if ((u.i & 0x80000000) != 0) + { + /* + * Map the negative value to range 0-7FFFFFFF. This flips the sign + * bit to 0 in the same instruction. + */ + Assert(f <= 0); /* can be -0 */ + u.i ^= 0xFFFFFFFF; + } + else + { + /* Map the positive value (or 0) to range 80000000-FFFFFFFF */ + u.i |= 0x80000000; + } + + return u.i; + } +} + +/* + * Compare the Z-order of points + */ +static int +gist_bbox_zorder_cmp(Datum a, Datum b, SortSupport ssup) +{ + Point *p1 = &(DatumGetBoxP(a)->low); + Point *p2 = &(DatumGetBoxP(b)->low); + uint64 z1; + uint64 z2; + + /* + * Do a quick check for equality first. It's not clear if this is worth it + * in general, but certainly is when used as tie-breaker with abbreviated + * keys, + */ + if (p1->x == p2->x && p1->y == p2->y) + return 0; + + z1 = point_zorder_internal(p1->x, p1->y); + z2 = point_zorder_internal(p2->x, p2->y); + if (z1 > z2) + return 1; + else if (z1 < z2) + return -1; + else + return 0; +} + +/* + * Abbreviated version of Z-order comparison + * + * The abbreviated format is a Z-order value computed from the two 32-bit + * floats. If SIZEOF_DATUM == 8, the 64-bit Z-order value fits fully in the + * abbreviated Datum, otherwise use its most significant bits. + */ +static Datum +gist_bbox_zorder_abbrev_convert(Datum original, SortSupport ssup) +{ + Point *p = &(DatumGetBoxP(original)->low); + uint64 z; + + z = point_zorder_internal(p->x, p->y); + +#if SIZEOF_DATUM == 8 + return (Datum) z; +#else + return (Datum) (z >> 32); +#endif +} + +static int +gist_bbox_zorder_cmp_abbrev(Datum z1, Datum z2, SortSupport ssup) +{ + /* + * Compare the pre-computed Z-orders as unsigned integers. Datum is a + * typedef for 'uintptr_t', so no casting is required. + */ + if (z1 > z2) + return 1; + else if (z1 < z2) + return -1; + else + return 0; +} + +/* + * We never consider aborting the abbreviation. + * + * On 64-bit systems, the abbreviation is not lossy so it is always + * worthwhile. (Perhaps it's not on 32-bit systems, but we don't bother + * with logic to decide.) + */ +static bool +gist_bbox_zorder_abbrev_abort(int memtupcount, SortSupport ssup) +{ + return false; +} + +/* + * Sort support routine for fast GiST index build by sorting. + */ +Datum +gist_point_sortsupport(PG_FUNCTION_ARGS) +{ + SortSupport ssup = (SortSupport) PG_GETARG_POINTER(0); + + if (ssup->abbreviate) + { + ssup->comparator = gist_bbox_zorder_cmp_abbrev; + ssup->abbrev_converter = gist_bbox_zorder_abbrev_convert; + ssup->abbrev_abort = gist_bbox_zorder_abbrev_abort; + ssup->abbrev_full_comparator = gist_bbox_zorder_cmp; + } + else + { + ssup->comparator = gist_bbox_zorder_cmp; + } + PG_RETURN_VOID(); +} diff --git a/src/backend/access/gist/gistutil.c b/src/backend/access/gist/gistutil.c index 0516059e3ddc..615b5ade2331 100644 --- a/src/backend/access/gist/gistutil.c +++ b/src/backend/access/gist/gistutil.c @@ -572,12 +572,31 @@ gistdentryinit(GISTSTATE *giststate, int nkey, GISTENTRY *e, IndexTuple gistFormTuple(GISTSTATE *giststate, Relation r, - Datum attdata[], bool isnull[], bool isleaf) + Datum *attdata, bool *isnull, bool isleaf) { Datum compatt[INDEX_MAX_KEYS]; - int i; IndexTuple res; + gistCompressValues(giststate, r, attdata, isnull, isleaf, compatt); + + res = index_form_tuple(isleaf ? giststate->leafTupdesc : + giststate->nonLeafTupdesc, + compatt, isnull); + + /* + * The offset number on tuples on internal pages is unused. For historical + * reasons, it is set to 0xffff. + */ + ItemPointerSetOffsetNumber(&(res->t_tid), 0xffff); + return res; +} + +void +gistCompressValues(GISTSTATE *giststate, Relation r, + Datum *attdata, bool *isnull, bool isleaf, Datum *compatt) +{ + int i; + /* * Call the compress method on each attribute. */ @@ -617,17 +636,6 @@ gistFormTuple(GISTSTATE *giststate, Relation r, compatt[i] = attdata[i]; } } - - res = index_form_tuple(isleaf ? giststate->leafTupdesc : - giststate->nonLeafTupdesc, - compatt, isnull); - - /* - * The offset number on tuples on internal pages is unused. For historical - * reasons, it is set to 0xffff. - */ - ItemPointerSetOffsetNumber(&(res->t_tid), 0xffff); - return res; } /* @@ -745,14 +753,11 @@ gistpenalty(GISTSTATE *giststate, int attno, * Initialize a new index page */ void -GISTInitBuffer(Buffer b, uint32 f) +gistinitpage(Page page, uint32 f) { GISTPageOpaque opaque; - Page page; - Size pageSize; + Size pageSize = BLCKSZ; - pageSize = BufferGetPageSize(b); - page = BufferGetPage(b); PageInit(page, pageSize, sizeof(GISTPageOpaqueData)); opaque = GistPageGetOpaque(page); @@ -763,6 +768,18 @@ GISTInitBuffer(Buffer b, uint32 f) opaque->gist_page_id = GIST_PAGE_ID; } +/* + * Initialize a new index buffer + */ +void +GISTInitBuffer(Buffer b, uint32 f) +{ + Page page; + + page = BufferGetPage(b); + gistinitpage(page, f); +} + /* * Verify that a freshly-read page looks sane. */ diff --git a/src/backend/access/gist/gistvalidate.c b/src/backend/access/gist/gistvalidate.c index 2b9ab693be18..8a14620fab27 100644 --- a/src/backend/access/gist/gistvalidate.c +++ b/src/backend/access/gist/gistvalidate.c @@ -143,6 +143,10 @@ gistvalidate(Oid opclassoid) case GIST_OPTIONS_PROC: ok = check_amoptsproc_signature(procform->amproc); break; + case GIST_SORTSUPPORT_PROC: + ok = check_amproc_signature(procform->amproc, VOIDOID, true, + 1, 1, INTERNALOID); + break; default: ereport(INFO, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), @@ -263,7 +267,7 @@ gistvalidate(Oid opclassoid) continue; /* got it */ if (i == GIST_DISTANCE_PROC || i == GIST_FETCH_PROC || i == GIST_COMPRESS_PROC || i == GIST_DECOMPRESS_PROC || - i == GIST_OPTIONS_PROC) + i == GIST_OPTIONS_PROC || i == GIST_SORTSUPPORT_PROC) continue; /* optional methods */ ereport(INFO, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index c526bb19281e..1f0e4e01e69b 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -1019,6 +1019,63 @@ log_newpage(RelFileNode *rnode, ForkNumber forkNum, BlockNumber blkno, return recptr; } +/* + * Like log_newpage(), but allows logging multiple pages in one operation. + * It is more efficient than calling log_newpage() for each page separately, + * because we can write multiple pages in a single WAL record. + */ +void +log_newpages(RelFileNode *rnode, ForkNumber forkNum, int num_pages, + BlockNumber *blknos, Page *pages, bool page_std) +{ + int flags; + XLogRecPtr recptr; + int i; + int j; + + flags = REGBUF_FORCE_IMAGE; + if (page_std) + flags |= REGBUF_STANDARD; + + /* + * Iterate over all the pages. They are collected into batches of + * XLR_MAX_BLOCK_ID pages, and a single WAL-record is written for each + * batch. + */ + XLogEnsureRecordSpace(XLR_MAX_BLOCK_ID - 1, 0); + + i = 0; + while (i < num_pages) + { + int batch_start = i; + int nbatch; + + XLogBeginInsert(); + + nbatch = 0; + while (nbatch < XLR_MAX_BLOCK_ID && i < num_pages) + { + XLogRegisterBlock(nbatch, rnode, forkNum, blknos[i], pages[i], flags); + i++; + nbatch++; + } + + recptr = XLogInsert(RM_XLOG_ID, XLOG_FPI); + + for (j = batch_start; j < i; j++) + { + /* + * The page may be uninitialized. If so, we can't set the LSN because that + * would corrupt the page. + */ + if (!PageIsNew(pages[j])) + { + PageSetLSN(pages[j], recptr); + } + } + } +} + /* * Write a WAL record containing a full image of a page. * diff --git a/src/backend/utils/sort/sortsupport.c b/src/backend/utils/sort/sortsupport.c index fcfe6e831a19..c436fbb4ce1e 100644 --- a/src/backend/utils/sort/sortsupport.c +++ b/src/backend/utils/sort/sortsupport.c @@ -15,6 +15,7 @@ #include "postgres.h" +#include "access/gist.h" #include "access/nbtree.h" #include "catalog/pg_am.h" #include "fmgr.h" @@ -175,3 +176,36 @@ PrepareSortSupportFromIndexRel(Relation indexRel, int16 strategy, FinishSortSupportFunction(opfamily, opcintype, ssup); } + +/* + * Fill in SortSupport given a GiST index relation + * + * Caller must previously have zeroed the SortSupportData structure and then + * filled in ssup_cxt, ssup_attno, ssup_collation, and ssup_nulls_first. This + * will fill in ssup_reverse (always false for GiST index build), as well as + * the comparator function pointer. + */ +void +PrepareSortSupportFromGistIndexRel(Relation indexRel, SortSupport ssup) +{ + Oid opfamily = indexRel->rd_opfamily[ssup->ssup_attno - 1]; + Oid opcintype = indexRel->rd_opcintype[ssup->ssup_attno - 1]; + Oid sortSupportFunction; + + Assert(ssup->comparator == NULL); + + if (indexRel->rd_rel->relam != GIST_AM_OID) + elog(ERROR, "unexpected non-gist AM: %u", indexRel->rd_rel->relam); + ssup->ssup_reverse = false; + + /* + * Look up the sort support function. This is simpler than for B-tree + * indexes because we don't support the old-style btree comparators. + */ + sortSupportFunction = get_opfamily_proc(opfamily, opcintype, opcintype, + GIST_SORTSUPPORT_PROC); + if (!OidIsValid(sortSupportFunction)) + elog(ERROR, "missing support function %d(%u,%u) in opfamily %u", + GIST_SORTSUPPORT_PROC, opcintype, opcintype, opfamily); + OidFunctionCall1(sortSupportFunction, PointerGetDatum(ssup)); +} diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c index cbda911f4652..d0cc04a878a1 100644 --- a/src/backend/utils/sort/tuplesort.c +++ b/src/backend/utils/sort/tuplesort.c @@ -1167,6 +1167,63 @@ tuplesort_begin_index_hash(Relation heapRel, return state; } +Tuplesortstate * +tuplesort_begin_index_gist(Relation heapRel, + Relation indexRel, + int workMem, + SortCoordinate coordinate, + bool randomAccess) +{ + Tuplesortstate *state = tuplesort_begin_common(workMem, coordinate, + randomAccess); + MemoryContext oldcontext; + int i; + + oldcontext = MemoryContextSwitchTo(state->sortcontext); + +#ifdef TRACE_SORT + if (trace_sort) + elog(LOG, + "begin index sort: workMem = %d, randomAccess = %c", + workMem, randomAccess ? 't' : 'f'); +#endif + + state->nKeys = IndexRelationGetNumberOfKeyAttributes(indexRel); + + state->comparetup = comparetup_index_btree; + state->copytup = copytup_index; + state->writetup = writetup_index; + state->readtup = readtup_index; + + state->heapRel = heapRel; + state->indexRel = indexRel; + + /* Prepare SortSupport data for each column */ + state->sortKeys = (SortSupport) palloc0(state->nKeys * + sizeof(SortSupportData)); + + for (i = 0; i < state->nKeys; i++) + { + SortSupport sortKey = state->sortKeys + i; + + sortKey->ssup_cxt = CurrentMemoryContext; + sortKey->ssup_collation = indexRel->rd_indcollation[i]; + sortKey->ssup_nulls_first = false; + sortKey->ssup_attno = i + 1; + /* Convey if abbreviation optimization is applicable in principle */ + sortKey->abbreviate = (i == 0); + + AssertState(sortKey->ssup_attno != 0); + + /* Look for a sort support function */ + PrepareSortSupportFromGistIndexRel(indexRel, sortKey); + } + + MemoryContextSwitchTo(oldcontext); + + return state; +} + Tuplesortstate * tuplesort_begin_datum(Oid datumType, Oid sortOperator, Oid sortCollation, bool nullsFirstFlag, int workMem, diff --git a/src/include/access/gist.h b/src/include/access/gist.h index 4994351697c3..4f6dae9a76b0 100644 --- a/src/include/access/gist.h +++ b/src/include/access/gist.h @@ -37,7 +37,8 @@ #define GIST_DISTANCE_PROC 8 #define GIST_FETCH_PROC 9 #define GIST_OPTIONS_PROC 10 -#define GISTNProcs 10 +#define GIST_SORTSUPPORT_PROC 11 +#define GISTNProcs 11 /* * Page opaque data in a GiST index page. diff --git a/src/include/access/gist_private.h b/src/include/access/gist_private.h index 02e985549f63..b68c01a5f246 100644 --- a/src/include/access/gist_private.h +++ b/src/include/access/gist_private.h @@ -501,12 +501,15 @@ extern IndexTuple gistgetadjusted(Relation r, GISTSTATE *giststate); extern IndexTuple gistFormTuple(GISTSTATE *giststate, Relation r, Datum *attdata, bool *isnull, bool isleaf); +extern void gistCompressValues(GISTSTATE *giststate, Relation r, + Datum *attdata, bool *isnull, bool isleaf, Datum *compatt); extern OffsetNumber gistchoose(Relation r, Page p, IndexTuple it, GISTSTATE *giststate); extern void GISTInitBuffer(Buffer b, uint32 f); +extern void gistinitpage(Page page, uint32 f); extern void gistdentryinit(GISTSTATE *giststate, int nkey, GISTENTRY *e, Datum k, Relation r, Page pg, OffsetNumber o, bool l, bool isNull); diff --git a/src/include/access/xloginsert.h b/src/include/access/xloginsert.h index 63df25ae90fe..4ba2c56be60b 100644 --- a/src/include/access/xloginsert.h +++ b/src/include/access/xloginsert.h @@ -54,6 +54,8 @@ extern bool XLogCheckBufferNeedsBackup(Buffer buffer); extern XLogRecPtr log_newpage(RelFileNode *rnode, ForkNumber forkNum, BlockNumber blk, char *page, bool page_std); +extern void log_newpages(RelFileNode *rnode, ForkNumber forkNum, int num_pages, + BlockNumber *blknos, char **pages, bool page_std); extern XLogRecPtr log_newpage_buffer(Buffer buffer, bool page_std); extern void log_newpage_range(Relation rel, ForkNumber forkNum, BlockNumber startblk, BlockNumber endblk, bool page_std); diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 0bbe0a122afd..06ddb1f16b43 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,7 @@ */ /* yyyymmddN */ +/* FIXME: bump this before pushing! */ #define CATALOG_VERSION_NO 202009031 #endif diff --git a/src/include/catalog/pg_amproc.dat b/src/include/catalog/pg_amproc.dat index 37b580883fcb..a8e0c4ff8a52 100644 --- a/src/include/catalog/pg_amproc.dat +++ b/src/include/catalog/pg_amproc.dat @@ -480,6 +480,8 @@ amproc => 'gist_point_distance' }, { amprocfamily => 'gist/point_ops', amproclefttype => 'point', amprocrighttype => 'point', amprocnum => '9', amproc => 'gist_point_fetch' }, +{ amprocfamily => 'gist/point_ops', amproclefttype => 'point', + amprocrighttype => 'point', amprocnum => '11', amproc => 'gist_point_sortsupport' }, { amprocfamily => 'gist/box_ops', amproclefttype => 'box', amprocrighttype => 'box', amprocnum => '1', amproc => 'gist_box_consistent' }, { amprocfamily => 'gist/box_ops', amproclefttype => 'box', diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 687509ba9265..96d7efd4270c 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -8062,6 +8062,9 @@ proname => 'gist_poly_distance', prorettype => 'float8', proargtypes => 'internal polygon int2 oid internal', prosrc => 'gist_poly_distance' }, +{ oid => '3435', descr => 'sort support', + proname => 'gist_point_sortsupport', prorettype => 'void', + proargtypes => 'internal', prosrc => 'gist_point_sortsupport' }, # GIN array support { oid => '2743', descr => 'GIN array support', diff --git a/src/include/utils/sortsupport.h b/src/include/utils/sortsupport.h index 264aec820b1c..fb262c6e8d42 100644 --- a/src/include/utils/sortsupport.h +++ b/src/include/utils/sortsupport.h @@ -272,5 +272,6 @@ extern void PrepareSortSupportComparisonShim(Oid cmpFunc, SortSupport ssup); extern void PrepareSortSupportFromOrderingOp(Oid orderingOp, SortSupport ssup); extern void PrepareSortSupportFromIndexRel(Relation indexRel, int16 strategy, SortSupport ssup); +extern void PrepareSortSupportFromGistIndexRel(Relation indexRel, SortSupport ssup); #endif /* SORTSUPPORT_H */ diff --git a/src/include/utils/tuplesort.h b/src/include/utils/tuplesort.h index 9e76666fe948..c69b36e209ad 100644 --- a/src/include/utils/tuplesort.h +++ b/src/include/utils/tuplesort.h @@ -217,6 +217,10 @@ extern Tuplesortstate *tuplesort_begin_index_hash(Relation heapRel, uint32 max_buckets, int workMem, SortCoordinate coordinate, bool randomAccess); +extern Tuplesortstate *tuplesort_begin_index_gist(Relation heapRel, + Relation indexRel, + int workMem, SortCoordinate coordinate, + bool randomAccess); extern Tuplesortstate *tuplesort_begin_datum(Oid datumType, Oid sortOperator, Oid sortCollation, bool nullsFirstFlag, diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 64c0c668593e..6ace7662ee1f 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -523,8 +523,8 @@ SELECT * FROM point_tbl ORDER BY f1 <-> '0,1'; SELECT * FROM point_tbl ORDER BY f1 <-> '0,1'; f1 ------------------- - (0,0) (1e-300,-1e-300) + (0,0) (-3,4) (-10,0) (10,10) @@ -561,8 +561,8 @@ SELECT * FROM point_tbl WHERE f1 IS NOT NULL ORDER BY f1 <-> '0,1'; SELECT * FROM point_tbl WHERE f1 IS NOT NULL ORDER BY f1 <-> '0,1'; f1 ------------------- - (0,0) (1e-300,-1e-300) + (0,0) (-3,4) (-10,0) (10,10) @@ -584,8 +584,8 @@ SELECT * FROM point_tbl WHERE f1 <@ '(-10,-10),(10,10)':: box ORDER BY f1 <-> '0 SELECT * FROM point_tbl WHERE f1 <@ '(-10,-10),(10,10)':: box ORDER BY f1 <-> '0,1'; f1 ------------------ - (0,0) (1e-300,-1e-300) + (0,0) (-3,4) (-10,0) (10,10) From 45b9805706fdc726906fc9187c9a7b38c559755c Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 17 Sep 2020 11:39:28 +0200 Subject: [PATCH 165/589] Allow CURRENT_ROLE where CURRENT_USER is accepted In the particular case of GRANTED BY, this is specified in the SQL standard. Since in PostgreSQL, CURRENT_ROLE is equivalent to CURRENT_USER, and CURRENT_USER is already supported here, adding CURRENT_ROLE is trivial. The other cases are PostgreSQL extensions, but for the same reason it also makes sense there. Reviewed-by: Vik Fearing Reviewed-by: Asif Rehman Reviewed-by: Alvaro Herrera Discussion: https://www.postgresql.org/message-id/flat/f2feac44-b4c5-f38f-3699-2851d6a76dc9%402ndquadrant.com --- doc/src/sgml/ref/alter_aggregate.sgml | 2 +- doc/src/sgml/ref/alter_collation.sgml | 2 +- doc/src/sgml/ref/alter_conversion.sgml | 2 +- doc/src/sgml/ref/alter_database.sgml | 2 +- doc/src/sgml/ref/alter_domain.sgml | 2 +- doc/src/sgml/ref/alter_event_trigger.sgml | 2 +- .../sgml/ref/alter_foreign_data_wrapper.sgml | 2 +- doc/src/sgml/ref/alter_foreign_table.sgml | 2 +- doc/src/sgml/ref/alter_function.sgml | 2 +- doc/src/sgml/ref/alter_group.sgml | 1 + doc/src/sgml/ref/alter_language.sgml | 2 +- doc/src/sgml/ref/alter_large_object.sgml | 2 +- doc/src/sgml/ref/alter_materialized_view.sgml | 2 +- doc/src/sgml/ref/alter_opclass.sgml | 2 +- doc/src/sgml/ref/alter_operator.sgml | 2 +- doc/src/sgml/ref/alter_opfamily.sgml | 2 +- doc/src/sgml/ref/alter_policy.sgml | 2 +- doc/src/sgml/ref/alter_procedure.sgml | 2 +- doc/src/sgml/ref/alter_publication.sgml | 2 +- doc/src/sgml/ref/alter_role.sgml | 2 + doc/src/sgml/ref/alter_routine.sgml | 2 +- doc/src/sgml/ref/alter_schema.sgml | 2 +- doc/src/sgml/ref/alter_sequence.sgml | 2 +- doc/src/sgml/ref/alter_server.sgml | 2 +- doc/src/sgml/ref/alter_statistics.sgml | 2 +- doc/src/sgml/ref/alter_subscription.sgml | 2 +- doc/src/sgml/ref/alter_table.sgml | 2 +- doc/src/sgml/ref/alter_tablespace.sgml | 2 +- doc/src/sgml/ref/alter_tsconfig.sgml | 2 +- doc/src/sgml/ref/alter_tsdictionary.sgml | 2 +- doc/src/sgml/ref/alter_type.sgml | 2 +- doc/src/sgml/ref/alter_user.sgml | 1 + doc/src/sgml/ref/alter_user_mapping.sgml | 4 +- doc/src/sgml/ref/alter_view.sgml | 2 +- doc/src/sgml/ref/create_policy.sgml | 2 +- doc/src/sgml/ref/create_schema.sgml | 1 + doc/src/sgml/ref/create_tablespace.sgml | 2 +- doc/src/sgml/ref/create_user_mapping.sgml | 4 +- doc/src/sgml/ref/drop_owned.sgml | 2 +- doc/src/sgml/ref/drop_user_mapping.sgml | 4 +- doc/src/sgml/ref/grant.sgml | 1 + doc/src/sgml/ref/reassign_owned.sgml | 4 +- doc/src/sgml/ref/revoke.sgml | 1 + src/backend/parser/gram.y | 11 + src/backend/utils/adt/acl.c | 2 + src/include/nodes/parsenodes.h | 1 + .../unsafe_tests/expected/rolenames.out | 622 ++++++++++-------- .../modules/unsafe_tests/sql/rolenames.sql | 225 ++++--- 48 files changed, 534 insertions(+), 416 deletions(-) diff --git a/doc/src/sgml/ref/alter_aggregate.sgml b/doc/src/sgml/ref/alter_aggregate.sgml index 2ad3e0440bf8..95934a100f87 100644 --- a/doc/src/sgml/ref/alter_aggregate.sgml +++ b/doc/src/sgml/ref/alter_aggregate.sgml @@ -23,7 +23,7 @@ PostgreSQL documentation ALTER AGGREGATE name ( aggregate_signature ) RENAME TO new_name ALTER AGGREGATE name ( aggregate_signature ) - OWNER TO { new_owner | CURRENT_USER | SESSION_USER } + OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER AGGREGATE name ( aggregate_signature ) SET SCHEMA new_schema where aggregate_signature is: diff --git a/doc/src/sgml/ref/alter_collation.sgml b/doc/src/sgml/ref/alter_collation.sgml index bee6f0dd3ca1..af9ff2867b72 100644 --- a/doc/src/sgml/ref/alter_collation.sgml +++ b/doc/src/sgml/ref/alter_collation.sgml @@ -24,7 +24,7 @@ PostgreSQL documentation ALTER COLLATION name REFRESH VERSION ALTER COLLATION name RENAME TO new_name -ALTER COLLATION name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER COLLATION name OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER COLLATION name SET SCHEMA new_schema diff --git a/doc/src/sgml/ref/alter_conversion.sgml b/doc/src/sgml/ref/alter_conversion.sgml index c42bd8b3e404..a128f20f3e8a 100644 --- a/doc/src/sgml/ref/alter_conversion.sgml +++ b/doc/src/sgml/ref/alter_conversion.sgml @@ -22,7 +22,7 @@ PostgreSQL documentation ALTER CONVERSION name RENAME TO new_name -ALTER CONVERSION name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER CONVERSION name OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER CONVERSION name SET SCHEMA new_schema diff --git a/doc/src/sgml/ref/alter_database.sgml b/doc/src/sgml/ref/alter_database.sgml index 7db878cf532c..81e37536a3f6 100644 --- a/doc/src/sgml/ref/alter_database.sgml +++ b/doc/src/sgml/ref/alter_database.sgml @@ -31,7 +31,7 @@ ALTER DATABASE name [ [ WITH ] name RENAME TO new_name -ALTER DATABASE name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER DATABASE name OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER DATABASE name SET TABLESPACE new_tablespace diff --git a/doc/src/sgml/ref/alter_domain.sgml b/doc/src/sgml/ref/alter_domain.sgml index 8201cbb65fcd..afa42b4926f0 100644 --- a/doc/src/sgml/ref/alter_domain.sgml +++ b/doc/src/sgml/ref/alter_domain.sgml @@ -36,7 +36,7 @@ ALTER DOMAIN name ALTER DOMAIN name VALIDATE CONSTRAINT constraint_name ALTER DOMAIN name - OWNER TO { new_owner | CURRENT_USER | SESSION_USER } + OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER DOMAIN name RENAME TO new_name ALTER DOMAIN name diff --git a/doc/src/sgml/ref/alter_event_trigger.sgml b/doc/src/sgml/ref/alter_event_trigger.sgml index 61919f7845db..ef5253bf37eb 100644 --- a/doc/src/sgml/ref/alter_event_trigger.sgml +++ b/doc/src/sgml/ref/alter_event_trigger.sgml @@ -23,7 +23,7 @@ PostgreSQL documentation ALTER EVENT TRIGGER name DISABLE ALTER EVENT TRIGGER name ENABLE [ REPLICA | ALWAYS ] -ALTER EVENT TRIGGER name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER EVENT TRIGGER name OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER EVENT TRIGGER name RENAME TO new_name diff --git a/doc/src/sgml/ref/alter_foreign_data_wrapper.sgml b/doc/src/sgml/ref/alter_foreign_data_wrapper.sgml index 14f3d616e71c..54f34c2c0151 100644 --- a/doc/src/sgml/ref/alter_foreign_data_wrapper.sgml +++ b/doc/src/sgml/ref/alter_foreign_data_wrapper.sgml @@ -25,7 +25,7 @@ ALTER FOREIGN DATA WRAPPER name [ HANDLER handler_function | NO HANDLER ] [ VALIDATOR validator_function | NO VALIDATOR ] [ OPTIONS ( [ ADD | SET | DROP ] option ['value'] [, ... ]) ] -ALTER FOREIGN DATA WRAPPER name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER FOREIGN DATA WRAPPER name OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER FOREIGN DATA WRAPPER name RENAME TO new_name diff --git a/doc/src/sgml/ref/alter_foreign_table.sgml b/doc/src/sgml/ref/alter_foreign_table.sgml index 0f11897c9977..04d53628ec2f 100644 --- a/doc/src/sgml/ref/alter_foreign_table.sgml +++ b/doc/src/sgml/ref/alter_foreign_table.sgml @@ -53,7 +53,7 @@ ALTER FOREIGN TABLE [ IF EXISTS ] nameparent_table NO INHERIT parent_table - OWNER TO { new_owner | CURRENT_USER | SESSION_USER } + OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } OPTIONS ( [ ADD | SET | DROP ] option ['value'] [, ... ]) diff --git a/doc/src/sgml/ref/alter_function.sgml b/doc/src/sgml/ref/alter_function.sgml index 70b1f24bc003..54e61e7d7885 100644 --- a/doc/src/sgml/ref/alter_function.sgml +++ b/doc/src/sgml/ref/alter_function.sgml @@ -26,7 +26,7 @@ ALTER FUNCTION name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] RENAME TO new_name ALTER FUNCTION name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] - OWNER TO { new_owner | CURRENT_USER | SESSION_USER } + OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER FUNCTION name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] SET SCHEMA new_schema ALTER FUNCTION name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] diff --git a/doc/src/sgml/ref/alter_group.sgml b/doc/src/sgml/ref/alter_group.sgml index 39cc2b88cfa2..f6e516310995 100644 --- a/doc/src/sgml/ref/alter_group.sgml +++ b/doc/src/sgml/ref/alter_group.sgml @@ -27,6 +27,7 @@ ALTER GROUP role_specification DROP where role_specification can be: role_name + | CURRENT_ROLE | CURRENT_USER | SESSION_USER diff --git a/doc/src/sgml/ref/alter_language.sgml b/doc/src/sgml/ref/alter_language.sgml index eac63dec1322..0b61c18aee36 100644 --- a/doc/src/sgml/ref/alter_language.sgml +++ b/doc/src/sgml/ref/alter_language.sgml @@ -22,7 +22,7 @@ PostgreSQL documentation ALTER [ PROCEDURAL ] LANGUAGE name RENAME TO new_name -ALTER [ PROCEDURAL ] LANGUAGE name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER [ PROCEDURAL ] LANGUAGE name OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } diff --git a/doc/src/sgml/ref/alter_large_object.sgml b/doc/src/sgml/ref/alter_large_object.sgml index 356f8a8eabf4..17ea1491ba37 100644 --- a/doc/src/sgml/ref/alter_large_object.sgml +++ b/doc/src/sgml/ref/alter_large_object.sgml @@ -21,7 +21,7 @@ PostgreSQL documentation -ALTER LARGE OBJECT large_object_oid OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER LARGE OBJECT large_object_oid OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } diff --git a/doc/src/sgml/ref/alter_materialized_view.sgml b/doc/src/sgml/ref/alter_materialized_view.sgml index 7321183dd0db..78ee99bb9cac 100644 --- a/doc/src/sgml/ref/alter_materialized_view.sgml +++ b/doc/src/sgml/ref/alter_materialized_view.sgml @@ -44,7 +44,7 @@ ALTER MATERIALIZED VIEW ALL IN TABLESPACE namestorage_parameter [= value] [, ... ] ) RESET ( storage_parameter [, ... ] ) - OWNER TO { new_owner | CURRENT_USER | SESSION_USER } + OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } diff --git a/doc/src/sgml/ref/alter_opclass.sgml b/doc/src/sgml/ref/alter_opclass.sgml index 59a64caa4fad..b1db459b113c 100644 --- a/doc/src/sgml/ref/alter_opclass.sgml +++ b/doc/src/sgml/ref/alter_opclass.sgml @@ -25,7 +25,7 @@ ALTER OPERATOR CLASS name USING index_method - OWNER TO { new_owner | CURRENT_USER | SESSION_USER } + OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER OPERATOR CLASS name USING index_method SET SCHEMA new_schema diff --git a/doc/src/sgml/ref/alter_operator.sgml b/doc/src/sgml/ref/alter_operator.sgml index b3bfa9ccbe97..ac35f229a0ca 100644 --- a/doc/src/sgml/ref/alter_operator.sgml +++ b/doc/src/sgml/ref/alter_operator.sgml @@ -22,7 +22,7 @@ PostgreSQL documentation ALTER OPERATOR name ( { left_type | NONE } , { right_type | NONE } ) - OWNER TO { new_owner | CURRENT_USER | SESSION_USER } + OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER OPERATOR name ( { left_type | NONE } , { right_type | NONE } ) SET SCHEMA new_schema diff --git a/doc/src/sgml/ref/alter_opfamily.sgml b/doc/src/sgml/ref/alter_opfamily.sgml index 4ac1cca95a3f..59d5bf107008 100644 --- a/doc/src/sgml/ref/alter_opfamily.sgml +++ b/doc/src/sgml/ref/alter_opfamily.sgml @@ -37,7 +37,7 @@ ALTER OPERATOR FAMILY name USING index_method - OWNER TO { new_owner | CURRENT_USER | SESSION_USER } + OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER OPERATOR FAMILY name USING index_method SET SCHEMA new_schema diff --git a/doc/src/sgml/ref/alter_policy.sgml b/doc/src/sgml/ref/alter_policy.sgml index a1c720a95693..1c38324b599d 100644 --- a/doc/src/sgml/ref/alter_policy.sgml +++ b/doc/src/sgml/ref/alter_policy.sgml @@ -24,7 +24,7 @@ PostgreSQL documentation ALTER POLICY name ON table_name RENAME TO new_name ALTER POLICY name ON table_name - [ TO { role_name | PUBLIC | CURRENT_USER | SESSION_USER } [, ...] ] + [ TO { role_name | PUBLIC | CURRENT_ROLE | CURRENT_USER | SESSION_USER } [, ...] ] [ USING ( using_expression ) ] [ WITH CHECK ( check_expression ) ] diff --git a/doc/src/sgml/ref/alter_procedure.sgml b/doc/src/sgml/ref/alter_procedure.sgml index dae80076d953..bcf45c7a85f9 100644 --- a/doc/src/sgml/ref/alter_procedure.sgml +++ b/doc/src/sgml/ref/alter_procedure.sgml @@ -26,7 +26,7 @@ ALTER PROCEDURE name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] RENAME TO new_name ALTER PROCEDURE name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] - OWNER TO { new_owner | CURRENT_USER | SESSION_USER } + OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER PROCEDURE name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] SET SCHEMA new_schema ALTER PROCEDURE name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] diff --git a/doc/src/sgml/ref/alter_publication.sgml b/doc/src/sgml/ref/alter_publication.sgml index 534e598d93e4..c2946dfe0f8c 100644 --- a/doc/src/sgml/ref/alter_publication.sgml +++ b/doc/src/sgml/ref/alter_publication.sgml @@ -25,7 +25,7 @@ ALTER PUBLICATION name ADD TABLE [ ALTER PUBLICATION name SET TABLE [ ONLY ] table_name [ * ] [, ...] ALTER PUBLICATION name DROP TABLE [ ONLY ] table_name [ * ] [, ...] ALTER PUBLICATION name SET ( publication_parameter [= value] [, ... ] ) -ALTER PUBLICATION name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER PUBLICATION name OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER PUBLICATION name RENAME TO new_name diff --git a/doc/src/sgml/ref/alter_role.sgml b/doc/src/sgml/ref/alter_role.sgml index dbf258ef50d0..d5f166c129ba 100644 --- a/doc/src/sgml/ref/alter_role.sgml +++ b/doc/src/sgml/ref/alter_role.sgml @@ -46,6 +46,7 @@ ALTER ROLE { role_specification | A where role_specification can be: role_name + | CURRENT_ROLE | CURRENT_USER | SESSION_USER @@ -134,6 +135,7 @@ ALTER ROLE { role_specification | A + CURRENT_ROLE CURRENT_USER diff --git a/doc/src/sgml/ref/alter_routine.sgml b/doc/src/sgml/ref/alter_routine.sgml index d1699691e10f..36acaff3198d 100644 --- a/doc/src/sgml/ref/alter_routine.sgml +++ b/doc/src/sgml/ref/alter_routine.sgml @@ -26,7 +26,7 @@ ALTER ROUTINE name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] RENAME TO new_name ALTER ROUTINE name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] - OWNER TO { new_owner | CURRENT_USER | SESSION_USER } + OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER ROUTINE name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] SET SCHEMA new_schema ALTER ROUTINE name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] diff --git a/doc/src/sgml/ref/alter_schema.sgml b/doc/src/sgml/ref/alter_schema.sgml index 2937214026ec..04624c5a5eb0 100644 --- a/doc/src/sgml/ref/alter_schema.sgml +++ b/doc/src/sgml/ref/alter_schema.sgml @@ -22,7 +22,7 @@ PostgreSQL documentation ALTER SCHEMA name RENAME TO new_name -ALTER SCHEMA name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER SCHEMA name OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } diff --git a/doc/src/sgml/ref/alter_sequence.sgml b/doc/src/sgml/ref/alter_sequence.sgml index bfd20af6d3d5..3cd9ece49f22 100644 --- a/doc/src/sgml/ref/alter_sequence.sgml +++ b/doc/src/sgml/ref/alter_sequence.sgml @@ -31,7 +31,7 @@ ALTER SEQUENCE [ IF EXISTS ] name [ RESTART [ [ WITH ] restart ] ] [ CACHE cache ] [ [ NO ] CYCLE ] [ OWNED BY { table_name.column_name | NONE } ] -ALTER SEQUENCE [ IF EXISTS ] name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER SEQUENCE [ IF EXISTS ] name OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER SEQUENCE [ IF EXISTS ] name RENAME TO new_name ALTER SEQUENCE [ IF EXISTS ] name SET SCHEMA new_schema diff --git a/doc/src/sgml/ref/alter_server.sgml b/doc/src/sgml/ref/alter_server.sgml index 17e55b093e93..186f38b5f82e 100644 --- a/doc/src/sgml/ref/alter_server.sgml +++ b/doc/src/sgml/ref/alter_server.sgml @@ -23,7 +23,7 @@ PostgreSQL documentation ALTER SERVER name [ VERSION 'new_version' ] [ OPTIONS ( [ ADD | SET | DROP ] option ['value'] [, ... ] ) ] -ALTER SERVER name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER SERVER name OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER SERVER name RENAME TO new_name diff --git a/doc/src/sgml/ref/alter_statistics.sgml b/doc/src/sgml/ref/alter_statistics.sgml index be4c3f1f0576..112f5d892497 100644 --- a/doc/src/sgml/ref/alter_statistics.sgml +++ b/doc/src/sgml/ref/alter_statistics.sgml @@ -23,7 +23,7 @@ PostgreSQL documentation -ALTER STATISTICS name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER STATISTICS name OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER STATISTICS name RENAME TO new_name ALTER STATISTICS name SET SCHEMA new_schema ALTER STATISTICS name SET STATISTICS new_target diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index a1666b370be9..db5e59f707c6 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -27,7 +27,7 @@ ALTER SUBSCRIPTION name REFRESH PUB ALTER SUBSCRIPTION name ENABLE ALTER SUBSCRIPTION name DISABLE ALTER SUBSCRIPTION name SET ( subscription_parameter [= value] [, ... ] ) -ALTER SUBSCRIPTION name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER SUBSCRIPTION name OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER SUBSCRIPTION name RENAME TO new_name diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index e9c6a8a6c15c..35971cd72333 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -82,7 +82,7 @@ ALTER TABLE [ IF EXISTS ] name NO INHERIT parent_table OF type_name NOT OF - OWNER TO { new_owner | CURRENT_USER | SESSION_USER } + OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } REPLICA IDENTITY { DEFAULT | USING INDEX index_name | FULL | NOTHING } and partition_bound_spec is: diff --git a/doc/src/sgml/ref/alter_tablespace.sgml b/doc/src/sgml/ref/alter_tablespace.sgml index 356fb9f93f32..6de80746d564 100644 --- a/doc/src/sgml/ref/alter_tablespace.sgml +++ b/doc/src/sgml/ref/alter_tablespace.sgml @@ -22,7 +22,7 @@ PostgreSQL documentation ALTER TABLESPACE name RENAME TO new_name -ALTER TABLESPACE name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER TABLESPACE name OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER TABLESPACE name SET ( tablespace_option = value [, ... ] ) ALTER TABLESPACE name RESET ( tablespace_option [, ... ] ) diff --git a/doc/src/sgml/ref/alter_tsconfig.sgml b/doc/src/sgml/ref/alter_tsconfig.sgml index ebe0b94b27e5..8fafcd3bbd82 100644 --- a/doc/src/sgml/ref/alter_tsconfig.sgml +++ b/doc/src/sgml/ref/alter_tsconfig.sgml @@ -32,7 +32,7 @@ ALTER TEXT SEARCH CONFIGURATION name ALTER TEXT SEARCH CONFIGURATION name DROP MAPPING [ IF EXISTS ] FOR token_type [, ... ] ALTER TEXT SEARCH CONFIGURATION name RENAME TO new_name -ALTER TEXT SEARCH CONFIGURATION name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER TEXT SEARCH CONFIGURATION name OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER TEXT SEARCH CONFIGURATION name SET SCHEMA new_schema diff --git a/doc/src/sgml/ref/alter_tsdictionary.sgml b/doc/src/sgml/ref/alter_tsdictionary.sgml index b29865e11e92..d1923ef1609f 100644 --- a/doc/src/sgml/ref/alter_tsdictionary.sgml +++ b/doc/src/sgml/ref/alter_tsdictionary.sgml @@ -25,7 +25,7 @@ ALTER TEXT SEARCH DICTIONARY name ( option [ = value ] [, ... ] ) ALTER TEXT SEARCH DICTIONARY name RENAME TO new_name -ALTER TEXT SEARCH DICTIONARY name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER TEXT SEARCH DICTIONARY name OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER TEXT SEARCH DICTIONARY name SET SCHEMA new_schema diff --git a/doc/src/sgml/ref/alter_type.sgml b/doc/src/sgml/ref/alter_type.sgml index f015fcd2689b..a4f09c660be3 100644 --- a/doc/src/sgml/ref/alter_type.sgml +++ b/doc/src/sgml/ref/alter_type.sgml @@ -23,7 +23,7 @@ PostgreSQL documentation -ALTER TYPE name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER TYPE name OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER TYPE name RENAME TO new_name ALTER TYPE name SET SCHEMA new_schema ALTER TYPE name RENAME ATTRIBUTE attribute_name TO new_attribute_name [ CASCADE | RESTRICT ] diff --git a/doc/src/sgml/ref/alter_user.sgml b/doc/src/sgml/ref/alter_user.sgml index 6769c8ecc4b7..51527cefb4c4 100644 --- a/doc/src/sgml/ref/alter_user.sgml +++ b/doc/src/sgml/ref/alter_user.sgml @@ -46,6 +46,7 @@ ALTER USER { role_specification | A where role_specification can be: role_name + | CURRENT_ROLE | CURRENT_USER | SESSION_USER diff --git a/doc/src/sgml/ref/alter_user_mapping.sgml b/doc/src/sgml/ref/alter_user_mapping.sgml index 7a9b5a188af4..ee5aee9bc9e5 100644 --- a/doc/src/sgml/ref/alter_user_mapping.sgml +++ b/doc/src/sgml/ref/alter_user_mapping.sgml @@ -21,7 +21,7 @@ PostgreSQL documentation -ALTER USER MAPPING FOR { user_name | USER | CURRENT_USER | SESSION_USER | PUBLIC } +ALTER USER MAPPING FOR { user_name | USER | CURRENT_ROLE | CURRENT_USER | SESSION_USER | PUBLIC } SERVER server_name OPTIONS ( [ ADD | SET | DROP ] option ['value'] [, ... ] ) @@ -51,7 +51,7 @@ ALTER USER MAPPING FOR { user_name user_name - User name of the mapping. CURRENT_USER + User name of the mapping. CURRENT_ROLE, CURRENT_USER, and USER match the name of the current user. PUBLIC is used to match all present and future user names in the system. diff --git a/doc/src/sgml/ref/alter_view.sgml b/doc/src/sgml/ref/alter_view.sgml index e8d9e11e0f6f..98c312c5bf6b 100644 --- a/doc/src/sgml/ref/alter_view.sgml +++ b/doc/src/sgml/ref/alter_view.sgml @@ -23,7 +23,7 @@ PostgreSQL documentation ALTER VIEW [ IF EXISTS ] name ALTER [ COLUMN ] column_name SET DEFAULT expression ALTER VIEW [ IF EXISTS ] name ALTER [ COLUMN ] column_name DROP DEFAULT -ALTER VIEW [ IF EXISTS ] name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER VIEW [ IF EXISTS ] name OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER VIEW [ IF EXISTS ] name RENAME [ COLUMN ] column_name TO new_column_name ALTER VIEW [ IF EXISTS ] name RENAME TO new_name ALTER VIEW [ IF EXISTS ] name SET SCHEMA new_schema diff --git a/doc/src/sgml/ref/create_policy.sgml b/doc/src/sgml/ref/create_policy.sgml index 2e1229c4f94c..b4f90561018c 100644 --- a/doc/src/sgml/ref/create_policy.sgml +++ b/doc/src/sgml/ref/create_policy.sgml @@ -24,7 +24,7 @@ PostgreSQL documentation CREATE POLICY name ON table_name [ AS { PERMISSIVE | RESTRICTIVE } ] [ FOR { ALL | SELECT | INSERT | UPDATE | DELETE } ] - [ TO { role_name | PUBLIC | CURRENT_USER | SESSION_USER } [, ...] ] + [ TO { role_name | PUBLIC | CURRENT_ROLE | CURRENT_USER | SESSION_USER } [, ...] ] [ USING ( using_expression ) ] [ WITH CHECK ( check_expression ) ] diff --git a/doc/src/sgml/ref/create_schema.sgml b/doc/src/sgml/ref/create_schema.sgml index ffbe1ba3bcc2..3c2dddb1631e 100644 --- a/doc/src/sgml/ref/create_schema.sgml +++ b/doc/src/sgml/ref/create_schema.sgml @@ -29,6 +29,7 @@ CREATE SCHEMA IF NOT EXISTS AUTHORIZATION role_sp where role_specification can be: user_name + | CURRENT_ROLE | CURRENT_USER | SESSION_USER diff --git a/doc/src/sgml/ref/create_tablespace.sgml b/doc/src/sgml/ref/create_tablespace.sgml index 462b8831c274..84fa7ee5e29e 100644 --- a/doc/src/sgml/ref/create_tablespace.sgml +++ b/doc/src/sgml/ref/create_tablespace.sgml @@ -22,7 +22,7 @@ PostgreSQL documentation CREATE TABLESPACE tablespace_name - [ OWNER { new_owner | CURRENT_USER | SESSION_USER } ] + [ OWNER { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ] LOCATION 'directory' [ WITH ( tablespace_option = value [, ... ] ) ] diff --git a/doc/src/sgml/ref/create_user_mapping.sgml b/doc/src/sgml/ref/create_user_mapping.sgml index 9719a4ff2c0d..55debd54012d 100644 --- a/doc/src/sgml/ref/create_user_mapping.sgml +++ b/doc/src/sgml/ref/create_user_mapping.sgml @@ -21,7 +21,7 @@ PostgreSQL documentation -CREATE USER MAPPING [ IF NOT EXISTS ] FOR { user_name | USER | CURRENT_USER | PUBLIC } +CREATE USER MAPPING [ IF NOT EXISTS ] FOR { user_name | USER | CURRENT_ROLE | CURRENT_USER | PUBLIC } SERVER server_name [ OPTIONS ( option 'value' [ , ... ] ) ] @@ -67,7 +67,7 @@ CREATE USER MAPPING [ IF NOT EXISTS ] FOR { user_ The name of an existing user that is mapped to foreign server. - CURRENT_USER and USER match the name of + CURRENT_ROLE, CURRENT_USER, and USER match the name of the current user. When PUBLIC is specified, a so-called public mapping is created that is used when no user-specific mapping is applicable. diff --git a/doc/src/sgml/ref/drop_owned.sgml b/doc/src/sgml/ref/drop_owned.sgml index 09107bef6474..dcc375f33bf8 100644 --- a/doc/src/sgml/ref/drop_owned.sgml +++ b/doc/src/sgml/ref/drop_owned.sgml @@ -21,7 +21,7 @@ PostgreSQL documentation -DROP OWNED BY { name | CURRENT_USER | SESSION_USER } [, ...] [ CASCADE | RESTRICT ] +DROP OWNED BY { name | CURRENT_ROLE | CURRENT_USER | SESSION_USER } [, ...] [ CASCADE | RESTRICT ] diff --git a/doc/src/sgml/ref/drop_user_mapping.sgml b/doc/src/sgml/ref/drop_user_mapping.sgml index 7cb09f1166dd..9e8896a307f7 100644 --- a/doc/src/sgml/ref/drop_user_mapping.sgml +++ b/doc/src/sgml/ref/drop_user_mapping.sgml @@ -21,7 +21,7 @@ PostgreSQL documentation -DROP USER MAPPING [ IF EXISTS ] FOR { user_name | USER | CURRENT_USER | PUBLIC } SERVER server_name +DROP USER MAPPING [ IF EXISTS ] FOR { user_name | USER | CURRENT_ROLE | CURRENT_USER | PUBLIC } SERVER server_name @@ -59,7 +59,7 @@ DROP USER MAPPING [ IF EXISTS ] FOR { user_nameuser_name - User name of the mapping. CURRENT_USER + User name of the mapping. CURRENT_ROLE, CURRENT_USER, and USER match the name of the current user. PUBLIC is used to match all present and future user names in the system. diff --git a/doc/src/sgml/ref/grant.sgml b/doc/src/sgml/ref/grant.sgml index bc573f7826b2..fe231aa30cd6 100644 --- a/doc/src/sgml/ref/grant.sgml +++ b/doc/src/sgml/ref/grant.sgml @@ -87,6 +87,7 @@ GRANT role_name [, ...] TO role_name | PUBLIC + | CURRENT_ROLE | CURRENT_USER | SESSION_USER diff --git a/doc/src/sgml/ref/reassign_owned.sgml b/doc/src/sgml/ref/reassign_owned.sgml index 42f72a726fd1..783389df4e8d 100644 --- a/doc/src/sgml/ref/reassign_owned.sgml +++ b/doc/src/sgml/ref/reassign_owned.sgml @@ -21,8 +21,8 @@ PostgreSQL documentation -REASSIGN OWNED BY { old_role | CURRENT_USER | SESSION_USER } [, ...] - TO { new_role | CURRENT_USER | SESSION_USER } +REASSIGN OWNED BY { old_role | CURRENT_ROLE | CURRENT_USER | SESSION_USER } [, ...] + TO { new_role | CURRENT_ROLE | CURRENT_USER | SESSION_USER } diff --git a/doc/src/sgml/ref/revoke.sgml b/doc/src/sgml/ref/revoke.sgml index b6bac21c57a3..b50f99dfe714 100644 --- a/doc/src/sgml/ref/revoke.sgml +++ b/doc/src/sgml/ref/revoke.sgml @@ -114,6 +114,7 @@ REVOKE [ ADMIN OPTION FOR ] [ GROUP ] role_name | PUBLIC + | CURRENT_ROLE | CURRENT_USER | SESSION_USER diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 9f47745ee246..4558c02f6a61 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -14919,6 +14919,13 @@ RoleId: RoleSpec "CURRENT_USER"), parser_errposition(@1))); break; + case ROLESPEC_CURRENT_ROLE: + ereport(ERROR, + (errcode(ERRCODE_RESERVED_NAME), + errmsg("%s cannot be used as a role name here", + "CURRENT_ROLE"), + parser_errposition(@1))); + break; } } ; @@ -14950,6 +14957,10 @@ RoleSpec: NonReservedWord } $$ = n; } + | CURRENT_ROLE + { + $$ = makeRoleSpec(ROLESPEC_CURRENT_ROLE, @1); + } | CURRENT_USER { $$ = makeRoleSpec(ROLESPEC_CURRENT_USER, @1); diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index de3f49637e22..f97489f0644a 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -5217,6 +5217,7 @@ get_rolespec_oid(const RoleSpec *role, bool missing_ok) oid = get_role_oid(role->rolename, missing_ok); break; + case ROLESPEC_CURRENT_ROLE: case ROLESPEC_CURRENT_USER: oid = GetUserId(); break; @@ -5259,6 +5260,7 @@ get_rolespec_tuple(const RoleSpec *role) errmsg("role \"%s\" does not exist", role->rolename))); break; + case ROLESPEC_CURRENT_ROLE: case ROLESPEC_CURRENT_USER: tuple = SearchSysCache1(AUTHOID, GetUserId()); if (!HeapTupleIsValid(tuple)) diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index e83329fd6d10..60c2f4546604 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -318,6 +318,7 @@ typedef struct CollateClause typedef enum RoleSpecType { ROLESPEC_CSTRING, /* role name is stored as a C string */ + ROLESPEC_CURRENT_ROLE, /* role spec is CURRENT_ROLE */ ROLESPEC_CURRENT_USER, /* role spec is CURRENT_USER */ ROLESPEC_SESSION_USER, /* role spec is SESSION_USER */ ROLESPEC_PUBLIC /* role name is "public" */ diff --git a/src/test/modules/unsafe_tests/expected/rolenames.out b/src/test/modules/unsafe_tests/expected/rolenames.out index ff6aa69fc097..9636e50e05c8 100644 --- a/src/test/modules/unsafe_tests/expected/rolenames.out +++ b/src/test/modules/unsafe_tests/expected/rolenames.out @@ -3,8 +3,10 @@ CREATE OR REPLACE FUNCTION chkrolattr() AS $$ SELECT r.rolname, v.keyword, r.rolcanlogin, r.rolreplication FROM pg_roles r - JOIN (VALUES(CURRENT_USER, 'current_user'), + JOIN (VALUES(CURRENT_ROLE, 'current_role'), + (CURRENT_USER, 'current_user'), (SESSION_USER, 'session_user'), + ('current_role', '-'), ('current_user', '-'), ('session_user', '-'), ('Public', '-'), @@ -21,8 +23,9 @@ SELECT COALESCE(d.datname, 'ALL'), COALESCE(r.rolname, 'ALL'), FROM pg_db_role_setting s LEFT JOIN pg_roles r ON (r.oid = s.setrole) LEFT JOIN pg_database d ON (d.oid = s.setdatabase) - LEFT JOIN (VALUES(CURRENT_USER, 'current_user'), - (SESSION_USER, 'session_user')) + LEFT JOIN (VALUES(CURRENT_ROLE, 'current_role'), + (CURRENT_USER, 'current_user'), + (SESSION_USER, 'session_user')) AS v(uname, keyword) ON (r.rolname = v.uname) WHERE (r.rolname) IN ('Public', 'current_user', 'regress_testrol1', 'regress_testrol2') @@ -46,6 +49,7 @@ $$ LANGUAGE SQL; SET client_min_messages = ERROR; CREATE ROLE "Public"; CREATE ROLE "None"; +CREATE ROLE "current_role"; CREATE ROLE "current_user"; CREATE ROLE "session_user"; CREATE ROLE "user"; @@ -55,7 +59,7 @@ ERROR: CURRENT_USER cannot be used as a role name here LINE 1: CREATE ROLE current_user; ^ CREATE ROLE current_role; -- error -ERROR: syntax error at or near "current_role" +ERROR: CURRENT_ROLE cannot be used as a role name here LINE 1: CREATE ROLE current_role; ^ CREATE ROLE session_user; -- error @@ -112,23 +116,56 @@ SELECT * FROM chkrolattr(); ------------------+--------------+----------+------------- None | - | f | f Public | - | f | f + current_role | - | f | f current_user | - | f | f regress_testrol1 | session_user | t | f + regress_testrol2 | current_role | f | f regress_testrol2 | current_user | f | f session_user | - | f | f -(6 rows) +(8 rows) + +ALTER ROLE CURRENT_ROLE WITH REPLICATION; +SELECT * FROM chkrolattr(); + role | rolekeyword | canlogin | replication +------------------+--------------+----------+------------- + None | - | f | f + Public | - | f | f + current_role | - | f | f + current_user | - | f | f + regress_testrol1 | session_user | t | f + regress_testrol2 | current_role | f | t + regress_testrol2 | current_user | f | t + session_user | - | f | f +(8 rows) +ALTER ROLE "current_role" WITH REPLICATION; +SELECT * FROM chkrolattr(); + role | rolekeyword | canlogin | replication +------------------+--------------+----------+------------- + None | - | f | f + Public | - | f | f + current_role | - | f | t + current_user | - | f | f + regress_testrol1 | session_user | t | f + regress_testrol2 | current_role | f | t + regress_testrol2 | current_user | f | t + session_user | - | f | f +(8 rows) + +ALTER ROLE CURRENT_ROLE WITH NOREPLICATION; ALTER ROLE CURRENT_USER WITH REPLICATION; SELECT * FROM chkrolattr(); role | rolekeyword | canlogin | replication ------------------+--------------+----------+------------- None | - | f | f Public | - | f | f + current_role | - | f | t current_user | - | f | f regress_testrol1 | session_user | t | f + regress_testrol2 | current_role | f | t regress_testrol2 | current_user | f | t session_user | - | f | f -(6 rows) +(8 rows) ALTER ROLE "current_user" WITH REPLICATION; SELECT * FROM chkrolattr(); @@ -136,11 +173,13 @@ SELECT * FROM chkrolattr(); ------------------+--------------+----------+------------- None | - | f | f Public | - | f | f + current_role | - | f | t current_user | - | f | t regress_testrol1 | session_user | t | f + regress_testrol2 | current_role | f | t regress_testrol2 | current_user | f | t session_user | - | f | f -(6 rows) +(8 rows) ALTER ROLE SESSION_USER WITH REPLICATION; SELECT * FROM chkrolattr(); @@ -148,11 +187,13 @@ SELECT * FROM chkrolattr(); ------------------+--------------+----------+------------- None | - | f | f Public | - | f | f + current_role | - | f | t current_user | - | f | t regress_testrol1 | session_user | t | t + regress_testrol2 | current_role | f | t regress_testrol2 | current_user | f | t session_user | - | f | f -(6 rows) +(8 rows) ALTER ROLE "session_user" WITH REPLICATION; SELECT * FROM chkrolattr(); @@ -160,11 +201,13 @@ SELECT * FROM chkrolattr(); ------------------+--------------+----------+------------- None | - | f | f Public | - | f | f + current_role | - | f | t current_user | - | f | t regress_testrol1 | session_user | t | t + regress_testrol2 | current_role | f | t regress_testrol2 | current_user | f | t session_user | - | f | t -(6 rows) +(8 rows) ALTER USER "Public" WITH REPLICATION; ALTER USER "None" WITH REPLICATION; @@ -173,11 +216,13 @@ SELECT * FROM chkrolattr(); ------------------+--------------+----------+------------- None | - | f | t Public | - | f | t + current_role | - | f | t current_user | - | f | t regress_testrol1 | session_user | t | t + regress_testrol2 | current_role | f | t regress_testrol2 | current_user | f | t session_user | - | f | t -(6 rows) +(8 rows) ALTER USER regress_testrol1 WITH NOREPLICATION; ALTER USER regress_testrol2 WITH NOREPLICATION; @@ -186,21 +231,19 @@ SELECT * FROM chkrolattr(); ------------------+--------------+----------+------------- None | - | f | t Public | - | f | t + current_role | - | f | t current_user | - | f | t regress_testrol1 | session_user | t | f + regress_testrol2 | current_role | f | f regress_testrol2 | current_user | f | f session_user | - | f | t -(6 rows) +(8 rows) ROLLBACK; ALTER ROLE USER WITH LOGIN; -- error ERROR: syntax error at or near "USER" LINE 1: ALTER ROLE USER WITH LOGIN; ^ -ALTER ROLE CURRENT_ROLE WITH LOGIN; --error -ERROR: syntax error at or near "CURRENT_ROLE" -LINE 1: ALTER ROLE CURRENT_ROLE WITH LOGIN; - ^ ALTER ROLE ALL WITH REPLICATION; -- error ERROR: syntax error at or near "WITH" LINE 1: ALTER ROLE ALL WITH REPLICATION; @@ -228,23 +271,56 @@ SELECT * FROM chkrolattr(); ------------------+--------------+----------+------------- None | - | f | f Public | - | f | f + current_role | - | f | f current_user | - | f | f regress_testrol1 | session_user | t | f + regress_testrol2 | current_role | f | f regress_testrol2 | current_user | f | f session_user | - | f | f -(6 rows) +(8 rows) + +ALTER USER CURRENT_ROLE WITH REPLICATION; +SELECT * FROM chkrolattr(); + role | rolekeyword | canlogin | replication +------------------+--------------+----------+------------- + None | - | f | f + Public | - | f | f + current_role | - | f | f + current_user | - | f | f + regress_testrol1 | session_user | t | f + regress_testrol2 | current_role | f | t + regress_testrol2 | current_user | f | t + session_user | - | f | f +(8 rows) + +ALTER USER "current_role" WITH REPLICATION; +SELECT * FROM chkrolattr(); + role | rolekeyword | canlogin | replication +------------------+--------------+----------+------------- + None | - | f | f + Public | - | f | f + current_role | - | f | t + current_user | - | f | f + regress_testrol1 | session_user | t | f + regress_testrol2 | current_role | f | t + regress_testrol2 | current_user | f | t + session_user | - | f | f +(8 rows) +ALTER USER CURRENT_ROLE WITH NOREPLICATION; ALTER USER CURRENT_USER WITH REPLICATION; SELECT * FROM chkrolattr(); role | rolekeyword | canlogin | replication ------------------+--------------+----------+------------- None | - | f | f Public | - | f | f + current_role | - | f | t current_user | - | f | f regress_testrol1 | session_user | t | f + regress_testrol2 | current_role | f | t regress_testrol2 | current_user | f | t session_user | - | f | f -(6 rows) +(8 rows) ALTER USER "current_user" WITH REPLICATION; SELECT * FROM chkrolattr(); @@ -252,11 +328,13 @@ SELECT * FROM chkrolattr(); ------------------+--------------+----------+------------- None | - | f | f Public | - | f | f + current_role | - | f | t current_user | - | f | t regress_testrol1 | session_user | t | f + regress_testrol2 | current_role | f | t regress_testrol2 | current_user | f | t session_user | - | f | f -(6 rows) +(8 rows) ALTER USER SESSION_USER WITH REPLICATION; SELECT * FROM chkrolattr(); @@ -264,11 +342,13 @@ SELECT * FROM chkrolattr(); ------------------+--------------+----------+------------- None | - | f | f Public | - | f | f + current_role | - | f | t current_user | - | f | t regress_testrol1 | session_user | t | t + regress_testrol2 | current_role | f | t regress_testrol2 | current_user | f | t session_user | - | f | f -(6 rows) +(8 rows) ALTER USER "session_user" WITH REPLICATION; SELECT * FROM chkrolattr(); @@ -276,11 +356,13 @@ SELECT * FROM chkrolattr(); ------------------+--------------+----------+------------- None | - | f | f Public | - | f | f + current_role | - | f | t current_user | - | f | t regress_testrol1 | session_user | t | t + regress_testrol2 | current_role | f | t regress_testrol2 | current_user | f | t session_user | - | f | t -(6 rows) +(8 rows) ALTER USER "Public" WITH REPLICATION; ALTER USER "None" WITH REPLICATION; @@ -289,11 +371,13 @@ SELECT * FROM chkrolattr(); ------------------+--------------+----------+------------- None | - | f | t Public | - | f | t + current_role | - | f | t current_user | - | f | t regress_testrol1 | session_user | t | t + regress_testrol2 | current_role | f | t regress_testrol2 | current_user | f | t session_user | - | f | t -(6 rows) +(8 rows) ALTER USER regress_testrol1 WITH NOREPLICATION; ALTER USER regress_testrol2 WITH NOREPLICATION; @@ -302,21 +386,19 @@ SELECT * FROM chkrolattr(); ------------------+--------------+----------+------------- None | - | f | t Public | - | f | t + current_role | - | f | t current_user | - | f | t regress_testrol1 | session_user | t | f + regress_testrol2 | current_role | f | f regress_testrol2 | current_user | f | f session_user | - | f | t -(6 rows) +(8 rows) ROLLBACK; ALTER USER USER WITH LOGIN; -- error ERROR: syntax error at or near "USER" LINE 1: ALTER USER USER WITH LOGIN; ^ -ALTER USER CURRENT_ROLE WITH LOGIN; -- error -ERROR: syntax error at or near "CURRENT_ROLE" -LINE 1: ALTER USER CURRENT_ROLE WITH LOGIN; - ^ ALTER USER ALL WITH REPLICATION; -- error ERROR: syntax error at or near "WITH" LINE 1: ALTER USER ALL WITH REPLICATION; @@ -343,6 +425,7 @@ SELECT * FROM chksetconfig(); ----+------+------------+----------- (0 rows) +ALTER ROLE CURRENT_ROLE SET application_name to 'BAZ'; ALTER ROLE CURRENT_USER SET application_name to 'FOO'; ALTER ROLE SESSION_USER SET application_name to 'BAR'; ALTER ROLE "current_user" SET application_name to 'FOOFOO'; @@ -355,7 +438,8 @@ SELECT * FROM chksetconfig(); ALL | current_user | - | {application_name=FOOFOO} ALL | regress_testrol1 | session_user | {application_name=BAR} ALL | regress_testrol2 | current_user | {application_name=FOO} -(4 rows) + ALL | regress_testrol2 | current_role | {application_name=FOO} +(5 rows) ALTER ROLE regress_testrol1 SET application_name to 'SLAM'; SELECT * FROM chksetconfig(); @@ -365,8 +449,10 @@ SELECT * FROM chksetconfig(); ALL | current_user | - | {application_name=FOOFOO} ALL | regress_testrol1 | session_user | {application_name=SLAM} ALL | regress_testrol2 | current_user | {application_name=FOO} -(4 rows) + ALL | regress_testrol2 | current_role | {application_name=FOO} +(5 rows) +ALTER ROLE CURRENT_ROLE RESET application_name; ALTER ROLE CURRENT_USER RESET application_name; ALTER ROLE SESSION_USER RESET application_name; ALTER ROLE "current_user" RESET application_name; @@ -377,10 +463,6 @@ SELECT * FROM chksetconfig(); ----+------+------------+----------- (0 rows) -ALTER ROLE CURRENT_ROLE SET application_name to 'BAZ'; -- error -ERROR: syntax error at or near "CURRENT_ROLE" -LINE 1: ALTER ROLE CURRENT_ROLE SET application_name to 'BAZ'; - ^ ALTER ROLE USER SET application_name to 'BOOM'; -- error ERROR: syntax error at or near "USER" LINE 1: ALTER ROLE USER SET application_name to 'BOOM'; @@ -395,6 +477,7 @@ SELECT * FROM chksetconfig(); ----+------+------------+----------- (0 rows) +ALTER USER CURRENT_ROLE SET application_name to 'BAZ'; ALTER USER CURRENT_USER SET application_name to 'FOO'; ALTER USER SESSION_USER SET application_name to 'BAR'; ALTER USER "current_user" SET application_name to 'FOOFOO'; @@ -407,7 +490,8 @@ SELECT * FROM chksetconfig(); ALL | current_user | - | {application_name=FOOFOO} ALL | regress_testrol1 | session_user | {application_name=BAR} ALL | regress_testrol2 | current_user | {application_name=FOO} -(4 rows) + ALL | regress_testrol2 | current_role | {application_name=FOO} +(5 rows) ALTER USER regress_testrol1 SET application_name to 'SLAM'; SELECT * FROM chksetconfig(); @@ -417,8 +501,10 @@ SELECT * FROM chksetconfig(); ALL | current_user | - | {application_name=FOOFOO} ALL | regress_testrol1 | session_user | {application_name=SLAM} ALL | regress_testrol2 | current_user | {application_name=FOO} -(4 rows) + ALL | regress_testrol2 | current_role | {application_name=FOO} +(5 rows) +ALTER USER CURRENT_ROLE RESET application_name; ALTER USER CURRENT_USER RESET application_name; ALTER USER SESSION_USER RESET application_name; ALTER USER "current_user" RESET application_name; @@ -429,10 +515,6 @@ SELECT * FROM chksetconfig(); ----+------+------------+----------- (0 rows) -ALTER USER CURRENT_ROLE SET application_name to 'BAZ'; -- error -ERROR: syntax error at or near "CURRENT_ROLE" -LINE 1: ALTER USER CURRENT_ROLE SET application_name to 'BAZ'; - ^ ALTER USER USER SET application_name to 'BOOM'; -- error ERROR: syntax error at or near "USER" LINE 1: ALTER USER USER SET application_name to 'BOOM'; @@ -448,26 +530,23 @@ ERROR: role "nonexistent" does not exist -- CREATE SCHEMA CREATE SCHEMA newschema1 AUTHORIZATION CURRENT_USER; CREATE SCHEMA newschema2 AUTHORIZATION "current_user"; -CREATE SCHEMA newschema3 AUTHORIZATION SESSION_USER; -CREATE SCHEMA newschema4 AUTHORIZATION regress_testrolx; -CREATE SCHEMA newschema5 AUTHORIZATION "Public"; -CREATE SCHEMA newschema6 AUTHORIZATION USER; -- error +CREATE SCHEMA newschema3 AUTHORIZATION CURRENT_ROLE; +CREATE SCHEMA newschema4 AUTHORIZATION SESSION_USER; +CREATE SCHEMA newschema5 AUTHORIZATION regress_testrolx; +CREATE SCHEMA newschema6 AUTHORIZATION "Public"; +CREATE SCHEMA newschemax AUTHORIZATION USER; -- error ERROR: syntax error at or near "USER" -LINE 1: CREATE SCHEMA newschema6 AUTHORIZATION USER; +LINE 1: CREATE SCHEMA newschemax AUTHORIZATION USER; ^ -CREATE SCHEMA newschema6 AUTHORIZATION CURRENT_ROLE; -- error -ERROR: syntax error at or near "CURRENT_ROLE" -LINE 1: CREATE SCHEMA newschema6 AUTHORIZATION CURRENT_ROLE; - ^ -CREATE SCHEMA newschema6 AUTHORIZATION PUBLIC; -- error +CREATE SCHEMA newschemax AUTHORIZATION PUBLIC; -- error ERROR: role "public" does not exist -CREATE SCHEMA newschema6 AUTHORIZATION "public"; -- error +CREATE SCHEMA newschemax AUTHORIZATION "public"; -- error ERROR: role "public" does not exist -CREATE SCHEMA newschema6 AUTHORIZATION NONE; -- error +CREATE SCHEMA newschemax AUTHORIZATION NONE; -- error ERROR: role name "none" is reserved -LINE 1: CREATE SCHEMA newschema6 AUTHORIZATION NONE; +LINE 1: CREATE SCHEMA newschemax AUTHORIZATION NONE; ^ -CREATE SCHEMA newschema6 AUTHORIZATION nonexistent; -- error +CREATE SCHEMA newschemax AUTHORIZATION nonexistent; -- error ERROR: role "nonexistent" does not exist SELECT n.nspname, r.rolname FROM pg_namespace n JOIN pg_roles r ON (r.oid = n.nspowner) @@ -476,38 +555,37 @@ SELECT n.nspname, r.rolname FROM pg_namespace n ------------+------------------ newschema1 | regress_testrol2 newschema2 | current_user - newschema3 | regress_testrol1 - newschema4 | regress_testrolx - newschema5 | Public -(5 rows) + newschema3 | regress_testrol2 + newschema4 | regress_testrol1 + newschema5 | regress_testrolx + newschema6 | Public +(6 rows) CREATE SCHEMA IF NOT EXISTS newschema1 AUTHORIZATION CURRENT_USER; NOTICE: schema "newschema1" already exists, skipping CREATE SCHEMA IF NOT EXISTS newschema2 AUTHORIZATION "current_user"; NOTICE: schema "newschema2" already exists, skipping -CREATE SCHEMA IF NOT EXISTS newschema3 AUTHORIZATION SESSION_USER; +CREATE SCHEMA IF NOT EXISTS newschema3 AUTHORIZATION CURRENT_ROLE; NOTICE: schema "newschema3" already exists, skipping -CREATE SCHEMA IF NOT EXISTS newschema4 AUTHORIZATION regress_testrolx; +CREATE SCHEMA IF NOT EXISTS newschema4 AUTHORIZATION SESSION_USER; NOTICE: schema "newschema4" already exists, skipping -CREATE SCHEMA IF NOT EXISTS newschema5 AUTHORIZATION "Public"; +CREATE SCHEMA IF NOT EXISTS newschema5 AUTHORIZATION regress_testrolx; NOTICE: schema "newschema5" already exists, skipping -CREATE SCHEMA IF NOT EXISTS newschema6 AUTHORIZATION USER; -- error +CREATE SCHEMA IF NOT EXISTS newschema6 AUTHORIZATION "Public"; +NOTICE: schema "newschema6" already exists, skipping +CREATE SCHEMA IF NOT EXISTS newschemax AUTHORIZATION USER; -- error ERROR: syntax error at or near "USER" -LINE 1: CREATE SCHEMA IF NOT EXISTS newschema6 AUTHORIZATION USER; - ^ -CREATE SCHEMA IF NOT EXISTS newschema6 AUTHORIZATION CURRENT_ROLE; -- error -ERROR: syntax error at or near "CURRENT_ROLE" -LINE 1: ...ATE SCHEMA IF NOT EXISTS newschema6 AUTHORIZATION CURRENT_RO... +LINE 1: CREATE SCHEMA IF NOT EXISTS newschemax AUTHORIZATION USER; ^ -CREATE SCHEMA IF NOT EXISTS newschema6 AUTHORIZATION PUBLIC; -- error +CREATE SCHEMA IF NOT EXISTS newschemax AUTHORIZATION PUBLIC; -- error ERROR: role "public" does not exist -CREATE SCHEMA IF NOT EXISTS newschema6 AUTHORIZATION "public"; -- error +CREATE SCHEMA IF NOT EXISTS newschemax AUTHORIZATION "public"; -- error ERROR: role "public" does not exist -CREATE SCHEMA IF NOT EXISTS newschema6 AUTHORIZATION NONE; -- error +CREATE SCHEMA IF NOT EXISTS newschemax AUTHORIZATION NONE; -- error ERROR: role name "none" is reserved -LINE 1: CREATE SCHEMA IF NOT EXISTS newschema6 AUTHORIZATION NONE; +LINE 1: CREATE SCHEMA IF NOT EXISTS newschemax AUTHORIZATION NONE; ^ -CREATE SCHEMA IF NOT EXISTS newschema6 AUTHORIZATION nonexistent; -- error +CREATE SCHEMA IF NOT EXISTS newschemax AUTHORIZATION nonexistent; -- error ERROR: role "nonexistent" does not exist SELECT n.nspname, r.rolname FROM pg_namespace n JOIN pg_roles r ON (r.oid = n.nspowner) @@ -516,10 +594,11 @@ SELECT n.nspname, r.rolname FROM pg_namespace n ------------+------------------ newschema1 | regress_testrol2 newschema2 | current_user - newschema3 | regress_testrol1 - newschema4 | regress_testrolx - newschema5 | Public -(5 rows) + newschema3 | regress_testrol2 + newschema4 | regress_testrol1 + newschema5 | regress_testrolx + newschema6 | Public +(6 rows) -- ALTER TABLE OWNER TO \c - @@ -530,27 +609,25 @@ CREATE TABLE testtab3 (a int); CREATE TABLE testtab4 (a int); CREATE TABLE testtab5 (a int); CREATE TABLE testtab6 (a int); +CREATE TABLE testtab7 (a int); \c - SET SESSION AUTHORIZATION regress_testrol1; SET ROLE regress_testrol2; ALTER TABLE testtab1 OWNER TO CURRENT_USER; ALTER TABLE testtab2 OWNER TO "current_user"; -ALTER TABLE testtab3 OWNER TO SESSION_USER; -ALTER TABLE testtab4 OWNER TO regress_testrolx; -ALTER TABLE testtab5 OWNER TO "Public"; -ALTER TABLE testtab6 OWNER TO CURRENT_ROLE; -- error -ERROR: syntax error at or near "CURRENT_ROLE" -LINE 1: ALTER TABLE testtab6 OWNER TO CURRENT_ROLE; - ^ -ALTER TABLE testtab6 OWNER TO USER; --error +ALTER TABLE testtab3 OWNER TO CURRENT_ROLE; +ALTER TABLE testtab4 OWNER TO SESSION_USER; +ALTER TABLE testtab5 OWNER TO regress_testrolx; +ALTER TABLE testtab6 OWNER TO "Public"; +ALTER TABLE testtab7 OWNER TO USER; --error ERROR: syntax error at or near "USER" -LINE 1: ALTER TABLE testtab6 OWNER TO USER; +LINE 1: ALTER TABLE testtab7 OWNER TO USER; ^ -ALTER TABLE testtab6 OWNER TO PUBLIC; -- error +ALTER TABLE testtab7 OWNER TO PUBLIC; -- error ERROR: role "public" does not exist -ALTER TABLE testtab6 OWNER TO "public"; -- error +ALTER TABLE testtab7 OWNER TO "public"; -- error ERROR: role "public" does not exist -ALTER TABLE testtab6 OWNER TO nonexistent; -- error +ALTER TABLE testtab7 OWNER TO nonexistent; -- error ERROR: role "nonexistent" does not exist SELECT c.relname, r.rolname FROM pg_class c JOIN pg_roles r ON (r.oid = c.relowner) @@ -560,11 +637,12 @@ SELECT c.relname, r.rolname ----------+------------------ testtab1 | regress_testrol2 testtab2 | current_user - testtab3 | regress_testrol1 - testtab4 | regress_testrolx - testtab5 | Public - testtab6 | regress_testrol0 -(6 rows) + testtab3 | regress_testrol2 + testtab4 | regress_testrol1 + testtab5 | regress_testrolx + testtab6 | Public + testtab7 | regress_testrol0 +(7 rows) -- ALTER TABLE, VIEW, MATERIALIZED VIEW, FOREIGN TABLE, SEQUENCE are -- changed their owner in the same way. @@ -580,27 +658,25 @@ CREATE AGGREGATE testagg6(int2) (SFUNC = int2_sum, STYPE = int8); CREATE AGGREGATE testagg7(int2) (SFUNC = int2_sum, STYPE = int8); CREATE AGGREGATE testagg8(int2) (SFUNC = int2_sum, STYPE = int8); CREATE AGGREGATE testagg9(int2) (SFUNC = int2_sum, STYPE = int8); +CREATE AGGREGATE testagga(int2) (SFUNC = int2_sum, STYPE = int8); \c - SET SESSION AUTHORIZATION regress_testrol1; SET ROLE regress_testrol2; ALTER AGGREGATE testagg1(int2) OWNER TO CURRENT_USER; ALTER AGGREGATE testagg2(int2) OWNER TO "current_user"; -ALTER AGGREGATE testagg3(int2) OWNER TO SESSION_USER; -ALTER AGGREGATE testagg4(int2) OWNER TO regress_testrolx; -ALTER AGGREGATE testagg5(int2) OWNER TO "Public"; -ALTER AGGREGATE testagg5(int2) OWNER TO CURRENT_ROLE; -- error -ERROR: syntax error at or near "CURRENT_ROLE" -LINE 1: ALTER AGGREGATE testagg5(int2) OWNER TO CURRENT_ROLE; - ^ -ALTER AGGREGATE testagg5(int2) OWNER TO USER; -- error +ALTER AGGREGATE testagg3(int2) OWNER TO CURRENT_ROLE; +ALTER AGGREGATE testagg4(int2) OWNER TO SESSION_USER; +ALTER AGGREGATE testagg5(int2) OWNER TO regress_testrolx; +ALTER AGGREGATE testagg6(int2) OWNER TO "Public"; +ALTER AGGREGATE testagg6(int2) OWNER TO USER; -- error ERROR: syntax error at or near "USER" -LINE 1: ALTER AGGREGATE testagg5(int2) OWNER TO USER; +LINE 1: ALTER AGGREGATE testagg6(int2) OWNER TO USER; ^ -ALTER AGGREGATE testagg5(int2) OWNER TO PUBLIC; -- error +ALTER AGGREGATE testagg6(int2) OWNER TO PUBLIC; -- error ERROR: role "public" does not exist -ALTER AGGREGATE testagg5(int2) OWNER TO "public"; -- error +ALTER AGGREGATE testagg6(int2) OWNER TO "public"; -- error ERROR: role "public" does not exist -ALTER AGGREGATE testagg5(int2) OWNER TO nonexistent; -- error +ALTER AGGREGATE testagg6(int2) OWNER TO nonexistent; -- error ERROR: role "nonexistent" does not exist SELECT p.proname, r.rolname FROM pg_proc p JOIN pg_roles r ON (r.oid = p.proowner) @@ -610,14 +686,15 @@ SELECT p.proname, r.rolname ----------+------------------ testagg1 | regress_testrol2 testagg2 | current_user - testagg3 | regress_testrol1 - testagg4 | regress_testrolx - testagg5 | Public - testagg6 | regress_testrol0 + testagg3 | regress_testrol2 + testagg4 | regress_testrol1 + testagg5 | regress_testrolx + testagg6 | Public testagg7 | regress_testrol0 testagg8 | regress_testrol0 testagg9 | regress_testrol0 -(9 rows) + testagga | regress_testrol0 +(10 rows) -- CREATE USER MAPPING CREATE FOREIGN DATA WRAPPER test_wrapper; @@ -630,58 +707,52 @@ CREATE SERVER sv6 FOREIGN DATA WRAPPER test_wrapper; CREATE SERVER sv7 FOREIGN DATA WRAPPER test_wrapper; CREATE SERVER sv8 FOREIGN DATA WRAPPER test_wrapper; CREATE SERVER sv9 FOREIGN DATA WRAPPER test_wrapper; +CREATE SERVER sv10 FOREIGN DATA WRAPPER test_wrapper; CREATE USER MAPPING FOR CURRENT_USER SERVER sv1 OPTIONS (user 'CURRENT_USER'); CREATE USER MAPPING FOR "current_user" SERVER sv2 OPTIONS (user '"current_user"'); -CREATE USER MAPPING FOR USER SERVER sv3 OPTIONS (user 'USER'); -CREATE USER MAPPING FOR "user" SERVER sv4 OPTIONS (user '"USER"'); -CREATE USER MAPPING FOR SESSION_USER SERVER sv5 OPTIONS (user 'SESSION_USER'); -CREATE USER MAPPING FOR PUBLIC SERVER sv6 OPTIONS (user 'PUBLIC'); -CREATE USER MAPPING FOR "Public" SERVER sv7 OPTIONS (user '"Public"'); -CREATE USER MAPPING FOR regress_testrolx SERVER sv8 OPTIONS (user 'regress_testrolx'); -CREATE USER MAPPING FOR CURRENT_ROLE SERVER sv9 - OPTIONS (user 'CURRENT_ROLE'); -- error -ERROR: syntax error at or near "CURRENT_ROLE" -LINE 1: CREATE USER MAPPING FOR CURRENT_ROLE SERVER sv9 - ^ -CREATE USER MAPPING FOR nonexistent SERVER sv9 - OPTIONS (user 'nonexistent'); -- error; +CREATE USER MAPPING FOR CURRENT_ROLE SERVER sv3 OPTIONS (user 'CURRENT_ROLE'); +CREATE USER MAPPING FOR USER SERVER sv4 OPTIONS (user 'USER'); +CREATE USER MAPPING FOR "user" SERVER sv5 OPTIONS (user '"USER"'); +CREATE USER MAPPING FOR SESSION_USER SERVER sv6 OPTIONS (user 'SESSION_USER'); +CREATE USER MAPPING FOR PUBLIC SERVER sv7 OPTIONS (user 'PUBLIC'); +CREATE USER MAPPING FOR "Public" SERVER sv8 OPTIONS (user '"Public"'); +CREATE USER MAPPING FOR regress_testrolx SERVER sv9 OPTIONS (user 'regress_testrolx'); +CREATE USER MAPPING FOR nonexistent SERVER sv10 OPTIONS (user 'nonexistent'); -- error; ERROR: role "nonexistent" does not exist SELECT * FROM chkumapping(); umname | umserver | umoptions ------------------+----------+--------------------------- regress_testrol2 | sv1 | {user=CURRENT_USER} current_user | sv2 | {"user=\"current_user\""} - regress_testrol2 | sv3 | {user=USER} - user | sv4 | {"user=\"USER\""} - regress_testrol1 | sv5 | {user=SESSION_USER} - | sv6 | {user=PUBLIC} - Public | sv7 | {"user=\"Public\""} - regress_testrolx | sv8 | {user=regress_testrolx} -(8 rows) + regress_testrol2 | sv3 | {user=CURRENT_ROLE} + regress_testrol2 | sv4 | {user=USER} + user | sv5 | {"user=\"USER\""} + regress_testrol1 | sv6 | {user=SESSION_USER} + | sv7 | {user=PUBLIC} + Public | sv8 | {"user=\"Public\""} + regress_testrolx | sv9 | {user=regress_testrolx} +(9 rows) -- ALTER USER MAPPING ALTER USER MAPPING FOR CURRENT_USER SERVER sv1 OPTIONS (SET user 'CURRENT_USER_alt'); ALTER USER MAPPING FOR "current_user" SERVER sv2 OPTIONS (SET user '"current_user"_alt'); -ALTER USER MAPPING FOR USER SERVER sv3 +ALTER USER MAPPING FOR CURRENT_ROLE SERVER sv3 + OPTIONS (SET user 'CURRENT_ROLE_alt'); +ALTER USER MAPPING FOR USER SERVER sv4 OPTIONS (SET user 'USER_alt'); -ALTER USER MAPPING FOR "user" SERVER sv4 +ALTER USER MAPPING FOR "user" SERVER sv5 OPTIONS (SET user '"user"_alt'); -ALTER USER MAPPING FOR SESSION_USER SERVER sv5 +ALTER USER MAPPING FOR SESSION_USER SERVER sv6 OPTIONS (SET user 'SESSION_USER_alt'); -ALTER USER MAPPING FOR PUBLIC SERVER sv6 +ALTER USER MAPPING FOR PUBLIC SERVER sv7 OPTIONS (SET user 'public_alt'); -ALTER USER MAPPING FOR "Public" SERVER sv7 +ALTER USER MAPPING FOR "Public" SERVER sv8 OPTIONS (SET user '"Public"_alt'); -ALTER USER MAPPING FOR regress_testrolx SERVER sv8 +ALTER USER MAPPING FOR regress_testrolx SERVER sv9 OPTIONS (SET user 'regress_testrolx_alt'); -ALTER USER MAPPING FOR CURRENT_ROLE SERVER sv9 - OPTIONS (SET user 'CURRENT_ROLE_alt'); -ERROR: syntax error at or near "CURRENT_ROLE" -LINE 1: ALTER USER MAPPING FOR CURRENT_ROLE SERVER sv9 - ^ -ALTER USER MAPPING FOR nonexistent SERVER sv9 +ALTER USER MAPPING FOR nonexistent SERVER sv10 OPTIONS (SET user 'nonexistent_alt'); -- error ERROR: role "nonexistent" does not exist SELECT * FROM chkumapping(); @@ -689,28 +760,26 @@ SELECT * FROM chkumapping(); ------------------+----------+------------------------------- regress_testrol2 | sv1 | {user=CURRENT_USER_alt} current_user | sv2 | {"user=\"current_user\"_alt"} - regress_testrol2 | sv3 | {user=USER_alt} - user | sv4 | {"user=\"user\"_alt"} - regress_testrol1 | sv5 | {user=SESSION_USER_alt} - | sv6 | {user=public_alt} - Public | sv7 | {"user=\"Public\"_alt"} - regress_testrolx | sv8 | {user=regress_testrolx_alt} -(8 rows) + regress_testrol2 | sv3 | {user=CURRENT_ROLE_alt} + regress_testrol2 | sv4 | {user=USER_alt} + user | sv5 | {"user=\"user\"_alt"} + regress_testrol1 | sv6 | {user=SESSION_USER_alt} + | sv7 | {user=public_alt} + Public | sv8 | {"user=\"Public\"_alt"} + regress_testrolx | sv9 | {user=regress_testrolx_alt} +(9 rows) -- DROP USER MAPPING DROP USER MAPPING FOR CURRENT_USER SERVER sv1; DROP USER MAPPING FOR "current_user" SERVER sv2; -DROP USER MAPPING FOR USER SERVER sv3; -DROP USER MAPPING FOR "user" SERVER sv4; -DROP USER MAPPING FOR SESSION_USER SERVER sv5; -DROP USER MAPPING FOR PUBLIC SERVER sv6; -DROP USER MAPPING FOR "Public" SERVER sv7; -DROP USER MAPPING FOR regress_testrolx SERVER sv8; -DROP USER MAPPING FOR CURRENT_ROLE SERVER sv9; -- error -ERROR: syntax error at or near "CURRENT_ROLE" -LINE 1: DROP USER MAPPING FOR CURRENT_ROLE SERVER sv9; - ^ -DROP USER MAPPING FOR nonexistent SERVER sv; -- error +DROP USER MAPPING FOR CURRENT_ROLE SERVER sv3; +DROP USER MAPPING FOR USER SERVER sv4; +DROP USER MAPPING FOR "user" SERVER sv5; +DROP USER MAPPING FOR SESSION_USER SERVER sv6; +DROP USER MAPPING FOR PUBLIC SERVER sv7; +DROP USER MAPPING FOR "Public" SERVER sv8; +DROP USER MAPPING FOR regress_testrolx SERVER sv9; +DROP USER MAPPING FOR nonexistent SERVER sv10; -- error ERROR: role "nonexistent" does not exist SELECT * FROM chkumapping(); umname | umserver | umoptions @@ -719,24 +788,26 @@ SELECT * FROM chkumapping(); CREATE USER MAPPING FOR CURRENT_USER SERVER sv1 OPTIONS (user 'CURRENT_USER'); CREATE USER MAPPING FOR "current_user" SERVER sv2 OPTIONS (user '"current_user"'); -CREATE USER MAPPING FOR USER SERVER sv3 OPTIONS (user 'USER'); -CREATE USER MAPPING FOR "user" SERVER sv4 OPTIONS (user '"USER"'); -CREATE USER MAPPING FOR SESSION_USER SERVER sv5 OPTIONS (user 'SESSION_USER'); -CREATE USER MAPPING FOR PUBLIC SERVER sv6 OPTIONS (user 'PUBLIC'); -CREATE USER MAPPING FOR "Public" SERVER sv7 OPTIONS (user '"Public"'); -CREATE USER MAPPING FOR regress_testrolx SERVER sv8 OPTIONS (user 'regress_testrolx'); +CREATE USER MAPPING FOR CURRENT_ROLE SERVER sv3 OPTIONS (user 'CURRENT_ROLE'); +CREATE USER MAPPING FOR USER SERVER sv4 OPTIONS (user 'USER'); +CREATE USER MAPPING FOR "user" SERVER sv5 OPTIONS (user '"USER"'); +CREATE USER MAPPING FOR SESSION_USER SERVER sv6 OPTIONS (user 'SESSION_USER'); +CREATE USER MAPPING FOR PUBLIC SERVER sv7 OPTIONS (user 'PUBLIC'); +CREATE USER MAPPING FOR "Public" SERVER sv8 OPTIONS (user '"Public"'); +CREATE USER MAPPING FOR regress_testrolx SERVER sv9 OPTIONS (user 'regress_testrolx'); SELECT * FROM chkumapping(); umname | umserver | umoptions ------------------+----------+--------------------------- regress_testrol2 | sv1 | {user=CURRENT_USER} current_user | sv2 | {"user=\"current_user\""} - regress_testrol2 | sv3 | {user=USER} - user | sv4 | {"user=\"USER\""} - regress_testrol1 | sv5 | {user=SESSION_USER} - | sv6 | {user=PUBLIC} - Public | sv7 | {"user=\"Public\""} - regress_testrolx | sv8 | {user=regress_testrolx} -(8 rows) + regress_testrol2 | sv3 | {user=CURRENT_ROLE} + regress_testrol2 | sv4 | {user=USER} + user | sv5 | {"user=\"USER\""} + regress_testrol1 | sv6 | {user=SESSION_USER} + | sv7 | {user=PUBLIC} + Public | sv8 | {"user=\"Public\""} + regress_testrolx | sv9 | {user=regress_testrolx} +(9 rows) -- DROP USER MAPPING IF EXISTS DROP USER MAPPING IF EXISTS FOR CURRENT_USER SERVER sv1; @@ -744,82 +815,92 @@ SELECT * FROM chkumapping(); umname | umserver | umoptions ------------------+----------+--------------------------- current_user | sv2 | {"user=\"current_user\""} - regress_testrol2 | sv3 | {user=USER} - user | sv4 | {"user=\"USER\""} - regress_testrol1 | sv5 | {user=SESSION_USER} - | sv6 | {user=PUBLIC} - Public | sv7 | {"user=\"Public\""} - regress_testrolx | sv8 | {user=regress_testrolx} -(7 rows) + regress_testrol2 | sv3 | {user=CURRENT_ROLE} + regress_testrol2 | sv4 | {user=USER} + user | sv5 | {"user=\"USER\""} + regress_testrol1 | sv6 | {user=SESSION_USER} + | sv7 | {user=PUBLIC} + Public | sv8 | {"user=\"Public\""} + regress_testrolx | sv9 | {user=regress_testrolx} +(8 rows) DROP USER MAPPING IF EXISTS FOR "current_user" SERVER sv2; SELECT * FROM chkumapping(); umname | umserver | umoptions ------------------+----------+------------------------- - regress_testrol2 | sv3 | {user=USER} - user | sv4 | {"user=\"USER\""} - regress_testrol1 | sv5 | {user=SESSION_USER} - | sv6 | {user=PUBLIC} - Public | sv7 | {"user=\"Public\""} - regress_testrolx | sv8 | {user=regress_testrolx} + regress_testrol2 | sv3 | {user=CURRENT_ROLE} + regress_testrol2 | sv4 | {user=USER} + user | sv5 | {"user=\"USER\""} + regress_testrol1 | sv6 | {user=SESSION_USER} + | sv7 | {user=PUBLIC} + Public | sv8 | {"user=\"Public\""} + regress_testrolx | sv9 | {user=regress_testrolx} +(7 rows) + +DROP USER MAPPING IF EXISTS FOR CURRENT_USER SERVER sv3; +SELECT * FROM chkumapping(); + umname | umserver | umoptions +------------------+----------+------------------------- + regress_testrol2 | sv4 | {user=USER} + user | sv5 | {"user=\"USER\""} + regress_testrol1 | sv6 | {user=SESSION_USER} + | sv7 | {user=PUBLIC} + Public | sv8 | {"user=\"Public\""} + regress_testrolx | sv9 | {user=regress_testrolx} (6 rows) -DROP USER MAPPING IF EXISTS FOR USER SERVER sv3; +DROP USER MAPPING IF EXISTS FOR USER SERVER sv4; SELECT * FROM chkumapping(); umname | umserver | umoptions ------------------+----------+------------------------- - user | sv4 | {"user=\"USER\""} - regress_testrol1 | sv5 | {user=SESSION_USER} - | sv6 | {user=PUBLIC} - Public | sv7 | {"user=\"Public\""} - regress_testrolx | sv8 | {user=regress_testrolx} + user | sv5 | {"user=\"USER\""} + regress_testrol1 | sv6 | {user=SESSION_USER} + | sv7 | {user=PUBLIC} + Public | sv8 | {"user=\"Public\""} + regress_testrolx | sv9 | {user=regress_testrolx} (5 rows) -DROP USER MAPPING IF EXISTS FOR "user" SERVER sv4; +DROP USER MAPPING IF EXISTS FOR "user" SERVER sv5; SELECT * FROM chkumapping(); umname | umserver | umoptions ------------------+----------+------------------------- - regress_testrol1 | sv5 | {user=SESSION_USER} - | sv6 | {user=PUBLIC} - Public | sv7 | {"user=\"Public\""} - regress_testrolx | sv8 | {user=regress_testrolx} + regress_testrol1 | sv6 | {user=SESSION_USER} + | sv7 | {user=PUBLIC} + Public | sv8 | {"user=\"Public\""} + regress_testrolx | sv9 | {user=regress_testrolx} (4 rows) -DROP USER MAPPING IF EXISTS FOR SESSION_USER SERVER sv5; +DROP USER MAPPING IF EXISTS FOR SESSION_USER SERVER sv6; SELECT * FROM chkumapping(); umname | umserver | umoptions ------------------+----------+------------------------- - | sv6 | {user=PUBLIC} - Public | sv7 | {"user=\"Public\""} - regress_testrolx | sv8 | {user=regress_testrolx} + | sv7 | {user=PUBLIC} + Public | sv8 | {"user=\"Public\""} + regress_testrolx | sv9 | {user=regress_testrolx} (3 rows) -DROP USER MAPPING IF EXISTS FOR PUBLIC SERVER sv6; +DROP USER MAPPING IF EXISTS FOR PUBLIC SERVER sv7; SELECT * FROM chkumapping(); umname | umserver | umoptions ------------------+----------+------------------------- - Public | sv7 | {"user=\"Public\""} - regress_testrolx | sv8 | {user=regress_testrolx} + Public | sv8 | {"user=\"Public\""} + regress_testrolx | sv9 | {user=regress_testrolx} (2 rows) -DROP USER MAPPING IF EXISTS FOR "Public" SERVER sv7; +DROP USER MAPPING IF EXISTS FOR "Public" SERVER sv8; SELECT * FROM chkumapping(); umname | umserver | umoptions ------------------+----------+------------------------- - regress_testrolx | sv8 | {user=regress_testrolx} + regress_testrolx | sv9 | {user=regress_testrolx} (1 row) -DROP USER MAPPING IF EXISTS FOR regress_testrolx SERVER sv8; +DROP USER MAPPING IF EXISTS FOR regress_testrolx SERVER sv9; SELECT * FROM chkumapping(); umname | umserver | umoptions --------+----------+----------- (0 rows) -DROP USER MAPPING IF EXISTS FOR CURRENT_ROLE SERVER sv9; --error -ERROR: syntax error at or near "CURRENT_ROLE" -LINE 1: DROP USER MAPPING IF EXISTS FOR CURRENT_ROLE SERVER sv9; - ^ -DROP USER MAPPING IF EXISTS FOR nonexistent SERVER sv9; -- error +DROP USER MAPPING IF EXISTS FOR nonexistent SERVER sv10; -- error NOTICE: role "nonexistent" does not exist, skipping -- GRANT/REVOKE GRANT regress_testrol0 TO pg_signal_backend; -- success @@ -840,7 +921,8 @@ SELECT proname, proacl FROM pg_proc WHERE proname LIKE 'testagg_'; testagg7 | testagg8 | testagg9 | -(9 rows) + testagga | +(10 rows) REVOKE ALL PRIVILEGES ON FUNCTION testagg1(int2) FROM PUBLIC; REVOKE ALL PRIVILEGES ON FUNCTION testagg2(int2) FROM PUBLIC; @@ -853,108 +935,106 @@ REVOKE ALL PRIVILEGES ON FUNCTION testagg8(int2) FROM PUBLIC; GRANT ALL PRIVILEGES ON FUNCTION testagg1(int2) TO PUBLIC; GRANT ALL PRIVILEGES ON FUNCTION testagg2(int2) TO CURRENT_USER; GRANT ALL PRIVILEGES ON FUNCTION testagg3(int2) TO "current_user"; -GRANT ALL PRIVILEGES ON FUNCTION testagg4(int2) TO SESSION_USER; -GRANT ALL PRIVILEGES ON FUNCTION testagg5(int2) TO "Public"; -GRANT ALL PRIVILEGES ON FUNCTION testagg6(int2) TO regress_testrolx; -GRANT ALL PRIVILEGES ON FUNCTION testagg7(int2) TO "public"; -GRANT ALL PRIVILEGES ON FUNCTION testagg8(int2) +GRANT ALL PRIVILEGES ON FUNCTION testagg4(int2) TO CURRENT_ROLE; +GRANT ALL PRIVILEGES ON FUNCTION testagg5(int2) TO SESSION_USER; +GRANT ALL PRIVILEGES ON FUNCTION testagg6(int2) TO "Public"; +GRANT ALL PRIVILEGES ON FUNCTION testagg7(int2) TO regress_testrolx; +GRANT ALL PRIVILEGES ON FUNCTION testagg8(int2) TO "public"; +GRANT ALL PRIVILEGES ON FUNCTION testagg9(int2) TO current_user, public, regress_testrolx; SELECT proname, proacl FROM pg_proc WHERE proname LIKE 'testagg_'; proname | proacl ----------+----------------------------------------------------------------------------------------------------------------------------------- testagg1 | {regress_testrol2=X/regress_testrol2,=X/regress_testrol2} testagg2 | {current_user=X/current_user,regress_testrol2=X/current_user} - testagg3 | {regress_testrol1=X/regress_testrol1,current_user=X/regress_testrol1} - testagg4 | {regress_testrolx=X/regress_testrolx,regress_testrol1=X/regress_testrolx} - testagg5 | {Public=X/Public} - testagg6 | {regress_testrol0=X/regress_testrol0,regress_testrolx=X/regress_testrol0} - testagg7 | {regress_testrol0=X/regress_testrol0,=X/regress_testrol0} - testagg8 | {regress_testrol0=X/regress_testrol0,regress_testrol2=X/regress_testrol0,=X/regress_testrol0,regress_testrolx=X/regress_testrol0} - testagg9 | -(9 rows) + testagg3 | {regress_testrol2=X/regress_testrol2,current_user=X/regress_testrol2} + testagg4 | {regress_testrol1=X/regress_testrol1,regress_testrol2=X/regress_testrol1} + testagg5 | {regress_testrolx=X/regress_testrolx,regress_testrol1=X/regress_testrolx} + testagg6 | {Public=X/Public} + testagg7 | {regress_testrol0=X/regress_testrol0,regress_testrolx=X/regress_testrol0} + testagg8 | {regress_testrol0=X/regress_testrol0,=X/regress_testrol0} + testagg9 | {=X/regress_testrol0,regress_testrol0=X/regress_testrol0,regress_testrol2=X/regress_testrol0,regress_testrolx=X/regress_testrol0} + testagga | +(10 rows) -GRANT ALL PRIVILEGES ON FUNCTION testagg9(int2) TO CURRENT_ROLE; --error -ERROR: syntax error at or near "CURRENT_ROLE" -LINE 1: ...RANT ALL PRIVILEGES ON FUNCTION testagg9(int2) TO CURRENT_RO... - ^ -GRANT ALL PRIVILEGES ON FUNCTION testagg9(int2) TO USER; --error +GRANT ALL PRIVILEGES ON FUNCTION testagga(int2) TO USER; --error ERROR: syntax error at or near "USER" -LINE 1: GRANT ALL PRIVILEGES ON FUNCTION testagg9(int2) TO USER; +LINE 1: GRANT ALL PRIVILEGES ON FUNCTION testagga(int2) TO USER; ^ -GRANT ALL PRIVILEGES ON FUNCTION testagg9(int2) TO NONE; --error +GRANT ALL PRIVILEGES ON FUNCTION testagga(int2) TO NONE; --error ERROR: role name "none" is reserved -LINE 1: GRANT ALL PRIVILEGES ON FUNCTION testagg9(int2) TO NONE; +LINE 1: GRANT ALL PRIVILEGES ON FUNCTION testagga(int2) TO NONE; ^ -GRANT ALL PRIVILEGES ON FUNCTION testagg9(int2) TO "none"; --error +GRANT ALL PRIVILEGES ON FUNCTION testagga(int2) TO "none"; --error ERROR: role name "none" is reserved -LINE 1: GRANT ALL PRIVILEGES ON FUNCTION testagg9(int2) TO "none"; +LINE 1: GRANT ALL PRIVILEGES ON FUNCTION testagga(int2) TO "none"; ^ SELECT proname, proacl FROM pg_proc WHERE proname LIKE 'testagg_'; proname | proacl ----------+----------------------------------------------------------------------------------------------------------------------------------- testagg1 | {regress_testrol2=X/regress_testrol2,=X/regress_testrol2} testagg2 | {current_user=X/current_user,regress_testrol2=X/current_user} - testagg3 | {regress_testrol1=X/regress_testrol1,current_user=X/regress_testrol1} - testagg4 | {regress_testrolx=X/regress_testrolx,regress_testrol1=X/regress_testrolx} - testagg5 | {Public=X/Public} - testagg6 | {regress_testrol0=X/regress_testrol0,regress_testrolx=X/regress_testrol0} - testagg7 | {regress_testrol0=X/regress_testrol0,=X/regress_testrol0} - testagg8 | {regress_testrol0=X/regress_testrol0,regress_testrol2=X/regress_testrol0,=X/regress_testrol0,regress_testrolx=X/regress_testrol0} - testagg9 | -(9 rows) + testagg3 | {regress_testrol2=X/regress_testrol2,current_user=X/regress_testrol2} + testagg4 | {regress_testrol1=X/regress_testrol1,regress_testrol2=X/regress_testrol1} + testagg5 | {regress_testrolx=X/regress_testrolx,regress_testrol1=X/regress_testrolx} + testagg6 | {Public=X/Public} + testagg7 | {regress_testrol0=X/regress_testrol0,regress_testrolx=X/regress_testrol0} + testagg8 | {regress_testrol0=X/regress_testrol0,=X/regress_testrol0} + testagg9 | {=X/regress_testrol0,regress_testrol0=X/regress_testrol0,regress_testrol2=X/regress_testrol0,regress_testrolx=X/regress_testrol0} + testagga | +(10 rows) REVOKE ALL PRIVILEGES ON FUNCTION testagg1(int2) FROM PUBLIC; REVOKE ALL PRIVILEGES ON FUNCTION testagg2(int2) FROM CURRENT_USER; REVOKE ALL PRIVILEGES ON FUNCTION testagg3(int2) FROM "current_user"; -REVOKE ALL PRIVILEGES ON FUNCTION testagg4(int2) FROM SESSION_USER; -REVOKE ALL PRIVILEGES ON FUNCTION testagg5(int2) FROM "Public"; -REVOKE ALL PRIVILEGES ON FUNCTION testagg6(int2) FROM regress_testrolx; -REVOKE ALL PRIVILEGES ON FUNCTION testagg7(int2) FROM "public"; -REVOKE ALL PRIVILEGES ON FUNCTION testagg8(int2) +REVOKE ALL PRIVILEGES ON FUNCTION testagg4(int2) FROM CURRENT_ROLE; +REVOKE ALL PRIVILEGES ON FUNCTION testagg5(int2) FROM SESSION_USER; +REVOKE ALL PRIVILEGES ON FUNCTION testagg6(int2) FROM "Public"; +REVOKE ALL PRIVILEGES ON FUNCTION testagg7(int2) FROM regress_testrolx; +REVOKE ALL PRIVILEGES ON FUNCTION testagg8(int2) FROM "public"; +REVOKE ALL PRIVILEGES ON FUNCTION testagg9(int2) FROM current_user, public, regress_testrolx; SELECT proname, proacl FROM pg_proc WHERE proname LIKE 'testagg_'; proname | proacl ----------+--------------------------------------- testagg1 | {regress_testrol2=X/regress_testrol2} testagg2 | {current_user=X/current_user} - testagg3 | {regress_testrol1=X/regress_testrol1} - testagg4 | {regress_testrolx=X/regress_testrolx} - testagg5 | {} - testagg6 | {regress_testrol0=X/regress_testrol0} + testagg3 | {regress_testrol2=X/regress_testrol2} + testagg4 | {regress_testrol1=X/regress_testrol1} + testagg5 | {regress_testrolx=X/regress_testrolx} + testagg6 | {} testagg7 | {regress_testrol0=X/regress_testrol0} testagg8 | {regress_testrol0=X/regress_testrol0} - testagg9 | -(9 rows) + testagg9 | {regress_testrol0=X/regress_testrol0} + testagga | +(10 rows) -REVOKE ALL PRIVILEGES ON FUNCTION testagg9(int2) FROM CURRENT_ROLE; --error -ERROR: syntax error at or near "CURRENT_ROLE" -LINE 1: ...KE ALL PRIVILEGES ON FUNCTION testagg9(int2) FROM CURRENT_RO... - ^ -REVOKE ALL PRIVILEGES ON FUNCTION testagg9(int2) FROM USER; --error +REVOKE ALL PRIVILEGES ON FUNCTION testagga(int2) FROM USER; --error ERROR: syntax error at or near "USER" -LINE 1: REVOKE ALL PRIVILEGES ON FUNCTION testagg9(int2) FROM USER; +LINE 1: REVOKE ALL PRIVILEGES ON FUNCTION testagga(int2) FROM USER; ^ -REVOKE ALL PRIVILEGES ON FUNCTION testagg9(int2) FROM NONE; --error +REVOKE ALL PRIVILEGES ON FUNCTION testagga(int2) FROM NONE; --error ERROR: role name "none" is reserved -LINE 1: REVOKE ALL PRIVILEGES ON FUNCTION testagg9(int2) FROM NONE; +LINE 1: REVOKE ALL PRIVILEGES ON FUNCTION testagga(int2) FROM NONE; ^ -REVOKE ALL PRIVILEGES ON FUNCTION testagg9(int2) FROM "none"; --error +REVOKE ALL PRIVILEGES ON FUNCTION testagga(int2) FROM "none"; --error ERROR: role name "none" is reserved -LINE 1: ...EVOKE ALL PRIVILEGES ON FUNCTION testagg9(int2) FROM "none"; +LINE 1: ...EVOKE ALL PRIVILEGES ON FUNCTION testagga(int2) FROM "none"; ^ SELECT proname, proacl FROM pg_proc WHERE proname LIKE 'testagg_'; proname | proacl ----------+--------------------------------------- testagg1 | {regress_testrol2=X/regress_testrol2} testagg2 | {current_user=X/current_user} - testagg3 | {regress_testrol1=X/regress_testrol1} - testagg4 | {regress_testrolx=X/regress_testrolx} - testagg5 | {} - testagg6 | {regress_testrol0=X/regress_testrol0} + testagg3 | {regress_testrol2=X/regress_testrol2} + testagg4 | {regress_testrol1=X/regress_testrol1} + testagg5 | {regress_testrolx=X/regress_testrolx} + testagg6 | {} testagg7 | {regress_testrol0=X/regress_testrol0} testagg8 | {regress_testrol0=X/regress_testrol0} - testagg9 | -(9 rows) + testagg9 | {regress_testrol0=X/regress_testrol0} + testagga | +(10 rows) -- DEFAULT MONITORING ROLES CREATE ROLE regress_role_haspriv; @@ -1005,7 +1085,7 @@ REVOKE pg_read_all_settings FROM regress_role_haspriv; -- clean up \c DROP SCHEMA test_roles_schema; -DROP OWNED BY regress_testrol0, "Public", "current_user", regress_testrol1, regress_testrol2, regress_testrolx CASCADE; +DROP OWNED BY regress_testrol0, "Public", "current_role", "current_user", regress_testrol1, regress_testrol2, regress_testrolx CASCADE; DROP ROLE regress_testrol0, regress_testrol1, regress_testrol2, regress_testrolx; -DROP ROLE "Public", "None", "current_user", "session_user", "user"; +DROP ROLE "Public", "None", "current_role", "current_user", "session_user", "user"; DROP ROLE regress_role_haspriv, regress_role_nopriv; diff --git a/src/test/modules/unsafe_tests/sql/rolenames.sql b/src/test/modules/unsafe_tests/sql/rolenames.sql index c3013c146498..638decda6800 100644 --- a/src/test/modules/unsafe_tests/sql/rolenames.sql +++ b/src/test/modules/unsafe_tests/sql/rolenames.sql @@ -3,8 +3,10 @@ CREATE OR REPLACE FUNCTION chkrolattr() AS $$ SELECT r.rolname, v.keyword, r.rolcanlogin, r.rolreplication FROM pg_roles r - JOIN (VALUES(CURRENT_USER, 'current_user'), + JOIN (VALUES(CURRENT_ROLE, 'current_role'), + (CURRENT_USER, 'current_user'), (SESSION_USER, 'session_user'), + ('current_role', '-'), ('current_user', '-'), ('session_user', '-'), ('Public', '-'), @@ -22,8 +24,9 @@ SELECT COALESCE(d.datname, 'ALL'), COALESCE(r.rolname, 'ALL'), FROM pg_db_role_setting s LEFT JOIN pg_roles r ON (r.oid = s.setrole) LEFT JOIN pg_database d ON (d.oid = s.setdatabase) - LEFT JOIN (VALUES(CURRENT_USER, 'current_user'), - (SESSION_USER, 'session_user')) + LEFT JOIN (VALUES(CURRENT_ROLE, 'current_role'), + (CURRENT_USER, 'current_user'), + (SESSION_USER, 'session_user')) AS v(uname, keyword) ON (r.rolname = v.uname) WHERE (r.rolname) IN ('Public', 'current_user', 'regress_testrol1', 'regress_testrol2') @@ -50,6 +53,7 @@ SET client_min_messages = ERROR; CREATE ROLE "Public"; CREATE ROLE "None"; +CREATE ROLE "current_role"; CREATE ROLE "current_user"; CREATE ROLE "session_user"; CREATE ROLE "user"; @@ -84,6 +88,11 @@ SET ROLE regress_testrol2; -- ALTER ROLE BEGIN; SELECT * FROM chkrolattr(); +ALTER ROLE CURRENT_ROLE WITH REPLICATION; +SELECT * FROM chkrolattr(); +ALTER ROLE "current_role" WITH REPLICATION; +SELECT * FROM chkrolattr(); +ALTER ROLE CURRENT_ROLE WITH NOREPLICATION; ALTER ROLE CURRENT_USER WITH REPLICATION; SELECT * FROM chkrolattr(); ALTER ROLE "current_user" WITH REPLICATION; @@ -101,7 +110,6 @@ SELECT * FROM chkrolattr(); ROLLBACK; ALTER ROLE USER WITH LOGIN; -- error -ALTER ROLE CURRENT_ROLE WITH LOGIN; --error ALTER ROLE ALL WITH REPLICATION; -- error ALTER ROLE SESSION_ROLE WITH NOREPLICATION; -- error ALTER ROLE PUBLIC WITH NOREPLICATION; -- error @@ -113,6 +121,11 @@ ALTER ROLE nonexistent WITH NOREPLICATION; -- error -- ALTER USER BEGIN; SELECT * FROM chkrolattr(); +ALTER USER CURRENT_ROLE WITH REPLICATION; +SELECT * FROM chkrolattr(); +ALTER USER "current_role" WITH REPLICATION; +SELECT * FROM chkrolattr(); +ALTER USER CURRENT_ROLE WITH NOREPLICATION; ALTER USER CURRENT_USER WITH REPLICATION; SELECT * FROM chkrolattr(); ALTER USER "current_user" WITH REPLICATION; @@ -130,7 +143,6 @@ SELECT * FROM chkrolattr(); ROLLBACK; ALTER USER USER WITH LOGIN; -- error -ALTER USER CURRENT_ROLE WITH LOGIN; -- error ALTER USER ALL WITH REPLICATION; -- error ALTER USER SESSION_ROLE WITH NOREPLICATION; -- error ALTER USER PUBLIC WITH NOREPLICATION; -- error @@ -141,6 +153,7 @@ ALTER USER nonexistent WITH NOREPLICATION; -- error -- ALTER ROLE SET/RESET SELECT * FROM chksetconfig(); +ALTER ROLE CURRENT_ROLE SET application_name to 'BAZ'; ALTER ROLE CURRENT_USER SET application_name to 'FOO'; ALTER ROLE SESSION_USER SET application_name to 'BAR'; ALTER ROLE "current_user" SET application_name to 'FOOFOO'; @@ -149,6 +162,7 @@ ALTER ROLE ALL SET application_name to 'SLAP'; SELECT * FROM chksetconfig(); ALTER ROLE regress_testrol1 SET application_name to 'SLAM'; SELECT * FROM chksetconfig(); +ALTER ROLE CURRENT_ROLE RESET application_name; ALTER ROLE CURRENT_USER RESET application_name; ALTER ROLE SESSION_USER RESET application_name; ALTER ROLE "current_user" RESET application_name; @@ -157,13 +171,13 @@ ALTER ROLE ALL RESET application_name; SELECT * FROM chksetconfig(); -ALTER ROLE CURRENT_ROLE SET application_name to 'BAZ'; -- error ALTER ROLE USER SET application_name to 'BOOM'; -- error ALTER ROLE PUBLIC SET application_name to 'BOMB'; -- error ALTER ROLE nonexistent SET application_name to 'BOMB'; -- error -- ALTER USER SET/RESET SELECT * FROM chksetconfig(); +ALTER USER CURRENT_ROLE SET application_name to 'BAZ'; ALTER USER CURRENT_USER SET application_name to 'FOO'; ALTER USER SESSION_USER SET application_name to 'BAR'; ALTER USER "current_user" SET application_name to 'FOOFOO'; @@ -172,6 +186,7 @@ ALTER USER ALL SET application_name to 'SLAP'; SELECT * FROM chksetconfig(); ALTER USER regress_testrol1 SET application_name to 'SLAM'; SELECT * FROM chksetconfig(); +ALTER USER CURRENT_ROLE RESET application_name; ALTER USER CURRENT_USER RESET application_name; ALTER USER SESSION_USER RESET application_name; ALTER USER "current_user" RESET application_name; @@ -180,7 +195,6 @@ ALTER USER ALL RESET application_name; SELECT * FROM chksetconfig(); -ALTER USER CURRENT_ROLE SET application_name to 'BAZ'; -- error ALTER USER USER SET application_name to 'BOOM'; -- error ALTER USER PUBLIC SET application_name to 'BOMB'; -- error ALTER USER NONE SET application_name to 'BOMB'; -- error @@ -189,16 +203,16 @@ ALTER USER nonexistent SET application_name to 'BOMB'; -- error -- CREATE SCHEMA CREATE SCHEMA newschema1 AUTHORIZATION CURRENT_USER; CREATE SCHEMA newschema2 AUTHORIZATION "current_user"; -CREATE SCHEMA newschema3 AUTHORIZATION SESSION_USER; -CREATE SCHEMA newschema4 AUTHORIZATION regress_testrolx; -CREATE SCHEMA newschema5 AUTHORIZATION "Public"; +CREATE SCHEMA newschema3 AUTHORIZATION CURRENT_ROLE; +CREATE SCHEMA newschema4 AUTHORIZATION SESSION_USER; +CREATE SCHEMA newschema5 AUTHORIZATION regress_testrolx; +CREATE SCHEMA newschema6 AUTHORIZATION "Public"; -CREATE SCHEMA newschema6 AUTHORIZATION USER; -- error -CREATE SCHEMA newschema6 AUTHORIZATION CURRENT_ROLE; -- error -CREATE SCHEMA newschema6 AUTHORIZATION PUBLIC; -- error -CREATE SCHEMA newschema6 AUTHORIZATION "public"; -- error -CREATE SCHEMA newschema6 AUTHORIZATION NONE; -- error -CREATE SCHEMA newschema6 AUTHORIZATION nonexistent; -- error +CREATE SCHEMA newschemax AUTHORIZATION USER; -- error +CREATE SCHEMA newschemax AUTHORIZATION PUBLIC; -- error +CREATE SCHEMA newschemax AUTHORIZATION "public"; -- error +CREATE SCHEMA newschemax AUTHORIZATION NONE; -- error +CREATE SCHEMA newschemax AUTHORIZATION nonexistent; -- error SELECT n.nspname, r.rolname FROM pg_namespace n JOIN pg_roles r ON (r.oid = n.nspowner) @@ -206,16 +220,16 @@ SELECT n.nspname, r.rolname FROM pg_namespace n CREATE SCHEMA IF NOT EXISTS newschema1 AUTHORIZATION CURRENT_USER; CREATE SCHEMA IF NOT EXISTS newschema2 AUTHORIZATION "current_user"; -CREATE SCHEMA IF NOT EXISTS newschema3 AUTHORIZATION SESSION_USER; -CREATE SCHEMA IF NOT EXISTS newschema4 AUTHORIZATION regress_testrolx; -CREATE SCHEMA IF NOT EXISTS newschema5 AUTHORIZATION "Public"; +CREATE SCHEMA IF NOT EXISTS newschema3 AUTHORIZATION CURRENT_ROLE; +CREATE SCHEMA IF NOT EXISTS newschema4 AUTHORIZATION SESSION_USER; +CREATE SCHEMA IF NOT EXISTS newschema5 AUTHORIZATION regress_testrolx; +CREATE SCHEMA IF NOT EXISTS newschema6 AUTHORIZATION "Public"; -CREATE SCHEMA IF NOT EXISTS newschema6 AUTHORIZATION USER; -- error -CREATE SCHEMA IF NOT EXISTS newschema6 AUTHORIZATION CURRENT_ROLE; -- error -CREATE SCHEMA IF NOT EXISTS newschema6 AUTHORIZATION PUBLIC; -- error -CREATE SCHEMA IF NOT EXISTS newschema6 AUTHORIZATION "public"; -- error -CREATE SCHEMA IF NOT EXISTS newschema6 AUTHORIZATION NONE; -- error -CREATE SCHEMA IF NOT EXISTS newschema6 AUTHORIZATION nonexistent; -- error +CREATE SCHEMA IF NOT EXISTS newschemax AUTHORIZATION USER; -- error +CREATE SCHEMA IF NOT EXISTS newschemax AUTHORIZATION PUBLIC; -- error +CREATE SCHEMA IF NOT EXISTS newschemax AUTHORIZATION "public"; -- error +CREATE SCHEMA IF NOT EXISTS newschemax AUTHORIZATION NONE; -- error +CREATE SCHEMA IF NOT EXISTS newschemax AUTHORIZATION nonexistent; -- error SELECT n.nspname, r.rolname FROM pg_namespace n JOIN pg_roles r ON (r.oid = n.nspowner) @@ -230,6 +244,7 @@ CREATE TABLE testtab3 (a int); CREATE TABLE testtab4 (a int); CREATE TABLE testtab5 (a int); CREATE TABLE testtab6 (a int); +CREATE TABLE testtab7 (a int); \c - SET SESSION AUTHORIZATION regress_testrol1; @@ -237,15 +252,15 @@ SET ROLE regress_testrol2; ALTER TABLE testtab1 OWNER TO CURRENT_USER; ALTER TABLE testtab2 OWNER TO "current_user"; -ALTER TABLE testtab3 OWNER TO SESSION_USER; -ALTER TABLE testtab4 OWNER TO regress_testrolx; -ALTER TABLE testtab5 OWNER TO "Public"; +ALTER TABLE testtab3 OWNER TO CURRENT_ROLE; +ALTER TABLE testtab4 OWNER TO SESSION_USER; +ALTER TABLE testtab5 OWNER TO regress_testrolx; +ALTER TABLE testtab6 OWNER TO "Public"; -ALTER TABLE testtab6 OWNER TO CURRENT_ROLE; -- error -ALTER TABLE testtab6 OWNER TO USER; --error -ALTER TABLE testtab6 OWNER TO PUBLIC; -- error -ALTER TABLE testtab6 OWNER TO "public"; -- error -ALTER TABLE testtab6 OWNER TO nonexistent; -- error +ALTER TABLE testtab7 OWNER TO USER; --error +ALTER TABLE testtab7 OWNER TO PUBLIC; -- error +ALTER TABLE testtab7 OWNER TO "public"; -- error +ALTER TABLE testtab7 OWNER TO nonexistent; -- error SELECT c.relname, r.rolname FROM pg_class c JOIN pg_roles r ON (r.oid = c.relowner) @@ -267,6 +282,7 @@ CREATE AGGREGATE testagg6(int2) (SFUNC = int2_sum, STYPE = int8); CREATE AGGREGATE testagg7(int2) (SFUNC = int2_sum, STYPE = int8); CREATE AGGREGATE testagg8(int2) (SFUNC = int2_sum, STYPE = int8); CREATE AGGREGATE testagg9(int2) (SFUNC = int2_sum, STYPE = int8); +CREATE AGGREGATE testagga(int2) (SFUNC = int2_sum, STYPE = int8); \c - SET SESSION AUTHORIZATION regress_testrol1; @@ -274,15 +290,15 @@ SET ROLE regress_testrol2; ALTER AGGREGATE testagg1(int2) OWNER TO CURRENT_USER; ALTER AGGREGATE testagg2(int2) OWNER TO "current_user"; -ALTER AGGREGATE testagg3(int2) OWNER TO SESSION_USER; -ALTER AGGREGATE testagg4(int2) OWNER TO regress_testrolx; -ALTER AGGREGATE testagg5(int2) OWNER TO "Public"; +ALTER AGGREGATE testagg3(int2) OWNER TO CURRENT_ROLE; +ALTER AGGREGATE testagg4(int2) OWNER TO SESSION_USER; +ALTER AGGREGATE testagg5(int2) OWNER TO regress_testrolx; +ALTER AGGREGATE testagg6(int2) OWNER TO "Public"; -ALTER AGGREGATE testagg5(int2) OWNER TO CURRENT_ROLE; -- error -ALTER AGGREGATE testagg5(int2) OWNER TO USER; -- error -ALTER AGGREGATE testagg5(int2) OWNER TO PUBLIC; -- error -ALTER AGGREGATE testagg5(int2) OWNER TO "public"; -- error -ALTER AGGREGATE testagg5(int2) OWNER TO nonexistent; -- error +ALTER AGGREGATE testagg6(int2) OWNER TO USER; -- error +ALTER AGGREGATE testagg6(int2) OWNER TO PUBLIC; -- error +ALTER AGGREGATE testagg6(int2) OWNER TO "public"; -- error +ALTER AGGREGATE testagg6(int2) OWNER TO nonexistent; -- error SELECT p.proname, r.rolname FROM pg_proc p JOIN pg_roles r ON (r.oid = p.proowner) @@ -300,20 +316,19 @@ CREATE SERVER sv6 FOREIGN DATA WRAPPER test_wrapper; CREATE SERVER sv7 FOREIGN DATA WRAPPER test_wrapper; CREATE SERVER sv8 FOREIGN DATA WRAPPER test_wrapper; CREATE SERVER sv9 FOREIGN DATA WRAPPER test_wrapper; +CREATE SERVER sv10 FOREIGN DATA WRAPPER test_wrapper; CREATE USER MAPPING FOR CURRENT_USER SERVER sv1 OPTIONS (user 'CURRENT_USER'); CREATE USER MAPPING FOR "current_user" SERVER sv2 OPTIONS (user '"current_user"'); -CREATE USER MAPPING FOR USER SERVER sv3 OPTIONS (user 'USER'); -CREATE USER MAPPING FOR "user" SERVER sv4 OPTIONS (user '"USER"'); -CREATE USER MAPPING FOR SESSION_USER SERVER sv5 OPTIONS (user 'SESSION_USER'); -CREATE USER MAPPING FOR PUBLIC SERVER sv6 OPTIONS (user 'PUBLIC'); -CREATE USER MAPPING FOR "Public" SERVER sv7 OPTIONS (user '"Public"'); -CREATE USER MAPPING FOR regress_testrolx SERVER sv8 OPTIONS (user 'regress_testrolx'); - -CREATE USER MAPPING FOR CURRENT_ROLE SERVER sv9 - OPTIONS (user 'CURRENT_ROLE'); -- error -CREATE USER MAPPING FOR nonexistent SERVER sv9 - OPTIONS (user 'nonexistent'); -- error; +CREATE USER MAPPING FOR CURRENT_ROLE SERVER sv3 OPTIONS (user 'CURRENT_ROLE'); +CREATE USER MAPPING FOR USER SERVER sv4 OPTIONS (user 'USER'); +CREATE USER MAPPING FOR "user" SERVER sv5 OPTIONS (user '"USER"'); +CREATE USER MAPPING FOR SESSION_USER SERVER sv6 OPTIONS (user 'SESSION_USER'); +CREATE USER MAPPING FOR PUBLIC SERVER sv7 OPTIONS (user 'PUBLIC'); +CREATE USER MAPPING FOR "Public" SERVER sv8 OPTIONS (user '"Public"'); +CREATE USER MAPPING FOR regress_testrolx SERVER sv9 OPTIONS (user 'regress_testrolx'); + +CREATE USER MAPPING FOR nonexistent SERVER sv10 OPTIONS (user 'nonexistent'); -- error; SELECT * FROM chkumapping(); @@ -322,22 +337,22 @@ ALTER USER MAPPING FOR CURRENT_USER SERVER sv1 OPTIONS (SET user 'CURRENT_USER_alt'); ALTER USER MAPPING FOR "current_user" SERVER sv2 OPTIONS (SET user '"current_user"_alt'); -ALTER USER MAPPING FOR USER SERVER sv3 +ALTER USER MAPPING FOR CURRENT_ROLE SERVER sv3 + OPTIONS (SET user 'CURRENT_ROLE_alt'); +ALTER USER MAPPING FOR USER SERVER sv4 OPTIONS (SET user 'USER_alt'); -ALTER USER MAPPING FOR "user" SERVER sv4 +ALTER USER MAPPING FOR "user" SERVER sv5 OPTIONS (SET user '"user"_alt'); -ALTER USER MAPPING FOR SESSION_USER SERVER sv5 +ALTER USER MAPPING FOR SESSION_USER SERVER sv6 OPTIONS (SET user 'SESSION_USER_alt'); -ALTER USER MAPPING FOR PUBLIC SERVER sv6 +ALTER USER MAPPING FOR PUBLIC SERVER sv7 OPTIONS (SET user 'public_alt'); -ALTER USER MAPPING FOR "Public" SERVER sv7 +ALTER USER MAPPING FOR "Public" SERVER sv8 OPTIONS (SET user '"Public"_alt'); -ALTER USER MAPPING FOR regress_testrolx SERVER sv8 +ALTER USER MAPPING FOR regress_testrolx SERVER sv9 OPTIONS (SET user 'regress_testrolx_alt'); -ALTER USER MAPPING FOR CURRENT_ROLE SERVER sv9 - OPTIONS (SET user 'CURRENT_ROLE_alt'); -ALTER USER MAPPING FOR nonexistent SERVER sv9 +ALTER USER MAPPING FOR nonexistent SERVER sv10 OPTIONS (SET user 'nonexistent_alt'); -- error SELECT * FROM chkumapping(); @@ -345,25 +360,26 @@ SELECT * FROM chkumapping(); -- DROP USER MAPPING DROP USER MAPPING FOR CURRENT_USER SERVER sv1; DROP USER MAPPING FOR "current_user" SERVER sv2; -DROP USER MAPPING FOR USER SERVER sv3; -DROP USER MAPPING FOR "user" SERVER sv4; -DROP USER MAPPING FOR SESSION_USER SERVER sv5; -DROP USER MAPPING FOR PUBLIC SERVER sv6; -DROP USER MAPPING FOR "Public" SERVER sv7; -DROP USER MAPPING FOR regress_testrolx SERVER sv8; - -DROP USER MAPPING FOR CURRENT_ROLE SERVER sv9; -- error -DROP USER MAPPING FOR nonexistent SERVER sv; -- error +DROP USER MAPPING FOR CURRENT_ROLE SERVER sv3; +DROP USER MAPPING FOR USER SERVER sv4; +DROP USER MAPPING FOR "user" SERVER sv5; +DROP USER MAPPING FOR SESSION_USER SERVER sv6; +DROP USER MAPPING FOR PUBLIC SERVER sv7; +DROP USER MAPPING FOR "Public" SERVER sv8; +DROP USER MAPPING FOR regress_testrolx SERVER sv9; + +DROP USER MAPPING FOR nonexistent SERVER sv10; -- error SELECT * FROM chkumapping(); CREATE USER MAPPING FOR CURRENT_USER SERVER sv1 OPTIONS (user 'CURRENT_USER'); CREATE USER MAPPING FOR "current_user" SERVER sv2 OPTIONS (user '"current_user"'); -CREATE USER MAPPING FOR USER SERVER sv3 OPTIONS (user 'USER'); -CREATE USER MAPPING FOR "user" SERVER sv4 OPTIONS (user '"USER"'); -CREATE USER MAPPING FOR SESSION_USER SERVER sv5 OPTIONS (user 'SESSION_USER'); -CREATE USER MAPPING FOR PUBLIC SERVER sv6 OPTIONS (user 'PUBLIC'); -CREATE USER MAPPING FOR "Public" SERVER sv7 OPTIONS (user '"Public"'); -CREATE USER MAPPING FOR regress_testrolx SERVER sv8 OPTIONS (user 'regress_testrolx'); +CREATE USER MAPPING FOR CURRENT_ROLE SERVER sv3 OPTIONS (user 'CURRENT_ROLE'); +CREATE USER MAPPING FOR USER SERVER sv4 OPTIONS (user 'USER'); +CREATE USER MAPPING FOR "user" SERVER sv5 OPTIONS (user '"USER"'); +CREATE USER MAPPING FOR SESSION_USER SERVER sv6 OPTIONS (user 'SESSION_USER'); +CREATE USER MAPPING FOR PUBLIC SERVER sv7 OPTIONS (user 'PUBLIC'); +CREATE USER MAPPING FOR "Public" SERVER sv8 OPTIONS (user '"Public"'); +CREATE USER MAPPING FOR regress_testrolx SERVER sv9 OPTIONS (user 'regress_testrolx'); SELECT * FROM chkumapping(); -- DROP USER MAPPING IF EXISTS @@ -371,21 +387,22 @@ DROP USER MAPPING IF EXISTS FOR CURRENT_USER SERVER sv1; SELECT * FROM chkumapping(); DROP USER MAPPING IF EXISTS FOR "current_user" SERVER sv2; SELECT * FROM chkumapping(); -DROP USER MAPPING IF EXISTS FOR USER SERVER sv3; +DROP USER MAPPING IF EXISTS FOR CURRENT_USER SERVER sv3; +SELECT * FROM chkumapping(); +DROP USER MAPPING IF EXISTS FOR USER SERVER sv4; SELECT * FROM chkumapping(); -DROP USER MAPPING IF EXISTS FOR "user" SERVER sv4; +DROP USER MAPPING IF EXISTS FOR "user" SERVER sv5; SELECT * FROM chkumapping(); -DROP USER MAPPING IF EXISTS FOR SESSION_USER SERVER sv5; +DROP USER MAPPING IF EXISTS FOR SESSION_USER SERVER sv6; SELECT * FROM chkumapping(); -DROP USER MAPPING IF EXISTS FOR PUBLIC SERVER sv6; +DROP USER MAPPING IF EXISTS FOR PUBLIC SERVER sv7; SELECT * FROM chkumapping(); -DROP USER MAPPING IF EXISTS FOR "Public" SERVER sv7; +DROP USER MAPPING IF EXISTS FOR "Public" SERVER sv8; SELECT * FROM chkumapping(); -DROP USER MAPPING IF EXISTS FOR regress_testrolx SERVER sv8; +DROP USER MAPPING IF EXISTS FOR regress_testrolx SERVER sv9; SELECT * FROM chkumapping(); -DROP USER MAPPING IF EXISTS FOR CURRENT_ROLE SERVER sv9; --error -DROP USER MAPPING IF EXISTS FOR nonexistent SERVER sv9; -- error +DROP USER MAPPING IF EXISTS FOR nonexistent SERVER sv10; -- error -- GRANT/REVOKE GRANT regress_testrol0 TO pg_signal_backend; -- success @@ -410,38 +427,38 @@ REVOKE ALL PRIVILEGES ON FUNCTION testagg8(int2) FROM PUBLIC; GRANT ALL PRIVILEGES ON FUNCTION testagg1(int2) TO PUBLIC; GRANT ALL PRIVILEGES ON FUNCTION testagg2(int2) TO CURRENT_USER; GRANT ALL PRIVILEGES ON FUNCTION testagg3(int2) TO "current_user"; -GRANT ALL PRIVILEGES ON FUNCTION testagg4(int2) TO SESSION_USER; -GRANT ALL PRIVILEGES ON FUNCTION testagg5(int2) TO "Public"; -GRANT ALL PRIVILEGES ON FUNCTION testagg6(int2) TO regress_testrolx; -GRANT ALL PRIVILEGES ON FUNCTION testagg7(int2) TO "public"; -GRANT ALL PRIVILEGES ON FUNCTION testagg8(int2) +GRANT ALL PRIVILEGES ON FUNCTION testagg4(int2) TO CURRENT_ROLE; +GRANT ALL PRIVILEGES ON FUNCTION testagg5(int2) TO SESSION_USER; +GRANT ALL PRIVILEGES ON FUNCTION testagg6(int2) TO "Public"; +GRANT ALL PRIVILEGES ON FUNCTION testagg7(int2) TO regress_testrolx; +GRANT ALL PRIVILEGES ON FUNCTION testagg8(int2) TO "public"; +GRANT ALL PRIVILEGES ON FUNCTION testagg9(int2) TO current_user, public, regress_testrolx; SELECT proname, proacl FROM pg_proc WHERE proname LIKE 'testagg_'; -GRANT ALL PRIVILEGES ON FUNCTION testagg9(int2) TO CURRENT_ROLE; --error -GRANT ALL PRIVILEGES ON FUNCTION testagg9(int2) TO USER; --error -GRANT ALL PRIVILEGES ON FUNCTION testagg9(int2) TO NONE; --error -GRANT ALL PRIVILEGES ON FUNCTION testagg9(int2) TO "none"; --error +GRANT ALL PRIVILEGES ON FUNCTION testagga(int2) TO USER; --error +GRANT ALL PRIVILEGES ON FUNCTION testagga(int2) TO NONE; --error +GRANT ALL PRIVILEGES ON FUNCTION testagga(int2) TO "none"; --error SELECT proname, proacl FROM pg_proc WHERE proname LIKE 'testagg_'; REVOKE ALL PRIVILEGES ON FUNCTION testagg1(int2) FROM PUBLIC; REVOKE ALL PRIVILEGES ON FUNCTION testagg2(int2) FROM CURRENT_USER; REVOKE ALL PRIVILEGES ON FUNCTION testagg3(int2) FROM "current_user"; -REVOKE ALL PRIVILEGES ON FUNCTION testagg4(int2) FROM SESSION_USER; -REVOKE ALL PRIVILEGES ON FUNCTION testagg5(int2) FROM "Public"; -REVOKE ALL PRIVILEGES ON FUNCTION testagg6(int2) FROM regress_testrolx; -REVOKE ALL PRIVILEGES ON FUNCTION testagg7(int2) FROM "public"; -REVOKE ALL PRIVILEGES ON FUNCTION testagg8(int2) +REVOKE ALL PRIVILEGES ON FUNCTION testagg4(int2) FROM CURRENT_ROLE; +REVOKE ALL PRIVILEGES ON FUNCTION testagg5(int2) FROM SESSION_USER; +REVOKE ALL PRIVILEGES ON FUNCTION testagg6(int2) FROM "Public"; +REVOKE ALL PRIVILEGES ON FUNCTION testagg7(int2) FROM regress_testrolx; +REVOKE ALL PRIVILEGES ON FUNCTION testagg8(int2) FROM "public"; +REVOKE ALL PRIVILEGES ON FUNCTION testagg9(int2) FROM current_user, public, regress_testrolx; SELECT proname, proacl FROM pg_proc WHERE proname LIKE 'testagg_'; -REVOKE ALL PRIVILEGES ON FUNCTION testagg9(int2) FROM CURRENT_ROLE; --error -REVOKE ALL PRIVILEGES ON FUNCTION testagg9(int2) FROM USER; --error -REVOKE ALL PRIVILEGES ON FUNCTION testagg9(int2) FROM NONE; --error -REVOKE ALL PRIVILEGES ON FUNCTION testagg9(int2) FROM "none"; --error +REVOKE ALL PRIVILEGES ON FUNCTION testagga(int2) FROM USER; --error +REVOKE ALL PRIVILEGES ON FUNCTION testagga(int2) FROM NONE; --error +REVOKE ALL PRIVILEGES ON FUNCTION testagga(int2) FROM "none"; --error SELECT proname, proacl FROM pg_proc WHERE proname LIKE 'testagg_'; @@ -481,7 +498,7 @@ REVOKE pg_read_all_settings FROM regress_role_haspriv; \c DROP SCHEMA test_roles_schema; -DROP OWNED BY regress_testrol0, "Public", "current_user", regress_testrol1, regress_testrol2, regress_testrolx CASCADE; +DROP OWNED BY regress_testrol0, "Public", "current_role", "current_user", regress_testrol1, regress_testrol2, regress_testrolx CASCADE; DROP ROLE regress_testrol0, regress_testrol1, regress_testrol2, regress_testrolx; -DROP ROLE "Public", "None", "current_user", "session_user", "user"; +DROP ROLE "Public", "None", "current_role", "current_user", "session_user", "user"; DROP ROLE regress_role_haspriv, regress_role_nopriv; From b7f2dd959a5082540adbeee1dea0c0b1c154374f Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Thu, 17 Sep 2020 15:16:46 +0530 Subject: [PATCH 166/589] Update parallel BTree scan state when the scan keys can't be satisfied. For parallel btree scan to work for array of scan keys, it should reach BTPARALLEL_DONE state once for every distinct combination of array keys. This is required to ensure that the parallel workers don't try to seize blocks at the same time for different scan keys. We missed to update this state when we discovered that the scan keys can't be satisfied. Author: James Hunter Reviewed-by: Amit Kapila Tested-by: Justin Pryzby Backpatch-through: 10, where it was introduced Discussion: https://postgr.es/m/4248CABC-25E3-4809-B4D0-128E1BAABC3C@amazon.com --- src/backend/access/nbtree/nbtsearch.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/backend/access/nbtree/nbtsearch.c b/src/backend/access/nbtree/nbtsearch.c index 1e628a33d77e..8f6575fdf15c 100644 --- a/src/backend/access/nbtree/nbtsearch.c +++ b/src/backend/access/nbtree/nbtsearch.c @@ -880,7 +880,11 @@ _bt_first(IndexScanDesc scan, ScanDirection dir) * never be satisfied (eg, x == 1 AND x > 2). */ if (!so->qual_ok) + { + /* Notify any other workers that we're done with this scan key. */ + _bt_parallel_done(scan); return false; + } /* * For parallel scans, get the starting page from shared state. If the From 99175141c9254318e5894ac30b9fdb622612acda Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 17 Sep 2020 12:52:18 -0400 Subject: [PATCH 167/589] Improve common/logging.c's support for multiple verbosity levels. Instead of hard-wiring specific verbosity levels into the option processing of client applications, invent pg_logging_increase_verbosity() and encourage clients to implement --verbose by calling that. Then, the common convention that more -v's gets you more verbosity just works. In particular, this allows resurrection of the debug-grade messages that have long existed in pg_dump and its siblings. They were unreachable before this commit due to lack of a way to select PG_LOG_DEBUG logging level. (It appears that they may have been unreachable for some time before common/logging.c was introduced, too, so I'm not specifically blaming cc8d41511 for the oversight. One reason for thinking that is that it's now apparent that _allocAH()'s message needs a null-pointer guard. Testing might have failed to reveal that before 96bf88d52.) Discussion: https://postgr.es/m/1173106.1600116625@sss.pgh.pa.us --- doc/src/sgml/ref/pg_dump.sgml | 2 ++ doc/src/sgml/ref/pg_dumpall.sgml | 4 +++- doc/src/sgml/ref/pg_restore.sgml | 7 ++++++- src/bin/pg_archivecleanup/pg_archivecleanup.c | 2 +- src/bin/pg_dump/pg_backup_archiver.c | 3 ++- src/bin/pg_dump/pg_dump.c | 2 +- src/bin/pg_dump/pg_dumpall.c | 2 +- src/bin/pg_dump/pg_restore.c | 2 +- src/bin/pg_rewind/pg_rewind.c | 2 +- src/bin/pgbench/pgbench.c | 2 +- src/common/logging.c | 18 ++++++++++++++++++ src/include/common/logging.h | 1 + 12 files changed, 38 insertions(+), 9 deletions(-) diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 0b2e2de87b6d..0ec99165c5c0 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -594,6 +594,8 @@ PostgreSQL documentation pg_dump to output detailed object comments and start/stop times to the dump file, and progress messages to standard error. + Repeating the option causes additional debug-level messages + to appear on standard error. diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml index 43abc530a0f6..5b514e4567b2 100644 --- a/doc/src/sgml/ref/pg_dumpall.sgml +++ b/doc/src/sgml/ref/pg_dumpall.sgml @@ -202,7 +202,9 @@ PostgreSQL documentation Specifies verbose mode. This will cause pg_dumpall to output start/stop times to the dump file, and progress messages to standard error. - It will also enable verbose output in pg_dump. + Repeating the option causes additional debug-level messages + to appear on standard error. + The option is also passed down to pg_dump. diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml index 27eab2f02a52..e0d9d2ad64f4 100644 --- a/doc/src/sgml/ref/pg_restore.sgml +++ b/doc/src/sgml/ref/pg_restore.sgml @@ -483,7 +483,12 @@ PostgreSQL documentation - Specifies verbose mode. + Specifies verbose mode. This will cause + pg_restore to output detailed object + comments and start/stop times to the output file, and progress + messages to standard error. + Repeating the option causes additional debug-level messages + to appear on standard error. diff --git a/src/bin/pg_archivecleanup/pg_archivecleanup.c b/src/bin/pg_archivecleanup/pg_archivecleanup.c index e454bae767de..12338e3bb2c2 100644 --- a/src/bin/pg_archivecleanup/pg_archivecleanup.c +++ b/src/bin/pg_archivecleanup/pg_archivecleanup.c @@ -302,7 +302,7 @@ main(int argc, char **argv) switch (c) { case 'd': /* Debug mode */ - pg_logging_set_level(PG_LOG_DEBUG); + pg_logging_increase_verbosity(); break; case 'n': /* Dry-Run mode */ dryrun = true; diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index c05a1fd6af0d..178b61d6cbc3 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -2278,7 +2278,8 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, { ArchiveHandle *AH; - pg_log_debug("allocating AH for %s, format %d", FileSpec, fmt); + pg_log_debug("allocating AH for %s, format %d", + FileSpec ? FileSpec : "(stdio)", fmt); AH = (ArchiveHandle *) pg_malloc0(sizeof(ArchiveHandle)); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 340638887217..320820165b7d 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -512,7 +512,7 @@ main(int argc, char **argv) case 'v': /* verbose */ g_verbose = true; - pg_logging_set_level(PG_LOG_INFO); + pg_logging_increase_verbosity(); break; case 'w': diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 97d2b8dac1c6..219ca963c3b7 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -283,7 +283,7 @@ main(int argc, char *argv[]) case 'v': verbose = true; - pg_logging_set_level(PG_LOG_INFO); + pg_logging_increase_verbosity(); appendPQExpBufferStr(pgdumpopts, " -v"); break; diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c index 544ae3bc5cdf..eebf0d300ba9 100644 --- a/src/bin/pg_dump/pg_restore.c +++ b/src/bin/pg_dump/pg_restore.c @@ -245,7 +245,7 @@ main(int argc, char **argv) case 'v': /* verbose */ opts->verbose = 1; - pg_logging_set_level(PG_LOG_INFO); + pg_logging_increase_verbosity(); break; case 'w': diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 23fc749e4451..0ec52cb03279 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -181,7 +181,7 @@ main(int argc, char **argv) case 3: debug = true; - pg_logging_set_level(PG_LOG_DEBUG); + pg_logging_increase_verbosity(); break; case 'D': /* -D or --target-pgdata */ diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 332eabf6379e..663d7d292a2a 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -5522,7 +5522,7 @@ main(int argc, char **argv) pgport = pg_strdup(optarg); break; case 'd': - pg_logging_set_level(PG_LOG_DEBUG); + pg_logging_increase_verbosity(); break; case 'c': benchmarking_option_set = true; diff --git a/src/common/logging.c b/src/common/logging.c index 6a3a437a34bd..d9632fffc8ad 100644 --- a/src/common/logging.c +++ b/src/common/logging.c @@ -157,12 +157,30 @@ pg_logging_config(int new_flags) log_flags = new_flags; } +/* + * pg_logging_init sets the default log level to INFO. Programs that prefer + * a different default should use this to set it, immediately afterward. + */ void pg_logging_set_level(enum pg_log_level new_level) { __pg_log_level = new_level; } +/* + * Command line switches such as --verbose should invoke this. + */ +void +pg_logging_increase_verbosity(void) +{ + /* + * The enum values are chosen such that we have to decrease __pg_log_level + * in order to become more verbose. + */ + if (__pg_log_level > PG_LOG_NOTSET + 1) + __pg_log_level--; +} + void pg_logging_set_pre_callback(void (*cb) (void)) { diff --git a/src/include/common/logging.h b/src/include/common/logging.h index 028149c7a152..3205b8fef9b7 100644 --- a/src/include/common/logging.h +++ b/src/include/common/logging.h @@ -66,6 +66,7 @@ extern enum pg_log_level __pg_log_level; void pg_logging_init(const char *argv0); void pg_logging_config(int new_flags); void pg_logging_set_level(enum pg_log_level new_level); +void pg_logging_increase_verbosity(void); void pg_logging_set_pre_callback(void (*cb) (void)); void pg_logging_set_locus_callback(void (*cb) (const char **filename, uint64 *lineno)); From 74d4608f506ba423e4c84609c1c0e03e1e92cb91 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 17 Sep 2020 14:16:18 -0400 Subject: [PATCH 168/589] Further improve pgindent's list of file exclusions. I despair of people keeping the README file's notes in sync with the actual exclusion list, so move the notes into the exclusion file. Adjust the pgindent script to explicitly ignore comments in the file, just in case (though it's hard to believe any would match filenames). Extend the list so that it doesn't process any files we'd not wish it to even in a fully-built-out development directory. (There are still a couple of derived files that it wants to reformat, but fixing the generator scripts for those seems like fit material for a separate patch.) Discussion: https://postgr.es/m/79ed5348-be7a-b647-dd40-742207186a22@2ndquadrant.com --- src/tools/pgindent/README | 30 +++-------------- src/tools/pgindent/exclude_file_patterns | 42 +++++++++++++++++++----- src/tools/pgindent/pgindent | 1 + 3 files changed, 39 insertions(+), 34 deletions(-) diff --git a/src/tools/pgindent/README b/src/tools/pgindent/README index 9b16b0ed504a..56f07c7ee4cd 100644 --- a/src/tools/pgindent/README +++ b/src/tools/pgindent/README @@ -138,33 +138,11 @@ Which files are processed The pgindent run processes (nearly) all PostgreSQL *.c and *.h files, but we currently exclude *.y and *.l files, as well as *.c and *.h files derived from *.y and *.l files. Additional exceptions are listed -in exclude_file_patterns: - -src/include/storage/s_lock.h and src/include/port/atomics/ are excluded -because they contain assembly code that pgindent tends to mess up. - -src/backend/utils/fmgrtab.c is excluded because it confuses pgindent -and it's a derived file anyway. - -src/interfaces/ecpg/test/expected/ is excluded to avoid breaking the ecpg -regression tests, since what ecpg generates is not necessarily formatted -as pgindent would do it. (Note that we do not exclude ecpg's header files -from the run; some of them get copied verbatim into ecpg's output, meaning -that the expected files may need to be updated to match.) - -src/include/snowball/libstemmer/ and src/backend/snowball/libstemmer/ -are excluded because those files are imported from an external project, -not maintained locally, and are machine-generated anyway. Likewise for -plperl/ppport.h. - -src/include/jit/llvmjit.h is excluded because it contains C++ constructs -that confuse pgindent. - -src/backend/utils/probes.h and its alias src/include/utils/probes.h -are excluded because that file is machine-generated by code not under -our control, and some versions of dtrace build files that confuse -pgindent. +in exclude_file_patterns; see the notes therein for rationale. +Note that we do not exclude ecpg's header files from the run. Some of them +get copied verbatim into ecpg's output, meaning that ecpg's expected files +may need to be updated to match. The perltidy run processes all *.pl and *.pm files, plus a few executable Perl scripts that are not named that way. See the "find" diff --git a/src/tools/pgindent/exclude_file_patterns b/src/tools/pgindent/exclude_file_patterns index a8f1a92f4b3b..b2e9d8f65478 100644 --- a/src/tools/pgindent/exclude_file_patterns +++ b/src/tools/pgindent/exclude_file_patterns @@ -1,11 +1,37 @@ -#list of file patterns to exclude from pgindent runs, see notes in README -/storage/s_lock\.h$ -/port/atomics/ -/utils/fmgrtab\.c$ -/ecpg/test/expected/ +# List of filename patterns to exclude from pgindent runs +# +# These contain assembly code that pgindent tends to mess up. +src/include/storage/s_lock\.h$ +src/include/port/atomics/ +# +# This contains C++ constructs that confuse pgindent. +src/include/jit/llvmjit\.h$ +# +# This confuses pgindent, and it's a derived file anyway. +src/backend/utils/fmgrtab\.c$ +# +# kwlist_d files are made by gen_keywordlist.pl. While we could insist that +# they match pgindent style, they'd look worse not better, so exclude them. +kwlist_d\.h$ +# +# Exclude ecpg test files to avoid breaking the ecpg regression tests +# (but include files at the top level of the ecpg/test/ directory). +src/interfaces/ecpg/test/.*/ +# +# src/include/snowball/libstemmer/ and src/backend/snowball/libstemmer/ +# are excluded because those files are imported from an external project, +# rather than maintained locally, and they are machine-generated anyway. /snowball/libstemmer/ -/pl/plperl/ppport\.h$ -/jit/llvmjit\.h$ -/utils/probes\.h$ +# +# These files are machine-generated by code not under our control, +# so we shouldn't expect them to conform to our style. +# (Some versions of dtrace build probes.h files that confuse pgindent, too.) +src/backend/utils/probes\.h$ +src/include/pg_config\.h$ +src/pl/plperl/ppport\.h$ +src/pl/plperl/SPI\.c$ +src/pl/plperl/Util\.c$ +# +# Exclude any temporary installations that may be in the tree. /tmp_check/ /tmp_install/ diff --git a/src/tools/pgindent/pgindent b/src/tools/pgindent/pgindent index 457e32882484..4124d27dea66 100755 --- a/src/tools/pgindent/pgindent +++ b/src/tools/pgindent/pgindent @@ -159,6 +159,7 @@ sub process_exclude while (my $line = <$eh>) { chomp $line; + next if $line =~ m/^#/; my $rgx = qr!$line!; @files = grep { $_ !~ /$rgx/ } @files if $rgx; } From 76f412ab310554acb970a0b73c8d1f37f35548c6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 17 Sep 2020 16:17:27 -0400 Subject: [PATCH 169/589] Remove factorial operators, leaving only the factorial() function. The "!" operator is our only built-in postfix operator. Remove it, on the way to removal of grammar support for postfix operators. There is also a "!!" prefix operator, but since it's been marked deprecated for most of its existence, we might as well remove it too. Also zap the SQL alias function numeric_fac(), which seems to have equally little reason to live. Mark Dilger, based on work by myself and Robert Haas; review by John Naylor Discussion: https://postgr.es/m/38ca86db-42ab-9b48-2902-337a0d6b8311@2ndquadrant.com --- .../postgres_fdw/expected/postgres_fdw.out | 8 ----- contrib/postgres_fdw/sql/postgres_fdw.sql | 1 - doc/src/sgml/func.sgml | 30 ------------------- src/include/catalog/catversion.h | 3 +- src/include/catalog/pg_operator.dat | 6 ---- src/include/catalog/pg_proc.dat | 4 --- src/test/regress/expected/create_operator.out | 20 ++++++------- src/test/regress/expected/numeric.out | 26 ++++++---------- src/test/regress/sql/create_operator.sql | 20 ++++++------- src/test/regress/sql/numeric.sql | 8 ++--- 10 files changed, 33 insertions(+), 93 deletions(-) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 84bc0ee38171..10e23d02ed56 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -653,14 +653,6 @@ EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = -c1; -- Op Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" WHERE (("C 1" = (- "C 1"))) (3 rows) -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE 1 = c1!; -- OpExpr(r) - QUERY PLAN ----------------------------------------------------------------------------------------------------------- - Foreign Scan on public.ft1 t1 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" WHERE ((1::numeric = ("C 1" !))) -(3 rows) - EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE (c1 IS NOT NULL) IS DISTINCT FROM (c1 IS NOT NULL); -- DistinctExpr QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------- diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index d452d063430a..78156d10b486 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -307,7 +307,6 @@ EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 IS NULL; -- Nu EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 IS NOT NULL; -- NullTest EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE round(abs(c1), 0) = 1; -- FuncExpr EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = -c1; -- OpExpr(l) -EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE 1 = c1!; -- OpExpr(r) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE (c1 IS NOT NULL) IS DISTINCT FROM (c1 IS NOT NULL); -- DistinctExpr EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = ANY(ARRAY[c2, 1, c1 + 0]); -- ScalarArrayOpExpr EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c1 = (ARRAY[c1,c2,3])[1]; -- SubscriptingRef diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index e2e618791ee0..d6283a35d84f 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -1048,36 +1048,6 @@ repeat('Pg', 4) PgPgPgPg - - - bigint ! - numeric - - - Factorial - (deprecated, use factorial() instead) - - - 5 ! - 120 - - - - - - !! bigint - numeric - - - Factorial as a prefix operator - (deprecated, use factorial() instead) - - - !! 5 - 120 - - - @ numeric_type diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 06ddb1f16b43..358935997087 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,7 +53,6 @@ */ /* yyyymmddN */ -/* FIXME: bump this before pushing! */ -#define CATALOG_VERSION_NO 202009031 +#define CATALOG_VERSION_NO 202009171 #endif diff --git a/src/include/catalog/pg_operator.dat b/src/include/catalog/pg_operator.dat index 4f8b9865effc..7cc812adda63 100644 --- a/src/include/catalog/pg_operator.dat +++ b/src/include/catalog/pg_operator.dat @@ -218,12 +218,6 @@ oprname => '>=', oprleft => 'xid8', oprright => 'xid8', oprresult => 'bool', oprcom => '<=(xid8,xid8)', oprnegate => '<(xid8,xid8)', oprcode => 'xid8ge', oprrest => 'scalargesel', oprjoin => 'scalargejoinsel' }, -{ oid => '388', descr => 'deprecated, use factorial() instead', - oprname => '!', oprkind => 'r', oprleft => 'int8', oprright => '0', - oprresult => 'numeric', oprcode => 'numeric_fac' }, -{ oid => '389', descr => 'deprecated, use factorial() instead', - oprname => '!!', oprkind => 'l', oprleft => '0', oprright => 'int8', - oprresult => 'numeric', oprcode => 'numeric_fac' }, { oid => '385', descr => 'equal', oprname => '=', oprcanhash => 't', oprleft => 'cid', oprright => 'cid', oprresult => 'bool', oprcom => '=(cid,cid)', oprcode => 'cideq', diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 96d7efd4270c..1ae428814662 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -327,10 +327,6 @@ { oid => '110', descr => 'I/O', proname => 'unknownout', prorettype => 'cstring', proargtypes => 'unknown', prosrc => 'unknownout' }, -{ oid => '111', - descr => 'implementation of deprecated ! and !! factorial operators', - proname => 'numeric_fac', prorettype => 'numeric', proargtypes => 'int8', - prosrc => 'numeric_fac' }, { oid => '115', proname => 'box_above_eq', prorettype => 'bool', proargtypes => 'box box', diff --git a/src/test/regress/expected/create_operator.out b/src/test/regress/expected/create_operator.out index 54e8b791595d..9e4d4e93fb79 100644 --- a/src/test/regress/expected/create_operator.out +++ b/src/test/regress/expected/create_operator.out @@ -16,15 +16,15 @@ CREATE OPERATOR <% ( ); CREATE OPERATOR @#@ ( rightarg = int8, -- left unary - procedure = numeric_fac + procedure = factorial ); CREATE OPERATOR #@# ( leftarg = int8, -- right unary - procedure = numeric_fac + procedure = factorial ); CREATE OPERATOR #%# ( leftarg = int8, -- right unary - procedure = numeric_fac + procedure = factorial ); -- Test operator created above SELECT point '(1,2)' <% widget '(0,0,3)' AS t, @@ -40,7 +40,7 @@ ERROR: operator does not exist: integer ###### -- => is disallowed now CREATE OPERATOR => ( leftarg = int8, -- right unary - procedure = numeric_fac + procedure = factorial ); ERROR: syntax error at or near "=>" LINE 1: CREATE OPERATOR => ( @@ -50,7 +50,7 @@ LINE 1: CREATE OPERATOR => ( -- this is legal because ! is not allowed in sql ops CREATE OPERATOR !=- ( leftarg = int8, -- right unary - procedure = numeric_fac + procedure = factorial ); SELECT 2 !=-; ?column? @@ -128,7 +128,7 @@ REVOKE USAGE ON SCHEMA schema_op1 FROM regress_rol_op1; SET ROLE regress_rol_op1; CREATE OPERATOR schema_op1.#*# ( leftarg = int8, -- right unary - procedure = numeric_fac + procedure = factorial ); ERROR: permission denied for schema schema_op1 ROLLBACK; @@ -136,7 +136,7 @@ ROLLBACK; BEGIN TRANSACTION; CREATE OPERATOR #*# ( leftarg = SETOF int8, - procedure = numeric_fac + procedure = factorial ); ERROR: SETOF type not allowed for operator argument ROLLBACK; @@ -144,7 +144,7 @@ ROLLBACK; BEGIN TRANSACTION; CREATE OPERATOR #*# ( rightarg = SETOF int8, - procedure = numeric_fac + procedure = factorial ); ERROR: SETOF type not allowed for operator argument ROLLBACK; @@ -168,13 +168,13 @@ ROLLBACK; -- Should fail. Invalid attribute CREATE OPERATOR #@%# ( leftarg = int8, -- right unary - procedure = numeric_fac, + procedure = factorial, invalid_att = int8 ); WARNING: operator attribute "invalid_att" not recognized -- Should fail. At least leftarg or rightarg should be mandatorily specified CREATE OPERATOR #@%# ( - procedure = numeric_fac + procedure = factorial ); ERROR: at least one of leftarg or rightarg must be specified -- Should fail. Procedure should be mandatorily specified diff --git a/src/test/regress/expected/numeric.out b/src/test/regress/expected/numeric.out index 86940ec68386..823f3fbccf51 100644 --- a/src/test/regress/expected/numeric.out +++ b/src/test/regress/expected/numeric.out @@ -2972,16 +2972,10 @@ ERROR: value overflows numeric format -- -- Tests for factorial -- -SELECT 4!; - ?column? ----------- - 24 -(1 row) - -SELECT !!3; - ?column? ----------- - 6 +SELECT factorial(4); + factorial +----------- + 24 (1 row) SELECT factorial(15); @@ -2990,16 +2984,14 @@ SELECT factorial(15); 1307674368000 (1 row) -SELECT 100000!; +SELECT factorial(100000); ERROR: value overflows numeric format -SELECT 0!; - ?column? ----------- - 1 +SELECT factorial(0); + factorial +----------- + 1 (1 row) -SELECT -4!; -ERROR: factorial of a negative number is undefined SELECT factorial(-4); ERROR: factorial of a negative number is undefined -- diff --git a/src/test/regress/sql/create_operator.sql b/src/test/regress/sql/create_operator.sql index 8b6fd0bb43d6..c32da8c066a7 100644 --- a/src/test/regress/sql/create_operator.sql +++ b/src/test/regress/sql/create_operator.sql @@ -19,17 +19,17 @@ CREATE OPERATOR <% ( CREATE OPERATOR @#@ ( rightarg = int8, -- left unary - procedure = numeric_fac + procedure = factorial ); CREATE OPERATOR #@# ( leftarg = int8, -- right unary - procedure = numeric_fac + procedure = factorial ); CREATE OPERATOR #%# ( leftarg = int8, -- right unary - procedure = numeric_fac + procedure = factorial ); -- Test operator created above @@ -42,7 +42,7 @@ COMMENT ON OPERATOR ###### (int4, NONE) IS 'bad right unary'; -- => is disallowed now CREATE OPERATOR => ( leftarg = int8, -- right unary - procedure = numeric_fac + procedure = factorial ); -- lexing of <=, >=, <>, != has a number of edge cases @@ -51,7 +51,7 @@ CREATE OPERATOR => ( -- this is legal because ! is not allowed in sql ops CREATE OPERATOR !=- ( leftarg = int8, -- right unary - procedure = numeric_fac + procedure = factorial ); SELECT 2 !=-; -- make sure lexer returns != as <> even in edge cases @@ -85,7 +85,7 @@ REVOKE USAGE ON SCHEMA schema_op1 FROM regress_rol_op1; SET ROLE regress_rol_op1; CREATE OPERATOR schema_op1.#*# ( leftarg = int8, -- right unary - procedure = numeric_fac + procedure = factorial ); ROLLBACK; @@ -94,7 +94,7 @@ ROLLBACK; BEGIN TRANSACTION; CREATE OPERATOR #*# ( leftarg = SETOF int8, - procedure = numeric_fac + procedure = factorial ); ROLLBACK; @@ -103,7 +103,7 @@ ROLLBACK; BEGIN TRANSACTION; CREATE OPERATOR #*# ( rightarg = SETOF int8, - procedure = numeric_fac + procedure = factorial ); ROLLBACK; @@ -129,13 +129,13 @@ ROLLBACK; -- Should fail. Invalid attribute CREATE OPERATOR #@%# ( leftarg = int8, -- right unary - procedure = numeric_fac, + procedure = factorial, invalid_att = int8 ); -- Should fail. At least leftarg or rightarg should be mandatorily specified CREATE OPERATOR #@%# ( - procedure = numeric_fac + procedure = factorial ); -- Should fail. Procedure should be mandatorily specified diff --git a/src/test/regress/sql/numeric.sql b/src/test/regress/sql/numeric.sql index febb096af23b..5eac895e86e1 100644 --- a/src/test/regress/sql/numeric.sql +++ b/src/test/regress/sql/numeric.sql @@ -1300,12 +1300,10 @@ SELECT lcm(9999 * (10::numeric)^131068 + (10::numeric^131068 - 1), 2); -- overfl -- -- Tests for factorial -- -SELECT 4!; -SELECT !!3; +SELECT factorial(4); SELECT factorial(15); -SELECT 100000!; -SELECT 0!; -SELECT -4!; +SELECT factorial(100000); +SELECT factorial(0); SELECT factorial(-4); -- From 1ed6b895634ce0dc5fd4bd040e87252b32182cba Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 17 Sep 2020 19:38:05 -0400 Subject: [PATCH 170/589] Remove support for postfix (right-unary) operators. This feature has been a thorn in our sides for a long time, causing many grammatical ambiguity problems. It doesn't seem worth the pain to continue to support it, so remove it. There are some follow-on improvements we can make in the grammar, but this commit only removes the bare minimum number of productions, plus assorted backend support code. Note that pg_dump and psql continue to have full support, since they may be used against older servers. However, pg_dump warns about postfix operators. There is also a check in pg_upgrade. Documentation-wise, I (tgl) largely removed the "left unary" terminology in favor of saying "prefix operator", which is a more standard and IMO less confusing term. I included a catversion bump, although no initial catalog data changes here, to mark the boundary at which oprkind = 'r' stopped being valid in pg_operator. Mark Dilger, based on work by myself and Robert Haas; review by John Naylor Discussion: https://postgr.es/m/38ca86db-42ab-9b48-2902-337a0d6b8311@2ndquadrant.com --- contrib/postgres_fdw/deparse.c | 19 +-- doc/src/sgml/catalogs.sgml | 8 +- doc/src/sgml/ref/alter_extension.sgml | 2 +- doc/src/sgml/ref/alter_operator.sgml | 9 +- doc/src/sgml/ref/alter_opfamily.sgml | 2 +- doc/src/sgml/ref/comment.sgml | 2 +- doc/src/sgml/ref/create_opclass.sgml | 2 +- doc/src/sgml/ref/create_operator.sgml | 22 +-- doc/src/sgml/ref/drop_operator.sgml | 16 +-- doc/src/sgml/syntax.sgml | 5 +- doc/src/sgml/typeconv.sgml | 6 +- doc/src/sgml/xoper.sgml | 12 +- src/backend/catalog/namespace.c | 7 +- src/backend/catalog/pg_operator.c | 4 +- src/backend/commands/operatorcmds.c | 14 +- src/backend/nodes/print.c | 1 - src/backend/parser/gram.y | 13 +- src/backend/parser/parse_expr.c | 38 ++---- src/backend/parser/parse_oper.c | 128 +++--------------- src/backend/utils/adt/ruleutils.c | 39 +----- src/bin/pg_dump/pg_dump.c | 8 +- src/bin/pg_upgrade/check.c | 106 +++++++++++++++ src/bin/psql/describe.c | 4 + src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_operator.h | 6 +- src/include/parser/parse_oper.h | 2 - src/test/regress/expected/create_operator.out | 50 ++++--- src/test/regress/expected/opr_sanity.out | 17 +-- src/test/regress/sql/create_operator.sql | 36 ++--- src/test/regress/sql/opr_sanity.sql | 14 +- src/tutorial/complex.source | 2 +- src/tutorial/syscat.source | 24 +--- 32 files changed, 280 insertions(+), 340 deletions(-) diff --git a/contrib/postgres_fdw/deparse.c b/contrib/postgres_fdw/deparse.c index ad37a7422133..2d44df19fee0 100644 --- a/contrib/postgres_fdw/deparse.c +++ b/contrib/postgres_fdw/deparse.c @@ -2706,7 +2706,6 @@ deparseOpExpr(OpExpr *node, deparse_expr_cxt *context) HeapTuple tuple; Form_pg_operator form; char oprkind; - ListCell *arg; /* Retrieve information about the operator from system catalog. */ tuple = SearchSysCache1(OPEROID, ObjectIdGetDatum(node->opno)); @@ -2716,18 +2715,16 @@ deparseOpExpr(OpExpr *node, deparse_expr_cxt *context) oprkind = form->oprkind; /* Sanity check. */ - Assert((oprkind == 'r' && list_length(node->args) == 1) || - (oprkind == 'l' && list_length(node->args) == 1) || + Assert((oprkind == 'l' && list_length(node->args) == 1) || (oprkind == 'b' && list_length(node->args) == 2)); /* Always parenthesize the expression. */ appendStringInfoChar(buf, '('); - /* Deparse left operand. */ - if (oprkind == 'r' || oprkind == 'b') + /* Deparse left operand, if any. */ + if (oprkind == 'b') { - arg = list_head(node->args); - deparseExpr(lfirst(arg), context); + deparseExpr(linitial(node->args), context); appendStringInfoChar(buf, ' '); } @@ -2735,12 +2732,8 @@ deparseOpExpr(OpExpr *node, deparse_expr_cxt *context) deparseOperatorName(buf, form); /* Deparse right operand. */ - if (oprkind == 'l' || oprkind == 'b') - { - arg = list_tail(node->args); - appendStringInfoChar(buf, ' '); - deparseExpr(lfirst(arg), context); - } + appendStringInfoChar(buf, ' '); + deparseExpr(llast(node->args), context); appendStringInfoChar(buf, ')'); diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 508bea3bc644..7e99928d0c1c 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -5159,8 +5159,8 @@ SCRAM-SHA-256$<iteration count>:&l oprkind char - b = infix (both), l = prefix - (left), r = postfix (right) + b = infix operator (both), + or l = prefix operator (left) @@ -5188,7 +5188,7 @@ SCRAM-SHA-256$<iteration count>:&l (references pg_type.oid) - Type of the left operand + Type of the left operand (0 if none) @@ -5266,7 +5266,7 @@ SCRAM-SHA-256$<iteration count>:&l
- Unused column contain zeroes. For example, oprleft + Unused columns contain zeroes. For example, oprleft is zero for a prefix operator. diff --git a/doc/src/sgml/ref/alter_extension.sgml b/doc/src/sgml/ref/alter_extension.sgml index a2d405d6cdfb..c819c7bb4e3c 100644 --- a/doc/src/sgml/ref/alter_extension.sgml +++ b/doc/src/sgml/ref/alter_extension.sgml @@ -251,7 +251,7 @@ ALTER EXTENSION name DROP The data type(s) of the operator's arguments (optionally schema-qualified). Write NONE for the missing argument - of a prefix or postfix operator. + of a prefix operator. diff --git a/doc/src/sgml/ref/alter_operator.sgml b/doc/src/sgml/ref/alter_operator.sgml index ac35f229a0ca..ad90c137f149 100644 --- a/doc/src/sgml/ref/alter_operator.sgml +++ b/doc/src/sgml/ref/alter_operator.sgml @@ -21,13 +21,13 @@ PostgreSQL documentation -ALTER OPERATOR name ( { left_type | NONE } , { right_type | NONE } ) +ALTER OPERATOR name ( { left_type | NONE } , right_type ) OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } -ALTER OPERATOR name ( { left_type | NONE } , { right_type | NONE } ) +ALTER OPERATOR name ( { left_type | NONE } , right_type ) SET SCHEMA new_schema -ALTER OPERATOR name ( { left_type | NONE } , { right_type | NONE } ) +ALTER OPERATOR name ( { left_type | NONE } , right_type ) SET ( { RESTRICT = { res_proc | NONE } | JOIN = { join_proc | NONE } } [, ... ] ) @@ -79,8 +79,7 @@ ALTER OPERATOR name ( { left_typeright_type - The data type of the operator's right operand; write - NONE if the operator has no right operand. + The data type of the operator's right operand. diff --git a/doc/src/sgml/ref/alter_opfamily.sgml b/doc/src/sgml/ref/alter_opfamily.sgml index 59d5bf107008..b3b5d61a852e 100644 --- a/doc/src/sgml/ref/alter_opfamily.sgml +++ b/doc/src/sgml/ref/alter_opfamily.sgml @@ -141,7 +141,7 @@ ALTER OPERATOR FAMILY name USING name [ DEFAUL In an OPERATOR clause, the operand data type(s) of the operator, or NONE to - signify a left-unary or right-unary operator. The operand data + signify a prefix operator. The operand data types can be omitted in the normal case where they are the same as the operator class's data type. diff --git a/doc/src/sgml/ref/create_operator.sgml b/doc/src/sgml/ref/create_operator.sgml index 66c34e0072f0..9462bc1e8caa 100644 --- a/doc/src/sgml/ref/create_operator.sgml +++ b/doc/src/sgml/ref/create_operator.sgml @@ -86,20 +86,9 @@ CREATE OPERATOR name ( - At least one of LEFTARG and RIGHTARG must be defined. For - binary operators, both must be defined. For right unary - operators, only LEFTARG should be defined, while for left - unary operators only RIGHTARG should be defined. - - - - - Right unary, also called postfix, operators are deprecated and will be - removed in PostgreSQL version 14. - - - - + For binary operators, both LEFTARG and + RIGHTARG must be defined. For prefix operators only + RIGHTARG should be defined. The function_name function must have been previously defined using CREATE FUNCTION and must be defined to accept the correct number @@ -160,7 +149,7 @@ CREATE OPERATOR name ( The data type of the operator's left operand, if any. - This option would be omitted for a left-unary operator. + This option would be omitted for a prefix operator. @@ -169,8 +158,7 @@ CREATE OPERATOR name ( right_type - The data type of the operator's right operand, if any. - This option would be omitted for a right-unary operator. + The data type of the operator's right operand. diff --git a/doc/src/sgml/ref/drop_operator.sgml b/doc/src/sgml/ref/drop_operator.sgml index 2dff050ecf22..7bcdd082ae70 100644 --- a/doc/src/sgml/ref/drop_operator.sgml +++ b/doc/src/sgml/ref/drop_operator.sgml @@ -21,7 +21,7 @@ PostgreSQL documentation -DROP OPERATOR [ IF EXISTS ] name ( { left_type | NONE } , { right_type | NONE } ) [, ...] [ CASCADE | RESTRICT ] +DROP OPERATOR [ IF EXISTS ] name ( { left_type | NONE } , right_type ) [, ...] [ CASCADE | RESTRICT ] @@ -73,8 +73,7 @@ DROP OPERATOR [ IF EXISTS ] name ( right_type - The data type of the operator's right operand; write - NONE if the operator has no right operand. + The data type of the operator's right operand. @@ -113,24 +112,17 @@ DROP OPERATOR ^ (integer, integer); - Remove the left unary bitwise complement operator + Remove the bitwise-complement prefix operator ~b for type bit: DROP OPERATOR ~ (none, bit); - - Remove the right unary factorial operator x! - for type bigint: - -DROP OPERATOR ! (bigint, none); - - Remove multiple operators in one command: -DROP OPERATOR ~ (none, bit), ! (bigint, none); +DROP OPERATOR ~ (none, bit), ^ (integer, integer); diff --git a/doc/src/sgml/syntax.sgml b/doc/src/sgml/syntax.sgml index b0ae5d2e127e..3fdd87823e00 100644 --- a/doc/src/sgml/syntax.sgml +++ b/doc/src/sgml/syntax.sgml @@ -836,7 +836,7 @@ CAST ( 'string' AS type ) When working with non-SQL-standard operator names, you will usually need to separate adjacent operators with spaces to avoid ambiguity. - For example, if you have defined a left unary operator named @, + For example, if you have defined a prefix operator named @, you cannot write X*@Y; you must write X* @Y to ensure that PostgreSQL reads it as two operator names @@ -1444,11 +1444,10 @@ $1.somecolumn - There are three possible syntaxes for an operator invocation: + There are two possible syntaxes for an operator invocation: expression operator expression (binary infix operator) operator expression (unary prefix operator) - expression operator (unary postfix operator) where the operator token follows the syntax rules of , or is one of the diff --git a/doc/src/sgml/typeconv.sgml b/doc/src/sgml/typeconv.sgml index 98662fc91fb6..cfeb851a507e 100644 --- a/doc/src/sgml/typeconv.sgml +++ b/doc/src/sgml/typeconv.sgml @@ -97,8 +97,8 @@ Operators PostgreSQL allows expressions with -prefix and postfix unary (one-argument) operators, -as well as binary (two-argument) operators. Like functions, operators can +prefix (one-argument) operators, +as well as infix (two-argument) operators. Like functions, operators can be overloaded, so the same problem of selecting the right operator exists. @@ -266,7 +266,7 @@ create objects. In such situations, cast arguments to force an exact match. If one argument of a binary operator invocation is of the unknown type, then assume it is the same type as the other argument for this check. -Invocations involving two unknown inputs, or a unary operator +Invocations involving two unknown inputs, or a prefix operator with an unknown input, will never find a match at this step. diff --git a/doc/src/sgml/xoper.sgml b/doc/src/sgml/xoper.sgml index 56b08491c96c..98f4c5c4aa46 100644 --- a/doc/src/sgml/xoper.sgml +++ b/doc/src/sgml/xoper.sgml @@ -20,8 +20,8 @@ - PostgreSQL supports left unary, right - unary, and binary operators. Operators can be + PostgreSQL supports prefix + and infix operators. Operators can be overloaded;overloadingoperators that is, the same operator name can be used for different operators that have different numbers and types of operands. When a query is @@ -64,9 +64,9 @@ SELECT (a + b) AS c FROM test_complex; - We've shown how to create a binary operator here. To create unary - operators, just omit one of leftarg (for left unary) or - rightarg (for right unary). The function + We've shown how to create a binary operator here. To create a prefix + operator, just omit the leftarg. + The function clause and the argument clauses are the only required items in CREATE OPERATOR. The commutator clause shown in the example is an optional hint to the query @@ -202,7 +202,7 @@ SELECT (a + b) AS c FROM test_complex; Unlike commutators, a pair of unary operators could validly be marked as each other's negators; that would mean (A x) equals NOT (B x) - for all x, or the equivalent for right unary operators. + for all x. diff --git a/src/backend/catalog/namespace.c b/src/backend/catalog/namespace.c index 0152e3869abe..391a9b225db7 100644 --- a/src/backend/catalog/namespace.c +++ b/src/backend/catalog/namespace.c @@ -1473,8 +1473,7 @@ FunctionIsVisible(Oid funcid) * Given a possibly-qualified operator name and exact input datatypes, * look up the operator. Returns InvalidOid if not found. * - * Pass oprleft = InvalidOid for a prefix op, oprright = InvalidOid for - * a postfix op. + * Pass oprleft = InvalidOid for a prefix op. * * If the operator name is not schema-qualified, it is sought in the current * namespace search path. If the name is schema-qualified and the given @@ -1580,8 +1579,8 @@ OpernameGetOprid(List *names, Oid oprleft, Oid oprright) * namespace case, we arrange for entries in earlier namespaces to mask * identical entries in later namespaces. * - * The returned items always have two args[] entries --- one or the other - * will be InvalidOid for a prefix or postfix oprkind. nargs is 2, too. + * The returned items always have two args[] entries --- the first will be + * InvalidOid for a prefix oprkind. nargs is always 2, too. */ FuncCandidateList OpernameGetCandidates(List *names, char oprkind, bool missing_schema_ok) diff --git a/src/backend/catalog/pg_operator.c b/src/backend/catalog/pg_operator.c index f7c07c9b5b89..904cb8ef820c 100644 --- a/src/backend/catalog/pg_operator.c +++ b/src/backend/catalog/pg_operator.c @@ -245,7 +245,7 @@ OperatorShellMake(const char *operatorName, values[Anum_pg_operator_oprname - 1] = NameGetDatum(&oname); values[Anum_pg_operator_oprnamespace - 1] = ObjectIdGetDatum(operatorNamespace); values[Anum_pg_operator_oprowner - 1] = ObjectIdGetDatum(GetUserId()); - values[Anum_pg_operator_oprkind - 1] = CharGetDatum(leftTypeId ? (rightTypeId ? 'b' : 'r') : 'l'); + values[Anum_pg_operator_oprkind - 1] = CharGetDatum(leftTypeId ? 'b' : 'l'); values[Anum_pg_operator_oprcanmerge - 1] = BoolGetDatum(false); values[Anum_pg_operator_oprcanhash - 1] = BoolGetDatum(false); values[Anum_pg_operator_oprleft - 1] = ObjectIdGetDatum(leftTypeId); @@ -494,7 +494,7 @@ OperatorCreate(const char *operatorName, values[Anum_pg_operator_oprname - 1] = NameGetDatum(&oname); values[Anum_pg_operator_oprnamespace - 1] = ObjectIdGetDatum(operatorNamespace); values[Anum_pg_operator_oprowner - 1] = ObjectIdGetDatum(GetUserId()); - values[Anum_pg_operator_oprkind - 1] = CharGetDatum(leftTypeId ? (rightTypeId ? 'b' : 'r') : 'l'); + values[Anum_pg_operator_oprkind - 1] = CharGetDatum(leftTypeId ? 'b' : 'l'); values[Anum_pg_operator_oprcanmerge - 1] = BoolGetDatum(canMerge); values[Anum_pg_operator_oprcanhash - 1] = BoolGetDatum(canHash); values[Anum_pg_operator_oprleft - 1] = ObjectIdGetDatum(leftTypeId); diff --git a/src/backend/commands/operatorcmds.c b/src/backend/commands/operatorcmds.c index bf23937849c9..a791e99092d5 100644 --- a/src/backend/commands/operatorcmds.c +++ b/src/backend/commands/operatorcmds.c @@ -168,10 +168,22 @@ DefineOperator(List *names, List *parameters) if (typeName2) typeId2 = typenameTypeId(NULL, typeName2); + /* + * If only the right argument is missing, the user is likely trying to + * create a postfix operator, so give them a hint about why that does not + * work. But if both arguments are missing, do not mention postfix + * operators, as the user most likely simply neglected to mention the + * arguments. + */ if (!OidIsValid(typeId1) && !OidIsValid(typeId2)) ereport(ERROR, (errcode(ERRCODE_INVALID_FUNCTION_DEFINITION), - errmsg("at least one of leftarg or rightarg must be specified"))); + errmsg("operator argument types must be specified"))); + if (!OidIsValid(typeId2)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_FUNCTION_DEFINITION), + errmsg("operator right argument type must be specified"), + errdetail("Postfix operators are not supported."))); if (typeName1) { diff --git a/src/backend/nodes/print.c b/src/backend/nodes/print.c index 42476724d88f..970a2d438402 100644 --- a/src/backend/nodes/print.c +++ b/src/backend/nodes/print.c @@ -394,7 +394,6 @@ print_expr(const Node *expr, const List *rtable) } else { - /* we print prefix and postfix ops the same... */ printf("%s ", ((opname != NULL) ? opname : "(invalid operator)")); print_expr(get_leftop((const Expr *) e), rtable); } diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 4558c02f6a61..b16ffb9bf7fd 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -741,10 +741,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %nonassoc '<' '>' '=' LESS_EQUALS GREATER_EQUALS NOT_EQUALS %nonassoc BETWEEN IN_P LIKE ILIKE SIMILAR NOT_LA %nonassoc ESCAPE /* ESCAPE must be just above LIKE/ILIKE/SIMILAR */ -%left POSTFIXOP /* dummy for postfix Op rules */ /* * To support target_el without AS, we must give IDENT an explicit priority - * between POSTFIXOP and Op. We can safely assign the same priority to + * between ESCAPE and Op. We can safely assign the same priority to * various unreserved keywords as needed to resolve ambiguities (this can't * have any bad effects since obviously the keywords will still behave the * same as if they weren't keywords). We need to do this: @@ -12993,8 +12992,6 @@ a_expr: c_expr { $$ = $1; } { $$ = (Node *) makeA_Expr(AEXPR_OP, $2, $1, $3, @2); } | qual_Op a_expr %prec Op { $$ = (Node *) makeA_Expr(AEXPR_OP, $1, NULL, $2, @1); } - | a_expr qual_Op %prec POSTFIXOP - { $$ = (Node *) makeA_Expr(AEXPR_OP, $2, $1, NULL, @2); } | a_expr AND a_expr { $$ = makeAndExpr($1, $3, @2); } @@ -13408,8 +13405,6 @@ b_expr: c_expr { $$ = (Node *) makeA_Expr(AEXPR_OP, $2, $1, $3, @2); } | qual_Op b_expr %prec Op { $$ = (Node *) makeA_Expr(AEXPR_OP, $1, NULL, $2, @1); } - | b_expr qual_Op %prec POSTFIXOP - { $$ = (Node *) makeA_Expr(AEXPR_OP, $2, $1, NULL, @2); } | b_expr IS DISTINCT FROM b_expr %prec IS { $$ = (Node *) makeSimpleA_Expr(AEXPR_DISTINCT, "=", $1, $5, @2); @@ -14665,11 +14660,7 @@ target_el: a_expr AS ColLabel } /* * We support omitting AS only for column labels that aren't - * any known keyword. There is an ambiguity against postfix - * operators: is "a ! b" an infix expression, or a postfix - * expression and a column label? We prefer to resolve this - * as an infix expression, which we accomplish by assigning - * IDENT a precedence higher than POSTFIXOP. + * any known keyword. */ | a_expr IDENT { diff --git a/src/backend/parser/parse_expr.c b/src/backend/parser/parse_expr.c index f69976cc8c98..d24420c58319 100644 --- a/src/backend/parser/parse_expr.c +++ b/src/backend/parser/parse_expr.c @@ -57,7 +57,7 @@ bool Transform_null_equals = false; #define PREC_GROUP_NOT_LIKE 9 /* NOT LIKE/ILIKE/SIMILAR */ #define PREC_GROUP_NOT_BETWEEN 10 /* NOT BETWEEN */ #define PREC_GROUP_NOT_IN 11 /* NOT IN */ -#define PREC_GROUP_POSTFIX_OP 12 /* generic postfix operators */ +#define PREC_GROUP_ANY_ALL 12 /* ANY/ALL */ #define PREC_GROUP_INFIX_OP 13 /* generic infix operators */ #define PREC_GROUP_PREFIX_OP 14 /* generic prefix operators */ @@ -71,7 +71,7 @@ bool Transform_null_equals = false; * 4. LIKE ILIKE SIMILAR * 5. BETWEEN * 6. IN - * 7. generic postfix Op + * 7. ANY ALL * 8. generic Op, including <= => <> * 9. generic prefix Op * 10. IS tests (NullTest, BooleanTest, etc) @@ -1031,7 +1031,7 @@ transformAExprOpAny(ParseState *pstate, A_Expr *a) Node *rexpr = a->rexpr; if (operator_precedence_warning) - emit_precedence_warnings(pstate, PREC_GROUP_POSTFIX_OP, + emit_precedence_warnings(pstate, PREC_GROUP_ANY_ALL, strVal(llast(a->name)), lexpr, NULL, a->location); @@ -1054,7 +1054,7 @@ transformAExprOpAll(ParseState *pstate, A_Expr *a) Node *rexpr = a->rexpr; if (operator_precedence_warning) - emit_precedence_warnings(pstate, PREC_GROUP_POSTFIX_OP, + emit_precedence_warnings(pstate, PREC_GROUP_ANY_ALL, strVal(llast(a->name)), lexpr, NULL, a->location); @@ -2019,7 +2019,7 @@ transformSubLink(ParseState *pstate, SubLink *sublink) sublink->testexpr, NULL, sublink->location); else - emit_precedence_warnings(pstate, PREC_GROUP_POSTFIX_OP, + emit_precedence_warnings(pstate, PREC_GROUP_ANY_ALL, strVal(llast(sublink->operName)), sublink->testexpr, NULL, sublink->location); @@ -3244,28 +3244,11 @@ operator_precedence_group(Node *node, const char **nodename) group = PREC_GROUP_PREFIX_OP; } } - else if (aexpr->kind == AEXPR_OP && - aexpr->lexpr != NULL && - aexpr->rexpr == NULL) - { - /* postfix operator */ - if (list_length(aexpr->name) == 1) - { - *nodename = strVal(linitial(aexpr->name)); - group = PREC_GROUP_POSTFIX_OP; - } - else - { - /* schema-qualified operator syntax */ - *nodename = "OPERATOR()"; - group = PREC_GROUP_POSTFIX_OP; - } - } else if (aexpr->kind == AEXPR_OP_ANY || aexpr->kind == AEXPR_OP_ALL) { *nodename = strVal(llast(aexpr->name)); - group = PREC_GROUP_POSTFIX_OP; + group = PREC_GROUP_ANY_ALL; } else if (aexpr->kind == AEXPR_DISTINCT || aexpr->kind == AEXPR_NOT_DISTINCT) @@ -3356,7 +3339,7 @@ operator_precedence_group(Node *node, const char **nodename) else { *nodename = strVal(llast(s->operName)); - group = PREC_GROUP_POSTFIX_OP; + group = PREC_GROUP_ANY_ALL; } } } @@ -3432,9 +3415,8 @@ emit_precedence_warnings(ParseState *pstate, * Complain if left child, which should be same or higher precedence * according to current rules, used to be lower precedence. * - * Exception to precedence rules: if left child is IN or NOT IN or a - * postfix operator, the grouping is syntactically forced regardless of - * precedence. + * Exception to precedence rules: if left child is IN or NOT IN the + * grouping is syntactically forced regardless of precedence. */ cgroup = operator_precedence_group(lchild, &copname); if (cgroup > 0) @@ -3442,7 +3424,7 @@ emit_precedence_warnings(ParseState *pstate, if (oldprecedence_l[cgroup] < oldprecedence_r[opgroup] && cgroup != PREC_GROUP_IN && cgroup != PREC_GROUP_NOT_IN && - cgroup != PREC_GROUP_POSTFIX_OP && + cgroup != PREC_GROUP_ANY_ALL && cgroup != PREC_GROUP_POSTFIX_IS) ereport(WARNING, (errmsg("operator precedence change: %s is now lower precedence than %s", diff --git a/src/backend/parser/parse_oper.c b/src/backend/parser/parse_oper.c index 2749974f6384..6613a3a8f879 100644 --- a/src/backend/parser/parse_oper.c +++ b/src/backend/parser/parse_oper.c @@ -52,7 +52,7 @@ typedef struct OprCacheKey { char oprname[NAMEDATALEN]; Oid left_arg; /* Left input OID, or 0 if prefix op */ - Oid right_arg; /* Right input OID, or 0 if postfix op */ + Oid right_arg; /* Right input OID */ Oid search_path[MAX_CACHED_PATH_LEN]; } OprCacheKey; @@ -88,8 +88,7 @@ static void InvalidateOprCacheCallBack(Datum arg, int cacheid, uint32 hashvalue) * Given a possibly-qualified operator name and exact input datatypes, * look up the operator. * - * Pass oprleft = InvalidOid for a prefix op, oprright = InvalidOid for - * a postfix op. + * Pass oprleft = InvalidOid for a prefix op. * * If the operator name is not schema-qualified, it is sought in the current * namespace search path. @@ -115,10 +114,16 @@ LookupOperName(ParseState *pstate, List *opername, Oid oprleft, Oid oprright, if (!OidIsValid(oprleft)) oprkind = 'l'; - else if (!OidIsValid(oprright)) - oprkind = 'r'; - else + else if (OidIsValid(oprright)) oprkind = 'b'; + else + { + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("postfix operators are not supported"), + parser_errposition(pstate, location))); + oprkind = 0; /* keep compiler quiet */ + } ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), @@ -507,85 +512,6 @@ compatible_oper_opid(List *op, Oid arg1, Oid arg2, bool noError) } -/* right_oper() -- search for a unary right operator (postfix operator) - * Given operator name and type of arg, return oper struct. - * - * IMPORTANT: the returned operator (if any) is only promised to be - * coercion-compatible with the input datatype. Do not use this if - * you need an exact- or binary-compatible match. - * - * If no matching operator found, return NULL if noError is true, - * raise an error if it is false. pstate and location are used only to report - * the error position; pass NULL/-1 if not available. - * - * NOTE: on success, the returned object is a syscache entry. The caller - * must ReleaseSysCache() the entry when done with it. - */ -Operator -right_oper(ParseState *pstate, List *op, Oid arg, bool noError, int location) -{ - Oid operOid; - OprCacheKey key; - bool key_ok; - FuncDetailCode fdresult = FUNCDETAIL_NOTFOUND; - HeapTuple tup = NULL; - - /* - * Try to find the mapping in the lookaside cache. - */ - key_ok = make_oper_cache_key(pstate, &key, op, arg, InvalidOid, location); - - if (key_ok) - { - operOid = find_oper_cache_entry(&key); - if (OidIsValid(operOid)) - { - tup = SearchSysCache1(OPEROID, ObjectIdGetDatum(operOid)); - if (HeapTupleIsValid(tup)) - return (Operator) tup; - } - } - - /* - * First try for an "exact" match. - */ - operOid = OpernameGetOprid(op, arg, InvalidOid); - if (!OidIsValid(operOid)) - { - /* - * Otherwise, search for the most suitable candidate. - */ - FuncCandidateList clist; - - /* Get postfix operators of given name */ - clist = OpernameGetCandidates(op, 'r', false); - - /* No operators found? Then fail... */ - if (clist != NULL) - { - /* - * We must run oper_select_candidate even if only one candidate, - * otherwise we may falsely return a non-type-compatible operator. - */ - fdresult = oper_select_candidate(1, &arg, clist, &operOid); - } - } - - if (OidIsValid(operOid)) - tup = SearchSysCache1(OPEROID, ObjectIdGetDatum(operOid)); - - if (HeapTupleIsValid(tup)) - { - if (key_ok) - make_oper_cache_entry(&key, operOid); - } - else if (!noError) - op_error(pstate, op, 'r', arg, InvalidOid, fdresult, location); - - return (Operator) tup; -} - - /* left_oper() -- search for a unary left operator (prefix operator) * Given operator name and type of arg, return oper struct. * @@ -696,8 +622,7 @@ op_signature_string(List *op, char oprkind, Oid arg1, Oid arg2) appendStringInfoString(&argbuf, NameListToString(op)); - if (oprkind != 'r') - appendStringInfo(&argbuf, " %s", format_type_be(arg2)); + appendStringInfo(&argbuf, " %s", format_type_be(arg2)); return argbuf.data; /* return palloc'd string buffer */ } @@ -758,17 +683,16 @@ make_op(ParseState *pstate, List *opname, Node *ltree, Node *rtree, Oid rettype; OpExpr *result; - /* Select the operator */ + /* Check it's not a postfix operator */ if (rtree == NULL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("postfix operators are not supported"))); + + /* Select the operator */ + if (ltree == NULL) { - /* right operator */ - ltypeId = exprType(ltree); - rtypeId = InvalidOid; - tup = right_oper(pstate, opname, ltypeId, false, location); - } - else if (ltree == NULL) - { - /* left operator */ + /* prefix operator */ rtypeId = exprType(rtree); ltypeId = InvalidOid; tup = left_oper(pstate, opname, rtypeId, false, location); @@ -795,17 +719,9 @@ make_op(ParseState *pstate, List *opname, Node *ltree, Node *rtree, parser_errposition(pstate, location))); /* Do typecasting and build the expression tree */ - if (rtree == NULL) - { - /* right operator */ - args = list_make1(ltree); - actual_arg_types[0] = ltypeId; - declared_arg_types[0] = opform->oprleft; - nargs = 1; - } - else if (ltree == NULL) + if (ltree == NULL) { - /* left operator */ + /* prefix operator */ args = list_make1(rtree); actual_arg_types[0] = rtypeId; declared_arg_types[0] = opform->oprright; diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index 60dd80c23c87..15877e37a609 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -9198,35 +9198,14 @@ get_oper_expr(OpExpr *expr, deparse_context *context) } else { - /* unary operator --- but which side? */ + /* prefix operator */ Node *arg = (Node *) linitial(args); - HeapTuple tp; - Form_pg_operator optup; - - tp = SearchSysCache1(OPEROID, ObjectIdGetDatum(opno)); - if (!HeapTupleIsValid(tp)) - elog(ERROR, "cache lookup failed for operator %u", opno); - optup = (Form_pg_operator) GETSTRUCT(tp); - switch (optup->oprkind) - { - case 'l': - appendStringInfo(buf, "%s ", - generate_operator_name(opno, - InvalidOid, - exprType(arg))); - get_rule_expr_paren(arg, context, true, (Node *) expr); - break; - case 'r': - get_rule_expr_paren(arg, context, true, (Node *) expr); - appendStringInfo(buf, " %s", - generate_operator_name(opno, - exprType(arg), - InvalidOid)); - break; - default: - elog(ERROR, "bogus oprkind: %d", optup->oprkind); - } - ReleaseSysCache(tp); + + appendStringInfo(buf, "%s ", + generate_operator_name(opno, + InvalidOid, + exprType(arg))); + get_rule_expr_paren(arg, context, true, (Node *) expr); } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); @@ -11087,10 +11066,6 @@ generate_operator_name(Oid operid, Oid arg1, Oid arg2) p_result = left_oper(NULL, list_make1(makeString(oprname)), arg2, true, -1); break; - case 'r': - p_result = right_oper(NULL, list_make1(makeString(oprname)), arg1, - true, -1); - break; default: elog(ERROR, "unrecognized oprkind: %d", operform->oprkind); p_result = NULL; /* keep compiler quiet */ diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 320820165b7d..dce6af09c9b7 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -12650,6 +12650,11 @@ dumpOpr(Archive *fout, OprInfo *oprinfo) oprcanmerge = PQgetvalue(res, 0, i_oprcanmerge); oprcanhash = PQgetvalue(res, 0, i_oprcanhash); + /* In PG14 upwards postfix operator support does not exist anymore. */ + if (strcmp(oprkind, "r") == 0) + pg_log_warning("postfix operators are not supported anymore (operator \"%s\")", + oprcode); + oprregproc = convertRegProcReference(fout, oprcode); if (oprregproc) { @@ -12662,7 +12667,8 @@ dumpOpr(Archive *fout, OprInfo *oprinfo) /* * right unary means there's a left arg and left unary means there's a - * right arg + * right arg. (Although the "r" case is dead code for PG14 and later, + * continue to support it in case we're dumping from an old server.) */ if (strcmp(oprkind, "r") == 0 || strcmp(oprkind, "b") == 0) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 00aef855dc07..2f7aa632c52b 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -22,6 +22,7 @@ static void check_is_install_user(ClusterInfo *cluster); static void check_proper_datallowconn(ClusterInfo *cluster); static void check_for_prepared_transactions(ClusterInfo *cluster); static void check_for_isn_and_int8_passing_mismatch(ClusterInfo *cluster); +static void check_for_user_defined_postfix_ops(ClusterInfo *cluster); static void check_for_tables_with_oids(ClusterInfo *cluster); static void check_for_reg_data_type_usage(ClusterInfo *cluster); static void check_for_jsonb_9_4_usage(ClusterInfo *cluster); @@ -100,6 +101,13 @@ check_and_dump_old_cluster(bool live_check) check_for_reg_data_type_usage(&old_cluster); check_for_isn_and_int8_passing_mismatch(&old_cluster); + /* + * Pre-PG 14 allowed user defined postfix operators, which are not + * supported anymore. Verify there are none, iff applicable. + */ + if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1300) + check_for_user_defined_postfix_ops(&old_cluster); + /* * Pre-PG 12 allowed tables to be declared WITH OIDS, which is not * supported anymore. Verify there are none, iff applicable. @@ -896,6 +904,104 @@ check_for_isn_and_int8_passing_mismatch(ClusterInfo *cluster) check_ok(); } +/* + * Verify that no user defined postfix operators exist. + */ +static void +check_for_user_defined_postfix_ops(ClusterInfo *cluster) +{ + int dbnum; + FILE *script = NULL; + bool found = false; + char output_path[MAXPGPATH]; + + prep_status("Checking for user-defined postfix operators"); + + snprintf(output_path, sizeof(output_path), + "postfix_ops.txt"); + + /* Find any user defined postfix operators */ + for (dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) + { + PGresult *res; + bool db_used = false; + int ntups; + int rowno; + int i_oproid, + i_oprnsp, + i_oprname, + i_typnsp, + i_typname; + DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; + PGconn *conn = connectToServer(cluster, active_db->db_name); + + /* + * The query below hardcodes FirstNormalObjectId as 16384 rather than + * interpolating that C #define into the query because, if that + * #define is ever changed, the cutoff we want to use is the value + * used by pre-version 14 servers, not that of some future version. + */ + res = executeQueryOrDie(conn, + "SELECT o.oid AS oproid, " + " n.nspname AS oprnsp, " + " o.oprname, " + " tn.nspname AS typnsp, " + " t.typname " + "FROM pg_catalog.pg_operator o, " + " pg_catalog.pg_namespace n, " + " pg_catalog.pg_type t, " + " pg_catalog.pg_namespace tn " + "WHERE o.oprnamespace = n.oid AND " + " o.oprleft = t.oid AND " + " t.typnamespace = tn.oid AND " + " o.oprright = 0 AND " + " o.oid >= 16384"); + ntups = PQntuples(res); + i_oproid = PQfnumber(res, "oproid"); + i_oprnsp = PQfnumber(res, "oprnsp"); + i_oprname = PQfnumber(res, "oprname"); + i_typnsp = PQfnumber(res, "typnsp"); + i_typname = PQfnumber(res, "typname"); + for (rowno = 0; rowno < ntups; rowno++) + { + found = true; + if (script == NULL && + (script = fopen_priv(output_path, "w")) == NULL) + pg_fatal("could not open file \"%s\": %s\n", + output_path, strerror(errno)); + if (!db_used) + { + fprintf(script, "In database: %s\n", active_db->db_name); + db_used = true; + } + fprintf(script, " (oid=%s) %s.%s (%s.%s, NONE)\n", + PQgetvalue(res, rowno, i_oproid), + PQgetvalue(res, rowno, i_oprnsp), + PQgetvalue(res, rowno, i_oprname), + PQgetvalue(res, rowno, i_typnsp), + PQgetvalue(res, rowno, i_typname)); + } + + PQclear(res); + + PQfinish(conn); + } + + if (script) + fclose(script); + + if (found) + { + pg_log(PG_REPORT, "fatal\n"); + pg_fatal("Your installation contains user-defined postfix operators, which are not\n" + "supported anymore. Consider dropping the postfix operators and replacing\n" + "them with prefix operators or function calls.\n" + "A list of user-defined postfix operators is in the file:\n" + " %s\n\n", output_path); + } + else + check_ok(); +} /* * Verify that no tables are declared WITH OIDS. diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index f22d907b1f9f..58de433fd302 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -799,6 +799,10 @@ describeOperators(const char *pattern, bool verbose, bool showSystem) * anyway, for now, because (1) third-party modules may still be following * the old convention, and (2) we'd need to do it anyway when talking to a * pre-9.1 server. + * + * The support for postfix operators in this query is dead code as of + * Postgres 14, but we need to keep it for as long as we support talking + * to pre-v14 servers. */ printfPQExpBuffer(&buf, diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 358935997087..365552635b84 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 202009171 +#define CATALOG_VERSION_NO 202009172 #endif diff --git a/src/include/catalog/pg_operator.h b/src/include/catalog/pg_operator.h index 1daa2638520f..62a7dbf23f69 100644 --- a/src/include/catalog/pg_operator.h +++ b/src/include/catalog/pg_operator.h @@ -41,7 +41,7 @@ CATALOG(pg_operator,2617,OperatorRelationId) /* operator owner */ Oid oprowner BKI_DEFAULT(PGUID); - /* 'l', 'r', or 'b' */ + /* 'l' for prefix or 'b' for infix */ char oprkind BKI_DEFAULT(b); /* can be used in merge join? */ @@ -50,10 +50,10 @@ CATALOG(pg_operator,2617,OperatorRelationId) /* can be used in hash join? */ bool oprcanhash BKI_DEFAULT(f); - /* left arg type, or 0 if 'l' oprkind */ + /* left arg type, or 0 if prefix operator */ Oid oprleft BKI_LOOKUP(pg_type); - /* right arg type, or 0 if 'r' oprkind */ + /* right arg type */ Oid oprright BKI_LOOKUP(pg_type); /* result datatype */ diff --git a/src/include/parser/parse_oper.h b/src/include/parser/parse_oper.h index bcd861e43ac3..09695a2765cf 100644 --- a/src/include/parser/parse_oper.h +++ b/src/include/parser/parse_oper.h @@ -31,8 +31,6 @@ extern Oid LookupOperWithArgs(ObjectWithArgs *oper, bool noError); /* NB: the selected operator may require coercion of the input types! */ extern Operator oper(ParseState *pstate, List *op, Oid arg1, Oid arg2, bool noError, int location); -extern Operator right_oper(ParseState *pstate, List *op, Oid arg, - bool noError, int location); extern Operator left_oper(ParseState *pstate, List *op, Oid arg, bool noError, int location); diff --git a/src/test/regress/expected/create_operator.out b/src/test/regress/expected/create_operator.out index 9e4d4e93fb79..530327759140 100644 --- a/src/test/regress/expected/create_operator.out +++ b/src/test/regress/expected/create_operator.out @@ -15,17 +15,15 @@ CREATE OPERATOR <% ( negator = >=% ); CREATE OPERATOR @#@ ( - rightarg = int8, -- left unary - procedure = factorial -); -CREATE OPERATOR #@# ( - leftarg = int8, -- right unary + rightarg = int8, -- prefix procedure = factorial ); CREATE OPERATOR #%# ( - leftarg = int8, -- right unary + leftarg = int8, -- fail, postfix is no longer supported procedure = factorial ); +ERROR: operator right argument type must be specified +DETAIL: Postfix operators are not supported. -- Test operator created above SELECT point '(1,2)' <% widget '(0,0,3)' AS t, point '(1,2)' <% widget '(0,0,1)' AS f; @@ -35,11 +33,22 @@ SELECT point '(1,2)' <% widget '(0,0,3)' AS t, (1 row) -- Test comments -COMMENT ON OPERATOR ###### (int4, NONE) IS 'bad right unary'; -ERROR: operator does not exist: integer ###### --- => is disallowed now +COMMENT ON OPERATOR ###### (NONE, int4) IS 'bad prefix'; +ERROR: operator does not exist: ###### integer +COMMENT ON OPERATOR ###### (int4, NONE) IS 'bad postfix'; +ERROR: postfix operators are not supported +COMMENT ON OPERATOR ###### (int4, int8) IS 'bad infix'; +ERROR: operator does not exist: integer ###### bigint +-- Check that DROP on a nonexistent op behaves sanely, too +DROP OPERATOR ###### (NONE, int4); +ERROR: operator does not exist: ###### integer +DROP OPERATOR ###### (int4, NONE); +ERROR: postfix operators are not supported +DROP OPERATOR ###### (int4, int8); +ERROR: operator does not exist: integer ###### bigint +-- => is disallowed as an operator name now CREATE OPERATOR => ( - leftarg = int8, -- right unary + rightarg = int8, procedure = factorial ); ERROR: syntax error at or near "=>" @@ -49,15 +58,20 @@ LINE 1: CREATE OPERATOR => ( -- (=> is tested elsewhere) -- this is legal because ! is not allowed in sql ops CREATE OPERATOR !=- ( - leftarg = int8, -- right unary + rightarg = int8, procedure = factorial ); -SELECT 2 !=-; +SELECT !=- 10; ?column? ---------- - 2 + 3628800 (1 row) +-- postfix operators don't work anymore +SELECT 10 !=-; +ERROR: syntax error at or near ";" +LINE 1: SELECT 10 !=-; + ^ -- make sure lexer returns != as <> even in edge cases SELECT 2 !=/**/ 1, 2 !=/**/ 2; ?column? | ?column? @@ -127,7 +141,7 @@ GRANT USAGE ON SCHEMA schema_op1 TO PUBLIC; REVOKE USAGE ON SCHEMA schema_op1 FROM regress_rol_op1; SET ROLE regress_rol_op1; CREATE OPERATOR schema_op1.#*# ( - leftarg = int8, -- right unary + rightarg = int8, procedure = factorial ); ERROR: permission denied for schema schema_op1 @@ -167,19 +181,19 @@ CREATE OPERATOR === ( ROLLBACK; -- Should fail. Invalid attribute CREATE OPERATOR #@%# ( - leftarg = int8, -- right unary + rightarg = int8, procedure = factorial, invalid_att = int8 ); WARNING: operator attribute "invalid_att" not recognized --- Should fail. At least leftarg or rightarg should be mandatorily specified +-- Should fail. At least rightarg should be mandatorily specified CREATE OPERATOR #@%# ( procedure = factorial ); -ERROR: at least one of leftarg or rightarg must be specified +ERROR: operator argument types must be specified -- Should fail. Procedure should be mandatorily specified CREATE OPERATOR #@%# ( - leftarg = int8 + rightarg = int8 ); ERROR: operator function must be specified -- Should fail. CREATE OPERATOR requires USAGE on TYPE diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index 1b3c146e4cc9..7825a765cd7b 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -1066,7 +1066,7 @@ WHERE condefault AND -- Look for illegal values in pg_operator fields. SELECT p1.oid, p1.oprname FROM pg_operator as p1 -WHERE (p1.oprkind != 'b' AND p1.oprkind != 'l' AND p1.oprkind != 'r') OR +WHERE (p1.oprkind != 'b' AND p1.oprkind != 'l') OR p1.oprresult = 0 OR p1.oprcode = 0; oid | oprname -----+--------- @@ -1077,8 +1077,7 @@ SELECT p1.oid, p1.oprname FROM pg_operator as p1 WHERE (p1.oprleft = 0 and p1.oprkind != 'l') OR (p1.oprleft != 0 and p1.oprkind = 'l') OR - (p1.oprright = 0 and p1.oprkind != 'r') OR - (p1.oprright != 0 and p1.oprkind = 'r'); + p1.oprright = 0; oid | oprname -----+--------- (0 rows) @@ -1285,18 +1284,6 @@ WHERE p1.oprcode = p2.oid AND -----+---------+-----+--------- (0 rows) -SELECT p1.oid, p1.oprname, p2.oid, p2.proname -FROM pg_operator AS p1, pg_proc AS p2 -WHERE p1.oprcode = p2.oid AND - p1.oprkind = 'r' AND - (p2.pronargs != 1 - OR NOT binary_coercible(p2.prorettype, p1.oprresult) - OR NOT binary_coercible(p1.oprleft, p2.proargtypes[0]) - OR p1.oprright != 0); - oid | oprname | oid | proname ------+---------+-----+--------- -(0 rows) - -- If the operator is mergejoinable or hashjoinable, its underlying function -- should not be volatile. SELECT p1.oid, p1.oprname, p2.oid, p2.proname diff --git a/src/test/regress/sql/create_operator.sql b/src/test/regress/sql/create_operator.sql index c32da8c066a7..4ff2c0ff2167 100644 --- a/src/test/regress/sql/create_operator.sql +++ b/src/test/regress/sql/create_operator.sql @@ -18,17 +18,12 @@ CREATE OPERATOR <% ( ); CREATE OPERATOR @#@ ( - rightarg = int8, -- left unary - procedure = factorial -); - -CREATE OPERATOR #@# ( - leftarg = int8, -- right unary + rightarg = int8, -- prefix procedure = factorial ); CREATE OPERATOR #%# ( - leftarg = int8, -- right unary + leftarg = int8, -- fail, postfix is no longer supported procedure = factorial ); @@ -37,11 +32,18 @@ SELECT point '(1,2)' <% widget '(0,0,3)' AS t, point '(1,2)' <% widget '(0,0,1)' AS f; -- Test comments -COMMENT ON OPERATOR ###### (int4, NONE) IS 'bad right unary'; +COMMENT ON OPERATOR ###### (NONE, int4) IS 'bad prefix'; +COMMENT ON OPERATOR ###### (int4, NONE) IS 'bad postfix'; +COMMENT ON OPERATOR ###### (int4, int8) IS 'bad infix'; + +-- Check that DROP on a nonexistent op behaves sanely, too +DROP OPERATOR ###### (NONE, int4); +DROP OPERATOR ###### (int4, NONE); +DROP OPERATOR ###### (int4, int8); --- => is disallowed now +-- => is disallowed as an operator name now CREATE OPERATOR => ( - leftarg = int8, -- right unary + rightarg = int8, procedure = factorial ); @@ -50,10 +52,12 @@ CREATE OPERATOR => ( -- this is legal because ! is not allowed in sql ops CREATE OPERATOR !=- ( - leftarg = int8, -- right unary + rightarg = int8, procedure = factorial ); -SELECT 2 !=-; +SELECT !=- 10; +-- postfix operators don't work anymore +SELECT 10 !=-; -- make sure lexer returns != as <> even in edge cases SELECT 2 !=/**/ 1, 2 !=/**/ 2; SELECT 2 !=-- comment to be removed by psql @@ -84,7 +88,7 @@ GRANT USAGE ON SCHEMA schema_op1 TO PUBLIC; REVOKE USAGE ON SCHEMA schema_op1 FROM regress_rol_op1; SET ROLE regress_rol_op1; CREATE OPERATOR schema_op1.#*# ( - leftarg = int8, -- right unary + rightarg = int8, procedure = factorial ); ROLLBACK; @@ -128,19 +132,19 @@ ROLLBACK; -- Should fail. Invalid attribute CREATE OPERATOR #@%# ( - leftarg = int8, -- right unary + rightarg = int8, procedure = factorial, invalid_att = int8 ); --- Should fail. At least leftarg or rightarg should be mandatorily specified +-- Should fail. At least rightarg should be mandatorily specified CREATE OPERATOR #@%# ( procedure = factorial ); -- Should fail. Procedure should be mandatorily specified CREATE OPERATOR #@%# ( - leftarg = int8 + rightarg = int8 ); -- Should fail. CREATE OPERATOR requires USAGE on TYPE diff --git a/src/test/regress/sql/opr_sanity.sql b/src/test/regress/sql/opr_sanity.sql index 7a9180b08152..307aab1deb76 100644 --- a/src/test/regress/sql/opr_sanity.sql +++ b/src/test/regress/sql/opr_sanity.sql @@ -571,7 +571,7 @@ WHERE condefault AND SELECT p1.oid, p1.oprname FROM pg_operator as p1 -WHERE (p1.oprkind != 'b' AND p1.oprkind != 'l' AND p1.oprkind != 'r') OR +WHERE (p1.oprkind != 'b' AND p1.oprkind != 'l') OR p1.oprresult = 0 OR p1.oprcode = 0; -- Look for missing or unwanted operand types @@ -580,8 +580,7 @@ SELECT p1.oid, p1.oprname FROM pg_operator as p1 WHERE (p1.oprleft = 0 and p1.oprkind != 'l') OR (p1.oprleft != 0 and p1.oprkind = 'l') OR - (p1.oprright = 0 and p1.oprkind != 'r') OR - (p1.oprright != 0 and p1.oprkind = 'r'); + p1.oprright = 0; -- Look for conflicting operator definitions (same names and input datatypes). @@ -715,15 +714,6 @@ WHERE p1.oprcode = p2.oid AND OR NOT binary_coercible(p1.oprright, p2.proargtypes[0]) OR p1.oprleft != 0); -SELECT p1.oid, p1.oprname, p2.oid, p2.proname -FROM pg_operator AS p1, pg_proc AS p2 -WHERE p1.oprcode = p2.oid AND - p1.oprkind = 'r' AND - (p2.pronargs != 1 - OR NOT binary_coercible(p2.prorettype, p1.oprresult) - OR NOT binary_coercible(p1.oprleft, p2.proargtypes[0]) - OR p1.oprright != 0); - -- If the operator is mergejoinable or hashjoinable, its underlying function -- should not be volatile. diff --git a/src/tutorial/complex.source b/src/tutorial/complex.source index 035592670162..d849ec0d4b70 100644 --- a/src/tutorial/complex.source +++ b/src/tutorial/complex.source @@ -111,7 +111,7 @@ CREATE FUNCTION complex_add(complex, complex) LANGUAGE C IMMUTABLE STRICT; -- we can now define the operator. We show a binary operator here but you --- can also define unary operators by omitting either of leftarg or rightarg. +-- can also define a prefix operator by omitting the leftarg. CREATE OPERATOR + ( leftarg = complex, rightarg = complex, diff --git a/src/tutorial/syscat.source b/src/tutorial/syscat.source index 3a1767f97be7..8a04d6a961f2 100644 --- a/src/tutorial/syscat.source +++ b/src/tutorial/syscat.source @@ -96,36 +96,22 @@ SELECT n.nspname, r.rolname, format_type(t.oid, null) as typname -- --- lists all left unary operators +-- lists all prefix operators -- -SELECT n.nspname, o.oprname AS left_unary, +SELECT n.nspname, o.oprname AS prefix_op, format_type(right_type.oid, null) AS operand, format_type(result.oid, null) AS return_type FROM pg_namespace n, pg_operator o, pg_type right_type, pg_type result WHERE o.oprnamespace = n.oid - and o.oprkind = 'l' -- left unary + and o.oprkind = 'l' -- prefix ("left unary") and o.oprright = right_type.oid and o.oprresult = result.oid ORDER BY nspname, operand; -- --- lists all right unary operators --- -SELECT n.nspname, o.oprname AS right_unary, - format_type(left_type.oid, null) AS operand, - format_type(result.oid, null) AS return_type - FROM pg_namespace n, pg_operator o, - pg_type left_type, pg_type result - WHERE o.oprnamespace = n.oid - and o.oprkind = 'r' -- right unary - and o.oprleft = left_type.oid - and o.oprresult = result.oid - ORDER BY nspname, operand; - --- --- lists all binary operators +-- lists all infix operators -- SELECT n.nspname, o.oprname AS binary_op, format_type(left_type.oid, null) AS left_opr, @@ -134,7 +120,7 @@ SELECT n.nspname, o.oprname AS binary_op, FROM pg_namespace n, pg_operator o, pg_type left_type, pg_type right_type, pg_type result WHERE o.oprnamespace = n.oid - and o.oprkind = 'b' -- binary + and o.oprkind = 'b' -- infix ("binary") and o.oprleft = left_type.oid and o.oprright = right_type.oid and o.oprresult = result.oid From e5209bf37a2752f0f7a34309ba53c210d21dbbb6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 17 Sep 2020 21:02:55 -0400 Subject: [PATCH 171/589] Try to stabilize output from rolenames regression test. It's not quite clear why commit 45b980570 has resulted in some instability here, though interference from concurrent autovacuum runs seems like a reasonable guess. What is clear is that the output ordering of the test queries is underdetermined for no very good reason. Extend the ORDER BY keys in hopes of fixing the buildfarm. Discussion: https://postgr.es/m/147499.1600351924@sss.pgh.pa.us --- src/test/modules/unsafe_tests/expected/rolenames.out | 10 +++++----- src/test/modules/unsafe_tests/sql/rolenames.sql | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/test/modules/unsafe_tests/expected/rolenames.out b/src/test/modules/unsafe_tests/expected/rolenames.out index 9636e50e05c8..6ddb1a83f67b 100644 --- a/src/test/modules/unsafe_tests/expected/rolenames.out +++ b/src/test/modules/unsafe_tests/expected/rolenames.out @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION chkrolattr() +CREATE FUNCTION chkrolattr() RETURNS TABLE ("role" name, rolekeyword text, canlogin bool, replication bool) AS $$ SELECT r.rolname, v.keyword, r.rolcanlogin, r.rolreplication @@ -13,9 +13,9 @@ SELECT r.rolname, v.keyword, r.rolcanlogin, r.rolreplication ('None', '-')) AS v(uname, keyword) ON (r.rolname = v.uname) - ORDER BY 1; + ORDER BY 1, 2; $$ LANGUAGE SQL; -CREATE OR REPLACE FUNCTION chksetconfig() +CREATE FUNCTION chksetconfig() RETURNS TABLE (db name, "role" name, rolkeyword text, setconfig text[]) AS $$ SELECT COALESCE(d.datname, 'ALL'), COALESCE(r.rolname, 'ALL'), @@ -31,14 +31,14 @@ SELECT COALESCE(d.datname, 'ALL'), COALESCE(r.rolname, 'ALL'), WHERE (r.rolname) IN ('Public', 'current_user', 'regress_testrol1', 'regress_testrol2') ORDER BY 1, 2; $$ LANGUAGE SQL; -CREATE OR REPLACE FUNCTION chkumapping() +CREATE FUNCTION chkumapping() RETURNS TABLE (umname name, umserver name, umoptions text[]) AS $$ SELECT r.rolname, s.srvname, m.umoptions FROM pg_user_mapping m LEFT JOIN pg_roles r ON (r.oid = m.umuser) JOIN pg_foreign_server s ON (s.oid = m.umserver) - ORDER BY 2; + ORDER BY 2, 1; $$ LANGUAGE SQL; -- -- We test creation and use of these role names to ensure that the server diff --git a/src/test/modules/unsafe_tests/sql/rolenames.sql b/src/test/modules/unsafe_tests/sql/rolenames.sql index 638decda6800..40dc86fdb9b3 100644 --- a/src/test/modules/unsafe_tests/sql/rolenames.sql +++ b/src/test/modules/unsafe_tests/sql/rolenames.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION chkrolattr() +CREATE FUNCTION chkrolattr() RETURNS TABLE ("role" name, rolekeyword text, canlogin bool, replication bool) AS $$ SELECT r.rolname, v.keyword, r.rolcanlogin, r.rolreplication @@ -13,10 +13,10 @@ SELECT r.rolname, v.keyword, r.rolcanlogin, r.rolreplication ('None', '-')) AS v(uname, keyword) ON (r.rolname = v.uname) - ORDER BY 1; + ORDER BY 1, 2; $$ LANGUAGE SQL; -CREATE OR REPLACE FUNCTION chksetconfig() +CREATE FUNCTION chksetconfig() RETURNS TABLE (db name, "role" name, rolkeyword text, setconfig text[]) AS $$ SELECT COALESCE(d.datname, 'ALL'), COALESCE(r.rolname, 'ALL'), @@ -33,14 +33,14 @@ SELECT COALESCE(d.datname, 'ALL'), COALESCE(r.rolname, 'ALL'), ORDER BY 1, 2; $$ LANGUAGE SQL; -CREATE OR REPLACE FUNCTION chkumapping() +CREATE FUNCTION chkumapping() RETURNS TABLE (umname name, umserver name, umoptions text[]) AS $$ SELECT r.rolname, s.srvname, m.umoptions FROM pg_user_mapping m LEFT JOIN pg_roles r ON (r.oid = m.umuser) JOIN pg_foreign_server s ON (s.oid = m.umserver) - ORDER BY 2; + ORDER BY 2, 1; $$ LANGUAGE SQL; -- From 0d32511eca5aec205cb6b609638ea67129ef6665 Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Fri, 18 Sep 2020 09:40:04 +0530 Subject: [PATCH 172/589] Fix comments in heapam.c. After commits 85f6b49c2c and 3ba59ccc89, we can allow parallel inserts which was earlier not possible as parallel group members won't conflict for relation extension and page lock. In those commits, we forgot to update comments at few places. Author: Amit Kapila Reviewed-by: Robert Haas and Dilip Kumar Backpatch-through: 13 Discussion: https://postgr.es/m/CAFiTN-tMrQh5FFMPx5aWJ+1gi1H6JxktEhq5mDwCHgnEO5oBkA@mail.gmail.com --- src/backend/access/heap/heapam.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 9b5f417eac44..1585861a021d 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -2044,12 +2044,10 @@ heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid, CommandId cid, int options) { /* - * Parallel operations are required to be strictly read-only in a parallel - * worker. Parallel inserts are not safe even in the leader in the - * general case, because group locking means that heavyweight locks for - * relation extension or GIN page locks will not conflict between members - * of a lock group, but we don't prohibit that case here because there are - * useful special cases that we can safely allow, such as CREATE TABLE AS. + * To allow parallel inserts, we need to ensure that they are safe to be + * performed in workers. We have the infrastructure to allow parallel + * inserts in general except for the cases where inserts generate a new + * CommandId (eg. inserts into a table having a foreign key column). */ if (IsParallelWorker()) ereport(ERROR, @@ -5725,10 +5723,10 @@ heap_inplace_update(Relation relation, HeapTuple tuple) uint32 newlen; /* - * For now, parallel operations are required to be strictly read-only. - * Unlike a regular update, this should never create a combo CID, so it - * might be possible to relax this restriction, but not without more - * thought and testing. It's not clear that it would be useful, anyway. + * For now, we don't allow parallel updates. Unlike a regular update, + * this should never create a combo CID, so it might be possible to relax + * this restriction, but not without more thought and testing. It's not + * clear that it would be useful, anyway. */ if (IsInParallelMode()) ereport(ERROR, From 24fb35e11148b2b2cae2bddc5f2d6c566675c2b2 Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Fri, 18 Sep 2020 10:14:30 +0530 Subject: [PATCH 173/589] Update file header comments for logical/relation.c. Author: Amit Langote Reviewed-by: Amit Kapila Discussion: https://postgr.es/m/CA+HiwqE20oZoix13JyCeALpTf_SmjarZWtBFe5sND6zz+iupAw@mail.gmail.com --- src/backend/replication/logical/relation.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/backend/replication/logical/relation.c b/src/backend/replication/logical/relation.c index 9ee70a2563e4..2bb8e7d57b36 100644 --- a/src/backend/replication/logical/relation.c +++ b/src/backend/replication/logical/relation.c @@ -1,6 +1,6 @@ /*------------------------------------------------------------------------- * relation.c - * PostgreSQL logical replication + * PostgreSQL logical replication relation mapping cache * * Copyright (c) 2016-2020, PostgreSQL Global Development Group * @@ -8,8 +8,9 @@ * src/backend/replication/logical/relation.c * * NOTES - * This file contains helper functions for logical replication relation - * mapping cache. + * Routines in this file mainly have to do with mapping the properties + * of local replication target relations to the properties of their + * remote counterpart. * *------------------------------------------------------------------------- */ From 0811f766fd740018a72e222521553f8b22e7b3d6 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Fri, 18 Sep 2020 13:26:48 -0400 Subject: [PATCH 174/589] pg_surgery: Try to stabilize regression tests. According to buildfarm member sungazer, the behavior of VACUUM can be unstable in these tests even if we prevent autovacuum from running on the tables in question, apparently because even a manual vacuum can behave differently depending on whether anything else is running that holds back the global xmin. So use a temporary table instead, which as of commit a7212be8b9e0885ee769e8c55f99ef742cda487b enables vacuuming using a more aggressive cutoff. This approach can't be used for the regression test that involves a materialized view, but that test doesn't run vacuum, so it shouldn't be prone to this particular failure mode. Analysis by Tom Lane. Patch by Ashutosh Sharma and me. Discussion: http://postgr.es/m/665524.1599948007@sss.pgh.pa.us --- contrib/pg_surgery/expected/heap_surgery.out | 10 ++++------ contrib/pg_surgery/sql/heap_surgery.sql | 11 ++++------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/contrib/pg_surgery/expected/heap_surgery.out b/contrib/pg_surgery/expected/heap_surgery.out index 9451c5747bc0..d4a757ffa014 100644 --- a/contrib/pg_surgery/expected/heap_surgery.out +++ b/contrib/pg_surgery/expected/heap_surgery.out @@ -1,8 +1,7 @@ create extension pg_surgery; -- create a normal heap table and insert some rows. --- note that we don't commit the transaction, so autovacuum can't interfere. -begin; -create table htab(a int); +-- use a temp table so that vacuum behavior doesn't depend on global xmin +create temp table htab (a int); insert into htab values (100), (200), (300), (400), (500); -- test empty TID array select heap_force_freeze('htab'::regclass, ARRAY[]::tid[]); @@ -85,9 +84,9 @@ NOTICE: skipping tid (0, 6) for relation "htab" because the item number is out (1 row) -rollback; -- set up a new table with a redirected line pointer -create table htab2(a int) with (autovacuum_enabled = off); +-- use a temp table so that vacuum behavior doesn't depend on global xmin +create temp table htab2(a int); insert into htab2 values (100); update htab2 set a = 200; vacuum htab2; @@ -175,6 +174,5 @@ ERROR: "vw" is not a table, materialized view, or TOAST table select heap_force_freeze('vw'::regclass, ARRAY['(0, 1)']::tid[]); ERROR: "vw" is not a table, materialized view, or TOAST table -- cleanup. -drop table htab2; drop view vw; drop extension pg_surgery; diff --git a/contrib/pg_surgery/sql/heap_surgery.sql b/contrib/pg_surgery/sql/heap_surgery.sql index 8a27214e9cd1..6526b27535de 100644 --- a/contrib/pg_surgery/sql/heap_surgery.sql +++ b/contrib/pg_surgery/sql/heap_surgery.sql @@ -1,9 +1,8 @@ create extension pg_surgery; -- create a normal heap table and insert some rows. --- note that we don't commit the transaction, so autovacuum can't interfere. -begin; -create table htab(a int); +-- use a temp table so that vacuum behavior doesn't depend on global xmin +create temp table htab (a int); insert into htab values (100), (200), (300), (400), (500); -- test empty TID array @@ -38,10 +37,9 @@ select ctid, xmax from htab where xmin = 2; -- out-of-range TIDs should be skipped select heap_force_freeze('htab'::regclass, ARRAY['(0, 0)', '(0, 6)']::tid[]); -rollback; - -- set up a new table with a redirected line pointer -create table htab2(a int) with (autovacuum_enabled = off); +-- use a temp table so that vacuum behavior doesn't depend on global xmin +create temp table htab2(a int); insert into htab2 values (100); update htab2 set a = 200; vacuum htab2; @@ -86,6 +84,5 @@ select heap_force_kill('vw'::regclass, ARRAY['(0, 1)']::tid[]); select heap_force_freeze('vw'::regclass, ARRAY['(0, 1)']::tid[]); -- cleanup. -drop table htab2; drop view vw; drop extension pg_surgery; From 06a7c3154f5bfad65549810cc84f0e3a77b408bf Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 18 Sep 2020 16:46:26 -0400 Subject: [PATCH 175/589] Allow most keywords to be used as column labels without requiring AS. Up to now, if you tried to omit "AS" before a column label in a SELECT list, it would only work if the column label was an IDENT, that is not any known keyword. This is rather unfriendly considering that we have so many keywords and are constantly growing more. In the wake of commit 1ed6b8956 it's possible to improve matters quite a bit. We'd originally tried to make this work by having some of the existing keyword categories be allowed without AS, but that didn't work too well, because each category contains a few special cases that don't work without AS. Instead, invent an entirely orthogonal keyword property "can be bare column label", and mark all keywords that way for which we don't get shift/reduce errors by doing so. It turns out that of our 450 current keywords, all but 39 can be made bare column labels, improving the situation by over 90%. This number might move around a little depending on future grammar work, but it's a pretty nice improvement. Mark Dilger, based on work by myself and Robert Haas; review by John Naylor Discussion: https://postgr.es/m/38ca86db-42ab-9b48-2902-337a0d6b8311@2ndquadrant.com --- doc/src/sgml/func.sgml | 13 +- doc/src/sgml/generate-keywords-table.pl | 13 +- doc/src/sgml/keywords.sgml | 24 +- doc/src/sgml/queries.sgml | 26 +- src/backend/parser/check_keywords.pl | 92 ++- src/backend/parser/gram.y | 440 +++++++++++- src/backend/parser/scan.l | 2 +- src/backend/utils/adt/misc.c | 31 +- src/common/keywords.c | 17 +- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.dat | 7 +- src/include/common/keywords.h | 2 + src/include/parser/kwlist.h | 904 ++++++++++++------------ src/interfaces/ecpg/preproc/keywords.c | 2 +- 14 files changed, 1055 insertions(+), 520 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index d6283a35d84f..5d486258ed29 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -22173,7 +22173,9 @@ SELECT pg_type_is_visible('myschema.widget'::regtype); setof record ( word text, catcode "char", - catdesc text ) + barelabel boolean, + catdesc text, + baredesc text ) Returns a set of records describing the SQL keywords recognized by the @@ -22183,8 +22185,15 @@ SELECT pg_type_is_visible('myschema.widget'::regtype); keyword, C for a keyword that can be a column name, T for a keyword that can be a type or function name, or R for a fully reserved keyword. + The barelabel column + contains true if the keyword can be used as + a bare column label in SELECT lists, + or false if it can only be used + after AS. The catdesc column contains a - possibly-localized string describing the category. + possibly-localized string describing the keyword's category. + The baredesc column contains a + possibly-localized string describing the keyword's column label status. diff --git a/doc/src/sgml/generate-keywords-table.pl b/doc/src/sgml/generate-keywords-table.pl index 824b324ef78a..6332d65aadc7 100644 --- a/doc/src/sgml/generate-keywords-table.pl +++ b/doc/src/sgml/generate-keywords-table.pl @@ -1,6 +1,7 @@ #!/usr/bin/perl # -# Generate the keywords table file +# Generate the keywords table for the documentation's SQL Key Words appendix +# # Copyright (c) 2019-2020, PostgreSQL Global Development Group use strict; @@ -11,8 +12,9 @@ my $srcdir = $ARGV[0]; my %keywords; +my %as_keywords; -# read SQL keywords +# read SQL-spec keywords foreach my $ver (@sql_versions) { @@ -39,9 +41,10 @@ while (<$fh>) { - if (/^PG_KEYWORD\("(\w+)", \w+, (\w+)_KEYWORD\)/) + if (/^PG_KEYWORD\("(\w+)", \w+, (\w+)_KEYWORD\, (\w+)\)/) { $keywords{ uc $1 }{'pg'}{ lc $2 } = 1; + $as_keywords{ uc $1 } = 1 if $3 eq 'AS_LABEL'; } } @@ -107,6 +110,10 @@ END { print "reserved"; } + if ($as_keywords{$word}) + { + print ", requires AS"; + } print "\n"; foreach my $ver (@sql_versions) diff --git a/doc/src/sgml/keywords.sgml b/doc/src/sgml/keywords.sgml index 57dcd6ae5c7b..a7bf30c50468 100644 --- a/doc/src/sgml/keywords.sgml +++ b/doc/src/sgml/keywords.sgml @@ -32,11 +32,11 @@ - In the PostgreSQL parser life is a bit + In the PostgreSQL parser, life is a bit more complicated. There are several different classes of tokens ranging from those that can never be used as an identifier to those - that have absolutely no special status in the parser as compared to - an ordinary identifier. (The latter is usually the case for + that have absolutely no special status in the parser, but are considered + ordinary identifiers. (The latter is usually the case for functions specified by SQL.) Even reserved key words are not completely reserved in PostgreSQL, but can be used as column labels (for example, SELECT 55 AS @@ -57,14 +57,24 @@ reserved are those tokens that are not allowed as column or table names. Some reserved key words are allowable as names for functions or data types; this is also shown in the - table. If not so marked, a reserved key word is only allowed as an - AS column label name. + table. If not so marked, a reserved key word is only allowed as a + column label. + A blank entry in this column means that the word is treated as an + ordinary identifier by PostgreSQL. + + + + Furthermore, while most key words can be used as bare + column labels without writing AS before them (as + described in ), there are a few + that require a leading AS to avoid ambiguity. These + are marked in the table as requires AS. As a general rule, if you get spurious parser errors for commands - that contain any of the listed key words as an identifier you should - try to quote the identifier to see if the problem goes away. + that use any of the listed key words as an identifier, you should + try quoting the identifier to see if the problem goes away. diff --git a/doc/src/sgml/queries.sgml b/doc/src/sgml/queries.sgml index 67ca71e56490..0a643ef59705 100644 --- a/doc/src/sgml/queries.sgml +++ b/doc/src/sgml/queries.sgml @@ -1496,21 +1496,25 @@ SELECT a AS value, b + c AS sum FROM ... - The AS keyword is optional, but only if the new column - name does not match any - PostgreSQL keyword (see ). To avoid an accidental match to - a keyword, you can double-quote the column name. For example, - VALUE is a keyword, so this does not work: + The AS key word is usually optional, but in some + cases where the desired column name matches a + PostgreSQL key word, you must write + AS or double-quote the column name in order to + avoid ambiguity. + ( shows which key words + require AS to be used as a column label.) + For example, FROM is one such key word, so this + does not work: -SELECT a value, b + c AS sum FROM ... +SELECT a from, b + c AS sum FROM ... - but this does: + but either of these do: -SELECT a "value", b + c AS sum FROM ... +SELECT a AS from, b + c AS sum FROM ... +SELECT a "from", b + c AS sum FROM ... - For protection against possible - future keyword additions, it is recommended that you always either + For greatest safety against possible + future key word additions, it is recommended that you always either write AS or double-quote the output column name. diff --git a/src/backend/parser/check_keywords.pl b/src/backend/parser/check_keywords.pl index 702c97bba2aa..3862db072780 100644 --- a/src/backend/parser/check_keywords.pl +++ b/src/backend/parser/check_keywords.pl @@ -21,6 +21,28 @@ sub error return; } +# Check alphabetical order of a set of keyword symbols +# (note these are NOT the actual keyword strings) +sub check_alphabetical_order +{ + my ($listname, $list) = @_; + my $prevkword = ''; + + foreach my $kword (@$list) + { + # Some symbols have a _P suffix. Remove it for the comparison. + my $bare_kword = $kword; + $bare_kword =~ s/_P$//; + if ($bare_kword le $prevkword) + { + error + "'$bare_kword' after '$prevkword' in $listname list is misplaced"; + } + $prevkword = $bare_kword; + } + return; +} + $, = ' '; # set output field separator $\ = "\n"; # set output record separator @@ -33,9 +55,11 @@ sub error open(my $gram, '<', $gram_filename) || die("Could not open : $gram_filename"); my $kcat; +my $in_bare_labels; my $comment; my @arr; my %keywords; +my @bare_label_keywords; line: while (my $S = <$gram>) { @@ -51,7 +75,7 @@ sub error $s = '[/][*]', $S =~ s#$s# /* #g; $s = '[*][/]', $S =~ s#$s# */ #g; - if (!($kcat)) + if (!($kcat) && !($in_bare_labels)) { # Is this the beginning of a keyword list? @@ -63,6 +87,10 @@ sub error next line; } } + + # Is this the beginning of the bare_label_keyword list? + $in_bare_labels = 1 if ($S =~ m/^bare_label_keyword:/); + next line; } @@ -97,7 +125,8 @@ sub error { # end of keyword list - $kcat = ''; + undef $kcat; + undef $in_bare_labels; next; } @@ -107,31 +136,21 @@ sub error } # Put this keyword into the right list - push @{ $keywords{$kcat} }, $arr[$fieldIndexer]; + if ($in_bare_labels) + { + push @bare_label_keywords, $arr[$fieldIndexer]; + } + else + { + push @{ $keywords{$kcat} }, $arr[$fieldIndexer]; + } } } close $gram; # Check that each keyword list is in alphabetical order (just for neatnik-ism) -my ($prevkword, $bare_kword); -foreach my $kcat (keys %keyword_categories) -{ - $prevkword = ''; - - foreach my $kword (@{ $keywords{$kcat} }) - { - - # Some keyword have a _P suffix. Remove it for the comparison. - $bare_kword = $kword; - $bare_kword =~ s/_P$//; - if ($bare_kword le $prevkword) - { - error - "'$bare_kword' after '$prevkword' in $kcat list is misplaced"; - } - $prevkword = $bare_kword; - } -} +check_alphabetical_order($_, $keywords{$_}) for (keys %keyword_categories); +check_alphabetical_order('bare_label_keyword', \@bare_label_keywords); # Transform the keyword lists into hashes. # kwhashes is a hash of hashes, keyed by keyword category id, @@ -147,6 +166,7 @@ sub error $kwhashes{$kcat_id} = $hash; } +my %bare_label_keywords = map { $_ => 1 } @bare_label_keywords; # Now read in kwlist.h @@ -160,11 +180,12 @@ sub error { my ($line) = $_; - if ($line =~ /^PG_KEYWORD\(\"(.*)\", (.*), (.*)\)/) + if ($line =~ /^PG_KEYWORD\(\"(.*)\", (.*), (.*), (.*)\)/) { my ($kwstring) = $1; my ($kwname) = $2; my ($kwcat_id) = $3; + my ($collabel) = $4; # Check that the list is in alphabetical order (critical!) if ($kwstring le $prevkwstring) @@ -197,7 +218,7 @@ sub error "keyword name '$kwname' doesn't match keyword string '$kwstring'"; } - # Check that the keyword is present in the grammar + # Check that the keyword is present in the right category list %kwhash = %{ $kwhashes{$kwcat_id} }; if (!(%kwhash)) @@ -219,6 +240,29 @@ sub error delete $kwhashes{$kwcat_id}->{$kwname}; } } + + # Check that the keyword's collabel property matches gram.y + if ($collabel eq 'BARE_LABEL') + { + unless ($bare_label_keywords{$kwname}) + { + error + "'$kwname' is marked as BARE_LABEL in kwlist.h, but it is missing from gram.y's bare_label_keyword rule"; + } + } + elsif ($collabel eq 'AS_LABEL') + { + if ($bare_label_keywords{$kwname}) + { + error + "'$kwname' is marked as AS_LABEL in kwlist.h, but it is listed in gram.y's bare_label_keyword rule"; + } + } + else + { + error + "'$collabel' not recognized in kwlist.h. Expected either 'BARE_LABEL' or 'AS_LABEL'"; + } } } close $kwlist; diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index b16ffb9bf7fd..017940bdcd62 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -540,14 +540,16 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type Sconst comment_text notify_payload %type RoleId opt_boolean_or_string %type var_list -%type ColId ColLabel var_name type_function_name param_name +%type ColId ColLabel BareColLabel %type NonReservedWord NonReservedWord_or_Sconst +%type var_name type_function_name param_name %type createdb_opt_name %type var_value zone_value %type auth_ident RoleSpec opt_granted_by %type unreserved_keyword type_func_name_keyword %type col_name_keyword reserved_keyword +%type bare_label_keyword %type TableConstraint TableLikeClause %type TableLikeOptionList TableLikeOption @@ -14658,11 +14660,7 @@ target_el: a_expr AS ColLabel $$->val = (Node *)$1; $$->location = @1; } - /* - * We support omitting AS only for column labels that aren't - * any known keyword. - */ - | a_expr IDENT + | a_expr BareColLabel { $$ = makeNode(ResTarget); $$->name = $2; @@ -15011,6 +15009,13 @@ ColLabel: IDENT { $$ = $1; } | reserved_keyword { $$ = pstrdup($1); } ; +/* Bare column label --- names that can be column labels without writing "AS". + * This classification is orthogonal to the other keyword categories. + */ +BareColLabel: IDENT { $$ = $1; } + | bare_label_keyword { $$ = pstrdup($1); } + ; + /* * Keyword category lists. Generally, every keyword present in @@ -15515,6 +15520,429 @@ reserved_keyword: | WITH ; +/* + * While all keywords can be used as column labels when preceded by AS, + * not all of them can be used as a "bare" column label without AS. + * Those that can be used as a bare label must be listed here, + * in addition to appearing in one of the category lists above. + * + * Always add a new keyword to this list if possible. Mark it BARE_LABEL + * in kwlist.h if it is included here, or AS_LABEL if it is not. + */ +bare_label_keyword: + ABORT_P + | ABSOLUTE_P + | ACCESS + | ACTION + | ADD_P + | ADMIN + | AFTER + | AGGREGATE + | ALL + | ALSO + | ALTER + | ALWAYS + | ANALYSE + | ANALYZE + | AND + | ANY + | ASC + | ASSERTION + | ASSIGNMENT + | ASYMMETRIC + | AT + | ATTACH + | ATTRIBUTE + | AUTHORIZATION + | BACKWARD + | BEFORE + | BEGIN_P + | BETWEEN + | BIGINT + | BINARY + | BIT + | BOOLEAN_P + | BOTH + | BY + | CACHE + | CALL + | CALLED + | CASCADE + | CASCADED + | CASE + | CAST + | CATALOG_P + | CHAIN + | CHARACTERISTICS + | CHECK + | CHECKPOINT + | CLASS + | CLOSE + | CLUSTER + | COALESCE + | COLLATE + | COLLATION + | COLUMN + | COLUMNS + | COMMENT + | COMMENTS + | COMMIT + | COMMITTED + | CONCURRENTLY + | CONFIGURATION + | CONFLICT + | CONNECTION + | CONSTRAINT + | CONSTRAINTS + | CONTENT_P + | CONTINUE_P + | CONVERSION_P + | COPY + | COST + | CROSS + | CSV + | CUBE + | CURRENT_P + | CURRENT_CATALOG + | CURRENT_DATE + | CURRENT_ROLE + | CURRENT_SCHEMA + | CURRENT_TIME + | CURRENT_TIMESTAMP + | CURRENT_USER + | CURSOR + | CYCLE + | DATA_P + | DATABASE + | DEALLOCATE + | DEC + | DECIMAL_P + | DECLARE + | DEFAULT + | DEFAULTS + | DEFERRABLE + | DEFERRED + | DEFINER + | DELETE_P + | DELIMITER + | DELIMITERS + | DEPENDS + | DESC + | DETACH + | DICTIONARY + | DISABLE_P + | DISCARD + | DISTINCT + | DO + | DOCUMENT_P + | DOMAIN_P + | DOUBLE_P + | DROP + | EACH + | ELSE + | ENABLE_P + | ENCODING + | ENCRYPTED + | END_P + | ENUM_P + | ESCAPE + | EVENT + | EXCLUDE + | EXCLUDING + | EXCLUSIVE + | EXECUTE + | EXISTS + | EXPLAIN + | EXPRESSION + | EXTENSION + | EXTERNAL + | EXTRACT + | FALSE_P + | FAMILY + | FIRST_P + | FLOAT_P + | FOLLOWING + | FORCE + | FOREIGN + | FORWARD + | FREEZE + | FULL + | FUNCTION + | FUNCTIONS + | GENERATED + | GLOBAL + | GRANTED + | GREATEST + | GROUPING + | GROUPS + | HANDLER + | HEADER_P + | HOLD + | IDENTITY_P + | IF_P + | ILIKE + | IMMEDIATE + | IMMUTABLE + | IMPLICIT_P + | IMPORT_P + | IN_P + | INCLUDE + | INCLUDING + | INCREMENT + | INDEX + | INDEXES + | INHERIT + | INHERITS + | INITIALLY + | INLINE_P + | INNER_P + | INOUT + | INPUT_P + | INSENSITIVE + | INSERT + | INSTEAD + | INT_P + | INTEGER + | INTERVAL + | INVOKER + | IS + | ISOLATION + | JOIN + | KEY + | LABEL + | LANGUAGE + | LARGE_P + | LAST_P + | LATERAL_P + | LEADING + | LEAKPROOF + | LEAST + | LEFT + | LEVEL + | LIKE + | LISTEN + | LOAD + | LOCAL + | LOCALTIME + | LOCALTIMESTAMP + | LOCATION + | LOCK_P + | LOCKED + | LOGGED + | MAPPING + | MATCH + | MATERIALIZED + | MAXVALUE + | METHOD + | MINVALUE + | MODE + | MOVE + | NAME_P + | NAMES + | NATIONAL + | NATURAL + | NCHAR + | NEW + | NEXT + | NFC + | NFD + | NFKC + | NFKD + | NO + | NONE + | NORMALIZE + | NORMALIZED + | NOT + | NOTHING + | NOTIFY + | NOWAIT + | NULL_P + | NULLIF + | NULLS_P + | NUMERIC + | OBJECT_P + | OF + | OFF + | OIDS + | OLD + | ONLY + | OPERATOR + | OPTION + | OPTIONS + | OR + | ORDINALITY + | OTHERS + | OUT_P + | OUTER_P + | OVERLAY + | OVERRIDING + | OWNED + | OWNER + | PARALLEL + | PARSER + | PARTIAL + | PARTITION + | PASSING + | PASSWORD + | PLACING + | PLANS + | POLICY + | POSITION + | PRECEDING + | PREPARE + | PREPARED + | PRESERVE + | PRIMARY + | PRIOR + | PRIVILEGES + | PROCEDURAL + | PROCEDURE + | PROCEDURES + | PROGRAM + | PUBLICATION + | QUOTE + | RANGE + | READ + | REAL + | REASSIGN + | RECHECK + | RECURSIVE + | REF + | REFERENCES + | REFERENCING + | REFRESH + | REINDEX + | RELATIVE_P + | RELEASE + | RENAME + | REPEATABLE + | REPLACE + | REPLICA + | RESET + | RESTART + | RESTRICT + | RETURNS + | REVOKE + | RIGHT + | ROLE + | ROLLBACK + | ROLLUP + | ROUTINE + | ROUTINES + | ROW + | ROWS + | RULE + | SAVEPOINT + | SCHEMA + | SCHEMAS + | SCROLL + | SEARCH + | SECURITY + | SELECT + | SEQUENCE + | SEQUENCES + | SERIALIZABLE + | SERVER + | SESSION + | SESSION_USER + | SET + | SETOF + | SETS + | SHARE + | SHOW + | SIMILAR + | SIMPLE + | SKIP + | SMALLINT + | SNAPSHOT + | SOME + | SQL_P + | STABLE + | STANDALONE_P + | START + | STATEMENT + | STATISTICS + | STDIN + | STDOUT + | STORAGE + | STORED + | STRICT_P + | STRIP_P + | SUBSCRIPTION + | SUBSTRING + | SUPPORT + | SYMMETRIC + | SYSID + | SYSTEM_P + | TABLE + | TABLES + | TABLESAMPLE + | TABLESPACE + | TEMP + | TEMPLATE + | TEMPORARY + | TEXT_P + | THEN + | TIES + | TIME + | TIMESTAMP + | TRAILING + | TRANSACTION + | TRANSFORM + | TREAT + | TRIGGER + | TRIM + | TRUE_P + | TRUNCATE + | TRUSTED + | TYPE_P + | TYPES_P + | UESCAPE + | UNBOUNDED + | UNCOMMITTED + | UNENCRYPTED + | UNIQUE + | UNKNOWN + | UNLISTEN + | UNLOGGED + | UNTIL + | UPDATE + | USER + | USING + | VACUUM + | VALID + | VALIDATE + | VALIDATOR + | VALUE_P + | VALUES + | VARCHAR + | VARIADIC + | VERBOSE + | VERSION_P + | VIEW + | VIEWS + | VOLATILE + | WHEN + | WHITESPACE_P + | WORK + | WRAPPER + | WRITE + | XML_P + | XMLATTRIBUTES + | XMLCONCAT + | XMLELEMENT + | XMLEXISTS + | XMLFOREST + | XMLNAMESPACES + | XMLPARSE + | XMLPI + | XMLROOT + | XMLSERIALIZE + | XMLTABLE + | YES_P + | ZONE + ; + %% /* diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l index b1ea0cb53846..4eab2980c990 100644 --- a/src/backend/parser/scan.l +++ b/src/backend/parser/scan.l @@ -73,7 +73,7 @@ bool standard_conforming_strings = true; * callers need to pass it to scanner_init, if they are using the * standard keyword list ScanKeywords. */ -#define PG_KEYWORD(kwname, value, category) value, +#define PG_KEYWORD(kwname, value, category, collabel) value, const uint16 ScanKeywordTokens[] = { #include "parser/kwlist.h" diff --git a/src/backend/utils/adt/misc.c b/src/backend/utils/adt/misc.c index 37c23c9155af..b2bf9fa8cbcf 100644 --- a/src/backend/utils/adt/misc.c +++ b/src/backend/utils/adt/misc.c @@ -416,12 +416,16 @@ pg_get_keywords(PG_FUNCTION_ARGS) funcctx = SRF_FIRSTCALL_INIT(); oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); - tupdesc = CreateTemplateTupleDesc(3); + tupdesc = CreateTemplateTupleDesc(5); TupleDescInitEntry(tupdesc, (AttrNumber) 1, "word", TEXTOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "catcode", CHAROID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "catdesc", + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "barelabel", + BOOLOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "catdesc", + TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "baredesc", TEXTOID, -1, 0); funcctx->attinmeta = TupleDescGetAttInMetadata(tupdesc); @@ -433,7 +437,7 @@ pg_get_keywords(PG_FUNCTION_ARGS) if (funcctx->call_cntr < ScanKeywords.num_keywords) { - char *values[3]; + char *values[5]; HeapTuple tuple; /* cast-away-const is ugly but alternatives aren't much better */ @@ -445,26 +449,37 @@ pg_get_keywords(PG_FUNCTION_ARGS) { case UNRESERVED_KEYWORD: values[1] = "U"; - values[2] = _("unreserved"); + values[3] = _("unreserved"); break; case COL_NAME_KEYWORD: values[1] = "C"; - values[2] = _("unreserved (cannot be function or type name)"); + values[3] = _("unreserved (cannot be function or type name)"); break; case TYPE_FUNC_NAME_KEYWORD: values[1] = "T"; - values[2] = _("reserved (can be function or type name)"); + values[3] = _("reserved (can be function or type name)"); break; case RESERVED_KEYWORD: values[1] = "R"; - values[2] = _("reserved"); + values[3] = _("reserved"); break; default: /* shouldn't be possible */ values[1] = NULL; - values[2] = NULL; + values[3] = NULL; break; } + if (ScanKeywordBareLabel[funcctx->call_cntr]) + { + values[2] = "true"; + values[4] = _("can be bare label"); + } + else + { + values[2] = "false"; + values[4] = _("requires AS"); + } + tuple = BuildTupleFromCStrings(funcctx->attinmeta, values); SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tuple)); diff --git a/src/common/keywords.c b/src/common/keywords.c index 54ed97709613..2de0c717a891 100644 --- a/src/common/keywords.c +++ b/src/common/keywords.c @@ -24,10 +24,25 @@ /* Keyword categories for SQL keywords */ -#define PG_KEYWORD(kwname, value, category) category, +#define PG_KEYWORD(kwname, value, category, collabel) category, const uint8 ScanKeywordCategories[SCANKEYWORDS_NUM_KEYWORDS] = { #include "parser/kwlist.h" }; #undef PG_KEYWORD + +/* Keyword can-be-bare-label flags for SQL keywords */ + +#define PG_KEYWORD(kwname, value, category, collabel) collabel, + +#define BARE_LABEL true +#define AS_LABEL false + +const bool ScanKeywordBareLabel[SCANKEYWORDS_NUM_KEYWORDS] = { +#include "parser/kwlist.h" +}; + +#undef PG_KEYWORD +#undef BARE_LABEL +#undef AS_LABEL diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 365552635b84..650e1f733ed2 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 202009172 +#define CATALOG_VERSION_NO 202009181 #endif diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 1ae428814662..f48f5fb4d99b 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -3683,10 +3683,11 @@ prosrc => 'pg_get_function_arg_default' }, { oid => '1686', descr => 'list of SQL keywords', - proname => 'pg_get_keywords', procost => '10', prorows => '400', + proname => 'pg_get_keywords', procost => '10', prorows => '500', proretset => 't', provolatile => 's', prorettype => 'record', - proargtypes => '', proallargtypes => '{text,char,text}', - proargmodes => '{o,o,o}', proargnames => '{word,catcode,catdesc}', + proargtypes => '', proallargtypes => '{text,char,bool,text,text}', + proargmodes => '{o,o,o,o,o}', + proargnames => '{word,catcode,barelabel,catdesc,baredesc}', prosrc => 'pg_get_keywords' }, { oid => '2289', descr => 'convert generic options array to name/value table', diff --git a/src/include/common/keywords.h b/src/include/common/keywords.h index 257c050903e3..c9f9a9f991a7 100644 --- a/src/include/common/keywords.h +++ b/src/include/common/keywords.h @@ -25,9 +25,11 @@ #ifndef FRONTEND extern PGDLLIMPORT const ScanKeywordList ScanKeywords; extern PGDLLIMPORT const uint8 ScanKeywordCategories[]; +extern PGDLLIMPORT const bool ScanKeywordBareLabel[]; #else extern const ScanKeywordList ScanKeywords; extern const uint8 ScanKeywordCategories[]; +extern const bool ScanKeywordBareLabel[]; #endif #endif /* KEYWORDS_H */ diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 08f22ce211dd..71dcdf28894d 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -19,459 +19,459 @@ /* there is deliberately not an #ifndef KWLIST_H here */ /* - * List of keyword (name, token-value, category) entries. + * List of keyword (name, token-value, category, bare-label-status) entries. * * Note: gen_keywordlist.pl requires the entries to appear in ASCII order. */ -/* name, value, category */ -PG_KEYWORD("abort", ABORT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("absolute", ABSOLUTE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("access", ACCESS, UNRESERVED_KEYWORD) -PG_KEYWORD("action", ACTION, UNRESERVED_KEYWORD) -PG_KEYWORD("add", ADD_P, UNRESERVED_KEYWORD) -PG_KEYWORD("admin", ADMIN, UNRESERVED_KEYWORD) -PG_KEYWORD("after", AFTER, UNRESERVED_KEYWORD) -PG_KEYWORD("aggregate", AGGREGATE, UNRESERVED_KEYWORD) -PG_KEYWORD("all", ALL, RESERVED_KEYWORD) -PG_KEYWORD("also", ALSO, UNRESERVED_KEYWORD) -PG_KEYWORD("alter", ALTER, UNRESERVED_KEYWORD) -PG_KEYWORD("always", ALWAYS, UNRESERVED_KEYWORD) -PG_KEYWORD("analyse", ANALYSE, RESERVED_KEYWORD) /* British spelling */ -PG_KEYWORD("analyze", ANALYZE, RESERVED_KEYWORD) -PG_KEYWORD("and", AND, RESERVED_KEYWORD) -PG_KEYWORD("any", ANY, RESERVED_KEYWORD) -PG_KEYWORD("array", ARRAY, RESERVED_KEYWORD) -PG_KEYWORD("as", AS, RESERVED_KEYWORD) -PG_KEYWORD("asc", ASC, RESERVED_KEYWORD) -PG_KEYWORD("assertion", ASSERTION, UNRESERVED_KEYWORD) -PG_KEYWORD("assignment", ASSIGNMENT, UNRESERVED_KEYWORD) -PG_KEYWORD("asymmetric", ASYMMETRIC, RESERVED_KEYWORD) -PG_KEYWORD("at", AT, UNRESERVED_KEYWORD) -PG_KEYWORD("attach", ATTACH, UNRESERVED_KEYWORD) -PG_KEYWORD("attribute", ATTRIBUTE, UNRESERVED_KEYWORD) -PG_KEYWORD("authorization", AUTHORIZATION, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("backward", BACKWARD, UNRESERVED_KEYWORD) -PG_KEYWORD("before", BEFORE, UNRESERVED_KEYWORD) -PG_KEYWORD("begin", BEGIN_P, UNRESERVED_KEYWORD) -PG_KEYWORD("between", BETWEEN, COL_NAME_KEYWORD) -PG_KEYWORD("bigint", BIGINT, COL_NAME_KEYWORD) -PG_KEYWORD("binary", BINARY, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("bit", BIT, COL_NAME_KEYWORD) -PG_KEYWORD("boolean", BOOLEAN_P, COL_NAME_KEYWORD) -PG_KEYWORD("both", BOTH, RESERVED_KEYWORD) -PG_KEYWORD("by", BY, UNRESERVED_KEYWORD) -PG_KEYWORD("cache", CACHE, UNRESERVED_KEYWORD) -PG_KEYWORD("call", CALL, UNRESERVED_KEYWORD) -PG_KEYWORD("called", CALLED, UNRESERVED_KEYWORD) -PG_KEYWORD("cascade", CASCADE, UNRESERVED_KEYWORD) -PG_KEYWORD("cascaded", CASCADED, UNRESERVED_KEYWORD) -PG_KEYWORD("case", CASE, RESERVED_KEYWORD) -PG_KEYWORD("cast", CAST, RESERVED_KEYWORD) -PG_KEYWORD("catalog", CATALOG_P, UNRESERVED_KEYWORD) -PG_KEYWORD("chain", CHAIN, UNRESERVED_KEYWORD) -PG_KEYWORD("char", CHAR_P, COL_NAME_KEYWORD) -PG_KEYWORD("character", CHARACTER, COL_NAME_KEYWORD) -PG_KEYWORD("characteristics", CHARACTERISTICS, UNRESERVED_KEYWORD) -PG_KEYWORD("check", CHECK, RESERVED_KEYWORD) -PG_KEYWORD("checkpoint", CHECKPOINT, UNRESERVED_KEYWORD) -PG_KEYWORD("class", CLASS, UNRESERVED_KEYWORD) -PG_KEYWORD("close", CLOSE, UNRESERVED_KEYWORD) -PG_KEYWORD("cluster", CLUSTER, UNRESERVED_KEYWORD) -PG_KEYWORD("coalesce", COALESCE, COL_NAME_KEYWORD) -PG_KEYWORD("collate", COLLATE, RESERVED_KEYWORD) -PG_KEYWORD("collation", COLLATION, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("column", COLUMN, RESERVED_KEYWORD) -PG_KEYWORD("columns", COLUMNS, UNRESERVED_KEYWORD) -PG_KEYWORD("comment", COMMENT, UNRESERVED_KEYWORD) -PG_KEYWORD("comments", COMMENTS, UNRESERVED_KEYWORD) -PG_KEYWORD("commit", COMMIT, UNRESERVED_KEYWORD) -PG_KEYWORD("committed", COMMITTED, UNRESERVED_KEYWORD) -PG_KEYWORD("concurrently", CONCURRENTLY, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("configuration", CONFIGURATION, UNRESERVED_KEYWORD) -PG_KEYWORD("conflict", CONFLICT, UNRESERVED_KEYWORD) -PG_KEYWORD("connection", CONNECTION, UNRESERVED_KEYWORD) -PG_KEYWORD("constraint", CONSTRAINT, RESERVED_KEYWORD) -PG_KEYWORD("constraints", CONSTRAINTS, UNRESERVED_KEYWORD) -PG_KEYWORD("content", CONTENT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("continue", CONTINUE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("conversion", CONVERSION_P, UNRESERVED_KEYWORD) -PG_KEYWORD("copy", COPY, UNRESERVED_KEYWORD) -PG_KEYWORD("cost", COST, UNRESERVED_KEYWORD) -PG_KEYWORD("create", CREATE, RESERVED_KEYWORD) -PG_KEYWORD("cross", CROSS, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("csv", CSV, UNRESERVED_KEYWORD) -PG_KEYWORD("cube", CUBE, UNRESERVED_KEYWORD) -PG_KEYWORD("current", CURRENT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("current_catalog", CURRENT_CATALOG, RESERVED_KEYWORD) -PG_KEYWORD("current_date", CURRENT_DATE, RESERVED_KEYWORD) -PG_KEYWORD("current_role", CURRENT_ROLE, RESERVED_KEYWORD) -PG_KEYWORD("current_schema", CURRENT_SCHEMA, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("current_time", CURRENT_TIME, RESERVED_KEYWORD) -PG_KEYWORD("current_timestamp", CURRENT_TIMESTAMP, RESERVED_KEYWORD) -PG_KEYWORD("current_user", CURRENT_USER, RESERVED_KEYWORD) -PG_KEYWORD("cursor", CURSOR, UNRESERVED_KEYWORD) -PG_KEYWORD("cycle", CYCLE, UNRESERVED_KEYWORD) -PG_KEYWORD("data", DATA_P, UNRESERVED_KEYWORD) -PG_KEYWORD("database", DATABASE, UNRESERVED_KEYWORD) -PG_KEYWORD("day", DAY_P, UNRESERVED_KEYWORD) -PG_KEYWORD("deallocate", DEALLOCATE, UNRESERVED_KEYWORD) -PG_KEYWORD("dec", DEC, COL_NAME_KEYWORD) -PG_KEYWORD("decimal", DECIMAL_P, COL_NAME_KEYWORD) -PG_KEYWORD("declare", DECLARE, UNRESERVED_KEYWORD) -PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD) -PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD) -PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD) -PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD) -PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD) -PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD) -PG_KEYWORD("delimiters", DELIMITERS, UNRESERVED_KEYWORD) -PG_KEYWORD("depends", DEPENDS, UNRESERVED_KEYWORD) -PG_KEYWORD("desc", DESC, RESERVED_KEYWORD) -PG_KEYWORD("detach", DETACH, UNRESERVED_KEYWORD) -PG_KEYWORD("dictionary", DICTIONARY, UNRESERVED_KEYWORD) -PG_KEYWORD("disable", DISABLE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("discard", DISCARD, UNRESERVED_KEYWORD) -PG_KEYWORD("distinct", DISTINCT, RESERVED_KEYWORD) -PG_KEYWORD("do", DO, RESERVED_KEYWORD) -PG_KEYWORD("document", DOCUMENT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("domain", DOMAIN_P, UNRESERVED_KEYWORD) -PG_KEYWORD("double", DOUBLE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("drop", DROP, UNRESERVED_KEYWORD) -PG_KEYWORD("each", EACH, UNRESERVED_KEYWORD) -PG_KEYWORD("else", ELSE, RESERVED_KEYWORD) -PG_KEYWORD("enable", ENABLE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("encoding", ENCODING, UNRESERVED_KEYWORD) -PG_KEYWORD("encrypted", ENCRYPTED, UNRESERVED_KEYWORD) -PG_KEYWORD("end", END_P, RESERVED_KEYWORD) -PG_KEYWORD("enum", ENUM_P, UNRESERVED_KEYWORD) -PG_KEYWORD("escape", ESCAPE, UNRESERVED_KEYWORD) -PG_KEYWORD("event", EVENT, UNRESERVED_KEYWORD) -PG_KEYWORD("except", EXCEPT, RESERVED_KEYWORD) -PG_KEYWORD("exclude", EXCLUDE, UNRESERVED_KEYWORD) -PG_KEYWORD("excluding", EXCLUDING, UNRESERVED_KEYWORD) -PG_KEYWORD("exclusive", EXCLUSIVE, UNRESERVED_KEYWORD) -PG_KEYWORD("execute", EXECUTE, UNRESERVED_KEYWORD) -PG_KEYWORD("exists", EXISTS, COL_NAME_KEYWORD) -PG_KEYWORD("explain", EXPLAIN, UNRESERVED_KEYWORD) -PG_KEYWORD("expression", EXPRESSION, UNRESERVED_KEYWORD) -PG_KEYWORD("extension", EXTENSION, UNRESERVED_KEYWORD) -PG_KEYWORD("external", EXTERNAL, UNRESERVED_KEYWORD) -PG_KEYWORD("extract", EXTRACT, COL_NAME_KEYWORD) -PG_KEYWORD("false", FALSE_P, RESERVED_KEYWORD) -PG_KEYWORD("family", FAMILY, UNRESERVED_KEYWORD) -PG_KEYWORD("fetch", FETCH, RESERVED_KEYWORD) -PG_KEYWORD("filter", FILTER, UNRESERVED_KEYWORD) -PG_KEYWORD("first", FIRST_P, UNRESERVED_KEYWORD) -PG_KEYWORD("float", FLOAT_P, COL_NAME_KEYWORD) -PG_KEYWORD("following", FOLLOWING, UNRESERVED_KEYWORD) -PG_KEYWORD("for", FOR, RESERVED_KEYWORD) -PG_KEYWORD("force", FORCE, UNRESERVED_KEYWORD) -PG_KEYWORD("foreign", FOREIGN, RESERVED_KEYWORD) -PG_KEYWORD("forward", FORWARD, UNRESERVED_KEYWORD) -PG_KEYWORD("freeze", FREEZE, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("from", FROM, RESERVED_KEYWORD) -PG_KEYWORD("full", FULL, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("function", FUNCTION, UNRESERVED_KEYWORD) -PG_KEYWORD("functions", FUNCTIONS, UNRESERVED_KEYWORD) -PG_KEYWORD("generated", GENERATED, UNRESERVED_KEYWORD) -PG_KEYWORD("global", GLOBAL, UNRESERVED_KEYWORD) -PG_KEYWORD("grant", GRANT, RESERVED_KEYWORD) -PG_KEYWORD("granted", GRANTED, UNRESERVED_KEYWORD) -PG_KEYWORD("greatest", GREATEST, COL_NAME_KEYWORD) -PG_KEYWORD("group", GROUP_P, RESERVED_KEYWORD) -PG_KEYWORD("grouping", GROUPING, COL_NAME_KEYWORD) -PG_KEYWORD("groups", GROUPS, UNRESERVED_KEYWORD) -PG_KEYWORD("handler", HANDLER, UNRESERVED_KEYWORD) -PG_KEYWORD("having", HAVING, RESERVED_KEYWORD) -PG_KEYWORD("header", HEADER_P, UNRESERVED_KEYWORD) -PG_KEYWORD("hold", HOLD, UNRESERVED_KEYWORD) -PG_KEYWORD("hour", HOUR_P, UNRESERVED_KEYWORD) -PG_KEYWORD("identity", IDENTITY_P, UNRESERVED_KEYWORD) -PG_KEYWORD("if", IF_P, UNRESERVED_KEYWORD) -PG_KEYWORD("ilike", ILIKE, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("immediate", IMMEDIATE, UNRESERVED_KEYWORD) -PG_KEYWORD("immutable", IMMUTABLE, UNRESERVED_KEYWORD) -PG_KEYWORD("implicit", IMPLICIT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("import", IMPORT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("in", IN_P, RESERVED_KEYWORD) -PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD) -PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD) -PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD) -PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD) -PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD) -PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD) -PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD) -PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD) -PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("inout", INOUT, COL_NAME_KEYWORD) -PG_KEYWORD("input", INPUT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("insensitive", INSENSITIVE, UNRESERVED_KEYWORD) -PG_KEYWORD("insert", INSERT, UNRESERVED_KEYWORD) -PG_KEYWORD("instead", INSTEAD, UNRESERVED_KEYWORD) -PG_KEYWORD("int", INT_P, COL_NAME_KEYWORD) -PG_KEYWORD("integer", INTEGER, COL_NAME_KEYWORD) -PG_KEYWORD("intersect", INTERSECT, RESERVED_KEYWORD) -PG_KEYWORD("interval", INTERVAL, COL_NAME_KEYWORD) -PG_KEYWORD("into", INTO, RESERVED_KEYWORD) -PG_KEYWORD("invoker", INVOKER, UNRESERVED_KEYWORD) -PG_KEYWORD("is", IS, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("isnull", ISNULL, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("isolation", ISOLATION, UNRESERVED_KEYWORD) -PG_KEYWORD("join", JOIN, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("key", KEY, UNRESERVED_KEYWORD) -PG_KEYWORD("label", LABEL, UNRESERVED_KEYWORD) -PG_KEYWORD("language", LANGUAGE, UNRESERVED_KEYWORD) -PG_KEYWORD("large", LARGE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("last", LAST_P, UNRESERVED_KEYWORD) -PG_KEYWORD("lateral", LATERAL_P, RESERVED_KEYWORD) -PG_KEYWORD("leading", LEADING, RESERVED_KEYWORD) -PG_KEYWORD("leakproof", LEAKPROOF, UNRESERVED_KEYWORD) -PG_KEYWORD("least", LEAST, COL_NAME_KEYWORD) -PG_KEYWORD("left", LEFT, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("level", LEVEL, UNRESERVED_KEYWORD) -PG_KEYWORD("like", LIKE, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("limit", LIMIT, RESERVED_KEYWORD) -PG_KEYWORD("listen", LISTEN, UNRESERVED_KEYWORD) -PG_KEYWORD("load", LOAD, UNRESERVED_KEYWORD) -PG_KEYWORD("local", LOCAL, UNRESERVED_KEYWORD) -PG_KEYWORD("localtime", LOCALTIME, RESERVED_KEYWORD) -PG_KEYWORD("localtimestamp", LOCALTIMESTAMP, RESERVED_KEYWORD) -PG_KEYWORD("location", LOCATION, UNRESERVED_KEYWORD) -PG_KEYWORD("lock", LOCK_P, UNRESERVED_KEYWORD) -PG_KEYWORD("locked", LOCKED, UNRESERVED_KEYWORD) -PG_KEYWORD("logged", LOGGED, UNRESERVED_KEYWORD) -PG_KEYWORD("mapping", MAPPING, UNRESERVED_KEYWORD) -PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD) -PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD) -PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD) -PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD) -PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("minvalue", MINVALUE, UNRESERVED_KEYWORD) -PG_KEYWORD("mode", MODE, UNRESERVED_KEYWORD) -PG_KEYWORD("month", MONTH_P, UNRESERVED_KEYWORD) -PG_KEYWORD("move", MOVE, UNRESERVED_KEYWORD) -PG_KEYWORD("name", NAME_P, UNRESERVED_KEYWORD) -PG_KEYWORD("names", NAMES, UNRESERVED_KEYWORD) -PG_KEYWORD("national", NATIONAL, COL_NAME_KEYWORD) -PG_KEYWORD("natural", NATURAL, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("nchar", NCHAR, COL_NAME_KEYWORD) -PG_KEYWORD("new", NEW, UNRESERVED_KEYWORD) -PG_KEYWORD("next", NEXT, UNRESERVED_KEYWORD) -PG_KEYWORD("nfc", NFC, UNRESERVED_KEYWORD) -PG_KEYWORD("nfd", NFD, UNRESERVED_KEYWORD) -PG_KEYWORD("nfkc", NFKC, UNRESERVED_KEYWORD) -PG_KEYWORD("nfkd", NFKD, UNRESERVED_KEYWORD) -PG_KEYWORD("no", NO, UNRESERVED_KEYWORD) -PG_KEYWORD("none", NONE, COL_NAME_KEYWORD) -PG_KEYWORD("normalize", NORMALIZE, COL_NAME_KEYWORD) -PG_KEYWORD("normalized", NORMALIZED, UNRESERVED_KEYWORD) -PG_KEYWORD("not", NOT, RESERVED_KEYWORD) -PG_KEYWORD("nothing", NOTHING, UNRESERVED_KEYWORD) -PG_KEYWORD("notify", NOTIFY, UNRESERVED_KEYWORD) -PG_KEYWORD("notnull", NOTNULL, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("nowait", NOWAIT, UNRESERVED_KEYWORD) -PG_KEYWORD("null", NULL_P, RESERVED_KEYWORD) -PG_KEYWORD("nullif", NULLIF, COL_NAME_KEYWORD) -PG_KEYWORD("nulls", NULLS_P, UNRESERVED_KEYWORD) -PG_KEYWORD("numeric", NUMERIC, COL_NAME_KEYWORD) -PG_KEYWORD("object", OBJECT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("of", OF, UNRESERVED_KEYWORD) -PG_KEYWORD("off", OFF, UNRESERVED_KEYWORD) -PG_KEYWORD("offset", OFFSET, RESERVED_KEYWORD) -PG_KEYWORD("oids", OIDS, UNRESERVED_KEYWORD) -PG_KEYWORD("old", OLD, UNRESERVED_KEYWORD) -PG_KEYWORD("on", ON, RESERVED_KEYWORD) -PG_KEYWORD("only", ONLY, RESERVED_KEYWORD) -PG_KEYWORD("operator", OPERATOR, UNRESERVED_KEYWORD) -PG_KEYWORD("option", OPTION, UNRESERVED_KEYWORD) -PG_KEYWORD("options", OPTIONS, UNRESERVED_KEYWORD) -PG_KEYWORD("or", OR, RESERVED_KEYWORD) -PG_KEYWORD("order", ORDER, RESERVED_KEYWORD) -PG_KEYWORD("ordinality", ORDINALITY, UNRESERVED_KEYWORD) -PG_KEYWORD("others", OTHERS, UNRESERVED_KEYWORD) -PG_KEYWORD("out", OUT_P, COL_NAME_KEYWORD) -PG_KEYWORD("outer", OUTER_P, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("over", OVER, UNRESERVED_KEYWORD) -PG_KEYWORD("overlaps", OVERLAPS, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("overlay", OVERLAY, COL_NAME_KEYWORD) -PG_KEYWORD("overriding", OVERRIDING, UNRESERVED_KEYWORD) -PG_KEYWORD("owned", OWNED, UNRESERVED_KEYWORD) -PG_KEYWORD("owner", OWNER, UNRESERVED_KEYWORD) -PG_KEYWORD("parallel", PARALLEL, UNRESERVED_KEYWORD) -PG_KEYWORD("parser", PARSER, UNRESERVED_KEYWORD) -PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD) -PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD) -PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD) -PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD) -PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD) -PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD) -PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD) -PG_KEYWORD("position", POSITION, COL_NAME_KEYWORD) -PG_KEYWORD("preceding", PRECEDING, UNRESERVED_KEYWORD) -PG_KEYWORD("precision", PRECISION, COL_NAME_KEYWORD) -PG_KEYWORD("prepare", PREPARE, UNRESERVED_KEYWORD) -PG_KEYWORD("prepared", PREPARED, UNRESERVED_KEYWORD) -PG_KEYWORD("preserve", PRESERVE, UNRESERVED_KEYWORD) -PG_KEYWORD("primary", PRIMARY, RESERVED_KEYWORD) -PG_KEYWORD("prior", PRIOR, UNRESERVED_KEYWORD) -PG_KEYWORD("privileges", PRIVILEGES, UNRESERVED_KEYWORD) -PG_KEYWORD("procedural", PROCEDURAL, UNRESERVED_KEYWORD) -PG_KEYWORD("procedure", PROCEDURE, UNRESERVED_KEYWORD) -PG_KEYWORD("procedures", PROCEDURES, UNRESERVED_KEYWORD) -PG_KEYWORD("program", PROGRAM, UNRESERVED_KEYWORD) -PG_KEYWORD("publication", PUBLICATION, UNRESERVED_KEYWORD) -PG_KEYWORD("quote", QUOTE, UNRESERVED_KEYWORD) -PG_KEYWORD("range", RANGE, UNRESERVED_KEYWORD) -PG_KEYWORD("read", READ, UNRESERVED_KEYWORD) -PG_KEYWORD("real", REAL, COL_NAME_KEYWORD) -PG_KEYWORD("reassign", REASSIGN, UNRESERVED_KEYWORD) -PG_KEYWORD("recheck", RECHECK, UNRESERVED_KEYWORD) -PG_KEYWORD("recursive", RECURSIVE, UNRESERVED_KEYWORD) -PG_KEYWORD("ref", REF, UNRESERVED_KEYWORD) -PG_KEYWORD("references", REFERENCES, RESERVED_KEYWORD) -PG_KEYWORD("referencing", REFERENCING, UNRESERVED_KEYWORD) -PG_KEYWORD("refresh", REFRESH, UNRESERVED_KEYWORD) -PG_KEYWORD("reindex", REINDEX, UNRESERVED_KEYWORD) -PG_KEYWORD("relative", RELATIVE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("release", RELEASE, UNRESERVED_KEYWORD) -PG_KEYWORD("rename", RENAME, UNRESERVED_KEYWORD) -PG_KEYWORD("repeatable", REPEATABLE, UNRESERVED_KEYWORD) -PG_KEYWORD("replace", REPLACE, UNRESERVED_KEYWORD) -PG_KEYWORD("replica", REPLICA, UNRESERVED_KEYWORD) -PG_KEYWORD("reset", RESET, UNRESERVED_KEYWORD) -PG_KEYWORD("restart", RESTART, UNRESERVED_KEYWORD) -PG_KEYWORD("restrict", RESTRICT, UNRESERVED_KEYWORD) -PG_KEYWORD("returning", RETURNING, RESERVED_KEYWORD) -PG_KEYWORD("returns", RETURNS, UNRESERVED_KEYWORD) -PG_KEYWORD("revoke", REVOKE, UNRESERVED_KEYWORD) -PG_KEYWORD("right", RIGHT, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("role", ROLE, UNRESERVED_KEYWORD) -PG_KEYWORD("rollback", ROLLBACK, UNRESERVED_KEYWORD) -PG_KEYWORD("rollup", ROLLUP, UNRESERVED_KEYWORD) -PG_KEYWORD("routine", ROUTINE, UNRESERVED_KEYWORD) -PG_KEYWORD("routines", ROUTINES, UNRESERVED_KEYWORD) -PG_KEYWORD("row", ROW, COL_NAME_KEYWORD) -PG_KEYWORD("rows", ROWS, UNRESERVED_KEYWORD) -PG_KEYWORD("rule", RULE, UNRESERVED_KEYWORD) -PG_KEYWORD("savepoint", SAVEPOINT, UNRESERVED_KEYWORD) -PG_KEYWORD("schema", SCHEMA, UNRESERVED_KEYWORD) -PG_KEYWORD("schemas", SCHEMAS, UNRESERVED_KEYWORD) -PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD) -PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD) -PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD) -PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD) -PG_KEYWORD("select", SELECT, RESERVED_KEYWORD) -PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD) -PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD) -PG_KEYWORD("serializable", SERIALIZABLE, UNRESERVED_KEYWORD) -PG_KEYWORD("server", SERVER, UNRESERVED_KEYWORD) -PG_KEYWORD("session", SESSION, UNRESERVED_KEYWORD) -PG_KEYWORD("session_user", SESSION_USER, RESERVED_KEYWORD) -PG_KEYWORD("set", SET, UNRESERVED_KEYWORD) -PG_KEYWORD("setof", SETOF, COL_NAME_KEYWORD) -PG_KEYWORD("sets", SETS, UNRESERVED_KEYWORD) -PG_KEYWORD("share", SHARE, UNRESERVED_KEYWORD) -PG_KEYWORD("show", SHOW, UNRESERVED_KEYWORD) -PG_KEYWORD("similar", SIMILAR, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("simple", SIMPLE, UNRESERVED_KEYWORD) -PG_KEYWORD("skip", SKIP, UNRESERVED_KEYWORD) -PG_KEYWORD("smallint", SMALLINT, COL_NAME_KEYWORD) -PG_KEYWORD("snapshot", SNAPSHOT, UNRESERVED_KEYWORD) -PG_KEYWORD("some", SOME, RESERVED_KEYWORD) -PG_KEYWORD("sql", SQL_P, UNRESERVED_KEYWORD) -PG_KEYWORD("stable", STABLE, UNRESERVED_KEYWORD) -PG_KEYWORD("standalone", STANDALONE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("start", START, UNRESERVED_KEYWORD) -PG_KEYWORD("statement", STATEMENT, UNRESERVED_KEYWORD) -PG_KEYWORD("statistics", STATISTICS, UNRESERVED_KEYWORD) -PG_KEYWORD("stdin", STDIN, UNRESERVED_KEYWORD) -PG_KEYWORD("stdout", STDOUT, UNRESERVED_KEYWORD) -PG_KEYWORD("storage", STORAGE, UNRESERVED_KEYWORD) -PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD) -PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD) -PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD) -PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD) -PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD) -PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD) -PG_KEYWORD("sysid", SYSID, UNRESERVED_KEYWORD) -PG_KEYWORD("system", SYSTEM_P, UNRESERVED_KEYWORD) -PG_KEYWORD("table", TABLE, RESERVED_KEYWORD) -PG_KEYWORD("tables", TABLES, UNRESERVED_KEYWORD) -PG_KEYWORD("tablesample", TABLESAMPLE, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("tablespace", TABLESPACE, UNRESERVED_KEYWORD) -PG_KEYWORD("temp", TEMP, UNRESERVED_KEYWORD) -PG_KEYWORD("template", TEMPLATE, UNRESERVED_KEYWORD) -PG_KEYWORD("temporary", TEMPORARY, UNRESERVED_KEYWORD) -PG_KEYWORD("text", TEXT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("then", THEN, RESERVED_KEYWORD) -PG_KEYWORD("ties", TIES, UNRESERVED_KEYWORD) -PG_KEYWORD("time", TIME, COL_NAME_KEYWORD) -PG_KEYWORD("timestamp", TIMESTAMP, COL_NAME_KEYWORD) -PG_KEYWORD("to", TO, RESERVED_KEYWORD) -PG_KEYWORD("trailing", TRAILING, RESERVED_KEYWORD) -PG_KEYWORD("transaction", TRANSACTION, UNRESERVED_KEYWORD) -PG_KEYWORD("transform", TRANSFORM, UNRESERVED_KEYWORD) -PG_KEYWORD("treat", TREAT, COL_NAME_KEYWORD) -PG_KEYWORD("trigger", TRIGGER, UNRESERVED_KEYWORD) -PG_KEYWORD("trim", TRIM, COL_NAME_KEYWORD) -PG_KEYWORD("true", TRUE_P, RESERVED_KEYWORD) -PG_KEYWORD("truncate", TRUNCATE, UNRESERVED_KEYWORD) -PG_KEYWORD("trusted", TRUSTED, UNRESERVED_KEYWORD) -PG_KEYWORD("type", TYPE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("types", TYPES_P, UNRESERVED_KEYWORD) -PG_KEYWORD("uescape", UESCAPE, UNRESERVED_KEYWORD) -PG_KEYWORD("unbounded", UNBOUNDED, UNRESERVED_KEYWORD) -PG_KEYWORD("uncommitted", UNCOMMITTED, UNRESERVED_KEYWORD) -PG_KEYWORD("unencrypted", UNENCRYPTED, UNRESERVED_KEYWORD) -PG_KEYWORD("union", UNION, RESERVED_KEYWORD) -PG_KEYWORD("unique", UNIQUE, RESERVED_KEYWORD) -PG_KEYWORD("unknown", UNKNOWN, UNRESERVED_KEYWORD) -PG_KEYWORD("unlisten", UNLISTEN, UNRESERVED_KEYWORD) -PG_KEYWORD("unlogged", UNLOGGED, UNRESERVED_KEYWORD) -PG_KEYWORD("until", UNTIL, UNRESERVED_KEYWORD) -PG_KEYWORD("update", UPDATE, UNRESERVED_KEYWORD) -PG_KEYWORD("user", USER, RESERVED_KEYWORD) -PG_KEYWORD("using", USING, RESERVED_KEYWORD) -PG_KEYWORD("vacuum", VACUUM, UNRESERVED_KEYWORD) -PG_KEYWORD("valid", VALID, UNRESERVED_KEYWORD) -PG_KEYWORD("validate", VALIDATE, UNRESERVED_KEYWORD) -PG_KEYWORD("validator", VALIDATOR, UNRESERVED_KEYWORD) -PG_KEYWORD("value", VALUE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("values", VALUES, COL_NAME_KEYWORD) -PG_KEYWORD("varchar", VARCHAR, COL_NAME_KEYWORD) -PG_KEYWORD("variadic", VARIADIC, RESERVED_KEYWORD) -PG_KEYWORD("varying", VARYING, UNRESERVED_KEYWORD) -PG_KEYWORD("verbose", VERBOSE, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("version", VERSION_P, UNRESERVED_KEYWORD) -PG_KEYWORD("view", VIEW, UNRESERVED_KEYWORD) -PG_KEYWORD("views", VIEWS, UNRESERVED_KEYWORD) -PG_KEYWORD("volatile", VOLATILE, UNRESERVED_KEYWORD) -PG_KEYWORD("when", WHEN, RESERVED_KEYWORD) -PG_KEYWORD("where", WHERE, RESERVED_KEYWORD) -PG_KEYWORD("whitespace", WHITESPACE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("window", WINDOW, RESERVED_KEYWORD) -PG_KEYWORD("with", WITH, RESERVED_KEYWORD) -PG_KEYWORD("within", WITHIN, UNRESERVED_KEYWORD) -PG_KEYWORD("without", WITHOUT, UNRESERVED_KEYWORD) -PG_KEYWORD("work", WORK, UNRESERVED_KEYWORD) -PG_KEYWORD("wrapper", WRAPPER, UNRESERVED_KEYWORD) -PG_KEYWORD("write", WRITE, UNRESERVED_KEYWORD) -PG_KEYWORD("xml", XML_P, UNRESERVED_KEYWORD) -PG_KEYWORD("xmlattributes", XMLATTRIBUTES, COL_NAME_KEYWORD) -PG_KEYWORD("xmlconcat", XMLCONCAT, COL_NAME_KEYWORD) -PG_KEYWORD("xmlelement", XMLELEMENT, COL_NAME_KEYWORD) -PG_KEYWORD("xmlexists", XMLEXISTS, COL_NAME_KEYWORD) -PG_KEYWORD("xmlforest", XMLFOREST, COL_NAME_KEYWORD) -PG_KEYWORD("xmlnamespaces", XMLNAMESPACES, COL_NAME_KEYWORD) -PG_KEYWORD("xmlparse", XMLPARSE, COL_NAME_KEYWORD) -PG_KEYWORD("xmlpi", XMLPI, COL_NAME_KEYWORD) -PG_KEYWORD("xmlroot", XMLROOT, COL_NAME_KEYWORD) -PG_KEYWORD("xmlserialize", XMLSERIALIZE, COL_NAME_KEYWORD) -PG_KEYWORD("xmltable", XMLTABLE, COL_NAME_KEYWORD) -PG_KEYWORD("year", YEAR_P, UNRESERVED_KEYWORD) -PG_KEYWORD("yes", YES_P, UNRESERVED_KEYWORD) -PG_KEYWORD("zone", ZONE, UNRESERVED_KEYWORD) +/* name, value, category, is-bare-label */ +PG_KEYWORD("abort", ABORT_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("absolute", ABSOLUTE_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("access", ACCESS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("action", ACTION, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("add", ADD_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("admin", ADMIN, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("after", AFTER, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("aggregate", AGGREGATE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("all", ALL, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("also", ALSO, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("alter", ALTER, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("always", ALWAYS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("analyse", ANALYSE, RESERVED_KEYWORD, BARE_LABEL) /* British spelling */ +PG_KEYWORD("analyze", ANALYZE, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("and", AND, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("any", ANY, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("array", ARRAY, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("as", AS, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("asc", ASC, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("assertion", ASSERTION, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("assignment", ASSIGNMENT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("asymmetric", ASYMMETRIC, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("at", AT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("attach", ATTACH, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("attribute", ATTRIBUTE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("authorization", AUTHORIZATION, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("backward", BACKWARD, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("before", BEFORE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("begin", BEGIN_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("between", BETWEEN, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("bigint", BIGINT, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("binary", BINARY, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("bit", BIT, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("boolean", BOOLEAN_P, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("both", BOTH, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("by", BY, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("cache", CACHE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("call", CALL, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("called", CALLED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("cascade", CASCADE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("cascaded", CASCADED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("case", CASE, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("cast", CAST, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("catalog", CATALOG_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("chain", CHAIN, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("char", CHAR_P, COL_NAME_KEYWORD, AS_LABEL) +PG_KEYWORD("character", CHARACTER, COL_NAME_KEYWORD, AS_LABEL) +PG_KEYWORD("characteristics", CHARACTERISTICS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("check", CHECK, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("checkpoint", CHECKPOINT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("class", CLASS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("close", CLOSE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("cluster", CLUSTER, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("coalesce", COALESCE, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("collate", COLLATE, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("collation", COLLATION, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("column", COLUMN, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("columns", COLUMNS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("comment", COMMENT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("comments", COMMENTS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("commit", COMMIT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("committed", COMMITTED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("concurrently", CONCURRENTLY, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("configuration", CONFIGURATION, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("conflict", CONFLICT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("connection", CONNECTION, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("constraint", CONSTRAINT, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("constraints", CONSTRAINTS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("content", CONTENT_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("continue", CONTINUE_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("conversion", CONVERSION_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("copy", COPY, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("cost", COST, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("create", CREATE, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("cross", CROSS, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("csv", CSV, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("cube", CUBE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("current", CURRENT_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("current_catalog", CURRENT_CATALOG, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("current_date", CURRENT_DATE, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("current_role", CURRENT_ROLE, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("current_schema", CURRENT_SCHEMA, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("current_time", CURRENT_TIME, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("current_timestamp", CURRENT_TIMESTAMP, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("current_user", CURRENT_USER, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("cursor", CURSOR, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("cycle", CYCLE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("data", DATA_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("database", DATABASE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("day", DAY_P, UNRESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("deallocate", DEALLOCATE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("dec", DEC, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("decimal", DECIMAL_P, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("declare", DECLARE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("delimiters", DELIMITERS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("depends", DEPENDS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("desc", DESC, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("detach", DETACH, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("dictionary", DICTIONARY, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("disable", DISABLE_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("discard", DISCARD, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("distinct", DISTINCT, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("do", DO, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("document", DOCUMENT_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("domain", DOMAIN_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("double", DOUBLE_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("drop", DROP, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("each", EACH, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("else", ELSE, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("enable", ENABLE_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("encoding", ENCODING, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("encrypted", ENCRYPTED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("end", END_P, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("enum", ENUM_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("escape", ESCAPE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("event", EVENT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("except", EXCEPT, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("exclude", EXCLUDE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("excluding", EXCLUDING, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("exclusive", EXCLUSIVE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("execute", EXECUTE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("exists", EXISTS, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("explain", EXPLAIN, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("expression", EXPRESSION, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("extension", EXTENSION, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("external", EXTERNAL, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("extract", EXTRACT, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("false", FALSE_P, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("family", FAMILY, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("fetch", FETCH, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("filter", FILTER, UNRESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("first", FIRST_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("float", FLOAT_P, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("following", FOLLOWING, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("for", FOR, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("force", FORCE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("foreign", FOREIGN, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("forward", FORWARD, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("freeze", FREEZE, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("from", FROM, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("full", FULL, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("function", FUNCTION, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("functions", FUNCTIONS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("generated", GENERATED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("global", GLOBAL, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("grant", GRANT, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("granted", GRANTED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("greatest", GREATEST, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("group", GROUP_P, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("grouping", GROUPING, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("groups", GROUPS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("handler", HANDLER, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("having", HAVING, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("header", HEADER_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("hold", HOLD, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("hour", HOUR_P, UNRESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("identity", IDENTITY_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("if", IF_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("ilike", ILIKE, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("immediate", IMMEDIATE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("immutable", IMMUTABLE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("implicit", IMPLICIT_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("import", IMPORT_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("inout", INOUT, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("input", INPUT_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("insensitive", INSENSITIVE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("insert", INSERT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("instead", INSTEAD, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("int", INT_P, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("integer", INTEGER, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("intersect", INTERSECT, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("interval", INTERVAL, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("into", INTO, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("invoker", INVOKER, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("is", IS, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("isnull", ISNULL, TYPE_FUNC_NAME_KEYWORD, AS_LABEL) +PG_KEYWORD("isolation", ISOLATION, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("join", JOIN, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("key", KEY, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("label", LABEL, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("language", LANGUAGE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("large", LARGE_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("last", LAST_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("lateral", LATERAL_P, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("leading", LEADING, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("leakproof", LEAKPROOF, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("least", LEAST, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("left", LEFT, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("level", LEVEL, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("like", LIKE, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("limit", LIMIT, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("listen", LISTEN, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("load", LOAD, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("local", LOCAL, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("localtime", LOCALTIME, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("localtimestamp", LOCALTIMESTAMP, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("location", LOCATION, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("lock", LOCK_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("locked", LOCKED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("logged", LOGGED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("mapping", MAPPING, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("minvalue", MINVALUE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("mode", MODE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("month", MONTH_P, UNRESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("move", MOVE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("name", NAME_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("names", NAMES, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("national", NATIONAL, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("natural", NATURAL, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("nchar", NCHAR, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("new", NEW, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("next", NEXT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("nfc", NFC, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("nfd", NFD, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("nfkc", NFKC, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("nfkd", NFKD, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("no", NO, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("none", NONE, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("normalize", NORMALIZE, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("normalized", NORMALIZED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("not", NOT, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("nothing", NOTHING, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("notify", NOTIFY, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("notnull", NOTNULL, TYPE_FUNC_NAME_KEYWORD, AS_LABEL) +PG_KEYWORD("nowait", NOWAIT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("null", NULL_P, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("nullif", NULLIF, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("nulls", NULLS_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("numeric", NUMERIC, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("object", OBJECT_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("of", OF, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("off", OFF, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("offset", OFFSET, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("oids", OIDS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("old", OLD, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("on", ON, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("only", ONLY, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("operator", OPERATOR, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("option", OPTION, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("options", OPTIONS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("or", OR, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("order", ORDER, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("ordinality", ORDINALITY, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("others", OTHERS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("out", OUT_P, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("outer", OUTER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("over", OVER, UNRESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("overlaps", OVERLAPS, TYPE_FUNC_NAME_KEYWORD, AS_LABEL) +PG_KEYWORD("overlay", OVERLAY, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("overriding", OVERRIDING, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("owned", OWNED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("owner", OWNER, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("parallel", PARALLEL, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("parser", PARSER, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("position", POSITION, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("preceding", PRECEDING, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("precision", PRECISION, COL_NAME_KEYWORD, AS_LABEL) +PG_KEYWORD("prepare", PREPARE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("prepared", PREPARED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("preserve", PRESERVE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("primary", PRIMARY, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("prior", PRIOR, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("privileges", PRIVILEGES, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("procedural", PROCEDURAL, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("procedure", PROCEDURE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("procedures", PROCEDURES, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("program", PROGRAM, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("publication", PUBLICATION, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("quote", QUOTE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("range", RANGE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("read", READ, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("real", REAL, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("reassign", REASSIGN, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("recheck", RECHECK, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("recursive", RECURSIVE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("ref", REF, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("references", REFERENCES, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("referencing", REFERENCING, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("refresh", REFRESH, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("reindex", REINDEX, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("relative", RELATIVE_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("release", RELEASE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("rename", RENAME, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("repeatable", REPEATABLE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("replace", REPLACE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("replica", REPLICA, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("reset", RESET, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("restart", RESTART, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("restrict", RESTRICT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("returning", RETURNING, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("returns", RETURNS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("revoke", REVOKE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("right", RIGHT, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("role", ROLE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("rollback", ROLLBACK, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("rollup", ROLLUP, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("routine", ROUTINE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("routines", ROUTINES, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("row", ROW, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("rows", ROWS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("rule", RULE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("savepoint", SAVEPOINT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("schema", SCHEMA, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("schemas", SCHEMAS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("serializable", SERIALIZABLE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("server", SERVER, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("session", SESSION, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("session_user", SESSION_USER, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("set", SET, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("setof", SETOF, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("sets", SETS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("share", SHARE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("show", SHOW, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("similar", SIMILAR, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("simple", SIMPLE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("skip", SKIP, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("smallint", SMALLINT, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("snapshot", SNAPSHOT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("some", SOME, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("sql", SQL_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("stable", STABLE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("standalone", STANDALONE_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("start", START, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("statement", STATEMENT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("statistics", STATISTICS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("stdin", STDIN, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("stdout", STDOUT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("storage", STORAGE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("sysid", SYSID, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("system", SYSTEM_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("table", TABLE, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("tables", TABLES, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("tablesample", TABLESAMPLE, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("tablespace", TABLESPACE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("temp", TEMP, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("template", TEMPLATE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("temporary", TEMPORARY, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("text", TEXT_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("then", THEN, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("ties", TIES, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("time", TIME, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("timestamp", TIMESTAMP, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("to", TO, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("trailing", TRAILING, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("transaction", TRANSACTION, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("transform", TRANSFORM, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("treat", TREAT, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("trigger", TRIGGER, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("trim", TRIM, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("true", TRUE_P, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("truncate", TRUNCATE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("trusted", TRUSTED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("type", TYPE_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("types", TYPES_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("uescape", UESCAPE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("unbounded", UNBOUNDED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("uncommitted", UNCOMMITTED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("unencrypted", UNENCRYPTED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("union", UNION, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("unique", UNIQUE, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("unknown", UNKNOWN, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("unlisten", UNLISTEN, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("unlogged", UNLOGGED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("until", UNTIL, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("update", UPDATE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("user", USER, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("using", USING, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("vacuum", VACUUM, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("valid", VALID, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("validate", VALIDATE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("validator", VALIDATOR, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("value", VALUE_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("values", VALUES, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("varchar", VARCHAR, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("variadic", VARIADIC, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("varying", VARYING, UNRESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("verbose", VERBOSE, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("version", VERSION_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("view", VIEW, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("views", VIEWS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("volatile", VOLATILE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("when", WHEN, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("where", WHERE, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("whitespace", WHITESPACE_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("window", WINDOW, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("with", WITH, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("within", WITHIN, UNRESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("without", WITHOUT, UNRESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("work", WORK, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("wrapper", WRAPPER, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("write", WRITE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("xml", XML_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("xmlattributes", XMLATTRIBUTES, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("xmlconcat", XMLCONCAT, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("xmlelement", XMLELEMENT, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("xmlexists", XMLEXISTS, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("xmlforest", XMLFOREST, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("xmlnamespaces", XMLNAMESPACES, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("xmlparse", XMLPARSE, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("xmlpi", XMLPI, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("xmlroot", XMLROOT, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("xmlserialize", XMLSERIALIZE, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("xmltable", XMLTABLE, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("year", YEAR_P, UNRESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("yes", YES_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("zone", ZONE, UNRESERVED_KEYWORD, BARE_LABEL) diff --git a/src/interfaces/ecpg/preproc/keywords.c b/src/interfaces/ecpg/preproc/keywords.c index f82764aeb97d..f1640d0062b6 100644 --- a/src/interfaces/ecpg/preproc/keywords.c +++ b/src/interfaces/ecpg/preproc/keywords.c @@ -29,7 +29,7 @@ #include "preproc_extern.h" #include "preproc.h" -#define PG_KEYWORD(kwname, value, category) value, +#define PG_KEYWORD(kwname, value, category, collabel) value, const uint16 SQLScanKeywordTokens[] = { #include "parser/kwlist.h" From be0a6666656ec3f68eb7d8e7abab5139fcd47012 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Sat, 19 Sep 2020 11:28:34 +1200 Subject: [PATCH 176/589] Remove large fill factor support from dynahash.c. Since ancient times we have had support for a fill factor (maximum load factor) to be set for a dynahash hash table, but: 1. It was an integer, whereas for in-memory hash tables interesting load factor targets are probably somewhere near the 0.75-1.0 range. 2. It was implemented in a way that performed an expensive division operation that regularly showed up in profiles. 3. We are not aware of anyone ever having used a non-default value. Therefore, remove support, effectively fixing it at 1. Author: Jakub Wartak Reviewed-by: Alvaro Herrera Reviewed-by: Tomas Vondra Reviewed-by: Thomas Munro Reviewed-by: David Rowley Discussion: https://postgr.es/m/VI1PR0701MB696044FC35013A96FECC7AC8F62D0%40VI1PR0701MB6960.eurprd07.prod.outlook.com --- src/backend/utils/hash/dynahash.c | 20 ++++++-------------- src/include/utils/hsearch.h | 2 -- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/src/backend/utils/hash/dynahash.c b/src/backend/utils/hash/dynahash.c index f4fbccdd7e44..1122e2e5e59a 100644 --- a/src/backend/utils/hash/dynahash.c +++ b/src/backend/utils/hash/dynahash.c @@ -122,7 +122,6 @@ #define DEF_SEGSIZE 256 #define DEF_SEGSIZE_SHIFT 8 /* must be log2(DEF_SEGSIZE) */ #define DEF_DIRSIZE 256 -#define DEF_FFACTOR 1 /* default fill factor */ /* Number of freelists to be used for a partitioned hash table. */ #define NUM_FREELISTS 32 @@ -191,7 +190,6 @@ struct HASHHDR Size keysize; /* hash key length in bytes */ Size entrysize; /* total user element size in bytes */ long num_partitions; /* # partitions (must be power of 2), or 0 */ - long ffactor; /* target fill factor */ long max_dsize; /* 'dsize' limit if directory is fixed size */ long ssize; /* segment size --- must be power of 2 */ int sshift; /* segment shift = log2(ssize) */ @@ -497,8 +495,6 @@ hash_create(const char *tabname, long nelem, HASHCTL *info, int flags) /* ssize had better be a power of 2 */ Assert(hctl->ssize == (1L << hctl->sshift)); } - if (flags & HASH_FFACTOR) - hctl->ffactor = info->ffactor; /* * SHM hash tables have fixed directory size passed by the caller. @@ -603,8 +599,6 @@ hdefault(HTAB *hashp) hctl->num_partitions = 0; /* not partitioned */ - hctl->ffactor = DEF_FFACTOR; - /* table has no fixed maximum size */ hctl->max_dsize = NO_MAX_DSIZE; @@ -670,11 +664,10 @@ init_htab(HTAB *hashp, long nelem) SpinLockInit(&(hctl->freeList[i].mutex)); /* - * Divide number of elements by the fill factor to determine a desired - * number of buckets. Allocate space for the next greater power of two - * number of buckets + * Allocate space for the next greater power of two number of buckets, + * assuming a desired maximum load factor of 1. */ - nbuckets = next_pow2_int((nelem - 1) / hctl->ffactor + 1); + nbuckets = next_pow2_int(nelem); /* * In a partitioned table, nbuckets must be at least equal to @@ -733,7 +726,6 @@ init_htab(HTAB *hashp, long nelem) "DIRECTORY SIZE ", hctl->dsize, "SEGMENT SIZE ", hctl->ssize, "SEGMENT SHIFT ", hctl->sshift, - "FILL FACTOR ", hctl->ffactor, "MAX BUCKET ", hctl->max_bucket, "HIGH MASK ", hctl->high_mask, "LOW MASK ", hctl->low_mask, @@ -761,7 +753,7 @@ hash_estimate_size(long num_entries, Size entrysize) elementAllocCnt; /* estimate number of buckets wanted */ - nBuckets = next_pow2_long((num_entries - 1) / DEF_FFACTOR + 1); + nBuckets = next_pow2_long(num_entries); /* # of segments needed for nBuckets */ nSegments = next_pow2_long((nBuckets - 1) / DEF_SEGSIZE + 1); /* directory entries */ @@ -804,7 +796,7 @@ hash_select_dirsize(long num_entries) nDirEntries; /* estimate number of buckets wanted */ - nBuckets = next_pow2_long((num_entries - 1) / DEF_FFACTOR + 1); + nBuckets = next_pow2_long(num_entries); /* # of segments needed for nBuckets */ nSegments = next_pow2_long((nBuckets - 1) / DEF_SEGSIZE + 1); /* directory entries */ @@ -975,7 +967,7 @@ hash_search_with_hash_value(HTAB *hashp, * order of these tests is to try to check cheaper conditions first. */ if (!IS_PARTITIONED(hctl) && !hashp->frozen && - hctl->freeList[0].nentries / (long) (hctl->max_bucket + 1) >= hctl->ffactor && + hctl->freeList[0].nentries > (long) (hctl->max_bucket + 1) && !has_seq_scans(hashp)) (void) expand_table(hashp); } diff --git a/src/include/utils/hsearch.h b/src/include/utils/hsearch.h index f1deb9beab04..bebf89b3c451 100644 --- a/src/include/utils/hsearch.h +++ b/src/include/utils/hsearch.h @@ -68,7 +68,6 @@ typedef struct HASHCTL long ssize; /* segment size */ long dsize; /* (initial) directory size */ long max_dsize; /* limit to dsize if dir size is limited */ - long ffactor; /* fill factor */ Size keysize; /* hash key length in bytes */ Size entrysize; /* total user element size in bytes */ HashValueFunc hash; /* hash function */ @@ -83,7 +82,6 @@ typedef struct HASHCTL #define HASH_PARTITION 0x0001 /* Hashtable is used w/partitioned locking */ #define HASH_SEGMENT 0x0002 /* Set segment size */ #define HASH_DIRSIZE 0x0004 /* Set directory size (initial and max) */ -#define HASH_FFACTOR 0x0008 /* Set fill factor */ #define HASH_ELEM 0x0010 /* Set keysize and entrysize */ #define HASH_BLOBS 0x0020 /* Select support functions for binary keys */ #define HASH_FUNCTION 0x0040 /* Set user defined hash function */ From ff28809feb442eabd303955277f05cd16d9c6d8a Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Sat, 19 Sep 2020 15:39:48 +1200 Subject: [PATCH 177/589] Code review for dynahash change. Commit be0a6666 left behind a comment about the order of some tests that didn't make sense without the expensive division, and in fact we might as well change the order to one that fails more cheaply most of the time as a micro-optimization. Also, remove the "+ 1" applied to max_bucket, to drop an instruction and match the original behavior. Per review from Tom Lane. Discussion: https://postgr.es/m/VI1PR0701MB696044FC35013A96FECC7AC8F62D0%40VI1PR0701MB6960.eurprd07.prod.outlook.com --- src/backend/utils/hash/dynahash.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/backend/utils/hash/dynahash.c b/src/backend/utils/hash/dynahash.c index 1122e2e5e59a..d14d875c9341 100644 --- a/src/backend/utils/hash/dynahash.c +++ b/src/backend/utils/hash/dynahash.c @@ -963,11 +963,10 @@ hash_search_with_hash_value(HTAB *hashp, { /* * Can't split if running in partitioned mode, nor if frozen, nor if - * table is the subject of any active hash_seq_search scans. Strange - * order of these tests is to try to check cheaper conditions first. + * table is the subject of any active hash_seq_search scans. */ - if (!IS_PARTITIONED(hctl) && !hashp->frozen && - hctl->freeList[0].nentries > (long) (hctl->max_bucket + 1) && + if (hctl->freeList[0].nentries > (long) hctl->max_bucket && + !IS_PARTITIONED(hctl) && !hashp->frozen && !has_seq_scans(hashp)) (void) expand_table(hashp); } From 8354e7b27ebec8bf78088e22b5b9c1caa46db8e1 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 25 Aug 2020 07:24:15 +0200 Subject: [PATCH 178/589] Remove unused parameters Remove various unused parameters in pg_dump code. These have all become unused over time or were never used. Discussion: https://www.postgresql.org/message-id/flat/511bb100-f829-ba21-2f10-9f952ec06ead%402ndquadrant.com --- src/bin/pg_dump/dumputils.c | 2 +- src/bin/pg_dump/dumputils.h | 2 +- src/bin/pg_dump/pg_backup_archiver.c | 20 ++++---- src/bin/pg_dump/pg_backup_tar.c | 8 ++-- src/bin/pg_dump/pg_dump.c | 71 +++++++++++----------------- src/bin/pg_dump/pg_dumpall.c | 2 +- 6 files changed, 44 insertions(+), 61 deletions(-) diff --git a/src/bin/pg_dump/dumputils.c b/src/bin/pg_dump/dumputils.c index 287d485d5f71..2d6ea13c4554 100644 --- a/src/bin/pg_dump/dumputils.c +++ b/src/bin/pg_dump/dumputils.c @@ -685,7 +685,7 @@ AddAcl(PQExpBuffer aclbuf, const char *keyword, const char *subname) * keep this file free of assumptions about how to deal with SQL errors.) */ void -buildShSecLabelQuery(PGconn *conn, const char *catalog_name, Oid objectId, +buildShSecLabelQuery(const char *catalog_name, Oid objectId, PQExpBuffer sql) { appendPQExpBuffer(sql, diff --git a/src/bin/pg_dump/dumputils.h b/src/bin/pg_dump/dumputils.h index cb1d98d873ee..d35d9d34d28c 100644 --- a/src/bin/pg_dump/dumputils.h +++ b/src/bin/pg_dump/dumputils.h @@ -46,7 +46,7 @@ extern bool buildDefaultACLCommands(const char *type, const char *nspname, const char *owner, int remoteVersion, PQExpBuffer sql); -extern void buildShSecLabelQuery(PGconn *conn, const char *catalog_name, +extern void buildShSecLabelQuery(const char *catalog_name, Oid objectId, PQExpBuffer sql); extern void emitShSecLabels(PGconn *conn, PGresult *res, PQExpBuffer buffer, const char *objtype, const char *objname); diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 178b61d6cbc3..bc4757829430 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,8 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, const int compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); -static void _getObjectDescription(PQExpBuffer buf, TocEntry *te, - ArchiveHandle *AH); +static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); static char *sanitize_line(const char *str, bool want_hyphen); static void _doSetFixedOutputState(ArchiveHandle *AH); @@ -91,7 +90,7 @@ static bool _tocEntryIsACL(TocEntry *te); static void _disableTriggersIfNecessary(ArchiveHandle *AH, TocEntry *te); static void _enableTriggersIfNecessary(ArchiveHandle *AH, TocEntry *te); static void buildTocEntryArrays(ArchiveHandle *AH); -static void _moveBefore(ArchiveHandle *AH, TocEntry *pos, TocEntry *te); +static void _moveBefore(TocEntry *pos, TocEntry *te); static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); @@ -121,8 +120,7 @@ static int TocEntrySizeCompare(const void *p1, const void *p2); static void move_to_ready_list(TocEntry *pending_list, ParallelReadyList *ready_list, RestorePass pass); -static TocEntry *pop_next_work_item(ArchiveHandle *AH, - ParallelReadyList *ready_list, +static TocEntry *pop_next_work_item(ParallelReadyList *ready_list, ParallelState *pstate); static void mark_dump_job_done(ArchiveHandle *AH, TocEntry *te, @@ -1442,7 +1440,7 @@ SortTocFromFile(Archive *AHX) * side-effects on the order in which restorable items actually get * restored. */ - _moveBefore(AH, AH->toc, te); + _moveBefore(AH->toc, te); } if (fclose(fh) != 0) @@ -1804,7 +1802,7 @@ _moveAfter(ArchiveHandle *AH, TocEntry *pos, TocEntry *te) #endif static void -_moveBefore(ArchiveHandle *AH, TocEntry *pos, TocEntry *te) +_moveBefore(TocEntry *pos, TocEntry *te) { /* Unlink te from list */ te->prev->next = te->next; @@ -3465,7 +3463,7 @@ _selectTableAccessMethod(ArchiveHandle *AH, const char *tableam) * This is used for ALTER ... OWNER TO. */ static void -_getObjectDescription(PQExpBuffer buf, TocEntry *te, ArchiveHandle *AH) +_getObjectDescription(PQExpBuffer buf, TocEntry *te) { const char *type = te->desc; @@ -3674,7 +3672,7 @@ _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData) PQExpBuffer temp = createPQExpBuffer(); appendPQExpBufferStr(temp, "ALTER "); - _getObjectDescription(temp, te, AH); + _getObjectDescription(temp, te); appendPQExpBuffer(temp, " OWNER TO %s;", fmtId(te->owner)); ahprintf(AH, "%s\n\n", temp->data); destroyPQExpBuffer(temp); @@ -4078,7 +4076,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate, for (;;) { /* Look for an item ready to be dispatched to a worker */ - next_work_item = pop_next_work_item(AH, &ready_list, pstate); + next_work_item = pop_next_work_item(&ready_list, pstate); if (next_work_item != NULL) { /* If not to be restored, don't waste time launching a worker */ @@ -4384,7 +4382,7 @@ move_to_ready_list(TocEntry *pending_list, * no remaining dependencies, but we have to check for lock conflicts. */ static TocEntry * -pop_next_work_item(ArchiveHandle *AH, ParallelReadyList *ready_list, +pop_next_work_item(ParallelReadyList *ready_list, ParallelState *pstate) { /* diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index c601ec07012a..54e708875cd1 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -107,7 +107,7 @@ static void tarClose(ArchiveHandle *AH, TAR_MEMBER *TH); #ifdef __NOT_USED__ static char *tarGets(char *buf, size_t len, TAR_MEMBER *th); #endif -static int tarPrintf(ArchiveHandle *AH, TAR_MEMBER *th, const char *fmt,...) pg_attribute_printf(3, 4); +static int tarPrintf(TAR_MEMBER *th, const char *fmt,...) pg_attribute_printf(2, 3); static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th); static TAR_MEMBER *_tarPositionTo(ArchiveHandle *AH, const char *filename); @@ -851,7 +851,7 @@ _CloseArchive(ArchiveHandle *AH) */ th = tarOpen(AH, "restore.sql", 'w'); - tarPrintf(AH, th, "--\n" + tarPrintf(th, "--\n" "-- NOTE:\n" "--\n" "-- File paths need to be edited. Search for $$PATH$$ and\n" @@ -964,7 +964,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) sprintf(fname, "blob_%u.dat%s", oid, sfx); - tarPrintf(AH, ctx->blobToc, "%u %s\n", oid, fname); + tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); tctx->TH = tarOpen(AH, fname, 'w'); } @@ -1008,7 +1008,7 @@ _EndBlobs(ArchiveHandle *AH, TocEntry *te) */ static int -tarPrintf(ArchiveHandle *AH, TAR_MEMBER *th, const char *fmt,...) +tarPrintf(TAR_MEMBER *th, const char *fmt,...) { int save_errno = errno; char *p; diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index dce6af09c9b7..f021bb72f40a 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -157,7 +157,7 @@ static void expand_table_name_patterns(Archive *fout, SimpleStringList *patterns, SimpleOidList *oids, bool strict_names); -static NamespaceInfo *findNamespace(Archive *fout, Oid nsoid); +static NamespaceInfo *findNamespace(Oid nsoid); static void dumpTableData(Archive *fout, TableDataInfo *tdinfo); static void refreshMatViewData(Archive *fout, TableDataInfo *tdinfo); static void guessConstraintInheritance(TableInfo *tblinfo, int numTables); @@ -250,9 +250,8 @@ static char *format_function_arguments_old(Archive *fout, char **argnames); static char *format_function_signature(Archive *fout, FuncInfo *finfo, bool honor_quotes); -static char *convertRegProcReference(Archive *fout, - const char *proc); -static char *getFormattedOperatorName(Archive *fout, const char *oproid); +static char *convertRegProcReference(const char *proc); +static char *getFormattedOperatorName(const char *oproid); static char *convertTSFunction(Archive *fout, Oid funcOid); static Oid findLastBuiltinOid_V71(Archive *fout); static char *getFormattedTypeName(Archive *fout, Oid oid, OidOptions opts); @@ -2974,7 +2973,7 @@ dumpDatabase(Archive *fout) seclabelQry = createPQExpBuffer(); - buildShSecLabelQuery(conn, "pg_database", dbCatId.oid, seclabelQry); + buildShSecLabelQuery("pg_database", dbCatId.oid, seclabelQry); shres = ExecuteSqlQuery(fout, seclabelQry->data, PGRES_TUPLES_OK); resetPQExpBuffer(seclabelQry); emitShSecLabels(conn, shres, seclabelQry, "DATABASE", datname); @@ -4811,7 +4810,7 @@ getNamespaces(Archive *fout, int *numNamespaces) * given a namespace OID, look up the info read by getNamespaces */ static NamespaceInfo * -findNamespace(Archive *fout, Oid nsoid) +findNamespace(Oid nsoid) { NamespaceInfo *nsinfo; @@ -5069,8 +5068,7 @@ getTypes(Archive *fout, int *numTypes) AssignDumpId(&tyinfo[i].dobj); tyinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_typname)); tyinfo[i].dobj.namespace = - findNamespace(fout, - atooid(PQgetvalue(res, i, i_typnamespace))); + findNamespace(atooid(PQgetvalue(res, i, i_typnamespace))); tyinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); tyinfo[i].typacl = pg_strdup(PQgetvalue(res, i, i_typacl)); tyinfo[i].rtypacl = pg_strdup(PQgetvalue(res, i, i_rtypacl)); @@ -5214,8 +5212,7 @@ getOperators(Archive *fout, int *numOprs) AssignDumpId(&oprinfo[i].dobj); oprinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oprname)); oprinfo[i].dobj.namespace = - findNamespace(fout, - atooid(PQgetvalue(res, i, i_oprnamespace))); + findNamespace(atooid(PQgetvalue(res, i, i_oprnamespace))); oprinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); oprinfo[i].oprkind = (PQgetvalue(res, i, i_oprkind))[0]; oprinfo[i].oprcode = atooid(PQgetvalue(res, i, i_oprcode)); @@ -5300,8 +5297,7 @@ getCollations(Archive *fout, int *numCollations) AssignDumpId(&collinfo[i].dobj); collinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_collname)); collinfo[i].dobj.namespace = - findNamespace(fout, - atooid(PQgetvalue(res, i, i_collnamespace))); + findNamespace(atooid(PQgetvalue(res, i, i_collnamespace))); collinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); /* Decide whether we want to dump it */ @@ -5373,8 +5369,7 @@ getConversions(Archive *fout, int *numConversions) AssignDumpId(&convinfo[i].dobj); convinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_conname)); convinfo[i].dobj.namespace = - findNamespace(fout, - atooid(PQgetvalue(res, i, i_connamespace))); + findNamespace(atooid(PQgetvalue(res, i, i_connamespace))); convinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); /* Decide whether we want to dump it */ @@ -5518,8 +5513,7 @@ getOpclasses(Archive *fout, int *numOpclasses) AssignDumpId(&opcinfo[i].dobj); opcinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_opcname)); opcinfo[i].dobj.namespace = - findNamespace(fout, - atooid(PQgetvalue(res, i, i_opcnamespace))); + findNamespace(atooid(PQgetvalue(res, i, i_opcnamespace))); opcinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); /* Decide whether we want to dump it */ @@ -5602,8 +5596,7 @@ getOpfamilies(Archive *fout, int *numOpfamilies) AssignDumpId(&opfinfo[i].dobj); opfinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_opfname)); opfinfo[i].dobj.namespace = - findNamespace(fout, - atooid(PQgetvalue(res, i, i_opfnamespace))); + findNamespace(atooid(PQgetvalue(res, i, i_opfnamespace))); opfinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); /* Decide whether we want to dump it */ @@ -5778,8 +5771,7 @@ getAggregates(Archive *fout, int *numAggs) AssignDumpId(&agginfo[i].aggfn.dobj); agginfo[i].aggfn.dobj.name = pg_strdup(PQgetvalue(res, i, i_aggname)); agginfo[i].aggfn.dobj.namespace = - findNamespace(fout, - atooid(PQgetvalue(res, i, i_aggnamespace))); + findNamespace(atooid(PQgetvalue(res, i, i_aggnamespace))); agginfo[i].aggfn.rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); if (strlen(agginfo[i].aggfn.rolname) == 0) pg_log_warning("owner of aggregate function \"%s\" appears to be invalid", @@ -6013,8 +6005,7 @@ getFuncs(Archive *fout, int *numFuncs) AssignDumpId(&finfo[i].dobj); finfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_proname)); finfo[i].dobj.namespace = - findNamespace(fout, - atooid(PQgetvalue(res, i, i_pronamespace))); + findNamespace(atooid(PQgetvalue(res, i, i_pronamespace))); finfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); finfo[i].lang = atooid(PQgetvalue(res, i, i_prolang)); finfo[i].prorettype = atooid(PQgetvalue(res, i, i_prorettype)); @@ -6751,8 +6742,7 @@ getTables(Archive *fout, int *numTables) AssignDumpId(&tblinfo[i].dobj); tblinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_relname)); tblinfo[i].dobj.namespace = - findNamespace(fout, - atooid(PQgetvalue(res, i, i_relnamespace))); + findNamespace(atooid(PQgetvalue(res, i, i_relnamespace))); tblinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); tblinfo[i].relacl = pg_strdup(PQgetvalue(res, i, i_relacl)); tblinfo[i].rrelacl = pg_strdup(PQgetvalue(res, i, i_rrelacl)); @@ -7413,8 +7403,7 @@ getExtendedStatistics(Archive *fout) AssignDumpId(&statsextinfo[i].dobj); statsextinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_stxname)); statsextinfo[i].dobj.namespace = - findNamespace(fout, - atooid(PQgetvalue(res, i, i_stxnamespace))); + findNamespace(atooid(PQgetvalue(res, i, i_stxnamespace))); statsextinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); statsextinfo[i].stattarget = atoi(PQgetvalue(res, i, i_stattarget)); @@ -8952,8 +8941,7 @@ getTSParsers(Archive *fout, int *numTSParsers) AssignDumpId(&prsinfo[i].dobj); prsinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_prsname)); prsinfo[i].dobj.namespace = - findNamespace(fout, - atooid(PQgetvalue(res, i, i_prsnamespace))); + findNamespace(atooid(PQgetvalue(res, i, i_prsnamespace))); prsinfo[i].prsstart = atooid(PQgetvalue(res, i, i_prsstart)); prsinfo[i].prstoken = atooid(PQgetvalue(res, i, i_prstoken)); prsinfo[i].prsend = atooid(PQgetvalue(res, i, i_prsend)); @@ -9035,8 +9023,7 @@ getTSDictionaries(Archive *fout, int *numTSDicts) AssignDumpId(&dictinfo[i].dobj); dictinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_dictname)); dictinfo[i].dobj.namespace = - findNamespace(fout, - atooid(PQgetvalue(res, i, i_dictnamespace))); + findNamespace(atooid(PQgetvalue(res, i, i_dictnamespace))); dictinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); dictinfo[i].dicttemplate = atooid(PQgetvalue(res, i, i_dicttemplate)); if (PQgetisnull(res, i, i_dictinitoption)) @@ -9115,8 +9102,7 @@ getTSTemplates(Archive *fout, int *numTSTemplates) AssignDumpId(&tmplinfo[i].dobj); tmplinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_tmplname)); tmplinfo[i].dobj.namespace = - findNamespace(fout, - atooid(PQgetvalue(res, i, i_tmplnamespace))); + findNamespace(atooid(PQgetvalue(res, i, i_tmplnamespace))); tmplinfo[i].tmplinit = atooid(PQgetvalue(res, i, i_tmplinit)); tmplinfo[i].tmpllexize = atooid(PQgetvalue(res, i, i_tmpllexize)); @@ -9192,8 +9178,7 @@ getTSConfigurations(Archive *fout, int *numTSConfigs) AssignDumpId(&cfginfo[i].dobj); cfginfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_cfgname)); cfginfo[i].dobj.namespace = - findNamespace(fout, - atooid(PQgetvalue(res, i, i_cfgnamespace))); + findNamespace(atooid(PQgetvalue(res, i, i_cfgnamespace))); cfginfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); cfginfo[i].cfgparser = atooid(PQgetvalue(res, i, i_cfgparser)); @@ -9640,7 +9625,7 @@ getDefaultACLs(Archive *fout, int *numDefaultACLs) daclinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_defaclobjtype)); if (nspid != InvalidOid) - daclinfo[i].dobj.namespace = findNamespace(fout, nspid); + daclinfo[i].dobj.namespace = findNamespace(nspid); else daclinfo[i].dobj.namespace = NULL; @@ -12655,7 +12640,7 @@ dumpOpr(Archive *fout, OprInfo *oprinfo) pg_log_warning("postfix operators are not supported anymore (operator \"%s\")", oprcode); - oprregproc = convertRegProcReference(fout, oprcode); + oprregproc = convertRegProcReference(oprcode); if (oprregproc) { appendPQExpBuffer(details, " FUNCTION = %s", oprregproc); @@ -12688,14 +12673,14 @@ dumpOpr(Archive *fout, OprInfo *oprinfo) else appendPQExpBufferStr(oprid, ", NONE)"); - oprref = getFormattedOperatorName(fout, oprcom); + oprref = getFormattedOperatorName(oprcom); if (oprref) { appendPQExpBuffer(details, ",\n COMMUTATOR = %s", oprref); free(oprref); } - oprref = getFormattedOperatorName(fout, oprnegate); + oprref = getFormattedOperatorName(oprnegate); if (oprref) { appendPQExpBuffer(details, ",\n NEGATOR = %s", oprref); @@ -12708,14 +12693,14 @@ dumpOpr(Archive *fout, OprInfo *oprinfo) if (strcmp(oprcanhash, "t") == 0) appendPQExpBufferStr(details, ",\n HASHES"); - oprregproc = convertRegProcReference(fout, oprrest); + oprregproc = convertRegProcReference(oprrest); if (oprregproc) { appendPQExpBuffer(details, ",\n RESTRICT = %s", oprregproc); free(oprregproc); } - oprregproc = convertRegProcReference(fout, oprjoin); + oprregproc = convertRegProcReference(oprjoin); if (oprregproc) { appendPQExpBuffer(details, ",\n JOIN = %s", oprregproc); @@ -12770,7 +12755,7 @@ dumpOpr(Archive *fout, OprInfo *oprinfo) * part. */ static char * -convertRegProcReference(Archive *fout, const char *proc) +convertRegProcReference(const char *proc) { char *name; char *paren; @@ -12811,7 +12796,7 @@ convertRegProcReference(Archive *fout, const char *proc) * are in different schemas. */ static char * -getFormattedOperatorName(Archive *fout, const char *oproid) +getFormattedOperatorName(const char *oproid) { OprInfo *oprInfo; @@ -14121,7 +14106,7 @@ dumpAgg(Archive *fout, AggInfo *agginfo) } } - aggsortconvop = getFormattedOperatorName(fout, aggsortop); + aggsortconvop = getFormattedOperatorName(aggsortop); if (aggsortconvop) { appendPQExpBuffer(details, ",\n SORTOP = %s", diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 219ca963c3b7..2fa11745cc33 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -1614,7 +1614,7 @@ buildShSecLabels(PGconn *conn, const char *catalog_name, Oid objectId, PQExpBuffer sql = createPQExpBuffer(); PGresult *res; - buildShSecLabelQuery(conn, catalog_name, objectId, sql); + buildShSecLabelQuery(catalog_name, objectId, sql); res = executeQuery(conn, sql->data); emitShSecLabels(conn, res, buffer, objtype, objname); From 28a61fc6c58fbfb83e416411f55e7d6ec47279ff Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 19 Sep 2020 15:11:26 -0400 Subject: [PATCH 179/589] Remove precedence hacks no longer needed without postfix operators. It's no longer necessary to assign explicit precedences to GENERATED, NULL_P, PRESERVE, or STRIP_P. Actually, we don't need to assign precedence to IDENT either; that was really just there to govern the behavior of target_el's "a_expr IDENT" production, which no longer ends with that terminal. However, it seems like a good idea to continue to do so, because it provides a reference point for a precedence level that we can assign to other unreserved keywords that lack a natural precedence level. Research by Peter Eisentraut and John Naylor; comment rewrite by me. Discussion: https://postgr.es/m/38ca86db-42ab-9b48-2902-337a0d6b8311@2ndquadrant.com --- src/backend/parser/gram.y | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 017940bdcd62..17653ef3a790 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -744,22 +744,15 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %nonassoc BETWEEN IN_P LIKE ILIKE SIMILAR NOT_LA %nonassoc ESCAPE /* ESCAPE must be just above LIKE/ILIKE/SIMILAR */ /* - * To support target_el without AS, we must give IDENT an explicit priority - * between ESCAPE and Op. We can safely assign the same priority to - * various unreserved keywords as needed to resolve ambiguities (this can't - * have any bad effects since obviously the keywords will still behave the - * same as if they weren't keywords). We need to do this: - * for PARTITION, RANGE, ROWS, GROUPS to support opt_existing_window_name; - * for RANGE, ROWS, GROUPS so that they can follow a_expr without creating - * postfix-operator problems; - * for GENERATED so that it can follow b_expr; - * and for NULL so that it can follow b_expr in ColQualList without creating - * postfix-operator problems. + * To support target_el without AS, it used to be necessary to assign IDENT an + * explicit precedence just less than Op. While that's not really necessary + * since we removed postfix operators, it's still helpful to do so because + * there are some other unreserved keywords that need precedence assignments. + * If those keywords have the same precedence as IDENT then they clearly act + * the same as non-keywords, reducing the risk of unwanted precedence effects. * - * To support CUBE and ROLLUP in GROUP BY without reserving them, we give them - * an explicit priority lower than '(', so that a rule with CUBE '(' will shift - * rather than reducing a conflicting rule that takes CUBE as a function name. - * Using the same precedence as IDENT seems right for the reasons given above. + * We need to do this for PARTITION, RANGE, ROWS, and GROUPS to support + * opt_existing_window_name (see comment there). * * The frame_bound productions UNBOUNDED PRECEDING and UNBOUNDED FOLLOWING * are even messier: since UNBOUNDED is an unreserved keyword (per spec!), @@ -769,9 +762,14 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); * appear to cause UNBOUNDED to be treated differently from other unreserved * keywords anywhere else in the grammar, but it's definitely risky. We can * blame any funny behavior of UNBOUNDED on the SQL standard, though. + * + * To support CUBE and ROLLUP in GROUP BY without reserving them, we give them + * an explicit priority lower than '(', so that a rule with CUBE '(' will shift + * rather than reducing a conflicting rule that takes CUBE as a function name. + * Using the same precedence as IDENT seems right for the reasons given above. */ -%nonassoc UNBOUNDED /* ideally should have same precedence as IDENT */ -%nonassoc IDENT GENERATED NULL_P PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP +%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */ +%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP %left Op OPERATOR /* multi-character ops and user-defined operators */ %left '+' '-' %left '*' '/' '%' @@ -792,8 +790,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); * left-associativity among the JOIN rules themselves. */ %left JOIN CROSS LEFT FULL RIGHT INNER_P NATURAL -/* kluge to keep xml_whitespace_option from causing shift/reduce conflicts */ -%right PRESERVE STRIP_P %% From 3d13867a2c0b0b9cd1215107d4d6aa94e975b33b Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sun, 20 Sep 2020 14:42:54 +0200 Subject: [PATCH 180/589] Fix whitespace --- src/backend/storage/ipc/procarray.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c index 802b119c4904..5aaeb6e2b558 100644 --- a/src/backend/storage/ipc/procarray.c +++ b/src/backend/storage/ipc/procarray.c @@ -1699,8 +1699,8 @@ ComputeXidHorizons(ComputeXidHorizonsResult *h) */ xmin = TransactionIdOlder(xmin, xid); - /* if neither is set, this proc doesn't influence the horizon */ - if (!TransactionIdIsValid(xmin)) + /* if neither is set, this proc doesn't influence the horizon */ + if (!TransactionIdIsValid(xmin)) continue; /* From c2bb287025189d93c8df16b28f2a3d9d07d25655 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 20 Sep 2020 17:08:49 -0400 Subject: [PATCH 181/589] Fix new GIST build code to work under CLOBBER_CACHE_ALWAYS. Can't say if this fixes *all* cases, but at least we get through the "point" regression test now, which hyrax's last run did not. Report: https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=hyrax&dt=2020-09-19%2021%3A27%3A23 --- src/backend/access/gist/gistbuild.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/access/gist/gistbuild.c b/src/backend/access/gist/gistbuild.c index 230625cf1e2c..5ecc4c87d119 100644 --- a/src/backend/access/gist/gistbuild.c +++ b/src/backend/access/gist/gistbuild.c @@ -408,6 +408,7 @@ gist_indexsortbuild(GISTBuildState *state) * replaced with the real root page at the end. */ page = palloc0(BLCKSZ); + RelationOpenSmgr(state->indexrel); smgrextend(state->indexrel->rd_smgr, MAIN_FORKNUM, GIST_ROOT_BLKNO, page, true); state->pages_allocated++; From c47a240fe6db8293e2de1565233faee40afa64b6 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 21 Sep 2020 14:50:07 +0300 Subject: [PATCH 182/589] Fix checksum calculation in the new sorting GiST build. Since we're bypassing the buffer manager, we need to call PageSetChecksumInplace() directly. As reported by Justin Pryzby. In the passing, add RelationOpenSmgr() calls before all smgrwrite() and smgrextend() calls. Tom added one before the first smgrextend() call in commit c2bb287025, which seems to be enough, but let's play it safe and do it before each one. That's how it's done in the similar code in nbtsort.c, too. Discussion: https://www.postgresql.org/message-id/20200920224446.GF30557@telsasoft.com --- src/backend/access/gist/gistbuild.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/backend/access/gist/gistbuild.c b/src/backend/access/gist/gistbuild.c index 5ecc4c87d119..188e33642f3d 100644 --- a/src/backend/access/gist/gistbuild.c +++ b/src/backend/access/gist/gistbuild.c @@ -449,7 +449,9 @@ gist_indexsortbuild(GISTBuildState *state) gist_indexsortbuild_flush_ready_pages(state); /* Write out the root */ + RelationOpenSmgr(state->indexrel); PageSetLSN(pagestate->page, GistBuildLSN); + PageSetChecksumInplace(pagestate->page, GIST_ROOT_BLKNO); smgrwrite(state->indexrel->rd_smgr, MAIN_FORKNUM, GIST_ROOT_BLKNO, pagestate->page, true); if (RelationNeedsWAL(state->indexrel)) @@ -546,21 +548,22 @@ gist_indexsortbuild_flush_ready_pages(GISTBuildState *state) if (state->ready_num_pages == 0) return; + RelationOpenSmgr(state->indexrel); + for (int i = 0; i < state->ready_num_pages; i++) { Page page = state->ready_pages[i]; + BlockNumber blkno = state->ready_blknos[i]; /* Currently, the blocks must be buffered in order. */ - if (state->ready_blknos[i] != state->pages_written) + if (blkno != state->pages_written) elog(ERROR, "unexpected block number to flush GiST sorting build"); PageSetLSN(page, GistBuildLSN); + PageSetChecksumInplace(page, blkno); + smgrextend(state->indexrel->rd_smgr, MAIN_FORKNUM, blkno, page, true); - smgrextend(state->indexrel->rd_smgr, - MAIN_FORKNUM, - state->pages_written++, - page, - true); + state->pages_written++; } if (RelationNeedsWAL(state->indexrel)) From 80fc96eceb817d89cdd11c02c8ab913c8d060a3e Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 21 Sep 2020 15:22:41 +0200 Subject: [PATCH 183/589] Standardize order of use strict and use warnings in Perl code The standard order in PostgreSQL and other code is use strict first, but some code was uselessly inconsistent about this. --- doc/src/sgml/generate-errcodes-table.pl | 2 +- src/backend/parser/check_keywords.pl | 2 +- src/backend/storage/lmgr/generate-lwlocknames.pl | 2 +- src/backend/utils/generate-errcodes.pl | 2 +- src/pl/plpgsql/src/generate-plerrcodes.pl | 2 +- src/pl/plpython/generate-spiexceptions.pl | 2 +- src/pl/tcl/generate-pltclerrcodes.pl | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/src/sgml/generate-errcodes-table.pl b/doc/src/sgml/generate-errcodes-table.pl index 9a1886059b2b..66a3ee00298a 100644 --- a/doc/src/sgml/generate-errcodes-table.pl +++ b/doc/src/sgml/generate-errcodes-table.pl @@ -3,8 +3,8 @@ # Generate the errcodes-table.sgml file from errcodes.txt # Copyright (c) 2000-2020, PostgreSQL Global Development Group -use warnings; use strict; +use warnings; print "\n"; diff --git a/src/backend/parser/check_keywords.pl b/src/backend/parser/check_keywords.pl index 3862db072780..e6c6c98fb5ec 100644 --- a/src/backend/parser/check_keywords.pl +++ b/src/backend/parser/check_keywords.pl @@ -6,8 +6,8 @@ # src/backend/parser/check_keywords.pl # Copyright (c) 2009-2020, PostgreSQL Global Development Group -use warnings; use strict; +use warnings; my $gram_filename = $ARGV[0]; my $kwlist_filename = $ARGV[1]; diff --git a/src/backend/storage/lmgr/generate-lwlocknames.pl b/src/backend/storage/lmgr/generate-lwlocknames.pl index ca54acdfb0f8..39cb97f5c3d9 100644 --- a/src/backend/storage/lmgr/generate-lwlocknames.pl +++ b/src/backend/storage/lmgr/generate-lwlocknames.pl @@ -3,8 +3,8 @@ # Generate lwlocknames.h and lwlocknames.c from lwlocknames.txt # Copyright (c) 2000-2020, PostgreSQL Global Development Group -use warnings; use strict; +use warnings; my $lastlockidx = -1; my $continue = "\n"; diff --git a/src/backend/utils/generate-errcodes.pl b/src/backend/utils/generate-errcodes.pl index 868a163578d7..1a071fbb1f43 100644 --- a/src/backend/utils/generate-errcodes.pl +++ b/src/backend/utils/generate-errcodes.pl @@ -3,8 +3,8 @@ # Generate the errcodes.h header from errcodes.txt # Copyright (c) 2000-2020, PostgreSQL Global Development Group -use warnings; use strict; +use warnings; print "/* autogenerated from src/backend/utils/errcodes.txt, do not edit */\n"; diff --git a/src/pl/plpgsql/src/generate-plerrcodes.pl b/src/pl/plpgsql/src/generate-plerrcodes.pl index f74dd0ef03f8..32d251c8c2f1 100644 --- a/src/pl/plpgsql/src/generate-plerrcodes.pl +++ b/src/pl/plpgsql/src/generate-plerrcodes.pl @@ -3,8 +3,8 @@ # Generate the plerrcodes.h header from errcodes.txt # Copyright (c) 2000-2020, PostgreSQL Global Development Group -use warnings; use strict; +use warnings; print "/* autogenerated from src/backend/utils/errcodes.txt, do not edit */\n"; diff --git a/src/pl/plpython/generate-spiexceptions.pl b/src/pl/plpython/generate-spiexceptions.pl index d7afb8516ded..fba603e9ec89 100644 --- a/src/pl/plpython/generate-spiexceptions.pl +++ b/src/pl/plpython/generate-spiexceptions.pl @@ -3,8 +3,8 @@ # Generate the spiexceptions.h header from errcodes.txt # Copyright (c) 2000-2020, PostgreSQL Global Development Group -use warnings; use strict; +use warnings; print "/* autogenerated from src/backend/utils/errcodes.txt, do not edit */\n"; diff --git a/src/pl/tcl/generate-pltclerrcodes.pl b/src/pl/tcl/generate-pltclerrcodes.pl index 59ea2baab7ed..43417a4bb4c2 100644 --- a/src/pl/tcl/generate-pltclerrcodes.pl +++ b/src/pl/tcl/generate-pltclerrcodes.pl @@ -3,8 +3,8 @@ # Generate the pltclerrcodes.h header from errcodes.txt # Copyright (c) 2000-2020, PostgreSQL Global Development Group -use warnings; use strict; +use warnings; print "/* autogenerated from src/backend/utils/errcodes.txt, do not edit */\n"; From 9436041ed848debb3d64fb5fbff6cdb35bc46d04 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 21 Sep 2020 12:43:42 -0400 Subject: [PATCH 184/589] Copy editing: fix a bunch of misspellings and poor wording. 99% of this is docs, but also a couple of comments. No code changes. Justin Pryzby Discussion: https://postgr.es/m/20200919175804.GE30557@telsasoft.com --- doc/src/sgml/btree.sgml | 2 +- doc/src/sgml/catalogs.sgml | 3 ++- doc/src/sgml/config.sgml | 22 ++++++++++---------- doc/src/sgml/ddl.sgml | 2 +- doc/src/sgml/func.sgml | 4 ++-- doc/src/sgml/libpq.sgml | 2 +- doc/src/sgml/logical-replication.sgml | 2 +- doc/src/sgml/monitoring.sgml | 4 ++-- doc/src/sgml/protocol.sgml | 4 ++-- doc/src/sgml/ref/alter_statistics.sgml | 3 ++- doc/src/sgml/ref/alter_table.sgml | 2 +- doc/src/sgml/ref/pg_basebackup.sgml | 4 ++-- doc/src/sgml/ref/pg_dump.sgml | 2 +- doc/src/sgml/ref/pg_rewind.sgml | 4 ++-- doc/src/sgml/ref/pgbench.sgml | 4 ++-- doc/src/sgml/ref/reindex.sgml | 2 +- doc/src/sgml/ref/reindexdb.sgml | 4 ++-- doc/src/sgml/ref/vacuumdb.sgml | 4 ++-- doc/src/sgml/runtime.sgml | 2 +- doc/src/sgml/sources.sgml | 2 +- src/backend/access/gin/README | 2 +- src/backend/utils/adt/jsonpath_exec.c | 2 +- src/test/regress/expected/partition_join.out | 2 +- src/test/regress/sql/partition_join.sql | 2 +- 24 files changed, 44 insertions(+), 42 deletions(-) diff --git a/doc/src/sgml/btree.sgml b/doc/src/sgml/btree.sgml index 20cabe921fb3..bb395e6a85c1 100644 --- a/doc/src/sgml/btree.sgml +++ b/doc/src/sgml/btree.sgml @@ -642,7 +642,7 @@ options(relopts local_relopts *) returns Deduplication works by periodically merging groups of duplicate - tuples together, forming a single posting list tuple for each + tuples together, forming a single posting list tuple for each group. The column key value(s) only appear once in this representation. This is followed by a sorted array of TIDs that point to rows in the table. This diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 7e99928d0c1c..de9bacd34f8d 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -7298,7 +7298,8 @@ SCRAM-SHA-256$<iteration count>:&l of statistics accumulated for this statistics object by . A zero value indicates that no statistics should be collected. - A negative value says to use the system default statistics target. + A negative value says to use the maximum of the statistics targets of + the referenced columns, if set, or the system default statistics target. Positive values of stxstattarget determine the target number of most common values to collect. diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 2c75876e3224..8eabf93834a4 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3813,7 +3813,7 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows servers or streaming base backup clients (i.e., the maximum number of simultaneously running WAL sender processes). The default is 10. The value 0 means - replication is disabled. Abrupt streaming client disconnection might + replication is disabled. Abrupt disconnection of a streaming client might leave an orphaned connection slot behind until a timeout is reached, so this parameter should be set slightly higher than the maximum number of expected clients so disconnected clients can immediately @@ -3902,9 +3902,9 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows slots are allowed to retain in the pg_wal directory at checkpoint time. If max_slot_wal_keep_size is -1 (the default), - replication slots retain unlimited amount of WAL files. If - restart_lsn of a replication slot gets behind more than that megabytes - from the current LSN, the standby using the slot may no longer be able + replication slots may retain an unlimited amount of WAL files. Otherwise, if + restart_lsn of a replication slot falls behind the current LSN by more + than the given size, the standby using the slot may no longer be able to continue replication due to removal of required WAL files. You can see the WAL availability of replication slots in pg_replication_slots. @@ -6847,9 +6847,9 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' - If greater than zero, each bind parameter value reported in - non-error statement-logging messages is trimmed to this many bytes. - Zero disables logging bind parameters with statements. + If greater than zero, each bind parameter value logged with a + non-error statement-logging message is trimmed to this many bytes. + Zero disables logging of bind parameters for non-error statement logs. -1 (the default) allows bind parameters to be logged in full. If this value is specified without units, it is taken as bytes. @@ -8224,10 +8224,10 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; a regular VACUUM in that it visits every page that might contain unfrozen XIDs or MXIDs, not just those that might contain dead tuples. The default is 150 million transactions. Although users can - set this value anywhere from zero to two billions, VACUUM + set this value anywhere from zero to two billion, VACUUM will silently limit the effective value to 95% of , so that a - periodical manual VACUUM has a chance to run before an + periodic manual VACUUM has a chance to run before an anti-wraparound autovacuum is launched for the table. For more information see . @@ -8271,10 +8271,10 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; a regular VACUUM in that it visits every page that might contain unfrozen XIDs or MXIDs, not just those that might contain dead tuples. The default is 150 million multixacts. - Although users can set this value anywhere from zero to two billions, + Although users can set this value anywhere from zero to two billion, VACUUM will silently limit the effective value to 95% of , so that a - periodical manual VACUUM has a chance to run before an + periodic manual VACUUM has a chance to run before an anti-wraparound is launched for the table. For more information see . diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index 6004dd1def9a..e890bdd10ad7 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -3992,7 +3992,7 @@ ALTER TABLE measurement ATTACH PARTITION measurement_y2008m02 Before running the ATTACH PARTITION command, it is recommended to create a CHECK constraint on the table to be attached matching the desired partition constraint. That way, - the system will be able to skip the scan to validate the implicit + the system will be able to skip the scan which is otherwise needed to validate the implicit partition constraint. Without the CHECK constraint, the table will be scanned to validate the partition constraint while holding an ACCESS EXCLUSIVE lock on that partition diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 5d486258ed29..461b748d890b 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -15626,11 +15626,11 @@ table2-mapping 'use_json_null'. - jsonb_set_lax('[{"f1":1,"f2":null},2,null,3]', '{0,f1}',null) + jsonb_set_lax('[{"f1":1,"f2":null},2,null,3]', '{0,f1}', null) [{"f1":null,"f2":null},2,null,3] - jsonb_set_lax('[{"f1":99,"f2":null},2]', '{0,f3}',null, true, 'return_target') + jsonb_set_lax('[{"f1":99,"f2":null},2]', '{0,f3}', null, true, 'return_target') [{"f1": 99, "f2": null}, 2] diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index b50391caee05..3315f1dd05b5 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -1227,7 +1227,7 @@ postgresql://%2Fvar%2Flib%2Fpostgresql/dbname connect_timeout - Maximum wait for connection, in seconds (write as a decimal integer, + Maximum time to wait while connecting, in seconds (write as a decimal integer, e.g., 10). Zero, negative, or not specified means wait indefinitely. The minimum allowed timeout is 2 seconds, therefore a value of 1 is interpreted as 2. diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index c35415801f6f..87eac7be5897 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -403,7 +403,7 @@ Replication is only supported by tables, including partitioned tables. - Attempts to replicate other types of relations such as views, materialized + Attempts to replicate other types of relations, such as views, materialized views, or foreign tables, will result in an error. diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 673a0e73e453..4e0193a967cb 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -6089,8 +6089,8 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid, waiting for checkpoint to finish The WAL sender process is currently performing - pg_start_backup to set up for - taking a base backup, and waiting for backup start + pg_start_backup to prepare to + take a base backup, and waiting for the start-of-backup checkpoint to finish. diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index c3cb7b4255fd..f5e33181061b 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2587,7 +2587,7 @@ The commands accepted in replication mode are: and sent along with the backup. The manifest is a list of every file present in the backup with the exception of any WAL files that may be included. It also stores the size, last modification time, and - an optional checksum for each file. + optionally a checksum for each file. A value of force-encode forces all filenames to be hex-encoded; otherwise, this type of encoding is performed only for files whose names are non-UTF8 octet sequences. @@ -2603,7 +2603,7 @@ The commands accepted in replication mode are: MANIFEST_CHECKSUMS checksum_algorithm - Specifies the algorithm that should be applied to each file included + Specifies the checksum algorithm that should be applied to each file included in the backup manifest. Currently, the available algorithms are NONE, CRC32C, SHA224, SHA256, diff --git a/doc/src/sgml/ref/alter_statistics.sgml b/doc/src/sgml/ref/alter_statistics.sgml index 112f5d892497..5f60ca8eac2a 100644 --- a/doc/src/sgml/ref/alter_statistics.sgml +++ b/doc/src/sgml/ref/alter_statistics.sgml @@ -101,7 +101,8 @@ ALTER STATISTICS name SET STATISTIC The statistic-gathering target for this statistics object for subsequent operations. The target can be set in the range 0 to 10000; alternatively, set it - to -1 to revert to using the system default statistics + to -1 to revert to using the maximum of the statistics target of the + referenced columns, if set, or the system default statistics target (). For more information on the use of statistics by the PostgreSQL query planner, refer to diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index 35971cd72333..f034e75ec099 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -669,7 +669,7 @@ WITH ( MODULUS numeric_literal, REM When applied to a partitioned table, nothing is moved, but any partitions created afterwards with CREATE TABLE PARTITION OF will use that tablespace, - unless the TABLESPACE clause is used to override it. + unless overridden by a TABLESPACE clause. diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index aa0b27c9f300..44e7b2444b7e 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -368,7 +368,7 @@ PostgreSQL documentation The following command-line options control the generation of the - backup and the running of the program: + backup and the invocation of the program: @@ -540,7 +540,7 @@ PostgreSQL documentation of each file for users who wish to verify that the backup has not been tampered with, while the CRC32C algorithm provides a checksum that is much faster to calculate; it is good at catching errors due to accidental - changes but is not resistant to targeted modifications. Note that, to + changes but is not resistant to malicious modifications. Note that, to be useful against an adversary who has access to the backup, the backup manifest would need to be stored securely elsewhere or otherwise verified not to have been modified since the backup was taken. diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 0ec99165c5c0..e09ed0a4c3ac 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -322,7 +322,7 @@ PostgreSQL documentation Run the dump in parallel by dumping njobs - tables simultaneously. This option reduces the time of the dump but it also + tables simultaneously. This option may reduce the time needed to perform the dump but it also increases the load on the database server. You can only use this option with the directory output format because this is the only output format where multiple processes can write their data at the same time. diff --git a/doc/src/sgml/ref/pg_rewind.sgml b/doc/src/sgml/ref/pg_rewind.sgml index ae23badc08c4..688acdcb06af 100644 --- a/doc/src/sgml/ref/pg_rewind.sgml +++ b/doc/src/sgml/ref/pg_rewind.sgml @@ -211,7 +211,7 @@ PostgreSQL documentation pg_rewind to return without waiting, which is faster, but means that a subsequent operating system crash can leave the synchronized data directory corrupt. Generally, this option is - useful for testing but should not be used when creating a production + useful for testing but should not be used on a production installation. @@ -322,7 +322,7 @@ GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text, bigint, bigint, b When executing pg_rewind using an online cluster as source which has been recently promoted, it is necessary - to execute a CHECKPOINT after promotion so as its + to execute a CHECKPOINT after promotion such that its control file reflects up-to-date timeline information, which is used by pg_rewind to check if the target cluster can be rewound using the designated source cluster. diff --git a/doc/src/sgml/ref/pgbench.sgml b/doc/src/sgml/ref/pgbench.sgml index 75575b6f0604..7180fedd6585 100644 --- a/doc/src/sgml/ref/pgbench.sgml +++ b/doc/src/sgml/ref/pgbench.sgml @@ -812,8 +812,8 @@ pgbench options d Common Options - pgbench accepts the following command-line - common arguments: + pgbench also accepts the following common command-line + arguments for connection parameters: diff --git a/doc/src/sgml/ref/reindex.sgml b/doc/src/sgml/ref/reindex.sgml index 33af4ae02a13..fa43e3a97202 100644 --- a/doc/src/sgml/ref/reindex.sgml +++ b/doc/src/sgml/ref/reindex.sgml @@ -252,7 +252,7 @@ REINDEX [ ( option [, ...] ) ] { IN Reindexing a single index or table requires being the owner of that index or table. Reindexing a schema or database requires being the - owner of that schema or database. Note that is therefore sometimes + owner of that schema or database. Note specifically that it's thus possible for non-superusers to rebuild indexes of tables owned by other users. However, as a special exception, when REINDEX DATABASE, REINDEX SCHEMA diff --git a/doc/src/sgml/ref/reindexdb.sgml b/doc/src/sgml/ref/reindexdb.sgml index 026fd018d93e..4f821c2095ef 100644 --- a/doc/src/sgml/ref/reindexdb.sgml +++ b/doc/src/sgml/ref/reindexdb.sgml @@ -174,8 +174,8 @@ PostgreSQL documentation Execute the reindex commands in parallel by running njobs - commands simultaneously. This option reduces the time of the - processing but it also increases the load on the database server. + commands simultaneously. This option may reduce the processing time + but it also increases the load on the database server. reindexdb will open diff --git a/doc/src/sgml/ref/vacuumdb.sgml b/doc/src/sgml/ref/vacuumdb.sgml index 95d6894cb03a..7b1395055244 100644 --- a/doc/src/sgml/ref/vacuumdb.sgml +++ b/doc/src/sgml/ref/vacuumdb.sgml @@ -155,8 +155,8 @@ PostgreSQL documentation Execute the vacuum or analyze commands in parallel by running njobs - commands simultaneously. This option reduces the time of the - processing but it also increases the load on the database server. + commands simultaneously. This option may reduce the processing time + but it also increases the load on the database server. vacuumdb will open diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index f5842319358b..f975406acd67 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -2575,7 +2575,7 @@ openssl x509 -req -in server.csr -text -days 365 \ The PostgreSQL server will listen for both normal and GSSAPI-encrypted connections on the same TCP - port, and will negotiate with any connecting client on whether to + port, and will negotiate with any connecting client whether to use GSSAPI for encryption (and for authentication). By default, this decision is up to the client (which means it can be downgraded by an attacker); see about diff --git a/doc/src/sgml/sources.sgml b/doc/src/sgml/sources.sgml index d4f73a03c3a7..38a516080c18 100644 --- a/doc/src/sgml/sources.sgml +++ b/doc/src/sgml/sources.sgml @@ -518,7 +518,7 @@ Hint: the addendum Use of Quotes - Use quotes always to delimit file names, user-supplied identifiers, and + Always use quotes to delimit file names, user-supplied identifiers, and other variables that might contain words. Do not use them to mark up variables that will not contain words (for example, operator names). diff --git a/src/backend/access/gin/README b/src/backend/access/gin/README index 125a82219b9e..41d4e1e8a093 100644 --- a/src/backend/access/gin/README +++ b/src/backend/access/gin/README @@ -413,7 +413,7 @@ leftmost leaf of the tree. Deletion algorithm keeps exclusive locks on left siblings of pages comprising currently investigated path. Thus, if current page is to be removed, all required pages to remove both downlink and rightlink are already locked. That -evades potential right to left page locking order, which could deadlock with +avoids potential right to left page locking order, which could deadlock with concurrent stepping right. A search concurrent to page deletion might already have read a pointer to the diff --git a/src/backend/utils/adt/jsonpath_exec.c b/src/backend/utils/adt/jsonpath_exec.c index 2c0b362502f3..0591c9effcb9 100644 --- a/src/backend/utils/adt/jsonpath_exec.c +++ b/src/backend/utils/adt/jsonpath_exec.c @@ -35,7 +35,7 @@ * executeItemOptUnwrapTarget() function have 'unwrap' argument, which indicates * whether unwrapping of array is needed. When unwrap == true, each of array * members is passed to executeItemOptUnwrapTarget() again but with unwrap == false - * in order to evade subsequent array unwrapping. + * in order to avoid subsequent array unwrapping. * * All boolean expressions (predicates) are evaluated by executeBoolItem() * function, which returns tri-state JsonPathBool. When error is occurred diff --git a/src/test/regress/expected/partition_join.out b/src/test/regress/expected/partition_join.out index 585e7243752c..0057f41caaf1 100644 --- a/src/test/regress/expected/partition_join.out +++ b/src/test/regress/expected/partition_join.out @@ -4567,7 +4567,7 @@ ANALYZE plt3_adv; -- This tests that when merging partitions from plt1_adv and plt2_adv in -- merge_list_bounds(), process_outer_partition() returns an already-assigned -- merged partition when re-called with plt1_adv_p1 for the second list value --- '0001' of that partitin +-- '0001' of that partition EXPLAIN (COSTS OFF) SELECT t1.a, t1.c, t2.a, t2.c, t3.a, t3.c FROM (plt1_adv t1 LEFT JOIN plt2_adv t2 ON (t1.c = t2.c)) FULL JOIN plt3_adv t3 ON (t1.c = t3.c) WHERE coalesce(t1.a, 0) % 5 != 3 AND coalesce(t1.a, 0) % 5 != 4 ORDER BY t1.c, t1.a, t2.a, t3.a; QUERY PLAN diff --git a/src/test/regress/sql/partition_join.sql b/src/test/regress/sql/partition_join.sql index 73606c86e514..d97b5b69ffc1 100644 --- a/src/test/regress/sql/partition_join.sql +++ b/src/test/regress/sql/partition_join.sql @@ -1090,7 +1090,7 @@ ANALYZE plt3_adv; -- This tests that when merging partitions from plt1_adv and plt2_adv in -- merge_list_bounds(), process_outer_partition() returns an already-assigned -- merged partition when re-called with plt1_adv_p1 for the second list value --- '0001' of that partitin +-- '0001' of that partition EXPLAIN (COSTS OFF) SELECT t1.a, t1.c, t2.a, t2.c, t3.a, t3.c FROM (plt1_adv t1 LEFT JOIN plt2_adv t2 ON (t1.c = t2.c)) FULL JOIN plt3_adv t3 ON (t1.c = t3.c) WHERE coalesce(t1.a, 0) % 5 != 3 AND coalesce(t1.a, 0) % 5 != 4 ORDER BY t1.c, t1.a, t2.a, t3.a; SELECT t1.a, t1.c, t2.a, t2.c, t3.a, t3.c FROM (plt1_adv t1 LEFT JOIN plt2_adv t2 ON (t1.c = t2.c)) FULL JOIN plt3_adv t3 ON (t1.c = t3.c) WHERE coalesce(t1.a, 0) % 5 != 3 AND coalesce(t1.a, 0) % 5 != 4 ORDER BY t1.c, t1.a, t2.a, t3.a; From f859c2ffa01d83a079652ce7d953cd61eb3d7171 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 21 Sep 2020 13:58:26 -0400 Subject: [PATCH 185/589] Fix a few more generator scripts to produce pgindent-clean output. This completes the project of making all our derived files be pgindent-clean (or else explicitly excluded from indentation), so that no surprises result when running pgindent in a built-out development tree. Discussion: https://postgr.es/m/79ed5348-be7a-b647-dd40-742207186a22@2ndquadrant.com --- src/backend/utils/sort/gen_qsort_tuple.pl | 7 ++++--- src/pl/plpgsql/src/generate-plerrcodes.pl | 2 +- src/pl/plpython/generate-spiexceptions.pl | 4 ++-- src/pl/tcl/generate-pltclerrcodes.pl | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/backend/utils/sort/gen_qsort_tuple.pl b/src/backend/utils/sort/gen_qsort_tuple.pl index eb0f7c5814f4..4c305806c7ce 100644 --- a/src/backend/utils/sort/gen_qsort_tuple.pl +++ b/src/backend/utils/sort/gen_qsort_tuple.pl @@ -115,7 +115,8 @@ sub emit_qsort_boilerplate { do { - SortTuple t = *a; + SortTuple t = *a; + *a++ = *b; *b++ = t; } while (--n > 0); @@ -143,9 +144,9 @@ sub emit_qsort_implementation { return cmp_$SUFFIX(a, b$CMPPARAMS) < 0 ? (cmp_$SUFFIX(b, c$CMPPARAMS) < 0 ? b : - (cmp_$SUFFIX(a, c$CMPPARAMS) < 0 ? c : a)) + (cmp_$SUFFIX(a, c$CMPPARAMS) < 0 ? c : a)) : (cmp_$SUFFIX(b, c$CMPPARAMS) > 0 ? b : - (cmp_$SUFFIX(a, c$CMPPARAMS) < 0 ? a : c)); + (cmp_$SUFFIX(a, c$CMPPARAMS) < 0 ? a : c)); } static void diff --git a/src/pl/plpgsql/src/generate-plerrcodes.pl b/src/pl/plpgsql/src/generate-plerrcodes.pl index 32d251c8c2f1..a50de66ef813 100644 --- a/src/pl/plpgsql/src/generate-plerrcodes.pl +++ b/src/pl/plpgsql/src/generate-plerrcodes.pl @@ -34,7 +34,7 @@ # Skip lines without PL/pgSQL condition names next unless defined($condition_name); - print "{\n\t\"$condition_name\", $errcode_macro\n},\n\n"; + print "\n{\n\t\"$condition_name\", $errcode_macro\n},\n"; } close $errcodes; diff --git a/src/pl/plpython/generate-spiexceptions.pl b/src/pl/plpython/generate-spiexceptions.pl index fba603e9ec89..14967ba3eef6 100644 --- a/src/pl/plpython/generate-spiexceptions.pl +++ b/src/pl/plpython/generate-spiexceptions.pl @@ -37,8 +37,8 @@ # Change some_error_condition to SomeErrorCondition $condition_name =~ s/([a-z])([^_]*)(?:_|$)/\u$1$2/g; - print "{ \"spiexceptions.$condition_name\", " - . "\"$condition_name\", $errcode_macro },\n"; + print "\n{\n\t\"spiexceptions.$condition_name\", " + . "\"$condition_name\", $errcode_macro\n},\n"; } close $errcodes; diff --git a/src/pl/tcl/generate-pltclerrcodes.pl b/src/pl/tcl/generate-pltclerrcodes.pl index 43417a4bb4c2..bb9eb8a824dc 100644 --- a/src/pl/tcl/generate-pltclerrcodes.pl +++ b/src/pl/tcl/generate-pltclerrcodes.pl @@ -34,7 +34,7 @@ # Skip lines without PL/pgSQL condition names next unless defined($condition_name); - print "{\n\t\"$condition_name\", $errcode_macro\n},\n\n"; + print "\n{\n\t\"$condition_name\", $errcode_macro\n},\n"; } close $errcodes; From ce90f075f0d831ca4085ba73891b7da2a2f7047e Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 22 Sep 2020 10:49:11 -0400 Subject: [PATCH 186/589] Improve the error message for an inappropriate column definition list. The existing message about "a column definition list is only allowed for functions returning "record"" could be given in some cases where it was fairly confusing; in particular, a function with multiple OUT parameters *does* return record according to pg_proc. Break it down into a couple more cases to deliver a more on-point complaint. Per complaint from Bruce Momjian. Discussion: https://postgr.es/m/798909.1600562993@sss.pgh.pa.us --- src/backend/parser/parse_relation.c | 44 ++++++++++++++++++++---- src/test/regress/expected/rangefuncs.out | 13 +++++++ src/test/regress/sql/rangefuncs.sql | 5 +++ 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/src/backend/parser/parse_relation.c b/src/backend/parser/parse_relation.c index b875a506463f..a56bd86181a7 100644 --- a/src/backend/parser/parse_relation.c +++ b/src/backend/parser/parse_relation.c @@ -1737,16 +1737,46 @@ addRangeTableEntryForFunction(ParseState *pstate, /* * A coldeflist is required if the function returns RECORD and hasn't - * got a predetermined record type, and is prohibited otherwise. + * got a predetermined record type, and is prohibited otherwise. This + * can be a bit confusing, so we expend some effort on delivering a + * relevant error message. */ if (coldeflist != NIL) { - if (functypclass != TYPEFUNC_RECORD) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("a column definition list is only allowed for functions returning \"record\""), - parser_errposition(pstate, - exprLocation((Node *) coldeflist)))); + switch (functypclass) + { + case TYPEFUNC_RECORD: + /* ok */ + break; + case TYPEFUNC_COMPOSITE: + case TYPEFUNC_COMPOSITE_DOMAIN: + + /* + * If the function's raw result type is RECORD, we must + * have resolved it using its OUT parameters. Otherwise, + * it must have a named composite type. + */ + if (exprType(funcexpr) == RECORDOID) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("a column definition list is redundant for a function with OUT parameters"), + parser_errposition(pstate, + exprLocation((Node *) coldeflist)))); + else + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("a column definition list is redundant for a function returning a named composite type"), + parser_errposition(pstate, + exprLocation((Node *) coldeflist)))); + break; + default: + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("a column definition list is only allowed for functions returning \"record\""), + parser_errposition(pstate, + exprLocation((Node *) coldeflist)))); + break; + } } else { diff --git a/src/test/regress/expected/rangefuncs.out b/src/test/regress/expected/rangefuncs.out index 7eced2845203..e618aec2ebca 100644 --- a/src/test/regress/expected/rangefuncs.out +++ b/src/test/regress/expected/rangefuncs.out @@ -2109,6 +2109,19 @@ select * from testrngfunc(); 7.136178 | 7.14 (1 row) +-- Check a couple of error cases while we're here +select * from testrngfunc() as t(f1 int8,f2 int8); -- fail, composite result +ERROR: a column definition list is redundant for a function returning a named composite type +LINE 1: select * from testrngfunc() as t(f1 int8,f2 int8); + ^ +select * from pg_get_keywords() as t(f1 int8,f2 int8); -- fail, OUT params +ERROR: a column definition list is redundant for a function with OUT parameters +LINE 1: select * from pg_get_keywords() as t(f1 int8,f2 int8); + ^ +select * from sin(3) as t(f1 int8,f2 int8); -- fail, scalar result type +ERROR: a column definition list is only allowed for functions returning "record" +LINE 1: select * from sin(3) as t(f1 int8,f2 int8); + ^ drop type rngfunc_type cascade; NOTICE: drop cascades to function testrngfunc() -- diff --git a/src/test/regress/sql/rangefuncs.sql b/src/test/regress/sql/rangefuncs.sql index ae3119a959eb..5f41cb2d8d0b 100644 --- a/src/test/regress/sql/rangefuncs.sql +++ b/src/test/regress/sql/rangefuncs.sql @@ -629,6 +629,11 @@ explain (verbose, costs off) select * from testrngfunc(); select * from testrngfunc(); +-- Check a couple of error cases while we're here +select * from testrngfunc() as t(f1 int8,f2 int8); -- fail, composite result +select * from pg_get_keywords() as t(f1 int8,f2 int8); -- fail, OUT params +select * from sin(3) as t(f1 int8,f2 int8); -- fail, scalar result type + drop type rngfunc_type cascade; -- From c4133ec169dfe47803656325dbfb8397f85a70ea Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 22 Sep 2020 11:32:10 -0400 Subject: [PATCH 187/589] Exclude fmgrprotos.h from pgindent processing. pgindent messes up entries in this file if their names match typedef names. While there's reason to avoid choosing conflicting names, we have some historical exceptions, and there's no guarantee that more duplicates won't appear in future. Since this is a derived file anyway, there's little harm in just excluding it. I said yesterday that all our derived files are pgindent-clean, or else explicitly excluded from indentation, but I'd forgotten about this one. Now that project is really done, as confirmed by a test run. Discussion: https://postgr.es/m/79ed5348-be7a-b647-dd40-742207186a22@2ndquadrant.com --- src/tools/pgindent/README | 4 ++++ src/tools/pgindent/exclude_file_patterns | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/src/tools/pgindent/README b/src/tools/pgindent/README index 56f07c7ee4cd..d36f5088279c 100644 --- a/src/tools/pgindent/README +++ b/src/tools/pgindent/README @@ -101,6 +101,10 @@ the comment block with some dashes: Odd spacing around typedef names might indicate an incomplete typedefs list. +pgindent will mangle both declaration and definition of a C function whose +name matches a typedef. Currently the best workaround is to choose +non-conflicting names. + pgindent can get confused by #if sequences that look correct to the compiler but have mismatched braces/parentheses when considered as a whole. Usually that looks pretty unreadable to humans too, so best practice is to rearrange diff --git a/src/tools/pgindent/exclude_file_patterns b/src/tools/pgindent/exclude_file_patterns index b2e9d8f65478..bfe103f1955a 100644 --- a/src/tools/pgindent/exclude_file_patterns +++ b/src/tools/pgindent/exclude_file_patterns @@ -10,6 +10,10 @@ src/include/jit/llvmjit\.h$ # This confuses pgindent, and it's a derived file anyway. src/backend/utils/fmgrtab\.c$ # +# pgindent might mangle entries in this that match typedef names. +# Since it's a derived file anyway, just exclude it. +src/backend/utils/fmgrprotos\.h$ +# # kwlist_d files are made by gen_keywordlist.pl. While we could insist that # they match pgindent style, they'd look worse not better, so exclude them. kwlist_d\.h$ @@ -35,3 +39,5 @@ src/pl/plperl/Util\.c$ # Exclude any temporary installations that may be in the tree. /tmp_check/ /tmp_install/ +# ... and for paranoia's sake, don't touch git stuff. +/\.git/ From 931487018c409a3102452f965ccaa48367244a41 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 22 Sep 2020 15:55:13 -0400 Subject: [PATCH 188/589] Rethink API for pg_get_line.c, one more time. Further experience says that the appending behavior offered by pg_get_line_append is useful to only a very small minority of callers. For most, the requirement to reset the buffer after each line is just an error-prone nuisance. Hence, invent another alternative call pg_get_line_buf, which takes care of that detail. Noted while reviewing a patch from Daniel Gustafsson. Discussion: https://postgr.es/m/48A4FA71-524E-41B9-953A-FD04EF36E2E7@yesql.se --- src/bin/initdb/initdb.c | 4 +-- src/common/pg_get_line.c | 29 +++++++++++++++++++++- src/include/common/string.h | 1 + src/interfaces/ecpg/test/pg_regress_ecpg.c | 3 +-- src/test/regress/pg_regress.c | 3 +-- 5 files changed, 32 insertions(+), 8 deletions(-) diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 37e0d7ceab93..118b282d1c5e 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -486,7 +486,7 @@ readfile(const char *path) result = (char **) pg_malloc(maxlines * sizeof(char *)); n = 0; - while (pg_get_line_append(infile, &line)) + while (pg_get_line_buf(infile, &line)) { /* make sure there will be room for a trailing NULL pointer */ if (n >= maxlines - 1) @@ -496,8 +496,6 @@ readfile(const char *path) } result[n++] = pg_strdup(line.data); - - resetStringInfo(&line); } result[n] = NULL; diff --git a/src/common/pg_get_line.c b/src/common/pg_get_line.c index 2fb8e198933d..9eb1a33bbb36 100644 --- a/src/common/pg_get_line.c +++ b/src/common/pg_get_line.c @@ -45,7 +45,8 @@ * Also note that the palloc'd buffer is usually a lot longer than * strictly necessary, so it may be inadvisable to use this function * to collect lots of long-lived data. A less memory-hungry option - * is to use pg_get_line_append() in a loop, then pstrdup() each line. + * is to use pg_get_line_buf() or pg_get_line_append() in a loop, + * then pstrdup() each line. */ char * pg_get_line(FILE *stream) @@ -67,11 +68,37 @@ pg_get_line(FILE *stream) return buf.data; } +/* + * pg_get_line_buf() + * + * This has similar behavior to pg_get_line(), and thence to fgets(), + * except that the collected data is returned in a caller-supplied + * StringInfo buffer. This is a convenient API for code that just + * wants to read and process one line at a time, without any artificial + * limit on line length. + * + * Returns true if a line was successfully collected (including the + * case of a non-newline-terminated line at EOF). Returns false if + * there was an I/O error or no data was available before EOF. + * (Check ferror(stream) to distinguish these cases.) + * + * In the false-result case, buf is reset to empty. + */ +bool +pg_get_line_buf(FILE *stream, StringInfo buf) +{ + /* We just need to drop any data from the previous call */ + resetStringInfo(buf); + return pg_get_line_append(stream, buf); +} + /* * pg_get_line_append() * * This has similar behavior to pg_get_line(), and thence to fgets(), * except that the collected data is appended to whatever is in *buf. + * This is useful in preference to pg_get_line_buf() if the caller wants + * to merge some lines together, e.g. to implement backslash continuation. * * Returns true if a line was successfully collected (including the * case of a non-newline-terminated line at EOF). Returns false if diff --git a/src/include/common/string.h b/src/include/common/string.h index 50c241a811b6..6a4baa6f3590 100644 --- a/src/include/common/string.h +++ b/src/include/common/string.h @@ -21,6 +21,7 @@ extern int pg_strip_crlf(char *str); /* functions in src/common/pg_get_line.c */ extern char *pg_get_line(FILE *stream); +extern bool pg_get_line_buf(FILE *stream, struct StringInfoData *buf); extern bool pg_get_line_append(FILE *stream, struct StringInfoData *buf); /* functions in src/common/sprompt.c */ diff --git a/src/interfaces/ecpg/test/pg_regress_ecpg.c b/src/interfaces/ecpg/test/pg_regress_ecpg.c index a2d7b70d9a30..6e1d25b1f4a3 100644 --- a/src/interfaces/ecpg/test/pg_regress_ecpg.c +++ b/src/interfaces/ecpg/test/pg_regress_ecpg.c @@ -49,7 +49,7 @@ ecpg_filter(const char *sourcefile, const char *outfile) initStringInfo(&linebuf); - while (pg_get_line_append(s, &linebuf)) + while (pg_get_line_buf(s, &linebuf)) { /* check for "#line " in the beginning */ if (strstr(linebuf.data, "#line ") == linebuf.data) @@ -69,7 +69,6 @@ ecpg_filter(const char *sourcefile, const char *outfile) } } fputs(linebuf.data, t); - resetStringInfo(&linebuf); } pfree(linebuf.data); diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c index 74fd026856ea..23d7d0beb2e8 100644 --- a/src/test/regress/pg_regress.c +++ b/src/test/regress/pg_regress.c @@ -566,7 +566,7 @@ convert_sourcefiles_in(const char *source_subdir, const char *dest_dir, const ch initStringInfo(&line); - while (pg_get_line_append(infile, &line)) + while (pg_get_line_buf(infile, &line)) { replace_string(&line, "@abs_srcdir@", inputdir); replace_string(&line, "@abs_builddir@", outputdir); @@ -574,7 +574,6 @@ convert_sourcefiles_in(const char *source_subdir, const char *dest_dir, const ch replace_string(&line, "@libdir@", dlpath); replace_string(&line, "@DLSUFFIX@", DLSUFFIX); fputs(line.data, outfile); - resetStringInfo(&line); } pfree(line.data); From c0cb87fbb6642222a99593785f77d318af06ef02 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 22 Sep 2020 15:59:23 -0400 Subject: [PATCH 189/589] Remove arbitrary line length limit for libpq service files. Use a StringInfo instead of a fixed-size buffer in parseServiceInfo(). While we've not heard complaints about the existing 255-byte limit, it certainly seems possible that complex cases could run afoul of it. Daniel Gustafsson Discussion: https://postgr.es/m/48A4FA71-524E-41B9-953A-FD04EF36E2E7@yesql.se --- src/interfaces/libpq/fe-connect.c | 68 ++++++++++++++----------------- 1 file changed, 31 insertions(+), 37 deletions(-) diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 724076a3103e..af27fee6b51e 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -28,6 +28,7 @@ #include "fe-auth.h" #include "libpq-fe.h" #include "libpq-int.h" +#include "lib/stringinfo.h" #include "mb/pg_wchar.h" #include "pg_config_paths.h" #include "port/pg_bswap.h" @@ -5011,8 +5012,6 @@ ldapServiceLookup(const char *purl, PQconninfoOption *options, #endif /* USE_LDAP */ -#define MAXBUFSIZE 256 - /* * parseServiceInfo: if a service name has been given, look it up and absorb * connection options from it into *options. @@ -5099,11 +5098,14 @@ parseServiceFile(const char *serviceFile, PQExpBuffer errorMessage, bool *group_found) { - int linenr = 0, + int result = 0, + linenr = 0, i; FILE *f; - char buf[MAXBUFSIZE], - *line; + char *line; + StringInfoData linebuf; + + *group_found = false; f = fopen(serviceFile, "r"); if (f == NULL) @@ -5113,26 +5115,18 @@ parseServiceFile(const char *serviceFile, return 1; } - while ((line = fgets(buf, sizeof(buf), f)) != NULL) - { - int len; + initStringInfo(&linebuf); + while (pg_get_line_buf(f, &linebuf)) + { linenr++; - if (strlen(line) >= sizeof(buf) - 1) - { - fclose(f); - printfPQExpBuffer(errorMessage, - libpq_gettext("line %d too long in service file \"%s\"\n"), - linenr, - serviceFile); - return 2; - } - /* ignore whitespace at end of line, especially the newline */ - len = strlen(line); - while (len > 0 && isspace((unsigned char) line[len - 1])) - line[--len] = '\0'; + while (linebuf.len > 0 && + isspace((unsigned char) linebuf.data[linebuf.len - 1])) + linebuf.data[--linebuf.len] = '\0'; + + line = linebuf.data; /* ignore leading whitespace too */ while (*line && isspace((unsigned char) line[0])) @@ -5147,9 +5141,8 @@ parseServiceFile(const char *serviceFile, { if (*group_found) { - /* group info already read */ - fclose(f); - return 0; + /* end of desired group reached; return success */ + goto exit; } if (strncmp(line + 1, service, strlen(service)) == 0 && @@ -5178,12 +5171,11 @@ parseServiceFile(const char *serviceFile, switch (rc) { case 0: - fclose(f); - return 0; + goto exit; case 1: case 3: - fclose(f); - return 3; + result = 3; + goto exit; case 2: continue; } @@ -5198,8 +5190,8 @@ parseServiceFile(const char *serviceFile, libpq_gettext("syntax error in service file \"%s\", line %d\n"), serviceFile, linenr); - fclose(f); - return 3; + result = 3; + goto exit; } *val++ = '\0'; @@ -5209,8 +5201,8 @@ parseServiceFile(const char *serviceFile, libpq_gettext("nested service specifications not supported in service file \"%s\", line %d\n"), serviceFile, linenr); - fclose(f); - return 3; + result = 3; + goto exit; } /* @@ -5228,8 +5220,8 @@ parseServiceFile(const char *serviceFile, { printfPQExpBuffer(errorMessage, libpq_gettext("out of memory\n")); - fclose(f); - return 3; + result = 3; + goto exit; } found_keyword = true; break; @@ -5242,16 +5234,18 @@ parseServiceFile(const char *serviceFile, libpq_gettext("syntax error in service file \"%s\", line %d\n"), serviceFile, linenr); - fclose(f); - return 3; + result = 3; + goto exit; } } } } +exit: fclose(f); + pfree(linebuf.data); - return 0; + return result; } From 2e3c19462da930d1d018caa3daabca00159b4b18 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 22 Sep 2020 16:03:32 -0400 Subject: [PATCH 190/589] Simplify SortTocFromFile() by removing fixed buffer-size limit. pg_restore previously coped with overlength TOC-file lines using some complicated logic to ignore additional bufferloads. While this isn't wrong, since we don't expect that the interesting part of a line would run to more than a dozen or so bytes, it's more complex than it needs to be. Use a StringInfo instead of a fixed-size buffer so that we can process long lines as single entities and thus not need the extra logic. Daniel Gustafsson Discussion: https://postgr.es/m/48A4FA71-524E-41B9-953A-FD04EF36E2E7@yesql.se --- src/bin/pg_dump/pg_backup_archiver.c | 41 +++++++++++----------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index bc4757829430..3567e9f365f5 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -30,8 +30,10 @@ #include #endif +#include "common/string.h" #include "dumputils.h" #include "fe_utils/string_utils.h" +#include "lib/stringinfo.h" #include "libpq/libpq-fs.h" #include "parallel.h" #include "pg_backup_archiver.h" @@ -1367,8 +1369,7 @@ SortTocFromFile(Archive *AHX) ArchiveHandle *AH = (ArchiveHandle *) AHX; RestoreOptions *ropt = AH->public.ropt; FILE *fh; - char buf[100]; - bool incomplete_line; + StringInfoData linebuf; /* Allocate space for the 'wanted' array, and init it */ ropt->idWanted = (bool *) pg_malloc0(sizeof(bool) * AH->maxDumpId); @@ -1378,45 +1379,33 @@ SortTocFromFile(Archive *AHX) if (!fh) fatal("could not open TOC file \"%s\": %m", ropt->tocFile); - incomplete_line = false; - while (fgets(buf, sizeof(buf), fh) != NULL) + initStringInfo(&linebuf); + + while (pg_get_line_buf(fh, &linebuf)) { - bool prev_incomplete_line = incomplete_line; - int buflen; char *cmnt; char *endptr; DumpId id; TocEntry *te; - /* - * Some lines in the file might be longer than sizeof(buf). This is - * no problem, since we only care about the leading numeric ID which - * can be at most a few characters; but we have to skip continuation - * bufferloads when processing a long line. - */ - buflen = strlen(buf); - if (buflen > 0 && buf[buflen - 1] == '\n') - incomplete_line = false; - else - incomplete_line = true; - if (prev_incomplete_line) - continue; - /* Truncate line at comment, if any */ - cmnt = strchr(buf, ';'); + cmnt = strchr(linebuf.data, ';'); if (cmnt != NULL) + { cmnt[0] = '\0'; + linebuf.len = cmnt - linebuf.data; + } /* Ignore if all blank */ - if (strspn(buf, " \t\r\n") == strlen(buf)) + if (strspn(linebuf.data, " \t\r\n") == linebuf.len) continue; /* Get an ID, check it's valid and not already seen */ - id = strtol(buf, &endptr, 10); - if (endptr == buf || id <= 0 || id > AH->maxDumpId || + id = strtol(linebuf.data, &endptr, 10); + if (endptr == linebuf.data || id <= 0 || id > AH->maxDumpId || ropt->idWanted[id - 1]) { - pg_log_warning("line ignored: %s", buf); + pg_log_warning("line ignored: %s", linebuf.data); continue; } @@ -1443,6 +1432,8 @@ SortTocFromFile(Archive *AHX) _moveBefore(AH->toc, te); } + pg_free(linebuf.data); + if (fclose(fh) != 0) fatal("could not close TOC file: %m"); } From 733fa9aa51c526582f100aa0d375e0eb9a6bce8b Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Wed, 23 Sep 2020 15:17:30 +1200 Subject: [PATCH 191/589] Allow WaitLatch() to be used without a latch. Due to flaws in commit 3347c982bab, using WaitLatch() without WL_LATCH_SET could cause an assertion failure or crash. Repair. While here, also add a check that the latch we're switching to belongs to this backend, when changing from one latch to another. Discussion: https://postgr.es/m/CA%2BhUKGK1607VmtrDUHQXrsooU%3Dap4g4R2yaoByWOOA3m8xevUQ%40mail.gmail.com --- src/backend/storage/ipc/latch.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/backend/storage/ipc/latch.c b/src/backend/storage/ipc/latch.c index 4153cc85579f..63c6c9753601 100644 --- a/src/backend/storage/ipc/latch.c +++ b/src/backend/storage/ipc/latch.c @@ -924,7 +924,22 @@ ModifyWaitEvent(WaitEventSet *set, int pos, uint32 events, Latch *latch) if (events == WL_LATCH_SET) { + if (latch && latch->owner_pid != MyProcPid) + elog(ERROR, "cannot wait on a latch owned by another process"); set->latch = latch; + /* + * On Unix, we don't need to modify the kernel object because the + * underlying pipe is the same for all latches so we can return + * immediately. On Windows, we need to update our array of handles, + * but we leave the old one in place and tolerate spurious wakeups if + * the latch is disabled. + */ +#if defined(WAIT_USE_WIN32) + if (!latch) + return; +#else + return; +#endif } #if defined(WAIT_USE_EPOLL) @@ -1386,7 +1401,7 @@ WaitEventSetWaitBlock(WaitEventSet *set, int cur_timeout, /* There's data in the self-pipe, clear it. */ drainSelfPipe(); - if (set->latch->is_set) + if (set->latch && set->latch->is_set) { occurred_events->fd = PGINVALID_SOCKET; occurred_events->events = WL_LATCH_SET; @@ -1536,7 +1551,7 @@ WaitEventSetWaitBlock(WaitEventSet *set, int cur_timeout, /* There's data in the self-pipe, clear it. */ drainSelfPipe(); - if (set->latch->is_set) + if (set->latch && set->latch->is_set) { occurred_events->fd = PGINVALID_SOCKET; occurred_events->events = WL_LATCH_SET; @@ -1645,7 +1660,7 @@ WaitEventSetWaitBlock(WaitEventSet *set, int cur_timeout, /* There's data in the self-pipe, clear it. */ drainSelfPipe(); - if (set->latch->is_set) + if (set->latch && set->latch->is_set) { occurred_events->fd = PGINVALID_SOCKET; occurred_events->events = WL_LATCH_SET; @@ -1812,7 +1827,7 @@ WaitEventSetWaitBlock(WaitEventSet *set, int cur_timeout, if (!ResetEvent(set->latch->event)) elog(ERROR, "ResetEvent failed: error code %lu", GetLastError()); - if (set->latch->is_set) + if (set->latch && set->latch->is_set) { occurred_events->fd = PGINVALID_SOCKET; occurred_events->events = WL_LATCH_SET; From 3ea7e9550e58c4016625c7e3e4a1340edfc9adf4 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 23 Sep 2020 11:36:13 -0400 Subject: [PATCH 192/589] Avoid possible dangling-pointer access in tsearch_readline_callback. tsearch_readline() saves the string pointer it returns to the caller for possible use in the associated error context callback. However, the caller will usually pfree that string sometime before it next calls tsearch_readline(), so that there is a window where an ereport will try to print an already-freed string. The built-in users of tsearch_readline() happen to all do that pfree at the bottoms of their loops, so that the window is effectively empty for them. However, this is not documented as a requirement, and contrib/dict_xsyn doesn't do it like that, so it seems likely that third-party dictionaries might have live bugs here. The practical consequences of this seem pretty limited in any case, since production builds wouldn't clobber the freed string immediately, besides which you'd not expect syntax errors in dictionary files being used in production. Still, it's clearly a bug waiting to bite somebody. Fix by pstrdup'ing the string to be saved for the error callback, and then pfree'ing it next time through. It's been like this for a long time, so back-patch to all supported branches. Discussion: https://postgr.es/m/48A4FA71-524E-41B9-953A-FD04EF36E2E7@yesql.se --- src/backend/tsearch/ts_locale.c | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/src/backend/tsearch/ts_locale.c b/src/backend/tsearch/ts_locale.c index a916dd6cb67b..9b199d0ac18b 100644 --- a/src/backend/tsearch/ts_locale.c +++ b/src/backend/tsearch/ts_locale.c @@ -147,10 +147,28 @@ tsearch_readline(tsearch_readline_state *stp) { char *result; + /* Advance line number to use in error reports */ stp->lineno++; - stp->curline = NULL; + + /* Clear curline, it's no longer relevant */ + if (stp->curline) + { + pfree(stp->curline); + stp->curline = NULL; + } + + /* Collect next line, if there is one */ result = t_readline(stp->fp); - stp->curline = result; + if (!result) + return NULL; + + /* + * Save a copy of the line for possible use in error reports. (We cannot + * just save "result", since it's likely to get pfree'd at some point by + * the caller; an error after that would try to access freed data.) + */ + stp->curline = pstrdup(result); + return result; } @@ -160,7 +178,16 @@ tsearch_readline(tsearch_readline_state *stp) void tsearch_readline_end(tsearch_readline_state *stp) { + /* Suppress use of curline in any error reported below */ + if (stp->curline) + { + pfree(stp->curline); + stp->curline = NULL; + } + + /* Release other resources */ FreeFile(stp->fp); + /* Pop the error context stack */ error_context_stack = stp->cb.previous; } From 6b2c4e59d016ea40c42fdc66367d6463e792c125 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 23 Sep 2020 18:04:53 -0400 Subject: [PATCH 193/589] Improve error cursor positions for problems with partition bounds. We failed to pass down the query string to check_new_partition_bound, so that its attempts to provide error cursor positions were for naught; one must have the query string to get parser_errposition to do anything. Adjust its API to require a ParseState to be passed down. Also, improve the logic inside check_new_partition_bound so that the cursor points at the partition bound for the specific column causing the issue, when one can be identified. That part is also for naught if we can't determine the query position of the column with the problem. Improve transformPartitionBoundValue so that it makes sure that const-simplified partition expressions will be properly labeled with positions. In passing, skip calling evaluate_expr if the value is already a Const, which is surely the most common case. Alexandra Wang, Ashwin Agrawal, Amit Langote; reviewed by Ashutosh Bapat Discussion: https://postgr.es/m/CACiyaSopZoqssfMzgHk6fAkp01cL6vnqBdmTw2C5_KJaFR_aMg@mail.gmail.com Discussion: https://postgr.es/m/CAJV4CdrZ5mKuaEsRSbLf2URQ3h6iMtKD=hik8MaF5WwdmC9uZw@mail.gmail.com --- src/backend/commands/tablecmds.c | 16 +++- src/backend/parser/parse_utilcmd.c | 53 +++++++---- src/backend/partitioning/partbounds.c | 101 ++++++++++++++------- src/include/partitioning/partbounds.h | 8 +- src/test/regress/expected/alter_table.out | 10 ++ src/test/regress/expected/create_table.out | 30 ++++++ 6 files changed, 160 insertions(+), 58 deletions(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index eab570a8675c..16285ad09faf 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -543,7 +543,8 @@ static void ComputePartitionAttrs(ParseState *pstate, Relation rel, List *partPa static void CreateInheritance(Relation child_rel, Relation parent_rel); static void RemoveInheritance(Relation child_rel, Relation parent_rel); static ObjectAddress ATExecAttachPartition(List **wqueue, Relation rel, - PartitionCmd *cmd); + PartitionCmd *cmd, + AlterTableUtilityContext *context); static void AttachPartitionEnsureIndexes(Relation rel, Relation attachrel); static void QueuePartitionConstraintValidation(List **wqueue, Relation scanrel, List *partConstraint, @@ -1007,7 +1008,7 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, * Check first that the new partition's bound is valid and does not * overlap with any of existing partitions of the parent. */ - check_new_partition_bound(relname, parent, bound); + check_new_partition_bound(relname, parent, bound, pstate); /* * If the default partition exists, its partition constraints will @@ -4718,7 +4719,8 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, cur_pass, context); Assert(cmd != NULL); if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) - ATExecAttachPartition(wqueue, rel, (PartitionCmd *) cmd->def); + ATExecAttachPartition(wqueue, rel, (PartitionCmd *) cmd->def, + context); else ATExecAttachPartitionIdx(wqueue, rel, ((PartitionCmd *) cmd->def)->name); @@ -16280,7 +16282,8 @@ QueuePartitionConstraintValidation(List **wqueue, Relation scanrel, * Return the address of the newly attached partition. */ static ObjectAddress -ATExecAttachPartition(List **wqueue, Relation rel, PartitionCmd *cmd) +ATExecAttachPartition(List **wqueue, Relation rel, PartitionCmd *cmd, + AlterTableUtilityContext *context) { Relation attachrel, catalog; @@ -16295,6 +16298,9 @@ ATExecAttachPartition(List **wqueue, Relation rel, PartitionCmd *cmd) const char *trigger_name; Oid defaultPartOid; List *partBoundConstraint; + ParseState *pstate = make_parsestate(NULL); + + pstate->p_sourcetext = context->queryString; /* * We must lock the default partition if one exists, because attaching a @@ -16460,7 +16466,7 @@ ATExecAttachPartition(List **wqueue, Relation rel, PartitionCmd *cmd) * error. */ check_new_partition_bound(RelationGetRelationName(attachrel), rel, - cmd->bound); + cmd->bound, pstate); /* OK to create inheritance. Rest of the checks performed there */ CreateInheritance(attachrel, rel); diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index ec944371dd36..164312d60e25 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -4164,7 +4164,7 @@ validateInfiniteBounds(ParseState *pstate, List *blist) } /* - * Transform one constant in a partition bound spec + * Transform one entry in a partition bound spec, producing a constant. */ static Const * transformPartitionBoundValue(ParseState *pstate, Node *val, @@ -4176,6 +4176,13 @@ transformPartitionBoundValue(ParseState *pstate, Node *val, /* Transform raw parsetree */ value = transformExpr(pstate, val, EXPR_KIND_PARTITION_BOUND); + /* + * transformExpr() should have already rejected column references, + * subqueries, aggregates, window functions, and SRFs, based on the + * EXPR_KIND_ of a partition bound expression. + */ + Assert(!contain_var_clause(value)); + /* * Check that the input expression's collation is compatible with one * specified for the parent's partition key (partcollation). Don't throw @@ -4220,7 +4227,11 @@ transformPartitionBoundValue(ParseState *pstate, Node *val, parser_errposition(pstate, exprLocation(value)))); } - /* Coerce to correct type */ + /* + * Coerce to the correct type. This might cause an explicit coercion step + * to be added on top of the expression, which must be evaluated before + * returning the result to the caller. + */ value = coerce_to_target_type(pstate, value, exprType(value), colType, @@ -4236,25 +4247,35 @@ transformPartitionBoundValue(ParseState *pstate, Node *val, format_type_be(colType), colName), parser_errposition(pstate, exprLocation(val)))); - /* Simplify the expression, in case we had a coercion */ - if (!IsA(value, Const)) - value = (Node *) expression_planner((Expr *) value); - /* - * transformExpr() should have already rejected column references, - * subqueries, aggregates, window functions, and SRFs, based on the - * EXPR_KIND_ for a default expression. + * Evaluate the expression, if needed, assigning the partition key's data + * type and collation to the resulting Const node. */ - Assert(!contain_var_clause(value)); + if (!IsA(value, Const)) + { + value = (Node *) expression_planner((Expr *) value); + value = (Node *) evaluate_expr((Expr *) value, colType, colTypmod, + partCollation); + if (!IsA(value, Const)) + elog(ERROR, "could not evaluate partition bound expression"); + } + else + { + /* + * If the expression is already a Const, as is often the case, we can + * skip the rather expensive steps above. But we still have to insert + * the right collation, since coerce_to_target_type doesn't handle + * that. + */ + ((Const *) value)->constcollid = partCollation; + } /* - * Evaluate the expression, assigning the partition key's collation to the - * resulting Const expression. + * Attach original expression's parse location to the Const, so that + * that's what will be reported for any later errors related to this + * partition bound. */ - value = (Node *) evaluate_expr((Expr *) value, colType, colTypmod, - partCollation); - if (!IsA(value, Const)) - elog(ERROR, "could not evaluate partition bound expression"); + ((Const *) value)->location = exprLocation(val); return (Const *) value; } diff --git a/src/backend/partitioning/partbounds.c b/src/backend/partitioning/partbounds.c index 58f9b46289a0..66c42b58986e 100644 --- a/src/backend/partitioning/partbounds.c +++ b/src/backend/partitioning/partbounds.c @@ -223,7 +223,7 @@ static int32 partition_rbound_cmp(int partnatts, FmgrInfo *partsupfunc, static int partition_range_bsearch(int partnatts, FmgrInfo *partsupfunc, Oid *partcollation, PartitionBoundInfo boundinfo, - PartitionRangeBound *probe, bool *is_equal); + PartitionRangeBound *probe, int32 *cmpval); static int get_partition_bound_num_indexes(PartitionBoundInfo b); static Expr *make_partition_op_expr(PartitionKey key, int keynum, uint16 strategy, Expr *arg1, Expr *arg2); @@ -2805,14 +2805,14 @@ partitions_are_ordered(PartitionBoundInfo boundinfo, int nparts) */ void check_new_partition_bound(char *relname, Relation parent, - PartitionBoundSpec *spec) + PartitionBoundSpec *spec, ParseState *pstate) { PartitionKey key = RelationGetPartitionKey(parent); PartitionDesc partdesc = RelationGetPartitionDesc(parent); PartitionBoundInfo boundinfo = partdesc->boundinfo; - ParseState *pstate = make_parsestate(NULL); int with = -1; bool overlap = false; + int overlap_location = -1; if (spec->is_default) { @@ -2907,6 +2907,7 @@ check_new_partition_bound(char *relname, Relation parent, if (boundinfo->indexes[remainder] != -1) { overlap = true; + overlap_location = spec->location; with = boundinfo->indexes[remainder]; break; } @@ -2935,6 +2936,7 @@ check_new_partition_bound(char *relname, Relation parent, { Const *val = castNode(Const, lfirst(cell)); + overlap_location = val->location; if (!val->constisnull) { int offset; @@ -2968,6 +2970,7 @@ check_new_partition_bound(char *relname, Relation parent, { PartitionRangeBound *lower, *upper; + int cmpval; Assert(spec->strategy == PARTITION_STRATEGY_RANGE); lower = make_one_partition_rbound(key, -1, spec->lowerdatums, true); @@ -2977,10 +2980,17 @@ check_new_partition_bound(char *relname, Relation parent, * First check if the resulting range would be empty with * specified lower and upper bounds */ - if (partition_rbound_cmp(key->partnatts, key->partsupfunc, - key->partcollation, lower->datums, - lower->kind, true, upper) >= 0) + cmpval = partition_rbound_cmp(key->partnatts, + key->partsupfunc, + key->partcollation, + lower->datums, lower->kind, + true, upper); + if (cmpval >= 0) { + /* Fetch the problematic key from the lower datums list. */ + PartitionRangeDatum *datum = list_nth(spec->lowerdatums, + cmpval - 1); + ereport(ERROR, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), errmsg("empty range bound specified for partition \"%s\"", @@ -2988,13 +2998,12 @@ check_new_partition_bound(char *relname, Relation parent, errdetail("Specified lower bound %s is greater than or equal to upper bound %s.", get_range_partbound_string(spec->lowerdatums), get_range_partbound_string(spec->upperdatums)), - parser_errposition(pstate, spec->location))); + parser_errposition(pstate, datum->location))); } if (partdesc->nparts > 0) { int offset; - bool equal; Assert(boundinfo && boundinfo->strategy == PARTITION_STRATEGY_RANGE && @@ -3020,7 +3029,7 @@ check_new_partition_bound(char *relname, Relation parent, key->partsupfunc, key->partcollation, boundinfo, lower, - &equal); + &cmpval); if (boundinfo->indexes[offset + 1] < 0) { @@ -3032,7 +3041,6 @@ check_new_partition_bound(char *relname, Relation parent, */ if (offset + 1 < boundinfo->ndatums) { - int32 cmpval; Datum *datums; PartitionRangeDatumKind *kind; bool is_lower; @@ -3048,12 +3056,20 @@ check_new_partition_bound(char *relname, Relation parent, is_lower, upper); if (cmpval < 0) { + /* + * Fetch the problematic key from the upper + * datums list. + */ + PartitionRangeDatum *datum = + list_nth(spec->upperdatums, -cmpval - 1); + /* * The new partition overlaps with the * existing partition between offset + 1 and * offset + 2. */ overlap = true; + overlap_location = datum->location; with = boundinfo->indexes[offset + 2]; } } @@ -3064,7 +3080,20 @@ check_new_partition_bound(char *relname, Relation parent, * The new partition overlaps with the existing * partition between offset and offset + 1. */ + PartitionRangeDatum *datum; + + /* + * Fetch the problematic key from the lower datums + * list. Given the way partition_range_bsearch() + * works, the new lower bound is certainly >= the + * bound at offset. If the bound matches exactly, we + * flag the 1st key. + */ + Assert(cmpval >= 0); + datum = cmpval == 0 ? linitial(spec->lowerdatums) : + list_nth(spec->lowerdatums, cmpval - 1); overlap = true; + overlap_location = datum->location; with = boundinfo->indexes[offset + 1]; } } @@ -3084,7 +3113,7 @@ check_new_partition_bound(char *relname, Relation parent, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), errmsg("partition \"%s\" would overlap partition \"%s\"", relname, get_rel_name(partdesc->oids[with])), - parser_errposition(pstate, spec->location))); + parser_errposition(pstate, overlap_location))); } } @@ -3317,8 +3346,12 @@ make_one_partition_rbound(PartitionKey key, int index, List *datums, bool lower) /* * partition_rbound_cmp * - * Return for two range bounds whether the 1st one (specified in datums1, - * kind1, and lower1) is <, =, or > the bound specified in *b2. + * For two range bounds this decides whether the 1st one (specified by + * datums1, kind1, and lower1) is <, =, or > the bound specified in *b2. + * + * 0 is returned if they are equal, otherwise a non-zero integer whose sign + * indicates the ordering, and whose absolute value gives the 1-based + * partition key number of the first mismatching column. * * partnatts, partsupfunc and partcollation give the number of attributes in the * bounds to be compared, comparison function to be used and the collations of @@ -3337,6 +3370,7 @@ partition_rbound_cmp(int partnatts, FmgrInfo *partsupfunc, Datum *datums1, PartitionRangeDatumKind *kind1, bool lower1, PartitionRangeBound *b2) { + int32 colnum = 0; int32 cmpval = 0; /* placate compiler */ int i; Datum *datums2 = b2->datums; @@ -3345,6 +3379,9 @@ partition_rbound_cmp(int partnatts, FmgrInfo *partsupfunc, for (i = 0; i < partnatts; i++) { + /* Track column number in case we need it for result */ + colnum++; + /* * First, handle cases where the column is unbounded, which should not * invoke the comparison procedure, and should not consider any later @@ -3352,9 +3389,9 @@ partition_rbound_cmp(int partnatts, FmgrInfo *partsupfunc, * compare the same way as the values they represent. */ if (kind1[i] < kind2[i]) - return -1; + return -colnum; else if (kind1[i] > kind2[i]) - return 1; + return colnum; else if (kind1[i] != PARTITION_RANGE_DATUM_VALUE) /* @@ -3381,7 +3418,7 @@ partition_rbound_cmp(int partnatts, FmgrInfo *partsupfunc, if (cmpval == 0 && lower1 != lower2) cmpval = lower1 ? 1 : -1; - return cmpval; + return cmpval == 0 ? 0 : (cmpval < 0 ? -colnum : colnum); } /* @@ -3393,7 +3430,6 @@ partition_rbound_cmp(int partnatts, FmgrInfo *partsupfunc, * n_tuple_datums, partsupfunc and partcollation give number of attributes in * the bounds to be compared, comparison function to be used and the collations * of attributes resp. - * */ int32 partition_rbound_datum_cmp(FmgrInfo *partsupfunc, Oid *partcollation, @@ -3486,14 +3522,17 @@ partition_list_bsearch(FmgrInfo *partsupfunc, Oid *partcollation, * equal to the given range bound or -1 if all of the range bounds are * greater * - * *is_equal is set to true if the range bound at the returned index is equal - * to the input range bound + * Upon return from this function, *cmpval is set to 0 if the bound at the + * returned index matches the input range bound exactly, otherwise a + * non-zero integer whose sign indicates the ordering, and whose absolute + * value gives the 1-based partition key number of the first mismatching + * column. */ static int partition_range_bsearch(int partnatts, FmgrInfo *partsupfunc, Oid *partcollation, PartitionBoundInfo boundinfo, - PartitionRangeBound *probe, bool *is_equal) + PartitionRangeBound *probe, int32 *cmpval) { int lo, hi, @@ -3503,21 +3542,17 @@ partition_range_bsearch(int partnatts, FmgrInfo *partsupfunc, hi = boundinfo->ndatums - 1; while (lo < hi) { - int32 cmpval; - mid = (lo + hi + 1) / 2; - cmpval = partition_rbound_cmp(partnatts, partsupfunc, - partcollation, - boundinfo->datums[mid], - boundinfo->kind[mid], - (boundinfo->indexes[mid] == -1), - probe); - if (cmpval <= 0) + *cmpval = partition_rbound_cmp(partnatts, partsupfunc, + partcollation, + boundinfo->datums[mid], + boundinfo->kind[mid], + (boundinfo->indexes[mid] == -1), + probe); + if (*cmpval <= 0) { lo = mid; - *is_equal = (cmpval == 0); - - if (*is_equal) + if (*cmpval == 0) break; } else @@ -3528,7 +3563,7 @@ partition_range_bsearch(int partnatts, FmgrInfo *partsupfunc, } /* - * partition_range_bsearch + * partition_range_datum_bsearch * Returns the index of the greatest range bound that is less than or * equal to the given tuple or -1 if all of the range bounds are greater * diff --git a/src/include/partitioning/partbounds.h b/src/include/partitioning/partbounds.h index dfc720720b93..192b0b1e2ad8 100644 --- a/src/include/partitioning/partbounds.h +++ b/src/include/partitioning/partbounds.h @@ -12,10 +12,9 @@ #define PARTBOUNDS_H #include "fmgr.h" -#include "nodes/parsenodes.h" -#include "nodes/pg_list.h" +#include "parser/parse_node.h" #include "partitioning/partdefs.h" -#include "utils/relcache.h" + struct RelOptInfo; /* avoid including pathnodes.h here */ @@ -98,7 +97,8 @@ extern PartitionBoundInfo partition_bounds_merge(int partnatts, List **inner_parts); extern bool partitions_are_ordered(PartitionBoundInfo boundinfo, int nparts); extern void check_new_partition_bound(char *relname, Relation parent, - PartitionBoundSpec *spec); + PartitionBoundSpec *spec, + ParseState *pstate); extern void check_default_partition_contents(Relation parent, Relation defaultRel, PartitionBoundSpec *new_spec); diff --git a/src/test/regress/expected/alter_table.out b/src/test/regress/expected/alter_table.out index f56615393ec3..0ce6ee4622d4 100644 --- a/src/test/regress/expected/alter_table.out +++ b/src/test/regress/expected/alter_table.out @@ -3868,6 +3868,8 @@ SELECT conislocal, coninhcount FROM pg_constraint WHERE conrelid = 'part_1'::reg CREATE TABLE fail_part (LIKE part_1 INCLUDING CONSTRAINTS); ALTER TABLE list_parted ATTACH PARTITION fail_part FOR VALUES IN (1); ERROR: partition "fail_part" would overlap partition "part_1" +LINE 1: ...LE list_parted ATTACH PARTITION fail_part FOR VALUES IN (1); + ^ DROP TABLE fail_part; -- check that an existing table can be attached as a default partition CREATE TABLE def_part (LIKE list_parted INCLUDING CONSTRAINTS); @@ -3877,6 +3879,8 @@ ALTER TABLE list_parted ATTACH PARTITION def_part DEFAULT; CREATE TABLE fail_def_part (LIKE part_1 INCLUDING CONSTRAINTS); ALTER TABLE list_parted ATTACH PARTITION fail_def_part DEFAULT; ERROR: partition "fail_def_part" conflicts with existing default partition "def_part" +LINE 1: ...ER TABLE list_parted ATTACH PARTITION fail_def_part DEFAULT; + ^ -- check validation when attaching list partitions CREATE TABLE list_parted2 ( a int, @@ -3946,6 +3950,8 @@ CREATE TABLE partr_def1 PARTITION OF range_parted DEFAULT; CREATE TABLE partr_def2 (LIKE part1 INCLUDING CONSTRAINTS); ALTER TABLE range_parted ATTACH PARTITION partr_def2 DEFAULT; ERROR: partition "partr_def2" conflicts with existing default partition "partr_def1" +LINE 1: ...LTER TABLE range_parted ATTACH PARTITION partr_def2 DEFAULT; + ^ -- Overlapping partitions cannot be attached, hence, following should give error INSERT INTO partr_def1 VALUES (2, 10); CREATE TABLE part3 (LIKE range_parted); @@ -4066,8 +4072,12 @@ CREATE TABLE hpart_1 PARTITION OF hash_parted FOR VALUES WITH (MODULUS 4, REMAIN CREATE TABLE fail_part (LIKE hpart_1); ALTER TABLE hash_parted ATTACH PARTITION fail_part FOR VALUES WITH (MODULUS 8, REMAINDER 4); ERROR: partition "fail_part" would overlap partition "hpart_1" +LINE 1: ...hash_parted ATTACH PARTITION fail_part FOR VALUES WITH (MODU... + ^ ALTER TABLE hash_parted ATTACH PARTITION fail_part FOR VALUES WITH (MODULUS 8, REMAINDER 0); ERROR: partition "fail_part" would overlap partition "hpart_1" +LINE 1: ...hash_parted ATTACH PARTITION fail_part FOR VALUES WITH (MODU... + ^ DROP TABLE fail_part; -- check validation when attaching hash partitions -- check that violating rows are correctly reported diff --git a/src/test/regress/expected/create_table.out b/src/test/regress/expected/create_table.out index 1c72f23bc93b..41dce69cc4c8 100644 --- a/src/test/regress/expected/create_table.out +++ b/src/test/regress/expected/create_table.out @@ -677,6 +677,8 @@ LINE 1: ...BLE fail_part PARTITION OF list_parted FOR VALUES WITH (MODU... CREATE TABLE part_default PARTITION OF list_parted DEFAULT; CREATE TABLE fail_default_part PARTITION OF list_parted DEFAULT; ERROR: partition "fail_default_part" conflicts with existing default partition "part_default" +LINE 1: ...TE TABLE fail_default_part PARTITION OF list_parted DEFAULT; + ^ -- specified literal can't be cast to the partition column data type CREATE TABLE bools ( a bool @@ -702,6 +704,8 @@ CREATE TABLE bigintp_10 PARTITION OF bigintp FOR VALUES IN (10); -- fails due to overlap: CREATE TABLE bigintp_10_2 PARTITION OF bigintp FOR VALUES IN ('10'); ERROR: partition "bigintp_10_2" would overlap partition "bigintp_10" +LINE 1: ...ABLE bigintp_10_2 PARTITION OF bigintp FOR VALUES IN ('10'); + ^ DROP TABLE bigintp; CREATE TABLE range_parted ( a date @@ -823,8 +827,12 @@ CREATE TABLE part_ab PARTITION OF list_parted2 FOR VALUES IN ('a', 'b'); CREATE TABLE list_parted2_def PARTITION OF list_parted2 DEFAULT; CREATE TABLE fail_part PARTITION OF list_parted2 FOR VALUES IN (null); ERROR: partition "fail_part" would overlap partition "part_null_z" +LINE 1: ...LE fail_part PARTITION OF list_parted2 FOR VALUES IN (null); + ^ CREATE TABLE fail_part PARTITION OF list_parted2 FOR VALUES IN ('b', 'c'); ERROR: partition "fail_part" would overlap partition "part_ab" +LINE 1: ...ail_part PARTITION OF list_parted2 FOR VALUES IN ('b', 'c'); + ^ -- check default partition overlap INSERT INTO list_parted2 VALUES('X'); CREATE TABLE fail_part PARTITION OF list_parted2 FOR VALUES IN ('W', 'X', 'Y'); @@ -835,28 +843,42 @@ CREATE TABLE range_parted2 ( -- trying to create range partition with empty range CREATE TABLE fail_part PARTITION OF range_parted2 FOR VALUES FROM (1) TO (0); ERROR: empty range bound specified for partition "fail_part" +LINE 1: ..._part PARTITION OF range_parted2 FOR VALUES FROM (1) TO (0); + ^ DETAIL: Specified lower bound (1) is greater than or equal to upper bound (0). -- note that the range '[1, 1)' has no elements CREATE TABLE fail_part PARTITION OF range_parted2 FOR VALUES FROM (1) TO (1); ERROR: empty range bound specified for partition "fail_part" +LINE 1: ..._part PARTITION OF range_parted2 FOR VALUES FROM (1) TO (1); + ^ DETAIL: Specified lower bound (1) is greater than or equal to upper bound (1). CREATE TABLE part0 PARTITION OF range_parted2 FOR VALUES FROM (minvalue) TO (1); CREATE TABLE fail_part PARTITION OF range_parted2 FOR VALUES FROM (minvalue) TO (2); ERROR: partition "fail_part" would overlap partition "part0" +LINE 1: ..._part PARTITION OF range_parted2 FOR VALUES FROM (minvalue) ... + ^ CREATE TABLE part1 PARTITION OF range_parted2 FOR VALUES FROM (1) TO (10); CREATE TABLE fail_part PARTITION OF range_parted2 FOR VALUES FROM (9) TO (maxvalue); ERROR: partition "fail_part" would overlap partition "part1" +LINE 1: ..._part PARTITION OF range_parted2 FOR VALUES FROM (9) TO (max... + ^ CREATE TABLE part2 PARTITION OF range_parted2 FOR VALUES FROM (20) TO (30); CREATE TABLE part3 PARTITION OF range_parted2 FOR VALUES FROM (30) TO (40); CREATE TABLE fail_part PARTITION OF range_parted2 FOR VALUES FROM (10) TO (30); ERROR: partition "fail_part" would overlap partition "part2" +LINE 1: ...art PARTITION OF range_parted2 FOR VALUES FROM (10) TO (30); + ^ CREATE TABLE fail_part PARTITION OF range_parted2 FOR VALUES FROM (10) TO (50); ERROR: partition "fail_part" would overlap partition "part2" +LINE 1: ...art PARTITION OF range_parted2 FOR VALUES FROM (10) TO (50); + ^ -- Create a default partition for range partitioned table CREATE TABLE range2_default PARTITION OF range_parted2 DEFAULT; -- More than one default partition is not allowed, so this should give error CREATE TABLE fail_default_part PARTITION OF range_parted2 DEFAULT; ERROR: partition "fail_default_part" conflicts with existing default partition "range2_default" +LINE 1: ... TABLE fail_default_part PARTITION OF range_parted2 DEFAULT; + ^ -- Check if the range for default partitions overlap INSERT INTO range_parted2 VALUES (85); CREATE TABLE fail_part PARTITION OF range_parted2 FOR VALUES FROM (80) TO (90); @@ -870,17 +892,23 @@ CREATE TABLE range_parted3 ( CREATE TABLE part00 PARTITION OF range_parted3 FOR VALUES FROM (0, minvalue) TO (0, maxvalue); CREATE TABLE fail_part PARTITION OF range_parted3 FOR VALUES FROM (0, minvalue) TO (0, 1); ERROR: partition "fail_part" would overlap partition "part00" +LINE 1: ..._part PARTITION OF range_parted3 FOR VALUES FROM (0, minvalu... + ^ CREATE TABLE part10 PARTITION OF range_parted3 FOR VALUES FROM (1, minvalue) TO (1, 1); CREATE TABLE part11 PARTITION OF range_parted3 FOR VALUES FROM (1, 1) TO (1, 10); CREATE TABLE part12 PARTITION OF range_parted3 FOR VALUES FROM (1, 10) TO (1, maxvalue); CREATE TABLE fail_part PARTITION OF range_parted3 FOR VALUES FROM (1, 10) TO (1, 20); ERROR: partition "fail_part" would overlap partition "part12" +LINE 1: ...rt PARTITION OF range_parted3 FOR VALUES FROM (1, 10) TO (1,... + ^ CREATE TABLE range3_default PARTITION OF range_parted3 DEFAULT; -- cannot create a partition that says column b is allowed to range -- from -infinity to +infinity, while there exist partitions that have -- more specific ranges CREATE TABLE fail_part PARTITION OF range_parted3 FOR VALUES FROM (1, minvalue) TO (1, maxvalue); ERROR: partition "fail_part" would overlap partition "part10" +LINE 1: ..._part PARTITION OF range_parted3 FOR VALUES FROM (1, minvalu... + ^ -- check for partition bound overlap and other invalid specifications for the hash partition CREATE TABLE hash_parted2 ( a varchar @@ -892,6 +920,8 @@ CREATE TABLE h2part_4 PARTITION OF hash_parted2 FOR VALUES WITH (MODULUS 8, REMA -- overlap with part_4 CREATE TABLE fail_part PARTITION OF hash_parted2 FOR VALUES WITH (MODULUS 2, REMAINDER 1); ERROR: partition "fail_part" would overlap partition "h2part_4" +LINE 1: ...LE fail_part PARTITION OF hash_parted2 FOR VALUES WITH (MODU... + ^ -- modulus must be greater than zero CREATE TABLE fail_part PARTITION OF hash_parted2 FOR VALUES WITH (MODULUS 0, REMAINDER 1); ERROR: modulus for hash partition must be a positive integer From aca74843e40f43d0cceffd314aec6aa6f80e83d5 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Thu, 24 Sep 2020 09:26:09 +1200 Subject: [PATCH 194/589] Fix missing fsync of SLRU directories. Harmonize behavior by moving reponsibility for fsyncing directories down into slru.c. In 10 and later, only the multixact directories were missed (see commit 1b02be21), and in older branches all SLRUs were missed. Back-patch to all supported releases. Reviewed-by: Andres Freund Reviewed-by: Michael Paquier Discussion: https://postgr.es/m/CA%2BhUKGLtsTUOScnNoSMZ-2ZLv%2BwGh01J6kAo_DM8mTRq1sKdSQ%40mail.gmail.com --- src/backend/access/transam/clog.c | 7 ------- src/backend/access/transam/commit_ts.c | 6 ------ src/backend/access/transam/slru.c | 4 ++++ 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/src/backend/access/transam/clog.c b/src/backend/access/transam/clog.c index 65aa8841f7ce..9e352d265835 100644 --- a/src/backend/access/transam/clog.c +++ b/src/backend/access/transam/clog.c @@ -836,13 +836,6 @@ CheckPointCLOG(void) /* Flush dirty CLOG pages to disk */ TRACE_POSTGRESQL_CLOG_CHECKPOINT_START(true); SimpleLruFlush(XactCtl, true); - - /* - * fsync pg_xact to ensure that any files flushed previously are durably - * on disk. - */ - fsync_fname("pg_xact", true); - TRACE_POSTGRESQL_CLOG_CHECKPOINT_DONE(true); } diff --git a/src/backend/access/transam/commit_ts.c b/src/backend/access/transam/commit_ts.c index 5244b06a2b65..f6a7329ba3a5 100644 --- a/src/backend/access/transam/commit_ts.c +++ b/src/backend/access/transam/commit_ts.c @@ -822,12 +822,6 @@ CheckPointCommitTs(void) { /* Flush dirty CommitTs pages to disk */ SimpleLruFlush(CommitTsCtl, true); - - /* - * fsync pg_commit_ts to ensure that any files flushed previously are - * durably on disk. - */ - fsync_fname("pg_commit_ts", true); } /* diff --git a/src/backend/access/transam/slru.c b/src/backend/access/transam/slru.c index 7640f153c227..fe7d759a8c1c 100644 --- a/src/backend/access/transam/slru.c +++ b/src/backend/access/transam/slru.c @@ -1187,6 +1187,10 @@ SimpleLruFlush(SlruCtl ctl, bool allow_redirtied) } if (!ok) SlruReportIOError(ctl, pageno, InvalidTransactionId); + + /* Ensure that directory entries for new files are on disk. */ + if (ctl->do_fsync) + fsync_fname(ctl->Dir, true); } /* From 83b61319a16ceabc7ea7d1143dcc045826c112d2 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 23 Sep 2020 20:26:58 -0400 Subject: [PATCH 195/589] Improve behavior of tsearch_readline(), and remove t_readline(). Commit fbeb9da22, which added the tsearch_readline APIs, left t_readline() in place as a compatibility measure. But that function has been unused and deprecated for twelve years now, so that seems like enough time to remove it. Doing so, and merging t_readline's code into tsearch_readline, aids in making several useful improvements: * The hard-wired 4K limit on line length in tsearch data files is removed, by using a StringInfo buffer instead of a fixed-size buffer. * We can buy back the per-line palloc/pfree added by 3ea7e9550 in the common case where encoding conversion is not required. * We no longer need a separate pg_verify_mbstr call, as that functionality was folded into encoding conversion some time ago. (We could have done some of this stuff while keeping t_readline as a separate API, but there seems little point, since there's no reason for anyone to still be using t_readline directly.) Discussion: https://postgr.es/m/48A4FA71-524E-41B9-953A-FD04EF36E2E7@yesql.se --- src/backend/tsearch/dict_thesaurus.c | 5 -- src/backend/tsearch/ts_locale.c | 78 ++++++++++------------------ src/include/tsearch/ts_locale.h | 7 +-- 3 files changed, 32 insertions(+), 58 deletions(-) diff --git a/src/backend/tsearch/dict_thesaurus.c b/src/backend/tsearch/dict_thesaurus.c index cb0835982d85..64c979086d1e 100644 --- a/src/backend/tsearch/dict_thesaurus.c +++ b/src/backend/tsearch/dict_thesaurus.c @@ -286,11 +286,6 @@ thesaurusRead(const char *filename, DictThesaurus *d) (errcode(ERRCODE_CONFIG_FILE_ERROR), errmsg("unexpected end of line"))); - /* - * Note: currently, tsearch_readline can't return lines exceeding 4KB, - * so overflow of the word counts is impossible. But that may not - * always be true, so let's check. - */ if (nwrd != (uint16) nwrd || posinsubst != (uint16) posinsubst) ereport(ERROR, (errcode(ERRCODE_CONFIG_FILE_ERROR), diff --git a/src/backend/tsearch/ts_locale.c b/src/backend/tsearch/ts_locale.c index 9b199d0ac18b..d362e86d61a2 100644 --- a/src/backend/tsearch/ts_locale.c +++ b/src/backend/tsearch/ts_locale.c @@ -14,6 +14,7 @@ #include "postgres.h" #include "catalog/pg_collation.h" +#include "common/string.h" #include "storage/fd.h" #include "tsearch/ts_locale.h" #include "tsearch/ts_public.h" @@ -128,6 +129,7 @@ tsearch_readline_begin(tsearch_readline_state *stp, return false; stp->filename = filename; stp->lineno = 0; + initStringInfo(&stp->buf); stp->curline = NULL; /* Setup error traceback support for ereport() */ stp->cb.callback = tsearch_readline_callback; @@ -145,7 +147,7 @@ tsearch_readline_begin(tsearch_readline_state *stp, char * tsearch_readline(tsearch_readline_state *stp) { - char *result; + char *recoded; /* Advance line number to use in error reports */ stp->lineno++; @@ -153,23 +155,35 @@ tsearch_readline(tsearch_readline_state *stp) /* Clear curline, it's no longer relevant */ if (stp->curline) { - pfree(stp->curline); + if (stp->curline != stp->buf.data) + pfree(stp->curline); stp->curline = NULL; } /* Collect next line, if there is one */ - result = t_readline(stp->fp); - if (!result) + if (!pg_get_line_buf(stp->fp, &stp->buf)) return NULL; + /* Validate the input as UTF-8, then convert to DB encoding if needed */ + recoded = pg_any_to_server(stp->buf.data, stp->buf.len, PG_UTF8); + + /* Save the correctly-encoded string for possible error reports */ + stp->curline = recoded; /* might be equal to buf.data */ + /* - * Save a copy of the line for possible use in error reports. (We cannot - * just save "result", since it's likely to get pfree'd at some point by - * the caller; an error after that would try to access freed data.) + * We always return a freshly pstrdup'd string. This is clearly necessary + * if pg_any_to_server() returned buf.data, and we need a second copy even + * if encoding conversion did occur. The caller is entitled to pfree the + * returned string at any time, which would leave curline pointing to + * recycled storage, causing problems if an error occurs after that point. + * (It's preferable to return the result of pstrdup instead of the output + * of pg_any_to_server, because the conversion result tends to be + * over-allocated. Since callers might save the result string directly + * into a long-lived dictionary structure, we don't want it to be a larger + * palloc chunk than necessary. We'll reclaim the conversion result on + * the next call.) */ - stp->curline = pstrdup(result); - - return result; + return pstrdup(recoded); } /* @@ -181,11 +195,13 @@ tsearch_readline_end(tsearch_readline_state *stp) /* Suppress use of curline in any error reported below */ if (stp->curline) { - pfree(stp->curline); + if (stp->curline != stp->buf.data) + pfree(stp->curline); stp->curline = NULL; } /* Release other resources */ + pfree(stp->buf.data); FreeFile(stp->fp); /* Pop the error context stack */ @@ -203,8 +219,7 @@ tsearch_readline_callback(void *arg) /* * We can't include the text of the config line for errors that occur - * during t_readline() itself. This is only partly a consequence of our - * arms-length use of that routine: the major cause of such errors is + * during tsearch_readline() itself. The major cause of such errors is * encoding violations, and we daren't try to print error messages * containing badly-encoded data. */ @@ -220,43 +235,6 @@ tsearch_readline_callback(void *arg) } -/* - * Read the next line from a tsearch data file (expected to be in UTF-8), and - * convert it to database encoding if needed. The returned string is palloc'd. - * NULL return means EOF. - * - * Note: direct use of this function is now deprecated. Go through - * tsearch_readline() to provide better error reporting. - */ -char * -t_readline(FILE *fp) -{ - int len; - char *recoded; - char buf[4096]; /* lines must not be longer than this */ - - if (fgets(buf, sizeof(buf), fp) == NULL) - return NULL; - - len = strlen(buf); - - /* Make sure the input is valid UTF-8 */ - (void) pg_verify_mbstr(PG_UTF8, buf, len, false); - - /* And convert */ - recoded = pg_any_to_server(buf, len, PG_UTF8); - if (recoded == buf) - { - /* - * conversion didn't pstrdup, so we must. We can use the length of the - * original string, because no conversion was done. - */ - recoded = pnstrdup(recoded, len); - } - - return recoded; -} - /* * lowerstr --- fold null-terminated string to lower case * diff --git a/src/include/tsearch/ts_locale.h b/src/include/tsearch/ts_locale.h index cc4bd9ab20d4..f1669fda2111 100644 --- a/src/include/tsearch/ts_locale.h +++ b/src/include/tsearch/ts_locale.h @@ -15,6 +15,7 @@ #include #include +#include "lib/stringinfo.h" #include "mb/pg_wchar.h" #include "utils/pg_locale.h" @@ -33,7 +34,9 @@ typedef struct FILE *fp; const char *filename; int lineno; - char *curline; + StringInfoData buf; /* current input line, in UTF-8 */ + char *curline; /* current input line, in DB's encoding */ + /* curline may be NULL, or equal to buf.data, or a palloc'd string */ ErrorContextCallback cb; } tsearch_readline_state; @@ -57,6 +60,4 @@ extern bool tsearch_readline_begin(tsearch_readline_state *stp, extern char *tsearch_readline(tsearch_readline_state *stp); extern void tsearch_readline_end(tsearch_readline_state *stp); -extern char *t_readline(FILE *fp); - #endif /* __TSLOCALE_H__ */ From fc5f107a8c08f55cc8b810d6f212dd723a07fe9a Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 24 Sep 2020 10:39:11 -0400 Subject: [PATCH 196/589] Doc: sync lobj.sgml's copy of testlo.c with the latter file. Zhijie Hou Discussion: https://postgr.es/m/ce2cd951fe9b448a9cda99dc1a871fb9@G08CNEXMBPEKD05.g08.fujitsu.local --- doc/src/sgml/lobj.sgml | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/src/sgml/lobj.sgml b/doc/src/sgml/lobj.sgml index cf4653fe0f94..6329cf0796be 100644 --- a/doc/src/sgml/lobj.sgml +++ b/doc/src/sgml/lobj.sgml @@ -901,8 +901,6 @@ exportFile(PGconn *conn, Oid lobjId, char *filename) lo_close(conn, lobj_fd); close(fd); - - return; } static void From f5ea92e8d620a1260d3427b92fa350bbf36594a2 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 24 Sep 2020 13:32:39 -0400 Subject: [PATCH 197/589] Expose oldSnapshotControl definition via new header. This makes it possible for code outside snapmgr.c to examine the contents of this data structure. This commit does not add any code which actually does so; a subsequent commit will make that change. Patch by me, reviewed by Thomas Munro, Dilip Kumar, Hamid Akhtar. Discussion: http://postgr.es/m/CA+TgmoY=aqf0zjTD+3dUWYkgMiNDegDLFjo+6ze=Wtpik+3XqA@mail.gmail.com --- src/backend/utils/time/snapmgr.c | 55 +---------------------- src/include/utils/old_snapshot.h | 75 ++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 53 deletions(-) create mode 100644 src/include/utils/old_snapshot.h diff --git a/src/backend/utils/time/snapmgr.c b/src/backend/utils/time/snapmgr.c index ed7f5239a059..650c2aa81595 100644 --- a/src/backend/utils/time/snapmgr.c +++ b/src/backend/utils/time/snapmgr.c @@ -64,6 +64,7 @@ #include "storage/spin.h" #include "utils/builtins.h" #include "utils/memutils.h" +#include "utils/old_snapshot.h" #include "utils/rel.h" #include "utils/resowner_private.h" #include "utils/snapmgr.h" @@ -76,59 +77,7 @@ */ int old_snapshot_threshold; /* number of minutes, -1 disables */ -/* - * Structure for dealing with old_snapshot_threshold implementation. - */ -typedef struct OldSnapshotControlData -{ - /* - * Variables for old snapshot handling are shared among processes and are - * only allowed to move forward. - */ - slock_t mutex_current; /* protect current_timestamp */ - TimestampTz current_timestamp; /* latest snapshot timestamp */ - slock_t mutex_latest_xmin; /* protect latest_xmin and next_map_update */ - TransactionId latest_xmin; /* latest snapshot xmin */ - TimestampTz next_map_update; /* latest snapshot valid up to */ - slock_t mutex_threshold; /* protect threshold fields */ - TimestampTz threshold_timestamp; /* earlier snapshot is old */ - TransactionId threshold_xid; /* earlier xid may be gone */ - - /* - * Keep one xid per minute for old snapshot error handling. - * - * Use a circular buffer with a head offset, a count of entries currently - * used, and a timestamp corresponding to the xid at the head offset. A - * count_used value of zero means that there are no times stored; a - * count_used value of OLD_SNAPSHOT_TIME_MAP_ENTRIES means that the buffer - * is full and the head must be advanced to add new entries. Use - * timestamps aligned to minute boundaries, since that seems less - * surprising than aligning based on the first usage timestamp. The - * latest bucket is effectively stored within latest_xmin. The circular - * buffer is updated when we get a new xmin value that doesn't fall into - * the same interval. - * - * It is OK if the xid for a given time slot is from earlier than - * calculated by adding the number of minutes corresponding to the - * (possibly wrapped) distance from the head offset to the time of the - * head entry, since that just results in the vacuuming of old tuples - * being slightly less aggressive. It would not be OK for it to be off in - * the other direction, since it might result in vacuuming tuples that are - * still expected to be there. - * - * Use of an SLRU was considered but not chosen because it is more - * heavyweight than is needed for this, and would probably not be any less - * code to implement. - * - * Persistence is not needed. - */ - int head_offset; /* subscript of oldest tracked time */ - TimestampTz head_timestamp; /* time corresponding to head xid */ - int count_used; /* how many slots are in use */ - TransactionId xid_by_minute[FLEXIBLE_ARRAY_MEMBER]; -} OldSnapshotControlData; - -static volatile OldSnapshotControlData *oldSnapshotControl; +volatile OldSnapshotControlData *oldSnapshotControl; /* diff --git a/src/include/utils/old_snapshot.h b/src/include/utils/old_snapshot.h new file mode 100644 index 000000000000..e6da1833a634 --- /dev/null +++ b/src/include/utils/old_snapshot.h @@ -0,0 +1,75 @@ +/*------------------------------------------------------------------------- + * + * old_snapshot.h + * Data structures for 'snapshot too old' + * + * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/include/utils/old_snapshot.h + * + *------------------------------------------------------------------------- + */ + +#ifndef OLD_SNAPSHOT_H +#define OLD_SNAPSHOT_H + +#include "datatype/timestamp.h" +#include "storage/s_lock.h" + +/* + * Structure for dealing with old_snapshot_threshold implementation. + */ +typedef struct OldSnapshotControlData +{ + /* + * Variables for old snapshot handling are shared among processes and are + * only allowed to move forward. + */ + slock_t mutex_current; /* protect current_timestamp */ + TimestampTz current_timestamp; /* latest snapshot timestamp */ + slock_t mutex_latest_xmin; /* protect latest_xmin and next_map_update */ + TransactionId latest_xmin; /* latest snapshot xmin */ + TimestampTz next_map_update; /* latest snapshot valid up to */ + slock_t mutex_threshold; /* protect threshold fields */ + TimestampTz threshold_timestamp; /* earlier snapshot is old */ + TransactionId threshold_xid; /* earlier xid may be gone */ + + /* + * Keep one xid per minute for old snapshot error handling. + * + * Use a circular buffer with a head offset, a count of entries currently + * used, and a timestamp corresponding to the xid at the head offset. A + * count_used value of zero means that there are no times stored; a + * count_used value of OLD_SNAPSHOT_TIME_MAP_ENTRIES means that the buffer + * is full and the head must be advanced to add new entries. Use + * timestamps aligned to minute boundaries, since that seems less + * surprising than aligning based on the first usage timestamp. The + * latest bucket is effectively stored within latest_xmin. The circular + * buffer is updated when we get a new xmin value that doesn't fall into + * the same interval. + * + * It is OK if the xid for a given time slot is from earlier than + * calculated by adding the number of minutes corresponding to the + * (possibly wrapped) distance from the head offset to the time of the + * head entry, since that just results in the vacuuming of old tuples + * being slightly less aggressive. It would not be OK for it to be off in + * the other direction, since it might result in vacuuming tuples that are + * still expected to be there. + * + * Use of an SLRU was considered but not chosen because it is more + * heavyweight than is needed for this, and would probably not be any less + * code to implement. + * + * Persistence is not needed. + */ + int head_offset; /* subscript of oldest tracked time */ + TimestampTz head_timestamp; /* time corresponding to head xid */ + int count_used; /* how many slots are in use */ + TransactionId xid_by_minute[FLEXIBLE_ARRAY_MEMBER]; +} OldSnapshotControlData; + +extern PGDLLIMPORT volatile OldSnapshotControlData *oldSnapshotControl; + +#endif From aecf5ee2bb36c597d3c6142e367e38d67816c777 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 24 Sep 2020 13:55:47 -0400 Subject: [PATCH 198/589] Add new 'old_snapshot' contrib module. You can use this to view the contents of the time to XID mapping which the server maintains when old_snapshot_threshold != -1. Being able to view that information may be interesting for users, and it's definitely useful for figuring out whether the mapping is being maintained correctly. It isn't, so that will need to be fixed in a subsequent commit. Patch by me, reviewed by Thomas Munro, Dilip Kumar, Hamid Akhtar. Discussion: http://postgr.es/m/CA+TgmoY=aqf0zjTD+3dUWYkgMiNDegDLFjo+6ze=Wtpik+3XqA@mail.gmail.com --- contrib/Makefile | 1 + contrib/old_snapshot/Makefile | 22 +++ contrib/old_snapshot/old_snapshot--1.0.sql | 14 ++ contrib/old_snapshot/old_snapshot.control | 5 + contrib/old_snapshot/time_mapping.c | 159 +++++++++++++++++++++ doc/src/sgml/contrib.sgml | 1 + doc/src/sgml/filelist.sgml | 1 + doc/src/sgml/oldsnapshot.sgml | 33 +++++ 8 files changed, 236 insertions(+) create mode 100644 contrib/old_snapshot/Makefile create mode 100644 contrib/old_snapshot/old_snapshot--1.0.sql create mode 100644 contrib/old_snapshot/old_snapshot.control create mode 100644 contrib/old_snapshot/time_mapping.c create mode 100644 doc/src/sgml/oldsnapshot.sgml diff --git a/contrib/Makefile b/contrib/Makefile index c8d2a1627358..7a4866e338db 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -27,6 +27,7 @@ SUBDIRS = \ lo \ ltree \ oid2name \ + old_snapshot \ pageinspect \ passwordcheck \ pg_buffercache \ diff --git a/contrib/old_snapshot/Makefile b/contrib/old_snapshot/Makefile new file mode 100644 index 000000000000..77c85df3225d --- /dev/null +++ b/contrib/old_snapshot/Makefile @@ -0,0 +1,22 @@ +# contrib/old_snapshot/Makefile + +MODULE_big = old_snapshot +OBJS = \ + $(WIN32RES) \ + time_mapping.o +PG_CPPFLAGS = -I$(libpq_srcdir) + +EXTENSION = old_snapshot +DATA = old_snapshot--1.0.sql +PGFILEDESC = "old_snapshot - utilities in support of old_snapshot_threshold" + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/old_snapshot +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/old_snapshot/old_snapshot--1.0.sql b/contrib/old_snapshot/old_snapshot--1.0.sql new file mode 100644 index 000000000000..9ebb8829e372 --- /dev/null +++ b/contrib/old_snapshot/old_snapshot--1.0.sql @@ -0,0 +1,14 @@ +/* contrib/old_snapshot/old_snapshot--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION old_snapshot" to load this file. \quit + +-- Show visibility map and page-level visibility information for each block. +CREATE FUNCTION pg_old_snapshot_time_mapping(array_offset OUT int4, + end_timestamp OUT timestamptz, + newest_xmin OUT xid) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_old_snapshot_time_mapping' +LANGUAGE C STRICT; + +-- XXX. Do we want REVOKE commands here? diff --git a/contrib/old_snapshot/old_snapshot.control b/contrib/old_snapshot/old_snapshot.control new file mode 100644 index 000000000000..491eec536cd6 --- /dev/null +++ b/contrib/old_snapshot/old_snapshot.control @@ -0,0 +1,5 @@ +# old_snapshot extension +comment = 'utilities in support of old_snapshot_threshold' +default_version = '1.0' +module_pathname = '$libdir/old_snapshot' +relocatable = true diff --git a/contrib/old_snapshot/time_mapping.c b/contrib/old_snapshot/time_mapping.c new file mode 100644 index 000000000000..37e0055a0086 --- /dev/null +++ b/contrib/old_snapshot/time_mapping.c @@ -0,0 +1,159 @@ +/*------------------------------------------------------------------------- + * + * time_mapping.c + * time to XID mapping information + * + * Copyright (c) 2020, PostgreSQL Global Development Group + * + * contrib/old_snapshot/time_mapping.c + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "funcapi.h" +#include "storage/lwlock.h" +#include "utils/old_snapshot.h" +#include "utils/snapmgr.h" +#include "utils/timestamp.h" + +/* + * Backend-private copy of the information from oldSnapshotControl which relates + * to the time to XID mapping, plus an index so that we can iterate. + * + * Note that the length of the xid_by_minute array is given by + * OLD_SNAPSHOT_TIME_MAP_ENTRIES (which is not a compile-time constant). + */ +typedef struct +{ + int current_index; + int head_offset; + TimestampTz head_timestamp; + int count_used; + TransactionId xid_by_minute[FLEXIBLE_ARRAY_MEMBER]; +} OldSnapshotTimeMapping; + +#define NUM_TIME_MAPPING_COLUMNS 3 + +PG_MODULE_MAGIC; +PG_FUNCTION_INFO_V1(pg_old_snapshot_time_mapping); + +static OldSnapshotTimeMapping *GetOldSnapshotTimeMapping(void); +static TupleDesc MakeOldSnapshotTimeMappingTupleDesc(void); +static HeapTuple MakeOldSnapshotTimeMappingTuple(TupleDesc tupdesc, + OldSnapshotTimeMapping *mapping); + +/* + * SQL-callable set-returning function. + */ +Datum +pg_old_snapshot_time_mapping(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; + OldSnapshotTimeMapping *mapping; + + if (SRF_IS_FIRSTCALL()) + { + MemoryContext oldcontext; + + funcctx = SRF_FIRSTCALL_INIT(); + oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + mapping = GetOldSnapshotTimeMapping(); + funcctx->user_fctx = mapping; + funcctx->tuple_desc = MakeOldSnapshotTimeMappingTupleDesc(); + MemoryContextSwitchTo(oldcontext); + } + + funcctx = SRF_PERCALL_SETUP(); + mapping = (OldSnapshotTimeMapping *) funcctx->user_fctx; + + while (mapping->current_index < mapping->count_used) + { + HeapTuple tuple; + + tuple = MakeOldSnapshotTimeMappingTuple(funcctx->tuple_desc, mapping); + ++mapping->current_index; + SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tuple)); + } + + SRF_RETURN_DONE(funcctx); +} + +/* + * Get the old snapshot time mapping data from shared memory. + */ +static OldSnapshotTimeMapping * +GetOldSnapshotTimeMapping(void) +{ + OldSnapshotTimeMapping *mapping; + + mapping = palloc(offsetof(OldSnapshotTimeMapping, xid_by_minute) + + sizeof(TransactionId) * OLD_SNAPSHOT_TIME_MAP_ENTRIES); + mapping->current_index = 0; + + LWLockAcquire(OldSnapshotTimeMapLock, LW_SHARED); + mapping->head_offset = oldSnapshotControl->head_offset; + mapping->head_timestamp = oldSnapshotControl->head_timestamp; + mapping->count_used = oldSnapshotControl->count_used; + for (int i = 0; i < OLD_SNAPSHOT_TIME_MAP_ENTRIES; ++i) + mapping->xid_by_minute[i] = oldSnapshotControl->xid_by_minute[i]; + LWLockRelease(OldSnapshotTimeMapLock); + + return mapping; +} + +/* + * Build a tuple descriptor for the pg_old_snapshot_time_mapping() SRF. + */ +static TupleDesc +MakeOldSnapshotTimeMappingTupleDesc(void) +{ + TupleDesc tupdesc; + + tupdesc = CreateTemplateTupleDesc(NUM_TIME_MAPPING_COLUMNS); + + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "array_offset", + INT4OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "end_timestamp", + TIMESTAMPTZOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "newest_xmin", + XIDOID, -1, 0); + + return BlessTupleDesc(tupdesc); +} + +/* + * Convert one entry from the old snapshot time mapping to a HeapTuple. + */ +static HeapTuple +MakeOldSnapshotTimeMappingTuple(TupleDesc tupdesc, OldSnapshotTimeMapping *mapping) +{ + Datum values[NUM_TIME_MAPPING_COLUMNS]; + bool nulls[NUM_TIME_MAPPING_COLUMNS]; + int array_position; + TimestampTz timestamp; + + /* + * Figure out the array position corresponding to the current index. + * + * Index 0 means the oldest entry in the mapping, which is stored at + * mapping->head_offset. Index 1 means the next-oldest entry, which is a the + * following index, and so on. We wrap around when we reach the end of the array. + */ + array_position = (mapping->head_offset + mapping->current_index) + % OLD_SNAPSHOT_TIME_MAP_ENTRIES; + + /* + * No explicit timestamp is stored for any entry other than the oldest one, + * but each entry corresponds to 1-minute period, so we can just add. + */ + timestamp = TimestampTzPlusMilliseconds(mapping->head_timestamp, + mapping->current_index * 60000); + + /* Initialize nulls and values arrays. */ + memset(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(array_position); + values[1] = TimestampTzGetDatum(timestamp); + values[2] = TransactionIdGetDatum(mapping->xid_by_minute[array_position]); + + return heap_form_tuple(tupdesc, values, nulls); +} diff --git a/doc/src/sgml/contrib.sgml b/doc/src/sgml/contrib.sgml index c82dde272639..4e833d79ef9b 100644 --- a/doc/src/sgml/contrib.sgml +++ b/doc/src/sgml/contrib.sgml @@ -116,6 +116,7 @@ CREATE EXTENSION module_name; &isn; &lo; <ree; + &oldsnapshot; &pageinspect; &passwordcheck; &pgbuffercache; diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 828396d4a9ee..47271addc14d 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -129,6 +129,7 @@ + diff --git a/doc/src/sgml/oldsnapshot.sgml b/doc/src/sgml/oldsnapshot.sgml new file mode 100644 index 000000000000..a665ae72e789 --- /dev/null +++ b/doc/src/sgml/oldsnapshot.sgml @@ -0,0 +1,33 @@ + + + + old_snapshot + + + old_snapshot + + + + The old_snapshot module allows inspection + of the server state that is used to implement + . + + + + Functions + + + + pg_old_snapshot_time_mapping(array_offset OUT int4, end_timestamp OUT timestamptz, newest_xmin OUT xid) returns setof record + + + Returns all of the entries in the server's timestamp to XID mapping. + Each entry represents the newest xmin of any snapshot taken in the + corresponding minute. + + + + + + + From c005eb00e7d878cb869854f592103f774e15d01e Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 24 Sep 2020 20:45:57 +0200 Subject: [PATCH 199/589] Standardize the printf format for st_size Existing code used various inconsistent ways to printf struct stat's st_size member. The type of that is off_t, which is in most cases a signed 64-bit integer, so use the long long int format for it. --- src/backend/access/transam/twophase.c | 12 ++++++------ src/backend/access/transam/xlogarchive.c | 6 +++--- src/bin/pg_basebackup/pg_receivewal.c | 4 ++-- src/bin/pg_verifybackup/pg_verifybackup.c | 8 ++++---- src/fe_utils/archive.c | 6 +++--- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index ef4f9981e359..794006044311 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1243,10 +1243,10 @@ ReadTwoPhaseFile(TransactionId xid, bool missing_ok) stat.st_size > MaxAllocSize) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg_plural("incorrect size of file \"%s\": %zu byte", - "incorrect size of file \"%s\": %zu bytes", - (Size) stat.st_size, path, - (Size) stat.st_size))); + errmsg_plural("incorrect size of file \"%s\": %lld byte", + "incorrect size of file \"%s\": %lld bytes", + (long long int) stat.st_size, path, + (long long int) stat.st_size))); crc_offset = stat.st_size - sizeof(pg_crc32c); if (crc_offset != MAXALIGN(crc_offset)) @@ -1270,8 +1270,8 @@ ReadTwoPhaseFile(TransactionId xid, bool missing_ok) errmsg("could not read file \"%s\": %m", path))); else ereport(ERROR, - (errmsg("could not read file \"%s\": read %d of %zu", - path, r, (Size) stat.st_size))); + (errmsg("could not read file \"%s\": read %d of %lld", + path, r, (long long int) stat.st_size))); } pgstat_report_wait_end(); diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c index 8f8734dc1d4e..cae93ab69dd7 100644 --- a/src/backend/access/transam/xlogarchive.c +++ b/src/backend/access/transam/xlogarchive.c @@ -202,10 +202,10 @@ RestoreArchivedFile(char *path, const char *xlogfname, else elevel = FATAL; ereport(elevel, - (errmsg("archive file \"%s\" has wrong size: %lu instead of %lu", + (errmsg("archive file \"%s\" has wrong size: %lld instead of %lld", xlogfname, - (unsigned long) stat_buf.st_size, - (unsigned long) expectedSize))); + (long long int) stat_buf.st_size, + (long long int) expectedSize))); return false; } else diff --git a/src/bin/pg_basebackup/pg_receivewal.c b/src/bin/pg_basebackup/pg_receivewal.c index cd05f5fede18..cddc896390da 100644 --- a/src/bin/pg_basebackup/pg_receivewal.c +++ b/src/bin/pg_basebackup/pg_receivewal.c @@ -269,8 +269,8 @@ FindStreamingStart(uint32 *tli) if (statbuf.st_size != WalSegSz) { - pg_log_warning("segment file \"%s\" has incorrect size %d, skipping", - dirent->d_name, (int) statbuf.st_size); + pg_log_warning("segment file \"%s\" has incorrect size %lld, skipping", + dirent->d_name, (long long int) statbuf.st_size); continue; } } diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c index 20140aa02748..bb3733b57e20 100644 --- a/src/bin/pg_verifybackup/pg_verifybackup.c +++ b/src/bin/pg_verifybackup/pg_verifybackup.c @@ -411,8 +411,8 @@ parse_manifest_file(char *manifest_path, manifest_files_hash **ht_p, report_fatal_error("could not read file \"%s\": %m", manifest_path); else - report_fatal_error("could not read file \"%s\": read %d of %zu", - manifest_path, rc, (size_t) statbuf.st_size); + report_fatal_error("could not read file \"%s\": read %d of %lld", + manifest_path, rc, (long long int) statbuf.st_size); } /* Close the manifest file. */ @@ -638,8 +638,8 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath) if (m->size != sb.st_size) { report_backup_error(context, - "\"%s\" has size %zu on disk but size %zu in the manifest", - relpath, (size_t) sb.st_size, m->size); + "\"%s\" has size %lld on disk but size %zu in the manifest", + relpath, (long long int) sb.st_size, m->size); m->bad = true; } diff --git a/src/fe_utils/archive.c b/src/fe_utils/archive.c index 1e9d994af63c..252dc0fb6a5d 100644 --- a/src/fe_utils/archive.c +++ b/src/fe_utils/archive.c @@ -71,9 +71,9 @@ RestoreArchivedFile(const char *path, const char *xlogfname, { if (expectedSize > 0 && stat_buf.st_size != expectedSize) { - pg_log_fatal("unexpected file size for \"%s\": %lu instead of %lu", - xlogfname, (unsigned long) stat_buf.st_size, - (unsigned long) expectedSize); + pg_log_fatal("unexpected file size for \"%s\": %lld instead of %lld", + xlogfname, (long long int) stat_buf.st_size, + (long long int) expectedSize); exit(1); } else From 55b7e2f4d78d8aa7b4a5eae9a0a810601d03c563 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 24 Sep 2020 15:27:19 -0400 Subject: [PATCH 200/589] Fix two bugs in MaintainOldSnapshotTimeMapping. The previous coding was confused about whether head_timestamp was intended to represent the timestamp for the newest bucket in the mapping or the oldest timestamp for the oldest bucket in the mapping. Decide that it's intended to be the oldest one, and repair accordingly. To do that, we need to do two things. First, when advancing to a new bucket, don't categorically set head_timestamp to the new timestamp. Do this only if we're blowing out the map completely because a lot of time has passed since we last maintained it. If we're replacing entries one by one, advance head_timestamp by 1 minute for each; if we're filling in unused entries, don't advance head_timestamp at all. Second, fix the computation of how many buckets we need to advance. The previous formula would be correct if head_timestamp were the timestamp for the new bucket, but we're now making all the code agree that it's the timestamp for the oldest bucket, so adjust the formula accordingly. This is certainly a bug fix, but I don't feel good about back-patching it without the introspection tools added by commit aecf5ee2bb36c597d3c6142e367e38d67816c777, and perhaps also some actual tests. Since back-patching the introspection tools might not attract sufficient support and since there are no automated tests of these fixes yet, I'm just committing this to master for now. Patch by me, reviewed by Thomas Munro, Dilip Kumar, Hamid Akhtar. Discussion: http://postgr.es/m/CA+TgmoY=aqf0zjTD+3dUWYkgMiNDegDLFjo+6ze=Wtpik+3XqA@mail.gmail.com --- src/backend/utils/time/snapmgr.c | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/backend/utils/time/snapmgr.c b/src/backend/utils/time/snapmgr.c index 650c2aa81595..8c41483e87c5 100644 --- a/src/backend/utils/time/snapmgr.c +++ b/src/backend/utils/time/snapmgr.c @@ -1949,10 +1949,32 @@ MaintainOldSnapshotTimeMapping(TimestampTz whenTaken, TransactionId xmin) else { /* We need a new bucket, but it might not be the very next one. */ - int advance = ((ts - oldSnapshotControl->head_timestamp) - / USECS_PER_MINUTE); + int distance_to_new_tail; + int distance_to_current_tail; + int advance; - oldSnapshotControl->head_timestamp = ts; + /* + * Our goal is for the new "tail" of the mapping, that is, the entry + * which is newest and thus furthest from the "head" entry, to + * correspond to "ts". Since there's one entry per minute, the + * distance between the current head and the new tail is just the + * number of minutes of difference between ts and the current + * head_timestamp. + * + * The distance from the current head to the current tail is one + * less than the number of entries in the mapping, because the + * entry at the head_offset is for 0 minutes after head_timestamp. + * + * The difference between these two values is the number of minutes + * by which we need to advance the mapping, either adding new entries + * or rotating old ones out. + */ + distance_to_new_tail = + (ts - oldSnapshotControl->head_timestamp) / USECS_PER_MINUTE; + distance_to_current_tail = + oldSnapshotControl->count_used - 1; + advance = distance_to_new_tail - distance_to_current_tail; + Assert(advance > 0); if (advance >= OLD_SNAPSHOT_TIME_MAP_ENTRIES) { @@ -1960,6 +1982,7 @@ MaintainOldSnapshotTimeMapping(TimestampTz whenTaken, TransactionId xmin) oldSnapshotControl->head_offset = 0; oldSnapshotControl->count_used = 1; oldSnapshotControl->xid_by_minute[0] = xmin; + oldSnapshotControl->head_timestamp = ts; } else { @@ -1978,6 +2001,7 @@ MaintainOldSnapshotTimeMapping(TimestampTz whenTaken, TransactionId xmin) else oldSnapshotControl->head_offset = old_head + 1; oldSnapshotControl->xid_by_minute[old_head] = xmin; + oldSnapshotControl->head_timestamp += USECS_PER_MINUTE; } else { From a45bc8a4f6495072bc48ad40a5aa0304979114f7 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 24 Sep 2020 18:19:38 -0400 Subject: [PATCH 201/589] Fix handling of -d "connection string" in pg_dump/pg_restore. Parallel pg_dump failed if its -d parameter was a connection string containing any essential information other than host, port, or username. The same was true for pg_restore with --create. The reason is that these scenarios failed to preserve the connection string from the command line; the code felt free to replace that with just the database name when reconnecting from a pg_dump parallel worker or after creating the target database. By chance, parallel pg_restore did not suffer this defect, as long as you didn't say --create. In practice it seems that the error would be obvious only if the connstring included essential, non-default SSL or GSS parameters. This may explain why it took us so long to notice. (It also makes it very difficult to craft a regression test case illustrating the problem, since the test would fail in builds without those options.) Fix by refactoring so that ConnectDatabase always receives all the relevant options directly from the command line, rather than reconstructed values. Inject a different database name, when necessary, by relying on libpq's rules for handling multiple "dbname" parameters. While here, let's get rid of the essentially duplicate _connectDB function, as well as some obsolete nearby cruft. Per bug #16604 from Zsolt Ero. Back-patch to all supported branches. Discussion: https://postgr.es/m/16604-933f4b8791227b15@postgresql.org --- src/bin/pg_dump/pg_backup.h | 35 ++-- src/bin/pg_dump/pg_backup_archiver.c | 96 +++-------- src/bin/pg_dump/pg_backup_archiver.h | 3 +- src/bin/pg_dump/pg_backup_db.c | 237 ++++++++------------------- src/bin/pg_dump/pg_dump.c | 24 +-- src/bin/pg_dump/pg_restore.c | 16 +- 6 files changed, 130 insertions(+), 281 deletions(-) diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 1017abbbe58c..a6a8e6f2fd86 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -58,6 +58,20 @@ typedef enum _teSection SECTION_POST_DATA /* stuff to be processed after data */ } teSection; +/* Parameters needed by ConnectDatabase; same for dump and restore */ +typedef struct _connParams +{ + /* These fields record the actual command line parameters */ + char *dbname; /* this may be a connstring! */ + char *pgport; + char *pghost; + char *username; + trivalue promptPassword; + /* If not NULL, this overrides the dbname obtained from command line */ + /* (but *only* the DB name, not anything else in the connstring) */ + char *override_dbname; +} ConnParams; + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -107,12 +121,9 @@ typedef struct _restoreOptions SimpleStringList tableNames; int useDB; - char *dbname; /* subject to expand_dbname */ - char *pgport; - char *pghost; - char *username; + ConnParams cparams; /* parameters to use if useDB */ + int noDataForFailedTables; - trivalue promptPassword; int exit_on_error; int compression; int suppressDumpWarnings; /* Suppress output of WARNING entries @@ -127,10 +138,7 @@ typedef struct _restoreOptions typedef struct _dumpOptions { - const char *dbname; /* subject to expand_dbname */ - const char *pghost; - const char *pgport; - const char *username; + ConnParams cparams; int binary_upgrade; @@ -247,12 +255,9 @@ typedef void (*SetupWorkerPtrType) (Archive *AH); * Main archiver interface. */ -extern void ConnectDatabase(Archive *AH, - const char *dbname, - const char *pghost, - const char *pgport, - const char *username, - trivalue prompt_password); +extern void ConnectDatabase(Archive *AHX, + const ConnParams *cparams, + bool isReconnect); extern void DisconnectDatabase(Archive *AHX); extern PGconn *GetConnection(Archive *AHX); diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 3567e9f365f5..d61b290d2a62 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -165,6 +165,7 @@ InitDumpOptions(DumpOptions *opts) memset(opts, 0, sizeof(DumpOptions)); /* set any fields that shouldn't default to zeroes */ opts->include_everything = true; + opts->cparams.promptPassword = TRI_DEFAULT; opts->dumpSections = DUMP_UNSECTIONED; } @@ -178,6 +179,11 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt) DumpOptions *dopt = NewDumpOptions(); /* this is the inverse of what's at the end of pg_dump.c's main() */ + dopt->cparams.dbname = ropt->cparams.dbname ? pg_strdup(ropt->cparams.dbname) : NULL; + dopt->cparams.pgport = ropt->cparams.pgport ? pg_strdup(ropt->cparams.pgport) : NULL; + dopt->cparams.pghost = ropt->cparams.pghost ? pg_strdup(ropt->cparams.pghost) : NULL; + dopt->cparams.username = ropt->cparams.username ? pg_strdup(ropt->cparams.username) : NULL; + dopt->cparams.promptPassword = ropt->cparams.promptPassword; dopt->outputClean = ropt->dropSchema; dopt->dataOnly = ropt->dataOnly; dopt->schemaOnly = ropt->schemaOnly; @@ -410,9 +416,7 @@ RestoreArchive(Archive *AHX) AHX->minRemoteVersion = 0; AHX->maxRemoteVersion = 9999999; - ConnectDatabase(AHX, ropt->dbname, - ropt->pghost, ropt->pgport, ropt->username, - ropt->promptPassword); + ConnectDatabase(AHX, &ropt->cparams, false); /* * If we're talking to the DB directly, don't send comments since they @@ -832,16 +836,8 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel) if (strcmp(te->desc, "DATABASE") == 0 || strcmp(te->desc, "DATABASE PROPERTIES") == 0) { - PQExpBufferData connstr; - - initPQExpBuffer(&connstr); - appendPQExpBufferStr(&connstr, "dbname="); - appendConnStrVal(&connstr, te->tag); - /* Abandon struct, but keep its buffer until process exit. */ - pg_log_info("connecting to new database \"%s\"", te->tag); _reconnectToDB(AH, te->tag); - ropt->dbname = connstr.data; } } @@ -973,7 +969,7 @@ NewRestoreOptions(void) /* set any fields that shouldn't default to zeroes */ opts->format = archUnknown; - opts->promptPassword = TRI_DEFAULT; + opts->cparams.promptPassword = TRI_DEFAULT; opts->dumpSections = DUMP_UNSECTIONED; return opts; @@ -2345,8 +2341,6 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, else AH->format = fmt; - AH->promptPassword = TRI_DEFAULT; - switch (AH->format) { case archCustom: @@ -3207,27 +3201,20 @@ _doSetSessionAuth(ArchiveHandle *AH, const char *user) * If we're currently restoring right into a database, this will * actually establish a connection. Otherwise it puts a \connect into * the script output. - * - * NULL dbname implies reconnecting to the current DB (pretty useless). */ static void _reconnectToDB(ArchiveHandle *AH, const char *dbname) { if (RestoringToDB(AH)) - ReconnectToServer(AH, dbname, NULL); + ReconnectToServer(AH, dbname); else { - if (dbname) - { - PQExpBufferData connectbuf; + PQExpBufferData connectbuf; - initPQExpBuffer(&connectbuf); - appendPsqlMetaConnect(&connectbuf, dbname); - ahprintf(AH, "%s\n", connectbuf.data); - termPQExpBuffer(&connectbuf); - } - else - ahprintf(AH, "%s\n", "\\connect -\n"); + initPQExpBuffer(&connectbuf); + appendPsqlMetaConnect(&connectbuf, dbname); + ahprintf(AH, "%s\n", connectbuf.data); + termPQExpBuffer(&connectbuf); } /* @@ -4159,9 +4146,7 @@ restore_toc_entries_postfork(ArchiveHandle *AH, TocEntry *pending_list) /* * Now reconnect the single parent connection. */ - ConnectDatabase((Archive *) AH, ropt->dbname, - ropt->pghost, ropt->pgport, ropt->username, - ropt->promptPassword); + ConnectDatabase((Archive *) AH, &ropt->cparams, true); /* re-establish fixed state */ _doSetFixedOutputState(AH); @@ -4823,54 +4808,15 @@ CloneArchive(ArchiveHandle *AH) clone->public.n_errors = 0; /* - * Connect our new clone object to the database: In parallel restore the - * parent is already disconnected, because we can connect the worker - * processes independently to the database (no snapshot sync required). In - * parallel backup we clone the parent's existing connection. + * Connect our new clone object to the database, using the same connection + * parameters used for the original connection. */ - if (AH->mode == archModeRead) - { - RestoreOptions *ropt = AH->public.ropt; - - Assert(AH->connection == NULL); - - /* this also sets clone->connection */ - ConnectDatabase((Archive *) clone, ropt->dbname, - ropt->pghost, ropt->pgport, ropt->username, - ropt->promptPassword); + ConnectDatabase((Archive *) clone, &clone->public.ropt->cparams, true); - /* re-establish fixed state */ + /* re-establish fixed state */ + if (AH->mode == archModeRead) _doSetFixedOutputState(clone); - } - else - { - PQExpBufferData connstr; - char *pghost; - char *pgport; - char *username; - - Assert(AH->connection != NULL); - - /* - * Even though we are technically accessing the parent's database - * object here, these functions are fine to be called like that - * because all just return a pointer and do not actually send/receive - * any data to/from the database. - */ - initPQExpBuffer(&connstr); - appendPQExpBufferStr(&connstr, "dbname="); - appendConnStrVal(&connstr, PQdb(AH->connection)); - pghost = PQhost(AH->connection); - pgport = PQport(AH->connection); - username = PQuser(AH->connection); - - /* this also sets clone->connection */ - ConnectDatabase((Archive *) clone, connstr.data, - pghost, pgport, username, TRI_NO); - - termPQExpBuffer(&connstr); - /* setupDumpWorker will fix up connection state */ - } + /* in write case, setupDumpWorker will fix up connection state */ /* Let the format-specific code have a chance too */ clone->ClonePtr(clone); diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index f9e6b42752af..fb8d226d487d 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -303,7 +303,6 @@ struct _archiveHandle /* Stuff for direct DB connection */ char *archdbname; /* DB name *read* from archive */ - trivalue promptPassword; char *savedPassword; /* password for ropt->username, if known */ char *use_role; PGconn *connection; @@ -471,7 +470,7 @@ extern void InitArchiveFmt_Tar(ArchiveHandle *AH); extern bool isValidTarHeader(char *header); -extern void ReconnectToServer(ArchiveHandle *AH, const char *dbname, const char *newUser); +extern void ReconnectToServer(ArchiveHandle *AH, const char *dbname); extern void DropBlobIfExists(ArchiveHandle *AH, Oid oid); void ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH); diff --git a/src/bin/pg_dump/pg_backup_db.c b/src/bin/pg_dump/pg_backup_db.c index 12899e26e292..5ba43441f50a 100644 --- a/src/bin/pg_dump/pg_backup_db.c +++ b/src/bin/pg_dump/pg_backup_db.c @@ -27,7 +27,6 @@ #include "pg_backup_utils.h" static void _check_database_version(ArchiveHandle *AH); -static PGconn *_connectDB(ArchiveHandle *AH, const char *newdbname, const char *newUser); static void notice_processor(void *arg, const char *message); static void @@ -73,211 +72,100 @@ _check_database_version(ArchiveHandle *AH) /* * Reconnect to the server. If dbname is not NULL, use that database, - * else the one associated with the archive handle. If username is - * not NULL, use that user name, else the one from the handle. + * else the one associated with the archive handle. */ void -ReconnectToServer(ArchiveHandle *AH, const char *dbname, const char *username) +ReconnectToServer(ArchiveHandle *AH, const char *dbname) { - PGconn *newConn; - const char *newdbname; - const char *newusername; - - if (!dbname) - newdbname = PQdb(AH->connection); - else - newdbname = dbname; - - if (!username) - newusername = PQuser(AH->connection); - else - newusername = username; - - newConn = _connectDB(AH, newdbname, newusername); - - /* Update ArchiveHandle's connCancel before closing old connection */ - set_archive_cancel_info(AH, newConn); - - PQfinish(AH->connection); - AH->connection = newConn; - - /* Start strict; later phases may override this. */ - PQclear(ExecuteSqlQueryForSingleRow((Archive *) AH, - ALWAYS_SECURE_SEARCH_PATH_SQL)); -} - -/* - * Connect to the db again. - * - * Note: it's not really all that sensible to use a single-entry password - * cache if the username keeps changing. In current usage, however, the - * username never does change, so one savedPassword is sufficient. We do - * update the cache on the off chance that the password has changed since the - * start of the run. - */ -static PGconn * -_connectDB(ArchiveHandle *AH, const char *reqdb, const char *requser) -{ - PQExpBufferData connstr; - PGconn *newConn; - const char *newdb; - const char *newuser; - char *password; - bool new_pass; - - if (!reqdb) - newdb = PQdb(AH->connection); - else - newdb = reqdb; - - if (!requser || strlen(requser) == 0) - newuser = PQuser(AH->connection); - else - newuser = requser; - - pg_log_info("connecting to database \"%s\" as user \"%s\"", - newdb, newuser); - - password = AH->savedPassword; - - if (AH->promptPassword == TRI_YES && password == NULL) - password = simple_prompt("Password: ", false); - - initPQExpBuffer(&connstr); - appendPQExpBufferStr(&connstr, "dbname="); - appendConnStrVal(&connstr, newdb); - - do - { - const char *keywords[7]; - const char *values[7]; - - keywords[0] = "host"; - values[0] = PQhost(AH->connection); - keywords[1] = "port"; - values[1] = PQport(AH->connection); - keywords[2] = "user"; - values[2] = newuser; - keywords[3] = "password"; - values[3] = password; - keywords[4] = "dbname"; - values[4] = connstr.data; - keywords[5] = "fallback_application_name"; - values[5] = progname; - keywords[6] = NULL; - values[6] = NULL; - - new_pass = false; - newConn = PQconnectdbParams(keywords, values, true); - - if (!newConn) - fatal("could not reconnect to database"); - - if (PQstatus(newConn) == CONNECTION_BAD) - { - if (!PQconnectionNeedsPassword(newConn)) - fatal("could not reconnect to database: %s", - PQerrorMessage(newConn)); - PQfinish(newConn); - - if (password) - fprintf(stderr, "Password incorrect\n"); - - fprintf(stderr, "Connecting to %s as %s\n", - newdb, newuser); - - if (AH->promptPassword != TRI_NO) - { - if (password && password != AH->savedPassword) - free(password); - password = simple_prompt("Password: ", false); - } - else - fatal("connection needs password"); - - new_pass = true; - } - } while (new_pass); - - if (password && password != AH->savedPassword) - free(password); + PGconn *oldConn = AH->connection; + RestoreOptions *ropt = AH->public.ropt; /* - * We want to remember connection's actual password, whether or not we got - * it by prompting. So we don't just store the password variable. + * Save the dbname, if given, in override_dbname so that it will also + * affect any later reconnection attempt. */ - if (PQconnectionUsedPassword(newConn)) - { - if (AH->savedPassword) - free(AH->savedPassword); - AH->savedPassword = pg_strdup(PQpass(newConn)); - } - - termPQExpBuffer(&connstr); + if (dbname) + ropt->cparams.override_dbname = pg_strdup(dbname); - /* check for version mismatch */ - _check_database_version(AH); + /* + * Note: we want to establish the new connection, and in particular update + * ArchiveHandle's connCancel, before closing old connection. Otherwise + * an ill-timed SIGINT could try to access a dead connection. + */ + AH->connection = NULL; /* dodge error check in ConnectDatabase */ - PQsetNoticeProcessor(newConn, notice_processor, NULL); + ConnectDatabase((Archive *) AH, &ropt->cparams, true); - return newConn; + PQfinish(oldConn); } - /* - * Make a database connection with the given parameters. The - * connection handle is returned, the parameters are stored in AHX. - * An interactive password prompt is automatically issued if required. + * Make, or remake, a database connection with the given parameters. + * + * The resulting connection handle is stored in AHX->connection. * + * An interactive password prompt is automatically issued if required. + * We store the results of that in AHX->savedPassword. * Note: it's not really all that sensible to use a single-entry password * cache if the username keeps changing. In current usage, however, the * username never does change, so one savedPassword is sufficient. */ void ConnectDatabase(Archive *AHX, - const char *dbname, - const char *pghost, - const char *pgport, - const char *username, - trivalue prompt_password) + const ConnParams *cparams, + bool isReconnect) { ArchiveHandle *AH = (ArchiveHandle *) AHX; + trivalue prompt_password; char *password; bool new_pass; if (AH->connection) fatal("already connected to a database"); + /* Never prompt for a password during a reconnection */ + prompt_password = isReconnect ? TRI_NO : cparams->promptPassword; + password = AH->savedPassword; if (prompt_password == TRI_YES && password == NULL) password = simple_prompt("Password: ", false); - AH->promptPassword = prompt_password; - /* * Start the connection. Loop until we have a password if requested by * backend. */ do { - const char *keywords[7]; - const char *values[7]; - - keywords[0] = "host"; - values[0] = pghost; - keywords[1] = "port"; - values[1] = pgport; - keywords[2] = "user"; - values[2] = username; - keywords[3] = "password"; - values[3] = password; - keywords[4] = "dbname"; - values[4] = dbname; - keywords[5] = "fallback_application_name"; - values[5] = progname; - keywords[6] = NULL; - values[6] = NULL; + const char *keywords[8]; + const char *values[8]; + int i = 0; + + /* + * If dbname is a connstring, its entries can override the other + * values obtained from cparams; but in turn, override_dbname can + * override the dbname component of it. + */ + keywords[i] = "host"; + values[i++] = cparams->pghost; + keywords[i] = "port"; + values[i++] = cparams->pgport; + keywords[i] = "user"; + values[i++] = cparams->username; + keywords[i] = "password"; + values[i++] = password; + keywords[i] = "dbname"; + values[i++] = cparams->dbname; + if (cparams->override_dbname) + { + keywords[i] = "dbname"; + values[i++] = cparams->override_dbname; + } + keywords[i] = "fallback_application_name"; + values[i++] = progname; + keywords[i] = NULL; + values[i++] = NULL; + Assert(i <= lengthof(keywords)); new_pass = false; AH->connection = PQconnectdbParams(keywords, values, true); @@ -298,9 +186,16 @@ ConnectDatabase(Archive *AHX, /* check to see that the backend connection was successfully made */ if (PQstatus(AH->connection) == CONNECTION_BAD) - fatal("connection to database \"%s\" failed: %s", - PQdb(AH->connection) ? PQdb(AH->connection) : "", - PQerrorMessage(AH->connection)); + { + if (isReconnect) + fatal("reconnection to database \"%s\" failed: %s", + PQdb(AH->connection) ? PQdb(AH->connection) : "", + PQerrorMessage(AH->connection)); + else + fatal("connection to database \"%s\" failed: %s", + PQdb(AH->connection) ? PQdb(AH->connection) : "", + PQerrorMessage(AH->connection)); + } /* Start strict; later phases may override this. */ PQclear(ExecuteSqlQueryForSingleRow((Archive *) AH, diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index f021bb72f40a..76320468baa2 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -314,7 +314,6 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - trivalue prompt_password = TRI_DEFAULT; int compressLevel = -1; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; @@ -444,7 +443,7 @@ main(int argc, char **argv) break; case 'd': /* database name */ - dopt.dbname = pg_strdup(optarg); + dopt.cparams.dbname = pg_strdup(optarg); break; case 'E': /* Dump encoding */ @@ -460,7 +459,7 @@ main(int argc, char **argv) break; case 'h': /* server host */ - dopt.pghost = pg_strdup(optarg); + dopt.cparams.pghost = pg_strdup(optarg); break; case 'j': /* number of dump jobs */ @@ -481,7 +480,7 @@ main(int argc, char **argv) break; case 'p': /* server port */ - dopt.pgport = pg_strdup(optarg); + dopt.cparams.pgport = pg_strdup(optarg); break; case 'R': @@ -506,7 +505,7 @@ main(int argc, char **argv) break; case 'U': - dopt.username = pg_strdup(optarg); + dopt.cparams.username = pg_strdup(optarg); break; case 'v': /* verbose */ @@ -515,11 +514,11 @@ main(int argc, char **argv) break; case 'w': - prompt_password = TRI_NO; + dopt.cparams.promptPassword = TRI_NO; break; case 'W': - prompt_password = TRI_YES; + dopt.cparams.promptPassword = TRI_YES; break; case 'x': /* skip ACL dump */ @@ -613,8 +612,8 @@ main(int argc, char **argv) * Non-option argument specifies database name as long as it wasn't * already specified with -d / --dbname */ - if (optind < argc && dopt.dbname == NULL) - dopt.dbname = argv[optind++]; + if (optind < argc && dopt.cparams.dbname == NULL) + dopt.cparams.dbname = argv[optind++]; /* Complain if any arguments remain */ if (optind < argc) @@ -740,7 +739,7 @@ main(int argc, char **argv) * Open the database using the Archiver, so it knows about it. Errors mean * death. */ - ConnectDatabase(fout, dopt.dbname, dopt.pghost, dopt.pgport, dopt.username, prompt_password); + ConnectDatabase(fout, &dopt.cparams, false); setup_connection(fout, dumpencoding, dumpsnapshot, use_role); /* @@ -918,6 +917,11 @@ main(int argc, char **argv) ropt->filename = filename; /* if you change this list, see dumpOptionsFromRestoreOptions */ + ropt->cparams.dbname = dopt.cparams.dbname ? pg_strdup(dopt.cparams.dbname) : NULL; + ropt->cparams.pgport = dopt.cparams.pgport ? pg_strdup(dopt.cparams.pgport) : NULL; + ropt->cparams.pghost = dopt.cparams.pghost ? pg_strdup(dopt.cparams.pghost) : NULL; + ropt->cparams.username = dopt.cparams.username ? pg_strdup(dopt.cparams.username) : NULL; + ropt->cparams.promptPassword = dopt.cparams.promptPassword; ropt->dropSchema = dopt.outputClean; ropt->dataOnly = dopt.dataOnly; ropt->schemaOnly = dopt.schemaOnly; diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c index eebf0d300ba9..589b4aed5398 100644 --- a/src/bin/pg_dump/pg_restore.c +++ b/src/bin/pg_dump/pg_restore.c @@ -163,7 +163,7 @@ main(int argc, char **argv) opts->createDB = 1; break; case 'd': - opts->dbname = pg_strdup(optarg); + opts->cparams.dbname = pg_strdup(optarg); break; case 'e': opts->exit_on_error = true; @@ -177,7 +177,7 @@ main(int argc, char **argv) break; case 'h': if (strlen(optarg) != 0) - opts->pghost = pg_strdup(optarg); + opts->cparams.pghost = pg_strdup(optarg); break; case 'j': /* number of restore jobs */ @@ -206,7 +206,7 @@ main(int argc, char **argv) case 'p': if (strlen(optarg) != 0) - opts->pgport = pg_strdup(optarg); + opts->cparams.pgport = pg_strdup(optarg); break; case 'R': /* no-op, still accepted for backwards compatibility */ @@ -240,7 +240,7 @@ main(int argc, char **argv) break; case 'U': - opts->username = pg_strdup(optarg); + opts->cparams.username = pg_strdup(optarg); break; case 'v': /* verbose */ @@ -249,11 +249,11 @@ main(int argc, char **argv) break; case 'w': - opts->promptPassword = TRI_NO; + opts->cparams.promptPassword = TRI_NO; break; case 'W': - opts->promptPassword = TRI_YES; + opts->cparams.promptPassword = TRI_YES; break; case 'x': /* skip ACL dump */ @@ -303,14 +303,14 @@ main(int argc, char **argv) } /* Complain if neither -f nor -d was specified (except if dumping TOC) */ - if (!opts->dbname && !opts->filename && !opts->tocSummary) + if (!opts->cparams.dbname && !opts->filename && !opts->tocSummary) { pg_log_error("one of -d/--dbname and -f/--file must be specified"); exit_nicely(1); } /* Should get at most one of -d and -f, else user is confused */ - if (opts->dbname) + if (opts->cparams.dbname) { if (opts->filename) { From ca7f8e2b86e5f15a40b67e6199d714f45a467ff1 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Fri, 25 Sep 2020 10:25:55 +0900 Subject: [PATCH 202/589] Remove custom memory allocation layer in pgcrypto PX_OWN_ALLOC was intended as a way to disable the use of palloc(), and over the time new palloc() or equivalent calls have been added like in 32984d8, making this extra layer losing its original purpose. This simplifies on the way some code paths to use palloc0() rather than palloc() followed by memset(0). Author: Daniel Gustafsson Discussion: https://postgr.es/m/A5BFAA1A-B2E8-4CBC-895E-7B1B9475A527@yesql.se --- contrib/pgcrypto/imath.c | 10 ++++----- contrib/pgcrypto/internal-sha2.c | 28 +++++++++++-------------- contrib/pgcrypto/internal.c | 32 ++++++++++++----------------- contrib/pgcrypto/mbuf.c | 30 +++++++++++++-------------- contrib/pgcrypto/openssl.c | 8 ++++---- contrib/pgcrypto/pgp-cfb.c | 5 ++--- contrib/pgcrypto/pgp-compress.c | 18 ++++++++-------- contrib/pgcrypto/pgp-decrypt.c | 11 +++++----- contrib/pgcrypto/pgp-encrypt.c | 9 ++++---- contrib/pgcrypto/pgp-mpi-internal.c | 6 +++--- contrib/pgcrypto/pgp-mpi.c | 4 ++-- contrib/pgcrypto/pgp-pubenc.c | 12 +++++------ contrib/pgcrypto/pgp-pubkey.c | 5 ++--- contrib/pgcrypto/pgp.c | 5 ++--- contrib/pgcrypto/px-hmac.c | 21 +++++++++---------- contrib/pgcrypto/px.c | 32 +++++++++++++---------------- contrib/pgcrypto/px.h | 13 ------------ 17 files changed, 106 insertions(+), 143 deletions(-) diff --git a/contrib/pgcrypto/imath.c b/contrib/pgcrypto/imath.c index da4cdede76fb..9deaa797c1a0 100644 --- a/contrib/pgcrypto/imath.c +++ b/contrib/pgcrypto/imath.c @@ -478,7 +478,7 @@ mp_int_init(mp_int z) mp_int mp_int_alloc(void) { - mp_int out = px_alloc(sizeof(mpz_t)); + mp_int out = palloc(sizeof(mpz_t)); if (out != NULL) mp_int_init(out); @@ -604,7 +604,7 @@ mp_int_free(mp_int z) assert(z != NULL); mp_int_clear(z); - px_free(z); /* note: NOT s_free() */ + pfree(z); /* note: NOT s_free() */ } mp_result @@ -2212,7 +2212,7 @@ static const mp_digit fill = (mp_digit) 0xdeadbeefabad1dea; static mp_digit * s_alloc(mp_size num) { - mp_digit *out = px_alloc(num * sizeof(mp_digit)); + mp_digit *out = palloc(num * sizeof(mp_digit)); assert(out != NULL); @@ -2235,7 +2235,7 @@ s_realloc(mp_digit *old, mp_size osize, mp_size nsize) new[ix] = fill; memcpy(new, old, osize * sizeof(mp_digit)); #else - mp_digit *new = px_realloc(old, nsize * sizeof(mp_digit)); + mp_digit *new = repalloc(old, nsize * sizeof(mp_digit)); assert(new != NULL); #endif @@ -2246,7 +2246,7 @@ s_realloc(mp_digit *old, mp_size osize, mp_size nsize) static void s_free(void *ptr) { - px_free(ptr); + pfree(ptr); } static bool diff --git a/contrib/pgcrypto/internal-sha2.c b/contrib/pgcrypto/internal-sha2.c index e06f55445eff..9fa940b5bbbb 100644 --- a/contrib/pgcrypto/internal-sha2.c +++ b/contrib/pgcrypto/internal-sha2.c @@ -85,8 +85,8 @@ int_sha224_free(PX_MD *h) pg_sha224_ctx *ctx = (pg_sha224_ctx *) h->p.ptr; px_memset(ctx, 0, sizeof(*ctx)); - px_free(ctx); - px_free(h); + pfree(ctx); + pfree(h); } /* SHA256 */ @@ -133,8 +133,8 @@ int_sha256_free(PX_MD *h) pg_sha256_ctx *ctx = (pg_sha256_ctx *) h->p.ptr; px_memset(ctx, 0, sizeof(*ctx)); - px_free(ctx); - px_free(h); + pfree(ctx); + pfree(h); } /* SHA384 */ @@ -181,8 +181,8 @@ int_sha384_free(PX_MD *h) pg_sha384_ctx *ctx = (pg_sha384_ctx *) h->p.ptr; px_memset(ctx, 0, sizeof(*ctx)); - px_free(ctx); - px_free(h); + pfree(ctx); + pfree(h); } /* SHA512 */ @@ -229,8 +229,8 @@ int_sha512_free(PX_MD *h) pg_sha512_ctx *ctx = (pg_sha512_ctx *) h->p.ptr; px_memset(ctx, 0, sizeof(*ctx)); - px_free(ctx); - px_free(h); + pfree(ctx); + pfree(h); } /* init functions */ @@ -240,8 +240,7 @@ init_sha224(PX_MD *md) { pg_sha224_ctx *ctx; - ctx = px_alloc(sizeof(*ctx)); - memset(ctx, 0, sizeof(*ctx)); + ctx = palloc0(sizeof(*ctx)); md->p.ptr = ctx; @@ -260,8 +259,7 @@ init_sha256(PX_MD *md) { pg_sha256_ctx *ctx; - ctx = px_alloc(sizeof(*ctx)); - memset(ctx, 0, sizeof(*ctx)); + ctx = palloc0(sizeof(*ctx)); md->p.ptr = ctx; @@ -280,8 +278,7 @@ init_sha384(PX_MD *md) { pg_sha384_ctx *ctx; - ctx = px_alloc(sizeof(*ctx)); - memset(ctx, 0, sizeof(*ctx)); + ctx = palloc0(sizeof(*ctx)); md->p.ptr = ctx; @@ -300,8 +297,7 @@ init_sha512(PX_MD *md) { pg_sha512_ctx *ctx; - ctx = px_alloc(sizeof(*ctx)); - memset(ctx, 0, sizeof(*ctx)); + ctx = palloc0(sizeof(*ctx)); md->p.ptr = ctx; diff --git a/contrib/pgcrypto/internal.c b/contrib/pgcrypto/internal.c index a12d7b417832..06469d41c0a7 100644 --- a/contrib/pgcrypto/internal.c +++ b/contrib/pgcrypto/internal.c @@ -123,8 +123,8 @@ int_md5_free(PX_MD *h) MD5_CTX *ctx = (MD5_CTX *) h->p.ptr; px_memset(ctx, 0, sizeof(*ctx)); - px_free(ctx); - px_free(h); + pfree(ctx); + pfree(h); } /* SHA1 */ @@ -171,8 +171,8 @@ int_sha1_free(PX_MD *h) SHA1_CTX *ctx = (SHA1_CTX *) h->p.ptr; px_memset(ctx, 0, sizeof(*ctx)); - px_free(ctx); - px_free(h); + pfree(ctx); + pfree(h); } /* init functions */ @@ -182,8 +182,7 @@ init_md5(PX_MD *md) { MD5_CTX *ctx; - ctx = px_alloc(sizeof(*ctx)); - memset(ctx, 0, sizeof(*ctx)); + ctx = palloc0(sizeof(*ctx)); md->p.ptr = ctx; @@ -202,8 +201,7 @@ init_sha1(PX_MD *md) { SHA1_CTX *ctx; - ctx = px_alloc(sizeof(*ctx)); - memset(ctx, 0, sizeof(*ctx)); + ctx = palloc0(sizeof(*ctx)); md->p.ptr = ctx; @@ -246,9 +244,9 @@ intctx_free(PX_Cipher *c) if (cx) { px_memset(cx, 0, sizeof *cx); - px_free(cx); + pfree(cx); } - px_free(c); + pfree(c); } /* @@ -373,8 +371,7 @@ rj_load(int mode) PX_Cipher *c; struct int_ctx *cx; - c = px_alloc(sizeof *c); - memset(c, 0, sizeof *c); + c = palloc0(sizeof *c); c->block_size = rj_block_size; c->key_size = rj_key_size; @@ -384,8 +381,7 @@ rj_load(int mode) c->decrypt = rj_decrypt; c->free = intctx_free; - cx = px_alloc(sizeof *cx); - memset(cx, 0, sizeof *cx); + cx = palloc0(sizeof *cx); cx->mode = mode; c->ptr = cx; @@ -482,8 +478,7 @@ bf_load(int mode) PX_Cipher *c; struct int_ctx *cx; - c = px_alloc(sizeof *c); - memset(c, 0, sizeof *c); + c = palloc0(sizeof *c); c->block_size = bf_block_size; c->key_size = bf_key_size; @@ -493,8 +488,7 @@ bf_load(int mode) c->decrypt = bf_decrypt; c->free = intctx_free; - cx = px_alloc(sizeof *cx); - memset(cx, 0, sizeof *cx); + cx = palloc0(sizeof *cx); cx->mode = mode; c->ptr = cx; return c; @@ -564,7 +558,7 @@ px_find_digest(const char *name, PX_MD **res) for (p = int_digest_list; p->name; p++) if (pg_strcasecmp(p->name, name) == 0) { - h = px_alloc(sizeof(*h)); + h = palloc(sizeof(*h)); p->init(h); *res = h; diff --git a/contrib/pgcrypto/mbuf.c b/contrib/pgcrypto/mbuf.c index 548ef6209745..bc668a0e802f 100644 --- a/contrib/pgcrypto/mbuf.c +++ b/contrib/pgcrypto/mbuf.c @@ -70,9 +70,9 @@ mbuf_free(MBuf *mbuf) if (mbuf->own_data) { px_memset(mbuf->data, 0, mbuf->buf_end - mbuf->data); - px_free(mbuf->data); + pfree(mbuf->data); } - px_free(mbuf); + pfree(mbuf); return 0; } @@ -88,7 +88,7 @@ prepare_room(MBuf *mbuf, int block_len) newlen = (mbuf->buf_end - mbuf->data) + ((block_len + STEP + STEP - 1) & -STEP); - newbuf = px_realloc(mbuf->data, newlen); + newbuf = repalloc(mbuf->data, newlen); mbuf->buf_end = newbuf + newlen; mbuf->data_end = newbuf + (mbuf->data_end - mbuf->data); @@ -121,8 +121,8 @@ mbuf_create(int len) if (!len) len = 8192; - mbuf = px_alloc(sizeof *mbuf); - mbuf->data = px_alloc(len); + mbuf = palloc(sizeof *mbuf); + mbuf->data = palloc(len); mbuf->buf_end = mbuf->data + len; mbuf->data_end = mbuf->data; mbuf->read_pos = mbuf->data; @@ -138,7 +138,7 @@ mbuf_create_from_data(uint8 *data, int len) { MBuf *mbuf; - mbuf = px_alloc(sizeof *mbuf); + mbuf = palloc(sizeof *mbuf); mbuf->data = (uint8 *) data; mbuf->buf_end = mbuf->data + len; mbuf->data_end = mbuf->data + len; @@ -219,15 +219,14 @@ pullf_create(PullFilter **pf_p, const PullFilterOps *op, void *init_arg, PullFil res = 0; } - pf = px_alloc(sizeof(*pf)); - memset(pf, 0, sizeof(*pf)); + pf = palloc0(sizeof(*pf)); pf->buflen = res; pf->op = op; pf->priv = priv; pf->src = src; if (pf->buflen > 0) { - pf->buf = px_alloc(pf->buflen); + pf->buf = palloc(pf->buflen); pf->pos = 0; } else @@ -248,11 +247,11 @@ pullf_free(PullFilter *pf) if (pf->buf) { px_memset(pf->buf, 0, pf->buflen); - px_free(pf->buf); + pfree(pf->buf); } px_memset(pf, 0, sizeof(*pf)); - px_free(pf); + pfree(pf); } /* may return less data than asked, 0 means eof */ @@ -386,15 +385,14 @@ pushf_create(PushFilter **mp_p, const PushFilterOps *op, void *init_arg, PushFil res = 0; } - mp = px_alloc(sizeof(*mp)); - memset(mp, 0, sizeof(*mp)); + mp = palloc0(sizeof(*mp)); mp->block_size = res; mp->op = op; mp->priv = priv; mp->next = next; if (mp->block_size > 0) { - mp->buf = px_alloc(mp->block_size); + mp->buf = palloc(mp->block_size); mp->pos = 0; } else @@ -415,11 +413,11 @@ pushf_free(PushFilter *mp) if (mp->buf) { px_memset(mp->buf, 0, mp->block_size); - px_free(mp->buf); + pfree(mp->buf); } px_memset(mp, 0, sizeof(*mp)); - px_free(mp); + pfree(mp); } void diff --git a/contrib/pgcrypto/openssl.c b/contrib/pgcrypto/openssl.c index 3057afb33909..90951a8ae7b0 100644 --- a/contrib/pgcrypto/openssl.c +++ b/contrib/pgcrypto/openssl.c @@ -156,7 +156,7 @@ digest_free(PX_MD *h) OSSLDigest *digest = (OSSLDigest *) h->p.ptr; free_openssl_digest(digest); - px_free(h); + pfree(h); } static int px_openssl_initialized = 0; @@ -214,7 +214,7 @@ px_find_digest(const char *name, PX_MD **res) open_digests = digest; /* The PX_MD object is allocated in the current memory context. */ - h = px_alloc(sizeof(*h)); + h = palloc(sizeof(*h)); h->result_size = digest_result_size; h->block_size = digest_block_size; h->reset = digest_reset; @@ -353,7 +353,7 @@ gen_ossl_free(PX_Cipher *c) OSSLCipher *od = (OSSLCipher *) c->ptr; free_openssl_cipher(od); - px_free(c); + pfree(c); } static int @@ -790,7 +790,7 @@ px_find_cipher(const char *name, PX_Cipher **res) od->evp_ciph = i->ciph->cipher_func(); /* The PX_Cipher is allocated in current memory context */ - c = px_alloc(sizeof(*c)); + c = palloc(sizeof(*c)); c->block_size = gen_ossl_block_size; c->key_size = gen_ossl_key_size; c->iv_size = gen_ossl_iv_size; diff --git a/contrib/pgcrypto/pgp-cfb.c b/contrib/pgcrypto/pgp-cfb.c index 8ae7c8608fb5..dafa562daa12 100644 --- a/contrib/pgcrypto/pgp-cfb.c +++ b/contrib/pgcrypto/pgp-cfb.c @@ -67,8 +67,7 @@ pgp_cfb_create(PGP_CFB **ctx_p, int algo, const uint8 *key, int key_len, return res; } - ctx = px_alloc(sizeof(*ctx)); - memset(ctx, 0, sizeof(*ctx)); + ctx = palloc0(sizeof(*ctx)); ctx->ciph = ciph; ctx->block_size = px_cipher_block_size(ciph); ctx->resync = resync; @@ -85,7 +84,7 @@ pgp_cfb_free(PGP_CFB *ctx) { px_cipher_free(ctx->ciph); px_memset(ctx, 0, sizeof(*ctx)); - px_free(ctx); + pfree(ctx); } /* diff --git a/contrib/pgcrypto/pgp-compress.c b/contrib/pgcrypto/pgp-compress.c index 3636a662b076..7e8ddba18735 100644 --- a/contrib/pgcrypto/pgp-compress.c +++ b/contrib/pgcrypto/pgp-compress.c @@ -57,13 +57,13 @@ struct ZipStat static void * z_alloc(void *priv, unsigned n_items, unsigned item_len) { - return px_alloc(n_items * item_len); + return palloc(n_items * item_len); } static void z_free(void *priv, void *addr) { - px_free(addr); + pfree(addr); } static int @@ -80,8 +80,7 @@ compress_init(PushFilter *next, void *init_arg, void **priv_p) /* * init */ - st = px_alloc(sizeof(*st)); - memset(st, 0, sizeof(*st)); + st = palloc0(sizeof(*st)); st->buf_len = ZIP_OUT_BUF; st->stream.zalloc = z_alloc; st->stream.zfree = z_free; @@ -93,7 +92,7 @@ compress_init(PushFilter *next, void *init_arg, void **priv_p) res = deflateInit(&st->stream, ctx->compress_level); if (res != Z_OK) { - px_free(st); + pfree(st); return PXE_PGP_COMPRESSION_ERROR; } *priv_p = st; @@ -174,7 +173,7 @@ compress_free(void *priv) deflateEnd(&st->stream); px_memset(st, 0, sizeof(*st)); - px_free(st); + pfree(st); } static const PushFilterOps @@ -212,8 +211,7 @@ decompress_init(void **priv_p, void *arg, PullFilter *src) && ctx->compress_algo != PGP_COMPR_ZIP) return PXE_PGP_UNSUPPORTED_COMPR; - dec = px_alloc(sizeof(*dec)); - memset(dec, 0, sizeof(*dec)); + dec = palloc0(sizeof(*dec)); dec->buf_len = ZIP_OUT_BUF; *priv_p = dec; @@ -226,7 +224,7 @@ decompress_init(void **priv_p, void *arg, PullFilter *src) res = inflateInit(&dec->stream); if (res != Z_OK) { - px_free(dec); + pfree(dec); px_debug("decompress_init: inflateInit error"); return PXE_PGP_COMPRESSION_ERROR; } @@ -318,7 +316,7 @@ decompress_free(void *priv) inflateEnd(&dec->stream); px_memset(dec, 0, sizeof(*dec)); - px_free(dec); + pfree(dec); } static const PullFilterOps diff --git a/contrib/pgcrypto/pgp-decrypt.c b/contrib/pgcrypto/pgp-decrypt.c index 3ecbf9c0c259..d12dcad19452 100644 --- a/contrib/pgcrypto/pgp-decrypt.c +++ b/contrib/pgcrypto/pgp-decrypt.c @@ -211,7 +211,7 @@ pktreader_free(void *priv) struct PktData *pkt = priv; px_memset(pkt, 0, sizeof(*pkt)); - px_free(pkt); + pfree(pkt); } static struct PullFilterOps pktreader_filter = { @@ -224,13 +224,13 @@ pgp_create_pkt_reader(PullFilter **pf_p, PullFilter *src, int len, int pkttype, PGP_Context *ctx) { int res; - struct PktData *pkt = px_alloc(sizeof(*pkt)); + struct PktData *pkt = palloc(sizeof(*pkt)); pkt->type = pkttype; pkt->len = len; res = pullf_create(pf_p, &pktreader_filter, pkt, src); if (res < 0) - px_free(pkt); + pfree(pkt); return res; } @@ -447,8 +447,7 @@ mdcbuf_init(void **priv_p, void *arg, PullFilter *src) PGP_Context *ctx = arg; struct MDCBufData *st; - st = px_alloc(sizeof(*st)); - memset(st, 0, sizeof(*st)); + st = palloc0(sizeof(*st)); st->buflen = sizeof(st->buf); st->ctx = ctx; *priv_p = st; @@ -576,7 +575,7 @@ mdcbuf_free(void *priv) px_md_free(st->ctx->mdc_ctx); st->ctx->mdc_ctx = NULL; px_memset(st, 0, sizeof(*st)); - px_free(st); + pfree(st); } static struct PullFilterOps mdcbuf_filter = { diff --git a/contrib/pgcrypto/pgp-encrypt.c b/contrib/pgcrypto/pgp-encrypt.c index 46518942ac2a..f7467c9b1cb1 100644 --- a/contrib/pgcrypto/pgp-encrypt.c +++ b/contrib/pgcrypto/pgp-encrypt.c @@ -178,8 +178,7 @@ encrypt_init(PushFilter *next, void *init_arg, void **priv_p) if (res < 0) return res; - st = px_alloc(sizeof(*st)); - memset(st, 0, sizeof(*st)); + st = palloc0(sizeof(*st)); st->ciph = ciph; *priv_p = st; @@ -219,7 +218,7 @@ encrypt_free(void *priv) if (st->ciph) pgp_cfb_free(st->ciph); px_memset(st, 0, sizeof(*st)); - px_free(st); + pfree(st); } static const PushFilterOps encrypt_filter = { @@ -241,7 +240,7 @@ pkt_stream_init(PushFilter *next, void *init_arg, void **priv_p) { struct PktStreamStat *st; - st = px_alloc(sizeof(*st)); + st = palloc(sizeof(*st)); st->final_done = 0; st->pkt_block = 1 << STREAM_BLOCK_SHIFT; *priv_p = st; @@ -301,7 +300,7 @@ pkt_stream_free(void *priv) struct PktStreamStat *st = priv; px_memset(st, 0, sizeof(*st)); - px_free(st); + pfree(st); } static const PushFilterOps pkt_stream_filter = { diff --git a/contrib/pgcrypto/pgp-mpi-internal.c b/contrib/pgcrypto/pgp-mpi-internal.c index 0cea51418058..5b94e654521b 100644 --- a/contrib/pgcrypto/pgp-mpi-internal.c +++ b/contrib/pgcrypto/pgp-mpi-internal.c @@ -60,10 +60,10 @@ mp_px_rand(uint32 bits, mpz_t *res) int last_bits = bits & 7; uint8 *buf; - buf = px_alloc(bytes); + buf = palloc(bytes); if (!pg_strong_random(buf, bytes)) { - px_free(buf); + pfree(buf); return PXE_NO_RANDOM; } @@ -78,7 +78,7 @@ mp_px_rand(uint32 bits, mpz_t *res) mp_int_read_unsigned(res, buf, bytes); - px_free(buf); + pfree(buf); return 0; } diff --git a/contrib/pgcrypto/pgp-mpi.c b/contrib/pgcrypto/pgp-mpi.c index 36a6d361ab31..03be27973bec 100644 --- a/contrib/pgcrypto/pgp-mpi.c +++ b/contrib/pgcrypto/pgp-mpi.c @@ -44,7 +44,7 @@ pgp_mpi_alloc(int bits, PGP_MPI **mpi) px_debug("pgp_mpi_alloc: unreasonable request: bits=%d", bits); return PXE_PGP_CORRUPT_DATA; } - n = px_alloc(sizeof(*n) + len); + n = palloc(sizeof(*n) + len); n->bits = bits; n->bytes = len; n->data = (uint8 *) (n) + sizeof(*n); @@ -72,7 +72,7 @@ pgp_mpi_free(PGP_MPI *mpi) if (mpi == NULL) return 0; px_memset(mpi, 0, sizeof(*mpi) + mpi->bytes); - px_free(mpi); + pfree(mpi); return 0; } diff --git a/contrib/pgcrypto/pgp-pubenc.c b/contrib/pgcrypto/pgp-pubenc.c index 9fdcf7c31c77..c254a3727506 100644 --- a/contrib/pgcrypto/pgp-pubenc.c +++ b/contrib/pgcrypto/pgp-pubenc.c @@ -46,12 +46,12 @@ pad_eme_pkcs1_v15(uint8 *data, int data_len, int res_len, uint8 **res_p) if (pad_len < 8) return PXE_BUG; - buf = px_alloc(res_len); + buf = palloc(res_len); buf[0] = 0x02; if (!pg_strong_random(buf + 1, pad_len)) { - px_free(buf); + pfree(buf); return PXE_NO_RANDOM; } @@ -64,7 +64,7 @@ pad_eme_pkcs1_v15(uint8 *data, int data_len, int res_len, uint8 **res_p) if (!pg_strong_random(p, 1)) { px_memset(buf, 0, res_len); - px_free(buf); + pfree(buf); return PXE_NO_RANDOM; } } @@ -97,7 +97,7 @@ create_secmsg(PGP_Context *ctx, PGP_MPI **msg_p, int full_bytes) /* * create "secret message" */ - secmsg = px_alloc(klen + 3); + secmsg = palloc(klen + 3); secmsg[0] = ctx->cipher_algo; memcpy(secmsg + 1, ctx->sess_key, klen); secmsg[klen + 1] = (cksum >> 8) & 0xFF; @@ -118,10 +118,10 @@ create_secmsg(PGP_Context *ctx, PGP_MPI **msg_p, int full_bytes) if (padded) { px_memset(padded, 0, full_bytes); - px_free(padded); + pfree(padded); } px_memset(secmsg, 0, klen + 3); - px_free(secmsg); + pfree(secmsg); if (res >= 0) *msg_p = m; diff --git a/contrib/pgcrypto/pgp-pubkey.c b/contrib/pgcrypto/pgp-pubkey.c index d447e5fd4fed..9a6561caf9dd 100644 --- a/contrib/pgcrypto/pgp-pubkey.c +++ b/contrib/pgcrypto/pgp-pubkey.c @@ -39,8 +39,7 @@ pgp_key_alloc(PGP_PubKey **pk_p) { PGP_PubKey *pk; - pk = px_alloc(sizeof(*pk)); - memset(pk, 0, sizeof(*pk)); + pk = palloc0(sizeof(*pk)); *pk_p = pk; return 0; } @@ -78,7 +77,7 @@ pgp_key_free(PGP_PubKey *pk) break; } px_memset(pk, 0, sizeof(*pk)); - px_free(pk); + pfree(pk); } static int diff --git a/contrib/pgcrypto/pgp.c b/contrib/pgcrypto/pgp.c index 9b245fee61bb..3e9c2fef9bc6 100644 --- a/contrib/pgcrypto/pgp.c +++ b/contrib/pgcrypto/pgp.c @@ -200,8 +200,7 @@ pgp_init(PGP_Context **ctx_p) { PGP_Context *ctx; - ctx = px_alloc(sizeof *ctx); - memset(ctx, 0, sizeof *ctx); + ctx = palloc0(sizeof *ctx); ctx->cipher_algo = def_cipher_algo; ctx->s2k_cipher_algo = def_s2k_cipher_algo; @@ -226,7 +225,7 @@ pgp_free(PGP_Context *ctx) if (ctx->pub_key) pgp_key_free(ctx->pub_key); px_memset(ctx, 0, sizeof *ctx); - px_free(ctx); + pfree(ctx); return 0; } diff --git a/contrib/pgcrypto/px-hmac.c b/contrib/pgcrypto/px-hmac.c index 06e5148f1b42..99174d265517 100644 --- a/contrib/pgcrypto/px-hmac.c +++ b/contrib/pgcrypto/px-hmac.c @@ -57,8 +57,7 @@ hmac_init(PX_HMAC *h, const uint8 *key, unsigned klen) PX_MD *md = h->md; bs = px_md_block_size(md); - keybuf = px_alloc(bs); - memset(keybuf, 0, bs); + keybuf = palloc0(bs); if (klen > bs) { @@ -76,7 +75,7 @@ hmac_init(PX_HMAC *h, const uint8 *key, unsigned klen) } px_memset(keybuf, 0, bs); - px_free(keybuf); + pfree(keybuf); px_md_update(md, h->p.ipad, bs); } @@ -108,7 +107,7 @@ hmac_finish(PX_HMAC *h, uint8 *dst) bs = px_md_block_size(md); hlen = px_md_result_size(md); - buf = px_alloc(hlen); + buf = palloc(hlen); px_md_finish(md, buf); @@ -118,7 +117,7 @@ hmac_finish(PX_HMAC *h, uint8 *dst) px_md_finish(md, dst); px_memset(buf, 0, hlen); - px_free(buf); + pfree(buf); } static void @@ -131,9 +130,9 @@ hmac_free(PX_HMAC *h) px_memset(h->p.ipad, 0, bs); px_memset(h->p.opad, 0, bs); - px_free(h->p.ipad); - px_free(h->p.opad); - px_free(h); + pfree(h->p.ipad); + pfree(h->p.opad); + pfree(h); } @@ -158,9 +157,9 @@ px_find_hmac(const char *name, PX_HMAC **res) return PXE_HASH_UNUSABLE_FOR_HMAC; } - h = px_alloc(sizeof(*h)); - h->p.ipad = px_alloc(bs); - h->p.opad = px_alloc(bs); + h = palloc(sizeof(*h)); + h->p.ipad = palloc(bs); + h->p.opad = palloc(bs); h->md = md; h->result_size = hmac_result_size; diff --git a/contrib/pgcrypto/px.c b/contrib/pgcrypto/px.c index 0f02fb56c4fb..6a4681dae989 100644 --- a/contrib/pgcrypto/px.c +++ b/contrib/pgcrypto/px.c @@ -196,8 +196,7 @@ combo_init(PX_Combo *cx, const uint8 *key, unsigned klen, ivs = px_cipher_iv_size(c); if (ivs > 0) { - ivbuf = px_alloc(ivs); - memset(ivbuf, 0, ivs); + ivbuf = palloc0(ivs); if (ivlen > ivs) memcpy(ivbuf, iv, ivs); else @@ -206,15 +205,15 @@ combo_init(PX_Combo *cx, const uint8 *key, unsigned klen, if (klen > ks) klen = ks; - keybuf = px_alloc(ks); + keybuf = palloc0(ks); memset(keybuf, 0, ks); memcpy(keybuf, key, klen); err = px_cipher_init(c, keybuf, klen, ivbuf); if (ivbuf) - px_free(ivbuf); - px_free(keybuf); + pfree(ivbuf); + pfree(keybuf); return err; } @@ -238,7 +237,7 @@ combo_encrypt(PX_Combo *cx, const uint8 *data, unsigned dlen, /* encrypt */ if (bs > 1) { - bbuf = px_alloc(bs * 4); + bbuf = palloc(bs * 4); bpos = dlen % bs; *rlen = dlen - bpos; memcpy(bbuf, data + *rlen, bpos); @@ -283,7 +282,7 @@ combo_encrypt(PX_Combo *cx, const uint8 *data, unsigned dlen, } out: if (bbuf) - px_free(bbuf); + pfree(bbuf); return err; } @@ -351,7 +350,7 @@ combo_free(PX_Combo *cx) if (cx->cipher) px_cipher_free(cx->cipher); px_memset(cx, 0, sizeof(*cx)); - px_free(cx); + pfree(cx); } /* PARSER */ @@ -408,17 +407,14 @@ px_find_combo(const char *name, PX_Combo **res) PX_Combo *cx; - cx = px_alloc(sizeof(*cx)); - memset(cx, 0, sizeof(*cx)); - - buf = px_alloc(strlen(name) + 1); - strcpy(buf, name); + cx = palloc0(sizeof(*cx)); + buf = pstrdup(name); err = parse_cipher_name(buf, &s_cipher, &s_pad); if (err) { - px_free(buf); - px_free(cx); + pfree(buf); + pfree(cx); return err; } @@ -445,7 +441,7 @@ px_find_combo(const char *name, PX_Combo **res) cx->decrypt_len = combo_decrypt_len; cx->free = combo_free; - px_free(buf); + pfree(buf); *res = cx; @@ -454,7 +450,7 @@ px_find_combo(const char *name, PX_Combo **res) err1: if (cx->cipher) px_cipher_free(cx->cipher); - px_free(cx); - px_free(buf); + pfree(cx); + pfree(buf); return PXE_NO_CIPHER; } diff --git a/contrib/pgcrypto/px.h b/contrib/pgcrypto/px.h index 0d4722a04a0a..5487923edb3e 100644 --- a/contrib/pgcrypto/px.h +++ b/contrib/pgcrypto/px.h @@ -37,19 +37,6 @@ /* keep debug messages? */ #define PX_DEBUG -/* a way to disable palloc - * - useful if compiled into standalone - */ -#ifndef PX_OWN_ALLOC -#define px_alloc(s) palloc(s) -#define px_realloc(p, s) repalloc(p, s) -#define px_free(p) pfree(p) -#else -void *px_alloc(size_t s); -void *px_realloc(void *p, size_t s); -void px_free(void *p); -#endif - /* max salt returned */ #define PX_MAX_SALT_LEN 128 From dee663f7843902535a15ae366cede8b4089f1144 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Fri, 25 Sep 2020 18:49:43 +1200 Subject: [PATCH 203/589] Defer flushing of SLRU files. Previously, we called fsync() after writing out individual pg_xact, pg_multixact and pg_commit_ts pages due to cache pressure, leading to regular I/O stalls in user backends and recovery. Collapse requests for the same file into a single system call as part of the next checkpoint, as we already did for relation files, using the infrastructure developed by commit 3eb77eba. This can cause a significant improvement to recovery performance, especially when it's otherwise CPU-bound. Hoist ProcessSyncRequests() up into CheckPointGuts() to make it clearer that it applies to all the SLRU mini-buffer-pools as well as the main buffer pool. Rearrange things so that data collected in CheckpointStats includes SLRU activity. Also remove the Shutdown{CLOG,CommitTS,SUBTRANS,MultiXact}() functions, because they were redundant after the shutdown checkpoint that immediately precedes them. (I'm not sure if they were ever needed, but they aren't now.) Reviewed-by: Tom Lane (parts) Tested-by: Jakub Wartak Discussion: https://postgr.es/m/CA+hUKGLJ=84YT+NvhkEEDAuUtVHMfQ9i-N7k_o50JmQ6Rpj_OQ@mail.gmail.com --- src/backend/access/transam/clog.c | 40 +++---- src/backend/access/transam/commit_ts.c | 36 +++--- src/backend/access/transam/multixact.c | 57 +++++---- src/backend/access/transam/slru.c | 154 +++++++++++++++++-------- src/backend/access/transam/subtrans.c | 25 +--- src/backend/access/transam/xlog.c | 28 +++-- src/backend/commands/async.c | 5 +- src/backend/storage/buffer/bufmgr.c | 7 -- src/backend/storage/lmgr/predicate.c | 8 +- src/backend/storage/sync/sync.c | 28 ++++- src/include/access/clog.h | 3 + src/include/access/commit_ts.h | 3 + src/include/access/multixact.h | 4 + src/include/access/slru.h | 14 ++- src/include/storage/sync.h | 7 +- src/tools/pgindent/typedefs.list | 5 +- 16 files changed, 252 insertions(+), 172 deletions(-) diff --git a/src/backend/access/transam/clog.c b/src/backend/access/transam/clog.c index 9e352d265835..034349aa7b98 100644 --- a/src/backend/access/transam/clog.c +++ b/src/backend/access/transam/clog.c @@ -42,6 +42,7 @@ #include "pg_trace.h" #include "pgstat.h" #include "storage/proc.h" +#include "storage/sync.h" /* * Defines for CLOG page sizes. A page is the same BLCKSZ as is used @@ -691,7 +692,8 @@ CLOGShmemInit(void) { XactCtl->PagePrecedes = CLOGPagePrecedes; SimpleLruInit(XactCtl, "Xact", CLOGShmemBuffers(), CLOG_LSNS_PER_PAGE, - XactSLRULock, "pg_xact", LWTRANCHE_XACT_BUFFER); + XactSLRULock, "pg_xact", LWTRANCHE_XACT_BUFFER, + SYNC_HANDLER_CLOG); } /* @@ -808,34 +810,19 @@ TrimCLOG(void) LWLockRelease(XactSLRULock); } -/* - * This must be called ONCE during postmaster or standalone-backend shutdown - */ -void -ShutdownCLOG(void) -{ - /* Flush dirty CLOG pages to disk */ - TRACE_POSTGRESQL_CLOG_CHECKPOINT_START(false); - SimpleLruFlush(XactCtl, false); - - /* - * fsync pg_xact to ensure that any files flushed previously are durably - * on disk. - */ - fsync_fname("pg_xact", true); - - TRACE_POSTGRESQL_CLOG_CHECKPOINT_DONE(false); -} - /* * Perform a checkpoint --- either during shutdown, or on-the-fly */ void CheckPointCLOG(void) { - /* Flush dirty CLOG pages to disk */ + /* + * Write dirty CLOG pages to disk. This may result in sync requests + * queued for later handling by ProcessSyncRequests(), as part of the + * checkpoint. + */ TRACE_POSTGRESQL_CLOG_CHECKPOINT_START(true); - SimpleLruFlush(XactCtl, true); + SimpleLruWriteAll(XactCtl, true); TRACE_POSTGRESQL_CLOG_CHECKPOINT_DONE(true); } @@ -1026,3 +1013,12 @@ clog_redo(XLogReaderState *record) else elog(PANIC, "clog_redo: unknown op code %u", info); } + +/* + * Entrypoint for sync.c to sync clog files. + */ +int +clogsyncfiletag(const FileTag *ftag, char *path) +{ + return SlruSyncFileTag(XactCtl, ftag, path); +} diff --git a/src/backend/access/transam/commit_ts.c b/src/backend/access/transam/commit_ts.c index f6a7329ba3a5..cb8a96880187 100644 --- a/src/backend/access/transam/commit_ts.c +++ b/src/backend/access/transam/commit_ts.c @@ -555,7 +555,8 @@ CommitTsShmemInit(void) CommitTsCtl->PagePrecedes = CommitTsPagePrecedes; SimpleLruInit(CommitTsCtl, "CommitTs", CommitTsShmemBuffers(), 0, CommitTsSLRULock, "pg_commit_ts", - LWTRANCHE_COMMITTS_BUFFER); + LWTRANCHE_COMMITTS_BUFFER, + SYNC_HANDLER_COMMIT_TS); commitTsShared = ShmemInitStruct("CommitTs shared", sizeof(CommitTimestampShared), @@ -798,30 +799,18 @@ DeactivateCommitTs(void) LWLockRelease(CommitTsSLRULock); } -/* - * This must be called ONCE during postmaster or standalone-backend shutdown - */ -void -ShutdownCommitTs(void) -{ - /* Flush dirty CommitTs pages to disk */ - SimpleLruFlush(CommitTsCtl, false); - - /* - * fsync pg_commit_ts to ensure that any files flushed previously are - * durably on disk. - */ - fsync_fname("pg_commit_ts", true); -} - /* * Perform a checkpoint --- either during shutdown, or on-the-fly */ void CheckPointCommitTs(void) { - /* Flush dirty CommitTs pages to disk */ - SimpleLruFlush(CommitTsCtl, true); + /* + * Write dirty CommitTs pages to disk. This may result in sync requests + * queued for later handling by ProcessSyncRequests(), as part of the + * checkpoint. + */ + SimpleLruWriteAll(CommitTsCtl, true); } /* @@ -1077,3 +1066,12 @@ commit_ts_redo(XLogReaderState *record) else elog(PANIC, "commit_ts_redo: unknown op code %u", info); } + +/* + * Entrypoint for sync.c to sync commit_ts files. + */ +int +committssyncfiletag(const FileTag *ftag, char *path) +{ + return SlruSyncFileTag(CommitTsCtl, ftag, path); +} diff --git a/src/backend/access/transam/multixact.c b/src/backend/access/transam/multixact.c index b8bedca04a4d..a2ce617c8ce2 100644 --- a/src/backend/access/transam/multixact.c +++ b/src/backend/access/transam/multixact.c @@ -1831,11 +1831,13 @@ MultiXactShmemInit(void) SimpleLruInit(MultiXactOffsetCtl, "MultiXactOffset", NUM_MULTIXACTOFFSET_BUFFERS, 0, MultiXactOffsetSLRULock, "pg_multixact/offsets", - LWTRANCHE_MULTIXACTOFFSET_BUFFER); + LWTRANCHE_MULTIXACTOFFSET_BUFFER, + SYNC_HANDLER_MULTIXACT_OFFSET); SimpleLruInit(MultiXactMemberCtl, "MultiXactMember", NUM_MULTIXACTMEMBER_BUFFERS, 0, MultiXactMemberSLRULock, "pg_multixact/members", - LWTRANCHE_MULTIXACTMEMBER_BUFFER); + LWTRANCHE_MULTIXACTMEMBER_BUFFER, + SYNC_HANDLER_MULTIXACT_MEMBER); /* Initialize our shared state struct */ MultiXactState = ShmemInitStruct("Shared MultiXact State", @@ -2100,19 +2102,6 @@ TrimMultiXact(void) SetMultiXactIdLimit(oldestMXact, oldestMXactDB, true); } -/* - * This must be called ONCE during postmaster or standalone-backend shutdown - */ -void -ShutdownMultiXact(void) -{ - /* Flush dirty MultiXact pages to disk */ - TRACE_POSTGRESQL_MULTIXACT_CHECKPOINT_START(false); - SimpleLruFlush(MultiXactOffsetCtl, false); - SimpleLruFlush(MultiXactMemberCtl, false); - TRACE_POSTGRESQL_MULTIXACT_CHECKPOINT_DONE(false); -} - /* * Get the MultiXact data to save in a checkpoint record */ @@ -2143,9 +2132,13 @@ CheckPointMultiXact(void) { TRACE_POSTGRESQL_MULTIXACT_CHECKPOINT_START(true); - /* Flush dirty MultiXact pages to disk */ - SimpleLruFlush(MultiXactOffsetCtl, true); - SimpleLruFlush(MultiXactMemberCtl, true); + /* + * Write dirty MultiXact pages to disk. This may result in sync requests + * queued for later handling by ProcessSyncRequests(), as part of the + * checkpoint. + */ + SimpleLruWriteAll(MultiXactOffsetCtl, true); + SimpleLruWriteAll(MultiXactMemberCtl, true); TRACE_POSTGRESQL_MULTIXACT_CHECKPOINT_DONE(true); } @@ -2728,14 +2721,10 @@ find_multixact_start(MultiXactId multi, MultiXactOffset *result) entryno = MultiXactIdToOffsetEntry(multi); /* - * Flush out dirty data, so PhysicalPageExists can work correctly. - * SimpleLruFlush() is a pretty big hammer for that. Alternatively we - * could add an in-memory version of page exists, but find_multixact_start - * is called infrequently, and it doesn't seem bad to flush buffers to - * disk before truncation. + * Write out dirty data, so PhysicalPageExists can work correctly. */ - SimpleLruFlush(MultiXactOffsetCtl, true); - SimpleLruFlush(MultiXactMemberCtl, true); + SimpleLruWriteAll(MultiXactOffsetCtl, true); + SimpleLruWriteAll(MultiXactMemberCtl, true); if (!SimpleLruDoesPhysicalPageExist(MultiXactOffsetCtl, pageno)) return false; @@ -3386,3 +3375,21 @@ pg_get_multixact_members(PG_FUNCTION_ARGS) SRF_RETURN_DONE(funccxt); } + +/* + * Entrypoint for sync.c to sync offsets files. + */ +int +multixactoffsetssyncfiletag(const FileTag *ftag, char *path) +{ + return SlruSyncFileTag(MultiXactOffsetCtl, ftag, path); +} + +/* + * Entrypoint for sync.c to sync members files. + */ +int +multixactmemberssyncfiletag(const FileTag *ftag, char *path) +{ + return SlruSyncFileTag(MultiXactMemberCtl, ftag, path); +} diff --git a/src/backend/access/transam/slru.c b/src/backend/access/transam/slru.c index fe7d759a8c1c..16a78986971f 100644 --- a/src/backend/access/transam/slru.c +++ b/src/backend/access/transam/slru.c @@ -63,22 +63,33 @@ snprintf(path, MAXPGPATH, "%s/%04X", (ctl)->Dir, seg) /* - * During SimpleLruFlush(), we will usually not need to write/fsync more - * than one or two physical files, but we may need to write several pages - * per file. We can consolidate the I/O requests by leaving files open - * until control returns to SimpleLruFlush(). This data structure remembers - * which files are open. + * During SimpleLruWriteAll(), we will usually not need to write more than one + * or two physical files, but we may need to write several pages per file. We + * can consolidate the I/O requests by leaving files open until control returns + * to SimpleLruWriteAll(). This data structure remembers which files are open. */ -#define MAX_FLUSH_BUFFERS 16 +#define MAX_WRITEALL_BUFFERS 16 -typedef struct SlruFlushData +typedef struct SlruWriteAllData { int num_files; /* # files actually open */ - int fd[MAX_FLUSH_BUFFERS]; /* their FD's */ - int segno[MAX_FLUSH_BUFFERS]; /* their log seg#s */ -} SlruFlushData; + int fd[MAX_WRITEALL_BUFFERS]; /* their FD's */ + int segno[MAX_WRITEALL_BUFFERS]; /* their log seg#s */ +} SlruWriteAllData; -typedef struct SlruFlushData *SlruFlush; +typedef struct SlruWriteAllData *SlruWriteAll; + +/* + * Populate a file tag describing a segment file. We only use the segment + * number, since we can derive everything else we need by having separate + * sync handler functions for clog, multixact etc. + */ +#define INIT_SLRUFILETAG(a,xx_handler,xx_segno) \ +( \ + memset(&(a), 0, sizeof(FileTag)), \ + (a).handler = (xx_handler), \ + (a).segno = (xx_segno) \ +) /* * Macro to mark a buffer slot "most recently used". Note multiple evaluation @@ -125,10 +136,10 @@ static int slru_errno; static void SimpleLruZeroLSNs(SlruCtl ctl, int slotno); static void SimpleLruWaitIO(SlruCtl ctl, int slotno); -static void SlruInternalWritePage(SlruCtl ctl, int slotno, SlruFlush fdata); +static void SlruInternalWritePage(SlruCtl ctl, int slotno, SlruWriteAll fdata); static bool SlruPhysicalReadPage(SlruCtl ctl, int pageno, int slotno); static bool SlruPhysicalWritePage(SlruCtl ctl, int pageno, int slotno, - SlruFlush fdata); + SlruWriteAll fdata); static void SlruReportIOError(SlruCtl ctl, int pageno, TransactionId xid); static int SlruSelectLRUPage(SlruCtl ctl, int pageno); @@ -173,7 +184,8 @@ SimpleLruShmemSize(int nslots, int nlsns) */ void SimpleLruInit(SlruCtl ctl, const char *name, int nslots, int nlsns, - LWLock *ctllock, const char *subdir, int tranche_id) + LWLock *ctllock, const char *subdir, int tranche_id, + SyncRequestHandler sync_handler) { SlruShared shared; bool found; @@ -251,7 +263,7 @@ SimpleLruInit(SlruCtl ctl, const char *name, int nslots, int nlsns, * assume caller set PagePrecedes. */ ctl->shared = shared; - ctl->do_fsync = true; /* default behavior */ + ctl->sync_handler = sync_handler; strlcpy(ctl->Dir, subdir, sizeof(ctl->Dir)); } @@ -523,7 +535,7 @@ SimpleLruReadPage_ReadOnly(SlruCtl ctl, int pageno, TransactionId xid) * Control lock must be held at entry, and will be held at exit. */ static void -SlruInternalWritePage(SlruCtl ctl, int slotno, SlruFlush fdata) +SlruInternalWritePage(SlruCtl ctl, int slotno, SlruWriteAll fdata) { SlruShared shared = ctl->shared; int pageno = shared->page_number[slotno]; @@ -587,6 +599,10 @@ SlruInternalWritePage(SlruCtl ctl, int slotno, SlruFlush fdata) /* Now it's okay to ereport if we failed */ if (!ok) SlruReportIOError(ctl, pageno, InvalidTransactionId); + + /* If part of a checkpoint, count this as a buffer written. */ + if (fdata) + CheckpointStats.ckpt_bufs_written++; } /* @@ -730,13 +746,13 @@ SlruPhysicalReadPage(SlruCtl ctl, int pageno, int slotno) * * For now, assume it's not worth keeping a file pointer open across * independent read/write operations. We do batch operations during - * SimpleLruFlush, though. + * SimpleLruWriteAll, though. * * fdata is NULL for a standalone write, pointer to open-file info during - * SimpleLruFlush. + * SimpleLruWriteAll. */ static bool -SlruPhysicalWritePage(SlruCtl ctl, int pageno, int slotno, SlruFlush fdata) +SlruPhysicalWritePage(SlruCtl ctl, int pageno, int slotno, SlruWriteAll fdata) { SlruShared shared = ctl->shared; int segno = pageno / SLRU_PAGES_PER_SEGMENT; @@ -791,7 +807,7 @@ SlruPhysicalWritePage(SlruCtl ctl, int pageno, int slotno, SlruFlush fdata) } /* - * During a Flush, we may already have the desired file open. + * During a WriteAll, we may already have the desired file open. */ if (fdata) { @@ -837,7 +853,7 @@ SlruPhysicalWritePage(SlruCtl ctl, int pageno, int slotno, SlruFlush fdata) if (fdata) { - if (fdata->num_files < MAX_FLUSH_BUFFERS) + if (fdata->num_files < MAX_WRITEALL_BUFFERS) { fdata->fd[fdata->num_files] = fd; fdata->segno[fdata->num_files] = segno; @@ -870,23 +886,31 @@ SlruPhysicalWritePage(SlruCtl ctl, int pageno, int slotno, SlruFlush fdata) } pgstat_report_wait_end(); - /* - * If not part of Flush, need to fsync now. We assume this happens - * infrequently enough that it's not a performance issue. - */ - if (!fdata) + /* Queue up a sync request for the checkpointer. */ + if (ctl->sync_handler != SYNC_HANDLER_NONE) { - pgstat_report_wait_start(WAIT_EVENT_SLRU_SYNC); - if (ctl->do_fsync && pg_fsync(fd) != 0) + FileTag tag; + + INIT_SLRUFILETAG(tag, ctl->sync_handler, segno); + if (!RegisterSyncRequest(&tag, SYNC_REQUEST, false)) { + /* No space to enqueue sync request. Do it synchronously. */ + pgstat_report_wait_start(WAIT_EVENT_SLRU_SYNC); + if (pg_fsync(fd) != 0) + { + pgstat_report_wait_end(); + slru_errcause = SLRU_FSYNC_FAILED; + slru_errno = errno; + CloseTransientFile(fd); + return false; + } pgstat_report_wait_end(); - slru_errcause = SLRU_FSYNC_FAILED; - slru_errno = errno; - CloseTransientFile(fd); - return false; } - pgstat_report_wait_end(); + } + /* Close file, unless part of flush request. */ + if (!fdata) + { if (CloseTransientFile(fd) != 0) { slru_errcause = SLRU_CLOSE_FAILED; @@ -1122,13 +1146,16 @@ SlruSelectLRUPage(SlruCtl ctl, int pageno) } /* - * Flush dirty pages to disk during checkpoint or database shutdown + * Write dirty pages to disk during checkpoint or database shutdown. Flushing + * is deferred until the next call to ProcessSyncRequests(), though we do fsync + * the containing directory here to make sure that newly created directory + * entries are on disk. */ void -SimpleLruFlush(SlruCtl ctl, bool allow_redirtied) +SimpleLruWriteAll(SlruCtl ctl, bool allow_redirtied) { SlruShared shared = ctl->shared; - SlruFlushData fdata; + SlruWriteAllData fdata; int slotno; int pageno = 0; int i; @@ -1162,21 +1189,11 @@ SimpleLruFlush(SlruCtl ctl, bool allow_redirtied) LWLockRelease(shared->ControlLock); /* - * Now fsync and close any files that were open + * Now close any files that were open */ ok = true; for (i = 0; i < fdata.num_files; i++) { - pgstat_report_wait_start(WAIT_EVENT_SLRU_FLUSH_SYNC); - if (ctl->do_fsync && pg_fsync(fdata.fd[i]) != 0) - { - slru_errcause = SLRU_FSYNC_FAILED; - slru_errno = errno; - pageno = fdata.segno[i] * SLRU_PAGES_PER_SEGMENT; - ok = false; - } - pgstat_report_wait_end(); - if (CloseTransientFile(fdata.fd[i]) != 0) { slru_errcause = SLRU_CLOSE_FAILED; @@ -1189,7 +1206,7 @@ SimpleLruFlush(SlruCtl ctl, bool allow_redirtied) SlruReportIOError(ctl, pageno, InvalidTransactionId); /* Ensure that directory entries for new files are on disk. */ - if (ctl->do_fsync) + if (ctl->sync_handler != SYNC_HANDLER_NONE) fsync_fname(ctl->Dir, true); } @@ -1350,6 +1367,19 @@ SlruDeleteSegment(SlruCtl ctl, int segno) snprintf(path, MAXPGPATH, "%s/%04X", ctl->Dir, segno); ereport(DEBUG2, (errmsg("removing file \"%s\"", path))); + + /* + * Tell the checkpointer to forget any sync requests, before we unlink the + * file. + */ + if (ctl->sync_handler != SYNC_HANDLER_NONE) + { + FileTag tag; + + INIT_SLRUFILETAG(tag, ctl->sync_handler, segno); + RegisterSyncRequest(&tag, SYNC_FORGET_REQUEST, true); + } + unlink(path); LWLockRelease(shared->ControlLock); @@ -1448,3 +1478,31 @@ SlruScanDirectory(SlruCtl ctl, SlruScanCallback callback, void *data) return retval; } + +/* + * Individual SLRUs (clog, ...) have to provide a sync.c handler function so + * that they can provide the correct "SlruCtl" (otherwise we don't know how to + * build the path), but they just forward to this common implementation that + * performs the fsync. + */ +int +SlruSyncFileTag(SlruCtl ctl, const FileTag *ftag, char *path) +{ + int fd; + int save_errno; + int result; + + SlruFileName(ctl, path, ftag->segno); + + fd = OpenTransientFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + return -1; + + result = pg_fsync(fd); + save_errno = errno; + + CloseTransientFile(fd); + + errno = save_errno; + return result; +} diff --git a/src/backend/access/transam/subtrans.c b/src/backend/access/transam/subtrans.c index a50f60b99af2..0111e867c79a 100644 --- a/src/backend/access/transam/subtrans.c +++ b/src/backend/access/transam/subtrans.c @@ -193,9 +193,7 @@ SUBTRANSShmemInit(void) SubTransCtl->PagePrecedes = SubTransPagePrecedes; SimpleLruInit(SubTransCtl, "Subtrans", NUM_SUBTRANS_BUFFERS, 0, SubtransSLRULock, "pg_subtrans", - LWTRANCHE_SUBTRANS_BUFFER); - /* Override default assumption that writes should be fsync'd */ - SubTransCtl->do_fsync = false; + LWTRANCHE_SUBTRANS_BUFFER, SYNC_HANDLER_NONE); } /* @@ -278,23 +276,6 @@ StartupSUBTRANS(TransactionId oldestActiveXID) LWLockRelease(SubtransSLRULock); } -/* - * This must be called ONCE during postmaster or standalone-backend shutdown - */ -void -ShutdownSUBTRANS(void) -{ - /* - * Flush dirty SUBTRANS pages to disk - * - * This is not actually necessary from a correctness point of view. We do - * it merely as a debugging aid. - */ - TRACE_POSTGRESQL_SUBTRANS_CHECKPOINT_START(false); - SimpleLruFlush(SubTransCtl, false); - TRACE_POSTGRESQL_SUBTRANS_CHECKPOINT_DONE(false); -} - /* * Perform a checkpoint --- either during shutdown, or on-the-fly */ @@ -302,14 +283,14 @@ void CheckPointSUBTRANS(void) { /* - * Flush dirty SUBTRANS pages to disk + * Write dirty SUBTRANS pages to disk * * This is not actually necessary from a correctness point of view. We do * it merely to improve the odds that writing of dirty pages is done by * the checkpoint process and not by backends. */ TRACE_POSTGRESQL_SUBTRANS_CHECKPOINT_START(true); - SimpleLruFlush(SubTransCtl, true); + SimpleLruWriteAll(SubTransCtl, true); TRACE_POSTGRESQL_SUBTRANS_CHECKPOINT_DONE(true); } diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 61754312e269..79a77ebbfe24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -8528,10 +8528,6 @@ ShutdownXLOG(int code, Datum arg) CreateCheckPoint(CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_IMMEDIATE); } - ShutdownCLOG(); - ShutdownCommitTs(); - ShutdownSUBTRANS(); - ShutdownMultiXact(); } /* @@ -9176,17 +9172,29 @@ CreateEndOfRecoveryRecord(void) static void CheckPointGuts(XLogRecPtr checkPointRedo, int flags) { - CheckPointCLOG(); - CheckPointCommitTs(); - CheckPointSUBTRANS(); - CheckPointMultiXact(); - CheckPointPredicate(); CheckPointRelationMap(); CheckPointReplicationSlots(); CheckPointSnapBuild(); CheckPointLogicalRewriteHeap(); - CheckPointBuffers(flags); /* performs all required fsyncs */ CheckPointReplicationOrigin(); + + /* Write out all dirty data in SLRUs and the main buffer pool */ + TRACE_POSTGRESQL_BUFFER_CHECKPOINT_START(flags); + CheckpointStats.ckpt_write_t = GetCurrentTimestamp(); + CheckPointCLOG(); + CheckPointCommitTs(); + CheckPointSUBTRANS(); + CheckPointMultiXact(); + CheckPointPredicate(); + CheckPointBuffers(flags); + + /* Perform all queued up fsyncs */ + TRACE_POSTGRESQL_BUFFER_CHECKPOINT_SYNC_START(); + CheckpointStats.ckpt_sync_t = GetCurrentTimestamp(); + ProcessSyncRequests(); + CheckpointStats.ckpt_sync_end_t = GetCurrentTimestamp(); + TRACE_POSTGRESQL_BUFFER_CHECKPOINT_DONE(); + /* We deliberately delay 2PC checkpointing as long as possible */ CheckPointTwoPhase(checkPointRedo); } diff --git a/src/backend/commands/async.c b/src/backend/commands/async.c index cb341365df46..8dbcace3f931 100644 --- a/src/backend/commands/async.c +++ b/src/backend/commands/async.c @@ -554,9 +554,8 @@ AsyncShmemInit(void) */ NotifyCtl->PagePrecedes = asyncQueuePagePrecedes; SimpleLruInit(NotifyCtl, "Notify", NUM_NOTIFY_BUFFERS, 0, - NotifySLRULock, "pg_notify", LWTRANCHE_NOTIFY_BUFFER); - /* Override default assumption that writes should be fsync'd */ - NotifyCtl->do_fsync = false; + NotifySLRULock, "pg_notify", LWTRANCHE_NOTIFY_BUFFER, + SYNC_HANDLER_NONE); if (!found) { diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index a2a963bd5b41..e549fa1d309f 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -2636,14 +2636,7 @@ PrintBufferLeakWarning(Buffer buffer) void CheckPointBuffers(int flags) { - TRACE_POSTGRESQL_BUFFER_CHECKPOINT_START(flags); - CheckpointStats.ckpt_write_t = GetCurrentTimestamp(); BufferSync(flags); - CheckpointStats.ckpt_sync_t = GetCurrentTimestamp(); - TRACE_POSTGRESQL_BUFFER_CHECKPOINT_SYNC_START(); - ProcessSyncRequests(); - CheckpointStats.ckpt_sync_end_t = GetCurrentTimestamp(); - TRACE_POSTGRESQL_BUFFER_CHECKPOINT_DONE(); } diff --git a/src/backend/storage/lmgr/predicate.c b/src/backend/storage/lmgr/predicate.c index a2f8e7524b49..8a365b400c6b 100644 --- a/src/backend/storage/lmgr/predicate.c +++ b/src/backend/storage/lmgr/predicate.c @@ -821,9 +821,7 @@ SerialInit(void) SerialSlruCtl->PagePrecedes = SerialPagePrecedesLogically; SimpleLruInit(SerialSlruCtl, "Serial", NUM_SERIAL_BUFFERS, 0, SerialSLRULock, "pg_serial", - LWTRANCHE_SERIAL_BUFFER); - /* Override default assumption that writes should be fsync'd */ - SerialSlruCtl->do_fsync = false; + LWTRANCHE_SERIAL_BUFFER, SYNC_HANDLER_NONE); /* * Create or attach to the SerialControl structure. @@ -1052,7 +1050,7 @@ CheckPointPredicate(void) SimpleLruTruncate(SerialSlruCtl, tailPage); /* - * Flush dirty SLRU pages to disk + * Write dirty SLRU pages to disk * * This is not actually necessary from a correctness point of view. We do * it merely as a debugging aid. @@ -1061,7 +1059,7 @@ CheckPointPredicate(void) * before deleting the file in which they sit, which would be completely * pointless. */ - SimpleLruFlush(SerialSlruCtl, true); + SimpleLruWriteAll(SerialSlruCtl, true); } /*------------------------------------------------------------------------*/ diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c index 3ded2cdd716b..1d635d596cad 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c @@ -18,6 +18,9 @@ #include #include +#include "access/commit_ts.h" +#include "access/clog.h" +#include "access/multixact.h" #include "access/xlog.h" #include "access/xlogutils.h" #include "commands/tablespace.h" @@ -90,12 +93,31 @@ typedef struct SyncOps const FileTag *candidate); } SyncOps; +/* + * These indexes must correspond to the values of the SyncRequestHandler enum. + */ static const SyncOps syncsw[] = { /* magnetic disk */ - { + [SYNC_HANDLER_MD] = { .sync_syncfiletag = mdsyncfiletag, .sync_unlinkfiletag = mdunlinkfiletag, .sync_filetagmatches = mdfiletagmatches + }, + /* pg_xact */ + [SYNC_HANDLER_CLOG] = { + .sync_syncfiletag = clogsyncfiletag + }, + /* pg_commit_ts */ + [SYNC_HANDLER_COMMIT_TS] = { + .sync_syncfiletag = committssyncfiletag + }, + /* pg_multixact/offsets */ + [SYNC_HANDLER_MULTIXACT_OFFSET] = { + .sync_syncfiletag = multixactoffsetssyncfiletag + }, + /* pg_multixact/members */ + [SYNC_HANDLER_MULTIXACT_MEMBER] = { + .sync_syncfiletag = multixactmemberssyncfiletag } }; @@ -505,8 +527,8 @@ RememberSyncRequest(const FileTag *ftag, SyncRequestType type) (void *) ftag, HASH_ENTER, &found); - /* if new entry, initialize it */ - if (!found) + /* if new entry, or was previously canceled, initialize it */ + if (!found || entry->canceled) { entry->cycle_ctr = sync_cycle_ctr; entry->canceled = false; diff --git a/src/include/access/clog.h b/src/include/access/clog.h index 2db8acb189f5..6c840cbf299b 100644 --- a/src/include/access/clog.h +++ b/src/include/access/clog.h @@ -12,6 +12,7 @@ #define CLOG_H #include "access/xlogreader.h" +#include "storage/sync.h" #include "lib/stringinfo.h" /* @@ -50,6 +51,8 @@ extern void CheckPointCLOG(void); extern void ExtendCLOG(TransactionId newestXact); extern void TruncateCLOG(TransactionId oldestXact, Oid oldestxid_datoid); +extern int clogsyncfiletag(const FileTag *ftag, char *path); + /* XLOG stuff */ #define CLOG_ZEROPAGE 0x00 #define CLOG_TRUNCATE 0x10 diff --git a/src/include/access/commit_ts.h b/src/include/access/commit_ts.h index 2740c02a84fa..2d1724952257 100644 --- a/src/include/access/commit_ts.h +++ b/src/include/access/commit_ts.h @@ -14,6 +14,7 @@ #include "access/xlog.h" #include "datatype/timestamp.h" #include "replication/origin.h" +#include "storage/sync.h" #include "utils/guc.h" @@ -45,6 +46,8 @@ extern void SetCommitTsLimit(TransactionId oldestXact, TransactionId newestXact); extern void AdvanceOldestCommitTsXid(TransactionId oldestXact); +extern int committssyncfiletag(const FileTag *ftag, char *path); + /* XLOG stuff */ #define COMMIT_TS_ZEROPAGE 0x00 #define COMMIT_TS_TRUNCATE 0x10 diff --git a/src/include/access/multixact.h b/src/include/access/multixact.h index 6d729008c600..58c42ffe1fe6 100644 --- a/src/include/access/multixact.h +++ b/src/include/access/multixact.h @@ -13,6 +13,7 @@ #include "access/xlogreader.h" #include "lib/stringinfo.h" +#include "storage/sync.h" /* @@ -116,6 +117,9 @@ extern bool MultiXactIdPrecedes(MultiXactId multi1, MultiXactId multi2); extern bool MultiXactIdPrecedesOrEquals(MultiXactId multi1, MultiXactId multi2); +extern int multixactoffsetssyncfiletag(const FileTag *ftag, char *path); +extern int multixactmemberssyncfiletag(const FileTag *ftag, char *path); + extern void AtEOXact_MultiXact(void); extern void AtPrepare_MultiXact(void); extern void PostPrepare_MultiXact(TransactionId xid); diff --git a/src/include/access/slru.h b/src/include/access/slru.h index 61fbc80ef0d6..b39b43504d80 100644 --- a/src/include/access/slru.h +++ b/src/include/access/slru.h @@ -15,6 +15,7 @@ #include "access/xlogdefs.h" #include "storage/lwlock.h" +#include "storage/sync.h" /* @@ -111,10 +112,10 @@ typedef struct SlruCtlData SlruShared shared; /* - * This flag tells whether to fsync writes (true for pg_xact and multixact - * stuff, false for pg_subtrans and pg_notify). + * Which sync handler function to use when handing sync requests over to + * the checkpointer. SYNC_HANDLER_NONE to disable fsync (eg pg_notify). */ - bool do_fsync; + SyncRequestHandler sync_handler; /* * Decide which of two page numbers is "older" for truncation purposes. We @@ -135,14 +136,15 @@ typedef SlruCtlData *SlruCtl; extern Size SimpleLruShmemSize(int nslots, int nlsns); extern void SimpleLruInit(SlruCtl ctl, const char *name, int nslots, int nlsns, - LWLock *ctllock, const char *subdir, int tranche_id); + LWLock *ctllock, const char *subdir, int tranche_id, + SyncRequestHandler sync_handler); extern int SimpleLruZeroPage(SlruCtl ctl, int pageno); extern int SimpleLruReadPage(SlruCtl ctl, int pageno, bool write_ok, TransactionId xid); extern int SimpleLruReadPage_ReadOnly(SlruCtl ctl, int pageno, TransactionId xid); extern void SimpleLruWritePage(SlruCtl ctl, int slotno); -extern void SimpleLruFlush(SlruCtl ctl, bool allow_redirtied); +extern void SimpleLruWriteAll(SlruCtl ctl, bool allow_redirtied); extern void SimpleLruTruncate(SlruCtl ctl, int cutoffPage); extern bool SimpleLruDoesPhysicalPageExist(SlruCtl ctl, int pageno); @@ -151,6 +153,8 @@ typedef bool (*SlruScanCallback) (SlruCtl ctl, char *filename, int segpage, extern bool SlruScanDirectory(SlruCtl ctl, SlruScanCallback callback, void *data); extern void SlruDeleteSegment(SlruCtl ctl, int segno); +extern int SlruSyncFileTag(SlruCtl ctl, const FileTag *ftag, char *path); + /* SlruScanDirectory public callbacks */ extern bool SlruScanDirCbReportPresence(SlruCtl ctl, char *filename, int segpage, void *data); diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h index e16ab8e711c4..f32e412e751d 100644 --- a/src/include/storage/sync.h +++ b/src/include/storage/sync.h @@ -34,7 +34,12 @@ typedef enum SyncRequestType */ typedef enum SyncRequestHandler { - SYNC_HANDLER_MD = 0 /* md smgr */ + SYNC_HANDLER_MD = 0, + SYNC_HANDLER_CLOG, + SYNC_HANDLER_COMMIT_TS, + SYNC_HANDLER_MULTIXACT_OFFSET, + SYNC_HANDLER_MULTIXACT_MEMBER, + SYNC_HANDLER_NONE } SyncRequestHandler; /* diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index b1afb345c368..9cd1179af63c 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2295,12 +2295,12 @@ SlotNumber SlruCtl SlruCtlData SlruErrorCause -SlruFlush -SlruFlushData SlruPageStatus SlruScanCallback SlruShared SlruSharedData +SlruWriteAll +SlruWriteAllData SnapBuild SnapBuildOnDisk SnapBuildState @@ -2404,6 +2404,7 @@ Syn SyncOps SyncRepConfigData SyncRepStandbyData +SyncRequestHandler SyncRequestType SysScanDesc SyscacheCallbackFunction From 079d0cacf4fefc326bf776ecc2df5dba995f50de Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Sat, 26 Sep 2020 10:08:00 +0530 Subject: [PATCH 204/589] Fix the logical replication from HEAD to lower versions. Commit 464824323e changed the logical replication protocol to allow the streaming of in-progress transactions and used the new version of protocol irrespective of the server version. Use the appropriate version of the protocol based on the server version. Reported-by: Ashutosh Sharma Author: Dilip Kumar Reviewed-by: Ashutosh Sharma and Amit Kapila Discussion: https://postgr.es/m/CAE9k0P=9OpXcNrcU5Gsvd5MZ8GFpiN833vNHzX6Uc=8+h1ft1Q@mail.gmail.com --- src/backend/replication/logical/worker.c | 4 +++- src/backend/replication/pgoutput/pgoutput.c | 4 ++-- src/include/replication/logicalproto.h | 8 +++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index d239d28c094b..9c6fdeeb56c4 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -3087,7 +3087,9 @@ ApplyWorkerMain(Datum main_arg) options.logical = true; options.startpoint = origin_startpos; options.slotname = myslotname; - options.proto.logical.proto_version = LOGICALREP_PROTO_VERSION_NUM; + options.proto.logical.proto_version = + walrcv_server_version(wrconn) >= 140000 ? + LOGICALREP_PROTO_STREAM_VERSION_NUM : LOGICALREP_PROTO_VERSION_NUM; options.proto.logical.publication_names = MySubscription->publications; options.proto.logical.binary = MySubscription->binary; options.proto.logical.streaming = MySubscription->stream; diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c index eb1f23004e77..9c997aed8367 100644 --- a/src/backend/replication/pgoutput/pgoutput.c +++ b/src/backend/replication/pgoutput/pgoutput.c @@ -272,11 +272,11 @@ pgoutput_startup(LogicalDecodingContext *ctx, OutputPluginOptions *opt, &enable_streaming); /* Check if we support requested protocol */ - if (data->protocol_version > LOGICALREP_PROTO_VERSION_NUM) + if (data->protocol_version > LOGICALREP_PROTO_MAX_VERSION_NUM) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("client sent proto_version=%d but we only support protocol %d or lower", - data->protocol_version, LOGICALREP_PROTO_VERSION_NUM))); + data->protocol_version, LOGICALREP_PROTO_MAX_VERSION_NUM))); if (data->protocol_version < LOGICALREP_PROTO_MIN_VERSION_NUM) ereport(ERROR, diff --git a/src/include/replication/logicalproto.h b/src/include/replication/logicalproto.h index 607a728508bb..0c2cda264e14 100644 --- a/src/include/replication/logicalproto.h +++ b/src/include/replication/logicalproto.h @@ -19,8 +19,9 @@ /* * Protocol capabilities * - * LOGICALREP_PROTO_VERSION_NUM is our native protocol and the greatest version - * we can support. LOGICALREP_PROTO_MIN_VERSION_NUM is the oldest version we + * LOGICALREP_PROTO_VERSION_NUM is our native protocol. + * LOGICALREP_PROTO_MAX_VERSION_NUM is the greatest version we can support. + * LOGICALREP_PROTO_MIN_VERSION_NUM is the oldest version we * have backwards compatibility for. The client requests protocol version at * connect time. * @@ -28,8 +29,9 @@ * support for streaming large transactions. */ #define LOGICALREP_PROTO_MIN_VERSION_NUM 1 +#define LOGICALREP_PROTO_VERSION_NUM 1 #define LOGICALREP_PROTO_STREAM_VERSION_NUM 2 -#define LOGICALREP_PROTO_VERSION_NUM 2 +#define LOGICALREP_PROTO_MAX_VERSION_NUM LOGICALREP_PROTO_STREAM_VERSION_NUM /* * This struct stores a tuple received via logical replication. From e55f718fc429fb7971eb7351debfa5931f9811bf Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 26 Sep 2020 16:04:06 -0400 Subject: [PATCH 205/589] Revise RelationBuildRowSecurity() to avoid memory leaks. This function leaked some memory while loading qual clauses for an RLS policy. While ordinarily negligible, that could build up in some repeated-reload cases, as reported by Konstantin Knizhnik. We can improve matters by borrowing the coding long used in RelationBuildRuleLock: build stringToNode's result directly in the target context, and remember to explicitly pfree the input string. This patch by no means completely guarantees zero leaks within this function, since we have no real guarantee that the catalog- reading subroutines it calls don't leak anything. However, practical tests suggest that this is enough to resolve the issue. In any case, any remaining leaks are similar to those risked by RelationBuildRuleLock and other relcache-loading subroutines. If we need to fix them, we should adopt a more global approach such as that used by the RECOVER_RELATION_BUILD_MEMORY hack. While here, let's remove the need for an expensive PG_TRY block by using MemoryContextSetParent to reparent an initially-short-lived context for the RLS data. Back-patch to all supported branches. Discussion: https://postgr.es/m/21356c12-8917-8249-b35f-1c447231922b@postgrespro.ru --- src/backend/commands/policy.c | 216 +++++++++++++++------------------- 1 file changed, 98 insertions(+), 118 deletions(-) diff --git a/src/backend/commands/policy.c b/src/backend/commands/policy.c index 4b4e4694930e..d3f8e8f06c13 100644 --- a/src/backend/commands/policy.c +++ b/src/backend/commands/policy.c @@ -187,159 +187,139 @@ policy_role_list_to_array(List *roles, int *num_roles) /* * Load row security policy from the catalog, and store it in * the relation's relcache entry. + * + * Note that caller should have verified that pg_class.relrowsecurity + * is true for this relation. */ void RelationBuildRowSecurity(Relation relation) { MemoryContext rscxt; MemoryContext oldcxt = CurrentMemoryContext; - RowSecurityDesc *volatile rsdesc = NULL; + RowSecurityDesc *rsdesc; + Relation catalog; + ScanKeyData skey; + SysScanDesc sscan; + HeapTuple tuple; /* * Create a memory context to hold everything associated with this * relation's row security policy. This makes it easy to clean up during - * a relcache flush. + * a relcache flush. However, to cover the possibility of an error + * partway through, we don't make the context long-lived till we're done. */ - rscxt = AllocSetContextCreate(CacheMemoryContext, + rscxt = AllocSetContextCreate(CurrentMemoryContext, "row security descriptor", ALLOCSET_SMALL_SIZES); + MemoryContextCopyAndSetIdentifier(rscxt, + RelationGetRelationName(relation)); + + rsdesc = MemoryContextAllocZero(rscxt, sizeof(RowSecurityDesc)); + rsdesc->rscxt = rscxt; /* - * Since rscxt lives under CacheMemoryContext, it is long-lived. Use a - * PG_TRY block to ensure it'll get freed if we fail partway through. + * Now scan pg_policy for RLS policies associated with this relation. + * Because we use the index on (polrelid, polname), we should consistently + * visit the rel's policies in name order, at least when system indexes + * aren't disabled. This simplifies equalRSDesc(). */ - PG_TRY(); - { - Relation catalog; - ScanKeyData skey; - SysScanDesc sscan; - HeapTuple tuple; - - MemoryContextCopyAndSetIdentifier(rscxt, - RelationGetRelationName(relation)); + catalog = table_open(PolicyRelationId, AccessShareLock); - rsdesc = MemoryContextAllocZero(rscxt, sizeof(RowSecurityDesc)); - rsdesc->rscxt = rscxt; + ScanKeyInit(&skey, + Anum_pg_policy_polrelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(RelationGetRelid(relation))); - catalog = table_open(PolicyRelationId, AccessShareLock); + sscan = systable_beginscan(catalog, PolicyPolrelidPolnameIndexId, true, + NULL, 1, &skey); - ScanKeyInit(&skey, - Anum_pg_policy_polrelid, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(RelationGetRelid(relation))); + while (HeapTupleIsValid(tuple = systable_getnext(sscan))) + { + Form_pg_policy policy_form = (Form_pg_policy) GETSTRUCT(tuple); + RowSecurityPolicy *policy; + Datum datum; + bool isnull; + char *str_value; - sscan = systable_beginscan(catalog, PolicyPolrelidPolnameIndexId, true, - NULL, 1, &skey); + policy = MemoryContextAllocZero(rscxt, sizeof(RowSecurityPolicy)); /* - * Loop through the row level security policies for this relation, if - * any. + * Note: we must be sure that pass-by-reference data gets copied into + * rscxt. We avoid making that context current over wider spans than + * we have to, though. */ - while (HeapTupleIsValid(tuple = systable_getnext(sscan))) - { - Datum value_datum; - char cmd_value; - bool permissive_value; - Datum roles_datum; - char *qual_value; - Expr *qual_expr; - char *with_check_value; - Expr *with_check_qual; - char *policy_name_value; - bool isnull; - RowSecurityPolicy *policy; - - /* - * Note: all the pass-by-reference data we collect here is either - * still stored in the tuple, or constructed in the caller's - * short-lived memory context. We must copy it into rscxt - * explicitly below. - */ - - /* Get policy command */ - value_datum = heap_getattr(tuple, Anum_pg_policy_polcmd, - RelationGetDescr(catalog), &isnull); - Assert(!isnull); - cmd_value = DatumGetChar(value_datum); - - /* Get policy permissive or restrictive */ - value_datum = heap_getattr(tuple, Anum_pg_policy_polpermissive, - RelationGetDescr(catalog), &isnull); - Assert(!isnull); - permissive_value = DatumGetBool(value_datum); - - /* Get policy name */ - value_datum = heap_getattr(tuple, Anum_pg_policy_polname, - RelationGetDescr(catalog), &isnull); - Assert(!isnull); - policy_name_value = NameStr(*(DatumGetName(value_datum))); - - /* Get policy roles */ - roles_datum = heap_getattr(tuple, Anum_pg_policy_polroles, - RelationGetDescr(catalog), &isnull); - /* shouldn't be null, but initdb doesn't mark it so, so check */ - if (isnull) - elog(ERROR, "unexpected null value in pg_policy.polroles"); - - /* Get policy qual */ - value_datum = heap_getattr(tuple, Anum_pg_policy_polqual, - RelationGetDescr(catalog), &isnull); - if (!isnull) - { - qual_value = TextDatumGetCString(value_datum); - qual_expr = (Expr *) stringToNode(qual_value); - } - else - qual_expr = NULL; - /* Get WITH CHECK qual */ - value_datum = heap_getattr(tuple, Anum_pg_policy_polwithcheck, - RelationGetDescr(catalog), &isnull); - if (!isnull) - { - with_check_value = TextDatumGetCString(value_datum); - with_check_qual = (Expr *) stringToNode(with_check_value); - } - else - with_check_qual = NULL; + /* Get policy command */ + policy->polcmd = policy_form->polcmd; - /* Now copy everything into the cache context */ - MemoryContextSwitchTo(rscxt); + /* Get policy, permissive or restrictive */ + policy->permissive = policy_form->polpermissive; - policy = palloc0(sizeof(RowSecurityPolicy)); - policy->policy_name = pstrdup(policy_name_value); - policy->polcmd = cmd_value; - policy->permissive = permissive_value; - policy->roles = DatumGetArrayTypePCopy(roles_datum); - policy->qual = copyObject(qual_expr); - policy->with_check_qual = copyObject(with_check_qual); - policy->hassublinks = checkExprHasSubLink((Node *) qual_expr) || - checkExprHasSubLink((Node *) with_check_qual); + /* Get policy name */ + policy->policy_name = + MemoryContextStrdup(rscxt, NameStr(policy_form->polname)); - rsdesc->policies = lcons(policy, rsdesc->policies); + /* Get policy roles */ + datum = heap_getattr(tuple, Anum_pg_policy_polroles, + RelationGetDescr(catalog), &isnull); + /* shouldn't be null, but let's check for luck */ + if (isnull) + elog(ERROR, "unexpected null value in pg_policy.polroles"); + MemoryContextSwitchTo(rscxt); + policy->roles = DatumGetArrayTypePCopy(datum); + MemoryContextSwitchTo(oldcxt); + /* Get policy qual */ + datum = heap_getattr(tuple, Anum_pg_policy_polqual, + RelationGetDescr(catalog), &isnull); + if (!isnull) + { + str_value = TextDatumGetCString(datum); + MemoryContextSwitchTo(rscxt); + policy->qual = (Expr *) stringToNode(str_value); MemoryContextSwitchTo(oldcxt); + pfree(str_value); + } + else + policy->qual = NULL; - /* clean up some (not all) of the junk ... */ - if (qual_expr != NULL) - pfree(qual_expr); - if (with_check_qual != NULL) - pfree(with_check_qual); + /* Get WITH CHECK qual */ + datum = heap_getattr(tuple, Anum_pg_policy_polwithcheck, + RelationGetDescr(catalog), &isnull); + if (!isnull) + { + str_value = TextDatumGetCString(datum); + MemoryContextSwitchTo(rscxt); + policy->with_check_qual = (Expr *) stringToNode(str_value); + MemoryContextSwitchTo(oldcxt); + pfree(str_value); } + else + policy->with_check_qual = NULL; - systable_endscan(sscan); - table_close(catalog, AccessShareLock); - } - PG_CATCH(); - { - /* Delete rscxt, first making sure it isn't active */ + /* We want to cache whether there are SubLinks in these expressions */ + policy->hassublinks = checkExprHasSubLink((Node *) policy->qual) || + checkExprHasSubLink((Node *) policy->with_check_qual); + + /* + * Add this object to list. For historical reasons, the list is built + * in reverse order. + */ + MemoryContextSwitchTo(rscxt); + rsdesc->policies = lcons(policy, rsdesc->policies); MemoryContextSwitchTo(oldcxt); - MemoryContextDelete(rscxt); - PG_RE_THROW(); } - PG_END_TRY(); - /* Success --- attach the policy descriptor to the relcache entry */ + systable_endscan(sscan); + table_close(catalog, AccessShareLock); + + /* + * Success. Reparent the descriptor's memory context under + * CacheMemoryContext so that it will live indefinitely, then attach the + * policy descriptor to the relcache entry. + */ + MemoryContextSetParent(rscxt, CacheMemoryContext); + relation->rd_rsdesc = rsdesc; } From 3c8819955023694feeaa456ee60853d0d6d0e60a Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 26 Sep 2020 17:42:20 -0400 Subject: [PATCH 206/589] Further stabilize output from rolenames regression test. Commit e5209bf37 didn't quite get the job done, as I failed to notice that chksetconfig() also needed to have its ORDER BY extended. Per buildfarm member dory. Report: https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=dory&dt=2020-09-26%2020%3A10%3A13 --- src/test/modules/unsafe_tests/expected/rolenames.out | 10 +++++----- src/test/modules/unsafe_tests/sql/rolenames.sql | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/modules/unsafe_tests/expected/rolenames.out b/src/test/modules/unsafe_tests/expected/rolenames.out index 6ddb1a83f67b..eb608fdc2eaf 100644 --- a/src/test/modules/unsafe_tests/expected/rolenames.out +++ b/src/test/modules/unsafe_tests/expected/rolenames.out @@ -29,7 +29,7 @@ SELECT COALESCE(d.datname, 'ALL'), COALESCE(r.rolname, 'ALL'), AS v(uname, keyword) ON (r.rolname = v.uname) WHERE (r.rolname) IN ('Public', 'current_user', 'regress_testrol1', 'regress_testrol2') -ORDER BY 1, 2; +ORDER BY 1, 2, 3; $$ LANGUAGE SQL; CREATE FUNCTION chkumapping() RETURNS TABLE (umname name, umserver name, umoptions text[]) @@ -437,8 +437,8 @@ SELECT * FROM chksetconfig(); ALL | Public | - | {application_name=BARBAR} ALL | current_user | - | {application_name=FOOFOO} ALL | regress_testrol1 | session_user | {application_name=BAR} - ALL | regress_testrol2 | current_user | {application_name=FOO} ALL | regress_testrol2 | current_role | {application_name=FOO} + ALL | regress_testrol2 | current_user | {application_name=FOO} (5 rows) ALTER ROLE regress_testrol1 SET application_name to 'SLAM'; @@ -448,8 +448,8 @@ SELECT * FROM chksetconfig(); ALL | Public | - | {application_name=BARBAR} ALL | current_user | - | {application_name=FOOFOO} ALL | regress_testrol1 | session_user | {application_name=SLAM} - ALL | regress_testrol2 | current_user | {application_name=FOO} ALL | regress_testrol2 | current_role | {application_name=FOO} + ALL | regress_testrol2 | current_user | {application_name=FOO} (5 rows) ALTER ROLE CURRENT_ROLE RESET application_name; @@ -489,8 +489,8 @@ SELECT * FROM chksetconfig(); ALL | Public | - | {application_name=BARBAR} ALL | current_user | - | {application_name=FOOFOO} ALL | regress_testrol1 | session_user | {application_name=BAR} - ALL | regress_testrol2 | current_user | {application_name=FOO} ALL | regress_testrol2 | current_role | {application_name=FOO} + ALL | regress_testrol2 | current_user | {application_name=FOO} (5 rows) ALTER USER regress_testrol1 SET application_name to 'SLAM'; @@ -500,8 +500,8 @@ SELECT * FROM chksetconfig(); ALL | Public | - | {application_name=BARBAR} ALL | current_user | - | {application_name=FOOFOO} ALL | regress_testrol1 | session_user | {application_name=SLAM} - ALL | regress_testrol2 | current_user | {application_name=FOO} ALL | regress_testrol2 | current_role | {application_name=FOO} + ALL | regress_testrol2 | current_user | {application_name=FOO} (5 rows) ALTER USER CURRENT_ROLE RESET application_name; diff --git a/src/test/modules/unsafe_tests/sql/rolenames.sql b/src/test/modules/unsafe_tests/sql/rolenames.sql index 40dc86fdb9b3..adac36536db4 100644 --- a/src/test/modules/unsafe_tests/sql/rolenames.sql +++ b/src/test/modules/unsafe_tests/sql/rolenames.sql @@ -30,7 +30,7 @@ SELECT COALESCE(d.datname, 'ALL'), COALESCE(r.rolname, 'ALL'), AS v(uname, keyword) ON (r.rolname = v.uname) WHERE (r.rolname) IN ('Public', 'current_user', 'regress_testrol1', 'regress_testrol2') -ORDER BY 1, 2; +ORDER BY 1, 2, 3; $$ LANGUAGE SQL; CREATE FUNCTION chkumapping() From 41efb8340877e8ffd0023bb6b2ef22ffd1ca014d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 27 Sep 2020 12:51:28 -0400 Subject: [PATCH 207/589] Move resolution of AlternativeSubPlan choices to the planner. When commit bd3daddaf introduced AlternativeSubPlans, I had some ambitions towards allowing the choice of subplan to change during execution. That has not happened, or even been thought about, in the ensuing twelve years; so it seems like a failed experiment. So let's rip that out and resolve the choice of subplan at the end of planning (in setrefs.c) rather than during executor startup. This has a number of positive benefits: * Removal of a few hundred lines of executor code, since AlternativeSubPlans need no longer be supported there. * Removal of executor-startup overhead (particularly, initialization of subplans that won't be used). * Removal of incidental costs of having a larger plan tree, such as tree-scanning and copying costs in the plancache; not to mention setrefs.c's own costs of processing the discarded subplans. * EXPLAIN no longer has to print a weird (and undocumented) representation of an AlternativeSubPlan choice; it sees only the subplan actually used. This should mean less confusion for users. * Since setrefs.c knows which subexpression of a plan node it's working on at any instant, it's possible to adjust the estimated number of executions of the subplan based on that. For example, we should usually estimate more executions of a qual expression than a targetlist expression. The implementation used here is pretty simplistic, because we don't want to expend a lot of cycles on the issue; but it's better than ignoring the point entirely, as the executor had to. That last point might possibly result in shifting the choice between hashed and non-hashed EXISTS subplans in a few cases, but in general this patch isn't meant to change planner choices. Since we're doing the resolution so late, it's really impossible to change any plan choices outside the AlternativeSubPlan itself. Patch by me; thanks to David Rowley for review. Discussion: https://postgr.es/m/1992952.1592785225@sss.pgh.pa.us --- src/backend/executor/execExpr.c | 17 - src/backend/executor/execExprInterp.c | 23 -- src/backend/executor/nodeSubplan.c | 80 ---- src/backend/jit/llvm/llvmjit_expr.c | 6 - src/backend/jit/llvm/llvmjit_types.c | 1 - src/backend/nodes/outfuncs.c | 1 + src/backend/optimizer/plan/planner.c | 5 +- src/backend/optimizer/plan/setrefs.c | 372 +++++++++++++----- src/backend/optimizer/plan/subselect.c | 46 ++- src/backend/utils/adt/ruleutils.c | 7 +- src/include/executor/execExpr.h | 10 - src/include/executor/nodeSubplan.h | 4 - src/include/nodes/execnodes.h | 12 - src/include/nodes/nodes.h | 1 - src/include/nodes/pathnodes.h | 1 + src/include/nodes/primnodes.h | 3 + src/test/regress/expected/insert_conflict.out | 6 +- src/test/regress/expected/subselect.out | 47 +++ src/test/regress/expected/updatable_views.out | 48 +-- src/test/regress/sql/subselect.sql | 17 + 20 files changed, 412 insertions(+), 295 deletions(-) diff --git a/src/backend/executor/execExpr.c b/src/backend/executor/execExpr.c index 236413f62aaf..868f8b0858f8 100644 --- a/src/backend/executor/execExpr.c +++ b/src/backend/executor/execExpr.c @@ -1104,23 +1104,6 @@ ExecInitExprRec(Expr *node, ExprState *state, break; } - case T_AlternativeSubPlan: - { - AlternativeSubPlan *asplan = (AlternativeSubPlan *) node; - AlternativeSubPlanState *asstate; - - if (!state->parent) - elog(ERROR, "AlternativeSubPlan found with no parent plan"); - - asstate = ExecInitAlternativeSubPlan(asplan, state->parent); - - scratch.opcode = EEOP_ALTERNATIVE_SUBPLAN; - scratch.d.alternative_subplan.asstate = asstate; - - ExprEvalPushStep(state, &scratch); - break; - } - case T_FieldSelect: { FieldSelect *fselect = (FieldSelect *) node; diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c index b812bbaceef8..26c2b4963215 100644 --- a/src/backend/executor/execExprInterp.c +++ b/src/backend/executor/execExprInterp.c @@ -431,7 +431,6 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull) &&CASE_EEOP_GROUPING_FUNC, &&CASE_EEOP_WINDOW_FUNC, &&CASE_EEOP_SUBPLAN, - &&CASE_EEOP_ALTERNATIVE_SUBPLAN, &&CASE_EEOP_AGG_STRICT_DESERIALIZE, &&CASE_EEOP_AGG_DESERIALIZE, &&CASE_EEOP_AGG_STRICT_INPUT_CHECK_ARGS, @@ -1536,14 +1535,6 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull) EEO_NEXT(); } - EEO_CASE(EEOP_ALTERNATIVE_SUBPLAN) - { - /* too complex for an inline implementation */ - ExecEvalAlternativeSubPlan(state, op, econtext); - - EEO_NEXT(); - } - /* evaluate a strict aggregate deserialization function */ EEO_CASE(EEOP_AGG_STRICT_DESERIALIZE) { @@ -3868,20 +3859,6 @@ ExecEvalSubPlan(ExprState *state, ExprEvalStep *op, ExprContext *econtext) *op->resvalue = ExecSubPlan(sstate, econtext, op->resnull); } -/* - * Hand off evaluation of an alternative subplan to nodeSubplan.c - */ -void -ExecEvalAlternativeSubPlan(ExprState *state, ExprEvalStep *op, ExprContext *econtext) -{ - AlternativeSubPlanState *asstate = op->d.alternative_subplan.asstate; - - /* could potentially be nested, so make sure there's enough stack */ - check_stack_depth(); - - *op->resvalue = ExecAlternativeSubPlan(asstate, econtext, op->resnull); -} - /* * Evaluate a wholerow Var expression. * diff --git a/src/backend/executor/nodeSubplan.c b/src/backend/executor/nodeSubplan.c index 9a7962518ee6..9a706df5f061 100644 --- a/src/backend/executor/nodeSubplan.c +++ b/src/backend/executor/nodeSubplan.c @@ -1303,83 +1303,3 @@ ExecReScanSetParamPlan(SubPlanState *node, PlanState *parent) parent->chgParam = bms_add_member(parent->chgParam, paramid); } } - - -/* - * ExecInitAlternativeSubPlan - * - * Initialize for execution of one of a set of alternative subplans. - */ -AlternativeSubPlanState * -ExecInitAlternativeSubPlan(AlternativeSubPlan *asplan, PlanState *parent) -{ - AlternativeSubPlanState *asstate = makeNode(AlternativeSubPlanState); - double num_calls; - SubPlan *subplan1; - SubPlan *subplan2; - Cost cost1; - Cost cost2; - ListCell *lc; - - asstate->subplan = asplan; - - /* - * Initialize subplans. (Can we get away with only initializing the one - * we're going to use?) - */ - foreach(lc, asplan->subplans) - { - SubPlan *sp = lfirst_node(SubPlan, lc); - SubPlanState *sps = ExecInitSubPlan(sp, parent); - - asstate->subplans = lappend(asstate->subplans, sps); - parent->subPlan = lappend(parent->subPlan, sps); - } - - /* - * Select the one to be used. For this, we need an estimate of the number - * of executions of the subplan. We use the number of output rows - * expected from the parent plan node. This is a good estimate if we are - * in the parent's targetlist, and an underestimate (but probably not by - * more than a factor of 2) if we are in the qual. - */ - num_calls = parent->plan->plan_rows; - - /* - * The planner saved enough info so that we don't have to work very hard - * to estimate the total cost, given the number-of-calls estimate. - */ - Assert(list_length(asplan->subplans) == 2); - subplan1 = (SubPlan *) linitial(asplan->subplans); - subplan2 = (SubPlan *) lsecond(asplan->subplans); - - cost1 = subplan1->startup_cost + num_calls * subplan1->per_call_cost; - cost2 = subplan2->startup_cost + num_calls * subplan2->per_call_cost; - - if (cost1 < cost2) - asstate->active = 0; - else - asstate->active = 1; - - return asstate; -} - -/* - * ExecAlternativeSubPlan - * - * Execute one of a set of alternative subplans. - * - * Note: in future we might consider changing to different subplans on the - * fly, in case the original rowcount estimate turns out to be way off. - */ -Datum -ExecAlternativeSubPlan(AlternativeSubPlanState *node, - ExprContext *econtext, - bool *isNull) -{ - /* Just pass control to the active subplan */ - SubPlanState *activesp = list_nth_node(SubPlanState, - node->subplans, node->active); - - return ExecSubPlan(activesp, econtext, isNull); -} diff --git a/src/backend/jit/llvm/llvmjit_expr.c b/src/backend/jit/llvm/llvmjit_expr.c index cca5c117a0ee..eb1dea658cb2 100644 --- a/src/backend/jit/llvm/llvmjit_expr.c +++ b/src/backend/jit/llvm/llvmjit_expr.c @@ -1918,12 +1918,6 @@ llvm_compile_expr(ExprState *state) LLVMBuildBr(b, opblocks[opno + 1]); break; - case EEOP_ALTERNATIVE_SUBPLAN: - build_EvalXFunc(b, mod, "ExecEvalAlternativeSubPlan", - v_state, op, v_econtext); - LLVMBuildBr(b, opblocks[opno + 1]); - break; - case EEOP_AGG_STRICT_DESERIALIZE: case EEOP_AGG_DESERIALIZE: { diff --git a/src/backend/jit/llvm/llvmjit_types.c b/src/backend/jit/llvm/llvmjit_types.c index 0a93d5f6658c..1ed3cafa2f23 100644 --- a/src/backend/jit/llvm/llvmjit_types.c +++ b/src/backend/jit/llvm/llvmjit_types.c @@ -102,7 +102,6 @@ void *referenced_functions[] = ExecAggTransReparent, ExecEvalAggOrderedTransDatum, ExecEvalAggOrderedTransTuple, - ExecEvalAlternativeSubPlan, ExecEvalArrayCoerce, ExecEvalArrayExpr, ExecEvalConstraintCheck, diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index e2f177515dac..f0386480ab89 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2254,6 +2254,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_BOOL_FIELD(hasLateralRTEs); WRITE_BOOL_FIELD(hasHavingQual); WRITE_BOOL_FIELD(hasPseudoConstantQuals); + WRITE_BOOL_FIELD(hasAlternativeSubPlans); WRITE_BOOL_FIELD(hasRecursion); WRITE_INT_FIELD(wt_param_id); WRITE_BITMAPSET_FIELD(curOuterRels); diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 8007e205ed77..3e2b4965c4a8 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -629,6 +629,8 @@ subquery_planner(PlannerGlobal *glob, Query *parse, root->minmax_aggs = NIL; root->qual_security_level = 0; root->inhTargetKind = INHKIND_NONE; + root->hasPseudoConstantQuals = false; + root->hasAlternativeSubPlans = false; root->hasRecursion = hasRecursion; if (hasRecursion) root->wt_param_id = assign_special_exec_param(root); @@ -759,9 +761,6 @@ subquery_planner(PlannerGlobal *glob, Query *parse, */ root->hasHavingQual = (parse->havingQual != NULL); - /* Clear this flag; might get set in distribute_qual_to_rels */ - root->hasPseudoConstantQuals = false; - /* * Do expression preprocessing on targetlist and quals, as well as other * random expressions in the querytree. Note that we do not need to diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index baefe0e94617..dd8e2e966ddd 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -49,6 +49,7 @@ typedef struct { PlannerInfo *root; int rtoffset; + double num_exec; } fix_scan_expr_context; typedef struct @@ -58,6 +59,7 @@ typedef struct indexed_tlist *inner_itlist; Index acceptable_rel; int rtoffset; + double num_exec; } fix_join_expr_context; typedef struct @@ -66,8 +68,28 @@ typedef struct indexed_tlist *subplan_itlist; Index newvarno; int rtoffset; + double num_exec; } fix_upper_expr_context; +/* + * Selecting the best alternative in an AlternativeSubPlan expression requires + * estimating how many times that expression will be evaluated. For an + * expression in a plan node's targetlist, the plan's estimated number of + * output rows is clearly what to use, but for an expression in a qual it's + * far less clear. Since AlternativeSubPlans aren't heavily used, we don't + * want to expend a lot of cycles making such estimates. What we use is twice + * the number of output rows. That's not entirely unfounded: we know that + * clause_selectivity() would fall back to a default selectivity estimate + * of 0.5 for any SubPlan, so if the qual containing the SubPlan is the last + * to be applied (which it likely would be, thanks to order_qual_clauses()), + * this matches what we could have estimated in a far more laborious fashion. + * Obviously there are many other scenarios, but it's probably not worth the + * trouble to try to improve on this estimate, especially not when we don't + * have a better estimate for the selectivity of the SubPlan qual itself. + */ +#define NUM_EXEC_TLIST(parentplan) ((parentplan)->plan_rows) +#define NUM_EXEC_QUAL(parentplan) ((parentplan)->plan_rows * 2.0) + /* * Check if a Const node is a regclass value. We accept plain OID too, * since a regclass Const will get folded to that type if it's an argument @@ -79,8 +101,8 @@ typedef struct (((con)->consttype == REGCLASSOID || (con)->consttype == OIDOID) && \ !(con)->constisnull) -#define fix_scan_list(root, lst, rtoffset) \ - ((List *) fix_scan_expr(root, (Node *) (lst), rtoffset)) +#define fix_scan_list(root, lst, rtoffset, num_exec) \ + ((List *) fix_scan_expr(root, (Node *) (lst), rtoffset, num_exec)) static void add_rtes_to_flat_rtable(PlannerInfo *root, bool recursing); static void flatten_unplanned_rtes(PlannerGlobal *glob, RangeTblEntry *rte); @@ -109,7 +131,8 @@ static Plan *set_mergeappend_references(PlannerInfo *root, int rtoffset); static void set_hash_references(PlannerInfo *root, Plan *plan, int rtoffset); static Relids offset_relid_set(Relids relids, int rtoffset); -static Node *fix_scan_expr(PlannerInfo *root, Node *node, int rtoffset); +static Node *fix_scan_expr(PlannerInfo *root, Node *node, + int rtoffset, double num_exec); static Node *fix_scan_expr_mutator(Node *node, fix_scan_expr_context *context); static bool fix_scan_expr_walker(Node *node, fix_scan_expr_context *context); static void set_join_references(PlannerInfo *root, Join *join, int rtoffset); @@ -133,14 +156,15 @@ static List *fix_join_expr(PlannerInfo *root, List *clauses, indexed_tlist *outer_itlist, indexed_tlist *inner_itlist, - Index acceptable_rel, int rtoffset); + Index acceptable_rel, + int rtoffset, double num_exec); static Node *fix_join_expr_mutator(Node *node, fix_join_expr_context *context); static Node *fix_upper_expr(PlannerInfo *root, Node *node, indexed_tlist *subplan_itlist, Index newvarno, - int rtoffset); + int rtoffset, double num_exec); static Node *fix_upper_expr_mutator(Node *node, fix_upper_expr_context *context); static List *set_returning_clause_references(PlannerInfo *root, @@ -177,17 +201,20 @@ static List *set_returning_clause_references(PlannerInfo *root, * 5. PARAM_MULTIEXPR Params are replaced by regular PARAM_EXEC Params, * now that we have finished planning all MULTIEXPR subplans. * - * 6. We compute regproc OIDs for operators (ie, we look up the function + * 6. AlternativeSubPlan expressions are replaced by just one of their + * alternatives, using an estimate of how many times they'll be executed. + * + * 7. We compute regproc OIDs for operators (ie, we look up the function * that implements each op). * - * 7. We create lists of specific objects that the plan depends on. + * 8. We create lists of specific objects that the plan depends on. * This will be used by plancache.c to drive invalidation of cached plans. * Relation dependencies are represented by OIDs, and everything else by * PlanInvalItems (this distinction is motivated by the shared-inval APIs). * Currently, relations, user-defined functions, and domains are the only * types of objects that are explicitly tracked this way. * - * 8. We assign every plan node in the tree a unique ID. + * 9. We assign every plan node in the tree a unique ID. * * We also perform one final optimization step, which is to delete * SubqueryScan, Append, and MergeAppend plan nodes that aren't doing @@ -490,9 +517,11 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) splan->scanrelid += rtoffset; splan->plan.targetlist = - fix_scan_list(root, splan->plan.targetlist, rtoffset); + fix_scan_list(root, splan->plan.targetlist, + rtoffset, NUM_EXEC_TLIST(plan)); splan->plan.qual = - fix_scan_list(root, splan->plan.qual, rtoffset); + fix_scan_list(root, splan->plan.qual, + rtoffset, NUM_EXEC_QUAL(plan)); } break; case T_SampleScan: @@ -501,11 +530,14 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) splan->scan.scanrelid += rtoffset; splan->scan.plan.targetlist = - fix_scan_list(root, splan->scan.plan.targetlist, rtoffset); + fix_scan_list(root, splan->scan.plan.targetlist, + rtoffset, NUM_EXEC_TLIST(plan)); splan->scan.plan.qual = - fix_scan_list(root, splan->scan.plan.qual, rtoffset); + fix_scan_list(root, splan->scan.plan.qual, + rtoffset, NUM_EXEC_QUAL(plan)); splan->tablesample = (TableSampleClause *) - fix_scan_expr(root, (Node *) splan->tablesample, rtoffset); + fix_scan_expr(root, (Node *) splan->tablesample, + rtoffset, 1); } break; case T_IndexScan: @@ -514,17 +546,23 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) splan->scan.scanrelid += rtoffset; splan->scan.plan.targetlist = - fix_scan_list(root, splan->scan.plan.targetlist, rtoffset); + fix_scan_list(root, splan->scan.plan.targetlist, + rtoffset, NUM_EXEC_TLIST(plan)); splan->scan.plan.qual = - fix_scan_list(root, splan->scan.plan.qual, rtoffset); + fix_scan_list(root, splan->scan.plan.qual, + rtoffset, NUM_EXEC_QUAL(plan)); splan->indexqual = - fix_scan_list(root, splan->indexqual, rtoffset); + fix_scan_list(root, splan->indexqual, + rtoffset, 1); splan->indexqualorig = - fix_scan_list(root, splan->indexqualorig, rtoffset); + fix_scan_list(root, splan->indexqualorig, + rtoffset, NUM_EXEC_QUAL(plan)); splan->indexorderby = - fix_scan_list(root, splan->indexorderby, rtoffset); + fix_scan_list(root, splan->indexorderby, + rtoffset, 1); splan->indexorderbyorig = - fix_scan_list(root, splan->indexorderbyorig, rtoffset); + fix_scan_list(root, splan->indexorderbyorig, + rtoffset, NUM_EXEC_QUAL(plan)); } break; case T_IndexOnlyScan: @@ -543,9 +581,10 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) Assert(splan->scan.plan.targetlist == NIL); Assert(splan->scan.plan.qual == NIL); splan->indexqual = - fix_scan_list(root, splan->indexqual, rtoffset); + fix_scan_list(root, splan->indexqual, rtoffset, 1); splan->indexqualorig = - fix_scan_list(root, splan->indexqualorig, rtoffset); + fix_scan_list(root, splan->indexqualorig, + rtoffset, NUM_EXEC_QUAL(plan)); } break; case T_BitmapHeapScan: @@ -554,11 +593,14 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) splan->scan.scanrelid += rtoffset; splan->scan.plan.targetlist = - fix_scan_list(root, splan->scan.plan.targetlist, rtoffset); + fix_scan_list(root, splan->scan.plan.targetlist, + rtoffset, NUM_EXEC_TLIST(plan)); splan->scan.plan.qual = - fix_scan_list(root, splan->scan.plan.qual, rtoffset); + fix_scan_list(root, splan->scan.plan.qual, + rtoffset, NUM_EXEC_QUAL(plan)); splan->bitmapqualorig = - fix_scan_list(root, splan->bitmapqualorig, rtoffset); + fix_scan_list(root, splan->bitmapqualorig, + rtoffset, NUM_EXEC_QUAL(plan)); } break; case T_TidScan: @@ -567,11 +609,14 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) splan->scan.scanrelid += rtoffset; splan->scan.plan.targetlist = - fix_scan_list(root, splan->scan.plan.targetlist, rtoffset); + fix_scan_list(root, splan->scan.plan.targetlist, + rtoffset, NUM_EXEC_TLIST(plan)); splan->scan.plan.qual = - fix_scan_list(root, splan->scan.plan.qual, rtoffset); + fix_scan_list(root, splan->scan.plan.qual, + rtoffset, NUM_EXEC_QUAL(plan)); splan->tidquals = - fix_scan_list(root, splan->tidquals, rtoffset); + fix_scan_list(root, splan->tidquals, + rtoffset, 1); } break; case T_SubqueryScan: @@ -585,11 +630,13 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) splan->scan.scanrelid += rtoffset; splan->scan.plan.targetlist = - fix_scan_list(root, splan->scan.plan.targetlist, rtoffset); + fix_scan_list(root, splan->scan.plan.targetlist, + rtoffset, NUM_EXEC_TLIST(plan)); splan->scan.plan.qual = - fix_scan_list(root, splan->scan.plan.qual, rtoffset); + fix_scan_list(root, splan->scan.plan.qual, + rtoffset, NUM_EXEC_QUAL(plan)); splan->functions = - fix_scan_list(root, splan->functions, rtoffset); + fix_scan_list(root, splan->functions, rtoffset, 1); } break; case T_TableFuncScan: @@ -598,11 +645,14 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) splan->scan.scanrelid += rtoffset; splan->scan.plan.targetlist = - fix_scan_list(root, splan->scan.plan.targetlist, rtoffset); + fix_scan_list(root, splan->scan.plan.targetlist, + rtoffset, NUM_EXEC_TLIST(plan)); splan->scan.plan.qual = - fix_scan_list(root, splan->scan.plan.qual, rtoffset); + fix_scan_list(root, splan->scan.plan.qual, + rtoffset, NUM_EXEC_QUAL(plan)); splan->tablefunc = (TableFunc *) - fix_scan_expr(root, (Node *) splan->tablefunc, rtoffset); + fix_scan_expr(root, (Node *) splan->tablefunc, + rtoffset, 1); } break; case T_ValuesScan: @@ -611,11 +661,14 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) splan->scan.scanrelid += rtoffset; splan->scan.plan.targetlist = - fix_scan_list(root, splan->scan.plan.targetlist, rtoffset); + fix_scan_list(root, splan->scan.plan.targetlist, + rtoffset, NUM_EXEC_TLIST(plan)); splan->scan.plan.qual = - fix_scan_list(root, splan->scan.plan.qual, rtoffset); + fix_scan_list(root, splan->scan.plan.qual, + rtoffset, NUM_EXEC_QUAL(plan)); splan->values_lists = - fix_scan_list(root, splan->values_lists, rtoffset); + fix_scan_list(root, splan->values_lists, + rtoffset, 1); } break; case T_CteScan: @@ -624,9 +677,11 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) splan->scan.scanrelid += rtoffset; splan->scan.plan.targetlist = - fix_scan_list(root, splan->scan.plan.targetlist, rtoffset); + fix_scan_list(root, splan->scan.plan.targetlist, + rtoffset, NUM_EXEC_TLIST(plan)); splan->scan.plan.qual = - fix_scan_list(root, splan->scan.plan.qual, rtoffset); + fix_scan_list(root, splan->scan.plan.qual, + rtoffset, NUM_EXEC_QUAL(plan)); } break; case T_NamedTuplestoreScan: @@ -635,9 +690,11 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) splan->scan.scanrelid += rtoffset; splan->scan.plan.targetlist = - fix_scan_list(root, splan->scan.plan.targetlist, rtoffset); + fix_scan_list(root, splan->scan.plan.targetlist, + rtoffset, NUM_EXEC_TLIST(plan)); splan->scan.plan.qual = - fix_scan_list(root, splan->scan.plan.qual, rtoffset); + fix_scan_list(root, splan->scan.plan.qual, + rtoffset, NUM_EXEC_QUAL(plan)); } break; case T_WorkTableScan: @@ -646,9 +703,11 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) splan->scan.scanrelid += rtoffset; splan->scan.plan.targetlist = - fix_scan_list(root, splan->scan.plan.targetlist, rtoffset); + fix_scan_list(root, splan->scan.plan.targetlist, + rtoffset, NUM_EXEC_TLIST(plan)); splan->scan.plan.qual = - fix_scan_list(root, splan->scan.plan.qual, rtoffset); + fix_scan_list(root, splan->scan.plan.qual, + rtoffset, NUM_EXEC_QUAL(plan)); } break; case T_ForeignScan: @@ -732,9 +791,9 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) Assert(splan->plan.qual == NIL); splan->limitOffset = - fix_scan_expr(root, splan->limitOffset, rtoffset); + fix_scan_expr(root, splan->limitOffset, rtoffset, 1); splan->limitCount = - fix_scan_expr(root, splan->limitCount, rtoffset); + fix_scan_expr(root, splan->limitCount, rtoffset, 1); } break; case T_Agg: @@ -775,9 +834,9 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) * variable refs, so fix_scan_expr works for them. */ wplan->startOffset = - fix_scan_expr(root, wplan->startOffset, rtoffset); + fix_scan_expr(root, wplan->startOffset, rtoffset, 1); wplan->endOffset = - fix_scan_expr(root, wplan->endOffset, rtoffset); + fix_scan_expr(root, wplan->endOffset, rtoffset, 1); } break; case T_Result: @@ -793,13 +852,15 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) else { splan->plan.targetlist = - fix_scan_list(root, splan->plan.targetlist, rtoffset); + fix_scan_list(root, splan->plan.targetlist, + rtoffset, NUM_EXEC_TLIST(plan)); splan->plan.qual = - fix_scan_list(root, splan->plan.qual, rtoffset); + fix_scan_list(root, splan->plan.qual, + rtoffset, NUM_EXEC_QUAL(plan)); } /* resconstantqual can't contain any subplan variable refs */ splan->resconstantqual = - fix_scan_expr(root, splan->resconstantqual, rtoffset); + fix_scan_expr(root, splan->resconstantqual, rtoffset, 1); } break; case T_ProjectSet: @@ -813,7 +874,8 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) Assert(splan->plan.qual == NIL); splan->withCheckOptionLists = - fix_scan_list(root, splan->withCheckOptionLists, rtoffset); + fix_scan_list(root, splan->withCheckOptionLists, + rtoffset, 1); if (splan->returningLists) { @@ -874,18 +936,18 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) fix_join_expr(root, splan->onConflictSet, NULL, itlist, linitial_int(splan->resultRelations), - rtoffset); + rtoffset, NUM_EXEC_QUAL(plan)); splan->onConflictWhere = (Node *) fix_join_expr(root, (List *) splan->onConflictWhere, NULL, itlist, linitial_int(splan->resultRelations), - rtoffset); + rtoffset, NUM_EXEC_QUAL(plan)); pfree(itlist); splan->exclRelTlist = - fix_scan_list(root, splan->exclRelTlist, rtoffset); + fix_scan_list(root, splan->exclRelTlist, rtoffset, 1); } splan->nominalRelation += rtoffset; @@ -1026,19 +1088,24 @@ set_indexonlyscan_references(PlannerInfo *root, (Node *) plan->scan.plan.targetlist, index_itlist, INDEX_VAR, - rtoffset); + rtoffset, + NUM_EXEC_TLIST((Plan *) plan)); plan->scan.plan.qual = (List *) fix_upper_expr(root, (Node *) plan->scan.plan.qual, index_itlist, INDEX_VAR, - rtoffset); + rtoffset, + NUM_EXEC_QUAL((Plan *) plan)); /* indexqual is already transformed to reference index columns */ - plan->indexqual = fix_scan_list(root, plan->indexqual, rtoffset); + plan->indexqual = fix_scan_list(root, plan->indexqual, + rtoffset, 1); /* indexorderby is already transformed to reference index columns */ - plan->indexorderby = fix_scan_list(root, plan->indexorderby, rtoffset); + plan->indexorderby = fix_scan_list(root, plan->indexorderby, + rtoffset, 1); /* indextlist must NOT be transformed to reference index columns */ - plan->indextlist = fix_scan_list(root, plan->indextlist, rtoffset); + plan->indextlist = fix_scan_list(root, plan->indextlist, + rtoffset, NUM_EXEC_TLIST((Plan *) plan)); pfree(index_itlist); @@ -1084,9 +1151,11 @@ set_subqueryscan_references(PlannerInfo *root, */ plan->scan.scanrelid += rtoffset; plan->scan.plan.targetlist = - fix_scan_list(root, plan->scan.plan.targetlist, rtoffset); + fix_scan_list(root, plan->scan.plan.targetlist, + rtoffset, NUM_EXEC_TLIST((Plan *) plan)); plan->scan.plan.qual = - fix_scan_list(root, plan->scan.plan.qual, rtoffset); + fix_scan_list(root, plan->scan.plan.qual, + rtoffset, NUM_EXEC_QUAL((Plan *) plan)); result = (Plan *) plan; } @@ -1202,29 +1271,34 @@ set_foreignscan_references(PlannerInfo *root, (Node *) fscan->scan.plan.targetlist, itlist, INDEX_VAR, - rtoffset); + rtoffset, + NUM_EXEC_TLIST((Plan *) fscan)); fscan->scan.plan.qual = (List *) fix_upper_expr(root, (Node *) fscan->scan.plan.qual, itlist, INDEX_VAR, - rtoffset); + rtoffset, + NUM_EXEC_QUAL((Plan *) fscan)); fscan->fdw_exprs = (List *) fix_upper_expr(root, (Node *) fscan->fdw_exprs, itlist, INDEX_VAR, - rtoffset); + rtoffset, + NUM_EXEC_QUAL((Plan *) fscan)); fscan->fdw_recheck_quals = (List *) fix_upper_expr(root, (Node *) fscan->fdw_recheck_quals, itlist, INDEX_VAR, - rtoffset); + rtoffset, + NUM_EXEC_QUAL((Plan *) fscan)); pfree(itlist); /* fdw_scan_tlist itself just needs fix_scan_list() adjustments */ fscan->fdw_scan_tlist = - fix_scan_list(root, fscan->fdw_scan_tlist, rtoffset); + fix_scan_list(root, fscan->fdw_scan_tlist, + rtoffset, NUM_EXEC_TLIST((Plan *) fscan)); } else { @@ -1233,13 +1307,17 @@ set_foreignscan_references(PlannerInfo *root, * way */ fscan->scan.plan.targetlist = - fix_scan_list(root, fscan->scan.plan.targetlist, rtoffset); + fix_scan_list(root, fscan->scan.plan.targetlist, + rtoffset, NUM_EXEC_TLIST((Plan *) fscan)); fscan->scan.plan.qual = - fix_scan_list(root, fscan->scan.plan.qual, rtoffset); + fix_scan_list(root, fscan->scan.plan.qual, + rtoffset, NUM_EXEC_QUAL((Plan *) fscan)); fscan->fdw_exprs = - fix_scan_list(root, fscan->fdw_exprs, rtoffset); + fix_scan_list(root, fscan->fdw_exprs, + rtoffset, NUM_EXEC_QUAL((Plan *) fscan)); fscan->fdw_recheck_quals = - fix_scan_list(root, fscan->fdw_recheck_quals, rtoffset); + fix_scan_list(root, fscan->fdw_recheck_quals, + rtoffset, NUM_EXEC_QUAL((Plan *) fscan)); } fscan->fs_relids = offset_relid_set(fscan->fs_relids, rtoffset); @@ -1270,33 +1348,40 @@ set_customscan_references(PlannerInfo *root, (Node *) cscan->scan.plan.targetlist, itlist, INDEX_VAR, - rtoffset); + rtoffset, + NUM_EXEC_TLIST((Plan *) cscan)); cscan->scan.plan.qual = (List *) fix_upper_expr(root, (Node *) cscan->scan.plan.qual, itlist, INDEX_VAR, - rtoffset); + rtoffset, + NUM_EXEC_QUAL((Plan *) cscan)); cscan->custom_exprs = (List *) fix_upper_expr(root, (Node *) cscan->custom_exprs, itlist, INDEX_VAR, - rtoffset); + rtoffset, + NUM_EXEC_QUAL((Plan *) cscan)); pfree(itlist); /* custom_scan_tlist itself just needs fix_scan_list() adjustments */ cscan->custom_scan_tlist = - fix_scan_list(root, cscan->custom_scan_tlist, rtoffset); + fix_scan_list(root, cscan->custom_scan_tlist, + rtoffset, NUM_EXEC_TLIST((Plan *) cscan)); } else { /* Adjust tlist, qual, custom_exprs in the standard way */ cscan->scan.plan.targetlist = - fix_scan_list(root, cscan->scan.plan.targetlist, rtoffset); + fix_scan_list(root, cscan->scan.plan.targetlist, + rtoffset, NUM_EXEC_TLIST((Plan *) cscan)); cscan->scan.plan.qual = - fix_scan_list(root, cscan->scan.plan.qual, rtoffset); + fix_scan_list(root, cscan->scan.plan.qual, + rtoffset, NUM_EXEC_QUAL((Plan *) cscan)); cscan->custom_exprs = - fix_scan_list(root, cscan->custom_exprs, rtoffset); + fix_scan_list(root, cscan->custom_exprs, + rtoffset, NUM_EXEC_QUAL((Plan *) cscan)); } /* Adjust child plan-nodes recursively, if needed */ @@ -1458,7 +1543,8 @@ set_hash_references(PlannerInfo *root, Plan *plan, int rtoffset) (Node *) hplan->hashkeys, outer_itlist, OUTER_VAR, - rtoffset); + rtoffset, + NUM_EXEC_QUAL(plan)); /* Hash doesn't project */ set_dummy_tlist_references(plan, rtoffset); @@ -1623,6 +1709,69 @@ fix_param_node(PlannerInfo *root, Param *p) return (Node *) copyObject(p); } +/* + * fix_alternative_subplan + * Do set_plan_references processing on an AlternativeSubPlan + * + * Choose one of the alternative implementations and return just that one, + * discarding the rest of the AlternativeSubPlan structure. + * Note: caller must still recurse into the result! + * + * We don't make any attempt to fix up cost estimates in the parent plan + * node or higher-level nodes. However, we do remove the rejected subplan(s) + * from root->glob->subplans, to minimize cycles expended on them later. + */ +static Node * +fix_alternative_subplan(PlannerInfo *root, AlternativeSubPlan *asplan, + double num_exec) +{ + SubPlan *bestplan = NULL; + Cost bestcost = 0; + ListCell *lc; + + /* + * Compute the estimated cost of each subplan assuming num_exec + * executions, and keep the cheapest one. Replace discarded subplans with + * NULL pointers in the global subplans list. In event of exact equality + * of estimates, we prefer the later plan; this is a bit arbitrary, but in + * current usage it biases us to break ties against fast-start subplans. + */ + Assert(asplan->subplans != NIL); + + foreach(lc, asplan->subplans) + { + SubPlan *curplan = (SubPlan *) lfirst(lc); + Cost curcost; + + curcost = curplan->startup_cost + num_exec * curplan->per_call_cost; + if (bestplan == NULL) + { + bestplan = curplan; + bestcost = curcost; + } + else if (curcost <= bestcost) + { + /* drop old bestplan */ + ListCell *lc2 = list_nth_cell(root->glob->subplans, + bestplan->plan_id - 1); + + lfirst(lc2) = NULL; + bestplan = curplan; + bestcost = curcost; + } + else + { + /* drop curplan */ + ListCell *lc2 = list_nth_cell(root->glob->subplans, + curplan->plan_id - 1); + + lfirst(lc2) = NULL; + } + } + + return (Node *) bestplan; +} + /* * fix_scan_expr * Do set_plan_references processing on a scan-level expression @@ -1630,21 +1779,24 @@ fix_param_node(PlannerInfo *root, Param *p) * This consists of incrementing all Vars' varnos by rtoffset, * replacing PARAM_MULTIEXPR Params, expanding PlaceHolderVars, * replacing Aggref nodes that should be replaced by initplan output Params, + * choosing the best implementation for AlternativeSubPlans, * looking up operator opcode info for OpExpr and related nodes, * and adding OIDs from regclass Const nodes into root->glob->relationOids. */ static Node * -fix_scan_expr(PlannerInfo *root, Node *node, int rtoffset) +fix_scan_expr(PlannerInfo *root, Node *node, int rtoffset, double num_exec) { fix_scan_expr_context context; context.root = root; context.rtoffset = rtoffset; + context.num_exec = num_exec; if (rtoffset != 0 || root->multiexpr_params != NIL || root->glob->lastPHId != 0 || - root->minmax_aggs != NIL) + root->minmax_aggs != NIL || + root->hasAlternativeSubPlans) { return fix_scan_expr_mutator(node, &context); } @@ -1655,7 +1807,8 @@ fix_scan_expr(PlannerInfo *root, Node *node, int rtoffset) * are no MULTIEXPR subqueries then we don't need to replace * PARAM_MULTIEXPR Params, and if there are no placeholders anywhere * we won't need to remove them, and if there are no minmax Aggrefs we - * won't need to replace them. Then it's OK to just scribble on the + * won't need to replace them, and if there are no AlternativeSubPlans + * we won't need to remove them. Then it's OK to just scribble on the * input node tree instead of copying (since the only change, filling * in any unset opfuncid fields, is harmless). This saves just enough * cycles to be noticeable on trivial queries. @@ -1729,6 +1882,11 @@ fix_scan_expr_mutator(Node *node, fix_scan_expr_context *context) return fix_scan_expr_mutator((Node *) phv->phexpr, context); } + if (IsA(node, AlternativeSubPlan)) + return fix_scan_expr_mutator(fix_alternative_subplan(context->root, + (AlternativeSubPlan *) node, + context->num_exec), + context); fix_expr_common(context->root, node); return expression_tree_mutator(node, fix_scan_expr_mutator, (void *) context); @@ -1740,6 +1898,7 @@ fix_scan_expr_walker(Node *node, fix_scan_expr_context *context) if (node == NULL) return false; Assert(!IsA(node, PlaceHolderVar)); + Assert(!IsA(node, AlternativeSubPlan)); fix_expr_common(context->root, node); return expression_tree_walker(node, fix_scan_expr_walker, (void *) context); @@ -1776,7 +1935,8 @@ set_join_references(PlannerInfo *root, Join *join, int rtoffset) outer_itlist, inner_itlist, (Index) 0, - rtoffset); + rtoffset, + NUM_EXEC_QUAL((Plan *) join)); /* Now do join-type-specific stuff */ if (IsA(join, NestLoop)) @@ -1792,7 +1952,8 @@ set_join_references(PlannerInfo *root, Join *join, int rtoffset) (Node *) nlp->paramval, outer_itlist, OUTER_VAR, - rtoffset); + rtoffset, + NUM_EXEC_TLIST(outer_plan)); /* Check we replaced any PlaceHolderVar with simple Var */ if (!(IsA(nlp->paramval, Var) && nlp->paramval->varno == OUTER_VAR)) @@ -1808,7 +1969,8 @@ set_join_references(PlannerInfo *root, Join *join, int rtoffset) outer_itlist, inner_itlist, (Index) 0, - rtoffset); + rtoffset, + NUM_EXEC_QUAL((Plan *) join)); } else if (IsA(join, HashJoin)) { @@ -1819,7 +1981,8 @@ set_join_references(PlannerInfo *root, Join *join, int rtoffset) outer_itlist, inner_itlist, (Index) 0, - rtoffset); + rtoffset, + NUM_EXEC_QUAL((Plan *) join)); /* * HashJoin's hashkeys are used to look for matching tuples from its @@ -1829,7 +1992,8 @@ set_join_references(PlannerInfo *root, Join *join, int rtoffset) (Node *) hj->hashkeys, outer_itlist, OUTER_VAR, - rtoffset); + rtoffset, + NUM_EXEC_QUAL((Plan *) join)); } /* @@ -1867,13 +2031,15 @@ set_join_references(PlannerInfo *root, Join *join, int rtoffset) outer_itlist, inner_itlist, (Index) 0, - rtoffset); + rtoffset, + NUM_EXEC_TLIST((Plan *) join)); join->plan.qual = fix_join_expr(root, join->plan.qual, outer_itlist, inner_itlist, (Index) 0, - rtoffset); + rtoffset, + NUM_EXEC_QUAL((Plan *) join)); pfree(outer_itlist); pfree(inner_itlist); @@ -1926,14 +2092,16 @@ set_upper_references(PlannerInfo *root, Plan *plan, int rtoffset) (Node *) tle->expr, subplan_itlist, OUTER_VAR, - rtoffset); + rtoffset, + NUM_EXEC_TLIST(plan)); } else newexpr = fix_upper_expr(root, (Node *) tle->expr, subplan_itlist, OUTER_VAR, - rtoffset); + rtoffset, + NUM_EXEC_TLIST(plan)); tle = flatCopyTargetEntry(tle); tle->expr = (Expr *) newexpr; output_targetlist = lappend(output_targetlist, tle); @@ -1945,7 +2113,8 @@ set_upper_references(PlannerInfo *root, Plan *plan, int rtoffset) (Node *) plan->qual, subplan_itlist, OUTER_VAR, - rtoffset); + rtoffset, + NUM_EXEC_QUAL(plan)); pfree(subplan_itlist); } @@ -2389,6 +2558,7 @@ search_indexed_tlist_for_sortgroupref(Expr *node, * 'acceptable_rel' is either zero or the rangetable index of a relation * whose Vars may appear in the clause without provoking an error * 'rtoffset': how much to increment varnos by + * 'num_exec': estimated number of executions of expression * * Returns the new expression tree. The original clause structure is * not modified. @@ -2399,7 +2569,8 @@ fix_join_expr(PlannerInfo *root, indexed_tlist *outer_itlist, indexed_tlist *inner_itlist, Index acceptable_rel, - int rtoffset) + int rtoffset, + double num_exec) { fix_join_expr_context context; @@ -2408,6 +2579,7 @@ fix_join_expr(PlannerInfo *root, context.inner_itlist = inner_itlist; context.acceptable_rel = acceptable_rel; context.rtoffset = rtoffset; + context.num_exec = num_exec; return (List *) fix_join_expr_mutator((Node *) clauses, &context); } @@ -2502,6 +2674,11 @@ fix_join_expr_mutator(Node *node, fix_join_expr_context *context) /* Special cases (apply only AFTER failing to match to lower tlist) */ if (IsA(node, Param)) return fix_param_node(context->root, (Param *) node); + if (IsA(node, AlternativeSubPlan)) + return fix_join_expr_mutator(fix_alternative_subplan(context->root, + (AlternativeSubPlan *) node, + context->num_exec), + context); fix_expr_common(context->root, node); return expression_tree_mutator(node, fix_join_expr_mutator, @@ -2533,6 +2710,7 @@ fix_join_expr_mutator(Node *node, fix_join_expr_context *context) * 'subplan_itlist': indexed target list for subplan (or index) * 'newvarno': varno to use for Vars referencing tlist elements * 'rtoffset': how much to increment varnos by + * 'num_exec': estimated number of executions of expression * * The resulting tree is a copy of the original in which all Var nodes have * varno = newvarno, varattno = resno of corresponding targetlist element. @@ -2543,7 +2721,8 @@ fix_upper_expr(PlannerInfo *root, Node *node, indexed_tlist *subplan_itlist, Index newvarno, - int rtoffset) + int rtoffset, + double num_exec) { fix_upper_expr_context context; @@ -2551,6 +2730,7 @@ fix_upper_expr(PlannerInfo *root, context.subplan_itlist = subplan_itlist; context.newvarno = newvarno; context.rtoffset = rtoffset; + context.num_exec = num_exec; return fix_upper_expr_mutator(node, &context); } @@ -2623,6 +2803,11 @@ fix_upper_expr_mutator(Node *node, fix_upper_expr_context *context) } /* If no match, just fall through to process it normally */ } + if (IsA(node, AlternativeSubPlan)) + return fix_upper_expr_mutator(fix_alternative_subplan(context->root, + (AlternativeSubPlan *) node, + context->num_exec), + context); fix_expr_common(context->root, node); return expression_tree_mutator(node, fix_upper_expr_mutator, @@ -2687,7 +2872,8 @@ set_returning_clause_references(PlannerInfo *root, itlist, NULL, resultRelation, - rtoffset); + rtoffset, + NUM_EXEC_TLIST(topplan)); pfree(itlist); diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c index 6eb794669fe3..fcce81926b7d 100644 --- a/src/backend/optimizer/plan/subselect.c +++ b/src/backend/optimizer/plan/subselect.c @@ -81,6 +81,7 @@ static Node *convert_testexpr(PlannerInfo *root, static Node *convert_testexpr_mutator(Node *node, convert_testexpr_context *context); static bool subplan_is_hashable(Plan *plan); +static bool subpath_is_hashable(Path *path); static bool testexpr_is_hashable(Node *testexpr, List *param_ids); static bool test_opexpr_is_hashable(OpExpr *testexpr, List *param_ids); static bool hash_ok_operator(OpExpr *expr); @@ -247,7 +248,7 @@ make_subplan(PlannerInfo *root, Query *orig_subquery, * likely to be better (it depends on the expected number of executions of * the EXISTS qual, and we are much too early in planning the outer query * to be able to guess that). So we generate both plans, if possible, and - * leave it to the executor to decide which to use. + * leave it to setrefs.c to decide which to use. */ if (simple_exists && IsA(result, SubPlan)) { @@ -273,20 +274,20 @@ make_subplan(PlannerInfo *root, Query *orig_subquery, plan_params = root->plan_params; root->plan_params = NIL; - /* Select best Path and turn it into a Plan */ + /* Select best Path */ final_rel = fetch_upper_rel(subroot, UPPERREL_FINAL, NULL); best_path = final_rel->cheapest_total_path; - plan = create_plan(subroot, best_path); - /* Now we can check if it'll fit in hash_mem */ - /* XXX can we check this at the Path stage? */ - if (subplan_is_hashable(plan)) + if (subpath_is_hashable(best_path)) { SubPlan *hashplan; AlternativeSubPlan *asplan; - /* OK, convert to SubPlan format. */ + /* OK, finish planning the ANY subquery */ + plan = create_plan(subroot, best_path); + + /* ... and convert to SubPlan format */ hashplan = castNode(SubPlan, build_subplan(root, plan, subroot, plan_params, @@ -298,10 +299,11 @@ make_subplan(PlannerInfo *root, Query *orig_subquery, Assert(hashplan->parParam == NIL); Assert(hashplan->useHashTable); - /* Leave it to the executor to decide which plan to use */ + /* Leave it to setrefs.c to decide which plan to use */ asplan = makeNode(AlternativeSubPlan); asplan->subplans = list_make2(result, hashplan); result = (Node *) asplan; + root->hasAlternativeSubPlans = true; } } } @@ -714,6 +716,9 @@ convert_testexpr_mutator(Node *node, /* * subplan_is_hashable: can we implement an ANY subplan by hashing? + * + * This is not responsible for checking whether the combining testexpr + * is suitable for hashing. We only look at the subquery itself. */ static bool subplan_is_hashable(Plan *plan) @@ -735,6 +740,31 @@ subplan_is_hashable(Plan *plan) return true; } +/* + * subpath_is_hashable: can we implement an ANY subplan by hashing? + * + * Identical to subplan_is_hashable, but work from a Path for the subplan. + */ +static bool +subpath_is_hashable(Path *path) +{ + double subquery_size; + int hash_mem = get_hash_mem(); + + /* + * The estimated size of the subquery result must fit in hash_mem. (Note: + * we use heap tuple overhead here even though the tuples will actually be + * stored as MinimalTuples; this provides some fudge factor for hashtable + * overhead.) + */ + subquery_size = path->rows * + (MAXALIGN(path->pathtarget->width) + MAXALIGN(SizeofHeapTupleHeader)); + if (subquery_size > hash_mem * 1024L) + return false; + + return true; +} + /* * testexpr_is_hashable: is an ANY SubLink's test expression hashable? * diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index 15877e37a609..03cf24199637 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -8192,7 +8192,12 @@ get_rule_expr(Node *node, deparse_context *context, AlternativeSubPlan *asplan = (AlternativeSubPlan *) node; ListCell *lc; - /* As above, this can only happen during EXPLAIN */ + /* + * This case cannot be reached in normal usage, since no + * AlternativeSubPlan can appear either in parsetrees or + * finished plan trees. We keep it just in case somebody + * wants to use this code to print planner data structures. + */ appendStringInfoString(buf, "(alternatives: "); foreach(lc, asplan->subplans) { diff --git a/src/include/executor/execExpr.h b/src/include/executor/execExpr.h index dbe8649a5763..b792de1bc95e 100644 --- a/src/include/executor/execExpr.h +++ b/src/include/executor/execExpr.h @@ -218,7 +218,6 @@ typedef enum ExprEvalOp EEOP_GROUPING_FUNC, EEOP_WINDOW_FUNC, EEOP_SUBPLAN, - EEOP_ALTERNATIVE_SUBPLAN, /* aggregation related nodes */ EEOP_AGG_STRICT_DESERIALIZE, @@ -589,13 +588,6 @@ typedef struct ExprEvalStep SubPlanState *sstate; } subplan; - /* for EEOP_ALTERNATIVE_SUBPLAN */ - struct - { - /* out-of-line state, created by nodeSubplan.c */ - AlternativeSubPlanState *asstate; - } alternative_subplan; - /* for EEOP_AGG_*DESERIALIZE */ struct { @@ -734,8 +726,6 @@ extern void ExecEvalXmlExpr(ExprState *state, ExprEvalStep *op); extern void ExecEvalGroupingFunc(ExprState *state, ExprEvalStep *op); extern void ExecEvalSubPlan(ExprState *state, ExprEvalStep *op, ExprContext *econtext); -extern void ExecEvalAlternativeSubPlan(ExprState *state, ExprEvalStep *op, - ExprContext *econtext); extern void ExecEvalWholeRowVar(ExprState *state, ExprEvalStep *op, ExprContext *econtext); extern void ExecEvalSysVar(ExprState *state, ExprEvalStep *op, diff --git a/src/include/executor/nodeSubplan.h b/src/include/executor/nodeSubplan.h index 83e90b3d07b9..b629af1f5fb9 100644 --- a/src/include/executor/nodeSubplan.h +++ b/src/include/executor/nodeSubplan.h @@ -18,12 +18,8 @@ extern SubPlanState *ExecInitSubPlan(SubPlan *subplan, PlanState *parent); -extern AlternativeSubPlanState *ExecInitAlternativeSubPlan(AlternativeSubPlan *asplan, PlanState *parent); - extern Datum ExecSubPlan(SubPlanState *node, ExprContext *econtext, bool *isNull); -extern Datum ExecAlternativeSubPlan(AlternativeSubPlanState *node, ExprContext *econtext, bool *isNull); - extern void ExecReScanSetParamPlan(SubPlanState *node, PlanState *parent); extern void ExecSetParamPlan(SubPlanState *node, ExprContext *econtext); diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index a5ab1aed14d5..ef448d67c77d 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -877,18 +877,6 @@ typedef struct SubPlanState ExprState *cur_eq_comp; /* equality comparator for LHS vs. table */ } SubPlanState; -/* ---------------- - * AlternativeSubPlanState node - * ---------------- - */ -typedef struct AlternativeSubPlanState -{ - NodeTag type; - AlternativeSubPlan *subplan; /* expression plan node */ - List *subplans; /* SubPlanStates of alternative subplans */ - int active; /* list index of the one we're using */ -} AlternativeSubPlanState; - /* * DomainConstraintState - one item to check during CoerceToDomain * diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 381d84b4e4f8..7ddd8c011bfc 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -213,7 +213,6 @@ typedef enum NodeTag T_WindowFuncExprState, T_SetExprState, T_SubPlanState, - T_AlternativeSubPlanState, T_DomainConstraintState, /* diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 485d1b06c910..dbe86e7af657 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -347,6 +347,7 @@ struct PlannerInfo bool hasHavingQual; /* true if havingQual was non-null */ bool hasPseudoConstantQuals; /* true if any RestrictInfo has * pseudoconstant = true */ + bool hasAlternativeSubPlans; /* true if we've made any of those */ bool hasRecursion; /* true if planning a recursive WITH item */ /* These fields are used only when hasRecursion is true: */ diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h index d73be2ad46cc..fd65ee8f9c59 100644 --- a/src/include/nodes/primnodes.h +++ b/src/include/nodes/primnodes.h @@ -736,6 +736,9 @@ typedef struct SubPlan /* * AlternativeSubPlan - expression node for a choice among SubPlans * + * This is used only transiently during planning: by the time the plan + * reaches the executor, all AlternativeSubPlan nodes have been removed. + * * The subplans are given as a List so that the node definition need not * change if there's ever more than two alternatives. For the moment, * though, there are always exactly two; and the first one is the fast-start diff --git a/src/test/regress/expected/insert_conflict.out b/src/test/regress/expected/insert_conflict.out index 1338b2b23e17..ff157ceb1c19 100644 --- a/src/test/regress/expected/insert_conflict.out +++ b/src/test/regress/expected/insert_conflict.out @@ -50,14 +50,12 @@ explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on con Insert on insertconflicttest Conflict Resolution: UPDATE Conflict Arbiter Indexes: op_index_key, collation_index_key, both_index_key - Conflict Filter: (alternatives: SubPlan 1 or hashed SubPlan 2) + Conflict Filter: (SubPlan 1) -> Result SubPlan 1 -> Index Only Scan using both_index_expr_key on insertconflicttest ii Index Cond: (key = excluded.key) - SubPlan 2 - -> Seq Scan on insertconflicttest ii_1 -(10 rows) +(8 rows) -- Neither collation nor operator class specifications are required -- -- supplying them merely *limits* matches to indexes with matching opclasses diff --git a/src/test/regress/expected/subselect.out b/src/test/regress/expected/subselect.out index b81923f2e741..9d56cdacf37d 100644 --- a/src/test/regress/expected/subselect.out +++ b/src/test/regress/expected/subselect.out @@ -874,6 +874,53 @@ select * from int8_tbl where q1 in (select c1 from inner_text); (2 rows) rollback; -- to get rid of the bogus operator +-- +-- Test resolution of hashed vs non-hashed implementation of EXISTS subplan +-- +explain (costs off) +select count(*) from tenk1 t +where (exists(select 1 from tenk1 k where k.unique1 = t.unique2) or ten < 0); + QUERY PLAN +-------------------------------------------------------------- + Aggregate + -> Seq Scan on tenk1 t + Filter: ((hashed SubPlan 2) OR (ten < 0)) + SubPlan 2 + -> Index Only Scan using tenk1_unique1 on tenk1 k +(5 rows) + +select count(*) from tenk1 t +where (exists(select 1 from tenk1 k where k.unique1 = t.unique2) or ten < 0); + count +------- + 10000 +(1 row) + +explain (costs off) +select count(*) from tenk1 t +where (exists(select 1 from tenk1 k where k.unique1 = t.unique2) or ten < 0) + and thousand = 1; + QUERY PLAN +-------------------------------------------------------------- + Aggregate + -> Bitmap Heap Scan on tenk1 t + Recheck Cond: (thousand = 1) + Filter: ((SubPlan 1) OR (ten < 0)) + -> Bitmap Index Scan on tenk1_thous_tenthous + Index Cond: (thousand = 1) + SubPlan 1 + -> Index Only Scan using tenk1_unique1 on tenk1 k + Index Cond: (unique1 = t.unique2) +(9 rows) + +select count(*) from tenk1 t +where (exists(select 1 from tenk1 k where k.unique1 = t.unique2) or ten < 0) + and thousand = 1; + count +------- + 10 +(1 row) + -- -- Test case for planner bug with nested EXISTS handling -- diff --git a/src/test/regress/expected/updatable_views.out b/src/test/regress/expected/updatable_views.out index 5de53f2782aa..caed1c19ec76 100644 --- a/src/test/regress/expected/updatable_views.out +++ b/src/test/regress/expected/updatable_views.out @@ -1869,9 +1869,7 @@ EXPLAIN (costs off) INSERT INTO rw_view1 VALUES (5); SubPlan 1 -> Index Only Scan using ref_tbl_pkey on ref_tbl r Index Cond: (a = b.a) - SubPlan 2 - -> Seq Scan on ref_tbl r_1 -(7 rows) +(5 rows) EXPLAIN (costs off) UPDATE rw_view1 SET a = a + 5; QUERY PLAN @@ -1885,9 +1883,7 @@ EXPLAIN (costs off) UPDATE rw_view1 SET a = a + 5; SubPlan 1 -> Index Only Scan using ref_tbl_pkey on ref_tbl r_1 Index Cond: (a = b.a) - SubPlan 2 - -> Seq Scan on ref_tbl r_2 -(11 rows) +(9 rows) DROP TABLE base_tbl, ref_tbl CASCADE; NOTICE: drop cascades to view rw_view1 @@ -2301,8 +2297,8 @@ SELECT * FROM v1 WHERE a=8; EXPLAIN (VERBOSE, COSTS OFF) UPDATE v1 SET a=100 WHERE snoop(a) AND leakproof(a) AND a < 7 AND a != 6; - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------------------------- Update on public.t1 Update on public.t1 Update on public.t11 t1_1 @@ -2311,32 +2307,26 @@ UPDATE v1 SET a=100 WHERE snoop(a) AND leakproof(a) AND a < 7 AND a != 6; -> Index Scan using t1_a_idx on public.t1 Output: 100, t1.b, t1.c, t1.ctid Index Cond: ((t1.a > 5) AND (t1.a < 7)) - Filter: ((t1.a <> 6) AND (alternatives: SubPlan 1 or hashed SubPlan 2) AND snoop(t1.a) AND leakproof(t1.a)) + Filter: ((t1.a <> 6) AND (SubPlan 1) AND snoop(t1.a) AND leakproof(t1.a)) SubPlan 1 -> Append -> Seq Scan on public.t12 t12_1 Filter: (t12_1.a = t1.a) -> Seq Scan on public.t111 t12_2 Filter: (t12_2.a = t1.a) - SubPlan 2 - -> Append - -> Seq Scan on public.t12 t12_4 - Output: t12_4.a - -> Seq Scan on public.t111 t12_5 - Output: t12_5.a -> Index Scan using t11_a_idx on public.t11 t1_1 Output: 100, t1_1.b, t1_1.c, t1_1.d, t1_1.ctid Index Cond: ((t1_1.a > 5) AND (t1_1.a < 7)) - Filter: ((t1_1.a <> 6) AND (alternatives: SubPlan 1 or hashed SubPlan 2) AND snoop(t1_1.a) AND leakproof(t1_1.a)) + Filter: ((t1_1.a <> 6) AND (SubPlan 1) AND snoop(t1_1.a) AND leakproof(t1_1.a)) -> Index Scan using t12_a_idx on public.t12 t1_2 Output: 100, t1_2.b, t1_2.c, t1_2.e, t1_2.ctid Index Cond: ((t1_2.a > 5) AND (t1_2.a < 7)) - Filter: ((t1_2.a <> 6) AND (alternatives: SubPlan 1 or hashed SubPlan 2) AND snoop(t1_2.a) AND leakproof(t1_2.a)) + Filter: ((t1_2.a <> 6) AND (SubPlan 1) AND snoop(t1_2.a) AND leakproof(t1_2.a)) -> Index Scan using t111_a_idx on public.t111 t1_3 Output: 100, t1_3.b, t1_3.c, t1_3.d, t1_3.e, t1_3.ctid Index Cond: ((t1_3.a > 5) AND (t1_3.a < 7)) - Filter: ((t1_3.a <> 6) AND (alternatives: SubPlan 1 or hashed SubPlan 2) AND snoop(t1_3.a) AND leakproof(t1_3.a)) -(33 rows) + Filter: ((t1_3.a <> 6) AND (SubPlan 1) AND snoop(t1_3.a) AND leakproof(t1_3.a)) +(27 rows) UPDATE v1 SET a=100 WHERE snoop(a) AND leakproof(a) AND a < 7 AND a != 6; SELECT * FROM v1 WHERE a=100; -- Nothing should have been changed to 100 @@ -2351,8 +2341,8 @@ SELECT * FROM t1 WHERE a=100; -- Nothing should have been changed to 100 EXPLAIN (VERBOSE, COSTS OFF) UPDATE v1 SET a=a+1 WHERE snoop(a) AND leakproof(a) AND a = 8; - QUERY PLAN ---------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------- Update on public.t1 Update on public.t1 Update on public.t11 t1_1 @@ -2361,32 +2351,26 @@ UPDATE v1 SET a=a+1 WHERE snoop(a) AND leakproof(a) AND a = 8; -> Index Scan using t1_a_idx on public.t1 Output: (t1.a + 1), t1.b, t1.c, t1.ctid Index Cond: ((t1.a > 5) AND (t1.a = 8)) - Filter: ((alternatives: SubPlan 1 or hashed SubPlan 2) AND snoop(t1.a) AND leakproof(t1.a)) + Filter: ((SubPlan 1) AND snoop(t1.a) AND leakproof(t1.a)) SubPlan 1 -> Append -> Seq Scan on public.t12 t12_1 Filter: (t12_1.a = t1.a) -> Seq Scan on public.t111 t12_2 Filter: (t12_2.a = t1.a) - SubPlan 2 - -> Append - -> Seq Scan on public.t12 t12_4 - Output: t12_4.a - -> Seq Scan on public.t111 t12_5 - Output: t12_5.a -> Index Scan using t11_a_idx on public.t11 t1_1 Output: (t1_1.a + 1), t1_1.b, t1_1.c, t1_1.d, t1_1.ctid Index Cond: ((t1_1.a > 5) AND (t1_1.a = 8)) - Filter: ((alternatives: SubPlan 1 or hashed SubPlan 2) AND snoop(t1_1.a) AND leakproof(t1_1.a)) + Filter: ((SubPlan 1) AND snoop(t1_1.a) AND leakproof(t1_1.a)) -> Index Scan using t12_a_idx on public.t12 t1_2 Output: (t1_2.a + 1), t1_2.b, t1_2.c, t1_2.e, t1_2.ctid Index Cond: ((t1_2.a > 5) AND (t1_2.a = 8)) - Filter: ((alternatives: SubPlan 1 or hashed SubPlan 2) AND snoop(t1_2.a) AND leakproof(t1_2.a)) + Filter: ((SubPlan 1) AND snoop(t1_2.a) AND leakproof(t1_2.a)) -> Index Scan using t111_a_idx on public.t111 t1_3 Output: (t1_3.a + 1), t1_3.b, t1_3.c, t1_3.d, t1_3.e, t1_3.ctid Index Cond: ((t1_3.a > 5) AND (t1_3.a = 8)) - Filter: ((alternatives: SubPlan 1 or hashed SubPlan 2) AND snoop(t1_3.a) AND leakproof(t1_3.a)) -(33 rows) + Filter: ((SubPlan 1) AND snoop(t1_3.a) AND leakproof(t1_3.a)) +(27 rows) UPDATE v1 SET a=a+1 WHERE snoop(a) AND leakproof(a) AND a = 8; NOTICE: snooped value: 8 diff --git a/src/test/regress/sql/subselect.sql b/src/test/regress/sql/subselect.sql index cce8ebdb3d9f..a25cb6fc5c53 100644 --- a/src/test/regress/sql/subselect.sql +++ b/src/test/regress/sql/subselect.sql @@ -509,6 +509,23 @@ select * from int8_tbl where q1 in (select c1 from inner_text); rollback; -- to get rid of the bogus operator +-- +-- Test resolution of hashed vs non-hashed implementation of EXISTS subplan +-- +explain (costs off) +select count(*) from tenk1 t +where (exists(select 1 from tenk1 k where k.unique1 = t.unique2) or ten < 0); +select count(*) from tenk1 t +where (exists(select 1 from tenk1 k where k.unique1 = t.unique2) or ten < 0); + +explain (costs off) +select count(*) from tenk1 t +where (exists(select 1 from tenk1 k where k.unique1 = t.unique2) or ten < 0) + and thousand = 1; +select count(*) from tenk1 t +where (exists(select 1 from tenk1 k where k.unique1 = t.unique2) or ten < 0) + and thousand = 1; + -- -- Test case for planner bug with nested EXISTS handling -- From 4d29e6dbd0bb6d8c3a48d0f3c7d65dc1def1b07e Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 28 Sep 2020 10:13:59 +0900 Subject: [PATCH 208/589] Improve range checks of options for pg_test_fsync and pg_test_timing Both tools never had safeguard checks for the options provided, and it was possible to make pg_test_fsync run an infinite amount of time or pass down buggy values to pg_test_timing. These behaviors have existed for a long time, with no actual complaints, so no backpatch is done. Basic TAP tests are introduced for both tools. Author: Michael Paquier Reviewed-by: Peter Eisentraut Discussion: https://postgr.es/m/20200806062759.GE16470@paquier.xyz --- src/bin/pg_test_fsync/.gitignore | 2 + src/bin/pg_test_fsync/Makefile | 6 +++ src/bin/pg_test_fsync/pg_test_fsync.c | 28 ++++++++++++-- src/bin/pg_test_fsync/t/001_basic.pl | 25 ++++++++++++ src/bin/pg_test_timing/.gitignore | 2 + src/bin/pg_test_timing/Makefile | 6 +++ src/bin/pg_test_timing/pg_test_timing.c | 51 +++++++++++++++---------- src/bin/pg_test_timing/t/001_basic.pl | 25 ++++++++++++ 8 files changed, 121 insertions(+), 24 deletions(-) create mode 100644 src/bin/pg_test_fsync/t/001_basic.pl create mode 100644 src/bin/pg_test_timing/t/001_basic.pl diff --git a/src/bin/pg_test_fsync/.gitignore b/src/bin/pg_test_fsync/.gitignore index f3b593249859..5eb5085f4524 100644 --- a/src/bin/pg_test_fsync/.gitignore +++ b/src/bin/pg_test_fsync/.gitignore @@ -1 +1,3 @@ /pg_test_fsync + +/tmp_check/ diff --git a/src/bin/pg_test_fsync/Makefile b/src/bin/pg_test_fsync/Makefile index 7632c94eb7f6..c4f9ae066484 100644 --- a/src/bin/pg_test_fsync/Makefile +++ b/src/bin/pg_test_fsync/Makefile @@ -22,6 +22,12 @@ install: all installdirs installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' +check: + $(prove_check) + +installcheck: + $(prove_installcheck) + uninstall: rm -f '$(DESTDIR)$(bindir)/pg_test_fsync$(X)' diff --git a/src/bin/pg_test_fsync/pg_test_fsync.c b/src/bin/pg_test_fsync/pg_test_fsync.c index 6e4729312331..3eddd983c63b 100644 --- a/src/bin/pg_test_fsync/pg_test_fsync.c +++ b/src/bin/pg_test_fsync/pg_test_fsync.c @@ -5,6 +5,7 @@ #include "postgres_fe.h" +#include #include #include #include @@ -62,7 +63,7 @@ do { \ static const char *progname; -static int secs_per_test = 5; +static unsigned int secs_per_test = 5; static int needs_unlink = 0; static char full_buf[DEFAULT_XLOG_SEG_SIZE], *buf, @@ -148,6 +149,8 @@ handle_args(int argc, char *argv[]) int option; /* Command line option */ int optindex = 0; /* used by getopt_long */ + unsigned long optval; /* used for option parsing */ + char *endptr; if (argc > 1) { @@ -173,7 +176,24 @@ handle_args(int argc, char *argv[]) break; case 's': - secs_per_test = atoi(optarg); + errno = 0; + optval = strtoul(optarg, &endptr, 10); + + if (endptr == optarg || *endptr != '\0' || + errno != 0 || optval != (unsigned int) optval) + { + pg_log_error("invalid argument for option %s", "--secs-per-test"); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); + } + + secs_per_test = (unsigned int) optval; + if (secs_per_test == 0) + { + pg_log_error("%s must be in range %u..%u", + "--secs-per-test", 1, UINT_MAX); + exit(1); + } break; default: @@ -193,8 +213,8 @@ handle_args(int argc, char *argv[]) exit(1); } - printf(ngettext("%d second per test\n", - "%d seconds per test\n", + printf(ngettext("%u second per test\n", + "%u seconds per test\n", secs_per_test), secs_per_test); #if PG_O_DIRECT != 0 diff --git a/src/bin/pg_test_fsync/t/001_basic.pl b/src/bin/pg_test_fsync/t/001_basic.pl new file mode 100644 index 000000000000..fe9c295c4976 --- /dev/null +++ b/src/bin/pg_test_fsync/t/001_basic.pl @@ -0,0 +1,25 @@ +use strict; +use warnings; + +use Config; +use TestLib; +use Test::More tests => 12; + +######################################### +# Basic checks + +program_help_ok('pg_test_fsync'); +program_version_ok('pg_test_fsync'); +program_options_handling_ok('pg_test_fsync'); + +######################################### +# Test invalid option combinations + +command_fails_like( + [ 'pg_test_fsync', '--secs-per-test', 'a' ], + qr/\Qpg_test_fsync: error: invalid argument for option --secs-per-test\E/, + 'pg_test_fsync: invalid argument for option --secs-per-test'); +command_fails_like( + [ 'pg_test_fsync', '--secs-per-test', '0' ], + qr/\Qpg_test_fsync: error: --secs-per-test must be in range 1..4294967295\E/, + 'pg_test_fsync: --secs-per-test must be in range'); diff --git a/src/bin/pg_test_timing/.gitignore b/src/bin/pg_test_timing/.gitignore index f6c664c76576..e5aac2ab120f 100644 --- a/src/bin/pg_test_timing/.gitignore +++ b/src/bin/pg_test_timing/.gitignore @@ -1 +1,3 @@ /pg_test_timing + +/tmp_check/ diff --git a/src/bin/pg_test_timing/Makefile b/src/bin/pg_test_timing/Makefile index 334d6ff5c00d..52994b4103c3 100644 --- a/src/bin/pg_test_timing/Makefile +++ b/src/bin/pg_test_timing/Makefile @@ -22,6 +22,12 @@ install: all installdirs installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' +check: + $(prove_check) + +installcheck: + $(prove_installcheck) + uninstall: rm -f '$(DESTDIR)$(bindir)/pg_test_timing$(X)' diff --git a/src/bin/pg_test_timing/pg_test_timing.c b/src/bin/pg_test_timing/pg_test_timing.c index e14802372bd6..c29d6f876294 100644 --- a/src/bin/pg_test_timing/pg_test_timing.c +++ b/src/bin/pg_test_timing/pg_test_timing.c @@ -6,15 +6,17 @@ #include "postgres_fe.h" +#include + #include "getopt_long.h" #include "portability/instr_time.h" static const char *progname; -static int32 test_duration = 3; +static unsigned int test_duration = 3; static void handle_args(int argc, char *argv[]); -static uint64 test_timing(int32); +static uint64 test_timing(unsigned int duration); static void output(uint64 loop_count); /* record duration in powers of 2 microseconds */ @@ -47,6 +49,8 @@ handle_args(int argc, char *argv[]) int option; /* Command line option */ int optindex = 0; /* used by getopt_long */ + unsigned long optval; /* used for option parsing */ + char *endptr; if (argc > 1) { @@ -68,7 +72,25 @@ handle_args(int argc, char *argv[]) switch (option) { case 'd': - test_duration = atoi(optarg); + errno = 0; + optval = strtoul(optarg, &endptr, 10); + + if (endptr == optarg || *endptr != '\0' || + errno != 0 || optval != (unsigned int) optval) + { + fprintf(stderr, _("%s: invalid argument for option %s\n"), + progname, "--duration"); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); + } + + test_duration = (unsigned int) optval; + if (test_duration == 0) + { + fprintf(stderr, _("%s: %s must be in range %u..%u\n"), + progname, "--duration", 1, UINT_MAX); + exit(1); + } break; default: @@ -89,26 +111,15 @@ handle_args(int argc, char *argv[]) exit(1); } - if (test_duration > 0) - { - printf(ngettext("Testing timing overhead for %d second.\n", - "Testing timing overhead for %d seconds.\n", - test_duration), - test_duration); - } - else - { - fprintf(stderr, - _("%s: duration must be a positive integer (duration is \"%d\")\n"), - progname, test_duration); - fprintf(stderr, _("Try \"%s --help\" for more information.\n"), - progname); - exit(1); - } + + printf(ngettext("Testing timing overhead for %u second.\n", + "Testing timing overhead for %u seconds.\n", + test_duration), + test_duration); } static uint64 -test_timing(int32 duration) +test_timing(unsigned int duration) { uint64 total_time; int64 time_elapsed = 0; diff --git a/src/bin/pg_test_timing/t/001_basic.pl b/src/bin/pg_test_timing/t/001_basic.pl new file mode 100644 index 000000000000..8bad19c7fad9 --- /dev/null +++ b/src/bin/pg_test_timing/t/001_basic.pl @@ -0,0 +1,25 @@ +use strict; +use warnings; + +use Config; +use TestLib; +use Test::More tests => 12; + +######################################### +# Basic checks + +program_help_ok('pg_test_timing'); +program_version_ok('pg_test_timing'); +program_options_handling_ok('pg_test_timing'); + +######################################### +# Test invalid option combinations + +command_fails_like( + [ 'pg_test_timing', '--duration', 'a' ], + qr/\Qpg_test_timing: invalid argument for option --duration\E/, + 'pg_test_timing: invalid argument for option --duration'); +command_fails_like( + [ 'pg_test_timing', '--duration', '0' ], + qr/\Qpg_test_timing: --duration must be in range 1..4294967295\E/, + 'pg_test_timing: --duration must be in range'); From cc99baa43e0ed83ac18fcbde31f2ab7274eb26cf Mon Sep 17 00:00:00 2001 From: David Rowley Date: Mon, 28 Sep 2020 14:47:19 +1300 Subject: [PATCH 209/589] Improve pg_list.h's linitial(), lsecond() and co macros Prior to this commit, the linitial(), lsecond(), lthird(), lfourth() macros and their int and Oid list cousins would call their corresponding inlined function to fetch the cell of interest. Those inline functions were kind enough to return NULL if the particular cell did not exist. Unfortunately, the care that these functions took was of no relevance to the calling macros as they proceeded to directly dereference the returned value without any regard to whether that value was NULL or not. If it had been, we'd have segfaulted. Of course, the fact that we would have segfaulted on misuse of these macros just goes to prove that nobody is relying on the empty or list too small checks. So here we just get rid of those checks completely. The existing inline functions have been left alone as someone may be using those directly. We just replace the call within each macro to use list_nth_cell(). For the llast*() case we require a new list_last_cell() inline function to get away from the multiple evaluation hazard that we'd get if we fetched ->length on the macro's parameter. Author: David Rowley Reviewed-by: Tom Lane Discussion: https://postgr.es/m/CAApHDvpo1zj9KhEpU2cCRZfSM3Q6XGdhzuAS2v79PH7WJBkYVA@mail.gmail.com --- src/include/nodes/pg_list.h | 41 ++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/src/include/nodes/pg_list.h b/src/include/nodes/pg_list.h index 14ea2766adb3..104df4174abc 100644 --- a/src/include/nodes/pg_list.h +++ b/src/include/nodes/pg_list.h @@ -186,35 +186,34 @@ list_length(const List *l) * linitial() than lfirst(): given a List, lsecond() returns the data * in the second list cell. */ - #define lfirst(lc) ((lc)->ptr_value) #define lfirst_int(lc) ((lc)->int_value) #define lfirst_oid(lc) ((lc)->oid_value) #define lfirst_node(type,lc) castNode(type, lfirst(lc)) -#define linitial(l) lfirst(list_head(l)) -#define linitial_int(l) lfirst_int(list_head(l)) -#define linitial_oid(l) lfirst_oid(list_head(l)) +#define linitial(l) lfirst(list_nth_cell(l, 0)) +#define linitial_int(l) lfirst_int(list_nth_cell(l, 0)) +#define linitial_oid(l) lfirst_oid(list_nth_cell(l, 0)) #define linitial_node(type,l) castNode(type, linitial(l)) -#define lsecond(l) lfirst(list_second_cell(l)) -#define lsecond_int(l) lfirst_int(list_second_cell(l)) -#define lsecond_oid(l) lfirst_oid(list_second_cell(l)) +#define lsecond(l) lfirst(list_nth_cell(l, 1)) +#define lsecond_int(l) lfirst_int(list_nth_cell(l, 1)) +#define lsecond_oid(l) lfirst_oid(list_nth_cell(l, 1)) #define lsecond_node(type,l) castNode(type, lsecond(l)) -#define lthird(l) lfirst(list_third_cell(l)) -#define lthird_int(l) lfirst_int(list_third_cell(l)) -#define lthird_oid(l) lfirst_oid(list_third_cell(l)) +#define lthird(l) lfirst(list_nth_cell(l, 2)) +#define lthird_int(l) lfirst_int(list_nth_cell(l, 2)) +#define lthird_oid(l) lfirst_oid(list_nth_cell(l, 2)) #define lthird_node(type,l) castNode(type, lthird(l)) -#define lfourth(l) lfirst(list_fourth_cell(l)) -#define lfourth_int(l) lfirst_int(list_fourth_cell(l)) -#define lfourth_oid(l) lfirst_oid(list_fourth_cell(l)) +#define lfourth(l) lfirst(list_nth_cell(l, 3)) +#define lfourth_int(l) lfirst_int(list_nth_cell(l, 3)) +#define lfourth_oid(l) lfirst_oid(list_nth_cell(l, 3)) #define lfourth_node(type,l) castNode(type, lfourth(l)) -#define llast(l) lfirst(list_tail(l)) -#define llast_int(l) lfirst_int(list_tail(l)) -#define llast_oid(l) lfirst_oid(list_tail(l)) +#define llast(l) lfirst(list_last_cell(l)) +#define llast_int(l) lfirst_int(list_last_cell(l)) +#define llast_oid(l) lfirst_oid(list_last_cell(l)) #define llast_node(type,l) castNode(type, llast(l)) /* @@ -269,6 +268,16 @@ list_nth_cell(const List *list, int n) return &list->elements[n]; } +/* + * Return the last cell in a non-NIL List. + */ +static inline ListCell * +list_last_cell(const List *list) +{ + Assert(list != NIL); + return &list->elements[list->length - 1]; +} + /* * Return the pointer value contained in the n'th element of the * specified list. (List elements begin at 0.) From 0baf82fa0cbe98fe317483f1e1c6612fc41e1dad Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Mon, 28 Sep 2020 11:23:15 +0900 Subject: [PATCH 210/589] Improve tab-completion for DEALLOCATE. Author: Naoki Nakamichi Reviewed-by: Fujii Masao Discussion: https://postgr.es/m/ec1a45b06edfce13706f2c765778d8c2@oss.nttdata.com --- src/bin/psql/tab-complete.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 9c6f5ecb6a8c..24c7b414cf3a 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -2911,7 +2911,8 @@ psql_completion(const char *text, int start, int end) /* DEALLOCATE */ else if (Matches("DEALLOCATE")) - COMPLETE_WITH_QUERY(Query_for_list_of_prepared_statements); + COMPLETE_WITH_QUERY(Query_for_list_of_prepared_statements + " UNION SELECT 'ALL'"); /* DECLARE */ else if (Matches("DECLARE", MatchAny)) From 9d299a492454f9ffdf0fd8fe1fd6303c8ddf805a Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 27 Sep 2020 22:30:44 -0400 Subject: [PATCH 211/589] Minor mop-up for List improvements. Fix a few places that were using written-out versions of the pg_list.h macros that commit cc99baa43 just improved, making them also use those macros so as to gain whatever performance improvement is to be had. Discussion: https://postgr.es/m/CAApHDvpo1zj9KhEpU2cCRZfSM3Q6XGdhzuAS2v79PH7WJBkYVA@mail.gmail.com --- src/backend/catalog/objectaddress.c | 2 +- src/backend/commands/sequence.c | 2 +- src/backend/nodes/list.c | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c index 6dfe1be2cc00..4815f6ca7e3a 100644 --- a/src/backend/catalog/objectaddress.c +++ b/src/backend/catalog/objectaddress.c @@ -1506,7 +1506,7 @@ get_object_address_attribute(ObjectType objtype, List *object, ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("column name must be qualified"))); - attname = strVal(lfirst(list_tail(object))); + attname = strVal(llast(object)); relname = list_truncate(list_copy(object), list_length(object) - 1); /* XXX no missing_ok support here */ relation = relation_openrv(makeRangeVarFromNameList(relname), lockmode); diff --git a/src/backend/commands/sequence.c b/src/backend/commands/sequence.c index 6aab73bfd447..632b34af6100 100644 --- a/src/backend/commands/sequence.c +++ b/src/backend/commands/sequence.c @@ -1668,7 +1668,7 @@ process_owned_by(Relation seqrel, List *owned_by, bool for_identity) /* Separate relname and attr name */ relname = list_truncate(list_copy(owned_by), nnames - 1); - attrname = strVal(lfirst(list_tail(owned_by))); + attrname = strVal(llast(owned_by)); /* Open and lock rel to ensure it won't go away meanwhile */ rel = makeRangeVarFromNameList(relname); diff --git a/src/backend/nodes/list.c b/src/backend/nodes/list.c index 80fa8c84e49a..efa44342c4b8 100644 --- a/src/backend/nodes/list.c +++ b/src/backend/nodes/list.c @@ -327,7 +327,7 @@ lappend(List *list, void *datum) else new_tail_cell(list); - lfirst(list_tail(list)) = datum; + llast(list) = datum; check_list_invariants(list); return list; } @@ -345,7 +345,7 @@ lappend_int(List *list, int datum) else new_tail_cell(list); - lfirst_int(list_tail(list)) = datum; + llast_int(list) = datum; check_list_invariants(list); return list; } @@ -363,7 +363,7 @@ lappend_oid(List *list, Oid datum) else new_tail_cell(list); - lfirst_oid(list_tail(list)) = datum; + llast_oid(list) = datum; check_list_invariants(list); return list; } @@ -459,7 +459,7 @@ lcons(void *datum, List *list) else new_head_cell(list); - lfirst(list_head(list)) = datum; + linitial(list) = datum; check_list_invariants(list); return list; } @@ -477,7 +477,7 @@ lcons_int(int datum, List *list) else new_head_cell(list); - lfirst_int(list_head(list)) = datum; + linitial_int(list) = datum; check_list_invariants(list); return list; } @@ -495,7 +495,7 @@ lcons_oid(Oid datum, List *list) else new_head_cell(list); - lfirst_oid(list_head(list)) = datum; + linitial_oid(list) = datum; check_list_invariants(list); return list; } From e21cbb4b893b85b5f1cf203b9a77ca0d9ee671d1 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 28 Sep 2020 12:47:13 +0900 Subject: [PATCH 212/589] Change SHA2 implementation based on OpenSSL to use EVP digest routines The use of low-level hash routines is not recommended by upstream OpenSSL since 2000, and pgcrypto already switched to EVP as of 5ff4a67. Note that this also fixes a failure with SCRAM authentication when using FIPS in OpenSSL, but as there have been few complaints about this problem and as this causes an ABI breakage, no backpatch is done. Author: Michael Paquier, Alessandro Gherardi Reviewed-by: Daniel Gustafsson Discussion: https://postgr.es/m/20200924025314.GE7405@paquier.xyz Discussion: https://postgr.es/m/20180911030250.GA27115@paquier.xyz --- src/common/sha2_openssl.c | 63 ++++++++++++++++++++++++++++++--------- src/include/common/sha2.h | 10 +++---- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/src/common/sha2_openssl.c b/src/common/sha2_openssl.c index 41673b3a8886..30f5f816f8ea 100644 --- a/src/common/sha2_openssl.c +++ b/src/common/sha2_openssl.c @@ -20,83 +20,118 @@ #include "postgres_fe.h" #endif -#include - #include "common/sha2.h" +#ifdef FRONTEND +#include "common/logging.h" +#else +#include "miscadmin.h" +#endif + +#ifdef FRONTEND +#define sha2_log_and_abort(...) \ + do { pg_log_fatal(__VA_ARGS__); exit(1); } while(0) +#else +#define sha2_log_and_abort(...) elog(ERROR, __VA_ARGS__) +#endif + +static void +digest_init(EVP_MD_CTX **ctx, const EVP_MD *type) +{ + *ctx = EVP_MD_CTX_create(); + if (*ctx == NULL) + sha2_log_and_abort("could not create EVP digest context"); + if (EVP_DigestInit_ex(*ctx, type, NULL) <= 0) + sha2_log_and_abort("could not initialize EVP digest context"); +} + +static void +digest_update(EVP_MD_CTX **ctx, const uint8 *data, size_t len) +{ + if (EVP_DigestUpdate(*ctx, data, len) <= 0) + sha2_log_and_abort("could not update EVP digest context"); +} + +static void +digest_final(EVP_MD_CTX **ctx, uint8 *dest) +{ + if (EVP_DigestFinal_ex(*ctx, dest, 0) <= 0) + sha2_log_and_abort("could not finalize EVP digest context"); + EVP_MD_CTX_destroy(*ctx); +} /* Interface routines for SHA-256 */ void pg_sha256_init(pg_sha256_ctx *ctx) { - SHA256_Init((SHA256_CTX *) ctx); + digest_init(ctx, EVP_sha256()); } void pg_sha256_update(pg_sha256_ctx *ctx, const uint8 *data, size_t len) { - SHA256_Update((SHA256_CTX *) ctx, data, len); + digest_update(ctx, data, len); } void pg_sha256_final(pg_sha256_ctx *ctx, uint8 *dest) { - SHA256_Final(dest, (SHA256_CTX *) ctx); + digest_final(ctx, dest); } /* Interface routines for SHA-512 */ void pg_sha512_init(pg_sha512_ctx *ctx) { - SHA512_Init((SHA512_CTX *) ctx); + digest_init(ctx, EVP_sha512()); } void pg_sha512_update(pg_sha512_ctx *ctx, const uint8 *data, size_t len) { - SHA512_Update((SHA512_CTX *) ctx, data, len); + digest_update(ctx, data, len); } void pg_sha512_final(pg_sha512_ctx *ctx, uint8 *dest) { - SHA512_Final(dest, (SHA512_CTX *) ctx); + digest_final(ctx, dest); } /* Interface routines for SHA-384 */ void pg_sha384_init(pg_sha384_ctx *ctx) { - SHA384_Init((SHA512_CTX *) ctx); + digest_init(ctx, EVP_sha384()); } void pg_sha384_update(pg_sha384_ctx *ctx, const uint8 *data, size_t len) { - SHA384_Update((SHA512_CTX *) ctx, data, len); + digest_update(ctx, data, len); } void pg_sha384_final(pg_sha384_ctx *ctx, uint8 *dest) { - SHA384_Final(dest, (SHA512_CTX *) ctx); + digest_final(ctx, dest); } /* Interface routines for SHA-224 */ void pg_sha224_init(pg_sha224_ctx *ctx) { - SHA224_Init((SHA256_CTX *) ctx); + digest_init(ctx, EVP_sha224()); } void pg_sha224_update(pg_sha224_ctx *ctx, const uint8 *data, size_t len) { - SHA224_Update((SHA256_CTX *) ctx, data, len); + digest_update(ctx, data, len); } void pg_sha224_final(pg_sha224_ctx *ctx, uint8 *dest) { - SHA224_Final(dest, (SHA256_CTX *) ctx); + digest_final(ctx, dest); } diff --git a/src/include/common/sha2.h b/src/include/common/sha2.h index 9c4abf777d43..2c5283816153 100644 --- a/src/include/common/sha2.h +++ b/src/include/common/sha2.h @@ -51,7 +51,7 @@ #define _PG_SHA2_H_ #ifdef USE_OPENSSL -#include +#include #endif /*** SHA224/256/384/512 Various Length Definitions ***********************/ @@ -70,10 +70,10 @@ /* Context Structures for SHA224/256/384/512 */ #ifdef USE_OPENSSL -typedef SHA256_CTX pg_sha256_ctx; -typedef SHA512_CTX pg_sha512_ctx; -typedef SHA256_CTX pg_sha224_ctx; -typedef SHA512_CTX pg_sha384_ctx; +typedef EVP_MD_CTX *pg_sha256_ctx; +typedef EVP_MD_CTX *pg_sha512_ctx; +typedef EVP_MD_CTX *pg_sha224_ctx; +typedef EVP_MD_CTX *pg_sha384_ctx; #else typedef struct pg_sha256_ctx { From 0a87ddff5c83589e90de236bd55e6a19b017fe9a Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 28 Sep 2020 12:05:03 -0400 Subject: [PATCH 213/589] Cache the result of converting now() to a struct pg_tm. SQL operations such as CURRENT_DATE, CURRENT_TIME, LOCALTIME, and conversion of "now" in a datetime input string have to obtain the transaction start timestamp ("now()") as a broken-down struct pg_tm. This is a remarkably expensive conversion, and since now() does not change intra-transaction, it doesn't really need to be done more than once per transaction. Introducing a simple cache provides visible speedups in queries that compute these values many times, for example insertion of many rows that use a default value of CURRENT_DATE. Peter Smith, with a bit of kibitzing by me Discussion: https://postgr.es/m/CAHut+Pu89TWjq530V2gY5O6SWi=OEJMQ_VHMt8bdZB_9JFna5A@mail.gmail.com --- src/backend/utils/adt/date.c | 49 ++++++++++++----------- src/backend/utils/adt/datetime.c | 67 ++++++++++++++++++++++++++------ 2 files changed, 80 insertions(+), 36 deletions(-) diff --git a/src/backend/utils/adt/date.c b/src/backend/utils/adt/date.c index eaaffa7137dc..057051fa8550 100644 --- a/src/backend/utils/adt/date.c +++ b/src/backend/utils/adt/date.c @@ -299,20 +299,31 @@ EncodeSpecialDate(DateADT dt, char *str) DateADT GetSQLCurrentDate(void) { - TimestampTz ts; - struct pg_tm tt, - *tm = &tt; - fsec_t fsec; - int tz; + struct pg_tm tm; - ts = GetCurrentTransactionStartTimestamp(); + static int cache_year = 0; + static int cache_mon = 0; + static int cache_mday = 0; + static DateADT cache_date; - if (timestamp2tm(ts, &tz, tm, &fsec, NULL, NULL) != 0) - ereport(ERROR, - (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), - errmsg("timestamp out of range"))); + GetCurrentDateTime(&tm); - return date2j(tm->tm_year, tm->tm_mon, tm->tm_mday) - POSTGRES_EPOCH_JDATE; + /* + * date2j involves several integer divisions; moreover, unless our session + * lives across local midnight, we don't really have to do it more than + * once. So it seems worth having a separate cache here. + */ + if (tm.tm_year != cache_year || + tm.tm_mon != cache_mon || + tm.tm_mday != cache_mday) + { + cache_date = date2j(tm.tm_year, tm.tm_mon, tm.tm_mday) - POSTGRES_EPOCH_JDATE; + cache_year = tm.tm_year; + cache_mon = tm.tm_mon; + cache_mday = tm.tm_mday; + } + + return cache_date; } /* @@ -322,18 +333,12 @@ TimeTzADT * GetSQLCurrentTime(int32 typmod) { TimeTzADT *result; - TimestampTz ts; struct pg_tm tt, *tm = &tt; fsec_t fsec; int tz; - ts = GetCurrentTransactionStartTimestamp(); - - if (timestamp2tm(ts, &tz, tm, &fsec, NULL, NULL) != 0) - ereport(ERROR, - (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), - errmsg("timestamp out of range"))); + GetCurrentTimeUsec(tm, &fsec, &tz); result = (TimeTzADT *) palloc(sizeof(TimeTzADT)); tm2timetz(tm, fsec, tz, result); @@ -348,18 +353,12 @@ TimeADT GetSQLLocalTime(int32 typmod) { TimeADT result; - TimestampTz ts; struct pg_tm tt, *tm = &tt; fsec_t fsec; int tz; - ts = GetCurrentTransactionStartTimestamp(); - - if (timestamp2tm(ts, &tz, tm, &fsec, NULL, NULL) != 0) - ereport(ERROR, - (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), - errmsg("timestamp out of range"))); + GetCurrentTimeUsec(tm, &fsec, &tz); tm2time(tm, fsec, &result); AdjustTimeForTypmod(&result, typmod); diff --git a/src/backend/utils/adt/datetime.c b/src/backend/utils/adt/datetime.c index dec2fad82a68..91fab8cc9cb3 100644 --- a/src/backend/utils/adt/datetime.c +++ b/src/backend/utils/adt/datetime.c @@ -339,35 +339,80 @@ j2day(int date) /* * GetCurrentDateTime() * - * Get the transaction start time ("now()") broken down as a struct pg_tm. + * Get the transaction start time ("now()") broken down as a struct pg_tm, + * converted according to the session timezone setting. + * + * This is just a convenience wrapper for GetCurrentTimeUsec, to cover the + * case where caller doesn't need either fractional seconds or tz offset. */ void GetCurrentDateTime(struct pg_tm *tm) { - int tz; fsec_t fsec; - timestamp2tm(GetCurrentTransactionStartTimestamp(), &tz, tm, &fsec, - NULL, NULL); - /* Note: don't pass NULL tzp to timestamp2tm; affects behavior */ + GetCurrentTimeUsec(tm, &fsec, NULL); } /* * GetCurrentTimeUsec() * * Get the transaction start time ("now()") broken down as a struct pg_tm, - * including fractional seconds and timezone offset. + * including fractional seconds and timezone offset. The time is converted + * according to the session timezone setting. + * + * Callers may pass tzp = NULL if they don't need the offset, but this does + * not affect the conversion behavior (unlike timestamp2tm()). + * + * Internally, we cache the result, since this could be called many times + * in a transaction, within which now() doesn't change. */ void GetCurrentTimeUsec(struct pg_tm *tm, fsec_t *fsec, int *tzp) { - int tz; + TimestampTz cur_ts = GetCurrentTransactionStartTimestamp(); + + /* + * The cache key must include both current time and current timezone. By + * representing the timezone by just a pointer, we're assuming that + * distinct timezone settings could never have the same pointer value. + * This is true by virtue of the hashtable used inside pg_tzset(); + * however, it might need another look if we ever allow entries in that + * hash to be recycled. + */ + static TimestampTz cache_ts = 0; + static pg_tz *cache_timezone = NULL; + static struct pg_tm cache_tm; + static fsec_t cache_fsec; + static int cache_tz; + + if (cur_ts != cache_ts || session_timezone != cache_timezone) + { + /* + * Make sure cache is marked invalid in case of error after partial + * update within timestamp2tm. + */ + cache_timezone = NULL; + + /* + * Perform the computation, storing results into cache. We do not + * really expect any error here, since current time surely ought to be + * within range, but check just for sanity's sake. + */ + if (timestamp2tm(cur_ts, &cache_tz, &cache_tm, &cache_fsec, + NULL, session_timezone) != 0) + ereport(ERROR, + (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), + errmsg("timestamp out of range"))); + + /* OK, so mark the cache valid. */ + cache_ts = cur_ts; + cache_timezone = session_timezone; + } - timestamp2tm(GetCurrentTransactionStartTimestamp(), &tz, tm, fsec, - NULL, NULL); - /* Note: don't pass NULL tzp to timestamp2tm; affects behavior */ + *tm = cache_tm; + *fsec = cache_fsec; if (tzp != NULL) - *tzp = tz; + *tzp = cache_tz; } From 2dfa3fea88bc951d0812a18649d801f07964c9b9 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 28 Sep 2020 13:44:01 -0400 Subject: [PATCH 214/589] Remove complaints about COLLATE clauses in partition bound values. transformPartitionBoundValue went out of its way to do the wrong thing: there is no reason to complain about a non-matching COLLATE clause in a partition boundary expression. We're coercing the bound expression to the target column type as though by an implicit assignment, and the rules for implicit assignment say that collations can be implicitly converted. What we *do* need to do, and the code is not doing, is apply assign_expr_collations() to the bound expression. While this is merely a definition disagreement, that is a bug that needs to be back-patched, so I'll commit it separately. Discussion: https://postgr.es/m/CAJV4CdrZ5mKuaEsRSbLf2URQ3h6iMtKD=hik8MaF5WwdmC9uZw@mail.gmail.com --- src/backend/parser/parse_utilcmd.c | 44 ---------------------- src/test/regress/expected/create_table.out | 32 ++++++++-------- src/test/regress/sql/create_table.sql | 18 +++------ 3 files changed, 21 insertions(+), 73 deletions(-) diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index 164312d60e25..6d2f36da2df0 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -4183,50 +4183,6 @@ transformPartitionBoundValue(ParseState *pstate, Node *val, */ Assert(!contain_var_clause(value)); - /* - * Check that the input expression's collation is compatible with one - * specified for the parent's partition key (partcollation). Don't throw - * an error if it's the default collation which we'll replace with the - * parent's collation anyway. - */ - if (IsA(value, CollateExpr)) - { - Oid exprCollOid = exprCollation(value); - - /* - * Check we have a collation iff it is a collatable type. The only - * expected failures here are (1) COLLATE applied to a noncollatable - * type, or (2) partition bound expression had an unresolved - * collation. But we might as well code this to be a complete - * consistency check. - */ - if (type_is_collatable(colType)) - { - if (!OidIsValid(exprCollOid)) - ereport(ERROR, - (errcode(ERRCODE_INDETERMINATE_COLLATION), - errmsg("could not determine which collation to use for partition bound expression"), - errhint("Use the COLLATE clause to set the collation explicitly."))); - } - else - { - if (OidIsValid(exprCollOid)) - ereport(ERROR, - (errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("collations are not supported by type %s", - format_type_be(colType)))); - } - - if (OidIsValid(exprCollOid) && - exprCollOid != DEFAULT_COLLATION_OID && - exprCollOid != partCollation) - ereport(ERROR, - (errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("collation of partition bound value for column \"%s\" does not match partition key collation \"%s\"", - colName, get_collation_name(partCollation)), - parser_errposition(pstate, exprLocation(value)))); - } - /* * Coerce to the correct type. This might cause an explicit coercion step * to be added on top of the expression, which must be evaluated before diff --git a/src/test/regress/expected/create_table.out b/src/test/regress/expected/create_table.out index 41dce69cc4c8..8ed4fae934d0 100644 --- a/src/test/regress/expected/create_table.out +++ b/src/test/regress/expected/create_table.out @@ -652,8 +652,6 @@ CREATE TABLE part_bogus_expr_fail PARTITION OF list_parted FOR VALUES IN (genera ERROR: set-returning functions are not allowed in partition bound LINE 1: ...expr_fail PARTITION OF list_parted FOR VALUES IN (generate_s... ^ -CREATE TABLE part_bogus_expr_fail PARTITION OF list_parted FOR VALUES IN ('1' collate "POSIX"); -ERROR: collations are not supported by type integer CREATE TABLE part_bogus_expr_fail PARTITION OF list_parted FOR VALUES IN ((1+1) collate "POSIX"); ERROR: collations are not supported by type integer LINE 1: ...ail PARTITION OF list_parted FOR VALUES IN ((1+1) collate "P... @@ -1026,28 +1024,28 @@ create table parted_collate_must_match1 partition of parted_collate_must_match create table parted_collate_must_match2 partition of parted_collate_must_match (b collate "POSIX") for values from ('m') to ('z'); drop table parted_collate_must_match; --- check that specifying incompatible collations for partition bound --- expressions fails promptly +-- check that non-matching collations for partition bound +-- expressions are coerced to the right collation create table test_part_coll_posix (a text) partition by range (a collate "POSIX"); --- fail +-- ok, collation is implicitly coerced create table test_part_coll partition of test_part_coll_posix for values from ('a' collate "C") to ('g'); -ERROR: collation of partition bound value for column "a" does not match partition key collation "POSIX" -LINE 1: ...artition of test_part_coll_posix for values from ('a' collat... - ^ --- ok -create table test_part_coll partition of test_part_coll_posix for values from ('a' collate "POSIX") to ('g'); -- ok create table test_part_coll2 partition of test_part_coll_posix for values from ('g') to ('m'); --- using a cast expression uses the target type's default collation --- fail +-- ok, collation is implicitly coerced create table test_part_coll_cast partition of test_part_coll_posix for values from (name 'm' collate "C") to ('s'); -ERROR: collation of partition bound value for column "a" does not match partition key collation "POSIX" -LINE 1: ...ion of test_part_coll_posix for values from (name 'm' collat... - ^ --- ok -create table test_part_coll_cast partition of test_part_coll_posix for values from (name 'm' collate "POSIX") to ('s'); -- ok; partition collation silently overrides the default collation of type 'name' create table test_part_coll_cast2 partition of test_part_coll_posix for values from (name 's') to ('z'); +\d+ test_part_coll_posix + Partitioned table "public.test_part_coll_posix" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------+------+-----------+----------+---------+----------+--------------+------------- + a | text | | | | extended | | +Partition key: RANGE (a COLLATE "POSIX") +Partitions: test_part_coll FOR VALUES FROM ('a') TO ('g'), + test_part_coll2 FOR VALUES FROM ('g') TO ('m'), + test_part_coll_cast FOR VALUES FROM ('m') TO ('s'), + test_part_coll_cast2 FOR VALUES FROM ('s') TO ('z') + drop table test_part_coll_posix; -- Partition bound in describe output \d+ part_b diff --git a/src/test/regress/sql/create_table.sql b/src/test/regress/sql/create_table.sql index 9b1adcb8adda..e6f0188a5189 100644 --- a/src/test/regress/sql/create_table.sql +++ b/src/test/regress/sql/create_table.sql @@ -549,7 +549,6 @@ CREATE TABLE part_bogus_expr_fail PARTITION OF list_parted FOR VALUES IN (sum(so CREATE TABLE part_bogus_expr_fail PARTITION OF list_parted FOR VALUES IN (sum(1)); CREATE TABLE part_bogus_expr_fail PARTITION OF list_parted FOR VALUES IN ((select 1)); CREATE TABLE part_bogus_expr_fail PARTITION OF list_parted FOR VALUES IN (generate_series(4, 6)); -CREATE TABLE part_bogus_expr_fail PARTITION OF list_parted FOR VALUES IN ('1' collate "POSIX"); CREATE TABLE part_bogus_expr_fail PARTITION OF list_parted FOR VALUES IN ((1+1) collate "POSIX"); -- syntax does not allow empty list of values for list partitions @@ -813,26 +812,21 @@ create table parted_collate_must_match2 partition of parted_collate_must_match (b collate "POSIX") for values from ('m') to ('z'); drop table parted_collate_must_match; --- check that specifying incompatible collations for partition bound --- expressions fails promptly +-- check that non-matching collations for partition bound +-- expressions are coerced to the right collation create table test_part_coll_posix (a text) partition by range (a collate "POSIX"); --- fail +-- ok, collation is implicitly coerced create table test_part_coll partition of test_part_coll_posix for values from ('a' collate "C") to ('g'); -- ok -create table test_part_coll partition of test_part_coll_posix for values from ('a' collate "POSIX") to ('g'); --- ok create table test_part_coll2 partition of test_part_coll_posix for values from ('g') to ('m'); - --- using a cast expression uses the target type's default collation - --- fail +-- ok, collation is implicitly coerced create table test_part_coll_cast partition of test_part_coll_posix for values from (name 'm' collate "C") to ('s'); --- ok -create table test_part_coll_cast partition of test_part_coll_posix for values from (name 'm' collate "POSIX") to ('s'); -- ok; partition collation silently overrides the default collation of type 'name' create table test_part_coll_cast2 partition of test_part_coll_posix for values from (name 's') to ('z'); +\d+ test_part_coll_posix + drop table test_part_coll_posix; -- Partition bound in describe output From 72647ac3bf0f11732483eac2cd9b5cf4972f2e1f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 28 Sep 2020 14:12:38 -0400 Subject: [PATCH 215/589] Assign collations in partition bound expressions. Failure to do this can result in errors during evaluation of the bound expression, as illustrated by the new regression test. Back-patch to v12 where the ability for partition bounds to be expressions was added. Discussion: https://postgr.es/m/CAJV4CdrZ5mKuaEsRSbLf2URQ3h6iMtKD=hik8MaF5WwdmC9uZw@mail.gmail.com --- src/backend/parser/parse_utilcmd.c | 1 + src/test/regress/expected/create_table.out | 7 +++++++ src/test/regress/sql/create_table.sql | 8 ++++++++ 3 files changed, 16 insertions(+) diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index 6d2f36da2df0..0dc03dd98408 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -4209,6 +4209,7 @@ transformPartitionBoundValue(ParseState *pstate, Node *val, */ if (!IsA(value, Const)) { + assign_expr_collations(pstate, value); value = (Node *) expression_planner((Expr *) value); value = (Node *) evaluate_expr((Expr *) value, colType, colTypmod, partCollation); diff --git a/src/test/regress/expected/create_table.out b/src/test/regress/expected/create_table.out index 8ed4fae934d0..45f4a7344785 100644 --- a/src/test/regress/expected/create_table.out +++ b/src/test/regress/expected/create_table.out @@ -1014,6 +1014,13 @@ DETAIL: Failing row contains (1, null). Partition of: parted_notnull_inh_test FOR VALUES IN (1) drop table parted_notnull_inh_test; +-- check that collations are assigned in partition bound expressions +create table parted_boolean_col (a bool, b text) partition by list(a); +create table parted_boolean_less partition of parted_boolean_col + for values in ('foo' < 'bar'); +create table parted_boolean_greater partition of parted_boolean_col + for values in ('foo' > 'bar'); +drop table parted_boolean_col; -- check for a conflicting COLLATE clause create table parted_collate_must_match (a text collate "C", b text collate "C") partition by range (a); diff --git a/src/test/regress/sql/create_table.sql b/src/test/regress/sql/create_table.sql index e6f0188a5189..22602aae5d2a 100644 --- a/src/test/regress/sql/create_table.sql +++ b/src/test/regress/sql/create_table.sql @@ -801,6 +801,14 @@ insert into parted_notnull_inh_test (b) values (null); \d parted_notnull_inh_test1 drop table parted_notnull_inh_test; +-- check that collations are assigned in partition bound expressions +create table parted_boolean_col (a bool, b text) partition by list(a); +create table parted_boolean_less partition of parted_boolean_col + for values in ('foo' < 'bar'); +create table parted_boolean_greater partition of parted_boolean_col + for values in ('foo' > 'bar'); +drop table parted_boolean_col; + -- check for a conflicting COLLATE clause create table parted_collate_must_match (a text collate "C", b text collate "C") partition by range (a); From 042d8017ec7bca244db4a53bc3d72d218495136d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 28 Sep 2020 14:48:01 -0400 Subject: [PATCH 216/589] Stabilize create_table regression test. Adding \d+ to the test in commit 2dfa3fea8 was ill-advised, because the partitions' names are such that their sort order is locale dependent. We could rename them to avoid that, but it doesn't seem worth the trouble; just take \d+ out again. Per buildfarm. --- src/test/regress/expected/create_table.out | 11 ----------- src/test/regress/sql/create_table.sql | 2 -- 2 files changed, 13 deletions(-) diff --git a/src/test/regress/expected/create_table.out b/src/test/regress/expected/create_table.out index 45f4a7344785..1fc266dd65cb 100644 --- a/src/test/regress/expected/create_table.out +++ b/src/test/regress/expected/create_table.out @@ -1042,17 +1042,6 @@ create table test_part_coll2 partition of test_part_coll_posix for values from ( create table test_part_coll_cast partition of test_part_coll_posix for values from (name 'm' collate "C") to ('s'); -- ok; partition collation silently overrides the default collation of type 'name' create table test_part_coll_cast2 partition of test_part_coll_posix for values from (name 's') to ('z'); -\d+ test_part_coll_posix - Partitioned table "public.test_part_coll_posix" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | -Partition key: RANGE (a COLLATE "POSIX") -Partitions: test_part_coll FOR VALUES FROM ('a') TO ('g'), - test_part_coll2 FOR VALUES FROM ('g') TO ('m'), - test_part_coll_cast FOR VALUES FROM ('m') TO ('s'), - test_part_coll_cast2 FOR VALUES FROM ('s') TO ('z') - drop table test_part_coll_posix; -- Partition bound in describe output \d+ part_b diff --git a/src/test/regress/sql/create_table.sql b/src/test/regress/sql/create_table.sql index 22602aae5d2a..cee822aa8b6e 100644 --- a/src/test/regress/sql/create_table.sql +++ b/src/test/regress/sql/create_table.sql @@ -833,8 +833,6 @@ create table test_part_coll_cast partition of test_part_coll_posix for values fr -- ok; partition collation silently overrides the default collation of type 'name' create table test_part_coll_cast2 partition of test_part_coll_posix for values from (name 's') to ('z'); -\d+ test_part_coll_posix - drop table test_part_coll_posix; -- Partition bound in describe output From fe0a1dc52c7332a65b44db8e8408a5fd1d8fc8fb Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 29 Sep 2020 09:25:51 +0900 Subject: [PATCH 217/589] Revert "Change SHA2 implementation based on OpenSSL to use EVP digest routines" This reverts commit e21cbb4, as the switch to EVP routines requires a more careful design where we would need to have at least our wrapper routines return a status instead of issuing an error by themselves to let the caller do the error handling. The memory handling was also incorrect and could cause leaks in the backend if a failure happened, requiring most likely a callback to do the necessary cleanup as the only clean way to be able to allocate an EVP context requires the use of an allocation within OpenSSL. The potential rework of the wrappers also impacts the fallback implementation when not building with OpenSSL. Originally, prairiedog has reported a compilation failure, but after discussion with Tom Lane this needs a better design. Discussion: https://postgr.es/m/20200928073330.GC2316@paquier.xyz --- src/common/sha2_openssl.c | 63 +++++++++------------------------------ src/include/common/sha2.h | 10 +++---- 2 files changed, 19 insertions(+), 54 deletions(-) diff --git a/src/common/sha2_openssl.c b/src/common/sha2_openssl.c index 30f5f816f8ea..41673b3a8886 100644 --- a/src/common/sha2_openssl.c +++ b/src/common/sha2_openssl.c @@ -20,118 +20,83 @@ #include "postgres_fe.h" #endif -#include "common/sha2.h" - -#ifdef FRONTEND -#include "common/logging.h" -#else -#include "miscadmin.h" -#endif +#include -#ifdef FRONTEND -#define sha2_log_and_abort(...) \ - do { pg_log_fatal(__VA_ARGS__); exit(1); } while(0) -#else -#define sha2_log_and_abort(...) elog(ERROR, __VA_ARGS__) -#endif - -static void -digest_init(EVP_MD_CTX **ctx, const EVP_MD *type) -{ - *ctx = EVP_MD_CTX_create(); - if (*ctx == NULL) - sha2_log_and_abort("could not create EVP digest context"); - if (EVP_DigestInit_ex(*ctx, type, NULL) <= 0) - sha2_log_and_abort("could not initialize EVP digest context"); -} - -static void -digest_update(EVP_MD_CTX **ctx, const uint8 *data, size_t len) -{ - if (EVP_DigestUpdate(*ctx, data, len) <= 0) - sha2_log_and_abort("could not update EVP digest context"); -} +#include "common/sha2.h" -static void -digest_final(EVP_MD_CTX **ctx, uint8 *dest) -{ - if (EVP_DigestFinal_ex(*ctx, dest, 0) <= 0) - sha2_log_and_abort("could not finalize EVP digest context"); - EVP_MD_CTX_destroy(*ctx); -} /* Interface routines for SHA-256 */ void pg_sha256_init(pg_sha256_ctx *ctx) { - digest_init(ctx, EVP_sha256()); + SHA256_Init((SHA256_CTX *) ctx); } void pg_sha256_update(pg_sha256_ctx *ctx, const uint8 *data, size_t len) { - digest_update(ctx, data, len); + SHA256_Update((SHA256_CTX *) ctx, data, len); } void pg_sha256_final(pg_sha256_ctx *ctx, uint8 *dest) { - digest_final(ctx, dest); + SHA256_Final(dest, (SHA256_CTX *) ctx); } /* Interface routines for SHA-512 */ void pg_sha512_init(pg_sha512_ctx *ctx) { - digest_init(ctx, EVP_sha512()); + SHA512_Init((SHA512_CTX *) ctx); } void pg_sha512_update(pg_sha512_ctx *ctx, const uint8 *data, size_t len) { - digest_update(ctx, data, len); + SHA512_Update((SHA512_CTX *) ctx, data, len); } void pg_sha512_final(pg_sha512_ctx *ctx, uint8 *dest) { - digest_final(ctx, dest); + SHA512_Final(dest, (SHA512_CTX *) ctx); } /* Interface routines for SHA-384 */ void pg_sha384_init(pg_sha384_ctx *ctx) { - digest_init(ctx, EVP_sha384()); + SHA384_Init((SHA512_CTX *) ctx); } void pg_sha384_update(pg_sha384_ctx *ctx, const uint8 *data, size_t len) { - digest_update(ctx, data, len); + SHA384_Update((SHA512_CTX *) ctx, data, len); } void pg_sha384_final(pg_sha384_ctx *ctx, uint8 *dest) { - digest_final(ctx, dest); + SHA384_Final(dest, (SHA512_CTX *) ctx); } /* Interface routines for SHA-224 */ void pg_sha224_init(pg_sha224_ctx *ctx) { - digest_init(ctx, EVP_sha224()); + SHA224_Init((SHA256_CTX *) ctx); } void pg_sha224_update(pg_sha224_ctx *ctx, const uint8 *data, size_t len) { - digest_update(ctx, data, len); + SHA224_Update((SHA256_CTX *) ctx, data, len); } void pg_sha224_final(pg_sha224_ctx *ctx, uint8 *dest) { - digest_final(ctx, dest); + SHA224_Final(dest, (SHA256_CTX *) ctx); } diff --git a/src/include/common/sha2.h b/src/include/common/sha2.h index 2c5283816153..9c4abf777d43 100644 --- a/src/include/common/sha2.h +++ b/src/include/common/sha2.h @@ -51,7 +51,7 @@ #define _PG_SHA2_H_ #ifdef USE_OPENSSL -#include +#include #endif /*** SHA224/256/384/512 Various Length Definitions ***********************/ @@ -70,10 +70,10 @@ /* Context Structures for SHA224/256/384/512 */ #ifdef USE_OPENSSL -typedef EVP_MD_CTX *pg_sha256_ctx; -typedef EVP_MD_CTX *pg_sha512_ctx; -typedef EVP_MD_CTX *pg_sha224_ctx; -typedef EVP_MD_CTX *pg_sha384_ctx; +typedef SHA256_CTX pg_sha256_ctx; +typedef SHA512_CTX pg_sha512_ctx; +typedef SHA256_CTX pg_sha224_ctx; +typedef SHA512_CTX pg_sha384_ctx; #else typedef struct pg_sha256_ctx { From 56fe008996bc1a547ce60c8dddd2ca821cac163e Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 28 Sep 2020 20:32:53 -0400 Subject: [PATCH 218/589] Add for_each_from, to simplify loops starting from non-first list cells. We have a dozen or so places that need to iterate over all but the first cell of a List. Prior to v13 this was typically written as for_each_cell(lc, lnext(list_head(list))) Commit 1cff1b95a changed these to for_each_cell(lc, list, list_second_cell(list)) This patch introduces a new macro for_each_from() which expresses the start point as a list index, allowing these to be written as for_each_from(lc, list, 1) This is marginally more efficient, since ForEachState.i can be initialized directly instead of backing into it from a ListCell address. It also seems clearer and less typo-prone. Some of the remaining uses of for_each_cell() look like they could profitably be changed to for_each_from(), but here I confined myself to changing uses of list_second_cell(). Also, fix for_each_cell_setup() and for_both_cell_setup() to const-ify their arguments; that's a simple oversight in 1cff1b95a. Back-patch into v13, on the grounds that (1) the const-ification is a minor bug fix, and (2) it's better for back-patching purposes if we only have two ways to write these loops rather than three. In HEAD, also remove list_third_cell() and list_fourth_cell(), which were also introduced in 1cff1b95a, and are unused as of cc99baa43. It seems unlikely that any third-party code would have started to use them already; anyone who has can be directed to list_nth_cell instead. Discussion: https://postgr.es/m/CAApHDvpo1zj9KhEpU2cCRZfSM3Q6XGdhzuAS2v79PH7WJBkYVA@mail.gmail.com --- src/backend/commands/tablecmds.c | 2 +- src/backend/nodes/nodeFuncs.c | 4 +- src/backend/optimizer/plan/createplan.c | 2 +- src/backend/optimizer/plan/planner.c | 4 +- src/backend/parser/parse_agg.c | 4 +- src/backend/utils/adt/jsonpath_gram.y | 2 +- src/backend/utils/adt/ruleutils.c | 8 ++-- src/backend/utils/adt/selfuncs.c | 2 +- src/include/nodes/pg_list.h | 52 ++++++++++++++----------- 9 files changed, 42 insertions(+), 38 deletions(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 16285ad09faf..e0ac4e05e5f5 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -5732,7 +5732,7 @@ ATCheckPartitionsNotInUse(Relation rel, LOCKMODE lockmode) inh = find_all_inheritors(RelationGetRelid(rel), lockmode, NULL); /* first element is the parent rel; must ignore it */ - for_each_cell(cell, inh, list_second_cell(inh)) + for_each_from(cell, inh, 1) { Relation childrel; diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c index 9ce8f43385ec..1dc873ed255f 100644 --- a/src/backend/nodes/nodeFuncs.c +++ b/src/backend/nodes/nodeFuncs.c @@ -441,7 +441,7 @@ exprTypmod(const Node *expr) typmod = exprTypmod((Node *) linitial(cexpr->args)); if (typmod < 0) return -1; /* no point in trying harder */ - for_each_cell(arg, cexpr->args, list_second_cell(cexpr->args)) + for_each_from(arg, cexpr->args, 1) { Node *e = (Node *) lfirst(arg); @@ -469,7 +469,7 @@ exprTypmod(const Node *expr) typmod = exprTypmod((Node *) linitial(mexpr->args)); if (typmod < 0) return -1; /* no point in trying harder */ - for_each_cell(arg, mexpr->args, list_second_cell(mexpr->args)) + for_each_from(arg, mexpr->args, 1) { Node *e = (Node *) lfirst(arg); diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 99278eed9319..3d7a4e373fb5 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -2261,7 +2261,7 @@ create_groupingsets_plan(PlannerInfo *root, GroupingSetsPath *best_path) { bool is_first_sort = ((RollupData *) linitial(rollups))->is_hashed; - for_each_cell(lc, rollups, list_second_cell(rollups)) + for_each_from(lc, rollups, 1) { RollupData *rollup = lfirst(lc); AttrNumber *new_grpColIdx; diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 3e2b4965c4a8..f331f82a6c20 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -4430,7 +4430,7 @@ consider_groupingsets_paths(PlannerInfo *root, * below, must use the same condition. */ i = 0; - for_each_cell(lc, gd->rollups, list_second_cell(gd->rollups)) + for_each_from(lc, gd->rollups, 1) { RollupData *rollup = lfirst_node(RollupData, lc); @@ -4464,7 +4464,7 @@ consider_groupingsets_paths(PlannerInfo *root, rollups = list_make1(linitial(gd->rollups)); i = 0; - for_each_cell(lc, gd->rollups, list_second_cell(gd->rollups)) + for_each_from(lc, gd->rollups, 1) { RollupData *rollup = lfirst_node(RollupData, lc); diff --git a/src/backend/parser/parse_agg.c b/src/backend/parser/parse_agg.c index f813b587f186..783f3fe8f2d1 100644 --- a/src/backend/parser/parse_agg.c +++ b/src/backend/parser/parse_agg.c @@ -1083,7 +1083,7 @@ parseCheckAggregates(ParseState *pstate, Query *qry) if (gset_common) { - for_each_cell(l, gsets, list_second_cell(gsets)) + for_each_from(l, gsets, 1) { gset_common = list_intersection_int(gset_common, lfirst(l)); if (!gset_common) @@ -1774,7 +1774,7 @@ expand_grouping_sets(List *groupingSets, int limit) result = lappend(result, list_union_int(NIL, (List *) lfirst(lc))); } - for_each_cell(lc, expanded_groups, list_second_cell(expanded_groups)) + for_each_from(lc, expanded_groups, 1) { List *p = lfirst(lc); List *new_result = NIL; diff --git a/src/backend/utils/adt/jsonpath_gram.y b/src/backend/utils/adt/jsonpath_gram.y index 88ef9550e9db..53f422260c38 100644 --- a/src/backend/utils/adt/jsonpath_gram.y +++ b/src/backend/utils/adt/jsonpath_gram.y @@ -441,7 +441,7 @@ makeItemList(List *list) while (end->next) end = end->next; - for_each_cell(cell, list, list_second_cell(list)) + for_each_from(cell, list, 1) { JsonPathParseItem *c = (JsonPathParseItem *) lfirst(cell); diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index 03cf24199637..62023c20b21e 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -8113,7 +8113,7 @@ get_rule_expr(Node *node, deparse_context *context, { BoolExpr *expr = (BoolExpr *) node; Node *first_arg = linitial(expr->args); - ListCell *arg = list_second_cell(expr->args); + ListCell *arg; switch (expr->boolop) { @@ -8122,12 +8122,11 @@ get_rule_expr(Node *node, deparse_context *context, appendStringInfoChar(buf, '('); get_rule_expr_paren(first_arg, context, false, node); - while (arg) + for_each_from(arg, expr->args, 1) { appendStringInfoString(buf, " AND "); get_rule_expr_paren((Node *) lfirst(arg), context, false, node); - arg = lnext(expr->args, arg); } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); @@ -8138,12 +8137,11 @@ get_rule_expr(Node *node, deparse_context *context, appendStringInfoChar(buf, '('); get_rule_expr_paren(first_arg, context, false, node); - while (arg) + for_each_from(arg, expr->args, 1) { appendStringInfoString(buf, " OR "); get_rule_expr_paren((Node *) lfirst(arg), context, false, node); - arg = lnext(expr->args, arg); } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index 00c7afc66fc2..bec357fcef04 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -3519,7 +3519,7 @@ estimate_num_groups(PlannerInfo *root, List *groupExprs, double input_rows, * for remaining Vars on other rels. */ relvarinfos = lappend(relvarinfos, varinfo1); - for_each_cell(l, varinfos, list_second_cell(varinfos)) + for_each_from(l, varinfos, 1) { GroupVarInfo *varinfo2 = (GroupVarInfo *) lfirst(l); diff --git a/src/include/nodes/pg_list.h b/src/include/nodes/pg_list.h index 104df4174abc..ec231010ce4b 100644 --- a/src/include/nodes/pg_list.h +++ b/src/include/nodes/pg_list.h @@ -144,26 +144,6 @@ list_second_cell(const List *l) return NULL; } -/* Fetch address of list's third cell, if it has one, else NULL */ -static inline ListCell * -list_third_cell(const List *l) -{ - if (l && l->length >= 3) - return &l->elements[2]; - else - return NULL; -} - -/* Fetch address of list's fourth cell, if it has one, else NULL */ -static inline ListCell * -list_fourth_cell(const List *l) -{ - if (l && l->length >= 4) - return &l->elements[3]; - else - return NULL; -} - /* Fetch list's length */ static inline int list_length(const List *l) @@ -389,6 +369,32 @@ lnext(const List *l, const ListCell *c) */ #define foreach_current_index(cell) (cell##__state.i) +/* + * for_each_from - + * Like foreach(), but start from the N'th (zero-based) list element, + * not necessarily the first one. + * + * It's okay for N to exceed the list length, but not for it to be negative. + * + * The caveats for foreach() apply equally here. + */ +#define for_each_from(cell, lst, N) \ + for (ForEachState cell##__state = for_each_from_setup(lst, N); \ + (cell##__state.l != NIL && \ + cell##__state.i < cell##__state.l->length) ? \ + (cell = &cell##__state.l->elements[cell##__state.i], true) : \ + (cell = NULL, false); \ + cell##__state.i++) + +static inline ForEachState +for_each_from_setup(const List *lst, int N) +{ + ForEachState r = {lst, N}; + + Assert(N >= 0); + return r; +} + /* * for_each_cell - * a convenience macro which loops through a list starting from a @@ -405,7 +411,7 @@ lnext(const List *l, const ListCell *c) cell##__state.i++) static inline ForEachState -for_each_cell_setup(List *lst, ListCell *initcell) +for_each_cell_setup(const List *lst, const ListCell *initcell) { ForEachState r = {lst, initcell ? list_cell_number(lst, initcell) : list_length(lst)}; @@ -456,8 +462,8 @@ for_each_cell_setup(List *lst, ListCell *initcell) cell1##__state.i1++, cell1##__state.i2++) static inline ForBothCellState -for_both_cell_setup(List *list1, ListCell *initcell1, - List *list2, ListCell *initcell2) +for_both_cell_setup(const List *list1, const ListCell *initcell1, + const List *list2, const ListCell *initcell2) { ForBothCellState r = {list1, list2, initcell1 ? list_cell_number(list1, initcell1) : list_length(list1), From e66bcfb4c66ebc97020b1f7484b1529bd7993f23 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 29 Sep 2020 14:15:57 +0900 Subject: [PATCH 219/589] Fix progress reporting of REINDEX CONCURRENTLY This addresses a couple of issues with the so-said subject: - Report the correct parent relation with the index actually being rebuilt or validated. Previously, the command status remained set to the last index created for the progress of the index build and validation, which would be incorrect when working on a table that has more than one index. - Use the correct phase when waiting before the drop of the old indexes. Previously, this was reported with the same status as when waiting before the old indexes are marked as dead. Author: Matthias van de Meent, Michael Paquier Discussion: https://postgr.es/m/CAEze2WhqFgcwe1_tv=sFYhLWV2AdpfukumotJ6JNcAOQs3jufg@mail.gmail.com Backpatch-through: 12 --- src/backend/commands/indexcmds.c | 70 ++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 16 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index f1b5f87e6a8c..59e04b47dfb3 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -3015,6 +3015,13 @@ ReindexRelationConcurrently(Oid relationOid, int options) char *relationName = NULL; char *relationNamespace = NULL; PGRUsage ru0; + const int progress_index[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_ACCESS_METHOD_OID + }; + int64 progress_vals[4]; /* * Create a memory context that will survive forced transaction commits we @@ -3294,12 +3301,11 @@ ReindexRelationConcurrently(Oid relationOid, int options) pgstat_progress_start_command(PROGRESS_COMMAND_CREATE_INDEX, RelationGetRelid(heapRel)); - pgstat_progress_update_param(PROGRESS_CREATEIDX_COMMAND, - PROGRESS_CREATEIDX_COMMAND_REINDEX_CONCURRENTLY); - pgstat_progress_update_param(PROGRESS_CREATEIDX_INDEX_OID, - indexId); - pgstat_progress_update_param(PROGRESS_CREATEIDX_ACCESS_METHOD_OID, - indexRel->rd_rel->relam); + progress_vals[0] = PROGRESS_CREATEIDX_COMMAND_REINDEX_CONCURRENTLY; + progress_vals[1] = 0; /* initializing */ + progress_vals[2] = indexId; + progress_vals[3] = indexRel->rd_rel->relam; + pgstat_progress_update_multi_param(4, progress_index, progress_vals); /* Choose a temporary relation name for the new index */ concurrentName = ChooseRelationName(get_rel_name(indexId), @@ -3403,12 +3409,12 @@ ReindexRelationConcurrently(Oid relationOid, int options) WaitForLockersMultiple(lockTags, ShareLock, true); CommitTransactionCommand(); - forboth(lc, indexIds, lc2, newIndexIds) + foreach(lc, newIndexIds) { - Relation indexRel; - Oid oldIndexId = lfirst_oid(lc); - Oid newIndexId = lfirst_oid(lc2); + Relation newIndexRel; + Oid newIndexId = lfirst_oid(lc); Oid heapId; + Oid indexam; /* Start new transaction for this index's concurrent build */ StartTransactionCommand(); @@ -3427,9 +3433,21 @@ ReindexRelationConcurrently(Oid relationOid, int options) * Index relation has been closed by previous commit, so reopen it to * get its information. */ - indexRel = index_open(oldIndexId, ShareUpdateExclusiveLock); - heapId = indexRel->rd_index->indrelid; - index_close(indexRel, NoLock); + newIndexRel = index_open(newIndexId, ShareUpdateExclusiveLock); + heapId = newIndexRel->rd_index->indrelid; + indexam = newIndexRel->rd_rel->relam; + index_close(newIndexRel, NoLock); + + /* + * Update progress for the index to build, with the correct parent + * table involved. + */ + pgstat_progress_start_command(PROGRESS_COMMAND_CREATE_INDEX, heapId); + progress_vals[0] = PROGRESS_CREATEIDX_COMMAND_REINDEX_CONCURRENTLY; + progress_vals[1] = PROGRESS_CREATEIDX_PHASE_BUILD; + progress_vals[2] = newIndexId; + progress_vals[3] = indexam; + pgstat_progress_update_multi_param(4, progress_index, progress_vals); /* Perform concurrent build of new index */ index_concurrently_build(heapId, newIndexId); @@ -3458,6 +3476,8 @@ ReindexRelationConcurrently(Oid relationOid, int options) Oid heapId; TransactionId limitXmin; Snapshot snapshot; + Relation newIndexRel; + Oid indexam; StartTransactionCommand(); @@ -3468,8 +3488,6 @@ ReindexRelationConcurrently(Oid relationOid, int options) */ CHECK_FOR_INTERRUPTS(); - heapId = IndexGetRelation(newIndexId, false); - /* * Take the "reference snapshot" that will be used by validate_index() * to filter candidate tuples. @@ -3477,6 +3495,26 @@ ReindexRelationConcurrently(Oid relationOid, int options) snapshot = RegisterSnapshot(GetTransactionSnapshot()); PushActiveSnapshot(snapshot); + /* + * Index relation has been closed by previous commit, so reopen it to + * get its information. + */ + newIndexRel = index_open(newIndexId, ShareUpdateExclusiveLock); + heapId = newIndexRel->rd_index->indrelid; + indexam = newIndexRel->rd_rel->relam; + index_close(newIndexRel, NoLock); + + /* + * Update progress for the index to build, with the correct parent + * table involved. + */ + pgstat_progress_start_command(PROGRESS_COMMAND_CREATE_INDEX, heapId); + progress_vals[0] = PROGRESS_CREATEIDX_COMMAND_REINDEX_CONCURRENTLY; + progress_vals[1] = PROGRESS_CREATEIDX_PHASE_VALIDATE_IDXSCAN; + progress_vals[2] = newIndexId; + progress_vals[3] = indexam; + pgstat_progress_update_multi_param(4, progress_index, progress_vals); + validate_index(heapId, newIndexId, snapshot); /* @@ -3611,7 +3649,7 @@ ReindexRelationConcurrently(Oid relationOid, int options) */ pgstat_progress_update_param(PROGRESS_CREATEIDX_PHASE, - PROGRESS_CREATEIDX_PHASE_WAIT_4); + PROGRESS_CREATEIDX_PHASE_WAIT_5); WaitForLockersMultiple(lockTags, AccessExclusiveLock, true); PushActiveSnapshot(GetTransactionSnapshot()); From fd26f78231830b20d8b5f8391f97d2ccebd492b7 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Tue, 29 Sep 2020 16:21:46 +0900 Subject: [PATCH 220/589] Archive timeline history files in standby if archive_mode is set to "always". Previously the standby server didn't archive timeline history files streamed from the primary even when archive_mode is set to "always", while it archives the streamed WAL files. This could cause the PITR to fail because there was no required timeline history file in the archive. The cause of this issue was that walreceiver didn't mark those files as ready for archiving. This commit makes walreceiver mark those streamed timeline history files as ready for archiving if archive_mode=always. Then the archiver process archives the marked timeline history files. Back-patch to all supported versions. Reported-by: Grigory Smolkin Author: Grigory Smolkin, Fujii Masao Reviewed-by: David Zhang, Anastasia Lubennikova Discussion: https://postgr.es/m/54b059d4-2b48-13a4-6f43-95a087c92367@postgrespro.ru --- doc/src/sgml/high-availability.sgml | 3 ++- src/backend/replication/walreceiver.c | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index beb309e668e2..42f01c515f9b 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -1395,7 +1395,8 @@ synchronous_standby_names = 'ANY 2 (s1, s2, s3)' If archive_mode is set to on, the archiver is not enabled during recovery or standby mode. If the standby server is promoted, it will start archiving after the promotion, but - will not archive any WAL it did not generate itself. To get a complete + will not archive any WAL or timeline history files that + it did not generate itself. To get a complete series of WAL files in the archive, you must ensure that all WAL is archived, before it reaches the standby. This is inherently true with file-based log shipping, as the standby can only restore files that diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 17f1a49f8711..bb1d44ccb7a0 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -758,6 +758,15 @@ WalRcvFetchTimeLineHistoryFiles(TimeLineID first, TimeLineID last) */ writeTimeLineHistoryFile(tli, content, len); + /* + * Mark the streamed history file as ready for archiving + * if archive_mode is always. + */ + if (XLogArchiveMode != ARCHIVE_MODE_ALWAYS) + XLogArchiveForceDone(fname); + else + XLogArchiveNotify(fname); + pfree(fname); pfree(content); } From c2aa562ea5c2bf28c347503731434f08097cf1e5 Mon Sep 17 00:00:00 2001 From: Alexander Korotkov Date: Tue, 29 Sep 2020 11:00:22 +0300 Subject: [PATCH 221/589] Remove excess space from jsonpath .datetime() default format string bffe1bd684 has introduced jsonpath .datetime() method, but default formats for time and timestamp contain excess space between time and timezone. This commit removes this excess space making behavior of .datetime() method standard-compliant. Discussion: https://postgr.es/m/94321be0-cc96-1a81-b6df-796f437f7c66%40postgrespro.ru Author: Nikita Glukhov Backpatch-through: 13 --- src/backend/utils/adt/jsonpath_exec.c | 8 +-- src/test/regress/expected/jsonb_jsonpath.out | 76 ++++++++++---------- src/test/regress/sql/jsonb_jsonpath.sql | 76 ++++++++++---------- 3 files changed, 80 insertions(+), 80 deletions(-) diff --git a/src/backend/utils/adt/jsonpath_exec.c b/src/backend/utils/adt/jsonpath_exec.c index 0591c9effcb9..31b88d674166 100644 --- a/src/backend/utils/adt/jsonpath_exec.c +++ b/src/backend/utils/adt/jsonpath_exec.c @@ -1837,11 +1837,11 @@ executeDateTimeMethod(JsonPathExecContext *cxt, JsonPathItem *jsp, static const char *fmt_str[] = { "yyyy-mm-dd", - "HH24:MI:SS TZH:TZM", - "HH24:MI:SS TZH", + "HH24:MI:SSTZH:TZM", + "HH24:MI:SSTZH", "HH24:MI:SS", - "yyyy-mm-dd HH24:MI:SS TZH:TZM", - "yyyy-mm-dd HH24:MI:SS TZH", + "yyyy-mm-dd HH24:MI:SSTZH:TZM", + "yyyy-mm-dd HH24:MI:SSTZH", "yyyy-mm-dd HH24:MI:SS" }; diff --git a/src/test/regress/expected/jsonb_jsonpath.out b/src/test/regress/expected/jsonb_jsonpath.out index c870b7f99426..31d6f05a7c20 100644 --- a/src/test/regress/expected/jsonb_jsonpath.out +++ b/src/test/regress/expected/jsonb_jsonpath.out @@ -1877,25 +1877,25 @@ select jsonb_path_query('"2017-03-10 12:34:56"', '$.datetime()'); "2017-03-10T12:34:56" (1 row) -select jsonb_path_query('"2017-03-10 12:34:56 +3"', '$.datetime().type()'); +select jsonb_path_query('"2017-03-10 12:34:56+3"', '$.datetime().type()'); jsonb_path_query ---------------------------- "timestamp with time zone" (1 row) -select jsonb_path_query('"2017-03-10 12:34:56 +3"', '$.datetime()'); +select jsonb_path_query('"2017-03-10 12:34:56+3"', '$.datetime()'); jsonb_path_query ----------------------------- "2017-03-10T12:34:56+03:00" (1 row) -select jsonb_path_query('"2017-03-10 12:34:56 +3:10"', '$.datetime().type()'); +select jsonb_path_query('"2017-03-10 12:34:56+3:10"', '$.datetime().type()'); jsonb_path_query ---------------------------- "timestamp with time zone" (1 row) -select jsonb_path_query('"2017-03-10 12:34:56 +3:10"', '$.datetime()'); +select jsonb_path_query('"2017-03-10 12:34:56+3:10"', '$.datetime()'); jsonb_path_query ----------------------------- "2017-03-10T12:34:56+03:10" @@ -1913,25 +1913,25 @@ select jsonb_path_query('"12:34:56"', '$.datetime()'); "12:34:56" (1 row) -select jsonb_path_query('"12:34:56 +3"', '$.datetime().type()'); +select jsonb_path_query('"12:34:56+3"', '$.datetime().type()'); jsonb_path_query ----------------------- "time with time zone" (1 row) -select jsonb_path_query('"12:34:56 +3"', '$.datetime()'); +select jsonb_path_query('"12:34:56+3"', '$.datetime()'); jsonb_path_query ------------------ "12:34:56+03:00" (1 row) -select jsonb_path_query('"12:34:56 +3:10"', '$.datetime().type()'); +select jsonb_path_query('"12:34:56+3:10"', '$.datetime().type()'); jsonb_path_query ----------------------- "time with time zone" (1 row) -select jsonb_path_query('"12:34:56 +3:10"', '$.datetime()'); +select jsonb_path_query('"12:34:56+3:10"', '$.datetime()'); jsonb_path_query ------------------ "12:34:56+03:10" @@ -1940,22 +1940,22 @@ select jsonb_path_query('"12:34:56 +3:10"', '$.datetime()'); set time zone '+00'; -- date comparison select jsonb_path_query( - '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03 +04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03 +04", "2017-03-10 03:00:00 +03"]', + '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03+04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03+04", "2017-03-10 03:00:00+03"]', '$[*].datetime() ? (@ == "10.03.2017".datetime("dd.mm.yyyy"))'); ERROR: cannot convert value from date to timestamptz without time zone usage HINT: Use *_tz() function for time zone support. select jsonb_path_query( - '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03 +04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03 +04", "2017-03-10 03:00:00 +03"]', + '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03+04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03+04", "2017-03-10 03:00:00+03"]', '$[*].datetime() ? (@ >= "10.03.2017".datetime("dd.mm.yyyy"))'); ERROR: cannot convert value from date to timestamptz without time zone usage HINT: Use *_tz() function for time zone support. select jsonb_path_query( - '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03 +04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03 +04", "2017-03-10 03:00:00 +03"]', + '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03+04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03+04", "2017-03-10 03:00:00+03"]', '$[*].datetime() ? (@ < "10.03.2017".datetime("dd.mm.yyyy"))'); ERROR: cannot convert value from date to timestamptz without time zone usage HINT: Use *_tz() function for time zone support. select jsonb_path_query_tz( - '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03 +04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03 +04", "2017-03-10 03:00:00 +03"]', + '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03+04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03+04", "2017-03-10 03:00:00+03"]', '$[*].datetime() ? (@ == "10.03.2017".datetime("dd.mm.yyyy"))'); jsonb_path_query_tz ----------------------------- @@ -1965,7 +1965,7 @@ select jsonb_path_query_tz( (3 rows) select jsonb_path_query_tz( - '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03 +04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03 +04", "2017-03-10 03:00:00 +03"]', + '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03+04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03+04", "2017-03-10 03:00:00+03"]', '$[*].datetime() ? (@ >= "10.03.2017".datetime("dd.mm.yyyy"))'); jsonb_path_query_tz ----------------------------- @@ -1977,7 +1977,7 @@ select jsonb_path_query_tz( (5 rows) select jsonb_path_query_tz( - '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03 +04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03 +04", "2017-03-10 03:00:00 +03"]', + '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03+04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03+04", "2017-03-10 03:00:00+03"]', '$[*].datetime() ? (@ < "10.03.2017".datetime("dd.mm.yyyy"))'); jsonb_path_query_tz ----------------------------- @@ -1987,22 +1987,22 @@ select jsonb_path_query_tz( -- time comparison select jsonb_path_query( - '["12:34:00", "12:35:00", "12:36:00", "12:35:00 +00", "12:35:00 +01", "13:35:00 +01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +01"]', + '["12:34:00", "12:35:00", "12:36:00", "12:35:00+00", "12:35:00+01", "13:35:00+01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00+01"]', '$[*].datetime() ? (@ == "12:35".datetime("HH24:MI"))'); ERROR: cannot convert value from time to timetz without time zone usage HINT: Use *_tz() function for time zone support. select jsonb_path_query( - '["12:34:00", "12:35:00", "12:36:00", "12:35:00 +00", "12:35:00 +01", "13:35:00 +01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +01"]', + '["12:34:00", "12:35:00", "12:36:00", "12:35:00+00", "12:35:00+01", "13:35:00+01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00+01"]', '$[*].datetime() ? (@ >= "12:35".datetime("HH24:MI"))'); ERROR: cannot convert value from time to timetz without time zone usage HINT: Use *_tz() function for time zone support. select jsonb_path_query( - '["12:34:00", "12:35:00", "12:36:00", "12:35:00 +00", "12:35:00 +01", "13:35:00 +01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +01"]', + '["12:34:00", "12:35:00", "12:36:00", "12:35:00+00", "12:35:00+01", "13:35:00+01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00+01"]', '$[*].datetime() ? (@ < "12:35".datetime("HH24:MI"))'); ERROR: cannot convert value from time to timetz without time zone usage HINT: Use *_tz() function for time zone support. select jsonb_path_query_tz( - '["12:34:00", "12:35:00", "12:36:00", "12:35:00 +00", "12:35:00 +01", "13:35:00 +01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +01"]', + '["12:34:00", "12:35:00", "12:36:00", "12:35:00+00", "12:35:00+01", "13:35:00+01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00+01"]', '$[*].datetime() ? (@ == "12:35".datetime("HH24:MI"))'); jsonb_path_query_tz --------------------- @@ -2011,7 +2011,7 @@ select jsonb_path_query_tz( (2 rows) select jsonb_path_query_tz( - '["12:34:00", "12:35:00", "12:36:00", "12:35:00 +00", "12:35:00 +01", "13:35:00 +01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +01"]', + '["12:34:00", "12:35:00", "12:36:00", "12:35:00+00", "12:35:00+01", "13:35:00+01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00+01"]', '$[*].datetime() ? (@ >= "12:35".datetime("HH24:MI"))'); jsonb_path_query_tz --------------------- @@ -2021,7 +2021,7 @@ select jsonb_path_query_tz( (3 rows) select jsonb_path_query_tz( - '["12:34:00", "12:35:00", "12:36:00", "12:35:00 +00", "12:35:00 +01", "13:35:00 +01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +01"]', + '["12:34:00", "12:35:00", "12:36:00", "12:35:00+00", "12:35:00+01", "13:35:00+01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00+01"]', '$[*].datetime() ? (@ < "12:35".datetime("HH24:MI"))'); jsonb_path_query_tz --------------------- @@ -2032,22 +2032,22 @@ select jsonb_path_query_tz( -- timetz comparison select jsonb_path_query( - '["12:34:00 +01", "12:35:00 +01", "12:36:00 +01", "12:35:00 +02", "12:35:00 -02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', + '["12:34:00+01", "12:35:00+01", "12:36:00+01", "12:35:00+02", "12:35:00-02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', '$[*].datetime() ? (@ == "12:35 +1".datetime("HH24:MI TZH"))'); ERROR: cannot convert value from time to timetz without time zone usage HINT: Use *_tz() function for time zone support. select jsonb_path_query( - '["12:34:00 +01", "12:35:00 +01", "12:36:00 +01", "12:35:00 +02", "12:35:00 -02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', + '["12:34:00+01", "12:35:00+01", "12:36:00+01", "12:35:00+02", "12:35:00-02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', '$[*].datetime() ? (@ >= "12:35 +1".datetime("HH24:MI TZH"))'); ERROR: cannot convert value from time to timetz without time zone usage HINT: Use *_tz() function for time zone support. select jsonb_path_query( - '["12:34:00 +01", "12:35:00 +01", "12:36:00 +01", "12:35:00 +02", "12:35:00 -02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', + '["12:34:00+01", "12:35:00+01", "12:36:00+01", "12:35:00+02", "12:35:00-02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', '$[*].datetime() ? (@ < "12:35 +1".datetime("HH24:MI TZH"))'); ERROR: cannot convert value from time to timetz without time zone usage HINT: Use *_tz() function for time zone support. select jsonb_path_query_tz( - '["12:34:00 +01", "12:35:00 +01", "12:36:00 +01", "12:35:00 +02", "12:35:00 -02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', + '["12:34:00+01", "12:35:00+01", "12:36:00+01", "12:35:00+02", "12:35:00-02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', '$[*].datetime() ? (@ == "12:35 +1".datetime("HH24:MI TZH"))'); jsonb_path_query_tz --------------------- @@ -2055,7 +2055,7 @@ select jsonb_path_query_tz( (1 row) select jsonb_path_query_tz( - '["12:34:00 +01", "12:35:00 +01", "12:36:00 +01", "12:35:00 +02", "12:35:00 -02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', + '["12:34:00+01", "12:35:00+01", "12:36:00+01", "12:35:00+02", "12:35:00-02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', '$[*].datetime() ? (@ >= "12:35 +1".datetime("HH24:MI TZH"))'); jsonb_path_query_tz --------------------- @@ -2067,7 +2067,7 @@ select jsonb_path_query_tz( (5 rows) select jsonb_path_query_tz( - '["12:34:00 +01", "12:35:00 +01", "12:36:00 +01", "12:35:00 +02", "12:35:00 -02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', + '["12:34:00+01", "12:35:00+01", "12:36:00+01", "12:35:00+02", "12:35:00-02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', '$[*].datetime() ? (@ < "12:35 +1".datetime("HH24:MI TZH"))'); jsonb_path_query_tz --------------------- @@ -2078,22 +2078,22 @@ select jsonb_path_query_tz( -- timestamp comparison select jsonb_path_query( - '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00 +01", "2017-03-10 13:35:00 +01", "2017-03-10 12:35:00 -01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00+01", "2017-03-10 13:35:00+01", "2017-03-10 12:35:00-01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ == "10.03.2017 12:35".datetime("dd.mm.yyyy HH24:MI"))'); ERROR: cannot convert value from timestamp to timestamptz without time zone usage HINT: Use *_tz() function for time zone support. select jsonb_path_query( - '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00 +01", "2017-03-10 13:35:00 +01", "2017-03-10 12:35:00 -01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00+01", "2017-03-10 13:35:00+01", "2017-03-10 12:35:00-01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ >= "10.03.2017 12:35".datetime("dd.mm.yyyy HH24:MI"))'); ERROR: cannot convert value from timestamp to timestamptz without time zone usage HINT: Use *_tz() function for time zone support. select jsonb_path_query( - '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00 +01", "2017-03-10 13:35:00 +01", "2017-03-10 12:35:00 -01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00+01", "2017-03-10 13:35:00+01", "2017-03-10 12:35:00-01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ < "10.03.2017 12:35".datetime("dd.mm.yyyy HH24:MI"))'); ERROR: cannot convert value from timestamp to timestamptz without time zone usage HINT: Use *_tz() function for time zone support. select jsonb_path_query_tz( - '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00 +01", "2017-03-10 13:35:00 +01", "2017-03-10 12:35:00 -01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00+01", "2017-03-10 13:35:00+01", "2017-03-10 12:35:00-01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ == "10.03.2017 12:35".datetime("dd.mm.yyyy HH24:MI"))'); jsonb_path_query_tz ----------------------------- @@ -2102,7 +2102,7 @@ select jsonb_path_query_tz( (2 rows) select jsonb_path_query_tz( - '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00 +01", "2017-03-10 13:35:00 +01", "2017-03-10 12:35:00 -01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00+01", "2017-03-10 13:35:00+01", "2017-03-10 12:35:00-01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ >= "10.03.2017 12:35".datetime("dd.mm.yyyy HH24:MI"))'); jsonb_path_query_tz ----------------------------- @@ -2114,7 +2114,7 @@ select jsonb_path_query_tz( (5 rows) select jsonb_path_query_tz( - '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00 +01", "2017-03-10 13:35:00 +01", "2017-03-10 12:35:00 -01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00+01", "2017-03-10 13:35:00+01", "2017-03-10 12:35:00-01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ < "10.03.2017 12:35".datetime("dd.mm.yyyy HH24:MI"))'); jsonb_path_query_tz ----------------------------- @@ -2125,22 +2125,22 @@ select jsonb_path_query_tz( -- timestamptz comparison select jsonb_path_query( - '["2017-03-10 12:34:00 +01", "2017-03-10 12:35:00 +01", "2017-03-10 12:36:00 +01", "2017-03-10 12:35:00 +02", "2017-03-10 12:35:00 -02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00+01", "2017-03-10 12:35:00+01", "2017-03-10 12:36:00+01", "2017-03-10 12:35:00+02", "2017-03-10 12:35:00-02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ == "10.03.2017 12:35 +1".datetime("dd.mm.yyyy HH24:MI TZH"))'); ERROR: cannot convert value from timestamp to timestamptz without time zone usage HINT: Use *_tz() function for time zone support. select jsonb_path_query( - '["2017-03-10 12:34:00 +01", "2017-03-10 12:35:00 +01", "2017-03-10 12:36:00 +01", "2017-03-10 12:35:00 +02", "2017-03-10 12:35:00 -02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00+01", "2017-03-10 12:35:00+01", "2017-03-10 12:36:00+01", "2017-03-10 12:35:00+02", "2017-03-10 12:35:00-02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ >= "10.03.2017 12:35 +1".datetime("dd.mm.yyyy HH24:MI TZH"))'); ERROR: cannot convert value from timestamp to timestamptz without time zone usage HINT: Use *_tz() function for time zone support. select jsonb_path_query( - '["2017-03-10 12:34:00 +01", "2017-03-10 12:35:00 +01", "2017-03-10 12:36:00 +01", "2017-03-10 12:35:00 +02", "2017-03-10 12:35:00 -02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00+01", "2017-03-10 12:35:00+01", "2017-03-10 12:36:00+01", "2017-03-10 12:35:00+02", "2017-03-10 12:35:00-02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ < "10.03.2017 12:35 +1".datetime("dd.mm.yyyy HH24:MI TZH"))'); ERROR: cannot convert value from timestamp to timestamptz without time zone usage HINT: Use *_tz() function for time zone support. select jsonb_path_query_tz( - '["2017-03-10 12:34:00 +01", "2017-03-10 12:35:00 +01", "2017-03-10 12:36:00 +01", "2017-03-10 12:35:00 +02", "2017-03-10 12:35:00 -02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00+01", "2017-03-10 12:35:00+01", "2017-03-10 12:36:00+01", "2017-03-10 12:35:00+02", "2017-03-10 12:35:00-02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ == "10.03.2017 12:35 +1".datetime("dd.mm.yyyy HH24:MI TZH"))'); jsonb_path_query_tz ----------------------------- @@ -2149,7 +2149,7 @@ select jsonb_path_query_tz( (2 rows) select jsonb_path_query_tz( - '["2017-03-10 12:34:00 +01", "2017-03-10 12:35:00 +01", "2017-03-10 12:36:00 +01", "2017-03-10 12:35:00 +02", "2017-03-10 12:35:00 -02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00+01", "2017-03-10 12:35:00+01", "2017-03-10 12:36:00+01", "2017-03-10 12:35:00+02", "2017-03-10 12:35:00-02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ >= "10.03.2017 12:35 +1".datetime("dd.mm.yyyy HH24:MI TZH"))'); jsonb_path_query_tz ----------------------------- @@ -2162,7 +2162,7 @@ select jsonb_path_query_tz( (6 rows) select jsonb_path_query_tz( - '["2017-03-10 12:34:00 +01", "2017-03-10 12:35:00 +01", "2017-03-10 12:36:00 +01", "2017-03-10 12:35:00 +02", "2017-03-10 12:35:00 -02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00+01", "2017-03-10 12:35:00+01", "2017-03-10 12:36:00+01", "2017-03-10 12:35:00+02", "2017-03-10 12:35:00-02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ < "10.03.2017 12:35 +1".datetime("dd.mm.yyyy HH24:MI TZH"))'); jsonb_path_query_tz ----------------------------- diff --git a/src/test/regress/sql/jsonb_jsonpath.sql b/src/test/regress/sql/jsonb_jsonpath.sql index a50abed95da7..dc25ceb283df 100644 --- a/src/test/regress/sql/jsonb_jsonpath.sql +++ b/src/test/regress/sql/jsonb_jsonpath.sql @@ -404,117 +404,117 @@ select jsonb_path_query('"2017-03-10"', '$.datetime().type()'); select jsonb_path_query('"2017-03-10"', '$.datetime()'); select jsonb_path_query('"2017-03-10 12:34:56"', '$.datetime().type()'); select jsonb_path_query('"2017-03-10 12:34:56"', '$.datetime()'); -select jsonb_path_query('"2017-03-10 12:34:56 +3"', '$.datetime().type()'); -select jsonb_path_query('"2017-03-10 12:34:56 +3"', '$.datetime()'); -select jsonb_path_query('"2017-03-10 12:34:56 +3:10"', '$.datetime().type()'); -select jsonb_path_query('"2017-03-10 12:34:56 +3:10"', '$.datetime()'); +select jsonb_path_query('"2017-03-10 12:34:56+3"', '$.datetime().type()'); +select jsonb_path_query('"2017-03-10 12:34:56+3"', '$.datetime()'); +select jsonb_path_query('"2017-03-10 12:34:56+3:10"', '$.datetime().type()'); +select jsonb_path_query('"2017-03-10 12:34:56+3:10"', '$.datetime()'); select jsonb_path_query('"12:34:56"', '$.datetime().type()'); select jsonb_path_query('"12:34:56"', '$.datetime()'); -select jsonb_path_query('"12:34:56 +3"', '$.datetime().type()'); -select jsonb_path_query('"12:34:56 +3"', '$.datetime()'); -select jsonb_path_query('"12:34:56 +3:10"', '$.datetime().type()'); -select jsonb_path_query('"12:34:56 +3:10"', '$.datetime()'); +select jsonb_path_query('"12:34:56+3"', '$.datetime().type()'); +select jsonb_path_query('"12:34:56+3"', '$.datetime()'); +select jsonb_path_query('"12:34:56+3:10"', '$.datetime().type()'); +select jsonb_path_query('"12:34:56+3:10"', '$.datetime()'); set time zone '+00'; -- date comparison select jsonb_path_query( - '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03 +04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03 +04", "2017-03-10 03:00:00 +03"]', + '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03+04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03+04", "2017-03-10 03:00:00+03"]', '$[*].datetime() ? (@ == "10.03.2017".datetime("dd.mm.yyyy"))'); select jsonb_path_query( - '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03 +04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03 +04", "2017-03-10 03:00:00 +03"]', + '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03+04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03+04", "2017-03-10 03:00:00+03"]', '$[*].datetime() ? (@ >= "10.03.2017".datetime("dd.mm.yyyy"))'); select jsonb_path_query( - '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03 +04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03 +04", "2017-03-10 03:00:00 +03"]', + '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03+04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03+04", "2017-03-10 03:00:00+03"]', '$[*].datetime() ? (@ < "10.03.2017".datetime("dd.mm.yyyy"))'); select jsonb_path_query_tz( - '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03 +04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03 +04", "2017-03-10 03:00:00 +03"]', + '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03+04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03+04", "2017-03-10 03:00:00+03"]', '$[*].datetime() ? (@ == "10.03.2017".datetime("dd.mm.yyyy"))'); select jsonb_path_query_tz( - '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03 +04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03 +04", "2017-03-10 03:00:00 +03"]', + '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03+04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03+04", "2017-03-10 03:00:00+03"]', '$[*].datetime() ? (@ >= "10.03.2017".datetime("dd.mm.yyyy"))'); select jsonb_path_query_tz( - '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03 +04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03 +04", "2017-03-10 03:00:00 +03"]', + '["2017-03-10", "2017-03-11", "2017-03-09", "12:34:56", "01:02:03+04", "2017-03-10 00:00:00", "2017-03-10 12:34:56", "2017-03-10 01:02:03+04", "2017-03-10 03:00:00+03"]', '$[*].datetime() ? (@ < "10.03.2017".datetime("dd.mm.yyyy"))'); -- time comparison select jsonb_path_query( - '["12:34:00", "12:35:00", "12:36:00", "12:35:00 +00", "12:35:00 +01", "13:35:00 +01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +01"]', + '["12:34:00", "12:35:00", "12:36:00", "12:35:00+00", "12:35:00+01", "13:35:00+01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00+01"]', '$[*].datetime() ? (@ == "12:35".datetime("HH24:MI"))'); select jsonb_path_query( - '["12:34:00", "12:35:00", "12:36:00", "12:35:00 +00", "12:35:00 +01", "13:35:00 +01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +01"]', + '["12:34:00", "12:35:00", "12:36:00", "12:35:00+00", "12:35:00+01", "13:35:00+01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00+01"]', '$[*].datetime() ? (@ >= "12:35".datetime("HH24:MI"))'); select jsonb_path_query( - '["12:34:00", "12:35:00", "12:36:00", "12:35:00 +00", "12:35:00 +01", "13:35:00 +01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +01"]', + '["12:34:00", "12:35:00", "12:36:00", "12:35:00+00", "12:35:00+01", "13:35:00+01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00+01"]', '$[*].datetime() ? (@ < "12:35".datetime("HH24:MI"))'); select jsonb_path_query_tz( - '["12:34:00", "12:35:00", "12:36:00", "12:35:00 +00", "12:35:00 +01", "13:35:00 +01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +01"]', + '["12:34:00", "12:35:00", "12:36:00", "12:35:00+00", "12:35:00+01", "13:35:00+01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00+01"]', '$[*].datetime() ? (@ == "12:35".datetime("HH24:MI"))'); select jsonb_path_query_tz( - '["12:34:00", "12:35:00", "12:36:00", "12:35:00 +00", "12:35:00 +01", "13:35:00 +01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +01"]', + '["12:34:00", "12:35:00", "12:36:00", "12:35:00+00", "12:35:00+01", "13:35:00+01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00+01"]', '$[*].datetime() ? (@ >= "12:35".datetime("HH24:MI"))'); select jsonb_path_query_tz( - '["12:34:00", "12:35:00", "12:36:00", "12:35:00 +00", "12:35:00 +01", "13:35:00 +01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +01"]', + '["12:34:00", "12:35:00", "12:36:00", "12:35:00+00", "12:35:00+01", "13:35:00+01", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00+01"]', '$[*].datetime() ? (@ < "12:35".datetime("HH24:MI"))'); -- timetz comparison select jsonb_path_query( - '["12:34:00 +01", "12:35:00 +01", "12:36:00 +01", "12:35:00 +02", "12:35:00 -02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', + '["12:34:00+01", "12:35:00+01", "12:36:00+01", "12:35:00+02", "12:35:00-02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', '$[*].datetime() ? (@ == "12:35 +1".datetime("HH24:MI TZH"))'); select jsonb_path_query( - '["12:34:00 +01", "12:35:00 +01", "12:36:00 +01", "12:35:00 +02", "12:35:00 -02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', + '["12:34:00+01", "12:35:00+01", "12:36:00+01", "12:35:00+02", "12:35:00-02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', '$[*].datetime() ? (@ >= "12:35 +1".datetime("HH24:MI TZH"))'); select jsonb_path_query( - '["12:34:00 +01", "12:35:00 +01", "12:36:00 +01", "12:35:00 +02", "12:35:00 -02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', + '["12:34:00+01", "12:35:00+01", "12:36:00+01", "12:35:00+02", "12:35:00-02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', '$[*].datetime() ? (@ < "12:35 +1".datetime("HH24:MI TZH"))'); select jsonb_path_query_tz( - '["12:34:00 +01", "12:35:00 +01", "12:36:00 +01", "12:35:00 +02", "12:35:00 -02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', + '["12:34:00+01", "12:35:00+01", "12:36:00+01", "12:35:00+02", "12:35:00-02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', '$[*].datetime() ? (@ == "12:35 +1".datetime("HH24:MI TZH"))'); select jsonb_path_query_tz( - '["12:34:00 +01", "12:35:00 +01", "12:36:00 +01", "12:35:00 +02", "12:35:00 -02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', + '["12:34:00+01", "12:35:00+01", "12:36:00+01", "12:35:00+02", "12:35:00-02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', '$[*].datetime() ? (@ >= "12:35 +1".datetime("HH24:MI TZH"))'); select jsonb_path_query_tz( - '["12:34:00 +01", "12:35:00 +01", "12:36:00 +01", "12:35:00 +02", "12:35:00 -02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', + '["12:34:00+01", "12:35:00+01", "12:36:00+01", "12:35:00+02", "12:35:00-02", "10:35:00", "11:35:00", "12:35:00", "2017-03-10", "2017-03-10 12:35:00", "2017-03-10 12:35:00 +1"]', '$[*].datetime() ? (@ < "12:35 +1".datetime("HH24:MI TZH"))'); -- timestamp comparison select jsonb_path_query( - '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00 +01", "2017-03-10 13:35:00 +01", "2017-03-10 12:35:00 -01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00+01", "2017-03-10 13:35:00+01", "2017-03-10 12:35:00-01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ == "10.03.2017 12:35".datetime("dd.mm.yyyy HH24:MI"))'); select jsonb_path_query( - '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00 +01", "2017-03-10 13:35:00 +01", "2017-03-10 12:35:00 -01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00+01", "2017-03-10 13:35:00+01", "2017-03-10 12:35:00-01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ >= "10.03.2017 12:35".datetime("dd.mm.yyyy HH24:MI"))'); select jsonb_path_query( - '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00 +01", "2017-03-10 13:35:00 +01", "2017-03-10 12:35:00 -01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00+01", "2017-03-10 13:35:00+01", "2017-03-10 12:35:00-01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ < "10.03.2017 12:35".datetime("dd.mm.yyyy HH24:MI"))'); select jsonb_path_query_tz( - '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00 +01", "2017-03-10 13:35:00 +01", "2017-03-10 12:35:00 -01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00+01", "2017-03-10 13:35:00+01", "2017-03-10 12:35:00-01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ == "10.03.2017 12:35".datetime("dd.mm.yyyy HH24:MI"))'); select jsonb_path_query_tz( - '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00 +01", "2017-03-10 13:35:00 +01", "2017-03-10 12:35:00 -01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00+01", "2017-03-10 13:35:00+01", "2017-03-10 12:35:00-01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ >= "10.03.2017 12:35".datetime("dd.mm.yyyy HH24:MI"))'); select jsonb_path_query_tz( - '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00 +01", "2017-03-10 13:35:00 +01", "2017-03-10 12:35:00 -01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00", "2017-03-10 12:35:00", "2017-03-10 12:36:00", "2017-03-10 12:35:00+01", "2017-03-10 13:35:00+01", "2017-03-10 12:35:00-01", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ < "10.03.2017 12:35".datetime("dd.mm.yyyy HH24:MI"))'); -- timestamptz comparison select jsonb_path_query( - '["2017-03-10 12:34:00 +01", "2017-03-10 12:35:00 +01", "2017-03-10 12:36:00 +01", "2017-03-10 12:35:00 +02", "2017-03-10 12:35:00 -02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00+01", "2017-03-10 12:35:00+01", "2017-03-10 12:36:00+01", "2017-03-10 12:35:00+02", "2017-03-10 12:35:00-02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ == "10.03.2017 12:35 +1".datetime("dd.mm.yyyy HH24:MI TZH"))'); select jsonb_path_query( - '["2017-03-10 12:34:00 +01", "2017-03-10 12:35:00 +01", "2017-03-10 12:36:00 +01", "2017-03-10 12:35:00 +02", "2017-03-10 12:35:00 -02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00+01", "2017-03-10 12:35:00+01", "2017-03-10 12:36:00+01", "2017-03-10 12:35:00+02", "2017-03-10 12:35:00-02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ >= "10.03.2017 12:35 +1".datetime("dd.mm.yyyy HH24:MI TZH"))'); select jsonb_path_query( - '["2017-03-10 12:34:00 +01", "2017-03-10 12:35:00 +01", "2017-03-10 12:36:00 +01", "2017-03-10 12:35:00 +02", "2017-03-10 12:35:00 -02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00+01", "2017-03-10 12:35:00+01", "2017-03-10 12:36:00+01", "2017-03-10 12:35:00+02", "2017-03-10 12:35:00-02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ < "10.03.2017 12:35 +1".datetime("dd.mm.yyyy HH24:MI TZH"))'); select jsonb_path_query_tz( - '["2017-03-10 12:34:00 +01", "2017-03-10 12:35:00 +01", "2017-03-10 12:36:00 +01", "2017-03-10 12:35:00 +02", "2017-03-10 12:35:00 -02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00+01", "2017-03-10 12:35:00+01", "2017-03-10 12:36:00+01", "2017-03-10 12:35:00+02", "2017-03-10 12:35:00-02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ == "10.03.2017 12:35 +1".datetime("dd.mm.yyyy HH24:MI TZH"))'); select jsonb_path_query_tz( - '["2017-03-10 12:34:00 +01", "2017-03-10 12:35:00 +01", "2017-03-10 12:36:00 +01", "2017-03-10 12:35:00 +02", "2017-03-10 12:35:00 -02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00+01", "2017-03-10 12:35:00+01", "2017-03-10 12:36:00+01", "2017-03-10 12:35:00+02", "2017-03-10 12:35:00-02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ >= "10.03.2017 12:35 +1".datetime("dd.mm.yyyy HH24:MI TZH"))'); select jsonb_path_query_tz( - '["2017-03-10 12:34:00 +01", "2017-03-10 12:35:00 +01", "2017-03-10 12:36:00 +01", "2017-03-10 12:35:00 +02", "2017-03-10 12:35:00 -02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56 +01"]', + '["2017-03-10 12:34:00+01", "2017-03-10 12:35:00+01", "2017-03-10 12:36:00+01", "2017-03-10 12:35:00+02", "2017-03-10 12:35:00-02", "2017-03-10 10:35:00", "2017-03-10 11:35:00", "2017-03-10 12:35:00", "2017-03-10", "2017-03-11", "12:34:56", "12:34:56+01"]', '$[*].datetime() ? (@ < "10.03.2017 12:35 +1".datetime("dd.mm.yyyy HH24:MI TZH"))'); -- overflow during comparison From 927d9abb6538e441aa97bb3eea7a15d4cda70715 Mon Sep 17 00:00:00 2001 From: Alexander Korotkov Date: Tue, 29 Sep 2020 11:41:46 +0300 Subject: [PATCH 222/589] Support for ISO 8601 in the jsonpath .datetime() method The SQL standard doesn't require jsonpath .datetime() method to support the ISO 8601 format. But our to_json[b]() functions convert timestamps to text in the ISO 8601 format in the sake of compatibility with javascript. So, we add support of the ISO 8601 to the jsonpath .datetime() in the sake compatibility with to_json[b](). The standard mode of datetime parsing currently supports just template patterns and separators in the format string. In order to implement ISO 8601, we have to add support of the format string double quotes to the standard parsing mode. Discussion: https://postgr.es/m/94321be0-cc96-1a81-b6df-796f437f7c66%40postgrespro.ru Author: Nikita Glukhov, revised by me Backpatch-through: 13 --- src/backend/utils/adt/formatting.c | 20 +++++++++++++++++--- src/backend/utils/adt/jsonpath_exec.c | 8 +++++++- src/test/regress/expected/jsonb_jsonpath.out | 19 +++++++++++++++++++ src/test/regress/sql/jsonb_jsonpath.sql | 6 ++++++ 4 files changed, 49 insertions(+), 4 deletions(-) diff --git a/src/backend/utils/adt/formatting.c b/src/backend/utils/adt/formatting.c index f9aa968f0985..b91ff7bb8036 100644 --- a/src/backend/utils/adt/formatting.c +++ b/src/backend/utils/adt/formatting.c @@ -1381,10 +1381,12 @@ parse_format(FormatNode *node, const char *str, const KeyWord *kw, { int chlen; - if (flags & STD_FLAG) + if ((flags & STD_FLAG) && *str != '"') { /* - * Standard mode, allow only following separators: "-./,':; " + * Standard mode, allow only following separators: "-./,':; ". + * However, we support double quotes even in standard mode + * (see below). This is our extension of standard mode. */ if (strchr("-./,':; ", *str) == NULL) ereport(ERROR, @@ -3346,7 +3348,19 @@ DCH_from_char(FormatNode *node, const char *in, TmFromChar *out, } else { - s += pg_mblen(s); + int chlen = pg_mblen(s); + + /* + * Standard mode requires strict match of format characters. + */ + if (std && n->type == NODE_TYPE_CHAR && + strncmp(s, n->character, chlen) != 0) + RETURN_ERROR(ereport(ERROR, + (errcode(ERRCODE_INVALID_DATETIME_FORMAT), + errmsg("unmatched format character \"%s\"", + n->character)))); + + s += chlen; } continue; } diff --git a/src/backend/utils/adt/jsonpath_exec.c b/src/backend/utils/adt/jsonpath_exec.c index 31b88d674166..28be845770a6 100644 --- a/src/backend/utils/adt/jsonpath_exec.c +++ b/src/backend/utils/adt/jsonpath_exec.c @@ -1833,6 +1833,9 @@ executeDateTimeMethod(JsonPathExecContext *cxt, JsonPathItem *jsp, /* * According to SQL/JSON standard enumerate ISO formats for: date, * timetz, time, timestamptz, timestamp. + * + * We also support ISO 8601 for timestamps, because to_json[b]() + * functions use this format. */ static const char *fmt_str[] = { @@ -1842,7 +1845,10 @@ executeDateTimeMethod(JsonPathExecContext *cxt, JsonPathItem *jsp, "HH24:MI:SS", "yyyy-mm-dd HH24:MI:SSTZH:TZM", "yyyy-mm-dd HH24:MI:SSTZH", - "yyyy-mm-dd HH24:MI:SS" + "yyyy-mm-dd HH24:MI:SS", + "yyyy-mm-dd\"T\"HH24:MI:SSTZH:TZM", + "yyyy-mm-dd\"T\"HH24:MI:SSTZH", + "yyyy-mm-dd\"T\"HH24:MI:SS" }; /* cache for format texts */ diff --git a/src/test/regress/expected/jsonb_jsonpath.out b/src/test/regress/expected/jsonb_jsonpath.out index 31d6f05a7c20..508ddd797ed5 100644 --- a/src/test/regress/expected/jsonb_jsonpath.out +++ b/src/test/regress/expected/jsonb_jsonpath.out @@ -1722,6 +1722,16 @@ select jsonb_path_query('"12:34:56 +05:20"', '$.datetime("HH24:MI:SS TZH:TZM").t "time with time zone" (1 row) +select jsonb_path_query('"10-03-2017T12:34:56"', '$.datetime("dd-mm-yyyy\"T\"HH24:MI:SS")'); + jsonb_path_query +----------------------- + "2017-03-10T12:34:56" +(1 row) + +select jsonb_path_query('"10-03-2017t12:34:56"', '$.datetime("dd-mm-yyyy\"T\"HH24:MI:SS")'); +ERROR: unmatched format character "T" +select jsonb_path_query('"10-03-2017 12:34:56"', '$.datetime("dd-mm-yyyy\"T\"HH24:MI:SS")'); +ERROR: unmatched format character "T" set time zone '+00'; select jsonb_path_query('"10-03-2017 12:34"', '$.datetime("dd-mm-yyyy HH24:MI")'); jsonb_path_query @@ -1901,6 +1911,15 @@ select jsonb_path_query('"2017-03-10 12:34:56+3:10"', '$.datetime()'); "2017-03-10T12:34:56+03:10" (1 row) +select jsonb_path_query('"2017-03-10T12:34:56+3:10"', '$.datetime()'); + jsonb_path_query +----------------------------- + "2017-03-10T12:34:56+03:10" +(1 row) + +select jsonb_path_query('"2017-03-10t12:34:56+3:10"', '$.datetime()'); +ERROR: datetime format is not recognized: "2017-03-10t12:34:56+3:10" +HINT: Use a datetime template argument to specify the input data format. select jsonb_path_query('"12:34:56"', '$.datetime().type()'); jsonb_path_query -------------------------- diff --git a/src/test/regress/sql/jsonb_jsonpath.sql b/src/test/regress/sql/jsonb_jsonpath.sql index dc25ceb283df..60f73cb05906 100644 --- a/src/test/regress/sql/jsonb_jsonpath.sql +++ b/src/test/regress/sql/jsonb_jsonpath.sql @@ -368,6 +368,10 @@ select jsonb_path_query('"10-03-2017 12:34 +05:20"', '$.datetime("dd-mm-yyyy HH2 select jsonb_path_query('"12:34:56"', '$.datetime("HH24:MI:SS").type()'); select jsonb_path_query('"12:34:56 +05:20"', '$.datetime("HH24:MI:SS TZH:TZM").type()'); +select jsonb_path_query('"10-03-2017T12:34:56"', '$.datetime("dd-mm-yyyy\"T\"HH24:MI:SS")'); +select jsonb_path_query('"10-03-2017t12:34:56"', '$.datetime("dd-mm-yyyy\"T\"HH24:MI:SS")'); +select jsonb_path_query('"10-03-2017 12:34:56"', '$.datetime("dd-mm-yyyy\"T\"HH24:MI:SS")'); + set time zone '+00'; select jsonb_path_query('"10-03-2017 12:34"', '$.datetime("dd-mm-yyyy HH24:MI")'); @@ -408,6 +412,8 @@ select jsonb_path_query('"2017-03-10 12:34:56+3"', '$.datetime().type()'); select jsonb_path_query('"2017-03-10 12:34:56+3"', '$.datetime()'); select jsonb_path_query('"2017-03-10 12:34:56+3:10"', '$.datetime().type()'); select jsonb_path_query('"2017-03-10 12:34:56+3:10"', '$.datetime()'); +select jsonb_path_query('"2017-03-10T12:34:56+3:10"', '$.datetime()'); +select jsonb_path_query('"2017-03-10t12:34:56+3:10"', '$.datetime()'); select jsonb_path_query('"12:34:56"', '$.datetime().type()'); select jsonb_path_query('"12:34:56"', '$.datetime()'); select jsonb_path_query('"12:34:56+3"', '$.datetime().type()'); From a6b1f5365d58356b5d42829e9cd89a6c725d7a0a Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 29 Sep 2020 11:18:30 -0400 Subject: [PATCH 223/589] Fix memory leak in plpgsql's CALL processing. When executing a CALL or DO in a non-atomic context (i.e., not inside a function or query), plpgsql creates a new plan each time through, as a rather hacky solution to some resource management issues. But it failed to free this plan until exit of the current procedure or DO block, resulting in serious memory bloat in procedures that called other procedures many times. Fix by remembering to free the plan, and by being more honest about restoring the previous state (otherwise, recursive procedure calls have a problem). There was also a smaller leak associated with recalculation of the "target" list of output variables. Fix that by using the statement- lifespan context to hold non-permanent values. Back-patch to v11 where procedures were introduced. Pavel Stehule and Tom Lane Discussion: https://postgr.es/m/CAFj8pRDiiU1dqym+_P4_GuTWm76knJu7z9opWayBJTC0nQGUUA@mail.gmail.com --- src/pl/plpgsql/src/pl_exec.c | 91 +++++++++++++++++++++++++++--------- 1 file changed, 70 insertions(+), 21 deletions(-) diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c index d4a3d58daa9e..ccbc50fc457c 100644 --- a/src/pl/plpgsql/src/pl_exec.c +++ b/src/pl/plpgsql/src/pl_exec.c @@ -2145,40 +2145,60 @@ exec_stmt_perform(PLpgSQL_execstate *estate, PLpgSQL_stmt_perform *stmt) /* * exec_stmt_call + * + * NOTE: this is used for both CALL and DO statements. */ static int exec_stmt_call(PLpgSQL_execstate *estate, PLpgSQL_stmt_call *stmt) { PLpgSQL_expr *expr = stmt->expr; + SPIPlanPtr orig_plan = expr->plan; + bool local_plan; + PLpgSQL_variable *volatile cur_target = stmt->target; volatile LocalTransactionId before_lxid; LocalTransactionId after_lxid; volatile bool pushed_active_snap = false; volatile int rc; + /* + * If not in atomic context, we make a local plan that we'll just use for + * this invocation, and will free at the end. Otherwise, transaction ends + * would cause errors about plancache leaks. + * + * XXX This would be fixable with some plancache/resowner surgery + * elsewhere, but for now we'll just work around this here. + */ + local_plan = !estate->atomic; + /* PG_TRY to ensure we clear the plan link, if needed, on failure */ PG_TRY(); { SPIPlanPtr plan = expr->plan; ParamListInfo paramLI; - if (plan == NULL) + /* + * Make a plan if we don't have one, or if we need a local one. Note + * that we'll overwrite expr->plan either way; the PG_TRY block will + * ensure we undo that on the way out, if the plan is local. + */ + if (plan == NULL || local_plan) { + /* Don't let SPI save the plan if it's going to be local */ + exec_prepare_plan(estate, expr, 0, !local_plan); + plan = expr->plan; /* - * Don't save the plan if not in atomic context. Otherwise, - * transaction ends would cause errors about plancache leaks. - * - * XXX This would be fixable with some plancache/resowner surgery - * elsewhere, but for now we'll just work around this here. + * A CALL or DO can never be a simple expression. (If it could + * be, we'd have to worry about saving/restoring the previous + * values of the related expr fields, not just expr->plan.) */ - exec_prepare_plan(estate, expr, 0, estate->atomic); + Assert(!expr->expr_simple_expr); /* * The procedure call could end transactions, which would upset * the snapshot management in SPI_execute*, so don't let it do it. * Instead, we set the snapshots ourselves below. */ - plan = expr->plan; plan->no_snapshots = true; /* @@ -2186,14 +2206,21 @@ exec_stmt_call(PLpgSQL_execstate *estate, PLpgSQL_stmt_call *stmt) * case the procedure's argument list has changed. */ stmt->target = NULL; + cur_target = NULL; } /* * We construct a DTYPE_ROW datum representing the plpgsql variables * associated with the procedure's output arguments. Then we can use * exec_move_row() to do the assignments. + * + * If we're using a local plan, also make a local target; otherwise, + * since the above code will force a new plan each time through, we'd + * repeatedly leak the memory for the target. (Note: we also leak the + * target when a plan change is forced, but that isn't so likely to + * cause excessive memory leaks.) */ - if (stmt->is_call && stmt->target == NULL) + if (stmt->is_call && cur_target == NULL) { Node *node; FuncExpr *funcexpr; @@ -2208,6 +2235,9 @@ exec_stmt_call(PLpgSQL_execstate *estate, PLpgSQL_stmt_call *stmt) int i; ListCell *lc; + /* Use eval_mcontext for any cruft accumulated here */ + oldcontext = MemoryContextSwitchTo(get_eval_mcontext(estate)); + /* * Get the parsed CallStmt, and look up the called procedure */ @@ -2239,9 +2269,11 @@ exec_stmt_call(PLpgSQL_execstate *estate, PLpgSQL_stmt_call *stmt) ReleaseSysCache(func_tuple); /* - * Begin constructing row Datum + * Begin constructing row Datum; keep it in fn_cxt if it's to be + * long-lived. */ - oldcontext = MemoryContextSwitchTo(estate->func->fn_cxt); + if (!local_plan) + MemoryContextSwitchTo(estate->func->fn_cxt); row = (PLpgSQL_row *) palloc0(sizeof(PLpgSQL_row)); row->dtype = PLPGSQL_DTYPE_ROW; @@ -2249,7 +2281,8 @@ exec_stmt_call(PLpgSQL_execstate *estate, PLpgSQL_stmt_call *stmt) row->lineno = -1; row->varnos = (int *) palloc(sizeof(int) * list_length(funcargs)); - MemoryContextSwitchTo(oldcontext); + if (!local_plan) + MemoryContextSwitchTo(get_eval_mcontext(estate)); /* * Examine procedure's argument list. Each output arg position @@ -2293,7 +2326,13 @@ exec_stmt_call(PLpgSQL_execstate *estate, PLpgSQL_stmt_call *stmt) row->nfields = nfields; - stmt->target = (PLpgSQL_variable *) row; + cur_target = (PLpgSQL_variable *) row; + + /* We can save and re-use the target datum, if it's not local */ + if (!local_plan) + stmt->target = cur_target; + + MemoryContextSwitchTo(oldcontext); } paramLI = setup_param_list(estate, expr); @@ -2316,17 +2355,27 @@ exec_stmt_call(PLpgSQL_execstate *estate, PLpgSQL_stmt_call *stmt) PG_CATCH(); { /* - * If we aren't saving the plan, unset the pointer. Note that it - * could have been unset already, in case of a recursive call. + * If we are using a local plan, restore the old plan link. */ - if (expr->plan && !expr->plan->saved) - expr->plan = NULL; + if (local_plan) + expr->plan = orig_plan; PG_RE_THROW(); } PG_END_TRY(); - if (expr->plan && !expr->plan->saved) - expr->plan = NULL; + /* + * If we are using a local plan, restore the old plan link; then free the + * local plan to avoid memory leaks. (Note that the error exit path above + * just clears the link without risking calling SPI_freeplan; we expect + * that xact cleanup will take care of the mess in that case.) + */ + if (local_plan) + { + SPIPlanPtr plan = expr->plan; + + expr->plan = orig_plan; + SPI_freeplan(plan); + } if (rc < 0) elog(ERROR, "SPI_execute_plan_with_paramlist failed executing query \"%s\": %s", @@ -2363,10 +2412,10 @@ exec_stmt_call(PLpgSQL_execstate *estate, PLpgSQL_stmt_call *stmt) { SPITupleTable *tuptab = SPI_tuptable; - if (!stmt->target) + if (!cur_target) elog(ERROR, "DO statement returned a row"); - exec_move_row(estate, stmt->target, tuptab->vals[0], tuptab->tupdesc); + exec_move_row(estate, cur_target, tuptab->vals[0], tuptab->tupdesc); } else if (SPI_processed > 1) elog(ERROR, "procedure call returned more than one row"); From a094c8ff53523e88ff9dd28ad467618039e27b58 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 29 Sep 2020 13:48:06 -0400 Subject: [PATCH 224/589] Fix make_timestamp[tz] to accept negative years as meaning BC. Previously we threw an error. But make_date already allowed the case, so it is inconsistent as well as unhelpful for make_timestamp not to. Both functions continue to reject year zero. Code and test fixes by Peter Eisentraut, doc changes by me Discussion: https://postgr.es/m/13c0992c-f15a-a0ca-d839-91d3efd965d9@2ndquadrant.com --- doc/src/sgml/func.sgml | 12 ++++++++++-- src/backend/utils/adt/timestamp.c | 14 +++++++++----- src/test/regress/expected/date.out | 2 ++ src/test/regress/expected/timestamp.out | 11 ++++++++++- src/test/regress/sql/date.sql | 1 + src/test/regress/sql/timestamp.sql | 5 ++++- 6 files changed, 36 insertions(+), 9 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 461b748d890b..62dd7382303d 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -8939,6 +8939,7 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); Create date from year, month and day fields + (negative years signify BC) make_date(2013, 7, 15) @@ -9004,6 +9005,7 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); Create timestamp from year, month, day, hour, minute and seconds fields + (negative years signify BC) make_timestamp(2013, 7, 15, 8, 15, 23.5) @@ -9027,12 +9029,18 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); Create timestamp with time zone from year, month, day, hour, minute - and seconds fields; if timezone is not - specified, the current time zone is used + and seconds fields (negative years signify BC). + If timezone is not + specified, the current time zone is used; the examples assume the + session time zone is Europe/London make_timestamptz(2013, 7, 15, 8, 15, 23.5) 2013-07-15 08:15:23.5+01 + + + make_timestamptz(2013, 7, 15, 8, 15, 23.5, 'America/New_York') + 2013-07-15 13:15:23.5+01 diff --git a/src/backend/utils/adt/timestamp.c b/src/backend/utils/adt/timestamp.c index 5fe304cea753..4128e3a73925 100644 --- a/src/backend/utils/adt/timestamp.c +++ b/src/backend/utils/adt/timestamp.c @@ -556,17 +556,21 @@ make_timestamp_internal(int year, int month, int day, TimeOffset date; TimeOffset time; int dterr; + bool bc = false; Timestamp result; tm.tm_year = year; tm.tm_mon = month; tm.tm_mday = day; - /* - * Note: we'll reject zero or negative year values. Perhaps negatives - * should be allowed to represent BC years? - */ - dterr = ValidateDate(DTK_DATE_M, false, false, false, &tm); + /* Handle negative years as BC */ + if (tm.tm_year < 0) + { + bc = true; + tm.tm_year = -tm.tm_year; + } + + dterr = ValidateDate(DTK_DATE_M, false, false, bc, &tm); if (dterr != 0) ereport(ERROR, diff --git a/src/test/regress/expected/date.out b/src/test/regress/expected/date.out index d035fe1f1e0a..1b921ce215b5 100644 --- a/src/test/regress/expected/date.out +++ b/src/test/regress/expected/date.out @@ -1607,6 +1607,8 @@ select make_time(8, 20, 0.0); (1 row) -- should fail +select make_date(0, 7, 15); +ERROR: date field value out of range: 0-07-15 select make_date(2013, 2, 30); ERROR: date field value out of range: 2013-02-30 select make_date(2013, 13, 1); diff --git a/src/test/regress/expected/timestamp.out b/src/test/regress/expected/timestamp.out index 5f97505a3074..96551160901d 100644 --- a/src/test/regress/expected/timestamp.out +++ b/src/test/regress/expected/timestamp.out @@ -1704,9 +1704,18 @@ SELECT '' AS to_char_12, to_char(d, 'FF1 FF2 FF3 FF4 FF5 FF6 ff1 ff2 ff3 ff4 ff (4 rows) -- timestamp numeric fields constructor -SELECT make_timestamp(2014,12,28,6,30,45.887); +SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887); make_timestamp ------------------------------ Sun Dec 28 06:30:45.887 2014 (1 row) +SELECT make_timestamp(-44, 3, 15, 12, 30, 15); + make_timestamp +----------------------------- + Fri Mar 15 12:30:15 0044 BC +(1 row) + +-- should fail +select make_timestamp(0, 7, 15, 12, 30, 15); +ERROR: date field value out of range: 0-07-15 diff --git a/src/test/regress/sql/date.sql b/src/test/regress/sql/date.sql index 488f5faa076f..7a734fb1a056 100644 --- a/src/test/regress/sql/date.sql +++ b/src/test/regress/sql/date.sql @@ -378,6 +378,7 @@ select make_date(2013, 7, 15); select make_date(-44, 3, 15); select make_time(8, 20, 0.0); -- should fail +select make_date(0, 7, 15); select make_date(2013, 2, 30); select make_date(2013, 13, 1); select make_date(2013, 11, -1); diff --git a/src/test/regress/sql/timestamp.sql b/src/test/regress/sql/timestamp.sql index 7b58c3cfa5fc..727ee500845c 100644 --- a/src/test/regress/sql/timestamp.sql +++ b/src/test/regress/sql/timestamp.sql @@ -240,4 +240,7 @@ SELECT '' AS to_char_12, to_char(d, 'FF1 FF2 FF3 FF4 FF5 FF6 ff1 ff2 ff3 ff4 ff ) d(d); -- timestamp numeric fields constructor -SELECT make_timestamp(2014,12,28,6,30,45.887); +SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887); +SELECT make_timestamp(-44, 3, 15, 12, 30, 15); +-- should fail +select make_timestamp(0, 7, 15, 12, 30, 15); From 2b888647d864c5341fd0fb628c3773304282dc1d Mon Sep 17 00:00:00 2001 From: David Rowley Date: Wed, 30 Sep 2020 13:02:08 +1300 Subject: [PATCH 225/589] Doc: Improve clarity on partitioned table limitations Explicitly mention that primary key constraints are also included in the limitation that the constraint columns must be a superset of the partition key columns. Wording suggestion from Tom Lane. Discussion: https://postgr.es/m/64062533.78364.1601415362244@mail.yahoo.com Backpatch-through: 11, where unique constraints on partitioned tables were added --- doc/src/sgml/ddl.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index e890bdd10ad7..9fb2be6674f9 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -4056,8 +4056,8 @@ ALTER INDEX measurement_city_id_logdate_key - Unique constraints on partitioned tables must include all the - partition key columns. This limitation exists because + Unique constraints (and hence primary keys) on partitioned tables must + include all the partition key columns. This limitation exists because PostgreSQL can only enforce uniqueness in each partition individually. From 151c0c5f7277fd572ec02cb4ee992fdd23365a2f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 29 Sep 2020 20:02:58 -0400 Subject: [PATCH 226/589] Remove obsolete replication settings within TAP tests. PostgresNode.pm set "max_wal_senders = 5" for replication testing, but this seems to be slightly too low for our current test suite. Slower buildfarm members frequently report "number of requested standby connections exceeds max_wal_senders" failures, due to old walsenders not exiting instantaneously. Usually, the test does not fail overall because of automatic walreceiver restart, but sometimes the failure becomes visible; and in any case such retries slow down the test. That value came in with commit 89ac7004d, but was soon obsoleted by f6d6d2920, which raised the built-in default from zero to 10; so that PostgresNode.pm is actually setting it to less than the conservative built-in default. That seems pretty pointless, so let's remove the special setting and let the default prevail, in hopes of making the TAP tests more robust. Likewise, the setting "max_replication_slots = 5" is obsolete and can be removed. While here, reverse-engineer a comment about why we're choosing less-than-default values for some other settings. (Note: before v12, max_wal_senders counted against max_connections so that the latter setting also needs some fiddling with.) Back-patch to v10 where the subscription tests were added. It's likely that the older branches aren't pushing the boundaries of max_wal_senders, but I'm disinclined to spend time trying to figure out exactly when it started to be a problem. Discussion: https://postgr.es/m/723911.1601417626@sss.pgh.pa.us --- src/test/perl/PostgresNode.pm | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index 1488bffa2ba3..47b5e58f24f8 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -469,12 +469,11 @@ sub init { print $conf "wal_level = replica\n"; } - print $conf "max_wal_senders = 5\n"; - print $conf "max_replication_slots = 5\n"; print $conf "max_wal_size = 128MB\n"; - print $conf "shared_buffers = 1MB\n"; print $conf "wal_log_hints = on\n"; print $conf "hot_standby = on\n"; + # conservative settings to ensure we can run multiple postmasters: + print $conf "shared_buffers = 1MB\n"; print $conf "max_connections = 10\n"; } else From 300b6984a58ad48d0ff2eb2f35c745613f07ad9c Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 30 Sep 2020 07:39:38 +0200 Subject: [PATCH 227/589] Fix XML id to match GUC name For some reason, the id of the description of max_parallel_maintenance_workers has been guc-max-parallel-workers-maintenance since the beginning. Flip that around to make it consistent. --- doc/src/sgml/config.sgml | 6 +++--- doc/src/sgml/ref/create_index.sgml | 2 +- doc/src/sgml/ref/vacuum.sgml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 8eabf93834a4..06405f359cdf 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -2367,7 +2367,7 @@ include_dir 'conf.d' When changing this value, consider also adjusting , - , and + , and . @@ -2415,7 +2415,7 @@ include_dir 'conf.d' - + max_parallel_maintenance_workers (integer) max_parallel_maintenance_workers configuration parameter @@ -2464,7 +2464,7 @@ include_dir 'conf.d' Sets the maximum number of workers that the system can support for parallel operations. The default value is 8. When increasing or decreasing this value, consider also adjusting - and + and . Also, note that a setting for this value which is higher than will have no effect, diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d58..7fa79f4cbfb0 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -771,7 +771,7 @@ Indexes: least a 32MB share of the total maintenance_work_mem budget. There must also be a remaining 32MB share for the leader process. - Increasing + Increasing may allow more workers to be used, which will reduce the time needed for index creation, so long as the index build is not already I/O bound. Of course, there should also be sufficient diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index a48f75ad7baf..26ede69bb31d 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -239,7 +239,7 @@ VACUUM [ FULL ] [ FREEZE ] [ VERBOSE ] [ ANALYZE ] [ . An index + limited by . An index can participate in parallel vacuum if and only if the size of the index is more than . Please note that it is not guaranteed that the number of parallel workers specified in From 9796f455c38e9e79847443529ff2166b1a6f714f Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 30 Sep 2020 10:58:09 +0300 Subject: [PATCH 228/589] pgbench: Use PQExpBuffer to simplify code that constructs SQL. Author: Fabien Coelho Reviewed-by: Jeevan Ladhe Discussion: https://www.postgresql.org/message-id/alpine.DEB.2.21.1910220826570.15559%40lancre --- src/bin/pgbench/pgbench.c | 198 +++++++++++++++++++------------------- 1 file changed, 98 insertions(+), 100 deletions(-) diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 663d7d292a2a..cd39f23d5b93 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -603,7 +603,6 @@ static void doLog(TState *thread, CState *st, StatsData *agg, bool skipped, double latency, double lag); static void processXactStats(TState *thread, CState *st, instr_time *now, bool skipped, StatsData *agg); -static void append_fillfactor(char *opts, int len); static void addScript(ParsedScript script); static void *threadRun(void *arg); static void finishCon(CState *st); @@ -3630,30 +3629,26 @@ initDropTables(PGconn *con) static void createPartitions(PGconn *con) { - char ff[64]; - - ff[0] = '\0'; - - /* - * Per ddlinfo in initCreateTables, fillfactor is needed on table - * pgbench_accounts. - */ - append_fillfactor(ff, sizeof(ff)); + PQExpBufferData query; /* we must have to create some partitions */ Assert(partitions > 0); fprintf(stderr, "creating %d partitions...\n", partitions); + initPQExpBuffer(&query); + for (int p = 1; p <= partitions; p++) { - char query[256]; - if (partition_method == PART_RANGE) { int64 part_size = (naccounts * (int64) scale + partitions - 1) / partitions; - char minvalue[32], - maxvalue[32]; + + printfPQExpBuffer(&query, + "create%s table pgbench_accounts_%d\n" + " partition of pgbench_accounts\n" + " for values from (", + unlogged_tables ? " unlogged" : "", p); /* * For RANGE, we use open-ended partitions at the beginning and @@ -3662,34 +3657,39 @@ createPartitions(PGconn *con) * scale, it is more generic and the performance is better. */ if (p == 1) - sprintf(minvalue, "minvalue"); + appendPQExpBufferStr(&query, "minvalue"); else - sprintf(minvalue, INT64_FORMAT, (p - 1) * part_size + 1); + appendPQExpBuffer(&query, INT64_FORMAT, (p - 1) * part_size + 1); + + appendPQExpBufferStr(&query, ") to ("); if (p < partitions) - sprintf(maxvalue, INT64_FORMAT, p * part_size + 1); + appendPQExpBuffer(&query, INT64_FORMAT, p * part_size + 1); else - sprintf(maxvalue, "maxvalue"); - - snprintf(query, sizeof(query), - "create%s table pgbench_accounts_%d\n" - " partition of pgbench_accounts\n" - " for values from (%s) to (%s)%s\n", - unlogged_tables ? " unlogged" : "", p, - minvalue, maxvalue, ff); + appendPQExpBufferStr(&query, "maxvalue"); + + appendPQExpBufferChar(&query, ')'); } else if (partition_method == PART_HASH) - snprintf(query, sizeof(query), - "create%s table pgbench_accounts_%d\n" - " partition of pgbench_accounts\n" - " for values with (modulus %d, remainder %d)%s\n", - unlogged_tables ? " unlogged" : "", p, - partitions, p - 1, ff); + printfPQExpBuffer(&query, + "create%s table pgbench_accounts_%d\n" + " partition of pgbench_accounts\n" + " for values with (modulus %d, remainder %d)", + unlogged_tables ? " unlogged" : "", p, + partitions, p - 1); else /* cannot get there */ Assert(0); - executeStatement(con, query); + /* + * Per ddlinfo in initCreateTables, fillfactor is needed on table + * pgbench_accounts. + */ + appendPQExpBuffer(&query, " with (fillfactor=%d)", fillfactor); + + executeStatement(con, query.data); } + + termPQExpBuffer(&query); } /* @@ -3743,63 +3743,50 @@ initCreateTables(PGconn *con) } }; int i; + PQExpBufferData query; fprintf(stderr, "creating tables...\n"); + initPQExpBuffer(&query); + for (i = 0; i < lengthof(DDLs); i++) { - char opts[256]; - char buffer[256]; const struct ddlinfo *ddl = &DDLs[i]; - const char *cols; /* Construct new create table statement. */ - opts[0] = '\0'; + printfPQExpBuffer(&query, "create%s table %s(%s)", + unlogged_tables ? " unlogged" : "", + ddl->table, + (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols); /* Partition pgbench_accounts table */ if (partition_method != PART_NONE && strcmp(ddl->table, "pgbench_accounts") == 0) - snprintf(opts + strlen(opts), sizeof(opts) - strlen(opts), - " partition by %s (aid)", PARTITION_METHOD[partition_method]); + appendPQExpBuffer(&query, + " partition by %s (aid)", PARTITION_METHOD[partition_method]); else if (ddl->declare_fillfactor) + { /* fillfactor is only expected on actual tables */ - append_fillfactor(opts, sizeof(opts)); + appendPQExpBuffer(&query, " with (fillfactor=%d)", fillfactor); + } if (tablespace != NULL) { char *escape_tablespace; - escape_tablespace = PQescapeIdentifier(con, tablespace, - strlen(tablespace)); - snprintf(opts + strlen(opts), sizeof(opts) - strlen(opts), - " tablespace %s", escape_tablespace); + escape_tablespace = PQescapeIdentifier(con, tablespace, strlen(tablespace)); + appendPQExpBuffer(&query, " tablespace %s", escape_tablespace); PQfreemem(escape_tablespace); } - cols = (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols; - - snprintf(buffer, sizeof(buffer), "create%s table %s(%s)%s", - unlogged_tables ? " unlogged" : "", - ddl->table, cols, opts); - - executeStatement(con, buffer); + executeStatement(con, query.data); } + termPQExpBuffer(&query); + if (partition_method != PART_NONE) createPartitions(con); } -/* - * add fillfactor percent option. - * - * XXX - As default is 100, it could be removed in this case. - */ -static void -append_fillfactor(char *opts, int len) -{ - snprintf(opts + strlen(opts), len - strlen(opts), - " with (fillfactor=%d)", fillfactor); -} - /* * Truncate away any old data, in one command in case there are foreign keys */ @@ -3819,7 +3806,7 @@ initTruncateTables(PGconn *con) static void initGenerateDataClientSide(PGconn *con) { - char sql[256]; + PQExpBufferData sql; PGresult *res; int i; int64 k; @@ -3845,6 +3832,8 @@ initGenerateDataClientSide(PGconn *con) /* truncate away any old data */ initTruncateTables(con); + initPQExpBuffer(&sql); + /* * fill branches, tellers, accounts in that order in case foreign keys * already exist @@ -3852,19 +3841,19 @@ initGenerateDataClientSide(PGconn *con) for (i = 0; i < nbranches * scale; i++) { /* "filler" column defaults to NULL */ - snprintf(sql, sizeof(sql), - "insert into pgbench_branches(bid,bbalance) values(%d,0)", - i + 1); - executeStatement(con, sql); + printfPQExpBuffer(&sql, + "insert into pgbench_branches(bid,bbalance) values(%d,0)", + i + 1); + executeStatement(con, sql.data); } for (i = 0; i < ntellers * scale; i++) { /* "filler" column defaults to NULL */ - snprintf(sql, sizeof(sql), - "insert into pgbench_tellers(tid,bid,tbalance) values (%d,%d,0)", - i + 1, i / ntellers + 1); - executeStatement(con, sql); + printfPQExpBuffer(&sql, + "insert into pgbench_tellers(tid,bid,tbalance) values (%d,%d,0)", + i + 1, i / ntellers + 1); + executeStatement(con, sql.data); } /* @@ -3885,10 +3874,10 @@ initGenerateDataClientSide(PGconn *con) int64 j = k + 1; /* "filler" column defaults to blank padded empty string */ - snprintf(sql, sizeof(sql), - INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n", - j, k / naccounts + 1, 0); - if (PQputline(con, sql)) + printfPQExpBuffer(&sql, + INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n", + j, k / naccounts + 1, 0); + if (PQputline(con, sql.data)) { pg_log_fatal("PQputline failed"); exit(1); @@ -3950,6 +3939,8 @@ initGenerateDataClientSide(PGconn *con) exit(1); } + termPQExpBuffer(&sql); + executeStatement(con, "commit"); } @@ -3963,7 +3954,7 @@ initGenerateDataClientSide(PGconn *con) static void initGenerateDataServerSide(PGconn *con) { - char sql[256]; + PQExpBufferData sql; fprintf(stderr, "generating data (server-side)...\n"); @@ -3976,24 +3967,28 @@ initGenerateDataServerSide(PGconn *con) /* truncate away any old data */ initTruncateTables(con); - snprintf(sql, sizeof(sql), - "insert into pgbench_branches(bid,bbalance) " - "select bid, 0 " - "from generate_series(1, %d) as bid", nbranches * scale); - executeStatement(con, sql); - - snprintf(sql, sizeof(sql), - "insert into pgbench_tellers(tid,bid,tbalance) " - "select tid, (tid - 1) / %d + 1, 0 " - "from generate_series(1, %d) as tid", ntellers, ntellers * scale); - executeStatement(con, sql); - - snprintf(sql, sizeof(sql), - "insert into pgbench_accounts(aid,bid,abalance,filler) " - "select aid, (aid - 1) / %d + 1, 0, '' " - "from generate_series(1, " INT64_FORMAT ") as aid", - naccounts, (int64) naccounts * scale); - executeStatement(con, sql); + initPQExpBuffer(&sql); + + printfPQExpBuffer(&sql, + "insert into pgbench_branches(bid,bbalance) " + "select bid, 0 " + "from generate_series(1, %d) as bid", nbranches * scale); + executeStatement(con, sql.data); + + printfPQExpBuffer(&sql, + "insert into pgbench_tellers(tid,bid,tbalance) " + "select tid, (tid - 1) / %d + 1, 0 " + "from generate_series(1, %d) as tid", ntellers, ntellers * scale); + executeStatement(con, sql.data); + + printfPQExpBuffer(&sql, + "insert into pgbench_accounts(aid,bid,abalance,filler) " + "select aid, (aid - 1) / %d + 1, 0, '' " + "from generate_series(1, " INT64_FORMAT ") as aid", + naccounts, (int64) naccounts * scale); + executeStatement(con, sql.data); + + termPQExpBuffer(&sql); executeStatement(con, "commit"); } @@ -4023,13 +4018,15 @@ initCreatePKeys(PGconn *con) "alter table pgbench_accounts add primary key (aid)" }; int i; + PQExpBufferData query; fprintf(stderr, "creating primary keys...\n"); + initPQExpBuffer(&query); + for (i = 0; i < lengthof(DDLINDEXes); i++) { - char buffer[256]; - - strlcpy(buffer, DDLINDEXes[i], sizeof(buffer)); + resetPQExpBuffer(&query); + appendPQExpBufferStr(&query, DDLINDEXes[i]); if (index_tablespace != NULL) { @@ -4037,13 +4034,14 @@ initCreatePKeys(PGconn *con) escape_tablespace = PQescapeIdentifier(con, index_tablespace, strlen(index_tablespace)); - snprintf(buffer + strlen(buffer), sizeof(buffer) - strlen(buffer), - " using index tablespace %s", escape_tablespace); + appendPQExpBuffer(&query, " using index tablespace %s", escape_tablespace); PQfreemem(escape_tablespace); } - executeStatement(con, buffer); + executeStatement(con, query.data); } + + termPQExpBuffer(&query); } /* From 489c9c3407cbfd473c2f8d7863ffaaf6d2e8fcf8 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 30 Sep 2020 15:40:23 -0400 Subject: [PATCH 229/589] Fix handling of BC years in to_date/to_timestamp. Previously, a conversion such as to_date('-44-02-01','YYYY-MM-DD') would result in '0045-02-01 BC', as the code attempted to interpret the negative year as BC, but failed to apply the correction needed for our internal handling of BC years. Fix the off-by-one problem. Also, arrange for the combination of a negative year and an explicit "BC" marker to cancel out and produce AD. This is how the negative-century case works, so it seems sane to do likewise. Continue to read "year 0000" as 1 BC. Oracle would throw an error, but we've accepted that case for a long time so I'm hesitant to change it in a back-patch. Per bug #16419 from Saeed Hubaishan. Back-patch to all supported branches. Dar Alathar-Yemen and Tom Lane Discussion: https://postgr.es/m/16419-d8d9db0a7553f01b@postgresql.org --- doc/src/sgml/func.sgml | 9 ++++++ src/backend/utils/adt/formatting.c | 7 ++-- src/test/regress/expected/horology.out | 45 ++++++++++++++++++++++++++ src/test/regress/sql/horology.sql | 12 +++++++ 4 files changed, 71 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 62dd7382303d..ec8451d1b9b9 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -7678,6 +7678,15 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); + + + In to_timestamp and to_date, + negative years are treated as signifying BC. If you write both a + negative year and an explicit BC field, you get AD + again. An input of year zero is treated as 1 BC. + + + In to_timestamp and to_date, diff --git a/src/backend/utils/adt/formatting.c b/src/backend/utils/adt/formatting.c index b91ff7bb8036..3bb01cdb65ab 100644 --- a/src/backend/utils/adt/formatting.c +++ b/src/backend/utils/adt/formatting.c @@ -4569,8 +4569,11 @@ do_to_timestamp(text *date_txt, text *fmt, Oid collid, bool std, { /* If a 4-digit year is provided, we use that and ignore CC. */ tm->tm_year = tmfc.year; - if (tmfc.bc && tm->tm_year > 0) - tm->tm_year = -(tm->tm_year - 1); + if (tmfc.bc) + tm->tm_year = -tm->tm_year; + /* correct for our representation of BC years */ + if (tm->tm_year < 0) + tm->tm_year++; } fmask |= DTK_M(YEAR); } diff --git a/src/test/regress/expected/horology.out b/src/test/regress/expected/horology.out index c8c33a0fc067..7f82dcfbfef3 100644 --- a/src/test/regress/expected/horology.out +++ b/src/test/regress/expected/horology.out @@ -2916,6 +2916,45 @@ SELECT to_date('2458872', 'J'); 01-23-2020 (1 row) +-- +-- Check handling of BC dates +-- +SELECT to_date('44-02-01 BC','YYYY-MM-DD BC'); + to_date +--------------- + 02-01-0044 BC +(1 row) + +SELECT to_date('-44-02-01','YYYY-MM-DD'); + to_date +--------------- + 02-01-0044 BC +(1 row) + +SELECT to_date('-44-02-01 BC','YYYY-MM-DD BC'); + to_date +------------ + 02-01-0044 +(1 row) + +SELECT to_timestamp('44-02-01 11:12:13 BC','YYYY-MM-DD HH24:MI:SS BC'); + to_timestamp +--------------------------------- + Fri Feb 01 11:12:13 0044 PST BC +(1 row) + +SELECT to_timestamp('-44-02-01 11:12:13','YYYY-MM-DD HH24:MI:SS'); + to_timestamp +--------------------------------- + Fri Feb 01 11:12:13 0044 PST BC +(1 row) + +SELECT to_timestamp('-44-02-01 11:12:13 BC','YYYY-MM-DD HH24:MI:SS BC'); + to_timestamp +------------------------------ + Mon Feb 01 11:12:13 0044 PST +(1 row) + -- -- Check handling of multiple spaces in format and/or input -- @@ -3183,6 +3222,12 @@ SELECT to_date('2016 366', 'YYYY DDD'); -- ok SELECT to_date('2016 367', 'YYYY DDD'); ERROR: date/time field value out of range: "2016 367" +SELECT to_date('0000-02-01','YYYY-MM-DD'); -- allowed, though it shouldn't be + to_date +--------------- + 02-01-0001 BC +(1 row) + -- -- Check behavior with SQL-style fixed-GMT-offset time zone (cf bug #8572) -- diff --git a/src/test/regress/sql/horology.sql b/src/test/regress/sql/horology.sql index c464e6766c69..fed21a53c867 100644 --- a/src/test/regress/sql/horology.sql +++ b/src/test/regress/sql/horology.sql @@ -426,6 +426,17 @@ SELECT to_date('1 4 1902', 'Q MM YYYY'); -- Q is ignored SELECT to_date('3 4 21 01', 'W MM CC YY'); SELECT to_date('2458872', 'J'); +-- +-- Check handling of BC dates +-- + +SELECT to_date('44-02-01 BC','YYYY-MM-DD BC'); +SELECT to_date('-44-02-01','YYYY-MM-DD'); +SELECT to_date('-44-02-01 BC','YYYY-MM-DD BC'); +SELECT to_timestamp('44-02-01 11:12:13 BC','YYYY-MM-DD HH24:MI:SS BC'); +SELECT to_timestamp('-44-02-01 11:12:13','YYYY-MM-DD HH24:MI:SS'); +SELECT to_timestamp('-44-02-01 11:12:13 BC','YYYY-MM-DD HH24:MI:SS BC'); + -- -- Check handling of multiple spaces in format and/or input -- @@ -511,6 +522,7 @@ SELECT to_date('2015 366', 'YYYY DDD'); SELECT to_date('2016 365', 'YYYY DDD'); -- ok SELECT to_date('2016 366', 'YYYY DDD'); -- ok SELECT to_date('2016 367', 'YYYY DDD'); +SELECT to_date('0000-02-01','YYYY-MM-DD'); -- allowed, though it shouldn't be -- -- Check behavior with SQL-style fixed-GMT-offset time zone (cf bug #8572) From 9fc21227128980f88b6eb6d5306625af0428ef45 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Wed, 30 Sep 2020 18:25:23 -0300 Subject: [PATCH 230/589] Reword partitioning error message The error message about columns in the primary key not including all of the partition key was unclear; reword it. Backpatch all the way to pg11, where it appeared. Reported-by: Nagaraj Raj Discussion: https://postgr.es/m/64062533.78364.1601415362244@mail.yahoo.com --- src/backend/commands/indexcmds.c | 3 +-- src/test/regress/expected/indexing.out | 18 +++++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 59e04b47dfb3..75552c64ed23 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -1002,8 +1002,7 @@ DefineIndex(Oid relationId, key->partattrs[i] - 1); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("insufficient columns in %s constraint definition", - constraint_type), + errmsg("unique constraint on partitioned table must include all partitioning columns"), errdetail("%s constraint on table \"%s\" lacks column \"%s\" which is part of the partition key.", constraint_type, RelationGetRelationName(rel), NameStr(att->attname)))); diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index 7e78a07af8b8..c93f4470c92c 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -907,16 +907,16 @@ Indexes: drop table idxpart; -- Failing to use the full partition key is not allowed create table idxpart (a int unique, b int) partition by range (a, b); -ERROR: insufficient columns in UNIQUE constraint definition +ERROR: unique constraint on partitioned table must include all partitioning columns DETAIL: UNIQUE constraint on table "idxpart" lacks column "b" which is part of the partition key. create table idxpart (a int, b int unique) partition by range (a, b); -ERROR: insufficient columns in UNIQUE constraint definition +ERROR: unique constraint on partitioned table must include all partitioning columns DETAIL: UNIQUE constraint on table "idxpart" lacks column "a" which is part of the partition key. create table idxpart (a int primary key, b int) partition by range (b, a); -ERROR: insufficient columns in PRIMARY KEY constraint definition +ERROR: unique constraint on partitioned table must include all partitioning columns DETAIL: PRIMARY KEY constraint on table "idxpart" lacks column "b" which is part of the partition key. create table idxpart (a int, b int primary key) partition by range (b, a); -ERROR: insufficient columns in PRIMARY KEY constraint definition +ERROR: unique constraint on partitioned table must include all partitioning columns DETAIL: PRIMARY KEY constraint on table "idxpart" lacks column "a" which is part of the partition key. -- OK if you use them in some other order create table idxpart (a int, b int, c text, primary key (a, b, c)) partition by range (b, c, a); @@ -936,7 +936,7 @@ DETAIL: UNIQUE constraints cannot be used when partition keys include expressio -- use ALTER TABLE to add a primary key create table idxpart (a int, b int, c text) partition by range (a, b); alter table idxpart add primary key (a); -- not an incomplete one though -ERROR: insufficient columns in PRIMARY KEY constraint definition +ERROR: unique constraint on partitioned table must include all partitioning columns DETAIL: PRIMARY KEY constraint on table "idxpart" lacks column "b" which is part of the partition key. alter table idxpart add primary key (a, b); -- this works \d idxpart @@ -967,7 +967,7 @@ drop table idxpart; -- use ALTER TABLE to add a unique constraint create table idxpart (a int, b int) partition by range (a, b); alter table idxpart add unique (a); -- not an incomplete one though -ERROR: insufficient columns in UNIQUE constraint definition +ERROR: unique constraint on partitioned table must include all partitioning columns DETAIL: UNIQUE constraint on table "idxpart" lacks column "b" which is part of the partition key. alter table idxpart add unique (b, a); -- this works \d idxpart @@ -1017,7 +1017,7 @@ drop table idxpart; create table idxpart (a int, b int, primary key (a)) partition by range (a); create table idxpart2 partition of idxpart for values from (0) to (1000) partition by range (b); -- fail -ERROR: insufficient columns in PRIMARY KEY constraint definition +ERROR: unique constraint on partitioned table must include all partitioning columns DETAIL: PRIMARY KEY constraint on table "idxpart2" lacks column "b" which is part of the partition key. drop table idxpart; -- Ditto for the ATTACH PARTITION case @@ -1025,7 +1025,7 @@ create table idxpart (a int unique, b int) partition by range (a); create table idxpart1 (a int not null, b int, unique (a, b)) partition by range (a, b); alter table idxpart attach partition idxpart1 for values from (1) to (1000); -ERROR: insufficient columns in UNIQUE constraint definition +ERROR: unique constraint on partitioned table must include all partitioning columns DETAIL: UNIQUE constraint on table "idxpart1" lacks column "b" which is part of the partition key. DROP TABLE idxpart, idxpart1; -- Multi-layer partitioning works correctly in this case: @@ -1278,7 +1278,7 @@ insert into covidxpart values (4, 1); ERROR: duplicate key value violates unique constraint "covidxpart4_a_b_idx" DETAIL: Key (a)=(4) already exists. create unique index on covidxpart (b) include (a); -- should fail -ERROR: insufficient columns in UNIQUE constraint definition +ERROR: unique constraint on partitioned table must include all partitioning columns DETAIL: UNIQUE constraint on table "covidxpart" lacks column "a" which is part of the partition key. -- check that detaching a partition also detaches the primary key constraint create table parted_pk_detach_test (a int primary key) partition by list (a); From 7b28913bcab8d1bf3dbf59c9d8fb4b51cef57664 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Wed, 30 Sep 2020 17:28:51 -0700 Subject: [PATCH 231/589] Fix and test snapshot behavior on standby. I (Andres) broke this in 623a9CA79bx, because I didn't think about the way snapshots are built on standbys sufficiently. Unfortunately our existing tests did not catch this, as they are all just querying with psql (therefore ending up with fresh snapshots). The fix is trivial, we just need to increment the transaction completion counter in ExpireTreeKnownAssignedTransactionIds(), which is the equivalent of ProcArrayEndTransaction() during recovery. This commit also adds a new test doing some basic testing of the correctness of snapshots built on standbys. To avoid the aforementioned issue of one-shot psql's not exercising the snapshot caching, the test uses a long lived psqls, similar to 013_crash_restart.pl. It'd be good to extend the test further. Reported-By: Ian Barwick Author: Andres Freund Author: Ian Barwick Discussion: https://postgr.es/m/61291ffe-d611-f889-68b5-c298da9fb18f@2ndquadrant.com --- src/backend/storage/ipc/procarray.c | 3 + src/test/recovery/t/021_row_visibility.pl | 192 ++++++++++++++++++++++ 2 files changed, 195 insertions(+) create mode 100644 src/test/recovery/t/021_row_visibility.pl diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c index 5aaeb6e2b558..07c5eeb74951 100644 --- a/src/backend/storage/ipc/procarray.c +++ b/src/backend/storage/ipc/procarray.c @@ -4280,6 +4280,9 @@ ExpireTreeKnownAssignedTransactionIds(TransactionId xid, int nsubxids, /* As in ProcArrayEndTransaction, advance latestCompletedXid */ MaintainLatestCompletedXidRecovery(max_xid); + /* ... and xactCompletionCount */ + ShmemVariableCache->xactCompletionCount++; + LWLockRelease(ProcArrayLock); } diff --git a/src/test/recovery/t/021_row_visibility.pl b/src/test/recovery/t/021_row_visibility.pl new file mode 100644 index 000000000000..95516b05d011 --- /dev/null +++ b/src/test/recovery/t/021_row_visibility.pl @@ -0,0 +1,192 @@ +# Checks that snapshots on standbys behave in a minimally reasonable +# way. +use strict; +use warnings; + +use PostgresNode; +use TestLib; +use Test::More tests => 10; + +# Initialize primary node +my $node_primary = get_new_node('primary'); +$node_primary->init(allows_streaming => 1); +$node_primary->append_conf('postgresql.conf', 'max_prepared_transactions=10'); +$node_primary->start; + +# Initialize with empty test table +$node_primary->safe_psql('postgres', + 'CREATE TABLE public.test_visibility (data text not null)'); + +# Take backup +my $backup_name = 'my_backup'; +$node_primary->backup($backup_name); + +# Create streaming standby from backup +my $node_standby = get_new_node('standby'); +$node_standby->init_from_backup($node_primary, $backup_name, + has_streaming => 1); +$node_standby->append_conf('postgresql.conf', 'max_prepared_transactions=10'); +$node_standby->start; + +# To avoid hanging while expecting some specific input from a psql +# instance being driven by us, add a timeout high enough that it +# should never trigger even on very slow machines, unless something +# is really wrong. +my $psql_timeout = IPC::Run::timer(30); + +# One psql to primary and standby each, for all queries. That allows +# to check uncommitted changes being replicated and such. +my %psql_primary = (stdin => '', stdout => '', stderr => ''); +$psql_primary{run} = + IPC::Run::start( + ['psql', '-XA', '-f', '-', '-d', $node_primary->connstr('postgres')], + '<', \$psql_primary{stdin}, + '>', \$psql_primary{stdout}, + '2>', \$psql_primary{stderr}, + $psql_timeout); + +my %psql_standby = ('stdin' => '', 'stdout' => '', 'stderr' => ''); +$psql_standby{run} = + IPC::Run::start( + ['psql', '-XA', '-f', '-', '-d', $node_standby->connstr('postgres')], + '<', \$psql_standby{stdin}, + '>', \$psql_standby{stdout}, + '2>', \$psql_standby{stderr}, + $psql_timeout); + +# +# 1. Check initial data is the same +# +ok(send_query_and_wait(\%psql_standby, + q/SELECT * FROM test_visibility ORDER BY data;/, + qr/^\(0 rows\)$/m), + 'data not visible'); + +# +# 2. Check if an INSERT is replayed and visible +# +$node_primary->psql('postgres', "INSERT INTO test_visibility VALUES ('first insert')"); +$node_primary->wait_for_catchup($node_standby, 'replay', + $node_primary->lsn('insert')); + +ok(send_query_and_wait(\%psql_standby, + q[SELECT * FROM test_visibility ORDER BY data;], + qr/first insert.*\n\(1 row\)/m), + 'insert visible'); + +# +# 3. Verify that uncommitted changes aren't visible. +# +ok(send_query_and_wait(\%psql_primary, + q[ +BEGIN; +UPDATE test_visibility SET data = 'first update' RETURNING data; + ], + qr/^UPDATE 1$/m), + 'UPDATE'); + +$node_primary->psql('postgres', "SELECT txid_current();"); # ensure WAL flush +$node_primary->wait_for_catchup($node_standby, 'replay', + $node_primary->lsn('insert')); + +ok(send_query_and_wait(\%psql_standby, + q[SELECT * FROM test_visibility ORDER BY data;], + qr/first insert.*\n\(1 row\)/m), + 'uncommitted update invisible'); + +# +# 4. That a commit turns 3. visible +# +ok(send_query_and_wait(\%psql_primary, + q[COMMIT;], + qr/^COMMIT$/m), + 'COMMIT'); + +$node_primary->wait_for_catchup($node_standby, 'replay', + $node_primary->lsn('insert')); + +ok(send_query_and_wait(\%psql_standby, + q[SELECT * FROM test_visibility ORDER BY data;], + qr/first update\n\(1 row\)$/m), + 'committed update visible'); + +# +# 5. Check that changes in prepared xacts is invisible +# +ok(send_query_and_wait(\%psql_primary, q[ +DELETE from test_visibility; -- delete old data, so we start with clean slate +BEGIN; +INSERT INTO test_visibility VALUES('inserted in prepared will_commit'); +PREPARE TRANSACTION 'will_commit';], + qr/^PREPARE TRANSACTION$/m), + 'prepared will_commit'); + +ok(send_query_and_wait(\%psql_primary, q[ +BEGIN; +INSERT INTO test_visibility VALUES('inserted in prepared will_abort'); +PREPARE TRANSACTION 'will_abort'; + ], + qr/^PREPARE TRANSACTION$/m), + 'prepared will_abort'); + +$node_primary->wait_for_catchup($node_standby, 'replay', + $node_primary->lsn('insert')); + +ok(send_query_and_wait(\%psql_standby, + q[SELECT * FROM test_visibility ORDER BY data;], + qr/^\(0 rows\)$/m), + 'uncommitted prepared invisible'); + +# For some variation, finish prepared xacts via separate connections +$node_primary->safe_psql('postgres', + "COMMIT PREPARED 'will_commit';"); +$node_primary->safe_psql('postgres', + "ROLLBACK PREPARED 'will_abort';"); +$node_primary->wait_for_catchup($node_standby, 'replay', + $node_primary->lsn('insert')); + +ok(send_query_and_wait(\%psql_standby, + q[SELECT * FROM test_visibility ORDER BY data;], + qr/will_commit.*\n\(1 row\)$/m), + 'finished prepared visible'); + +$node_primary->stop; +$node_standby->stop; + +# Send query, wait until string matches +sub send_query_and_wait +{ + my ($psql, $query, $untl) = @_; + my $ret; + + # send query + $$psql{stdin} .= $query; + $$psql{stdin} .= "\n"; + + # wait for query results + $$psql{run}->pump_nb(); + while (1) + { + last if $$psql{stdout} =~ /$untl/; + + if ($psql_timeout->is_expired) + { + BAIL_OUT("aborting wait: program timed out\n". + "stream contents: >>$$psql{stdout}<<\n". + "pattern searched for: $untl\n"); + return 0; + } + if (not $$psql{run}->pumpable()) + { + BAIL_OUT("aborting wait: program died\n". + "stream contents: >>$$psql{stdout}<<\n". + "pattern searched for: $untl\n"); + return 0; + } + $$psql{run}->pump(); + } + + $$psql{stdout} = ''; + + return 1; +} From 6b1c5cacec5e8305a3880b441526c47bd47c64cd Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 1 Oct 2020 10:37:34 +0900 Subject: [PATCH 232/589] Remove logging.c from the shared library of src/common/ As fe0a1dc has proved, it is not a good concept to add to libpq dependencies that would enforce the error output to a central logging facility because it breaks the promise of reporting the error back to an application in a consistent way, with the application to potentially exit() suddenly if using pieces from for example jsonapi.c. prairiedog has allowed to report an actual design problem with fe0a1dc, but it will not be around forever, so removing logging.c from libpgcommon_shlib is a simple and much better long-term way to prevent any attempt to load the central logging in libraries with general purposes. Author: Michael Paquier Reviewed-by: Tom Lane Discussion: https://postgr.es/m/20200928073330.GC2316@paquier.xyz --- src/common/Makefile | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/common/Makefile b/src/common/Makefile index f2817628851e..25c55bd6423c 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -88,16 +88,21 @@ OBJS_COMMON += sha2.o endif # A few files are currently only built for frontend, not server -# (Mkvcbuild.pm has a copy of this list, too) -OBJS_FRONTEND = \ +# (Mkvcbuild.pm has a copy of this list, too). logging.c is excluded +# from OBJS_FRONTEND_SHLIB (shared library) as a matter of policy, +# because it is not appropriate for general purpose libraries such +# as libpq to report errors directly. +OBJS_FRONTEND_SHLIB = \ $(OBJS_COMMON) \ fe_memutils.o \ - logging.o \ restricted_token.o \ sprompt.o +OBJS_FRONTEND = \ + $(OBJS_FRONTEND_SHLIB) \ + logging.o # foo.o, foo_shlib.o, and foo_srv.o are all built from foo.c -OBJS_SHLIB = $(OBJS_FRONTEND:%.o=%_shlib.o) +OBJS_SHLIB = $(OBJS_FRONTEND_SHLIB:%.o=%_shlib.o) OBJS_SRV = $(OBJS_COMMON:%.o=%_srv.o) # where to find gen_keywordlist.pl and subsidiary files From 265ea567852a692e644fe6b207def4a2fc812adb Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 1 Oct 2020 11:10:43 +0300 Subject: [PATCH 233/589] Set right-links during sorted GiST index build. This is not strictly necessary, as the right-links are only needed by scans that are concurrent with page splits, and neither scans or page splits can happen during sorted index build. But it seems like a good idea to set them anyway, if we e.g. want to add a check to amcheck in the future to verify that the chain of right-links is complete. Author: Andrey Borodin Discussion: https://www.postgresql.org/message-id/4D68C21F-9FB9-41DA-B663-FDFC8D143788%40yandex-team.ru --- src/backend/access/gist/gistbuild.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/backend/access/gist/gistbuild.c b/src/backend/access/gist/gistbuild.c index 188e33642f3d..28bc5855ad9f 100644 --- a/src/backend/access/gist/gistbuild.c +++ b/src/backend/access/gist/gistbuild.c @@ -540,6 +540,19 @@ gist_indexsortbuild_pagestate_flush(GISTBuildState *state, /* Re-initialize the page buffer for next page on this level. */ pagestate->page = palloc(BLCKSZ); gistinitpage(pagestate->page, isleaf ? F_LEAF : 0); + + /* + * Set the right link to point to the previous page. This is just for + * debugging purposes: GiST only follows the right link if a page is split + * concurrently to a scan, and that cannot happen during index build. + * + * It's a bit counterintuitive that we set the right link on the new page + * to point to the previous page, and not the other way round. But GiST + * pages are not ordered like B-tree pages are, so as long as the + * right-links form a chain through all the pages in the same level, the + * order doesn't matter. + */ + GistPageGetOpaque(pagestate->page)->rightlink = blkno; } static void From e1761871c096514ef08a211bf4a9b557753383dc Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 1 Oct 2020 11:48:48 +0300 Subject: [PATCH 234/589] Fix incorrect assertion on number of array dimensions. This has been wrong ever since the support for multi-dimensional arrays as PL/python function arguments and return values was introduced in commit 94aceed317. Backpatch-through: 10 Discussion: https://www.postgresql.org/message-id/61647b8e-961c-0362-d5d3-c8a18f4a7ec6%40iki.fi --- src/pl/plpython/plpy_typeio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pl/plpython/plpy_typeio.c b/src/pl/plpython/plpy_typeio.c index 7c844c2b8a10..9e4e9035f783 100644 --- a/src/pl/plpython/plpy_typeio.c +++ b/src/pl/plpython/plpy_typeio.c @@ -679,7 +679,7 @@ PLyList_FromArray(PLyDatumToOb *arg, Datum d) /* Array dimensions and left bounds */ ndim = ARR_NDIM(array); dims = ARR_DIMS(array); - Assert(ndim < MAXDIM); + Assert(ndim <= MAXDIM); /* * We iterate the SQL array in the physical order it's stored in the From 49642530486852f8fef248964e563aaab25ecae7 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 1 Oct 2020 10:59:20 -0400 Subject: [PATCH 235/589] Put back explicit setting of replication values within TAP tests. Commit 151c0c5f7 neglected the possibility that a TEMP_CONFIG file would explicitly set max_wal_senders=0; as indeed buildfarm member thorntail does, so that it can test wal_level=minimal in other test suites. Hence, rather than assuming that max_wal_senders=10 will prevail if we say nothing, set it explicitly. Set max_replication_slots=10 explicitly too, just to be safe. Back-patch to v10, like the previous patch. Discussion: https://postgr.es/m/723911.1601417626@sss.pgh.pa.us --- src/test/perl/PostgresNode.pm | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index 47b5e58f24f8..97e9d932ce70 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -469,12 +469,15 @@ sub init { print $conf "wal_level = replica\n"; } - print $conf "max_wal_size = 128MB\n"; + print $conf "max_wal_senders = 10\n"; + print $conf "max_replication_slots = 10\n"; print $conf "wal_log_hints = on\n"; print $conf "hot_standby = on\n"; # conservative settings to ensure we can run multiple postmasters: print $conf "shared_buffers = 1MB\n"; print $conf "max_connections = 10\n"; + # limit disk space consumption, too: + print $conf "max_wal_size = 128MB\n"; } else { From 9d0bd95fa90a7243047a74e29f265296a9fc556d Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Fri, 2 Oct 2020 09:31:50 +0900 Subject: [PATCH 236/589] Add block information in error context of WAL REDO apply loop Providing this information can be useful for example when diagnosing problems related to recovery conflicts or for recovery issues without having to go through the output generated by pg_waldump to get some information about the blocks a WAL record works on. The block information is printed in the same format as pg_waldump. This already existed in xlog.c for debugging purposes with -DWAL_DEBUG, so adding the block information in the callback has required just a small refactoring. Author: Bertrand Drouvot Reviewed-by: Michael Paquier, Masahiko Sawada Discussion: https://postgr.es/m/c31e2cba-efda-762c-f4ad-5c25e5dac3d0@amazon.com --- src/backend/access/transam/xlog.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 79a77ebbfe24..03c089b9a966 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -940,6 +940,7 @@ static bool CheckForStandbyTrigger(void); #ifdef WAL_DEBUG static void xlog_outrec(StringInfo buf, XLogReaderState *record); #endif +static void xlog_block_info(StringInfo buf, XLogReaderState *record); static void xlog_outdesc(StringInfo buf, XLogReaderState *record); static void pg_start_backup_callback(int code, Datum arg); static void pg_stop_backup_callback(int code, Datum arg); @@ -10258,6 +10259,19 @@ xlog_outrec(StringInfo buf, XLogReaderState *record) appendStringInfo(buf, "; len %u", XLogRecGetDataLen(record)); + xlog_block_info(buf, record); +} +#endif /* WAL_DEBUG */ + +/* + * Returns a string giving information about all the blocks in an + * XLogRecord. + */ +static void +xlog_block_info(StringInfo buf, XLogReaderState *record) +{ + int block_id; + /* decode block references */ for (block_id = 0; block_id <= record->max_block_id; block_id++) { @@ -10284,7 +10298,6 @@ xlog_outrec(StringInfo buf, XLogReaderState *record) appendStringInfoString(buf, " FPW"); } } -#endif /* WAL_DEBUG */ /* * Returns a string describing an XLogRecord, consisting of its identity @@ -11765,6 +11778,7 @@ rm_redo_error_callback(void *arg) initStringInfo(&buf); xlog_outdesc(&buf, record); + xlog_block_info(&buf, record); /* translator: %s is a WAL record description */ errcontext("WAL redo at %X/%X for %s", From 8d9a935965f01b7759a8c23ff6291000b670a2bf Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Fri, 2 Oct 2020 10:17:11 +0900 Subject: [PATCH 237/589] Add pg_stat_wal statistics view. This view shows the statistics about WAL activity. Currently it has only two columns: wal_buffers_full and stats_reset. wal_buffers_full column indicates the number of times WAL data was written to the disk because WAL buffers got full. This information is useful when tuning wal_buffers. stats_reset column indicates the time at which these statistics were last reset. pg_stat_wal view is also the basic infrastructure to expose other various statistics about WAL activity later. Bump PGSTAT_FILE_FORMAT_ID due to the change in pgstat format. Bump catalog version. Author: Masahiro Ikeda Reviewed-by: Takayuki Tsunakawa, Kyotaro Horiguchi, Amit Kapila, Fujii Masao Discussion: https://postgr.es/m/188bd3f2d2233cf97753b5ced02bb050@oss.nttdata.com --- doc/src/sgml/monitoring.sgml | 63 ++++++++++++- src/backend/access/transam/xlog.c | 1 + src/backend/catalog/system_views.sql | 6 ++ src/backend/postmaster/checkpointer.c | 3 + src/backend/postmaster/pgstat.c | 120 +++++++++++++++++++++++-- src/backend/utils/adt/pgstatfuncs.c | 36 ++++++++ src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.dat | 8 ++ src/include/pgstat.h | 33 ++++++- src/test/regress/expected/rules.out | 3 + src/test/regress/expected/sysviews.out | 7 ++ src/test/regress/sql/sysviews.sql | 3 + src/tools/pgindent/typedefs.list | 2 + 13 files changed, 276 insertions(+), 11 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 4e0193a967cb..495018009a2a 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -424,6 +424,14 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser + + pg_stat_walpg_stat_wal + One row only, showing statistics about WAL activity. See + + pg_stat_wal for details. + + + pg_stat_databasepg_stat_database One row per database, showing database-wide statistics. See @@ -3280,6 +3288,56 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i + + <structname>pg_stat_wal</structname> + + + pg_stat_wal + + + + The pg_stat_wal view will always have a + single row, containing data about WAL activity of the cluster. + + + + <structname>pg_stat_wal</structname> View + + + + + Column Type + + + Description + + + + + + + + wal_buffers_full bigint + + + Number of times WAL data was written to the disk because WAL buffers got full + + + + + + stats_reset timestamp with time zone + + + Time at which these statistics were last reset + + + + +
+ + + <structname>pg_stat_database</structname> @@ -4668,8 +4726,9 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i argument. The argument can be bgwriter to reset all the counters shown in the pg_stat_bgwriter - view, or archiver to reset all the counters shown in - the pg_stat_archiver view. + view, archiver to reset all the counters shown in + the pg_stat_archiver view or wal + to reset all the counters shown in the pg_stat_wal view. This function is restricted to superusers by default, but other users diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 03c089b9a966..8f11b1b9de18 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -2196,6 +2196,7 @@ AdvanceXLInsertBuffer(XLogRecPtr upto, bool opportunistic) WriteRqst.Flush = 0; XLogWrite(WriteRqst, false); LWLockRelease(WALWriteLock); + WalStats.m_wal_buffers_full++; TRACE_POSTGRESQL_WAL_BUFFER_WRITE_DIRTY_DONE(); } /* Re-acquire WALBufMappingLock and retry */ diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index ed4f3f142d87..923c2e2be1f6 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -979,6 +979,12 @@ CREATE VIEW pg_stat_bgwriter AS pg_stat_get_buf_alloc() AS buffers_alloc, pg_stat_get_bgwriter_stat_reset_time() AS stats_reset; +CREATE VIEW pg_stat_wal AS + SELECT + w.wal_buffers_full, + w.stats_reset + FROM pg_stat_get_wal() w; + CREATE VIEW pg_stat_progress_analyze AS SELECT S.pid AS pid, S.datid AS datid, D.datname AS datname, diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c index 3e7dcd4f764d..429c8010ef44 100644 --- a/src/backend/postmaster/checkpointer.c +++ b/src/backend/postmaster/checkpointer.c @@ -504,6 +504,9 @@ CheckpointerMain(void) */ pgstat_send_bgwriter(); + /* Send WAL statistics to the stats collector. */ + pgstat_send_wal(); + /* * If any checkpoint flags have been set, redo the loop to handle the * checkpoint without sleeping. diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index e6be2b7836a4..5294c7854942 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -135,11 +135,12 @@ char *pgstat_stat_filename = NULL; char *pgstat_stat_tmpname = NULL; /* - * BgWriter global statistics counters (unused in other processes). - * Stored directly in a stats message structure so it can be sent - * without needing to copy things around. We assume this inits to zeroes. + * BgWriter and WAL global statistics counters. + * Stored directly in a stats message structure so they can be sent + * without needing to copy things around. We assume these init to zeroes. */ PgStat_MsgBgWriter BgWriterStats; +PgStat_MsgWal WalStats; /* * List of SLRU names that we keep stats for. There is no central registry of @@ -281,6 +282,7 @@ static int localNumBackends = 0; */ static PgStat_ArchiverStats archiverStats; static PgStat_GlobalStats globalStats; +static PgStat_WalStats walStats; static PgStat_SLRUStats slruStats[SLRU_NUM_ELEMENTS]; /* @@ -353,6 +355,7 @@ static void pgstat_recv_vacuum(PgStat_MsgVacuum *msg, int len); static void pgstat_recv_analyze(PgStat_MsgAnalyze *msg, int len); static void pgstat_recv_archiver(PgStat_MsgArchiver *msg, int len); static void pgstat_recv_bgwriter(PgStat_MsgBgWriter *msg, int len); +static void pgstat_recv_wal(PgStat_MsgWal *msg, int len); static void pgstat_recv_slru(PgStat_MsgSLRU *msg, int len); static void pgstat_recv_funcstat(PgStat_MsgFuncstat *msg, int len); static void pgstat_recv_funcpurge(PgStat_MsgFuncpurge *msg, int len); @@ -938,6 +941,9 @@ pgstat_report_stat(bool force) /* Now, send function statistics */ pgstat_send_funcstats(); + /* Send WAL statistics */ + pgstat_send_wal(); + /* Finally send SLRU statistics */ pgstat_send_slru(); } @@ -1370,11 +1376,13 @@ pgstat_reset_shared_counters(const char *target) msg.m_resettarget = RESET_ARCHIVER; else if (strcmp(target, "bgwriter") == 0) msg.m_resettarget = RESET_BGWRITER; + else if (strcmp(target, "wal") == 0) + msg.m_resettarget = RESET_WAL; else ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unrecognized reset target: \"%s\"", target), - errhint("Target must be \"archiver\" or \"bgwriter\"."))); + errhint("Target must be \"archiver\", \"bgwriter\" or \"wal\"."))); pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_RESETSHAREDCOUNTER); pgstat_send(&msg, sizeof(msg)); @@ -2674,6 +2682,21 @@ pgstat_fetch_global(void) return &globalStats; } +/* + * --------- + * pgstat_fetch_stat_wal() - + * + * Support function for the SQL-callable pgstat* functions. Returns + * a pointer to the WAL statistics struct. + * --------- + */ +PgStat_WalStats * +pgstat_fetch_stat_wal(void) +{ + backend_read_statsfile(); + + return &walStats; +} /* * --------- @@ -4419,6 +4442,38 @@ pgstat_send_bgwriter(void) MemSet(&BgWriterStats, 0, sizeof(BgWriterStats)); } +/* ---------- + * pgstat_send_wal() - + * + * Send WAL statistics to the collector + * ---------- + */ +void +pgstat_send_wal(void) +{ + /* We assume this initializes to zeroes */ + static const PgStat_MsgWal all_zeroes; + + /* + * This function can be called even if nothing at all has happened. In + * this case, avoid sending a completely empty message to the stats + * collector. + */ + if (memcmp(&WalStats, &all_zeroes, sizeof(PgStat_MsgWal)) == 0) + return; + + /* + * Prepare and send the message + */ + pgstat_setheader(&WalStats.m_hdr, PGSTAT_MTYPE_WAL); + pgstat_send(&WalStats, sizeof(WalStats)); + + /* + * Clear out the statistics buffer, so it can be re-used. + */ + MemSet(&WalStats, 0, sizeof(WalStats)); +} + /* ---------- * pgstat_send_slru() - * @@ -4658,6 +4713,10 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_bgwriter(&msg.msg_bgwriter, len); break; + case PGSTAT_MTYPE_WAL: + pgstat_recv_wal(&msg.msg_wal, len); + break; + case PGSTAT_MTYPE_SLRU: pgstat_recv_slru(&msg.msg_slru, len); break; @@ -4927,6 +4986,12 @@ pgstat_write_statsfiles(bool permanent, bool allDbs) rc = fwrite(&archiverStats, sizeof(archiverStats), 1, fpout); (void) rc; /* we'll check for error with ferror */ + /* + * Write WAL stats struct + */ + rc = fwrite(&walStats, sizeof(walStats), 1, fpout); + (void) rc; /* we'll check for error with ferror */ + /* * Write SLRU stats struct */ @@ -5186,11 +5251,12 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); /* - * Clear out global and archiver statistics so they start from zero in - * case we can't load an existing statsfile. + * Clear out global, archiver, WAL and SLRU statistics so they start from + * zero in case we can't load an existing statsfile. */ memset(&globalStats, 0, sizeof(globalStats)); memset(&archiverStats, 0, sizeof(archiverStats)); + memset(&walStats, 0, sizeof(walStats)); memset(&slruStats, 0, sizeof(slruStats)); /* @@ -5199,6 +5265,7 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) */ globalStats.stat_reset_timestamp = GetCurrentTimestamp(); archiverStats.stat_reset_timestamp = globalStats.stat_reset_timestamp; + walStats.stat_reset_timestamp = globalStats.stat_reset_timestamp; /* * Set the same reset timestamp for all SLRU items too. @@ -5268,6 +5335,17 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) goto done; } + /* + * Read WAL stats struct + */ + if (fread(&walStats, 1, sizeof(walStats), fpin) != sizeof(walStats)) + { + ereport(pgStatRunningInCollector ? LOG : WARNING, + (errmsg("corrupted statistics file \"%s\"", statfile))); + memset(&walStats, 0, sizeof(walStats)); + goto done; + } + /* * Read SLRU stats struct */ @@ -5578,6 +5656,7 @@ pgstat_read_db_statsfile_timestamp(Oid databaseid, bool permanent, PgStat_StatDBEntry dbentry; PgStat_GlobalStats myGlobalStats; PgStat_ArchiverStats myArchiverStats; + PgStat_WalStats myWalStats; PgStat_SLRUStats mySLRUStats[SLRU_NUM_ELEMENTS]; FILE *fpin; int32 format_id; @@ -5633,6 +5712,17 @@ pgstat_read_db_statsfile_timestamp(Oid databaseid, bool permanent, return false; } + /* + * Read WAL stats struct + */ + if (fread(&myWalStats, 1, sizeof(myWalStats), fpin) != sizeof(myWalStats)) + { + ereport(pgStatRunningInCollector ? LOG : WARNING, + (errmsg("corrupted statistics file \"%s\"", statfile))); + FreeFile(fpin); + return false; + } + /* * Read SLRU stats struct */ @@ -6213,6 +6303,12 @@ pgstat_recv_resetsharedcounter(PgStat_MsgResetsharedcounter *msg, int len) memset(&archiverStats, 0, sizeof(archiverStats)); archiverStats.stat_reset_timestamp = GetCurrentTimestamp(); } + else if (msg->m_resettarget == RESET_WAL) + { + /* Reset the WAL statistics for the cluster. */ + memset(&walStats, 0, sizeof(walStats)); + walStats.stat_reset_timestamp = GetCurrentTimestamp(); + } /* * Presumably the sender of this message validated the target, don't @@ -6427,6 +6523,18 @@ pgstat_recv_bgwriter(PgStat_MsgBgWriter *msg, int len) globalStats.buf_alloc += msg->m_buf_alloc; } +/* ---------- + * pgstat_recv_wal() - + * + * Process a WAL message. + * ---------- + */ +static void +pgstat_recv_wal(PgStat_MsgWal *msg, int len) +{ + walStats.wal_buffers_full += msg->m_wal_buffers_full; +} + /* ---------- * pgstat_recv_slru() - * diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 95738a4e34ee..24e191ea30b6 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1697,6 +1697,42 @@ pg_stat_get_buf_alloc(PG_FUNCTION_ARGS) PG_RETURN_INT64(pgstat_fetch_global()->buf_alloc); } +/* + * Returns statistics of WAL activity + */ +Datum +pg_stat_get_wal(PG_FUNCTION_ARGS) +{ +#define PG_STAT_GET_WAL_COLS 2 + TupleDesc tupdesc; + Datum values[PG_STAT_GET_WAL_COLS]; + bool nulls[PG_STAT_GET_WAL_COLS]; + PgStat_WalStats *wal_stats; + + /* Initialise values and NULL flags arrays */ + MemSet(values, 0, sizeof(values)); + MemSet(nulls, 0, sizeof(nulls)); + + /* Initialise attributes information in the tuple descriptor */ + tupdesc = CreateTemplateTupleDesc(PG_STAT_GET_WAL_COLS); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "wal_buffers_full", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "stats_reset", + TIMESTAMPTZOID, -1, 0); + + BlessTupleDesc(tupdesc); + + /* Get statistics about WAL activity */ + wal_stats = pgstat_fetch_stat_wal(); + + /* Fill values and NULLs */ + values[0] = Int64GetDatum(wal_stats->wal_buffers_full); + values[1] = TimestampTzGetDatum(wal_stats->stat_reset_timestamp); + + /* Returns the record as Datum */ + PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); +} + /* * Returns statistics of SLRU caches. */ diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 650e1f733ed2..1be017972438 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 202009181 +#define CATALOG_VERSION_NO 202010021 #endif diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index f48f5fb4d99b..d6f3e2d286b4 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5481,6 +5481,14 @@ proname => 'pg_stat_get_buf_alloc', provolatile => 's', proparallel => 'r', prorettype => 'int8', proargtypes => '', prosrc => 'pg_stat_get_buf_alloc' }, +{ oid => '1136', descr => 'statistics: information about WAL activity', + proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'record', proargtypes => '', + proallargtypes => '{int8,timestamptz}', + proargmodes => '{o,o}', + proargnames => '{wal_buffers_full,stats_reset}', + prosrc => 'pg_stat_get_wal' }, + { oid => '2306', descr => 'statistics: information about SLRU caches', proname => 'pg_stat_get_slru', prorows => '100', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 0dfbac46b4b0..343eef507eab 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -61,6 +61,7 @@ typedef enum StatMsgType PGSTAT_MTYPE_ANALYZE, PGSTAT_MTYPE_ARCHIVER, PGSTAT_MTYPE_BGWRITER, + PGSTAT_MTYPE_WAL, PGSTAT_MTYPE_SLRU, PGSTAT_MTYPE_FUNCSTAT, PGSTAT_MTYPE_FUNCPURGE, @@ -122,7 +123,8 @@ typedef struct PgStat_TableCounts typedef enum PgStat_Shared_Reset_Target { RESET_ARCHIVER, - RESET_BGWRITER + RESET_BGWRITER, + RESET_WAL } PgStat_Shared_Reset_Target; /* Possible object types for resetting single counters */ @@ -436,6 +438,16 @@ typedef struct PgStat_MsgBgWriter PgStat_Counter m_checkpoint_sync_time; } PgStat_MsgBgWriter; +/* ---------- + * PgStat_MsgWal Sent by backends and background processes to update WAL statistics. + * ---------- + */ +typedef struct PgStat_MsgWal +{ + PgStat_MsgHdr m_hdr; + PgStat_Counter m_wal_buffers_full; +} PgStat_MsgWal; + /* ---------- * PgStat_MsgSLRU Sent by a backend to update SLRU statistics. * ---------- @@ -596,6 +608,7 @@ typedef union PgStat_Msg PgStat_MsgAnalyze msg_analyze; PgStat_MsgArchiver msg_archiver; PgStat_MsgBgWriter msg_bgwriter; + PgStat_MsgWal msg_wal; PgStat_MsgSLRU msg_slru; PgStat_MsgFuncstat msg_funcstat; PgStat_MsgFuncpurge msg_funcpurge; @@ -614,7 +627,7 @@ typedef union PgStat_Msg * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BC9D +#define PGSTAT_FILE_FORMAT_ID 0x01A5BC9E /* ---------- * PgStat_StatDBEntry The collector's data per database @@ -745,6 +758,15 @@ typedef struct PgStat_GlobalStats TimestampTz stat_reset_timestamp; } PgStat_GlobalStats; +/* + * WAL statistics kept in the stats collector + */ +typedef struct PgStat_WalStats +{ + PgStat_Counter wal_buffers_full; + TimestampTz stat_reset_timestamp; +} PgStat_WalStats; + /* * SLRU statistics kept in the stats collector */ @@ -1265,6 +1287,11 @@ extern char *pgstat_stat_filename; */ extern PgStat_MsgBgWriter BgWriterStats; +/* + * WAL statistics counter is updated by backends and background processes + */ +extern PgStat_MsgWal WalStats; + /* * Updated by pgstat_count_buffer_*_time macros */ @@ -1464,6 +1491,7 @@ extern void pgstat_twophase_postabort(TransactionId xid, uint16 info, extern void pgstat_send_archiver(const char *xlog, bool failed); extern void pgstat_send_bgwriter(void); +extern void pgstat_send_wal(void); /* ---------- * Support functions for the SQL-callable functions to @@ -1478,6 +1506,7 @@ extern PgStat_StatFuncEntry *pgstat_fetch_stat_funcentry(Oid funcid); extern int pgstat_fetch_stat_numbackends(void); extern PgStat_ArchiverStats *pgstat_fetch_stat_archiver(void); extern PgStat_GlobalStats *pgstat_fetch_global(void); +extern PgStat_WalStats *pgstat_fetch_stat_wal(void); extern PgStat_SLRUStats *pgstat_fetch_slru(void); extern void pgstat_count_slru_page_zeroed(int slru_idx); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 2a18dc423e2b..af4192f9a872 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2129,6 +2129,9 @@ pg_stat_user_tables| SELECT pg_stat_all_tables.relid, pg_stat_all_tables.autoanalyze_count FROM pg_stat_all_tables WHERE ((pg_stat_all_tables.schemaname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND (pg_stat_all_tables.schemaname !~ '^pg_toast'::text)); +pg_stat_wal| SELECT w.wal_buffers_full, + w.stats_reset + FROM pg_stat_get_wal() w(wal_buffers_full, stats_reset); pg_stat_wal_receiver| SELECT s.pid, s.status, s.receive_start_lsn, diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out index 1cffc3349d60..81bdacf59daa 100644 --- a/src/test/regress/expected/sysviews.out +++ b/src/test/regress/expected/sysviews.out @@ -76,6 +76,13 @@ select count(*) >= 0 as ok from pg_prepared_xacts; t (1 row) +-- There must be only one record +select count(*) = 1 as ok from pg_stat_wal; + ok +---- + t +(1 row) + -- This is to record the prevailing planner enable_foo settings during -- a regression test run. select name, setting from pg_settings where name like 'enable%'; diff --git a/src/test/regress/sql/sysviews.sql b/src/test/regress/sql/sysviews.sql index ac4a0e1cbba7..b9b875bc6abc 100644 --- a/src/test/regress/sql/sysviews.sql +++ b/src/test/regress/sql/sysviews.sql @@ -37,6 +37,9 @@ select count(*) = 0 as ok from pg_prepared_statements; -- See also prepared_xacts.sql select count(*) >= 0 as ok from pg_prepared_xacts; +-- There must be only one record +select count(*) = 1 as ok from pg_stat_wal; + -- This is to record the prevailing planner enable_foo settings during -- a regression test run. select name, setting from pg_settings where name like 'enable%'; diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 9cd1179af63c..4191f9486963 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1841,6 +1841,7 @@ PgStat_MsgTabpurge PgStat_MsgTabstat PgStat_MsgTempFile PgStat_MsgVacuum +PgStat_MsgWal PgStat_SLRUStats PgStat_Shared_Reset_Target PgStat_Single_Reset_Type @@ -1852,6 +1853,7 @@ PgStat_TableCounts PgStat_TableEntry PgStat_TableStatus PgStat_TableXactStatus +PgStat_WalStats PgXmlErrorContext PgXmlStrictness Pg_finfo_record From 8550cbd0bae7c3004034351cb3447b51a552e56c Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Fri, 2 Oct 2020 10:36:35 +0900 Subject: [PATCH 238/589] doc: Improve some documentation about HA and replication This clarifies some wording in the description of the options available as replication solutions. While on it, this replaces some instances of "master" with "primary", for consistency with recent changes like 9e101cf. Author: Robert Treat Reviewed-by: Magnus Hagander, Michael Paquier Discussion: https://postgr.es/m/CAJSLCQ2TPaK_K8raofCamrqELCxY-H6mJrpDNRzc-LKpPY7c+g@mail.gmail.com --- doc/src/sgml/high-availability.sgml | 46 ++++++++++++++--------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index 42f01c515f9b..86da84fce709 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -39,9 +39,9 @@ Some solutions deal with synchronization by allowing only one server to modify the data. Servers that can modify data are called read/write, master or primary servers. - Servers that track changes in the master are called standby + Servers that track changes in the primary are called standby or secondary servers. A standby server that cannot be connected - to until it is promoted to a master server is called a warm + to until it is promoted to a primary server is called a warm standby server, and one that can accept connections and serves read-only queries is called a hot standby server. @@ -165,10 +165,10 @@ protocol to make nodes agree on a serializable transactional order. Logical replication allows a database server to send a stream of data modifications to another server. PostgreSQL logical replication constructs a stream of logical data modifications - from the WAL. Logical replication allows the data changes from - individual tables to be replicated. Logical replication doesn't require - a particular server to be designated as a primary or a replica but allows - data to flow in multiple directions. For more information on logical + from the WAL. Logical replication allows replication of data changes on + a per-table basis. In addition, a server that is publishing its own + changes can also subscribe to changes from another server, allowing data + to flow in multiple directions. For more information on logical replication, see . Through the logical decoding interface (), third-party extensions can also provide similar functionality. @@ -177,22 +177,24 @@ protocol to make nodes agree on a serializable transactional order. - Trigger-Based Master-Standby Replication + Trigger-Based Primary-Standby Replication - A master-standby replication setup sends all data modification - queries to the master server. The master server asynchronously - sends data changes to the standby server. The standby can answer - read-only queries while the master server is running. The - standby server is ideal for data warehouse queries. + A trigger-based replication setup typically funnels data modification + queries to a designated primary server. Operating on a per-table basis, + the primary server sends data changes (typically) asynchronously to the + standby servers. Standby servers can answer queries while the primary is + running, and may allow some local data changes or write activity. This + form of replication is often used for offloading large analytical or data + warehouse queries. - Slony-I is an example of this type of replication, with per-table - granularity, and support for multiple standby servers. Because it - updates the standby server asynchronously (in batches), there is - possible data loss during fail over. + Slony-I is an example of this type of + replication, with per-table granularity, and support for multiple standby + servers. Because it updates the standby server asynchronously (in + batches), there is possible data loss during fail over. @@ -215,14 +217,10 @@ protocol to make nodes agree on a serializable transactional order. random(), CURRENT_TIMESTAMP, and sequences can have different values on different servers. This is because each server operates independently, and because - SQL queries are broadcast (and not actual modified rows). If + SQL queries are broadcast rather than actual data changes. If this is unacceptable, either the middleware or the application - must query such values from a single server and then use those - values in write queries. Another option is to use this replication - option with a traditional primary-standby setup, i.e., data modification - queries are sent only to the primary and are propagated to the - standby servers via primary-standby replication, not by the replication - middleware. Care must also be taken that all + must determine such values from a single source and then use those + values in write queries. Care must also be taken that all transactions either commit or abort on all servers, perhaps using two-phase commit ( and ). @@ -351,7 +349,7 @@ protocol to make nodes agree on a serializable transactional order. - Allows multiple master servers + Allows multiple primary servers From 26b8361518c393c2f152e9e3837daf605b34bef8 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 2 Oct 2020 18:23:39 +0300 Subject: [PATCH 239/589] Tidy up error reporting when converting PL/Python arrays. Use PLy_elog() only when a call to a Python C API function failed, and ereport() for other errors. Add an error code to the "wrong length of inner sequence" ereport(). Reviewed-by: Daniel Gustafsson Discussion: https://www.postgresql.org/message-id/B8B72889-D6D7-48FF-B782-D670A6CA4D37%40yesql.se --- src/pl/plpython/plpy_typeio.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/pl/plpython/plpy_typeio.c b/src/pl/plpython/plpy_typeio.c index 9e4e9035f783..b4aeb7fd5959 100644 --- a/src/pl/plpython/plpy_typeio.c +++ b/src/pl/plpython/plpy_typeio.c @@ -1173,18 +1173,25 @@ PLySequence_ToArray(PLyObToDatum *arg, PyObject *plrv, break; if (ndim == MAXDIM) - PLy_elog(ERROR, "number of array dimensions exceeds the maximum allowed (%d)", MAXDIM); + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("number of array dimensions exceeds the maximum allowed (%d)", + MAXDIM))); dims[ndim] = PySequence_Length(pyptr); if (dims[ndim] < 0) PLy_elog(ERROR, "could not determine sequence length for function return value"); if (dims[ndim] > MaxAllocSize) - PLy_elog(ERROR, "array size exceeds the maximum allowed"); + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("array size exceeds the maximum allowed"))); len *= dims[ndim]; if (len > MaxAllocSize) - PLy_elog(ERROR, "array size exceeds the maximum allowed"); + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("array size exceeds the maximum allowed"))); if (dims[ndim] == 0) { @@ -1210,7 +1217,9 @@ PLySequence_ToArray(PLyObToDatum *arg, PyObject *plrv, if (ndim == 0) { if (!PySequence_Check(plrv)) - PLy_elog(ERROR, "return value of function with array return type is not a Python sequence"); + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("return value of function with array return type is not a Python sequence"))); ndim = 1; len = dims[0] = PySequence_Length(plrv); @@ -1256,7 +1265,8 @@ PLySequence_ToArray_recurse(PLyObToDatum *elm, PyObject *list, if (PySequence_Length(list) != dims[dim]) ereport(ERROR, - (errmsg("wrong length of inner sequence: has length %d, but %d was expected", + (errcode(ERRCODE_ARRAY_SUBSCRIPT_ERROR), + errmsg("wrong length of inner sequence: has length %d, but %d was expected", (int) PySequence_Length(list), dims[dim]), (errdetail("To construct a multidimensional array, the inner sequences must all have the same length.")))); From 472e518a44eacd9caac7d618f1b6451672ca4481 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Fri, 2 Oct 2020 21:39:33 -0400 Subject: [PATCH 240/589] doc: clarify the use of ssh port forwarding Reported-by: karimelghazouly@gmail.com Discussion: https://postgr.es/m/159854511172.24991.4373145230066586863@wrigleys.postgresql.org Backpatch-through: 9.5 --- doc/src/sgml/runtime.sgml | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index f975406acd67..418aa3f85c7a 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -2611,34 +2611,39 @@ openssl x509 -req -in server.csr -text -days 365 \ First make sure that an SSH server is running properly on the same machine as the PostgreSQL server and that you can log in using - ssh as some user. Then you can establish a secure - tunnel with a command like this from the client machine: + ssh as some user; you then can establish a + secure tunnel to the remote server. A secure tunnel listens on a + local port and forwards all traffic to a port on the remote machine. + Traffic sent to the remote port can arrive on its + localhost address, or different bind + address if desired; it does not appear as coming from your + local machine. This command creates a secure tunnel from the client + machine to the remote machine foo.com: ssh -L 63333:localhost:5432 joe@foo.com The first number in the argument, 63333, is the - port number of your end of the tunnel; it can be any unused port. - (IANA reserves ports 49152 through 65535 for private use.) The - second number, 5432, is the remote end of the tunnel: the port - number your server is using. The name or IP address between the - port numbers is the host with the database server you are going to - connect to, as seen from the host you are logging in to, which - is foo.com in this example. In order to connect - to the database server using this tunnel, you connect to port 63333 - on the local machine: + local port number of the tunnel; it can be any unused port. (IANA + reserves ports 49152 through 65535 for private use.) The name or IP + address after this is the remote bind address you are connecting to, + i.e., localhost, which is the default. The second + number, 5432, is the remote end of the tunnel, e.g., the port number + your database server is using. In order to connect to the database + server using this tunnel, you connect to port 63333 on the local + machine: psql -h localhost -p 63333 postgres - To the database server it will then look as though you are really + To the database server it will then look as though you are user joe on host foo.com - connecting to localhost in that context, and it + connecting to the localhost bind address, and it will use whatever authentication procedure was configured for - connections from this user and host. Note that the server will not + connections by that user to that bind address. Note that the server will not think the connection is SSL-encrypted, since in fact it is not encrypted between the SSH server and the PostgreSQL server. This should not pose any - extra security risk as long as they are on the same machine. + extra security risk because they are on the same machine. @@ -2650,12 +2655,12 @@ psql -h localhost -p 63333 postgres - You could also have set up the port forwarding as + You could also have set up port forwarding as ssh -L 63333:foo.com:5432 joe@foo.com but then the database server will see the connection as coming in - on its foo.com interface, which is not opened by + on its foo.com bind address, which is not opened by the default setting listen_addresses = 'localhost'. This is usually not what you want. From 1a9388bd0fefccc81430192d0b2c87938bb62d38 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Fri, 2 Oct 2020 22:19:31 -0400 Subject: [PATCH 241/589] doc: libpq connection options can override command-line flags Reported-by: Alexander Lakhin Discussion: https://postgr.es/m/16486-b9c93d71c02c4907@postgresql.org Backpatch-through: 9.5 --- doc/src/sgml/ref/clusterdb.sgml | 5 ++++- doc/src/sgml/ref/pg_basebackup.sgml | 5 +++-- doc/src/sgml/ref/pg_dump.sgml | 12 ++++-------- doc/src/sgml/ref/pg_dumpall.sgml | 5 +++-- doc/src/sgml/ref/pg_isready.sgml | 14 +++++--------- doc/src/sgml/ref/pg_receivewal.sgml | 5 +++-- doc/src/sgml/ref/pg_recvlogical.sgml | 14 ++++++++------ doc/src/sgml/ref/pg_restore.sgml | 5 ++++- doc/src/sgml/ref/psql-ref.sgml | 25 +++++++++++-------------- doc/src/sgml/ref/reindexdb.sgml | 5 ++++- doc/src/sgml/ref/vacuumdb.sgml | 5 ++++- 11 files changed, 53 insertions(+), 47 deletions(-) diff --git a/doc/src/sgml/ref/clusterdb.sgml b/doc/src/sgml/ref/clusterdb.sgml index 177856ca74d2..c53bacf8648c 100644 --- a/doc/src/sgml/ref/clusterdb.sgml +++ b/doc/src/sgml/ref/clusterdb.sgml @@ -95,7 +95,10 @@ PostgreSQL documentation ) is not used, the database name is read from the environment variable PGDATABASE. If that is not set, the user name specified for the connection is - used. + used. The dbname can be a connection string. If so, + connection string parameters will override any conflicting command + line options. diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index 44e7b2444b7e..e993e8761c13 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -653,8 +653,9 @@ PostgreSQL documentation - Specifies parameters used to connect to the server, as a connection - string. See for more information. + Specifies parameters used to connect to the server, as a connction string; these + will override any conflicting command line options. The option is called --dbname for consistency with other diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index e09ed0a4c3ac..1400cf877599 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -1132,14 +1132,10 @@ PostgreSQL documentation Specifies the name of the database to connect to. This is equivalent to specifying dbname as the first non-option - argument on the command line. - - - If this parameter contains an = sign or starts - with a valid URI prefix - (postgresql:// - or postgres://), it is treated as a - conninfo string. See for more information. + argument on the command line. The dbname + can be a connection string. + If so, connection string parameters will override any conflicting + command line options. diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml index 5b514e4567b2..4360b2cf5773 100644 --- a/doc/src/sgml/ref/pg_dumpall.sgml +++ b/doc/src/sgml/ref/pg_dumpall.sgml @@ -552,8 +552,9 @@ PostgreSQL documentation - Specifies parameters used to connect to the server, as a connection - string. See for more information. + Specifies parameters used to connect to the server, as a connction string; these + will override any conflicting command line options. The option is called --dbname for consistency with other diff --git a/doc/src/sgml/ref/pg_isready.sgml b/doc/src/sgml/ref/pg_isready.sgml index 3d5b551b87f2..ba25ca65a40e 100644 --- a/doc/src/sgml/ref/pg_isready.sgml +++ b/doc/src/sgml/ref/pg_isready.sgml @@ -47,15 +47,11 @@ PostgreSQL documentation - Specifies the name of the database to connect to. - - - If this parameter contains an = sign or starts - with a valid URI prefix - (postgresql:// - or postgres://), it is treated as a - conninfo string. See for more information. + Specifies the name of the database to connect to. The + dbname can be a connection string. If so, + connection string parameters will override any conflicting command + line options. diff --git a/doc/src/sgml/ref/pg_receivewal.sgml b/doc/src/sgml/ref/pg_receivewal.sgml index 865ec8426219..26a66c0e19b8 100644 --- a/doc/src/sgml/ref/pg_receivewal.sgml +++ b/doc/src/sgml/ref/pg_receivewal.sgml @@ -252,8 +252,9 @@ PostgreSQL documentation - Specifies parameters used to connect to the server, as a connection - string. See for more information. + Specifies parameters used to connect to the server, as a connction string; these + will override any conflicting command line options. The option is called --dbname for consistency with other diff --git a/doc/src/sgml/ref/pg_recvlogical.sgml b/doc/src/sgml/ref/pg_recvlogical.sgml index 41508fdc1e56..6b1d98d06ef1 100644 --- a/doc/src/sgml/ref/pg_recvlogical.sgml +++ b/doc/src/sgml/ref/pg_recvlogical.sgml @@ -273,14 +273,16 @@ PostgreSQL documentation - - + + - The database to connect to. See the description of the actions for - what this means in detail. This can be a libpq connection string; - see for more information. Defaults - to user name. + The database to connect to. See the description + of the actions for what this means in detail. + The dbname can be a connection string. If so, + connection string parameters will override any conflicting + command line options. Defaults to the user name. diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml index e0d9d2ad64f4..93ea937ac8ea 100644 --- a/doc/src/sgml/ref/pg_restore.sgml +++ b/doc/src/sgml/ref/pg_restore.sgml @@ -156,7 +156,10 @@ PostgreSQL documentation Connect to database dbname and restore directly - into the database. + into the database. The dbname can + be a connection string. + If so, connection string parameters will override any conflicting + command line options. diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml index ef18fe27e03c..aed051f543c0 100644 --- a/doc/src/sgml/ref/psql-ref.sgml +++ b/doc/src/sgml/ref/psql-ref.sgml @@ -168,15 +168,10 @@ EOF Specifies the name of the database to connect to. This is equivalent to specifying dbname as the first non-option - argument on the command line. - - - If this parameter contains an = sign or starts - with a valid URI prefix - (postgresql:// - or postgres://), it is treated as a - conninfo string. See for more information. + argument on the command line. The dbname + can be a connection string. + If so, connection string parameters will override any conflicting + command line options. @@ -498,7 +493,7 @@ EOF Never issue a password prompt. If the server requires password - authentication and a password is not available by other means + authentication and a password is not available from other sources such as a .pgpass file, the connection attempt will fail. This option can be useful in batch jobs and scripts where no user is present to enter a password. @@ -518,13 +513,15 @@ EOF Force psql to prompt for a - password before connecting to a database. + password before connecting to a database, even if the password will + not be used. - This option is never essential, since psql - will automatically prompt for a password if the server demands - password authentication. However, psql + If the server requires password authentication and a password is not + available from other sources such as a .pgpass + file, psql will prompt for a + password in any case. However, psql will waste a connection attempt finding out that the server wants a password. In some cases it is worth typing to avoid the extra connection attempt. diff --git a/doc/src/sgml/ref/reindexdb.sgml b/doc/src/sgml/ref/reindexdb.sgml index 4f821c2095ef..0f3f12bfbfc6 100644 --- a/doc/src/sgml/ref/reindexdb.sgml +++ b/doc/src/sgml/ref/reindexdb.sgml @@ -139,7 +139,10 @@ PostgreSQL documentation ) is not used, the database name is read from the environment variable PGDATABASE. If that is not set, the user name specified for the connection is - used. + used. The dbname can be a connection string. If so, + connection string parameters will override any conflicting command + line options. diff --git a/doc/src/sgml/ref/vacuumdb.sgml b/doc/src/sgml/ref/vacuumdb.sgml index 7b1395055244..766c6882bd4c 100644 --- a/doc/src/sgml/ref/vacuumdb.sgml +++ b/doc/src/sgml/ref/vacuumdb.sgml @@ -97,7 +97,10 @@ PostgreSQL documentation ) is not used, the database name is read from the environment variable PGDATABASE. If that is not set, the user name specified for the connection is - used. + used. The dbname can be a connection string. If so, + connection string parameters will override any conflicting command + line options. From 9081bddbd75e4e8994ca243c820ca63387bd33f7 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sat, 3 Oct 2020 16:16:51 +0200 Subject: [PATCH 242/589] Improve vs. formatting in the documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SQL commands are generally marked up as , except when a link to a reference page is used using . But the latter doesn't create monospace markup, so this looks strange especially when a paragraph contains a mix of links and non-links. We considered putting in the on the target side, but that creates some formatting side effects elsewhere. Generally, it seems safer to solve this on the link source side. We can't put the inside the ; the DTD doesn't allow this. DocBook 5 would allow the to have the linkend attribute itself, but we are not there yet. So to solve this for now, convert the s to plus . This gives the correct look and also gives some more flexibility what we can put into the link text (e.g., subcommands or other clauses). In the future, these could then be converted to DocBook 5 style. I haven't converted absolutely all xrefs to SQL command reference pages, only those where we care about the appearance of the link text or where it was otherwise appropriate to make the appearance match a bit better. Also in some cases, the links where repetitive, so in those cases the links where just removed and replaced by a plain . In cases where we just want the link and don't specifically care about the generated link text (typically phrased "for further information see ") the xref is kept. Reported-by: Dagfinn Ilmari Mannsåker Discussion: https://www.postgresql.org/message-id/flat/87o8pco34z.fsf@wibble.ilmari.org --- doc/src/sgml/backup.sgml | 8 +- doc/src/sgml/catalogs.sgml | 98 +++++++++---------- doc/src/sgml/config.sgml | 24 ++--- doc/src/sgml/ddl.sgml | 22 ++--- doc/src/sgml/dml.sgml | 2 +- doc/src/sgml/ecpg.sgml | 7 +- doc/src/sgml/extend.sgml | 10 +- doc/src/sgml/logical-replication.sgml | 12 +-- doc/src/sgml/maintenance.sgml | 10 +- doc/src/sgml/monitoring.sgml | 6 +- doc/src/sgml/mvcc.sgml | 13 ++- doc/src/sgml/perform.sgml | 12 +-- doc/src/sgml/plpgsql.sgml | 2 +- doc/src/sgml/postgres-fdw.sgml | 10 +- doc/src/sgml/queries.sgml | 2 +- doc/src/sgml/ref/abort.sgml | 8 +- doc/src/sgml/ref/alter_aggregate.sgml | 2 +- doc/src/sgml/ref/alter_domain.sgml | 2 +- doc/src/sgml/ref/alter_foreign_table.sgml | 20 ++-- doc/src/sgml/ref/alter_group.sgml | 6 +- doc/src/sgml/ref/alter_index.sgml | 10 +- doc/src/sgml/ref/alter_materialized_view.sgml | 3 +- doc/src/sgml/ref/alter_role.sgml | 24 ++--- doc/src/sgml/ref/alter_statistics.sgml | 2 +- doc/src/sgml/ref/alter_table.sgml | 32 +++--- doc/src/sgml/ref/alter_trigger.sgml | 2 +- doc/src/sgml/ref/alter_type.sgml | 2 +- doc/src/sgml/ref/alter_user.sgml | 2 +- doc/src/sgml/ref/analyze.sgml | 11 +-- doc/src/sgml/ref/begin.sgml | 16 +-- doc/src/sgml/ref/close.sgml | 2 +- doc/src/sgml/ref/cluster.sgml | 4 +- doc/src/sgml/ref/copy.sgml | 8 +- doc/src/sgml/ref/create_aggregate.sgml | 2 +- doc/src/sgml/ref/create_cast.sgml | 2 +- doc/src/sgml/ref/create_database.sgml | 8 +- doc/src/sgml/ref/create_foreign_table.sgml | 4 +- doc/src/sgml/ref/create_function.sgml | 2 +- doc/src/sgml/ref/create_index.sgml | 6 +- doc/src/sgml/ref/create_language.sgml | 4 +- .../sgml/ref/create_materialized_view.sgml | 4 +- doc/src/sgml/ref/create_operator.sgml | 4 +- doc/src/sgml/ref/create_procedure.sgml | 2 +- doc/src/sgml/ref/create_role.sgml | 12 +-- doc/src/sgml/ref/create_table.sgml | 12 +-- doc/src/sgml/ref/create_table_as.sgml | 10 +- doc/src/sgml/ref/create_transform.sgml | 2 +- doc/src/sgml/ref/create_trigger.sgml | 4 +- doc/src/sgml/ref/create_type.sgml | 4 +- doc/src/sgml/ref/create_user.sgml | 2 +- doc/src/sgml/ref/create_view.sgml | 10 +- doc/src/sgml/ref/createdb.sgml | 4 +- doc/src/sgml/ref/createuser.sgml | 2 +- doc/src/sgml/ref/declare.sgml | 14 +-- doc/src/sgml/ref/delete.sgml | 2 +- doc/src/sgml/ref/drop_group.sgml | 2 +- doc/src/sgml/ref/drop_language.sgml | 2 +- doc/src/sgml/ref/drop_owned.sgml | 2 +- doc/src/sgml/ref/drop_role.sgml | 4 +- doc/src/sgml/ref/drop_table.sgml | 4 +- doc/src/sgml/ref/drop_user.sgml | 2 +- doc/src/sgml/ref/dropdb.sgml | 2 +- doc/src/sgml/ref/dropuser.sgml | 2 +- doc/src/sgml/ref/end.sgml | 8 +- doc/src/sgml/ref/explain.sgml | 2 +- doc/src/sgml/ref/fetch.sgml | 4 +- doc/src/sgml/ref/grant.sgml | 2 +- doc/src/sgml/ref/lock.sgml | 6 +- doc/src/sgml/ref/postgres-ref.sgml | 4 +- doc/src/sgml/ref/prepare.sgml | 4 +- doc/src/sgml/ref/prepare_transaction.sgml | 8 +- doc/src/sgml/ref/psql-ref.sgml | 30 +++--- doc/src/sgml/ref/reassign_owned.sgml | 2 +- .../sgml/ref/refresh_materialized_view.sgml | 2 +- doc/src/sgml/ref/reindexdb.sgml | 2 +- doc/src/sgml/ref/revoke.sgml | 4 +- doc/src/sgml/ref/rollback.sgml | 2 +- doc/src/sgml/ref/rollback_to.sgml | 2 +- doc/src/sgml/ref/savepoint.sgml | 4 +- doc/src/sgml/ref/select.sgml | 4 +- doc/src/sgml/ref/select_into.sgml | 6 +- doc/src/sgml/ref/set_role.sgml | 6 +- doc/src/sgml/ref/set_session_auth.sgml | 4 +- doc/src/sgml/ref/start_transaction.sgml | 4 +- doc/src/sgml/ref/vacuumdb.sgml | 2 +- doc/src/sgml/sepgsql.sgml | 8 +- doc/src/sgml/tsm-system-rows.sgml | 2 +- doc/src/sgml/tsm-system-time.sgml | 2 +- doc/src/sgml/user-manag.sgml | 24 ++--- doc/src/sgml/xaggr.sgml | 2 +- doc/src/sgml/xfunc.sgml | 12 +-- doc/src/sgml/xplang.sgml | 2 +- 92 files changed, 348 insertions(+), 346 deletions(-) diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml index b9331830f7d0..42a8ed328d88 100644 --- a/doc/src/sgml/backup.sgml +++ b/doc/src/sgml/backup.sgml @@ -177,8 +177,8 @@ pg_dump -h host1 dbname | - After restoring a backup, it is wise to run on each + After restoring a backup, it is wise to run ANALYZE on each database so the query optimizer has useful statistics; see and for more information. @@ -1594,7 +1594,7 @@ archive_command = 'local_backup_script.sh "%p" "%f"' - If a + If a CREATE DATABASE command is executed while a base backup is being taken, and then the template database that the CREATE DATABASE copied is modified while the base backup is still in progress, it is @@ -1607,7 +1607,7 @@ archive_command = 'local_backup_script.sh "%p" "%f"' - + CREATE TABLESPACE commands are WAL-logged with the literal absolute path, and will therefore be replayed as tablespace creations with the same absolute path. This might be undesirable if the log is being diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index de9bacd34f8d..0e580b157f53 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -639,8 +639,8 @@ - New aggregate functions are registered with the + New aggregate functions are registered with the CREATE AGGREGATE command. See for more information about writing aggregate functions and the meaning of the transition functions, etc. @@ -1161,7 +1161,7 @@ attstattarget controls the level of detail of statistics accumulated for this column by - . + ANALYZE. A zero value indicates that no statistics should be collected. A negative value says to use the system default statistics target. The exact meaning of positive values is data type-dependent. @@ -1966,9 +1966,9 @@ SCRAM-SHA-256$<iteration count>:&l Size of the on-disk representation of this table in pages (of size BLCKSZ). This is only an estimate used by the - planner. It is updated by , - , and a few DDL commands such as - . + planner. It is updated by VACUUM, + ANALYZE, and a few DDL commands such as + CREATE INDEX. @@ -1978,9 +1978,9 @@ SCRAM-SHA-256$<iteration count>:&l Number of live rows in the table. This is only an estimate used by - the planner. It is updated by , - , and a few DDL commands such as - . + the planner. It is updated by VACUUM, + ANALYZE, and a few DDL commands such as + CREATE INDEX. If the table has never yet been vacuumed or analyzed, reltuples contains -1 indicating that the row count is @@ -1995,9 +1995,9 @@ SCRAM-SHA-256$<iteration count>:&l Number of pages that are marked all-visible in the table's visibility map. This is only an estimate used by the - planner. It is updated by , - , and a few DDL commands such as - . + planner. It is updated by VACUUM, + ANALYZE, and a few DDL commands such as + CREATE INDEX. @@ -2241,8 +2241,8 @@ SCRAM-SHA-256$<iteration count>:&l lazily: they are guaranteed to be true if that's the correct state, but may not be reset to false immediately when the condition is no longer true. For example, relhasindex is set by - , but it is never cleared by - . Instead, clears + CREATE INDEX, but it is never cleared by + DROP INDEX. Instead, VACUUM clears relhasindex if it finds the table has no indexes. This arrangement avoids race conditions and improves concurrency. @@ -2848,8 +2848,8 @@ SCRAM-SHA-256$<iteration count>:&l The catalog pg_database stores information about - the available databases. Databases are created with the command. + the available databases. Databases are created with the CREATE DATABASE command. Consult for details about the meaning of some of the parameters. @@ -3425,7 +3425,7 @@ SCRAM-SHA-256$<iteration count>:&l the referenced object (see pg_extension). The dependent object can be dropped only via - on the referenced object. + DROP EXTENSION on the referenced object. Functionally this dependency type acts the same as an INTERNAL dependency, but it's kept separate for clarity and to simplify pg_dump. @@ -3492,7 +3492,7 @@ SCRAM-SHA-256$<iteration count>:&l The catalog pg_description stores optional descriptions (comments) for each database object. Descriptions can be manipulated - with the command and viewed with + with the COMMENT command and viewed with psql's \d commands. Descriptions of many built-in system objects are provided in the initial contents of pg_description. @@ -4285,7 +4285,7 @@ SCRAM-SHA-256$<iteration count>:&l If true, the index is currently valid for queries. False means the index is possibly incomplete: it must still be modified by - / operations, but it cannot safely + INSERT/UPDATE operations, but it cannot safely be used for queries. If it is unique, the uniqueness property is not guaranteed true either. @@ -4309,7 +4309,7 @@ SCRAM-SHA-256$<iteration count>:&l If true, the index is currently ready for inserts. False means the - index must be ignored by / + index must be ignored by INSERT/UPDATE operations. @@ -4504,11 +4504,11 @@ SCRAM-SHA-256$<iteration count>:&l Objects can have initial privileges either by having those privileges set when the system is initialized (by initdb) or when the - object is created during a and the - extension script sets initial privileges using the + object is created during a CREATE EXTENSION and the + extension script sets initial privileges using the GRANT system. Note that the system will automatically handle recording of the privileges during the extension script and that extension authors need - only use the and + only use the GRANT and REVOKE statements in their script to have the privileges recorded. The privtype column indicates if the initial privilege was set by initdb or during a @@ -6481,7 +6481,7 @@ SCRAM-SHA-256$<iteration count>:&l The catalog pg_seclabel stores security labels on database objects. Security labels can be manipulated - with the command. For an easier + with the SECURITY LABEL command. For an easier way to view security labels, see . @@ -6855,7 +6855,7 @@ SCRAM-SHA-256$<iteration count>:&l The catalog pg_shdescription stores optional descriptions (comments) for shared database objects. Descriptions can be - manipulated with the command and viewed with + manipulated with the COMMENT command and viewed with psql's \d commands. @@ -6931,7 +6931,7 @@ SCRAM-SHA-256$<iteration count>:&l The catalog pg_shseclabel stores security labels on shared database objects. Security labels can be manipulated - with the command. For an easier + with the SECURITY LABEL command. For an easier way to view security labels, see . @@ -7015,7 +7015,7 @@ SCRAM-SHA-256$<iteration count>:&l The catalog pg_statistic stores statistical data about the contents of the database. Entries are - created by + created by ANALYZE and subsequently used by the query planner. Note that all the statistical data is inherently approximate, even assuming that it is up-to-date. @@ -7223,7 +7223,7 @@ SCRAM-SHA-256$<iteration count>:&l The catalog pg_statistic_ext holds definitions of extended planner statistics. Each row in this catalog corresponds to a statistics object - created with . + created with CREATE STATISTICS. @@ -7296,7 +7296,7 @@ SCRAM-SHA-256$<iteration count>:&l stxstattarget controls the level of detail of statistics accumulated for this statistics object by - . + ANALYZE. A zero value indicates that no statistics should be collected. A negative value says to use the maximum of the statistics targets of the referenced columns, if set, or the system default statistics target. @@ -7337,9 +7337,9 @@ SCRAM-SHA-256$<iteration count>:&l The pg_statistic_ext entry is filled in - completely during , but the actual + completely during CREATE STATISTICS, but the actual statistical values are not computed then. - Subsequent commands compute the desired values + Subsequent ANALYZE commands compute the desired values and populate an entry in the pg_statistic_ext_data catalog. @@ -7358,7 +7358,7 @@ SCRAM-SHA-256$<iteration count>:&l holds data for extended planner statistics defined in pg_statistic_ext. Each row in this catalog corresponds to a statistics object - created with . + created with CREATE STATISTICS. @@ -7591,7 +7591,7 @@ SCRAM-SHA-256$<iteration count>:&l This catalog only contains tables known to the subscription after running - either or + either CREATE SUBSCRIPTION or ALTER SUBSCRIPTION ... REFRESH PUBLICATION. @@ -8569,9 +8569,9 @@ SCRAM-SHA-256$<iteration count>:&l The catalog pg_type stores information about data types. Base types and enum types (scalar types) are created with - , and + CREATE TYPE, and domains with - . + CREATE DOMAIN. A composite type is automatically created for each table in the database, to represent the row structure of the table. It is also possible to create composite types with CREATE TYPE AS. @@ -9791,7 +9791,7 @@ SCRAM-SHA-256$<iteration count>:&l - via the + via the DECLARE statement in SQL @@ -10917,7 +10917,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx pg_prepared_statements contains one row for each prepared statement. Rows are added to the view when a new prepared statement is created and removed when a prepared statement - is released (for example, via the command). + is released (for example, via the DEALLOCATE command).
@@ -10951,7 +10951,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx The query string submitted by the client to create this prepared statement. For prepared statements created via SQL, - this is the statement submitted by + this is the PREPARE statement submitted by the client. For prepared statements created via the frontend/backend protocol, this is the text of the prepared statement itself. @@ -10985,7 +10985,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx true if the prepared statement was created - via the SQL command; + via the PREPARE SQL command; false if the statement was prepared via the frontend/backend protocol @@ -11967,10 +11967,10 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx The view pg_settings provides access to run-time parameters of the server. It is essentially an alternative - interface to the - and commands. + interface to the SHOW + and SET commands. It also provides access to some facts about each parameter that are - not directly available from SHOW, such as minimum and + not directly available from SHOW, such as minimum and maximum values. @@ -12116,7 +12116,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx reset_valtext - Value that would reset the parameter to + Value that RESET would reset the parameter to in the current session @@ -12249,7 +12249,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx These settings can be set from postgresql.conf, - or within a session via the command; but only superusers + or within a session via the SET command; but only superusers can change them via SET. Changes in postgresql.conf will affect existing sessions only if no session-local value has been established with SET. @@ -12262,7 +12262,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx These settings can be set from postgresql.conf, - or within a session via the command. Any user is + or within a session via the SET command. Any user is allowed to change their session-local value. Changes in postgresql.conf will affect existing sessions only if no session-local value has been established with SET. @@ -12278,9 +12278,9 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx The pg_settings view cannot be inserted into or - deleted from, but it can be updated. An applied + deleted from, but it can be updated. An UPDATE applied to a row of pg_settings is equivalent to executing - the command on that named + the SET command on that named parameter. The change only affects the value used by the current session. If an UPDATE is issued within a transaction that is later aborted, the effects of the UPDATE command @@ -12622,7 +12622,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx If greater than zero, the estimated number of distinct values in the column. If less than zero, the negative of the number of distinct values divided by the number of rows. (The negated form is used when - believes that the number of distinct values is + ANALYZE believes that the number of distinct values is likely to increase as the table grows; the positive form is used when the column seems to have a fixed number of possible values.) For example, -1 indicates a unique column in which the number of distinct @@ -12846,7 +12846,7 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx than zero, the estimated number of distinct values in the combination. If less than zero, the negative of the number of distinct values divided by the number of rows. - (The negated form is used when believes that + (The negated form is used when ANALYZE believes that the number of distinct values is likely to increase as the table grows; the positive form is used when the column seems to have a fixed number of possible values.) For example, -1 indicates a unique combination of diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 06405f359cdf..ee914740cc4b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -189,7 +189,7 @@ shared_buffers = 128MB postgresql.auto.confpostgresql.auto.conf, which has the same format as postgresql.conf but is intended to be edited automatically, not manually. This file holds - settings provided through the command. + settings provided through the ALTER SYSTEM command. This file is read whenever postgresql.conf is, and its settings take effect in the same way. Settings in postgresql.auto.conf override those @@ -221,7 +221,7 @@ shared_buffers = 128MB PostgreSQL provides three SQL commands to establish configuration defaults. - The already-mentioned command + The already-mentioned ALTER SYSTEM command provides a SQL-accessible means of changing global defaults; it is functionally equivalent to editing postgresql.conf. In addition, there are two commands that allow setting of defaults @@ -231,14 +231,14 @@ shared_buffers = 128MB - The command allows global + The ALTER DATABASE command allows global settings to be overridden on a per-database basis. - The command allows both global and + The ALTER ROLE command allows both global and per-database settings to be overridden with user-specific values. @@ -262,7 +262,7 @@ shared_buffers = 128MB - The command allows inspection of the + The SHOW command allows inspection of the current value of all parameters. The corresponding function is current_setting(setting_name text). @@ -270,7 +270,7 @@ shared_buffers = 128MB - The command allows modification of the + The SET command allows modification of the current value of those parameters that can be set locally to a session; it has no effect on other sessions. The corresponding function is @@ -296,7 +296,7 @@ shared_buffers = 128MB - Using on this view, specifically + Using UPDATE on this view, specifically updating the setting column, is the equivalent of issuing SET commands. For example, the equivalent of @@ -4608,7 +4608,7 @@ ANY num_sync ( ), - running manually, increasing + running ANALYZE manually, increasing the value of the configuration parameter, and increasing the amount of statistics collected for @@ -8095,7 +8095,7 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; set it to replica when they are applying replicated changes. The effect of that will be that triggers and rules (that have not been altered from their default configuration) will not fire - on the replica. See the clauses + on the replica. See the ALTER TABLE clauses ENABLE TRIGGER and ENABLE RULE for more information. @@ -8781,7 +8781,7 @@ SET XML OPTION { DOCUMENT | CONTENT }; This variable specifies one or more shared libraries that are to be preloaded at connection start. It contains a comma-separated list of library names, where each name - is interpreted as for the command. + is interpreted as for the LOAD command. Whitespace between entries is ignored; surround a library name with double quotes if you need to include whitespace or commas in the name. The parameter value only takes effect at the start of the connection. @@ -8832,7 +8832,7 @@ SET XML OPTION { DOCUMENT | CONTENT }; This variable specifies one or more shared libraries that are to be preloaded at connection start. It contains a comma-separated list of library names, where each name - is interpreted as for the command. + is interpreted as for the LOAD command. Whitespace between entries is ignored; surround a library name with double quotes if you need to include whitespace or commas in the name. The parameter value only takes effect at the start of the connection. @@ -8874,7 +8874,7 @@ SET XML OPTION { DOCUMENT | CONTENT }; This variable specifies one or more shared libraries to be preloaded at server start. It contains a comma-separated list of library names, where each name - is interpreted as for the command. + is interpreted as for the LOAD command. Whitespace between entries is ignored; surround a library name with double quotes if you need to include whitespace or commas in the name. This parameter can only be set at server start. If a specified diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index 9fb2be6674f9..c4897d68c9b9 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -1675,12 +1675,12 @@ REVOKE ALL ON accounts FROM PUBLIC; SELECT - Allows from + Allows SELECT from any column, or specific column(s), of a table, view, materialized view, or other table-like object. - Also allows use of TO. + Also allows use of COPY TO. This privilege is also needed to reference existing column values in - or . + UPDATE or DELETE. For sequences, this privilege also allows use of the currval function. For large objects, this privilege allows the object to be read. @@ -1692,11 +1692,11 @@ REVOKE ALL ON accounts FROM PUBLIC; INSERT - Allows of a new row into a table, view, + Allows INSERT of a new row into a table, view, etc. Can be granted on specific column(s), in which case only those columns may be assigned to in the INSERT command (other columns will therefore receive default values). - Also allows use of FROM. + Also allows use of COPY FROM. @@ -1705,7 +1705,7 @@ REVOKE ALL ON accounts FROM PUBLIC; UPDATE - Allows of any + Allows UPDATE of any column, or specific column(s), of a table, view, etc. (In practice, any nontrivial UPDATE command will require SELECT privilege as well, since it must @@ -1727,7 +1727,7 @@ REVOKE ALL ON accounts FROM PUBLIC; DELETE - Allows of a row from a table, view, etc. + Allows DELETE of a row from a table, view, etc. (In practice, any nontrivial DELETE command will require SELECT privilege as well, since it must reference table columns to determine which rows to delete.) @@ -1739,7 +1739,7 @@ REVOKE ALL ON accounts FROM PUBLIC; TRUNCATE - Allows on a table, view, etc. + Allows TRUNCATE on a table, view, etc. @@ -3370,11 +3370,11 @@ VALUES ('Albany', NULL, NULL, 'NY'); Table inheritance is typically established when the child table is created, using the INHERITS clause of the - + CREATE TABLE statement. Alternatively, a table which is already defined in a compatible way can have a new parent relationship added, using the INHERIT - variant of . + variant of ALTER TABLE. To do this the new child table must already include columns with the same names and types as the columns of the parent. It must also include check constraints with the same names and check expressions as those of the @@ -3406,7 +3406,7 @@ VALUES ('Albany', NULL, NULL, 'NY'); - will + ALTER TABLE will propagate any changes in column data definitions and check constraints down the inheritance hierarchy. Again, dropping columns that are depended on by other tables is only possible when using diff --git a/doc/src/sgml/dml.sgml b/doc/src/sgml/dml.sgml index 3844e34a7dcc..971e6a36b515 100644 --- a/doc/src/sgml/dml.sgml +++ b/doc/src/sgml/dml.sgml @@ -245,7 +245,7 @@ UPDATE mytable SET a = 5, b = 3, c = 1 WHERE a > 0; You use the command to remove rows; the syntax is very similar to the - UPDATE command. For instance, to remove all + command. For instance, to remove all rows from the products table that have a price of 10, use: DELETE FROM products WHERE price = 10; diff --git a/doc/src/sgml/ecpg.sgml b/doc/src/sgml/ecpg.sgml index 7266e229a47d..6e3ca788f6ec 100644 --- a/doc/src/sgml/ecpg.sgml +++ b/doc/src/sgml/ecpg.sgml @@ -479,10 +479,9 @@ EXEC SQL COMMIT; - For more details about declaration of the cursor, - see , and - see for FETCH command - details. + For more details about declaring a cursor, see ; for more details about fetching rows from a + cursor, see . diff --git a/doc/src/sgml/extend.sgml b/doc/src/sgml/extend.sgml index e486006224cc..1c37026bb050 100644 --- a/doc/src/sgml/extend.sgml +++ b/doc/src/sgml/extend.sgml @@ -504,7 +504,7 @@ RETURNS anycompatible AS ... of the extension itself. If the extension includes C code, there will typically also be a shared library file into which the C code has been built. Once you have these files, a simple - command loads the objects into + CREATE EXTENSION command loads the objects into your database. @@ -513,7 +513,7 @@ RETURNS anycompatible AS ... SQL script to load a bunch of loose objects into your database, is that PostgreSQL will then understand that the objects of the extension go together. You can - drop all the objects with a single + drop all the objects with a single DROP EXTENSION command (no need to maintain a separate uninstall script). Even more useful, pg_dump knows that it should not dump the individual member objects of the extension — it will @@ -572,7 +572,7 @@ RETURNS anycompatible AS ... The kinds of SQL objects that can be members of an extension are shown in - the description of . Notably, objects + the description of ALTER EXTENSION. Notably, objects that are database-cluster-wide, such as databases, roles, and tablespaces, cannot be extension members since an extension is only known within one database. (Although an extension script is not prohibited from creating @@ -605,7 +605,7 @@ RETURNS anycompatible AS ... - The command relies on a control + The CREATE EXTENSION command relies on a control file for each extension, which must be named the same as the extension with a suffix of .control, and must be placed in the installation's SHAREDIR/extension directory. There @@ -1373,7 +1373,7 @@ include $(PGXS) Once the files are installed, use the - command to load the objects into + CREATE EXTENSION command to load the objects into any particular database. diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index 87eac7be5897..a560ad69b44b 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -147,13 +147,13 @@ - A publication is created using the + A publication is created using the CREATE PUBLICATION command and may later be altered or dropped using corresponding commands. The individual tables can be added and removed dynamically using - . Both the ADD + ALTER PUBLICATION. Both the ADD TABLE and DROP TABLE operations are transactional; so the table will start or stop replicating at the correct snapshot once the transaction has committed. @@ -207,10 +207,10 @@ - The subscription is added using and + The subscription is added using CREATE SUBSCRIPTION and can be stopped/resumed at any time using the - command and removed using - . + ALTER SUBSCRIPTION command and removed using + DROP SUBSCRIPTION. @@ -418,7 +418,7 @@ tables.) Publications can also specify that changes are to be replicated using the identity and schema of the partitioned root table instead of that of the individual leaf partitions in which the changes actually - originate (see ). + originate (see CREATE PUBLICATION). diff --git a/doc/src/sgml/maintenance.sgml b/doc/src/sgml/maintenance.sgml index de0794adeb90..4d8ad754f857 100644 --- a/doc/src/sgml/maintenance.sgml +++ b/doc/src/sgml/maintenance.sgml @@ -87,7 +87,7 @@ PostgreSQL's - command has to + VACUUM command has to process each table on a regular basis for several reasons: @@ -227,9 +227,9 @@ massive update or delete activity. If you have such a table and you need to reclaim the excess disk space it occupies, you will need to use VACUUM FULL, or alternatively - + CLUSTER or one of the table-rewriting variants of - . + ALTER TABLE. These commands rewrite an entire new copy of the table and build new indexes for it. All these options require exclusive lock. Note that they also temporarily use extra disk space approximately equal to the size @@ -242,7 +242,7 @@ If you have a table whose entire contents are deleted on a periodic basis, consider doing it with - rather + TRUNCATE rather than using DELETE followed by VACUUM. TRUNCATE removes the entire content of the table immediately, without requiring a @@ -269,7 +269,7 @@ The PostgreSQL query planner relies on statistical information about the contents of tables in order to generate good plans for queries. These statistics are gathered by - the command, + the ANALYZE command, which can be invoked by itself or as an optional step in VACUUM. It is important to have reasonably accurate statistics, otherwise poor choices of plans might diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 495018009a2a..171ba7049c7b 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -27,7 +27,7 @@ ps, top, iostat, and vmstat. Also, once one has identified a poorly-performing query, further investigation might be needed using - PostgreSQL's command. + PostgreSQL's EXPLAIN command. discusses EXPLAIN and other methods for understanding the behavior of an individual query. @@ -5278,8 +5278,8 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid, Note that when ANALYZE is run on a partitioned table, - all of its partitions are also recursively analyzed as also mentioned in - . In that case, ANALYZE + all of its partitions are also recursively analyzed. + In that case, ANALYZE progress is reported first for the parent table, whereby its inheritance statistics are collected, followed by that for each partition. diff --git a/doc/src/sgml/mvcc.sgml b/doc/src/sgml/mvcc.sgml index 6920913a260f..1d406176568d 100644 --- a/doc/src/sgml/mvcc.sgml +++ b/doc/src/sgml/mvcc.sgml @@ -950,10 +950,9 @@ ERROR: could not serialize access due to read/write dependencies among transact Acquired by VACUUM (without ), ANALYZE, CREATE INDEX CONCURRENTLY, REINDEX CONCURRENTLY, - CREATE STATISTICS, and certain ALTER - INDEX and ALTER TABLE variants (for full - details see and ). + CREATE STATISTICS, and certain ALTER + INDEX and ALTER TABLE variants (for full + details see the documentation of these commands). @@ -995,7 +994,7 @@ ERROR: could not serialize access due to read/write dependencies among transact Acquired by CREATE TRIGGER and some forms of - ALTER TABLE (see ). + ALTER TABLE. @@ -1723,8 +1722,8 @@ SELECT pg_advisory_lock(q.id) FROM Caveats - Some DDL commands, currently only and the - table-rewriting forms of , are not + Some DDL commands, currently only TRUNCATE and the + table-rewriting forms of ALTER TABLE, are not MVCC-safe. This means that after the truncation or rewrite commits, the table will appear empty to concurrent transactions, if they are using a snapshot taken before the DDL command committed. This will only be an diff --git a/doc/src/sgml/perform.sgml b/doc/src/sgml/perform.sgml index 1cd9f5092db1..117a1f7ff92a 100644 --- a/doc/src/sgml/perform.sgml +++ b/doc/src/sgml/perform.sgml @@ -31,7 +31,7 @@ plan to match the query structure and the properties of the data is absolutely critical for good performance, so the system includes a complex planner that tries to choose good plans. - You can use the command + You can use the EXPLAIN command to see what query plan the planner creates for any query. Plan-reading is an art that requires some experience to master, but this section attempts to cover the basics. @@ -1144,7 +1144,7 @@ WHERE tablename = 'road'; Statistics objects are created using the - command. + CREATE STATISTICS command. Creation of such an object merely creates a catalog entry expressing interest in the statistics. Actual data collection is performed by ANALYZE (either a manual command, or background @@ -1612,7 +1612,7 @@ SELECT * FROM x, y, a, b, c WHERE something AND somethingelse; Use <command>COPY</command> - Use to load + Use COPY to load all the rows in one command, instead of using a series of INSERT commands. The COPY command is optimized for loading large numbers of rows; it is less @@ -1623,8 +1623,8 @@ SELECT * FROM x, y, a, b, c WHERE something AND somethingelse; - If you cannot use COPY, it might help to use to create a + If you cannot use COPY, it might help to use PREPARE to create a prepared INSERT statement, and then use EXECUTE as many times as required. This avoids some of the overhead of repeatedly parsing and planning @@ -1763,7 +1763,7 @@ SELECT * FROM x, y, a, b, c WHERE something AND somethingelse; Whenever you have significantly altered the distribution of data - within a table, running is strongly recommended. This + within a table, running ANALYZE is strongly recommended. This includes bulk loading large amounts of data into the table. Running ANALYZE (or VACUUM ANALYZE) ensures that the planner has up-to-date statistics about the diff --git a/doc/src/sgml/plpgsql.sgml b/doc/src/sgml/plpgsql.sgml index 815912666dd0..c2bb3e326851 100644 --- a/doc/src/sgml/plpgsql.sgml +++ b/doc/src/sgml/plpgsql.sgml @@ -1299,7 +1299,7 @@ EXECUTE format('SELECT count(*) FROM %I ' The PL/pgSQL EXECUTE statement is not related to the - SQL + EXECUTE SQL statement supported by the PostgreSQL server. The server's EXECUTE statement cannot be used directly within diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml index 4efaf35d3c4e..e6fd2143c105 100644 --- a/doc/src/sgml/postgres-fdw.sgml +++ b/doc/src/sgml/postgres-fdw.sgml @@ -456,14 +456,14 @@ OPTIONS (ADD password_required 'false'); Note that constraints other than NOT NULL will never be imported from the remote tables. Although PostgreSQL - does support CHECK constraints on foreign tables, there is no + does support check constraints on foreign tables, there is no provision for importing them automatically, because of the risk that a constraint expression could evaluate differently on the local and remote - servers. Any such inconsistency in the behavior of a CHECK + servers. Any such inconsistency in the behavior of a check constraint could lead to hard-to-detect errors in query optimization. - So if you wish to import CHECK constraints, you must do so + So if you wish to import check constraints, you must do so manually, and you should verify the semantics of each one carefully. - For more detail about the treatment of CHECK constraints on + For more detail about the treatment of check constraints on foreign tables, see . @@ -705,7 +705,7 @@ CREATE FOREIGN TABLE foreign_table ( Column names must match as well, unless you attach column_name options to the individual columns to show how they are named in the remote table. - In many cases, use of is + In many cases, use of IMPORT FOREIGN SCHEMA is preferable to constructing foreign table definitions manually. diff --git a/doc/src/sgml/queries.sgml b/doc/src/sgml/queries.sgml index 0a643ef59705..875a4d84de02 100644 --- a/doc/src/sgml/queries.sgml +++ b/doc/src/sgml/queries.sgml @@ -24,7 +24,7 @@ The process of retrieving or the command to retrieve data from a database is called a query. In SQL the - command is + SELECT command is used to specify queries. The general syntax of the SELECT command is diff --git a/doc/src/sgml/ref/abort.sgml b/doc/src/sgml/ref/abort.sgml index 037291336516..16b5602487d7 100644 --- a/doc/src/sgml/ref/abort.sgml +++ b/doc/src/sgml/ref/abort.sgml @@ -33,7 +33,7 @@ ABORT [ WORK | TRANSACTION ] [ AND [ NO ] CHAIN ] all the updates made by the transaction to be discarded. This command is identical in behavior to the standard SQL command - , + ROLLBACK, and is present only for historical reasons. @@ -57,8 +57,8 @@ ABORT [ WORK | TRANSACTION ] [ AND [ NO ] CHAIN ] If AND CHAIN is specified, a new transaction is - immediately started with the same transaction characteristics (see ) as the just finished one. Otherwise, + immediately started with the same transaction characteristics (see SET TRANSACTION) as the just finished one. Otherwise, no new transaction is started. @@ -70,7 +70,7 @@ ABORT [ WORK | TRANSACTION ] [ AND [ NO ] CHAIN ] Notes - Use to + Use COMMIT to successfully terminate a transaction. diff --git a/doc/src/sgml/ref/alter_aggregate.sgml b/doc/src/sgml/ref/alter_aggregate.sgml index 95934a100f87..aee10a5ca2e0 100644 --- a/doc/src/sgml/ref/alter_aggregate.sgml +++ b/doc/src/sgml/ref/alter_aggregate.sgml @@ -142,7 +142,7 @@ ALTER AGGREGATE name ( aggregate_signatu The recommended syntax for referencing an ordered-set aggregate is to write ORDER BY between the direct and aggregated argument specifications, in the same style as in - . However, it will also work to + CREATE AGGREGATE. However, it will also work to omit ORDER BY and just run the direct and aggregated argument specifications into a single list. In this abbreviated form, if VARIADIC "any" was used in both the direct and diff --git a/doc/src/sgml/ref/alter_domain.sgml b/doc/src/sgml/ref/alter_domain.sgml index afa42b4926f0..2db53725139c 100644 --- a/doc/src/sgml/ref/alter_domain.sgml +++ b/doc/src/sgml/ref/alter_domain.sgml @@ -80,7 +80,7 @@ ALTER DOMAIN name This form adds a new constraint to a domain using the same syntax as - . + CREATE DOMAIN. When a new constraint is added to a domain, all columns using that domain will be checked against the newly added constraint. These checks can be suppressed by adding the new constraint using the diff --git a/doc/src/sgml/ref/alter_foreign_table.sgml b/doc/src/sgml/ref/alter_foreign_table.sgml index 04d53628ec2f..7ca03f3ac9f1 100644 --- a/doc/src/sgml/ref/alter_foreign_table.sgml +++ b/doc/src/sgml/ref/alter_foreign_table.sgml @@ -71,7 +71,7 @@ ALTER FOREIGN TABLE [ IF EXISTS ] name This form adds a new column to the foreign table, using the same syntax as - . + CREATE FOREIGN TABLE. Unlike the case when adding a column to a regular table, nothing happens to the underlying storage: this action simply declares that some new column is now accessible through the foreign table. @@ -133,8 +133,8 @@ ALTER FOREIGN TABLE [ IF EXISTS ] name This form sets the per-column statistics-gathering target for subsequent - operations. - See the similar form of + ANALYZE operations. + See the similar form of ALTER TABLE for more details. @@ -146,7 +146,7 @@ ALTER FOREIGN TABLE [ IF EXISTS ] name This form sets or resets per-attribute options. - See the similar form of + See the similar form of ALTER TABLE for more details. @@ -159,7 +159,7 @@ ALTER FOREIGN TABLE [ IF EXISTS ] name This form sets the storage mode for a column. - See the similar form of + See the similar form of ALTER TABLE for more details. Note that the storage mode has no effect unless the table's foreign-data wrapper chooses to pay attention to it. @@ -172,7 +172,7 @@ ALTER FOREIGN TABLE [ IF EXISTS ] name This form adds a new constraint to a foreign table, using the same - syntax as . + syntax as CREATE FOREIGN TABLE. Currently only CHECK constraints are supported. @@ -181,7 +181,7 @@ ALTER FOREIGN TABLE [ IF EXISTS ] name.) + in CREATE FOREIGN TABLE.) If the constraint is marked NOT VALID, then it isn't assumed to hold, but is only recorded for possible future use. @@ -216,7 +216,7 @@ ALTER FOREIGN TABLE [ IF EXISTS ] name These forms configure the firing of trigger(s) belonging to the foreign - table. See the similar form of for more + table. See the similar form of ALTER TABLE for more details. @@ -239,7 +239,7 @@ ALTER FOREIGN TABLE [ IF EXISTS ] name This form adds the target foreign table as a new child of the specified parent table. - See the similar form of + See the similar form of ALTER TABLE for more details. @@ -503,7 +503,7 @@ ALTER FOREIGN TABLE [ IF EXISTS ] name - Refer to for a further description of valid + Refer to CREATE FOREIGN TABLE for a further description of valid parameters. diff --git a/doc/src/sgml/ref/alter_group.sgml b/doc/src/sgml/ref/alter_group.sgml index f6e516310995..fa4a8df91249 100644 --- a/doc/src/sgml/ref/alter_group.sgml +++ b/doc/src/sgml/ref/alter_group.sgml @@ -51,14 +51,14 @@ ALTER GROUP group_name RENAME TO group for this purpose.) These variants are effectively equivalent to granting or revoking membership in the role named as the group; so the preferred way to do this is to use - or - . + GRANT or + REVOKE. The third variant changes the name of the group. This is exactly equivalent to renaming the role with - . + ALTER ROLE. diff --git a/doc/src/sgml/ref/alter_index.sgml b/doc/src/sgml/ref/alter_index.sgml index a5e3b06ee493..793119d2fc1a 100644 --- a/doc/src/sgml/ref/alter_index.sgml +++ b/doc/src/sgml/ref/alter_index.sgml @@ -81,7 +81,7 @@ ALTER INDEX ALL IN TABLESPACE name this command, use ALTER DATABASE or explicit ALTER INDEX invocations instead if desired. See also - . + CREATE TABLESPACE. @@ -118,11 +118,11 @@ ALTER INDEX ALL IN TABLESPACE name This form changes one or more index-method-specific storage parameters for the index. See - + CREATE INDEX for details on the available parameters. Note that the index contents will not be modified immediately by this command; depending on the parameter you might need to rebuild the index with - + REINDEX to get the desired effects. @@ -144,7 +144,7 @@ ALTER INDEX ALL IN TABLESPACE name This form sets the per-column statistics-gathering target for - subsequent operations, though can + subsequent ANALYZE operations, though can be used only on index columns that are defined as an expression. Since expressions lack a unique name, we refer to them using the ordinal number of the index column. @@ -252,7 +252,7 @@ ALTER INDEX ALL IN TABLESPACE name These operations are also possible using - . + ALTER TABLE. ALTER INDEX is in fact just an alias for the forms of ALTER TABLE that apply to indexes. diff --git a/doc/src/sgml/ref/alter_materialized_view.sgml b/doc/src/sgml/ref/alter_materialized_view.sgml index 78ee99bb9cac..bf379db77e38 100644 --- a/doc/src/sgml/ref/alter_materialized_view.sgml +++ b/doc/src/sgml/ref/alter_materialized_view.sgml @@ -72,7 +72,8 @@ ALTER MATERIALIZED VIEW ALL IN TABLESPACE nameALTER MATERIALIZED VIEW are a subset of those available for ALTER TABLE, and have the same meaning when used for - materialized views. See the descriptions for + materialized views. See the descriptions for + ALTER TABLE for details. diff --git a/doc/src/sgml/ref/alter_role.sgml b/doc/src/sgml/ref/alter_role.sgml index d5f166c129ba..aef30521bcc5 100644 --- a/doc/src/sgml/ref/alter_role.sgml +++ b/doc/src/sgml/ref/alter_role.sgml @@ -63,11 +63,11 @@ ALTER ROLE { role_specification | A The first variant of this command listed in the synopsis can change many of the role attributes that can be specified in - . + CREATE ROLE. (All the possible attributes are covered, except that there are no options for adding or removing memberships; use - and - for that.) + GRANT and + REVOKE for that.) Attributes not mentioned in the command retain their previous settings. Database superusers can change any of these settings for any role. Roles having CREATEROLE privilege can change any of these @@ -103,8 +103,8 @@ ALTER ROLE { role_specification | A default, overriding whatever setting is present in postgresql.conf or has been received from the postgres command line. This only happens at login time; executing - or - does not cause new + SET ROLE or + SET SESSION AUTHORIZATION does not cause new configuration values to be set. Settings set for all databases are overridden by database-specific settings attached to a role. Settings for specific databases or specific roles override @@ -176,7 +176,7 @@ ALTER ROLE { role_specification | A These clauses alter attributes originally set by - . For more information, see the + CREATE ROLE. For more information, see the CREATE ROLE reference page. @@ -220,8 +220,8 @@ ALTER ROLE { role_specification | A Role-specific variable settings take effect only at login; - and - + SET ROLE and + SET SESSION AUTHORIZATION do not process role-specific variable settings. @@ -239,14 +239,14 @@ ALTER ROLE { role_specification | A Notes - Use - to add new roles, and to remove a role. + Use CREATE ROLE + to add new roles, and DROP ROLE to remove a role. ALTER ROLE cannot change a role's memberships. - Use and - + Use GRANT and + REVOKE to do that. diff --git a/doc/src/sgml/ref/alter_statistics.sgml b/doc/src/sgml/ref/alter_statistics.sgml index 5f60ca8eac2a..ce6cdf2bb1ec 100644 --- a/doc/src/sgml/ref/alter_statistics.sgml +++ b/doc/src/sgml/ref/alter_statistics.sgml @@ -99,7 +99,7 @@ ALTER STATISTICS name SET STATISTIC The statistic-gathering target for this statistics object for subsequent - operations. + ANALYZE operations. The target can be set in the range 0 to 10000; alternatively, set it to -1 to revert to using the maximum of the statistics target of the referenced columns, if set, or the system default statistics diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index f034e75ec099..c25ef5abd6ad 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -152,7 +152,7 @@ WITH ( MODULUS numeric_literal, REM This form adds a new column to the table, using the same syntax as - . If IF NOT EXISTS + CREATE TABLE. If IF NOT EXISTS is specified and a column already exists with this name, no error is thrown. @@ -268,7 +268,7 @@ WITH ( MODULUS numeric_literal, REM These forms change whether a column is an identity column or change the generation attribute of an existing identity column. - See for details. + See CREATE TABLE for details. Like SET DEFAULT, these forms only affect the behavior of subsequent INSERT and UPDATE commands; they do not cause rows @@ -290,7 +290,7 @@ WITH ( MODULUS numeric_literal, REM These forms alter the sequence that underlies an existing identity column. sequence_option is an option - supported by such + supported by ALTER SEQUENCE such as INCREMENT BY. @@ -302,7 +302,7 @@ WITH ( MODULUS numeric_literal, REM This form sets the per-column statistics-gathering target for subsequent - operations. + ANALYZE operations. The target can be set in the range 0 to 10000; alternatively, set it to -1 to revert to using the system default statistics target (). @@ -326,7 +326,7 @@ WITH ( MODULUS numeric_literal, REM defined per-attribute options are n_distinct and n_distinct_inherited, which override the number-of-distinct-values estimates made by subsequent - + ANALYZE operations. n_distinct affects the statistics for the table itself, while n_distinct_inherited affects the statistics gathered for the table plus its inheritance children. When set to a @@ -388,7 +388,7 @@ WITH ( MODULUS numeric_literal, REM This form adds a new constraint to a table using the same constraint - syntax as , plus the option NOT + syntax as CREATE TABLE, plus the option NOT VALID, which is currently only allowed for foreign key and CHECK constraints. @@ -422,7 +422,7 @@ WITH ( MODULUS numeric_literal, REM Additional restrictions apply when unique or primary key constraints - are added to partitioned tables; see . + are added to partitioned tables; see CREATE TABLE. Also, foreign key constraints on partitioned tables may not be declared NOT VALID at present. @@ -598,7 +598,7 @@ WITH ( MODULUS numeric_literal, REM even if row level security is disabled. In this case, the policies will not be applied and the policies will be ignored. See also - . + CREATE POLICY. @@ -613,7 +613,7 @@ WITH ( MODULUS numeric_literal, REM disabled (the default) then row level security will not be applied when the user is the table owner. See also - . + CREATE POLICY. @@ -623,7 +623,7 @@ WITH ( MODULUS numeric_literal, REM This form selects the default index for future - + CLUSTER operations. It does not actually re-cluster the table. @@ -637,7 +637,7 @@ WITH ( MODULUS numeric_literal, REM This form removes the most recently used - + CLUSTER index specification from the table. This affects future cluster operations that don't specify an index. @@ -685,7 +685,7 @@ WITH ( MODULUS numeric_literal, REM information_schema relations are not considered part of the system catalogs and will be moved. See also - . + CREATE TABLESPACE. @@ -707,12 +707,12 @@ WITH ( MODULUS numeric_literal, REM This form changes one or more storage parameters for the table. See in the - documentation + CREATE TABLE documentation for details on the available parameters. Note that the table contents will not be modified immediately by this command; depending on the parameter you might need to rewrite the table to get the desired effects. - That can be done with VACUUM - FULL, or one of the forms + That can be done with VACUUM + FULL, CLUSTER or one of the forms of ALTER TABLE that forces a table rewrite. For planner related parameters, changes will take effect from the next time the table is locked so currently executing queries will not be @@ -878,7 +878,7 @@ WITH ( MODULUS numeric_literal, REM A partition using FOR VALUES uses same syntax for partition_bound_spec as - . The partition bound specification + CREATE TABLE. The partition bound specification must correspond to the partitioning strategy and partition key of the target table. The table to be attached must have all the same columns as the target table and no more; moreover, the column types must also diff --git a/doc/src/sgml/ref/alter_trigger.sgml b/doc/src/sgml/ref/alter_trigger.sgml index 6d4784c82f19..43a7da4f0bcf 100644 --- a/doc/src/sgml/ref/alter_trigger.sgml +++ b/doc/src/sgml/ref/alter_trigger.sgml @@ -93,7 +93,7 @@ ALTER TRIGGER name ON The ability to temporarily enable or disable a trigger is provided by - , not by + ALTER TABLE, not by ALTER TRIGGER, because ALTER TRIGGER has no convenient way to express the option of enabling or disabling all of a table's triggers at once. diff --git a/doc/src/sgml/ref/alter_type.sgml b/doc/src/sgml/ref/alter_type.sgml index a4f09c660be3..64bf266373d4 100644 --- a/doc/src/sgml/ref/alter_type.sgml +++ b/doc/src/sgml/ref/alter_type.sgml @@ -90,7 +90,7 @@ ALTER TYPE name SET ( This form adds a new attribute to a composite type, using the same syntax as - . + CREATE TYPE. diff --git a/doc/src/sgml/ref/alter_user.sgml b/doc/src/sgml/ref/alter_user.sgml index 51527cefb4c4..0ee89f54c5ce 100644 --- a/doc/src/sgml/ref/alter_user.sgml +++ b/doc/src/sgml/ref/alter_user.sgml @@ -57,7 +57,7 @@ ALTER USER { role_specification | A ALTER USER is now an alias for - . + ALTER ROLE. diff --git a/doc/src/sgml/ref/analyze.sgml b/doc/src/sgml/ref/analyze.sgml index 5ac3ba832193..7d816c87c603 100644 --- a/doc/src/sgml/ref/analyze.sgml +++ b/doc/src/sgml/ref/analyze.sgml @@ -174,7 +174,7 @@ ANALYZE [ VERBOSE ] [ table_and_columns + strategy for read-mostly databases is to run VACUUM and ANALYZE once a day during a low-usage time of day. (This will not be sufficient if there is heavy update activity.) @@ -205,7 +205,7 @@ ANALYZE [ VERBOSE ] [ table_and_columnsANALYZE is run, even if the actual table contents did not change. This might result in small changes in the planner's estimated costs shown by - . + EXPLAIN. In rare situations, this non-determinism will cause the planner's choices of query plans to change after ANALYZE is run. To avoid this, raise the amount of statistics collected by @@ -216,8 +216,8 @@ ANALYZE [ VERBOSE ] [ table_and_columns configuration variable, or on a column-by-column basis by setting the per-column statistics - target with ALTER TABLE ... ALTER COLUMN ... SET - STATISTICS (see ). + target with ALTER TABLE ... ALTER COLUMN ... SET + STATISTICS. The target value sets the maximum number of entries in the most-common-value list and the maximum number of bins in the histogram. The default target value @@ -246,8 +246,7 @@ ANALYZE [ VERBOSE ] [ table_and_columnsALTER TABLE ... ALTER COLUMN ... SET (n_distinct = ...) - (see ). + ALTER TABLE ... ALTER COLUMN ... SET (n_distinct = ...). diff --git a/doc/src/sgml/ref/begin.sgml b/doc/src/sgml/ref/begin.sgml index c23bbfb4e711..016b02148741 100644 --- a/doc/src/sgml/ref/begin.sgml +++ b/doc/src/sgml/ref/begin.sgml @@ -37,9 +37,9 @@ BEGIN [ WORK | TRANSACTION ] [ transaction_mode BEGIN initiates a transaction block, that is, all statements after a BEGIN command will be - executed in a single transaction until an explicit or is given. + executed in a single transaction until an explicit COMMIT or ROLLBACK is given. By default (without BEGIN), PostgreSQL executes transactions in autocommit mode, that is, each @@ -60,7 +60,7 @@ BEGIN [ WORK | TRANSACTION ] [ transaction_mode If the isolation level, read/write mode, or deferrable mode is specified, the new transaction has those characteristics, as if - + SET TRANSACTION was executed. @@ -90,13 +90,13 @@ BEGIN [ WORK | TRANSACTION ] [ transaction_modeNotes - has the same functionality + START TRANSACTION has the same functionality as BEGIN. - Use or - + Use COMMIT or + ROLLBACK to terminate a transaction block. @@ -131,7 +131,7 @@ BEGIN; BEGIN is a PostgreSQL language extension. It is equivalent to the SQL-standard command - , whose reference page + START TRANSACTION, whose reference page contains additional compatibility information. diff --git a/doc/src/sgml/ref/close.sgml b/doc/src/sgml/ref/close.sgml index e464df1965d9..32d20edd6aa4 100644 --- a/doc/src/sgml/ref/close.sgml +++ b/doc/src/sgml/ref/close.sgml @@ -84,7 +84,7 @@ CLOSE { name | ALL } PostgreSQL does not have an explicit OPEN cursor statement; a cursor is considered open when it is declared. Use the - + DECLARE statement to declare a cursor. diff --git a/doc/src/sgml/ref/cluster.sgml b/doc/src/sgml/ref/cluster.sgml index 4da60d8d56ab..b9450e7366ae 100644 --- a/doc/src/sgml/ref/cluster.sgml +++ b/doc/src/sgml/ref/cluster.sgml @@ -57,7 +57,7 @@ CLUSTER [VERBOSE] CLUSTER table_name reclusters the table using the same index as before. You can also use the CLUSTER or SET WITHOUT CLUSTER - forms of to set the index to be used for + forms of ALTER TABLE to set the index to be used for future cluster operations, or to clear any previous setting. @@ -159,7 +159,7 @@ CLUSTER [VERBOSE] Because the planner records statistics about the ordering of - tables, it is advisable to run + tables, it is advisable to run ANALYZE on the newly clustered table. Otherwise, the planner might make poor choices of query plans. diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml index 18189abc6c90..369342b74d5b 100644 --- a/doc/src/sgml/ref/copy.sgml +++ b/doc/src/sgml/ref/copy.sgml @@ -112,9 +112,11 @@ COPY { table_name [ ( query - A , , - , or - command whose results are to be + A SELECT, + VALUES, + INSERT, + UPDATE, or + DELETE command whose results are to be copied. Note that parentheses are required around the query. diff --git a/doc/src/sgml/ref/create_aggregate.sgml b/doc/src/sgml/ref/create_aggregate.sgml index a315fff8bd3f..222e0aa5c9d0 100644 --- a/doc/src/sgml/ref/create_aggregate.sgml +++ b/doc/src/sgml/ref/create_aggregate.sgml @@ -629,7 +629,7 @@ SELECT col FROM tab ORDER BY col USING sortop LIMIT 1; The meanings of PARALLEL SAFE, PARALLEL RESTRICTED, and PARALLEL UNSAFE are the same as - in . An aggregate will not be + in CREATE FUNCTION. An aggregate will not be considered for parallelization if it is marked PARALLEL UNSAFE (which is the default!) or PARALLEL RESTRICTED. Note that the parallel-safety markings of the aggregate's support diff --git a/doc/src/sgml/ref/create_cast.sgml b/doc/src/sgml/ref/create_cast.sgml index 2b4d4d557328..bad75bc1dce5 100644 --- a/doc/src/sgml/ref/create_cast.sgml +++ b/doc/src/sgml/ref/create_cast.sgml @@ -304,7 +304,7 @@ SELECT CAST ( 2 AS numeric ) + 4.0; Notes - Use to remove user-defined casts. + Use DROP CAST to remove user-defined casts. diff --git a/doc/src/sgml/ref/create_database.sgml b/doc/src/sgml/ref/create_database.sgml index 420576c5e83e..41cb4068ec2f 100644 --- a/doc/src/sgml/ref/create_database.sgml +++ b/doc/src/sgml/ref/create_database.sgml @@ -226,7 +226,7 @@ CREATE DATABASE name - Use to remove a database. + Use DROP DATABASE to remove a database. @@ -235,9 +235,9 @@ CREATE DATABASE name - Database-level configuration parameters (set via ) and database-level permissions (set via - ) are not copied from the template database. + Database-level configuration parameters (set via ALTER DATABASE) and database-level permissions (set via + GRANT) are not copied from the template database. diff --git a/doc/src/sgml/ref/create_foreign_table.sgml b/doc/src/sgml/ref/create_foreign_table.sgml index 3ee0f2d635d3..f9477efe58d7 100644 --- a/doc/src/sgml/ref/create_foreign_table.sgml +++ b/doc/src/sgml/ref/create_foreign_table.sgml @@ -159,7 +159,7 @@ CHECK ( expression ) [ NO INHERIT ] tables from which the new foreign table automatically inherits all columns. Parent tables can be plain tables or foreign tables. See the similar form of - for more details. + CREATE TABLE for more details. @@ -171,7 +171,7 @@ CHECK ( expression ) [ NO INHERIT ] This form can be used to create the foreign table as partition of the given parent table with specified partition bound values. See the similar form of - for more details. + CREATE TABLE for more details. Note that it is currently not allowed to create the foreign table as a partition of the parent table if there are UNIQUE indexes on the parent table. (See also diff --git a/doc/src/sgml/ref/create_function.sgml b/doc/src/sgml/ref/create_function.sgml index 97285b757841..3c1eaea651cb 100644 --- a/doc/src/sgml/ref/create_function.sgml +++ b/doc/src/sgml/ref/create_function.sgml @@ -557,7 +557,7 @@ CREATE [ OR REPLACE ] FUNCTION the SQL function. The string obj_file is the name of the shared library file containing the compiled C function, and is interpreted - as for the command. The string + as for the LOAD command. The string link_symbol is the function's link symbol, that is, the name of the function in the C language source code. If the link symbol is omitted, it is assumed to diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 7fa79f4cbfb0..847b8efcf4d8 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -779,8 +779,8 @@ Indexes: - Setting a value for parallel_workers via directly controls how many parallel + Setting a value for parallel_workers via ALTER TABLE directly controls how many parallel worker processes will be requested by a CREATE INDEX against the table. This bypasses the cost model completely, and prevents maintenance_work_mem @@ -808,7 +808,7 @@ Indexes: - Use + Use DROP INDEX to remove an index. diff --git a/doc/src/sgml/ref/create_language.sgml b/doc/src/sgml/ref/create_language.sgml index 10d1533d6d8c..102efe5a6c7f 100644 --- a/doc/src/sgml/ref/create_language.sgml +++ b/doc/src/sgml/ref/create_language.sgml @@ -137,7 +137,7 @@ CREATE [ OR REPLACE ] [ TRUSTED ] [ PROCEDURAL ] LANGUAGE inline_handler is the name of a previously registered function that will be called to execute an anonymous code block - ( command) + (DO command) in this language. If no inline_handler function is specified, the language does not support anonymous code @@ -183,7 +183,7 @@ CREATE [ OR REPLACE ] [ TRUSTED ] [ PROCEDURAL ] LANGUAGE to drop procedural languages. + Use DROP LANGUAGE to drop procedural languages. diff --git a/doc/src/sgml/ref/create_materialized_view.sgml b/doc/src/sgml/ref/create_materialized_view.sgml index de9f17655c63..5ba851b687a4 100644 --- a/doc/src/sgml/ref/create_materialized_view.sgml +++ b/doc/src/sgml/ref/create_materialized_view.sgml @@ -132,8 +132,8 @@ CREATE MATERIALIZED VIEW [ IF NOT EXISTS ] table_name query - A , TABLE, - or command. This query will run within a + A SELECT, TABLE, + or VALUES command. This query will run within a security-restricted operation; in particular, calls to functions that themselves create temporary tables will fail. diff --git a/doc/src/sgml/ref/create_operator.sgml b/doc/src/sgml/ref/create_operator.sgml index 9462bc1e8caa..e27512ff3919 100644 --- a/doc/src/sgml/ref/create_operator.sgml +++ b/doc/src/sgml/ref/create_operator.sgml @@ -251,8 +251,8 @@ COMMUTATOR = OPERATOR(myschema.===) , - Use to delete user-defined operators - from a database. Use to modify operators in a + Use DROP OPERATOR to delete user-defined operators + from a database. Use ALTER OPERATOR to modify operators in a database. diff --git a/doc/src/sgml/ref/create_procedure.sgml b/doc/src/sgml/ref/create_procedure.sgml index d225695626cb..36c307cadc79 100644 --- a/doc/src/sgml/ref/create_procedure.sgml +++ b/doc/src/sgml/ref/create_procedure.sgml @@ -285,7 +285,7 @@ CREATE [ OR REPLACE ] PROCEDURE the SQL procedure. The string obj_file is the name of the shared library file containing the compiled C procedure, and is interpreted - as for the command. The string + as for the LOAD command. The string link_symbol is the procedure's link symbol, that is, the name of the procedure in the C language source code. If the link symbol is omitted, it is assumed diff --git a/doc/src/sgml/ref/create_role.sgml b/doc/src/sgml/ref/create_role.sgml index 6e4148a17c48..d23133945db2 100644 --- a/doc/src/sgml/ref/create_role.sgml +++ b/doc/src/sgml/ref/create_role.sgml @@ -162,7 +162,7 @@ in sync when changing the above synopsis! If not specified, NOLOGIN is the default, except when CREATE ROLE is invoked through its alternative spelling - . + CREATE USER. @@ -335,8 +335,8 @@ in sync when changing the above synopsis! Notes - Use to - change the attributes of a role, and + Use ALTER ROLE to + change the attributes of a role, and DROP ROLE to remove a role. All the attributes specified by CREATE ROLE can be modified by later ALTER ROLE commands. @@ -345,8 +345,8 @@ in sync when changing the above synopsis! The preferred way to add and remove members of roles that are being used as groups is to use - and - . + GRANT and + REVOKE. @@ -364,7 +364,7 @@ in sync when changing the above synopsis! a member of a role with CREATEDB privilege does not immediately grant the ability to create databases, even if INHERIT is set; it would be necessary to become that role via - before + SET ROLE before creating a database. diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index 087cad184c0c..28f844071b06 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -1152,7 +1152,7 @@ WITH ( MODULUS numeric_literal, REM constraint that is not deferrable will be checked immediately after every command. Checking of constraints that are deferrable can be postponed until the end of the transaction - (using the command). + (using the SET CONSTRAINTS command). NOT DEFERRABLE is the default. Currently, only UNIQUE, PRIMARY KEY, EXCLUDE, and @@ -1176,7 +1176,7 @@ WITH ( MODULUS numeric_literal, REM statement. This is the default. If the constraint is INITIALLY DEFERRED, it is checked only at the end of the transaction. The constraint check time can be - altered with the command. + altered with the SET CONSTRAINTS command. @@ -1244,8 +1244,8 @@ WITH ( MODULUS numeric_literal, REM All rows in the temporary table will be deleted at the end - of each transaction block. Essentially, an automatic is done + of each transaction block. Essentially, an automatic TRUNCATE is done at each commit. When used on a partitioned table, this is not cascaded to its partitions. @@ -1430,7 +1430,7 @@ WITH ( MODULUS numeric_literal, REM Disabling index cleanup can speed up VACUUM very significantly, but may also lead to severely bloated indexes if table modifications are frequent. The INDEX_CLEANUP - parameter of , if specified, overrides + parameter of VACUUM, if specified, overrides the value of this option. @@ -1451,7 +1451,7 @@ WITH ( MODULUS numeric_literal, REM the truncated pages is returned to the operating system. Note that the truncation requires ACCESS EXCLUSIVE lock on the table. The TRUNCATE parameter - of , if specified, overrides the value + of VACUUM, if specified, overrides the value of this option. diff --git a/doc/src/sgml/ref/create_table_as.sgml b/doc/src/sgml/ref/create_table_as.sgml index a4640929cfba..bcbd73b2272c 100644 --- a/doc/src/sgml/ref/create_table_as.sgml +++ b/doc/src/sgml/ref/create_table_as.sgml @@ -185,8 +185,8 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI All rows in the temporary table will be deleted at the end - of each transaction block. Essentially, an automatic is done + of each transaction block. Essentially, an automatic TRUNCATE is done at each commit. @@ -222,9 +222,9 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI query - A , TABLE, or - command, or an command that runs a + A SELECT, TABLE, or VALUES + command, or an EXECUTE command that runs a prepared SELECT, TABLE, or VALUES query. diff --git a/doc/src/sgml/ref/create_transform.sgml b/doc/src/sgml/ref/create_transform.sgml index 5b46c23196db..3f81dc6bba2c 100644 --- a/doc/src/sgml/ref/create_transform.sgml +++ b/doc/src/sgml/ref/create_transform.sgml @@ -147,7 +147,7 @@ CREATE [ OR REPLACE ] TRANSFORM FOR type_name LANGUAG Notes - Use to remove transforms. + Use DROP TRANSFORM to remove transforms. diff --git a/doc/src/sgml/ref/create_trigger.sgml b/doc/src/sgml/ref/create_trigger.sgml index 289dd1d9da8e..60346e1e83b0 100644 --- a/doc/src/sgml/ref/create_trigger.sgml +++ b/doc/src/sgml/ref/create_trigger.sgml @@ -170,7 +170,7 @@ CREATE [ CONSTRAINT ] TRIGGER name When the CONSTRAINT option is specified, this command creates a constraint trigger. This is the same as a regular trigger except that the timing of the trigger firing can be adjusted using - . + SET CONSTRAINTS. Constraint triggers must be AFTER ROW triggers on plain tables (not foreign tables). They can be fired either at the end of the statement causing the triggering @@ -442,7 +442,7 @@ UPDATE OF column_name1 [, column_name2 - Use to remove a trigger. + Use DROP TRIGGER to remove a trigger. diff --git a/doc/src/sgml/ref/create_type.sgml b/doc/src/sgml/ref/create_type.sgml index 111f8e65d29d..970b517db9f2 100644 --- a/doc/src/sgml/ref/create_type.sgml +++ b/doc/src/sgml/ref/create_type.sgml @@ -121,8 +121,8 @@ CREATE TYPE name must be less than NAMEDATALEN bytes long (64 bytes in a standard PostgreSQL build). (It is possible to create an enumerated type with zero labels, but such a type cannot be used - to hold values before at least one label is added using .) + to hold values before at least one label is added using ALTER TYPE.) diff --git a/doc/src/sgml/ref/create_user.sgml b/doc/src/sgml/ref/create_user.sgml index 198e06e7230e..48d2089238c7 100644 --- a/doc/src/sgml/ref/create_user.sgml +++ b/doc/src/sgml/ref/create_user.sgml @@ -49,7 +49,7 @@ CREATE USER name [ [ WITH ] CREATE USER is now an alias for - . + CREATE ROLE. The only difference is that when the command is spelled CREATE USER, LOGIN is assumed by default, whereas NOLOGIN is assumed when diff --git a/doc/src/sgml/ref/create_view.sgml b/doc/src/sgml/ref/create_view.sgml index eb5591b63c73..4b5b1cf79531 100644 --- a/doc/src/sgml/ref/create_view.sgml +++ b/doc/src/sgml/ref/create_view.sgml @@ -137,8 +137,8 @@ CREATE VIEW [ schema . ] view_namelocal or cascaded, and is equivalent to specifying WITH [ CASCADED | LOCAL ] CHECK OPTION (see below). - This option can be changed on existing views using . + This option can be changed on existing views using ALTER VIEW. @@ -160,8 +160,8 @@ CREATE VIEW [ schema . ] view_namequery - A or - command + A SELECT or + VALUES command which will provide the columns and rows of the view. @@ -245,7 +245,7 @@ CREATE VIEW [ schema . ] view_nameNotes - Use the + Use the DROP VIEW statement to drop views. diff --git a/doc/src/sgml/ref/createdb.sgml b/doc/src/sgml/ref/createdb.sgml index d3c92943f071..95cc82dc88bd 100644 --- a/doc/src/sgml/ref/createdb.sgml +++ b/doc/src/sgml/ref/createdb.sgml @@ -46,7 +46,7 @@ PostgreSQL documentation createdb is a wrapper around the - SQL command . + SQL command CREATE DATABASE. There is no effective difference between creating databases via this utility and via other methods for accessing the server. @@ -197,7 +197,7 @@ PostgreSQL documentation The options , , , , and correspond to options of the underlying - SQL command ; see there for more information + SQL command CREATE DATABASE; see there for more information about them. diff --git a/doc/src/sgml/ref/createuser.sgml b/doc/src/sgml/ref/createuser.sgml index 9d24df8b7a88..4d60dc2cda12 100644 --- a/doc/src/sgml/ref/createuser.sgml +++ b/doc/src/sgml/ref/createuser.sgml @@ -49,7 +49,7 @@ PostgreSQL documentation createuser is a wrapper around the - SQL command . + SQL command CREATE ROLE. There is no effective difference between creating users via this utility and via other methods for accessing the server. diff --git a/doc/src/sgml/ref/declare.sgml b/doc/src/sgml/ref/declare.sgml index d6177dcd9c44..2152134635e4 100644 --- a/doc/src/sgml/ref/declare.sgml +++ b/doc/src/sgml/ref/declare.sgml @@ -39,7 +39,7 @@ DECLARE name [ BINARY ] [ INSENSITI can be used to retrieve a small number of rows at a time out of a larger query. After the cursor is created, rows are fetched from it using - . + FETCH. @@ -124,8 +124,8 @@ DECLARE name [ BINARY ] [ INSENSITI query - A or - command + A SELECT or + VALUES command which will provide the rows to be returned by the cursor. @@ -183,9 +183,9 @@ DECLARE name [ BINARY ] [ INSENSITI PostgreSQL reports an error if such a command is used outside a transaction block. Use - and - - (or ) + BEGIN and + COMMIT + (or ROLLBACK) to define a transaction block. @@ -244,7 +244,7 @@ DECLARE name [ BINARY ] [ INSENSITI If the cursor's query includes FOR UPDATE or FOR SHARE, then returned rows are locked at the time they are first fetched, in the same way as for a regular - command with + SELECT command with these options. In addition, the returned rows will be the most up-to-date versions; therefore these options provide the equivalent of what the SQL standard diff --git a/doc/src/sgml/ref/delete.sgml b/doc/src/sgml/ref/delete.sgml index ec3c40df2ea9..1b81b4e7d743 100644 --- a/doc/src/sgml/ref/delete.sgml +++ b/doc/src/sgml/ref/delete.sgml @@ -41,7 +41,7 @@ DELETE FROM [ ONLY ] table_name [ * - provides a + TRUNCATE provides a faster mechanism to remove all rows from a table. diff --git a/doc/src/sgml/ref/drop_group.sgml b/doc/src/sgml/ref/drop_group.sgml index 47d4a72121b6..eb7dc182c82b 100644 --- a/doc/src/sgml/ref/drop_group.sgml +++ b/doc/src/sgml/ref/drop_group.sgml @@ -30,7 +30,7 @@ DROP GROUP [ IF EXISTS ] name [, .. DROP GROUP is now an alias for - . + DROP ROLE. diff --git a/doc/src/sgml/ref/drop_language.sgml b/doc/src/sgml/ref/drop_language.sgml index 4705836ac79e..8ba6621bc4af 100644 --- a/doc/src/sgml/ref/drop_language.sgml +++ b/doc/src/sgml/ref/drop_language.sgml @@ -38,7 +38,7 @@ DROP [ PROCEDURAL ] LANGUAGE [ IF EXISTS ] name As of PostgreSQL 9.1, most procedural languages have been made into extensions, and should - therefore be removed with + therefore be removed with DROP EXTENSION not DROP LANGUAGE. diff --git a/doc/src/sgml/ref/drop_owned.sgml b/doc/src/sgml/ref/drop_owned.sgml index dcc375f33bf8..8fa8c414a10e 100644 --- a/doc/src/sgml/ref/drop_owned.sgml +++ b/doc/src/sgml/ref/drop_owned.sgml @@ -90,7 +90,7 @@ DROP OWNED BY { name | CURRENT_ROLE - The command is an alternative that + The REASSIGN OWNED command is an alternative that reassigns the ownership of all the database objects owned by one or more roles. However, REASSIGN OWNED does not deal with privileges for other objects. diff --git a/doc/src/sgml/ref/drop_role.sgml b/doc/src/sgml/ref/drop_role.sgml index 13079f3e1f4a..13dc1cc64998 100644 --- a/doc/src/sgml/ref/drop_role.sgml +++ b/doc/src/sgml/ref/drop_role.sgml @@ -40,7 +40,9 @@ DROP ROLE [ IF EXISTS ] name [, ... of the cluster; an error will be raised if so. Before dropping the role, you must drop all the objects it owns (or reassign their ownership) and revoke any privileges the role has been granted on other objects. - The and + The REASSIGN + OWNED and DROP + OWNED commands can be useful for this purpose; see for more discussion. diff --git a/doc/src/sgml/ref/drop_table.sgml b/doc/src/sgml/ref/drop_table.sgml index bf8996d19858..450458fd2a42 100644 --- a/doc/src/sgml/ref/drop_table.sgml +++ b/doc/src/sgml/ref/drop_table.sgml @@ -32,8 +32,8 @@ DROP TABLE [ IF EXISTS ] name [, .. DROP TABLE removes tables from the database. Only the table owner, the schema owner, and superuser can drop a table. To empty a table of rows - without destroying the table, use - or . + without destroying the table, use DELETE + or TRUNCATE. diff --git a/doc/src/sgml/ref/drop_user.sgml b/doc/src/sgml/ref/drop_user.sgml index 37ab856125d1..74e736b0ebd8 100644 --- a/doc/src/sgml/ref/drop_user.sgml +++ b/doc/src/sgml/ref/drop_user.sgml @@ -30,7 +30,7 @@ DROP USER [ IF EXISTS ] name [, ... DROP USER is simply an alternate spelling of - . + DROP ROLE. diff --git a/doc/src/sgml/ref/dropdb.sgml b/doc/src/sgml/ref/dropdb.sgml index ded85b0e232d..fe523a2ee1d3 100644 --- a/doc/src/sgml/ref/dropdb.sgml +++ b/doc/src/sgml/ref/dropdb.sgml @@ -41,7 +41,7 @@ PostgreSQL documentation dropdb is a wrapper around the - SQL command . + SQL command DROP DATABASE. There is no effective difference between dropping databases via this utility and via other methods for accessing the server. diff --git a/doc/src/sgml/ref/dropuser.sgml b/doc/src/sgml/ref/dropuser.sgml index f9aab340d3ba..81580507e826 100644 --- a/doc/src/sgml/ref/dropuser.sgml +++ b/doc/src/sgml/ref/dropuser.sgml @@ -42,7 +42,7 @@ PostgreSQL documentation dropuser is a wrapper around the - SQL command . + SQL command DROP ROLE. There is no effective difference between dropping users via this utility and via other methods for accessing the server. diff --git a/doc/src/sgml/ref/end.sgml b/doc/src/sgml/ref/end.sgml index 8b8f4f0dbb9f..498652919ad8 100644 --- a/doc/src/sgml/ref/end.sgml +++ b/doc/src/sgml/ref/end.sgml @@ -33,7 +33,7 @@ END [ WORK | TRANSACTION ] [ AND [ NO ] CHAIN ] made by the transaction become visible to others and are guaranteed to be durable if a crash occurs. This command is a PostgreSQL extension - that is equivalent to . + that is equivalent to COMMIT. @@ -69,7 +69,7 @@ END [ WORK | TRANSACTION ] [ AND [ NO ] CHAIN ] Notes - Use to + Use ROLLBACK to abort a transaction. @@ -94,8 +94,8 @@ END; END is a PostgreSQL - extension that provides functionality equivalent to , which is + extension that provides functionality equivalent to COMMIT, which is specified in the SQL standard. diff --git a/doc/src/sgml/ref/explain.sgml b/doc/src/sgml/ref/explain.sgml index 906b2ccd50a2..b0ccdd26e730 100644 --- a/doc/src/sgml/ref/explain.sgml +++ b/doc/src/sgml/ref/explain.sgml @@ -302,7 +302,7 @@ ROLLBACK; the autovacuum daemon will take care of that automatically. But if a table has recently had substantial changes in its contents, you might need to do a manual - rather than wait for autovacuum to catch up + ANALYZE rather than wait for autovacuum to catch up with the changes. diff --git a/doc/src/sgml/ref/fetch.sgml b/doc/src/sgml/ref/fetch.sgml index e802be61c8c6..ec843f568442 100644 --- a/doc/src/sgml/ref/fetch.sgml +++ b/doc/src/sgml/ref/fetch.sgml @@ -335,9 +335,9 @@ FETCH count - + DECLARE is used to define a cursor. Use - + MOVE to change cursor position without retrieving data. diff --git a/doc/src/sgml/ref/grant.sgml b/doc/src/sgml/ref/grant.sgml index fe231aa30cd6..c3db393bdea7 100644 --- a/doc/src/sgml/ref/grant.sgml +++ b/doc/src/sgml/ref/grant.sgml @@ -259,7 +259,7 @@ GRANT role_name [, ...] TO Notes - The command is used + The REVOKE command is used to revoke access privileges. diff --git a/doc/src/sgml/ref/lock.sgml b/doc/src/sgml/ref/lock.sgml index 0c4688603d9f..4cdfae2279e3 100644 --- a/doc/src/sgml/ref/lock.sgml +++ b/doc/src/sgml/ref/lock.sgml @@ -186,9 +186,9 @@ LOCK [ TABLE ] [ ONLY ] name [ * ] PostgreSQL reports an error if LOCK is used outside a transaction block. Use - and - - (or ) + BEGIN and + COMMIT + (or ROLLBACK) to define a transaction block. diff --git a/doc/src/sgml/ref/postgres-ref.sgml b/doc/src/sgml/ref/postgres-ref.sgml index 806949df42b7..fda678e345c8 100644 --- a/doc/src/sgml/ref/postgres-ref.sgml +++ b/doc/src/sgml/ref/postgres-ref.sgml @@ -143,8 +143,8 @@ PostgreSQL documentation This option is meant for other programs that interact with a server instance, such as , to query configuration - parameter values. User-facing applications should instead use or the pg_settings view. + parameter values. User-facing applications should instead use SHOW or the pg_settings view. diff --git a/doc/src/sgml/ref/prepare.sgml b/doc/src/sgml/ref/prepare.sgml index 1e484f6d2021..57a34ff83c79 100644 --- a/doc/src/sgml/ref/prepare.sgml +++ b/doc/src/sgml/ref/prepare.sgml @@ -66,7 +66,7 @@ PREPARE name [ ( command. + manually cleaned up using the DEALLOCATE command. @@ -163,7 +163,7 @@ PREPARE name [ ( To examine the query plan PostgreSQL is using - for a prepared statement, use , for example + for a prepared statement, use EXPLAIN, for example EXPLAIN EXECUTE name(parameter_values); diff --git a/doc/src/sgml/ref/prepare_transaction.sgml b/doc/src/sgml/ref/prepare_transaction.sgml index 18051983e160..f4f6118ac316 100644 --- a/doc/src/sgml/ref/prepare_transaction.sgml +++ b/doc/src/sgml/ref/prepare_transaction.sgml @@ -39,8 +39,8 @@ PREPARE TRANSACTION transaction_id Once prepared, a transaction can later be committed or rolled back - with - or , + with COMMIT PREPARED + or ROLLBACK PREPARED, respectively. Those commands can be issued from any session, not only the one that executed the original transaction. @@ -92,8 +92,8 @@ PREPARE TRANSACTION transaction_id - This command must be used inside a transaction block. Use to start one. + This command must be used inside a transaction block. Use BEGIN to start one. diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml index aed051f543c0..ee3fc0957798 100644 --- a/doc/src/sgml/ref/psql-ref.sgml +++ b/doc/src/sgml/ref/psql-ref.sgml @@ -756,8 +756,8 @@ testdb=> Whenever a command is executed, psql also polls for asynchronous notification events generated by - and - . + LISTEN and + NOTIFY. @@ -993,7 +993,7 @@ testdb=> Performs a frontend (client) copy. This is an operation that - runs an SQL + runs an SQL COPY command, but instead of the server reading or writing the specified file, psql reads or writes the file and @@ -1030,9 +1030,9 @@ testdb=> The syntax of this command is similar to that of the - SQL + SQL COPY command. All options other than the data source/destination are - as specified for . + as specified for COPY. Because of this, special parsing rules apply to the \copy meta-command. Unlike most other meta-commands, the entire remainder of the line is always taken to be the arguments of \copy, @@ -1397,8 +1397,8 @@ testdb=> - Descriptions for objects can be created with the + Descriptions for objects can be created with the COMMENT SQL command. @@ -1435,9 +1435,9 @@ testdb=> - The command is used to set - default access privileges. The meaning of the - privilege display is explained in + The ALTER DEFAULT + PRIVILEGES command is used to set default access + privileges. The meaning of the privilege display is explained in . @@ -1751,8 +1751,8 @@ testdb=> - The and - + The GRANT and + REVOKE commands are used to set access privileges. The meaning of the privilege display is explained in . @@ -1807,8 +1807,8 @@ testdb=> - The and - + The ALTER ROLE and + ALTER DATABASE commands are used to define per-role and per-database configuration settings. @@ -3179,7 +3179,7 @@ lo_import 152801 This command is unrelated to the SQL - command . + command SET. diff --git a/doc/src/sgml/ref/reassign_owned.sgml b/doc/src/sgml/ref/reassign_owned.sgml index 783389df4e8d..ab692bd06908 100644 --- a/doc/src/sgml/ref/reassign_owned.sgml +++ b/doc/src/sgml/ref/reassign_owned.sgml @@ -82,7 +82,7 @@ REASSIGN OWNED BY { old_role | CURR - The command is an alternative that + The DROP OWNED command is an alternative that simply drops all the database objects owned by one or more roles. diff --git a/doc/src/sgml/ref/refresh_materialized_view.sgml b/doc/src/sgml/ref/refresh_materialized_view.sgml index 8ae62671adab..3bf888444782 100644 --- a/doc/src/sgml/ref/refresh_materialized_view.sgml +++ b/doc/src/sgml/ref/refresh_materialized_view.sgml @@ -94,7 +94,7 @@ REFRESH MATERIALIZED VIEW [ CONCURRENTLY ] name While the default index for future - + CLUSTER operations is retained, REFRESH MATERIALIZED VIEW does not order the generated rows based on this property. If you want the data to be ordered upon generation, you must use an ORDER BY diff --git a/doc/src/sgml/ref/reindexdb.sgml b/doc/src/sgml/ref/reindexdb.sgml index 0f3f12bfbfc6..a6d93693c5d4 100644 --- a/doc/src/sgml/ref/reindexdb.sgml +++ b/doc/src/sgml/ref/reindexdb.sgml @@ -93,7 +93,7 @@ PostgreSQL documentation reindexdb is a wrapper around the SQL - command . + command REINDEX. There is no effective difference between reindexing databases via this utility and via other methods for accessing the server. diff --git a/doc/src/sgml/ref/revoke.sgml b/doc/src/sgml/ref/revoke.sgml index b50f99dfe714..35ff87a4f5e2 100644 --- a/doc/src/sgml/ref/revoke.sgml +++ b/doc/src/sgml/ref/revoke.sgml @@ -131,7 +131,7 @@ REVOKE [ ADMIN OPTION FOR ] - See the description of the command for + See the description of the GRANT command for the meaning of the privilege types. @@ -292,7 +292,7 @@ REVOKE admins FROM joe; Compatibility - The compatibility notes of the command + The compatibility notes of the GRANT command apply analogously to REVOKE. The keyword RESTRICT or CASCADE is required according to the standard, but PostgreSQL diff --git a/doc/src/sgml/ref/rollback.sgml b/doc/src/sgml/ref/rollback.sgml index 1357eaa8323a..142f71e77425 100644 --- a/doc/src/sgml/ref/rollback.sgml +++ b/doc/src/sgml/ref/rollback.sgml @@ -70,7 +70,7 @@ ROLLBACK [ WORK | TRANSACTION ] [ AND [ NO ] CHAIN ] Notes - Use to + Use COMMIT to successfully terminate a transaction. diff --git a/doc/src/sgml/ref/rollback_to.sgml b/doc/src/sgml/ref/rollback_to.sgml index 4d5647a302e2..3d5a241e1aa9 100644 --- a/doc/src/sgml/ref/rollback_to.sgml +++ b/doc/src/sgml/ref/rollback_to.sgml @@ -64,7 +64,7 @@ ROLLBACK [ WORK | TRANSACTION ] TO [ SAVEPOINT ] savepoint_nameNotes - Use to destroy a savepoint + Use RELEASE SAVEPOINT to destroy a savepoint without discarding the effects of commands executed after it was established. diff --git a/doc/src/sgml/ref/savepoint.sgml b/doc/src/sgml/ref/savepoint.sgml index 87243b1d2046..b17342a1ee6a 100644 --- a/doc/src/sgml/ref/savepoint.sgml +++ b/doc/src/sgml/ref/savepoint.sgml @@ -64,8 +64,8 @@ SAVEPOINT savepoint_name Notes - Use to - rollback to a savepoint. Use + Use ROLLBACK TO to + rollback to a savepoint. Use RELEASE SAVEPOINT to destroy a savepoint, keeping the effects of commands executed after it was established. diff --git a/doc/src/sgml/ref/select.sgml b/doc/src/sgml/ref/select.sgml index b93e4ca208b1..b4dea9b6acf4 100644 --- a/doc/src/sgml/ref/select.sgml +++ b/doc/src/sgml/ref/select.sgml @@ -446,7 +446,7 @@ TABLE [ ONLY ] table_name [ * ] sub-SELECT must be surrounded by parentheses, and an alias must be provided for it. A - command + VALUES command can also be used here. @@ -1534,7 +1534,7 @@ KEY SHARE to the row-level lock(s) — the required ROW SHARE table-level lock is still taken in the ordinary way (see ). You can use - + LOCK with the NOWAIT option first, if you need to acquire the table-level lock without waiting. diff --git a/doc/src/sgml/ref/select_into.sgml b/doc/src/sgml/ref/select_into.sgml index b1af52a4da12..6cfa706b5752 100644 --- a/doc/src/sgml/ref/select_into.sgml +++ b/doc/src/sgml/ref/select_into.sgml @@ -95,7 +95,7 @@ SELECT [ ALL | DISTINCT [ ON ( expressionNotes - is functionally similar to + CREATE TABLE AS is functionally similar to SELECT INTO. CREATE TABLE AS is the recommended syntax, since this form of SELECT INTO is not available in ECPG @@ -109,8 +109,8 @@ SELECT [ ALL | DISTINCT [ ON ( expressionCREATE TABLE AS, SELECT INTO does not allow to specify properties like a table's access method with or the table's - tablespace with . Use if necessary. Therefore, the default table + tablespace with . Use + CREATE TABLE AS if necessary. Therefore, the default table access method is chosen for the new table. See for more information. diff --git a/doc/src/sgml/ref/set_role.sgml b/doc/src/sgml/ref/set_role.sgml index a4842f363c8b..739f2c5cdfa5 100644 --- a/doc/src/sgml/ref/set_role.sgml +++ b/doc/src/sgml/ref/set_role.sgml @@ -48,7 +48,7 @@ RESET ROLE The SESSION and LOCAL modifiers act the same - as for the regular + as for the regular SET command. @@ -82,7 +82,7 @@ RESET ROLE SET ROLE has effects comparable to - , but the privilege + SET SESSION AUTHORIZATION, but the privilege checks involved are quite different. Also, SET SESSION AUTHORIZATION determines which roles are allowable for later SET ROLE commands, whereas changing @@ -92,7 +92,7 @@ RESET ROLE SET ROLE does not process session variables as specified by - the role's settings; this only happens during + the role's ALTER ROLE settings; this only happens during login. diff --git a/doc/src/sgml/ref/set_session_auth.sgml b/doc/src/sgml/ref/set_session_auth.sgml index 6a838e58b764..e44e78ed8d67 100644 --- a/doc/src/sgml/ref/set_session_auth.sgml +++ b/doc/src/sgml/ref/set_session_auth.sgml @@ -45,7 +45,7 @@ RESET SESSION AUTHORIZATION identifier is normally equal to the session user identifier, but might change temporarily in the context of SECURITY DEFINER functions and similar mechanisms; it can also be changed by - . + SET ROLE. The current user identifier is relevant for permission checking. @@ -58,7 +58,7 @@ RESET SESSION AUTHORIZATION The SESSION and LOCAL modifiers act the same - as for the regular + as for the regular SET command. diff --git a/doc/src/sgml/ref/start_transaction.sgml b/doc/src/sgml/ref/start_transaction.sgml index d6cd1d417792..74ccd7e3456c 100644 --- a/doc/src/sgml/ref/start_transaction.sgml +++ b/doc/src/sgml/ref/start_transaction.sgml @@ -37,8 +37,8 @@ START TRANSACTION [ transaction_mode This command begins a new transaction block. If the isolation level, read/write mode, or deferrable mode is specified, the new transaction has those - characteristics, as if was executed. This is the same - as the command. + characteristics, as if SET TRANSACTION was executed. This is the same + as the BEGIN command. diff --git a/doc/src/sgml/ref/vacuumdb.sgml b/doc/src/sgml/ref/vacuumdb.sgml index 766c6882bd4c..6dcdab9cafd6 100644 --- a/doc/src/sgml/ref/vacuumdb.sgml +++ b/doc/src/sgml/ref/vacuumdb.sgml @@ -62,7 +62,7 @@ PostgreSQL documentation vacuumdb is a wrapper around the SQL - command . + command VACUUM. There is no effective difference between vacuuming and analyzing databases via this utility and via other methods for accessing the server. diff --git a/doc/src/sgml/sepgsql.sgml b/doc/src/sgml/sepgsql.sgml index 9961569afc8a..e896a44ce591 100644 --- a/doc/src/sgml/sepgsql.sgml +++ b/doc/src/sgml/sepgsql.sgml @@ -51,7 +51,7 @@ - The statement allows assignment of + The SECURITY LABEL statement allows assignment of a security label to a database object. @@ -451,7 +451,7 @@ UPDATE t1 SET x = 2, y = func1(y) WHERE z = 100; - additionally requires + CREATE DATABASE additionally requires getattr permission for the source or template database. @@ -509,7 +509,7 @@ UPDATE t1 SET x = 2, y = func1(y) WHERE z = 100; - Using on an object additionally + Using SECURITY LABEL on an object additionally requires relabelfrom permission for the object in conjunction with its old security label and relabelto permission for the object in conjunction with its new security label. @@ -644,7 +644,7 @@ ERROR: SELinux: security policy violation Miscellaneous - We reject the command across the board, because + We reject the LOAD command across the board, because any module loaded could easily circumvent security policy enforcement. diff --git a/doc/src/sgml/tsm-system-rows.sgml b/doc/src/sgml/tsm-system-rows.sgml index 071ff301d07b..d960aa3e0fbc 100644 --- a/doc/src/sgml/tsm-system-rows.sgml +++ b/doc/src/sgml/tsm-system-rows.sgml @@ -10,7 +10,7 @@ The tsm_system_rows module provides the table sampling method SYSTEM_ROWS, which can be used in - the TABLESAMPLE clause of a + the TABLESAMPLE clause of a SELECT command. diff --git a/doc/src/sgml/tsm-system-time.sgml b/doc/src/sgml/tsm-system-time.sgml index cd074926d85a..df6e83a9236e 100644 --- a/doc/src/sgml/tsm-system-time.sgml +++ b/doc/src/sgml/tsm-system-time.sgml @@ -10,7 +10,7 @@ The tsm_system_time module provides the table sampling method SYSTEM_TIME, which can be used in - the TABLESAMPLE clause of a + the TABLESAMPLE clause of a SELECT command. diff --git a/doc/src/sgml/user-manag.sgml b/doc/src/sgml/user-manag.sgml index 829decd88390..cc082521a2ae 100644 --- a/doc/src/sgml/user-manag.sgml +++ b/doc/src/sgml/user-manag.sgml @@ -51,8 +51,8 @@ operating system users. In practice it might be convenient to maintain a correspondence, but this is not required. Database roles are global across a database cluster installation (and not - per individual database). To create a role use the SQL command: + per individual database). To create a role use the CREATE ROLE SQL command: CREATE ROLE name; @@ -61,7 +61,7 @@ CREATE ROLE name; double-quoted. (In practice, you will usually want to add additional options, such as LOGIN, to the command. More details appear below.) To remove an existing role, use the analogous - command: + DROP ROLE command: DROP ROLE name; @@ -303,8 +303,8 @@ CREATE ROLE name; Once the group role exists, you can add and remove members using the - and - commands: + GRANT and + REVOKE commands: GRANT group_role TO role1, ... ; REVOKE group_role FROM role1, ... ; @@ -319,7 +319,7 @@ REVOKE group_role FROM role1 The members of a group role can use the privileges of the role in two ways. First, every member of a group can explicitly do - to + SET ROLE to temporarily become the group role. In this state, the database session has access to the privileges of the group role rather than the original login role, and any database objects created are @@ -402,8 +402,8 @@ RESET ROLE; - To destroy a group role, use : + To destroy a group role, use DROP ROLE: DROP ROLE name; @@ -418,7 +418,7 @@ DROP ROLE name; Because roles can own database objects and can hold privileges to access other objects, dropping a role is often not just a matter of a - quick . Any objects owned by the role must + quick DROP ROLE. Any objects owned by the role must first be dropped or reassigned to other owners; and any permissions granted to the role must be revoked. @@ -429,7 +429,7 @@ DROP ROLE name; ALTER TABLE bobs_table OWNER TO alice; - Alternatively, the command can be + Alternatively, the REASSIGN OWNED command can be used to reassign ownership of all objects owned by the role-to-be-dropped to a single other role. Because REASSIGN OWNED cannot access objects in other databases, it is necessary to run it in each database @@ -442,7 +442,7 @@ ALTER TABLE bobs_table OWNER TO alice; Once any valuable objects have been transferred to new owners, any remaining objects owned by the role-to-be-dropped can be dropped with - the command. Again, this command cannot + the DROP OWNED command. Again, this command cannot access objects in other databases, so it is necessary to run it in each database that contains objects owned by the role. Also, DROP OWNED will not drop entire databases or tablespaces, so it is @@ -598,7 +598,7 @@ DROP ROLE doomed_role; Administrators can grant access to these roles to users using the - command, for example: + GRANT command, for example: GRANT pg_signal_backend TO admin_user; diff --git a/doc/src/sgml/xaggr.sgml b/doc/src/sgml/xaggr.sgml index f035866848c8..36c2d21101b9 100644 --- a/doc/src/sgml/xaggr.sgml +++ b/doc/src/sgml/xaggr.sgml @@ -490,7 +490,7 @@ SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY income) FROM households; Also, because the final function performs the sort, it is not possible to continue adding input rows by executing the transition function again later. This means the final function is not READ_ONLY; - it must be declared in + it must be declared in CREATE AGGREGATE as READ_WRITE, or as SHAREABLE if it's possible for additional final-function calls to make use of the already-sorted state. diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index 0f60a4a0ab6d..8c74c11d3b59 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -85,7 +85,7 @@ that a procedure does not return a value, so there is no return type declaration. While a function is called as part of a query or DML command, a procedure is called in isolation using - the command. If the CALL command is not + the CALL command. If the CALL command is not part of an explicit transaction, a procedure in many server-side languages can commit, rollback, and begin new transactions during its execution, which is not possible in functions. @@ -94,15 +94,15 @@ The explanations on how to define user-defined functions in the rest of this chapter apply to procedures as well, except that - the command is used instead, there is + the CREATE PROCEDURE command is used instead, there is no return type, and some other features such as strictness don't apply. Collectively, functions and procedures are also known as routinesroutine. - There are commands such as - and that can operate on functions and + There are commands such as ALTER ROUTINE + and DROP ROUTINE that can operate on functions and procedures without having to know which kind it is. Note, however, that there is no CREATE ROUTINE command. @@ -1531,7 +1531,7 @@ CREATE FUNCTION test(int, int) RETURNS int Every function has a volatility classification, with the possibilities being VOLATILE, STABLE, or IMMUTABLE. VOLATILE is the default if the - + CREATE FUNCTION command does not specify a category. The volatility category is a promise to the optimizer about the behavior of the function: @@ -3432,7 +3432,7 @@ if (!ptr) Some basic facts can be supplied by declarative annotations provided in - the command. Most important of + the CREATE FUNCTION command. Most important of these is the function's volatility category (IMMUTABLE, STABLE, or VOLATILE); one should always be careful to diff --git a/doc/src/sgml/xplang.sgml b/doc/src/sgml/xplang.sgml index 7f1409305629..31d403c4806b 100644 --- a/doc/src/sgml/xplang.sgml +++ b/doc/src/sgml/xplang.sgml @@ -103,7 +103,7 @@ CREATE FUNCTION handler_function_name() Optionally, the language handler can provide an inline handler function that executes anonymous code blocks - ( commands) + (DO commands) written in this language. If an inline handler function is provided by the language, declare it with a command like From 97b61448262eae5e1b4a631aeac63b11d902a474 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 4 Oct 2020 16:09:55 -0400 Subject: [PATCH 243/589] Make postgres.bki use the same literal-string syntax as postgresql.conf. The BKI file's string quoting conventions were previously quite weird, perhaps as a result of repurposing a function built to scan single-quoted strings to scan double-quoted ones. Change to use the same rules as we use in GUC files, allowing some simplifications in genbki.pl and initdb.c. While at it, completely remove the backend's scanstr() function, which was essentially a duplicate of the string dequoting code in guc-file.l. Instead export that one (under a less generic name than it had) and let bootscanner.l use it. Now we can clarify that scansup.c exists only to support the main lexer. We could alternatively have removed GUC_scanstr, but this way seems better since the previous arrangement could mislead a reader into thinking that scanstr() had something to do with the main lexer's handling of string literals. Maybe it did once, but if so it was a long time ago. This patch does not bump catversion, since the initially-installed catalog contents don't change. Note however that successful initdb after applying this patch will require up-to-date postgres.bki as well as postgres and initdb executables. In passing, remove a bunch of very-long-obsolete #include's in bootparse.y and bootscanner.l. John Naylor Discussion: https://postgr.es/m/CACPNZCtDpd18T0KATTmCggO2GdVC4ow86ypiq5ENff1VnauL8g@mail.gmail.com --- doc/src/sgml/bki.sgml | 10 +-- src/backend/bootstrap/bootparse.y | 19 ------ src/backend/bootstrap/bootscanner.l | 29 ++------- src/backend/catalog/genbki.pl | 6 +- src/backend/parser/scansup.c | 95 +---------------------------- src/backend/utils/misc/guc-file.l | 14 +++-- src/bin/initdb/initdb.c | 34 +++-------- src/include/parser/scansup.h | 5 +- src/include/utils/guc.h | 1 + 9 files changed, 31 insertions(+), 182 deletions(-) diff --git a/doc/src/sgml/bki.sgml b/doc/src/sgml/bki.sgml index 4e696d1d3ed0..62fc9cbea3dc 100644 --- a/doc/src/sgml/bki.sgml +++ b/doc/src/sgml/bki.sgml @@ -752,8 +752,8 @@ $ perl rewrite_dat_with_prokind.pl pg_proc.dat next token that syntactically cannot belong to the preceding command starts a new one. (Usually you would put a new command on a new line, for clarity.) Tokens can be certain key words, special - characters (parentheses, commas, etc.), numbers, or double-quoted - strings. Everything is case sensitive. + characters (parentheses, commas, etc.), identifiers, numbers, or + single-quoted strings. Everything is case sensitive. @@ -876,7 +876,9 @@ $ perl rewrite_dat_with_prokind.pl pg_proc.dat NULL values can be specified using the special key word _null_. Values that do not look like - identifiers or digit strings must be double quoted. + identifiers or digit strings must be single-quoted. + (To include a single quote in a value, write it twice. + Escape-string-style backslash escapes are allowed in the string, too.) @@ -1046,7 +1048,7 @@ $ perl rewrite_dat_with_prokind.pl pg_proc.dat create test_table 420 (oid = oid, cola = int4, colb = text) open test_table -insert ( 421 1 "value1" ) +insert ( 421 1 'value 1' ) insert ( 422 2 _null_ ) close test_table diff --git a/src/backend/bootstrap/bootparse.y b/src/backend/bootstrap/bootparse.y index 5eaca279ee83..6bb0c6ed1ea9 100644 --- a/src/backend/bootstrap/bootparse.y +++ b/src/backend/bootstrap/bootparse.y @@ -18,16 +18,10 @@ #include -#include "access/attnum.h" -#include "access/htup.h" -#include "access/itup.h" -#include "access/tupdesc.h" #include "bootstrap/bootstrap.h" -#include "catalog/catalog.h" #include "catalog/heap.h" #include "catalog/namespace.h" #include "catalog/pg_am.h" -#include "catalog/pg_attribute.h" #include "catalog/pg_authid.h" #include "catalog/pg_class.h" #include "catalog/pg_namespace.h" @@ -36,20 +30,7 @@ #include "commands/defrem.h" #include "miscadmin.h" #include "nodes/makefuncs.h" -#include "nodes/nodes.h" -#include "nodes/parsenodes.h" -#include "nodes/pg_list.h" -#include "nodes/primnodes.h" -#include "rewrite/prs2lock.h" -#include "storage/block.h" -#include "storage/fd.h" -#include "storage/ipc.h" -#include "storage/itemptr.h" -#include "storage/off.h" -#include "storage/smgr.h" -#include "tcop/dest.h" #include "utils/memutils.h" -#include "utils/rel.h" /* diff --git a/src/backend/bootstrap/bootscanner.l b/src/backend/bootstrap/bootscanner.l index 1048e70d0537..6a0bed6c8d60 100644 --- a/src/backend/bootstrap/bootscanner.l +++ b/src/backend/bootstrap/bootscanner.l @@ -15,25 +15,8 @@ */ #include "postgres.h" -#include "access/attnum.h" -#include "access/htup.h" -#include "access/itup.h" -#include "access/tupdesc.h" #include "bootstrap/bootstrap.h" -#include "catalog/pg_am.h" -#include "catalog/pg_attribute.h" -#include "catalog/pg_class.h" -#include "nodes/nodes.h" -#include "nodes/parsenodes.h" -#include "nodes/pg_list.h" -#include "nodes/primnodes.h" -#include "parser/scansup.h" -#include "rewrite/prs2lock.h" -#include "storage/block.h" -#include "storage/fd.h" -#include "storage/itemptr.h" -#include "storage/off.h" -#include "utils/rel.h" +#include "utils/guc.h" /* Not needed now that this file is compiled as part of bootparse. */ /* #include "bootparse.h" */ @@ -66,7 +49,7 @@ static int yyline = 1; /* line number for error reporting */ id [-A-Za-z0-9_]+ -sid \"([^\"])*\" +sid \'([^']|\'\')*\' /* * Keyword tokens return the keyword text (as a constant string) in yylval.kw, @@ -120,14 +103,12 @@ NOT { yylval.kw = "NOT"; return XNOT; } NULL { yylval.kw = "NULL"; return XNULL; } {id} { - yylval.str = scanstr(yytext); + yylval.str = pstrdup(yytext); return ID; } {sid} { - /* leading and trailing quotes are not passed to scanstr */ - yytext[strlen(yytext) - 1] = '\0'; - yylval.str = scanstr(yytext+1); - yytext[strlen(yytext)] = '"'; /* restore yytext */ + /* strip quotes and escapes */ + yylval.str = DeescapeQuotedString(yytext); return ID; } diff --git a/src/backend/catalog/genbki.pl b/src/backend/catalog/genbki.pl index dc5f442397a4..ef3105af44bb 100644 --- a/src/backend/catalog/genbki.pl +++ b/src/backend/catalog/genbki.pl @@ -845,17 +845,15 @@ sub print_bki_insert # since that represents a NUL char in C code. $bki_value = '' if $bki_value eq '\0'; - # Handle single quotes by doubling them, and double quotes by - # converting them to octal escapes, because that's what the + # Handle single quotes by doubling them, because that's what the # bootstrap scanner requires. We do not process backslashes # specially; this allows escape-string-style backslash escapes # to be used in catalog data. $bki_value =~ s/'/''/g; - $bki_value =~ s/"/\\042/g; # Quote value if needed. We need not quote values that satisfy # the "id" pattern in bootscanner.l, currently "[-A-Za-z0-9_]+". - $bki_value = sprintf(qq'"%s"', $bki_value) + $bki_value = sprintf("'%s'", $bki_value) if length($bki_value) == 0 or $bki_value =~ /[^-A-Za-z0-9_]/; diff --git a/src/backend/parser/scansup.c b/src/backend/parser/scansup.c index cac70d5df7af..d07cbafcee75 100644 --- a/src/backend/parser/scansup.c +++ b/src/backend/parser/scansup.c @@ -1,8 +1,7 @@ /*------------------------------------------------------------------------- * * scansup.c - * support routines for the lex/flex scanner, used by both the normal - * backend as well as the bootstrap backend + * scanner support routines used by the core lexer * * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California @@ -20,98 +19,6 @@ #include "mb/pg_wchar.h" #include "parser/scansup.h" -/* ---------------- - * scanstr - * - * if the string passed in has escaped codes, map the escape codes to actual - * chars - * - * the string returned is palloc'd and should eventually be pfree'd by the - * caller! - * ---------------- - */ - -char * -scanstr(const char *s) -{ - char *newStr; - int len, - i, - j; - - if (s == NULL || s[0] == '\0') - return pstrdup(""); - - len = strlen(s); - - newStr = palloc(len + 1); /* string cannot get longer */ - - for (i = 0, j = 0; i < len; i++) - { - if (s[i] == '\'') - { - /* - * Note: if scanner is working right, unescaped quotes can only - * appear in pairs, so there should be another character. - */ - i++; - /* The bootstrap parser is not as smart, so check here. */ - Assert(s[i] == '\''); - newStr[j] = s[i]; - } - else if (s[i] == '\\') - { - i++; - switch (s[i]) - { - case 'b': - newStr[j] = '\b'; - break; - case 'f': - newStr[j] = '\f'; - break; - case 'n': - newStr[j] = '\n'; - break; - case 'r': - newStr[j] = '\r'; - break; - case 't': - newStr[j] = '\t'; - break; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - { - int k; - long octVal = 0; - - for (k = 0; - s[i + k] >= '0' && s[i + k] <= '7' && k < 3; - k++) - octVal = (octVal << 3) + (s[i + k] - '0'); - i += k - 1; - newStr[j] = ((char) octVal); - } - break; - default: - newStr[j] = s[i]; - break; - } /* switch */ - } /* s[i] == '\\' */ - else - newStr[j] = s[i]; - j++; - } - newStr[j] = '\0'; - return newStr; -} - /* * downcase_truncate_identifier() --- do appropriate downcasing and diff --git a/src/backend/utils/misc/guc-file.l b/src/backend/utils/misc/guc-file.l index 268b74552838..c98e2202951c 100644 --- a/src/backend/utils/misc/guc-file.l +++ b/src/backend/utils/misc/guc-file.l @@ -55,7 +55,6 @@ static void record_config_file_error(const char *errmsg, ConfigVariable **tail_p); static int GUC_flex_fatal(const char *msg); -static char *GUC_scanstr(const char *s); /* LCOV_EXCL_START */ @@ -797,7 +796,7 @@ ParseConfigFp(FILE *fp, const char *config_file, int depth, int elevel, token != GUC_UNQUOTED_STRING) goto parse_error; if (token == GUC_STRING) /* strip quotes and escapes */ - opt_value = GUC_scanstr(yytext); + opt_value = DeescapeQuotedString(yytext); else opt_value = pstrdup(yytext); @@ -1132,22 +1131,25 @@ FreeConfigVariable(ConfigVariable *item) /* - * scanstr + * DeescapeQuotedString * * Strip the quotes surrounding the given string, and collapse any embedded * '' sequences and backslash escapes. * - * the string returned is palloc'd and should eventually be pfree'd by the + * The string returned is palloc'd and should eventually be pfree'd by the * caller. + * + * This is exported because it is also used by the bootstrap scanner. */ -static char * -GUC_scanstr(const char *s) +char * +DeescapeQuotedString(const char *s) { char *newStr; int len, i, j; + /* We just Assert that there are leading and trailing quotes */ Assert(s != NULL && s[0] == '\''); len = strlen(s); Assert(len >= 2); diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 118b282d1c5e..ee3bfa82f481 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -331,12 +331,9 @@ escape_quotes(const char *src) /* * Escape a field value to be inserted into the BKI data. - * Here, we first run the value through escape_quotes (which - * will be inverted by the backend's scanstr() function) and - * then overlay special processing of double quotes, which - * bootscanner.l will only accept as data if converted to octal - * representation ("\042"). We always wrap the value in double - * quotes, even if that isn't strictly necessary. + * Run the value through escape_quotes (which will be inverted + * by the backend's DeescapeQuotedString() function), then wrap + * the value in single quotes, even if that isn't strictly necessary. */ static char * escape_quotes_bki(const char *src) @@ -345,30 +342,13 @@ escape_quotes_bki(const char *src) char *data = escape_quotes(src); char *resultp; char *datap; - int nquotes = 0; - /* count double quotes in data */ - datap = data; - while ((datap = strchr(datap, '"')) != NULL) - { - nquotes++; - datap++; - } - - result = (char *) pg_malloc(strlen(data) + 3 + nquotes * 3); + result = (char *) pg_malloc(strlen(data) + 3); resultp = result; - *resultp++ = '"'; + *resultp++ = '\''; for (datap = data; *datap; datap++) - { - if (*datap == '"') - { - strcpy(resultp, "\\042"); - resultp += 4; - } - else - *resultp++ = *datap; - } - *resultp++ = '"'; + *resultp++ = *datap; + *resultp++ = '\''; *resultp = '\0'; free(data); diff --git a/src/include/parser/scansup.h b/src/include/parser/scansup.h index 7a6ee529ae0c..5bc426660df6 100644 --- a/src/include/parser/scansup.h +++ b/src/include/parser/scansup.h @@ -1,8 +1,7 @@ /*------------------------------------------------------------------------- * * scansup.h - * scanner support routines. used by both the bootstrap lexer - * as well as the normal lexer + * scanner support routines used by the core lexer * * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California @@ -15,8 +14,6 @@ #ifndef SCANSUP_H #define SCANSUP_H -extern char *scanstr(const char *s); - extern char *downcase_truncate_identifier(const char *ident, int len, bool warn); diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index 281928218167..073c8f3e0628 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -155,6 +155,7 @@ extern bool ParseConfigDirectory(const char *includedir, ConfigVariable **head_p, ConfigVariable **tail_p); extern void FreeConfigVariables(ConfigVariable *list); +extern char *DeescapeQuotedString(const char *s); /* * The possible values of an enum variable are specified by an array of From 10c5291cc2c6497d5abe04dde7df649d898becc4 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 5 Oct 2020 09:43:17 +0900 Subject: [PATCH 244/589] Fix handling of redundant options with COPY for "freeze" and "header" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The handling of those options was inconsistent, as the processing used directly the value assigned to the option to check if it was redundant, leading to patterns like this one to succeed (note that false is specified first): COPY hoge to '/path/to/file/' (header off, header on); And the opposite would fail correctly (note that true is first here): COPY hoge to '/path/to/file/' (header on, header off); While on it, add some tests to check for all redundant patterns with the options of COPY. I have gone through the code and did not notice similar mistakes for other commands. "header" got it wrong since b63990c, and "freeze" was wrong from the start as of 8de72b6. No backpatch is done per the lack of complaints. Reported-by: Rémi Lapeyre Discussion: https://postgr.es/m/20200929072433.GA15570@paquier.xyz Discussion: https://postgr.es/m/0B55BD07-83E4-439F-AACC-FA2D7CF50532@lenstra.fr --- src/backend/commands/copy.c | 8 +++-- src/test/regress/expected/copy2.out | 47 +++++++++++++++++++++++++++++ src/test/regress/sql/copy2.sql | 14 +++++++++ 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 2047557e5200..3c7dbad27a21 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -1159,6 +1159,8 @@ ProcessCopyOptions(ParseState *pstate, List *options) { bool format_specified = false; + bool freeze_specified = false; + bool header_specified = false; ListCell *option; /* Support external use for option sanity checking */ @@ -1198,11 +1200,12 @@ ProcessCopyOptions(ParseState *pstate, } else if (strcmp(defel->defname, "freeze") == 0) { - if (cstate->freeze) + if (freeze_specified) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("conflicting or redundant options"), parser_errposition(pstate, defel->location))); + freeze_specified = true; cstate->freeze = defGetBoolean(defel); } else if (strcmp(defel->defname, "delimiter") == 0) @@ -1225,11 +1228,12 @@ ProcessCopyOptions(ParseState *pstate, } else if (strcmp(defel->defname, "header") == 0) { - if (cstate->header_line) + if (header_specified) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("conflicting or redundant options"), parser_errposition(pstate, defel->location))); + header_specified = true; cstate->header_line = defGetBoolean(defel); } else if (strcmp(defel->defname, "quote") == 0) diff --git a/src/test/regress/expected/copy2.out b/src/test/regress/expected/copy2.out index e40287d25a41..c64f0719e7b6 100644 --- a/src/test/regress/expected/copy2.out +++ b/src/test/regress/expected/copy2.out @@ -28,6 +28,53 @@ COPY x (a, b, c, d, e) from stdin; -- non-existent column in column list: should fail COPY x (xyz) from stdin; ERROR: column "xyz" of relation "x" does not exist +-- redundant options +COPY x from stdin (format CSV, FORMAT CSV); +ERROR: conflicting or redundant options +LINE 1: COPY x from stdin (format CSV, FORMAT CSV); + ^ +COPY x from stdin (freeze off, freeze on); +ERROR: conflicting or redundant options +LINE 1: COPY x from stdin (freeze off, freeze on); + ^ +COPY x from stdin (delimiter ',', delimiter ','); +ERROR: conflicting or redundant options +LINE 1: COPY x from stdin (delimiter ',', delimiter ','); + ^ +COPY x from stdin (null ' ', null ' '); +ERROR: conflicting or redundant options +LINE 1: COPY x from stdin (null ' ', null ' '); + ^ +COPY x from stdin (header off, header on); +ERROR: conflicting or redundant options +LINE 1: COPY x from stdin (header off, header on); + ^ +COPY x from stdin (quote ':', quote ':'); +ERROR: conflicting or redundant options +LINE 1: COPY x from stdin (quote ':', quote ':'); + ^ +COPY x from stdin (escape ':', escape ':'); +ERROR: conflicting or redundant options +LINE 1: COPY x from stdin (escape ':', escape ':'); + ^ +COPY x from stdin (force_quote (a), force_quote *); +ERROR: conflicting or redundant options +LINE 1: COPY x from stdin (force_quote (a), force_quote *); + ^ +COPY x from stdin (force_not_null (a), force_not_null (b)); +ERROR: conflicting or redundant options +LINE 1: COPY x from stdin (force_not_null (a), force_not_null (b)); + ^ +COPY x from stdin (force_null (a), force_null (b)); +ERROR: conflicting or redundant options +COPY x from stdin (convert_selectively (a), convert_selectively (b)); +ERROR: conflicting or redundant options +LINE 1: COPY x from stdin (convert_selectively (a), convert_selectiv... + ^ +COPY x from stdin (encoding 'sql_ascii', encoding 'sql_ascii'); +ERROR: conflicting or redundant options +LINE 1: COPY x from stdin (encoding 'sql_ascii', encoding 'sql_ascii... + ^ -- too many columns in column list: should fail COPY x (a, b, c, d, e, d, c) from stdin; ERROR: column "d" specified more than once diff --git a/src/test/regress/sql/copy2.sql b/src/test/regress/sql/copy2.sql index 902f4fac19a4..b3c16af48eec 100644 --- a/src/test/regress/sql/copy2.sql +++ b/src/test/regress/sql/copy2.sql @@ -53,6 +53,20 @@ COPY x (a, b, c, d, e) from stdin; -- non-existent column in column list: should fail COPY x (xyz) from stdin; +-- redundant options +COPY x from stdin (format CSV, FORMAT CSV); +COPY x from stdin (freeze off, freeze on); +COPY x from stdin (delimiter ',', delimiter ','); +COPY x from stdin (null ' ', null ' '); +COPY x from stdin (header off, header on); +COPY x from stdin (quote ':', quote ':'); +COPY x from stdin (escape ':', escape ':'); +COPY x from stdin (force_quote (a), force_quote *); +COPY x from stdin (force_not_null (a), force_not_null (b)); +COPY x from stdin (force_null (a), force_null (b)); +COPY x from stdin (convert_selectively (a), convert_selectively (b)); +COPY x from stdin (encoding 'sql_ascii', encoding 'sql_ascii'); + -- too many columns in column list: should fail COPY x (a, b, c, d, e, d, c) from stdin; From e899742081fa24bf52d4a32103ef854a3a85865d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 4 Oct 2020 20:45:06 -0400 Subject: [PATCH 245/589] Improve stability of identity.sql regression test. I noticed while trying to run the regression tests under a low geqo_threshold that one query on information_schema.columns had unstable (as in, variable from one run to the next) output order. This is pretty unsurprising given the complexity of the underlying plan. Interestingly, of this test's three nigh-identical queries on information_schema.columns, the other two already had ORDER BY clauses guaranteeing stable output. Let's make this one look the same. Back-patch to v10 where this test was added. We've not heard field reports of the test failing, but this experience shows that it can happen when testing under even slightly unusual conditions. --- src/test/regress/expected/identity.out | 2 +- src/test/regress/sql/identity.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/regress/expected/identity.out b/src/test/regress/expected/identity.out index 7ac9df767f55..2238f896f9a4 100644 --- a/src/test/regress/expected/identity.out +++ b/src/test/regress/expected/identity.out @@ -335,7 +335,7 @@ SELECT * FROM itest6; 102 | (3 rows) -SELECT table_name, column_name, is_identity, identity_generation FROM information_schema.columns WHERE table_name = 'itest6'; +SELECT table_name, column_name, is_identity, identity_generation FROM information_schema.columns WHERE table_name = 'itest6' ORDER BY 1, 2; table_name | column_name | is_identity | identity_generation ------------+-------------+-------------+--------------------- itest6 | a | YES | BY DEFAULT diff --git a/src/test/regress/sql/identity.sql b/src/test/regress/sql/identity.sql index 1bf2a976eb0b..d4bc29ab5c30 100644 --- a/src/test/regress/sql/identity.sql +++ b/src/test/regress/sql/identity.sql @@ -208,7 +208,7 @@ INSERT INTO itest6 DEFAULT VALUES; INSERT INTO itest6 DEFAULT VALUES; SELECT * FROM itest6; -SELECT table_name, column_name, is_identity, identity_generation FROM information_schema.columns WHERE table_name = 'itest6'; +SELECT table_name, column_name, is_identity, identity_generation FROM information_schema.columns WHERE table_name = 'itest6' ORDER BY 1, 2; ALTER TABLE itest6 ALTER COLUMN b SET INCREMENT BY 2; -- fail, not identity From 2453ea142233ae57af452019c3b9a443dad1cdd0 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 5 Oct 2020 09:09:09 +0200 Subject: [PATCH 246/589] Support for OUT parameters in procedures Unlike for functions, OUT parameters for procedures are part of the signature. Therefore, they have to be listed in pg_proc.proargtypes as well as mentioned in ALTER PROCEDURE and DROP PROCEDURE. Reviewed-by: Andrew Dunstan Reviewed-by: Pavel Stehule Discussion: https://www.postgresql.org/message-id/flat/2b8490fe-51af-e671-c504-47359dc453c5@2ndquadrant.com --- doc/src/sgml/catalogs.sgml | 5 +- doc/src/sgml/plpgsql.sgml | 38 ++++++++ doc/src/sgml/ref/alter_extension.sgml | 11 ++- doc/src/sgml/ref/alter_procedure.sgml | 5 +- doc/src/sgml/ref/comment.sgml | 11 ++- doc/src/sgml/ref/create_procedure.sgml | 6 +- doc/src/sgml/ref/drop_procedure.sgml | 5 +- doc/src/sgml/ref/security_label.sgml | 11 ++- doc/src/sgml/xfunc.sgml | 59 ++++++++++++ src/backend/catalog/pg_proc.c | 9 +- src/backend/commands/functioncmds.c | 57 ++++++----- src/backend/executor/functions.c | 3 +- src/backend/parser/gram.y | 96 ++++++++++++------- src/backend/utils/fmgr/funcapi.c | 4 +- src/include/catalog/pg_proc.h | 2 +- src/include/funcapi.h | 3 +- src/pl/plperl/expected/plperl_call.out | 18 ++++ src/pl/plperl/sql/plperl_call.sql | 20 ++++ src/pl/plpgsql/src/expected/plpgsql_call.out | 19 ++++ src/pl/plpgsql/src/pl_comp.c | 1 + src/pl/plpgsql/src/sql/plpgsql_call.sql | 21 ++++ src/pl/plpython/expected/plpython_call.out | 17 ++++ src/pl/plpython/plpy_procedure.c | 4 +- src/pl/plpython/sql/plpython_call.sql | 19 ++++ src/pl/tcl/expected/pltcl_call.out | 17 ++++ src/pl/tcl/sql/pltcl_call.sql | 19 ++++ .../regress/expected/create_procedure.out | 16 +++- src/test/regress/sql/create_procedure.sql | 13 ++- 28 files changed, 416 insertions(+), 93 deletions(-) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 0e580b157f53..3927b1030df4 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -5875,8 +5875,9 @@ SCRAM-SHA-256$<iteration count>:&l An array with the data types of the function arguments. This includes only input arguments (including INOUT and - VARIADIC arguments), and thus represents - the call signature of the function. + VARIADIC arguments), as well as + OUT parameters of procedures, and thus represents + the call signature of the function or procedure. diff --git a/doc/src/sgml/plpgsql.sgml b/doc/src/sgml/plpgsql.sgml index c2bb3e326851..74b6b2587809 100644 --- a/doc/src/sgml/plpgsql.sgml +++ b/doc/src/sgml/plpgsql.sgml @@ -478,6 +478,14 @@ $$ LANGUAGE plpgsql; included it, but it would be redundant. + + To call a function with OUT parameters, omit the + output parameter in the function call: + +SELECT sales_tax(100.00); + + + Output parameters are most useful when returning multiple values. A trivial example is: @@ -489,6 +497,11 @@ BEGIN prod := x * y; END; $$ LANGUAGE plpgsql; + +SELECT * FROM sum_n_product(2, 4); + sum | prod +-----+------ + 6 | 8 As discussed in , this @@ -497,6 +510,31 @@ $$ LANGUAGE plpgsql; RETURNS record. + + This also works with procedures, for example: + + +CREATE PROCEDURE sum_n_product(x int, y int, OUT sum int, OUT prod int) AS $$ +BEGIN + sum := x + y; + prod := x * y; +END; +$$ LANGUAGE plpgsql; + + + In a call to a procedure, all the parameters must be specified. For + output parameters, NULL may be specified. + +CALL sum_n_product(2, 4, NULL, NULL); + sum | prod +-----+------ + 6 | 8 + + Output parameters in procedures become more interesting in nested calls, + where they can be assigned to variables. See for details. + + Another way to declare a PL/pgSQL function is with RETURNS TABLE, for example: diff --git a/doc/src/sgml/ref/alter_extension.sgml b/doc/src/sgml/ref/alter_extension.sgml index c819c7bb4e3c..38fd60128b78 100644 --- a/doc/src/sgml/ref/alter_extension.sgml +++ b/doc/src/sgml/ref/alter_extension.sgml @@ -212,11 +212,12 @@ ALTER EXTENSION name DROP IN, OUT, INOUT, or VARIADIC. If omitted, the default is IN. - Note that ALTER EXTENSION does not actually pay - any attention to OUT arguments, since only the input - arguments are needed to determine the function's identity. - So it is sufficient to list the IN, INOUT, - and VARIADIC arguments. + Note that ALTER EXTENSION does not actually pay any + attention to OUT arguments for functions and + aggregates (but not procedures), since only the input arguments are + needed to determine the function's identity. So it is sufficient to + list the IN, INOUT, and + VARIADIC arguments for functions and aggregates. diff --git a/doc/src/sgml/ref/alter_procedure.sgml b/doc/src/sgml/ref/alter_procedure.sgml index bcf45c7a85f9..5c176fb5d876 100644 --- a/doc/src/sgml/ref/alter_procedure.sgml +++ b/doc/src/sgml/ref/alter_procedure.sgml @@ -81,8 +81,9 @@ ALTER PROCEDURE name [ ( [ [ name [ - The mode of an argument: IN or VARIADIC. - If omitted, the default is IN. + The mode of an argument: IN, OUT, + INOUT, or VARIADIC. If omitted, + the default is IN. diff --git a/doc/src/sgml/ref/security_label.sgml b/doc/src/sgml/ref/security_label.sgml index e9688cce214b..9b87bcd51961 100644 --- a/doc/src/sgml/ref/security_label.sgml +++ b/doc/src/sgml/ref/security_label.sgml @@ -127,11 +127,12 @@ SECURITY LABEL [ FOR provider ] ON argument: IN, OUT, INOUT, or VARIADIC. If omitted, the default is IN. - Note that SECURITY LABEL does not actually - pay any attention to OUT arguments, since only the input - arguments are needed to determine the function's identity. - So it is sufficient to list the IN, INOUT, - and VARIADIC arguments. + Note that SECURITY LABEL does not actually pay any + attention to OUT arguments for functions and + aggregates (but not procedures), since only the input arguments are + needed to determine the function's identity. So it is sufficient to + list the IN, INOUT, and + VARIADIC arguments for functions and aggregates. diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index 8c74c11d3b59..2863f7c20657 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -179,6 +179,24 @@ SELECT clean_emp(); + + You can also write this as a procedure, thus avoiding the issue of the + return type. For example: + +CREATE PROCEDURE clean_emp() AS ' + DELETE FROM emp + WHERE salary < 0; +' LANGUAGE SQL; + +CALL clean_emp(); + + In simple cases like this, the difference between a function returning + void and a procedure is mostly stylistic. However, + procedures offer additional functionality such as transaction control + that is not available in functions. Also, procedures are SQL standard + whereas returning void is a PostgreSQL extension. + + The entire body of a SQL function is parsed before any of it is @@ -716,6 +734,47 @@ DROP FUNCTION sum_n_product (int, int); + + <acronym>SQL</acronym> Procedures with Output Parameters + + + procedures + output parameter + + + + Output parameters are also supported in procedures, but they work a bit + differently from functions. Notably, output parameters + are included in the signature of a procedure and + must be specified in the procedure call. + + + + For example, the bank account debiting routine from earlier could be + written like this: + +CREATE PROCEDURE tp1 (accountno integer, debit numeric, OUT new_balance numeric) AS $$ + UPDATE bank + SET balance = balance - debit + WHERE accountno = tp1.accountno + RETURNING balance; +$$ LANGUAGE SQL; + + To call this procedure, it is irrelevant what is passed as the argument + of the OUT parameter, so you could pass + NULL: + +CALL tp1(17, 100.0, NULL); + + + + + Procedures with output parameters are more useful in PL/pgSQL, where the + output parameters can be assigned to variables. See for details. + + + <acronym>SQL</acronym> Functions with Variable Numbers of Arguments diff --git a/src/backend/catalog/pg_proc.c b/src/backend/catalog/pg_proc.c index 40d65dc6bab1..f7dab9925b9e 100644 --- a/src/backend/catalog/pg_proc.c +++ b/src/backend/catalog/pg_proc.c @@ -249,6 +249,9 @@ ProcedureCreate(const char *procedureName, elog(ERROR, "variadic parameter must be last"); break; case PROARGMODE_OUT: + if (OidIsValid(variadicType) && prokind == PROKIND_PROCEDURE) + elog(ERROR, "variadic parameter must be last"); + break; case PROARGMODE_TABLE: /* okay */ break; @@ -462,10 +465,12 @@ ProcedureCreate(const char *procedureName, if (isnull) proargmodes = PointerGetDatum(NULL); /* just to be sure */ - n_old_arg_names = get_func_input_arg_names(proargnames, + n_old_arg_names = get_func_input_arg_names(prokind, + proargnames, proargmodes, &old_arg_names); - n_new_arg_names = get_func_input_arg_names(parameterNames, + n_new_arg_names = get_func_input_arg_names(prokind, + parameterNames, parameterModes, &new_arg_names); for (j = 0; j < n_old_arg_names; j++) diff --git a/src/backend/commands/functioncmds.c b/src/backend/commands/functioncmds.c index e236581a8e06..c3ce480c8f56 100644 --- a/src/backend/commands/functioncmds.c +++ b/src/backend/commands/functioncmds.c @@ -194,8 +194,8 @@ interpret_function_parameter_list(ParseState *pstate, Oid *requiredResultType) { int parameterCount = list_length(parameters); - Oid *inTypes; - int inCount = 0; + Oid *sigArgTypes; + int sigArgCount = 0; Datum *allTypes; Datum *paramModes; Datum *paramNames; @@ -209,7 +209,7 @@ interpret_function_parameter_list(ParseState *pstate, *variadicArgType = InvalidOid; /* default result */ *requiredResultType = InvalidOid; /* default result */ - inTypes = (Oid *) palloc(parameterCount * sizeof(Oid)); + sigArgTypes = (Oid *) palloc(parameterCount * sizeof(Oid)); allTypes = (Datum *) palloc(parameterCount * sizeof(Datum)); paramModes = (Datum *) palloc(parameterCount * sizeof(Datum)); paramNames = (Datum *) palloc0(parameterCount * sizeof(Datum)); @@ -281,25 +281,21 @@ interpret_function_parameter_list(ParseState *pstate, errmsg("functions cannot accept set arguments"))); } - if (objtype == OBJECT_PROCEDURE) - { - if (fp->mode == FUNC_PARAM_OUT) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("procedures cannot have OUT arguments"), - errhint("INOUT arguments are permitted."))); - } - /* handle input parameters */ if (fp->mode != FUNC_PARAM_OUT && fp->mode != FUNC_PARAM_TABLE) + isinput = true; + + /* handle signature parameters */ + if (fp->mode == FUNC_PARAM_IN || fp->mode == FUNC_PARAM_INOUT || + (objtype == OBJECT_PROCEDURE && fp->mode == FUNC_PARAM_OUT) || + fp->mode == FUNC_PARAM_VARIADIC) { - /* other input parameters can't follow a VARIADIC parameter */ + /* other signature parameters can't follow a VARIADIC parameter */ if (varCount > 0) ereport(ERROR, (errcode(ERRCODE_INVALID_FUNCTION_DEFINITION), - errmsg("VARIADIC parameter must be the last input parameter"))); - inTypes[inCount++] = toid; - isinput = true; + errmsg("VARIADIC parameter must be the last signature parameter"))); + sigArgTypes[sigArgCount++] = toid; } /* handle output parameters */ @@ -429,7 +425,7 @@ interpret_function_parameter_list(ParseState *pstate, } /* Now construct the proper outputs as needed */ - *parameterTypes = buildoidvector(inTypes, inCount); + *parameterTypes = buildoidvector(sigArgTypes, sigArgCount); if (outCount > 0 || varCount > 0) { @@ -2067,6 +2063,9 @@ ExecuteCallStmt(CallStmt *stmt, ParamListInfo params, bool atomic, DestReceiver int nargs; int i; AclResult aclresult; + Oid *argtypes; + char **argnames; + char *argmodes; FmgrInfo flinfo; CallContext *callcontext; EState *estate; @@ -2127,6 +2126,8 @@ ExecuteCallStmt(CallStmt *stmt, ParamListInfo params, bool atomic, DestReceiver tp); nargs = list_length(fexpr->args); + get_func_arg_info(tp, &argtypes, &argnames, &argmodes); + ReleaseSysCache(tp); /* safety check; see ExecInitFunc() */ @@ -2156,16 +2157,24 @@ ExecuteCallStmt(CallStmt *stmt, ParamListInfo params, bool atomic, DestReceiver i = 0; foreach(lc, fexpr->args) { - ExprState *exprstate; - Datum val; - bool isnull; + if (argmodes && argmodes[i] == PROARGMODE_OUT) + { + fcinfo->args[i].value = 0; + fcinfo->args[i].isnull = true; + } + else + { + ExprState *exprstate; + Datum val; + bool isnull; - exprstate = ExecPrepareExpr(lfirst(lc), estate); + exprstate = ExecPrepareExpr(lfirst(lc), estate); - val = ExecEvalExprSwitchContext(exprstate, econtext, &isnull); + val = ExecEvalExprSwitchContext(exprstate, econtext, &isnull); - fcinfo->args[i].value = val; - fcinfo->args[i].isnull = isnull; + fcinfo->args[i].value = val; + fcinfo->args[i].isnull = isnull; + } i++; } diff --git a/src/backend/executor/functions.c b/src/backend/executor/functions.c index f940f48c6dae..bf00a9c1e8da 100644 --- a/src/backend/executor/functions.c +++ b/src/backend/executor/functions.c @@ -259,7 +259,8 @@ prepare_sql_fn_parse_info(HeapTuple procedureTuple, if (isNull) proargmodes = PointerGetDatum(NULL); /* just to be sure */ - n_arg_names = get_func_input_arg_names(proargnames, proargmodes, + n_arg_names = get_func_input_arg_names(procedureStruct->prokind, + proargnames, proargmodes, &pinfo->argnames); /* Paranoia: ignore the result if too few array entries */ diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 17653ef3a790..0d101d817154 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -166,7 +166,7 @@ static RoleSpec *makeRoleSpec(RoleSpecType type, int location); static void check_qualified_name(List *names, core_yyscan_t yyscanner); static List *check_func_name(List *names, core_yyscan_t yyscanner); static List *check_indirection(List *indirection, core_yyscan_t yyscanner); -static List *extractArgTypes(List *parameters); +static List *extractArgTypes(ObjectType objtype, List *parameters); static List *extractAggrArgTypes(List *aggrargs); static List *makeOrderedSetArgs(List *directargs, List *orderedargs, core_yyscan_t yyscanner); @@ -375,8 +375,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type privilege %type privileges privilege_list %type privilege_target -%type function_with_argtypes aggregate_with_argtypes operator_with_argtypes -%type function_with_argtypes_list aggregate_with_argtypes_list operator_with_argtypes_list +%type function_with_argtypes aggregate_with_argtypes operator_with_argtypes procedure_with_argtypes function_with_argtypes_common +%type function_with_argtypes_list aggregate_with_argtypes_list operator_with_argtypes_list procedure_with_argtypes_list %type defacl_privilege_target %type DefACLOption %type DefACLOptionList @@ -4623,7 +4623,7 @@ AlterExtensionContentsStmt: n->object = (Node *) lcons(makeString($9), $7); $$ = (Node *)n; } - | ALTER EXTENSION name add_drop PROCEDURE function_with_argtypes + | ALTER EXTENSION name add_drop PROCEDURE procedure_with_argtypes { AlterExtensionContentsStmt *n = makeNode(AlterExtensionContentsStmt); n->extname = $3; @@ -4632,7 +4632,7 @@ AlterExtensionContentsStmt: n->object = (Node *) $6; $$ = (Node *)n; } - | ALTER EXTENSION name add_drop ROUTINE function_with_argtypes + | ALTER EXTENSION name add_drop ROUTINE procedure_with_argtypes { AlterExtensionContentsStmt *n = makeNode(AlterExtensionContentsStmt); n->extname = $3; @@ -6365,7 +6365,7 @@ CommentStmt: n->comment = $8; $$ = (Node *) n; } - | COMMENT ON PROCEDURE function_with_argtypes IS comment_text + | COMMENT ON PROCEDURE procedure_with_argtypes IS comment_text { CommentStmt *n = makeNode(CommentStmt); n->objtype = OBJECT_PROCEDURE; @@ -6373,7 +6373,7 @@ CommentStmt: n->comment = $6; $$ = (Node *) n; } - | COMMENT ON ROUTINE function_with_argtypes IS comment_text + | COMMENT ON ROUTINE procedure_with_argtypes IS comment_text { CommentStmt *n = makeNode(CommentStmt); n->objtype = OBJECT_ROUTINE; @@ -6519,7 +6519,7 @@ SecLabelStmt: n->label = $9; $$ = (Node *) n; } - | SECURITY LABEL opt_provider ON PROCEDURE function_with_argtypes + | SECURITY LABEL opt_provider ON PROCEDURE procedure_with_argtypes IS security_label { SecLabelStmt *n = makeNode(SecLabelStmt); @@ -6880,7 +6880,7 @@ privilege_target: n->objs = $2; $$ = n; } - | PROCEDURE function_with_argtypes_list + | PROCEDURE procedure_with_argtypes_list { PrivTarget *n = (PrivTarget *) palloc(sizeof(PrivTarget)); n->targtype = ACL_TARGET_OBJECT; @@ -6888,7 +6888,7 @@ privilege_target: n->objs = $2; $$ = n; } - | ROUTINE function_with_argtypes_list + | ROUTINE procedure_with_argtypes_list { PrivTarget *n = (PrivTarget *) palloc(sizeof(PrivTarget)); n->targtype = ACL_TARGET_OBJECT; @@ -7409,20 +7409,33 @@ function_with_argtypes_list: { $$ = lappend($1, $3); } ; +procedure_with_argtypes_list: + procedure_with_argtypes { $$ = list_make1($1); } + | procedure_with_argtypes_list ',' procedure_with_argtypes + { $$ = lappend($1, $3); } + ; + function_with_argtypes: func_name func_args { ObjectWithArgs *n = makeNode(ObjectWithArgs); n->objname = $1; - n->objargs = extractArgTypes($2); + n->objargs = extractArgTypes(OBJECT_FUNCTION, $2); $$ = n; } + | function_with_argtypes_common + { + $$ = $1; + } + ; + +function_with_argtypes_common: /* * Because of reduce/reduce conflicts, we can't use func_name * below, but we can write it out the long way, which actually * allows more cases. */ - | type_func_name_keyword + type_func_name_keyword { ObjectWithArgs *n = makeNode(ObjectWithArgs); n->objname = list_make1(makeString(pstrdup($1))); @@ -7446,6 +7459,24 @@ function_with_argtypes: } ; +/* + * This is different from function_with_argtypes in the call to + * extractArgTypes(). + */ +procedure_with_argtypes: + func_name func_args + { + ObjectWithArgs *n = makeNode(ObjectWithArgs); + n->objname = $1; + n->objargs = extractArgTypes(OBJECT_PROCEDURE, $2); + $$ = n; + } + | function_with_argtypes_common + { + $$ = $1; + } + ; + /* * func_args_with_defaults is separate because we only want to accept * defaults in CREATE FUNCTION, not in ALTER etc. @@ -7824,7 +7855,7 @@ AlterFunctionStmt: n->actions = $4; $$ = (Node *) n; } - | ALTER PROCEDURE function_with_argtypes alterfunc_opt_list opt_restrict + | ALTER PROCEDURE procedure_with_argtypes alterfunc_opt_list opt_restrict { AlterFunctionStmt *n = makeNode(AlterFunctionStmt); n->objtype = OBJECT_PROCEDURE; @@ -7832,7 +7863,7 @@ AlterFunctionStmt: n->actions = $4; $$ = (Node *) n; } - | ALTER ROUTINE function_with_argtypes alterfunc_opt_list opt_restrict + | ALTER ROUTINE procedure_with_argtypes alterfunc_opt_list opt_restrict { AlterFunctionStmt *n = makeNode(AlterFunctionStmt); n->objtype = OBJECT_ROUTINE; @@ -7888,7 +7919,7 @@ RemoveFuncStmt: n->concurrent = false; $$ = (Node *)n; } - | DROP PROCEDURE function_with_argtypes_list opt_drop_behavior + | DROP PROCEDURE procedure_with_argtypes_list opt_drop_behavior { DropStmt *n = makeNode(DropStmt); n->removeType = OBJECT_PROCEDURE; @@ -7898,7 +7929,7 @@ RemoveFuncStmt: n->concurrent = false; $$ = (Node *)n; } - | DROP PROCEDURE IF_P EXISTS function_with_argtypes_list opt_drop_behavior + | DROP PROCEDURE IF_P EXISTS procedure_with_argtypes_list opt_drop_behavior { DropStmt *n = makeNode(DropStmt); n->removeType = OBJECT_PROCEDURE; @@ -7908,7 +7939,7 @@ RemoveFuncStmt: n->concurrent = false; $$ = (Node *)n; } - | DROP ROUTINE function_with_argtypes_list opt_drop_behavior + | DROP ROUTINE procedure_with_argtypes_list opt_drop_behavior { DropStmt *n = makeNode(DropStmt); n->removeType = OBJECT_ROUTINE; @@ -7918,7 +7949,7 @@ RemoveFuncStmt: n->concurrent = false; $$ = (Node *)n; } - | DROP ROUTINE IF_P EXISTS function_with_argtypes_list opt_drop_behavior + | DROP ROUTINE IF_P EXISTS procedure_with_argtypes_list opt_drop_behavior { DropStmt *n = makeNode(DropStmt); n->removeType = OBJECT_ROUTINE; @@ -8393,7 +8424,7 @@ RenameStmt: ALTER AGGREGATE aggregate_with_argtypes RENAME TO name n->missing_ok = true; $$ = (Node *)n; } - | ALTER PROCEDURE function_with_argtypes RENAME TO name + | ALTER PROCEDURE procedure_with_argtypes RENAME TO name { RenameStmt *n = makeNode(RenameStmt); n->renameType = OBJECT_PROCEDURE; @@ -8411,7 +8442,7 @@ RenameStmt: ALTER AGGREGATE aggregate_with_argtypes RENAME TO name n->missing_ok = false; $$ = (Node *)n; } - | ALTER ROUTINE function_with_argtypes RENAME TO name + | ALTER ROUTINE procedure_with_argtypes RENAME TO name { RenameStmt *n = makeNode(RenameStmt); n->renameType = OBJECT_ROUTINE; @@ -8822,7 +8853,7 @@ AlterObjectDependsStmt: n->remove = $4; $$ = (Node *)n; } - | ALTER PROCEDURE function_with_argtypes opt_no DEPENDS ON EXTENSION name + | ALTER PROCEDURE procedure_with_argtypes opt_no DEPENDS ON EXTENSION name { AlterObjectDependsStmt *n = makeNode(AlterObjectDependsStmt); n->objectType = OBJECT_PROCEDURE; @@ -8831,7 +8862,7 @@ AlterObjectDependsStmt: n->remove = $4; $$ = (Node *)n; } - | ALTER ROUTINE function_with_argtypes opt_no DEPENDS ON EXTENSION name + | ALTER ROUTINE procedure_with_argtypes opt_no DEPENDS ON EXTENSION name { AlterObjectDependsStmt *n = makeNode(AlterObjectDependsStmt); n->objectType = OBJECT_ROUTINE; @@ -8962,7 +8993,7 @@ AlterObjectSchemaStmt: n->missing_ok = false; $$ = (Node *)n; } - | ALTER PROCEDURE function_with_argtypes SET SCHEMA name + | ALTER PROCEDURE procedure_with_argtypes SET SCHEMA name { AlterObjectSchemaStmt *n = makeNode(AlterObjectSchemaStmt); n->objectType = OBJECT_PROCEDURE; @@ -8971,7 +9002,7 @@ AlterObjectSchemaStmt: n->missing_ok = false; $$ = (Node *)n; } - | ALTER ROUTINE function_with_argtypes SET SCHEMA name + | ALTER ROUTINE procedure_with_argtypes SET SCHEMA name { AlterObjectSchemaStmt *n = makeNode(AlterObjectSchemaStmt); n->objectType = OBJECT_ROUTINE; @@ -9273,7 +9304,7 @@ AlterOwnerStmt: ALTER AGGREGATE aggregate_with_argtypes OWNER TO RoleSpec n->newowner = $9; $$ = (Node *)n; } - | ALTER PROCEDURE function_with_argtypes OWNER TO RoleSpec + | ALTER PROCEDURE procedure_with_argtypes OWNER TO RoleSpec { AlterOwnerStmt *n = makeNode(AlterOwnerStmt); n->objectType = OBJECT_PROCEDURE; @@ -9281,7 +9312,7 @@ AlterOwnerStmt: ALTER AGGREGATE aggregate_with_argtypes OWNER TO RoleSpec n->newowner = $6; $$ = (Node *)n; } - | ALTER ROUTINE function_with_argtypes OWNER TO RoleSpec + | ALTER ROUTINE procedure_with_argtypes OWNER TO RoleSpec { AlterOwnerStmt *n = makeNode(AlterOwnerStmt); n->objectType = OBJECT_ROUTINE; @@ -16218,13 +16249,14 @@ check_indirection(List *indirection, core_yyscan_t yyscanner) } /* extractArgTypes() + * * Given a list of FunctionParameter nodes, extract a list of just the - * argument types (TypeNames) for input parameters only. This is what - * is needed to look up an existing function, which is what is wanted by - * the productions that use this call. + * argument types (TypeNames) for signature parameters only (e.g., only input + * parameters for functions). This is what is needed to look up an existing + * function, which is what is wanted by the productions that use this call. */ static List * -extractArgTypes(List *parameters) +extractArgTypes(ObjectType objtype, List *parameters) { List *result = NIL; ListCell *i; @@ -16233,7 +16265,7 @@ extractArgTypes(List *parameters) { FunctionParameter *p = (FunctionParameter *) lfirst(i); - if (p->mode != FUNC_PARAM_OUT && p->mode != FUNC_PARAM_TABLE) + if ((p->mode != FUNC_PARAM_OUT || objtype == OBJECT_PROCEDURE) && p->mode != FUNC_PARAM_TABLE) result = lappend(result, p->argType); } return result; @@ -16246,7 +16278,7 @@ static List * extractAggrArgTypes(List *aggrargs) { Assert(list_length(aggrargs) == 2); - return extractArgTypes((List *) linitial(aggrargs)); + return extractArgTypes(OBJECT_AGGREGATE, (List *) linitial(aggrargs)); } /* makeOrderedSetArgs() diff --git a/src/backend/utils/fmgr/funcapi.c b/src/backend/utils/fmgr/funcapi.c index 78ed85720385..b9efa7729152 100644 --- a/src/backend/utils/fmgr/funcapi.c +++ b/src/backend/utils/fmgr/funcapi.c @@ -1233,7 +1233,8 @@ get_func_trftypes(HeapTuple procTup, * are set to NULL. You don't get anything if proargnames is NULL. */ int -get_func_input_arg_names(Datum proargnames, Datum proargmodes, +get_func_input_arg_names(char prokind, + Datum proargnames, Datum proargmodes, char ***arg_names) { ArrayType *arr; @@ -1291,6 +1292,7 @@ get_func_input_arg_names(Datum proargnames, Datum proargmodes, if (argmodes == NULL || argmodes[i] == PROARGMODE_IN || argmodes[i] == PROARGMODE_INOUT || + (argmodes[i] == PROARGMODE_OUT && prokind == PROKIND_PROCEDURE) || argmodes[i] == PROARGMODE_VARIADIC) { char *pname = TextDatumGetCString(argnames[i]); diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index b50fa25dbd86..268c81089653 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -91,7 +91,7 @@ CATALOG(pg_proc,1255,ProcedureRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(81,Proce * proargtypes */ - /* parameter types (excludes OUT params) */ + /* parameter types (excludes OUT params of functions) */ oidvector proargtypes BKI_LOOKUP(pg_type) BKI_FORCE_NOT_NULL; #ifdef CATALOG_VARLEN diff --git a/src/include/funcapi.h b/src/include/funcapi.h index b047acdc1a85..2f46442087e2 100644 --- a/src/include/funcapi.h +++ b/src/include/funcapi.h @@ -172,7 +172,8 @@ extern int get_func_arg_info(HeapTuple procTup, Oid **p_argtypes, char ***p_argnames, char **p_argmodes); -extern int get_func_input_arg_names(Datum proargnames, Datum proargmodes, +extern int get_func_input_arg_names(char prokind, + Datum proargnames, Datum proargmodes, char ***arg_names); extern int get_func_trftypes(HeapTuple procTup, Oid **p_trftypes); diff --git a/src/pl/plperl/expected/plperl_call.out b/src/pl/plperl/expected/plperl_call.out index c55c59cbceb3..a08b9ff795c6 100644 --- a/src/pl/plperl/expected/plperl_call.out +++ b/src/pl/plperl/expected/plperl_call.out @@ -48,6 +48,24 @@ CALL test_proc6(2, 3, 4); 6 | 8 (1 row) +-- OUT parameters +CREATE PROCEDURE test_proc9(IN a int, OUT b int) +LANGUAGE plperl +AS $$ +my ($a, $b) = @_; +elog(NOTICE, "a: $a, b: $b"); +return { b => $a * 2 }; +$$; +DO $$ +DECLARE _a int; _b int; +BEGIN + _a := 10; _b := 30; + CALL test_proc9(_a, _b); + RAISE NOTICE '_a: %, _b: %', _a, _b; +END +$$; +NOTICE: a: 10, b: +NOTICE: _a: 10, _b: 20 DROP PROCEDURE test_proc1; DROP PROCEDURE test_proc2; DROP PROCEDURE test_proc3; diff --git a/src/pl/plperl/sql/plperl_call.sql b/src/pl/plperl/sql/plperl_call.sql index 2cf5461fefde..bbea85fc9f50 100644 --- a/src/pl/plperl/sql/plperl_call.sql +++ b/src/pl/plperl/sql/plperl_call.sql @@ -51,6 +51,26 @@ $$; CALL test_proc6(2, 3, 4); +-- OUT parameters + +CREATE PROCEDURE test_proc9(IN a int, OUT b int) +LANGUAGE plperl +AS $$ +my ($a, $b) = @_; +elog(NOTICE, "a: $a, b: $b"); +return { b => $a * 2 }; +$$; + +DO $$ +DECLARE _a int; _b int; +BEGIN + _a := 10; _b := 30; + CALL test_proc9(_a, _b); + RAISE NOTICE '_a: %, _b: %', _a, _b; +END +$$; + + DROP PROCEDURE test_proc1; DROP PROCEDURE test_proc2; DROP PROCEDURE test_proc3; diff --git a/src/pl/plpgsql/src/expected/plpgsql_call.out b/src/pl/plpgsql/src/expected/plpgsql_call.out index d9c88e85c8d8..973857161105 100644 --- a/src/pl/plpgsql/src/expected/plpgsql_call.out +++ b/src/pl/plpgsql/src/expected/plpgsql_call.out @@ -264,6 +264,25 @@ END $$; ERROR: procedure parameter "c" is an output parameter but corresponding argument is not writable CONTEXT: PL/pgSQL function inline_code_block line 5 at CALL +-- OUT parameters +CREATE PROCEDURE test_proc9(IN a int, OUT b int) +LANGUAGE plpgsql +AS $$ +BEGIN + RAISE NOTICE 'a: %, b: %', a, b; + b := a * 2; +END; +$$; +DO $$ +DECLARE _a int; _b int; +BEGIN + _a := 10; _b := 30; + CALL test_proc9(_a, _b); + RAISE NOTICE '_a: %, _b: %', _a, _b; +END +$$; +NOTICE: a: 10, b: +NOTICE: _a: 10, _b: 20 -- transition variable assignment TRUNCATE test1; CREATE FUNCTION triggerfunc1() RETURNS trigger diff --git a/src/pl/plpgsql/src/pl_comp.c b/src/pl/plpgsql/src/pl_comp.c index e7f4a5f291d1..344627da956f 100644 --- a/src/pl/plpgsql/src/pl_comp.c +++ b/src/pl/plpgsql/src/pl_comp.c @@ -458,6 +458,7 @@ do_compile(FunctionCallInfo fcinfo, /* Remember arguments in appropriate arrays */ if (argmode == PROARGMODE_IN || argmode == PROARGMODE_INOUT || + (argmode == PROARGMODE_OUT && function->fn_prokind == PROKIND_PROCEDURE) || argmode == PROARGMODE_VARIADIC) in_arg_varnos[num_in_args++] = argvariable->dno; if (argmode == PROARGMODE_OUT || diff --git a/src/pl/plpgsql/src/sql/plpgsql_call.sql b/src/pl/plpgsql/src/sql/plpgsql_call.sql index 4702bd14d12e..d506809ddbf6 100644 --- a/src/pl/plpgsql/src/sql/plpgsql_call.sql +++ b/src/pl/plpgsql/src/sql/plpgsql_call.sql @@ -237,6 +237,27 @@ END $$; +-- OUT parameters + +CREATE PROCEDURE test_proc9(IN a int, OUT b int) +LANGUAGE plpgsql +AS $$ +BEGIN + RAISE NOTICE 'a: %, b: %', a, b; + b := a * 2; +END; +$$; + +DO $$ +DECLARE _a int; _b int; +BEGIN + _a := 10; _b := 30; + CALL test_proc9(_a, _b); + RAISE NOTICE '_a: %, _b: %', _a, _b; +END +$$; + + -- transition variable assignment TRUNCATE test1; diff --git a/src/pl/plpython/expected/plpython_call.out b/src/pl/plpython/expected/plpython_call.out index 07ae04e98ba2..c3f3c8e95e57 100644 --- a/src/pl/plpython/expected/plpython_call.out +++ b/src/pl/plpython/expected/plpython_call.out @@ -52,6 +52,23 @@ CALL test_proc6(2, 3, 4); 6 | 8 (1 row) +-- OUT parameters +CREATE PROCEDURE test_proc9(IN a int, OUT b int) +LANGUAGE plpythonu +AS $$ +plpy.notice("a: %s, b: %s" % (a, b)) +return (a * 2,) +$$; +DO $$ +DECLARE _a int; _b int; +BEGIN + _a := 10; _b := 30; + CALL test_proc9(_a, _b); + RAISE NOTICE '_a: %, _b: %', _a, _b; +END +$$; +NOTICE: a: 10, b: None +NOTICE: _a: 10, _b: 20 DROP PROCEDURE test_proc1; DROP PROCEDURE test_proc2; DROP PROCEDURE test_proc3; diff --git a/src/pl/plpython/plpy_procedure.c b/src/pl/plpython/plpy_procedure.c index 9e1583961115..ec47f52e61d6 100644 --- a/src/pl/plpython/plpy_procedure.c +++ b/src/pl/plpython/plpy_procedure.c @@ -273,7 +273,7 @@ PLy_procedure_create(HeapTuple procTup, Oid fn_oid, bool is_trigger) /* proc->nargs was initialized to 0 above */ for (i = 0; i < total; i++) { - if (modes[i] != PROARGMODE_OUT && + if ((modes[i] != PROARGMODE_OUT || proc->is_procedure) && modes[i] != PROARGMODE_TABLE) (proc->nargs)++; } @@ -289,7 +289,7 @@ PLy_procedure_create(HeapTuple procTup, Oid fn_oid, bool is_trigger) Form_pg_type argTypeStruct; if (modes && - (modes[i] == PROARGMODE_OUT || + ((modes[i] == PROARGMODE_OUT && !proc->is_procedure) || modes[i] == PROARGMODE_TABLE)) continue; /* skip OUT arguments */ diff --git a/src/pl/plpython/sql/plpython_call.sql b/src/pl/plpython/sql/plpython_call.sql index 2f792f92bd78..46e89b1a9e1c 100644 --- a/src/pl/plpython/sql/plpython_call.sql +++ b/src/pl/plpython/sql/plpython_call.sql @@ -54,6 +54,25 @@ $$; CALL test_proc6(2, 3, 4); +-- OUT parameters + +CREATE PROCEDURE test_proc9(IN a int, OUT b int) +LANGUAGE plpythonu +AS $$ +plpy.notice("a: %s, b: %s" % (a, b)) +return (a * 2,) +$$; + +DO $$ +DECLARE _a int; _b int; +BEGIN + _a := 10; _b := 30; + CALL test_proc9(_a, _b); + RAISE NOTICE '_a: %, _b: %', _a, _b; +END +$$; + + DROP PROCEDURE test_proc1; DROP PROCEDURE test_proc2; DROP PROCEDURE test_proc3; diff --git a/src/pl/tcl/expected/pltcl_call.out b/src/pl/tcl/expected/pltcl_call.out index d290c8fbd05c..f0eb356cf23a 100644 --- a/src/pl/tcl/expected/pltcl_call.out +++ b/src/pl/tcl/expected/pltcl_call.out @@ -49,6 +49,23 @@ CALL test_proc6(2, 3, 4); 6 | 8 (1 row) +-- OUT parameters +CREATE PROCEDURE test_proc9(IN a int, OUT b int) +LANGUAGE pltcl +AS $$ +elog NOTICE "a: $1, b: $2" +return [list b [expr {$1 * 2}]] +$$; +DO $$ +DECLARE _a int; _b int; +BEGIN + _a := 10; _b := 30; + CALL test_proc9(_a, _b); + RAISE NOTICE '_a: %, _b: %', _a, _b; +END +$$; +NOTICE: a: 10, b: +NOTICE: _a: 10, _b: 20 DROP PROCEDURE test_proc1; DROP PROCEDURE test_proc2; DROP PROCEDURE test_proc3; diff --git a/src/pl/tcl/sql/pltcl_call.sql b/src/pl/tcl/sql/pltcl_call.sql index 95791d08beea..963277e1fb87 100644 --- a/src/pl/tcl/sql/pltcl_call.sql +++ b/src/pl/tcl/sql/pltcl_call.sql @@ -52,6 +52,25 @@ $$; CALL test_proc6(2, 3, 4); +-- OUT parameters + +CREATE PROCEDURE test_proc9(IN a int, OUT b int) +LANGUAGE pltcl +AS $$ +elog NOTICE "a: $1, b: $2" +return [list b [expr {$1 * 2}]] +$$; + +DO $$ +DECLARE _a int; _b int; +BEGIN + _a := 10; _b := 30; + CALL test_proc9(_a, _b); + RAISE NOTICE '_a: %, _b: %', _a, _b; +END +$$; + + DROP PROCEDURE test_proc1; DROP PROCEDURE test_proc2; DROP PROCEDURE test_proc3; diff --git a/src/test/regress/expected/create_procedure.out b/src/test/regress/expected/create_procedure.out index 211a42cefa03..3838fa2324da 100644 --- a/src/test/regress/expected/create_procedure.out +++ b/src/test/regress/expected/create_procedure.out @@ -146,6 +146,19 @@ AS $$ SELECT a = b; $$; CALL ptest7(least('a', 'b'), 'a'); +-- OUT parameters +CREATE PROCEDURE ptest9(OUT a int) +LANGUAGE SQL +AS $$ +INSERT INTO cp_test VALUES (1, 'a'); +SELECT 1; +$$; +CALL ptest9(NULL); + a +--- + 1 +(1 row) + -- various error cases CALL version(); -- error: not a procedure ERROR: version() is not a procedure @@ -165,9 +178,6 @@ CREATE PROCEDURE ptestx() LANGUAGE SQL STRICT AS $$ INSERT INTO cp_test VALUES ( ERROR: invalid attribute in procedure definition LINE 1: CREATE PROCEDURE ptestx() LANGUAGE SQL STRICT AS $$ INSERT I... ^ -CREATE PROCEDURE ptestx(OUT a int) LANGUAGE SQL AS $$ INSERT INTO cp_test VALUES (1, 'a') $$; -ERROR: procedures cannot have OUT arguments -HINT: INOUT arguments are permitted. ALTER PROCEDURE ptest1(text) STRICT; ERROR: invalid attribute in procedure definition LINE 1: ALTER PROCEDURE ptest1(text) STRICT; diff --git a/src/test/regress/sql/create_procedure.sql b/src/test/regress/sql/create_procedure.sql index 89b96d580ffa..2ef1c82ceabe 100644 --- a/src/test/regress/sql/create_procedure.sql +++ b/src/test/regress/sql/create_procedure.sql @@ -112,6 +112,18 @@ $$; CALL ptest7(least('a', 'b'), 'a'); +-- OUT parameters + +CREATE PROCEDURE ptest9(OUT a int) +LANGUAGE SQL +AS $$ +INSERT INTO cp_test VALUES (1, 'a'); +SELECT 1; +$$; + +CALL ptest9(NULL); + + -- various error cases CALL version(); -- error: not a procedure @@ -119,7 +131,6 @@ CALL sum(1); -- error: not a procedure CREATE PROCEDURE ptestx() LANGUAGE SQL WINDOW AS $$ INSERT INTO cp_test VALUES (1, 'a') $$; CREATE PROCEDURE ptestx() LANGUAGE SQL STRICT AS $$ INSERT INTO cp_test VALUES (1, 'a') $$; -CREATE PROCEDURE ptestx(OUT a int) LANGUAGE SQL AS $$ INSERT INTO cp_test VALUES (1, 'a') $$; ALTER PROCEDURE ptest1(text) STRICT; ALTER FUNCTION ptest1(text) VOLATILE; -- error: not a function From 9cc3d614a9eb6ce17c0f6f6bf77c5d431617267e Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 5 Oct 2020 11:42:33 -0400 Subject: [PATCH 247/589] Doc: fix parameter names in the docs of a couple of functions. The descriptions of make_interval() and pg_options_to_table() were randomly different from the reality embedded in pg_proc. (These are not all the discrepancies I found in a quick search, but the others perhaps require more discussion, since there's at least a case to be made for changing pg_proc not the docs.) make_interval issue noted by Thomas Kellerer. Discussion: https://postgr.es/m/7b154ef0-9f22-90b9-7734-4bf23686695b@gmx.net --- doc/src/sgml/func.sgml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index ec8451d1b9b9..e7cff980ddf9 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -8960,13 +8960,13 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); make_interval - make_interval ( year int - , month int - , week int - , day int - , hour int - , min int - , sec double precision + make_interval ( years int + , months int + , weeks int + , days int + , hours int + , mins int + , secs double precision ) interval @@ -22416,7 +22416,7 @@ SELECT currval(pg_get_serial_sequence('sometable', 'id')); pg_options_to_table - pg_options_to_table ( reloptions text[] ) + pg_options_to_table ( options_array text[] ) setof record ( option_name text, option_value text ) From 53c6daff4364219256119fcd79b2d71b57c13e37 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 5 Oct 2020 13:15:39 -0400 Subject: [PATCH 248/589] Fix two latent(?) bugs in equivclass.c. get_eclass_for_sort_expr() computes expr_relids and nullable_relids early on, even though they won't be needed unless we make a new EquivalenceClass, which we often don't. Aside from the probably-minor inefficiency, there's a memory management problem: these bitmapsets will be built in the caller's context, leading to dangling pointers if that is shorter-lived than root->planner_cxt. This would be a live bug if get_eclass_for_sort_expr() could be called with create_it = true during GEQO join planning. So far as I can find, the core code never does that, but it's hard to be sure that no extensions do, especially since the comments make it clear that that's supposed to be a supported case. Fix by not computing these values until we've switched into planner_cxt to build the new EquivalenceClass. generate_join_implied_equalities() uses inner_rel->relids to look up relevant eclasses, but it ought to be using nominal_inner_relids. This is presently harmless because a child RelOptInfo will always have exactly the same eclass_indexes as its topmost parent; but that might not be true forever, and anyway it makes the code confusing. The first of these is old (introduced by me in f3b3b8d5b), so back-patch to all supported branches. The second only dates to v13, but we might as well back-patch it to keep the code looking similar across branches. Discussion: https://postgr.es/m/1508010.1601832581@sss.pgh.pa.us --- src/backend/optimizer/path/equivclass.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index b68a5a0ec717..d3d826b790e8 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -634,12 +634,6 @@ get_eclass_for_sort_expr(PlannerInfo *root, */ expr = canonicalize_ec_expression(expr, opcintype, collation); - /* - * Get the precise set of nullable relids appearing in the expression. - */ - expr_relids = pull_varnos((Node *) expr); - nullable_relids = bms_intersect(nullable_relids, expr_relids); - /* * Scan through the existing EquivalenceClasses for a match */ @@ -716,6 +710,12 @@ get_eclass_for_sort_expr(PlannerInfo *root, if (newec->ec_has_volatile && sortref == 0) /* should not happen */ elog(ERROR, "volatile EquivalenceClass has no sortref"); + /* + * Get the precise set of nullable relids appearing in the expression. + */ + expr_relids = pull_varnos((Node *) expr); + nullable_relids = bms_intersect(nullable_relids, expr_relids); + newem = add_eq_member(newec, copyObject(expr), expr_relids, nullable_relids, false, opcintype); @@ -1171,9 +1171,9 @@ generate_join_implied_equalities(PlannerInfo *root, } /* - * Get all eclasses in common between inner_rel's relids and outer_relids + * Get all eclasses that mention both inner and outer sides of the join */ - matching_ecs = get_common_eclass_indexes(root, inner_rel->relids, + matching_ecs = get_common_eclass_indexes(root, nominal_inner_relids, outer_relids); i = -1; From 18c170a08ee23d03a06d235ea628fecb057d974f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 5 Oct 2020 13:40:28 -0400 Subject: [PATCH 249/589] Include the process PID in assertion-failure messages. This should help to identify what happened when studying the postmaster log after-the-fact. While here, clean up some old comments in the same function. Discussion: https://postgr.es/m/1568983.1601845687@sss.pgh.pa.us --- src/backend/utils/error/assert.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/backend/utils/error/assert.c b/src/backend/utils/error/assert.c index 4dfa3269238f..a8c0a8ec487c 100644 --- a/src/backend/utils/error/assert.c +++ b/src/backend/utils/error/assert.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * assert.c - * Assert code. + * Assert support code. * * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California @@ -10,9 +10,6 @@ * IDENTIFICATION * src/backend/utils/error/assert.c * - * NOTE - * This should eventually work with elog() - * *------------------------------------------------------------------------- */ #include "postgres.h" @@ -24,6 +21,10 @@ /* * ExceptionalCondition - Handles the failure of an Assert() + * + * We intentionally do not go through elog() here, on the grounds of + * wanting to minimize the amount of infrastructure that has to be + * working to report an assertion failure. */ void ExceptionalCondition(const char *conditionName, @@ -31,20 +32,21 @@ ExceptionalCondition(const char *conditionName, const char *fileName, int lineNumber) { + /* Report the failure on stderr (or local equivalent) */ if (!PointerIsValid(conditionName) || !PointerIsValid(fileName) || !PointerIsValid(errorType)) - write_stderr("TRAP: ExceptionalCondition: bad arguments\n"); + write_stderr("TRAP: ExceptionalCondition: bad arguments in PID %d\n", + (int) getpid()); else - { - write_stderr("TRAP: %s(\"%s\", File: \"%s\", Line: %d)\n", + write_stderr("TRAP: %s(\"%s\", File: \"%s\", Line: %d, PID: %d)\n", errorType, conditionName, - fileName, lineNumber); - } + fileName, lineNumber, (int) getpid()); /* Usually this shouldn't be needed, but make sure the msg went out */ fflush(stderr); + /* If we have support for it, dump a simple backtrace */ #ifdef HAVE_BACKTRACE_SYMBOLS { void *buf[100]; @@ -55,12 +57,12 @@ ExceptionalCondition(const char *conditionName, } #endif -#ifdef SLEEP_ON_ASSERT - /* - * It would be nice to use pg_usleep() here, but only does 2000 sec or 33 - * minutes, which seems too short. + * If configured to do so, sleep indefinitely to allow user to attach a + * debugger. It would be nice to use pg_usleep() here, but that can sleep + * at most 2G usec or ~33 minutes, which seems too short. */ +#ifdef SLEEP_ON_ASSERT sleep(1000000); #endif From 253f1025da8c8d6e52f96f764658b76eb59290ad Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 5 Oct 2020 15:48:40 -0400 Subject: [PATCH 250/589] Overhaul pg_hba.conf clientcert's API Since PG 12, clientcert no longer supported only on/off, so remove 1/0 as possible values, and instead support only the text strings 'verify-ca' and 'verify-full'. Remove support for 'no-verify' since that is possible by just not specifying clientcert. Also, throw an error if 'verify-ca' is used and 'cert' authentication is used, since cert authentication requires verify-full. Also improve the docs. THIS IS A BACKWARD INCOMPATIBLE API CHANGE. Reported-by: Kyotaro Horiguchi Discussion: https://postgr.es/m/20200716.093012.1627751694396009053.horikyota.ntt@gmail.com Author: Kyotaro Horiguchi Backpatch-through: master --- doc/src/sgml/client-auth.sgml | 11 ++++------- doc/src/sgml/runtime.sgml | 5 ++--- src/backend/libpq/hba.c | 18 +++++++----------- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/doc/src/sgml/client-auth.sgml b/doc/src/sgml/client-auth.sgml index d62d1a061c9c..bad3c3469c95 100644 --- a/doc/src/sgml/client-auth.sgml +++ b/doc/src/sgml/client-auth.sgml @@ -2044,13 +2044,10 @@ host ... radius radiusservers="server1,server2" radiussecrets="""secret one"","" - In a pg_hba.conf record specifying certificate - authentication, the authentication option clientcert is - assumed to be verify-ca or verify-full, - and it cannot be turned off since a client certificate is necessary for this - method. What the cert method adds to the basic - clientcert certificate validity test is a check that the - cn attribute matches the database user name. + It is redundant to use the clientcert option with + cert authentication because cert + authentication is effectively trust authentication + with clientcert=verify-full. diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index 418aa3f85c7a..17e938148c5c 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -2345,9 +2345,8 @@ pg_dumpall -p 5432 | psql -d postgres -p 5433 The clientcert authentication option is available for all authentication methods, but only in pg_hba.conf lines specified as hostssl. When clientcert is - not specified or is set to no-verify, the server will still - verify any presented client certificates against its CA file, if one is - configured — but it will not insist that a client certificate be presented. + not specified, the server verifies the client certificate against its CA + file only if a client certificate is presented and the CA is configured. diff --git a/src/backend/libpq/hba.c b/src/backend/libpq/hba.c index 7b54ffc31ea1..4c86fb608748 100644 --- a/src/backend/libpq/hba.c +++ b/src/backend/libpq/hba.c @@ -1730,29 +1730,25 @@ parse_hba_auth_opt(char *name, char *val, HbaLine *hbaline, *err_msg = "clientcert can only be configured for \"hostssl\" rows"; return false; } - if (strcmp(val, "1") == 0 - || strcmp(val, "verify-ca") == 0) - { - hbaline->clientcert = clientCertCA; - } - else if (strcmp(val, "verify-full") == 0) + + if (strcmp(val, "verify-full") == 0) { hbaline->clientcert = clientCertFull; } - else if (strcmp(val, "0") == 0 - || strcmp(val, "no-verify") == 0) + else if (strcmp(val, "verify-ca") == 0) { if (hbaline->auth_method == uaCert) { ereport(elevel, (errcode(ERRCODE_CONFIG_FILE_ERROR), - errmsg("clientcert cannot be set to \"no-verify\" when using \"cert\" authentication"), + errmsg("clientcert only accepts \"verify-full\" when using \"cert\" authentication"), errcontext("line %d of configuration file \"%s\"", line_num, HbaFileName))); - *err_msg = "clientcert cannot be set to \"no-verify\" when using \"cert\" authentication"; + *err_msg = "clientcert can only be set to \"verify-full\" when using \"cert\" authentication"; return false; } - hbaline->clientcert = clientCertOff; + + hbaline->clientcert = clientCertCA; } else { From dd0a64ed435d4a266ed16adb8204e7222af6c164 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 5 Oct 2020 16:27:33 -0400 Subject: [PATCH 251/589] doc: show functions returning record types and use of ROWS FROM Previously it was unclear exactly how ROWS FROM behaved and how to cast the data types of columns returned by FROM functions. Also document that only non-OUT record functions can have their columns cast to data types. Reported-by: guyren@gmail.com Discussion: https://postgr.es/m/158638264419.662.2482095087061084020@wrigleys.postgresql.org Backpatch-through: 9.5 --- doc/src/sgml/queries.sgml | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/queries.sgml b/doc/src/sgml/queries.sgml index 875a4d84de02..77fb1991aebd 100644 --- a/doc/src/sgml/queries.sgml +++ b/doc/src/sgml/queries.sgml @@ -762,7 +762,8 @@ SELECT * FROM vw_getfoo; In some cases it is useful to define table functions that can return different column sets depending on how they are invoked. To support this, the table function can be declared as returning - the pseudo-type record. When such a function is used in + the pseudo-type record with no OUT + parameters. When such a function is used in a query, the expected row structure must be specified in the query itself, so that the system can know how to parse and plan the query. This syntax looks like: @@ -803,6 +804,33 @@ SELECT * that the parser knows, for example, what * should expand to. + + + This example uses ROWS FROM: + +SELECT * +FROM ROWS FROM + ( + json_to_recordset('[{"a":40,"b":"foo"},{"a":"100","b":"bar"}]') + AS (a INTEGER, b TEXT), + generate_series(1, 3) + ) AS x (p, q, s) +ORDER BY p; + + p | q | s +-----+-----+--- + 40 | foo | 1 + 100 | bar | 2 + | | 3 + + It joins two functions into a single FROM + target. json_to_recordset() is instructed + to return two columns, the first integer + and the second text. The result of + generate_series() is used directly. + The ORDER BY clause sorts the column values + as integers. + From 32a9c0bdf493cf5fc029ab44a22384d547290eff Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Tue, 6 Oct 2020 10:31:09 +0900 Subject: [PATCH 252/589] postgres_fdw: reestablish new connection if cached one is detected as broken. In postgres_fdw, once remote connections are established, they are cached and re-used for subsequent queries and transactions. There can be some cases where those cached connections are unavaiable, for example, by the restart of remote server. In these cases, previously an error was reported and the query accessing to remote server failed if new remote transaction failed to start because the cached connection was broken. This commit improves postgres_fdw so that new connection is remade if broken connection is detected when starting new remote transaction. This is useful to avoid unnecessary failure of queries when connection is broken but can be reestablished. Author: Bharath Rupireddy, tweaked a bit by Fujii Masao Reviewed-by: Ashutosh Bapat, Tatsuhito Kasahara, Fujii Masao Discussion: https://postgr.es/m/CALj2ACUAi23vf1WiHNar_LksM9EDOWXcbHCo-fD4Mbr1d=78YQ@mail.gmail.com --- contrib/postgres_fdw/connection.c | 54 ++++++++++++++----- .../postgres_fdw/expected/postgres_fdw.out | 48 +++++++++++++++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 41 ++++++++++++++ 3 files changed, 131 insertions(+), 12 deletions(-) diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c index 08daf26fdf08..76994f3820fc 100644 --- a/contrib/postgres_fdw/connection.c +++ b/contrib/postgres_fdw/connection.c @@ -108,6 +108,7 @@ PGconn * GetConnection(UserMapping *user, bool will_prep_stmt) { bool found; + volatile bool retry_conn = false; ConnCacheEntry *entry; ConnCacheKey key; @@ -159,23 +160,26 @@ GetConnection(UserMapping *user, bool will_prep_stmt) /* Reject further use of connections which failed abort cleanup. */ pgfdw_reject_incomplete_xact_state_change(entry); +retry: + /* * If the connection needs to be remade due to invalidation, disconnect as - * soon as we're out of all transactions. + * soon as we're out of all transactions. Also, if previous attempt to + * start new remote transaction failed on the cached connection, + * disconnect it to retry a new connection. */ - if (entry->conn != NULL && entry->invalidated && entry->xact_depth == 0) + if ((entry->conn != NULL && entry->invalidated && + entry->xact_depth == 0) || retry_conn) { - elog(DEBUG3, "closing connection %p for option changes to take effect", - entry->conn); + if (retry_conn) + elog(DEBUG3, "closing connection %p to reestablish a new one", + entry->conn); + else + elog(DEBUG3, "closing connection %p for option changes to take effect", + entry->conn); disconnect_pg_server(entry); } - /* - * We don't check the health of cached connection here, because it would - * require some overhead. Broken connection will be detected when the - * connection is actually used. - */ - /* * If cache entry doesn't have a connection, we have to establish a new * connection. (If connect_pg_server throws an error, the cache entry @@ -206,9 +210,35 @@ GetConnection(UserMapping *user, bool will_prep_stmt) } /* - * Start a new transaction or subtransaction if needed. + * We check the health of the cached connection here when starting a new + * remote transaction. If a broken connection is detected in the first + * attempt, we try to reestablish a new connection. If broken connection + * is detected again here, we give up getting a connection. */ - begin_remote_xact(entry); + PG_TRY(); + { + /* Start a new transaction or subtransaction if needed. */ + begin_remote_xact(entry); + retry_conn = false; + } + PG_CATCH(); + { + if (PQstatus(entry->conn) != CONNECTION_BAD || + entry->xact_depth > 0 || + retry_conn) + PG_RE_THROW(); + retry_conn = true; + } + PG_END_TRY(); + + if (retry_conn) + { + ereport(DEBUG3, + (errmsg_internal("could not start remote transaction on connection %p", + entry->conn)), + errdetail_internal("%s", pchomp(PQerrorMessage(entry->conn)))); + goto retry; + } /* Remember if caller will prepare statements */ entry->have_prep_stmt |= will_prep_stmt; diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 10e23d02ed56..2c5614073f94 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -8987,3 +8987,51 @@ PREPARE TRANSACTION 'fdw_tpc'; ERROR: cannot PREPARE a transaction that has operated on postgres_fdw foreign tables ROLLBACK; WARNING: there is no transaction in progress +-- =================================================================== +-- reestablish new connection +-- =================================================================== +-- Terminate the backend having the specified application_name and wait for +-- the termination to complete. +CREATE OR REPLACE PROCEDURE terminate_backend_and_wait(appname text) AS $$ +BEGIN + PERFORM pg_terminate_backend(pid) FROM pg_stat_activity + WHERE application_name = appname; + LOOP + PERFORM * FROM pg_stat_activity WHERE application_name = appname; + EXIT WHEN NOT FOUND; + PERFORM pg_sleep(1), pg_stat_clear_snapshot(); + END LOOP; +END; +$$ LANGUAGE plpgsql; +-- Change application_name of remote connection to special one +-- so that we can easily terminate the connection later. +ALTER SERVER loopback OPTIONS (application_name 'fdw_retry_check'); +SELECT 1 FROM ft1 LIMIT 1; + ?column? +---------- + 1 +(1 row) + +-- Terminate the remote connection. +CALL terminate_backend_and_wait('fdw_retry_check'); +-- This query should detect the broken connection when starting new remote +-- transaction, reestablish new connection, and then succeed. +BEGIN; +SELECT 1 FROM ft1 LIMIT 1; + ?column? +---------- + 1 +(1 row) + +-- If the query detects the broken connection when starting new remote +-- subtransaction, it doesn't reestablish new connection and should fail. +CALL terminate_backend_and_wait('fdw_retry_check'); +SAVEPOINT s; +SELECT 1 FROM ft1 LIMIT 1; -- should fail +ERROR: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +CONTEXT: remote SQL command: SAVEPOINT s2 +COMMIT; +-- Clean up +DROP PROCEDURE terminate_backend_and_wait(text); diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index 78156d10b486..4da1f78956e5 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2653,3 +2653,44 @@ SELECT count(*) FROM ft1; -- error here PREPARE TRANSACTION 'fdw_tpc'; ROLLBACK; + +-- =================================================================== +-- reestablish new connection +-- =================================================================== + +-- Terminate the backend having the specified application_name and wait for +-- the termination to complete. +CREATE OR REPLACE PROCEDURE terminate_backend_and_wait(appname text) AS $$ +BEGIN + PERFORM pg_terminate_backend(pid) FROM pg_stat_activity + WHERE application_name = appname; + LOOP + PERFORM * FROM pg_stat_activity WHERE application_name = appname; + EXIT WHEN NOT FOUND; + PERFORM pg_sleep(1), pg_stat_clear_snapshot(); + END LOOP; +END; +$$ LANGUAGE plpgsql; + +-- Change application_name of remote connection to special one +-- so that we can easily terminate the connection later. +ALTER SERVER loopback OPTIONS (application_name 'fdw_retry_check'); +SELECT 1 FROM ft1 LIMIT 1; + +-- Terminate the remote connection. +CALL terminate_backend_and_wait('fdw_retry_check'); + +-- This query should detect the broken connection when starting new remote +-- transaction, reestablish new connection, and then succeed. +BEGIN; +SELECT 1 FROM ft1 LIMIT 1; + +-- If the query detects the broken connection when starting new remote +-- subtransaction, it doesn't reestablish new connection and should fail. +CALL terminate_backend_and_wait('fdw_retry_check'); +SAVEPOINT s; +SELECT 1 FROM ft1 LIMIT 1; -- should fail +COMMIT; + +-- Clean up +DROP PROCEDURE terminate_backend_and_wait(text); From 1df2b50dbebb72a75708c3cfbc21bcaa71993147 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Mon, 5 Oct 2020 19:20:17 -0700 Subject: [PATCH 253/589] Try to unbreak 021_row_visibility.pl on mingw. Thanks to Andrew for proposing and testing this fix. It's possible that we should address this on a more fundamental basis, e.g. by configuring PerlIO to to CR/LF conversion for us, but this approach already exists in other places. And it's nice to unbreak the BF. Proposed-By: Andrew Dunstan Discussion: https://postgr.es/m/2355d1f0-0244-da9c-ef0c-7542b944e1ac@2ndQuadrant.com --- src/test/recovery/t/021_row_visibility.pl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test/recovery/t/021_row_visibility.pl b/src/test/recovery/t/021_row_visibility.pl index 95516b05d011..8a466e56e0fd 100644 --- a/src/test/recovery/t/021_row_visibility.pl +++ b/src/test/recovery/t/021_row_visibility.pl @@ -6,6 +6,7 @@ use PostgresNode; use TestLib; use Test::More tests => 10; +use Config; # Initialize primary node my $node_primary = get_new_node('primary'); @@ -167,6 +168,9 @@ sub send_query_and_wait $$psql{run}->pump_nb(); while (1) { + # See PostgresNode.pm's psql() + $$psql{stdout} =~ s/\r\n/\n/g if $Config{osname} eq 'msys'; + last if $$psql{stdout} =~ /$untl/; if ($psql_timeout->is_expired) From 0a3c864c32751fd29d021929cf70af421fd27370 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 6 Oct 2020 15:29:34 +0900 Subject: [PATCH 254/589] Fix compilation warning in xlog.c Oversight in 9d0bd95. Reported-by: Andres Freund Discussion: https://postgr.es/m/20201006023802.qqfi6m5bw5y77zql@alap3.anarazel.de --- src/backend/access/transam/xlog.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8f11b1b9de18..52a67b117015 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -10250,8 +10250,6 @@ xlog_redo(XLogReaderState *record) static void xlog_outrec(StringInfo buf, XLogReaderState *record) { - int block_id; - appendStringInfo(buf, "prev %X/%X; xid %u", (uint32) (XLogRecGetPrev(record) >> 32), (uint32) XLogRecGetPrev(record), From 5b36221c46f9ca9c5df6a6e108790b5c6c5d2cc3 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Tue, 6 Oct 2020 14:15:32 +0200 Subject: [PATCH 255/589] Expand installation documentation to cover binary installations Reviewed-By: David G. Johnston, Daniel Gustafsson --- doc/src/sgml/filelist.sgml | 1 + doc/src/sgml/install-binaries.sgml | 24 ++++++++++++++++++++++++ doc/src/sgml/install-windows.sgml | 3 ++- doc/src/sgml/installation.sgml | 2 +- doc/src/sgml/postgres.sgml | 1 + 5 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 doc/src/sgml/install-binaries.sgml diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 47271addc14d..38e8aa0bbf90 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -36,6 +36,7 @@ + diff --git a/doc/src/sgml/install-binaries.sgml b/doc/src/sgml/install-binaries.sgml new file mode 100644 index 000000000000..001c3c7be01f --- /dev/null +++ b/doc/src/sgml/install-binaries.sgml @@ -0,0 +1,24 @@ + + + Installation from Binaries + + + installation + binaries + + + + PostgreSQL is available in the form of binary + packages for most common operating systems today. When available, this is + the recommended way to install PostgreSQL for users of the system. Building + from source (see ) is only recommended for + people developing PostgreSQL or extensions. + + + + For an updated list of platforms providing binary packages, please visit + the download section on the PostgreSQL website at + and follow the + instructions for the specific platform. + + diff --git a/doc/src/sgml/install-windows.sgml b/doc/src/sgml/install-windows.sgml index 11f5957aca5b..844ef2cbd29a 100644 --- a/doc/src/sgml/install-windows.sgml +++ b/doc/src/sgml/install-windows.sgml @@ -11,7 +11,8 @@ It is recommended that most users download the binary distribution for Windows, available as a graphical installer package - from the PostgreSQL website. Building from source + from the PostgreSQL website at + . Building from source is only intended for people developing PostgreSQL or extensions. diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index b585f224085c..3f12acb7189d 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -21,7 +21,7 @@ documentation. See standalone-profile.xsl for details. PostgreSQL using the source code distribution. If you are installing a pre-packaged distribution, such as an RPM or Debian package, ignore this chapter - and read the packager's instructions instead. + and see instead. diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml index c41ce9499be4..730d5fdc3483 100644 --- a/doc/src/sgml/postgres.sgml +++ b/doc/src/sgml/postgres.sgml @@ -154,6 +154,7 @@ break is not needed in a wider output rendering. + &installbin; &installation; &installw; &runtime; From b8c4d385120cb7fe70006812234b2cbc950f4a61 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Tue, 6 Oct 2020 15:46:36 +0200 Subject: [PATCH 256/589] Clarify documentation around pg_dump -t option The behavior is different for different types of objects, so make that more clear. Author: Ian Barwick --- doc/src/sgml/ref/pg_dump.sgml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 1400cf877599..e783d5efa0c9 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -517,9 +517,7 @@ PostgreSQL documentation Dump only tables with names matching - pattern. - For this purpose, table includes views, materialized views, - sequences, and foreign tables. Multiple tables + pattern. Multiple tables can be selected by writing multiple switches. The pattern parameter is interpreted as a pattern according to the same rules used by @@ -531,6 +529,14 @@ PostgreSQL documentation below. + + As well as tables, this option can be used to dump views, materialized views, + foreign tables, and sequence definitions. However it will not dump the contents + of views or materialized views, and the contents of foreign tables will only be + dumped if the corresponding foreign server is specified with + . + + The and switches have no effect when is used, because tables selected by will From 1b22224945e1e7162fe92aecb1cd2b373fba24c2 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Tue, 6 Oct 2020 15:50:03 +0200 Subject: [PATCH 257/589] Further improvements on documentation for pg_dump -t Ian submitted an updated patch just as I was pushing the previous one, so use this newer wording instead. Author: Ian Barwick --- doc/src/sgml/ref/pg_dump.sgml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index e783d5efa0c9..a9d389455497 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -530,10 +530,10 @@ PostgreSQL documentation - As well as tables, this option can be used to dump views, materialized views, - foreign tables, and sequence definitions. However it will not dump the contents - of views or materialized views, and the contents of foreign tables will only be - dumped if the corresponding foreign server is specified with + As well as tables, this option can be used to dump the definition of matching + views, materialized views, foreign tables, and sequences. It will not dump the + contents of views or materialized views, and the contents of foreign tables will + only be dumped if the corresponding foreign server is specified with . From d7885b1f87fa066606acc54700edee28812be8c1 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 6 Oct 2020 11:43:53 -0400 Subject: [PATCH 258/589] Build EC members for child join rels in the right memory context. This patch prevents crashes or wrong plans when partition-wise joins are considered during GEQO planning, as a consequence of the EquivalenceClass data structures becoming corrupt after a GEQO context reset. A remaining problem is that successive GEQO cycles will make multiple copies of the required EC members, since add_child_join_rel_equivalences has no idea that such members might exist already. For now we'll just live with that. The lack of field complaints of crashes suggests that this is a mighty little-used situation. Back-patch to v12 where this code was introduced. Discussion: https://postgr.es/m/1683100.1601860653@sss.pgh.pa.us --- src/backend/optimizer/path/equivclass.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index d3d826b790e8..823422edad02 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -2380,6 +2380,7 @@ add_child_join_rel_equivalences(PlannerInfo *root, Relids top_parent_relids = child_joinrel->top_parent_relids; Relids child_relids = child_joinrel->relids; Bitmapset *matching_ecs; + MemoryContext oldcontext; int i; Assert(IS_JOIN_REL(child_joinrel) && IS_JOIN_REL(parent_joinrel)); @@ -2387,6 +2388,16 @@ add_child_join_rel_equivalences(PlannerInfo *root, /* We need consider only ECs that mention the parent joinrel */ matching_ecs = get_eclass_indexes_for_relids(root, top_parent_relids); + /* + * If we're being called during GEQO join planning, we still have to + * create any new EC members in the main planner context, to avoid having + * a corrupt EC data structure after the GEQO context is reset. This is + * problematic since we'll leak memory across repeated GEQO cycles. For + * now, though, bloat is better than crash. If it becomes a real issue + * we'll have to do something to avoid generating duplicate EC members. + */ + oldcontext = MemoryContextSwitchTo(root->planner_cxt); + i = -1; while ((i = bms_next_member(matching_ecs, i)) >= 0) { @@ -2486,6 +2497,8 @@ add_child_join_rel_equivalences(PlannerInfo *root, } } } + + MemoryContextSwitchTo(oldcontext); } From bc1fbc960bf5efbb692f4d1bf91bf9bc6390425a Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 6 Oct 2020 12:12:09 -0400 Subject: [PATCH 259/589] pg_upgrade; change major version comparisons to use <=, not < This makes checking for older major versions more consistent. Backpatch-through: 9.5 --- src/bin/pg_upgrade/check.c | 6 +++--- src/bin/pg_upgrade/controldata.c | 2 +- src/bin/pg_upgrade/exec.c | 6 +++--- src/bin/pg_upgrade/function.c | 4 ++-- src/bin/pg_upgrade/pg_upgrade.c | 6 +++--- src/bin/pg_upgrade/server.c | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 2f7aa632c52b..89f3b0d63cc3 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -275,7 +275,7 @@ check_cluster_versions(void) * upgrades */ - if (GET_MAJOR_VERSION(old_cluster.major_version) < 804) + if (GET_MAJOR_VERSION(old_cluster.major_version) <= 803) pg_fatal("This utility can only upgrade from PostgreSQL version 8.4 and later.\n"); /* Only current PG version is supported as a target */ @@ -312,7 +312,7 @@ check_cluster_compatibility(bool live_check) check_control_data(&old_cluster.controldata, &new_cluster.controldata); /* We read the real port number for PG >= 9.1 */ - if (live_check && GET_MAJOR_VERSION(old_cluster.major_version) < 901 && + if (live_check && GET_MAJOR_VERSION(old_cluster.major_version) <= 900 && old_cluster.port == DEF_PGUPORT) pg_fatal("When checking a pre-PG 9.1 live old server, " "you must specify the old server's port number.\n"); @@ -520,7 +520,7 @@ create_script_for_cluster_analyze(char **analyze_script_file_name) fprintf(script, "\"%s/vacuumdb\" %s--all --analyze-in-stages\n", new_cluster.bindir, user_specification.data); /* Did we copy the free space files? */ - if (GET_MAJOR_VERSION(old_cluster.major_version) < 804) + if (GET_MAJOR_VERSION(old_cluster.major_version) <= 803) fprintf(script, "\"%s/vacuumdb\" %s--all\n", new_cluster.bindir, user_specification.data); diff --git a/src/bin/pg_upgrade/controldata.c b/src/bin/pg_upgrade/controldata.c index 00d71e3a8a7c..39bcaa8fe1a2 100644 --- a/src/bin/pg_upgrade/controldata.c +++ b/src/bin/pg_upgrade/controldata.c @@ -180,7 +180,7 @@ get_control_data(ClusterInfo *cluster, bool live_check) } /* pg_resetxlog has been renamed to pg_resetwal in version 10 */ - if (GET_MAJOR_VERSION(cluster->bin_version) < 1000) + if (GET_MAJOR_VERSION(cluster->bin_version) <= 906) resetwal_bin = "pg_resetxlog\" -n"; else resetwal_bin = "pg_resetwal\" -n"; diff --git a/src/bin/pg_upgrade/exec.c b/src/bin/pg_upgrade/exec.c index b31cda8fec60..bdff13bb688f 100644 --- a/src/bin/pg_upgrade/exec.c +++ b/src/bin/pg_upgrade/exec.c @@ -341,13 +341,13 @@ check_data_dir(ClusterInfo *cluster) check_single_dir(pg_data, "pg_twophase"); /* pg_xlog has been renamed to pg_wal in v10 */ - if (GET_MAJOR_VERSION(cluster->major_version) < 1000) + if (GET_MAJOR_VERSION(cluster->major_version) <= 906) check_single_dir(pg_data, "pg_xlog"); else check_single_dir(pg_data, "pg_wal"); /* pg_clog has been renamed to pg_xact in v10 */ - if (GET_MAJOR_VERSION(cluster->major_version) < 1000) + if (GET_MAJOR_VERSION(cluster->major_version) <= 906) check_single_dir(pg_data, "pg_clog"); else check_single_dir(pg_data, "pg_xact"); @@ -387,7 +387,7 @@ check_bin_dir(ClusterInfo *cluster) get_bin_version(cluster); /* pg_resetxlog has been renamed to pg_resetwal in version 10 */ - if (GET_MAJOR_VERSION(cluster->bin_version) < 1000) + if (GET_MAJOR_VERSION(cluster->bin_version) <= 906) validate_exec(cluster->bindir, "pg_resetxlog"); else validate_exec(cluster->bindir, "pg_resetwal"); diff --git a/src/bin/pg_upgrade/function.c b/src/bin/pg_upgrade/function.c index d163cb2dde71..e0bc368e1e16 100644 --- a/src/bin/pg_upgrade/function.c +++ b/src/bin/pg_upgrade/function.c @@ -90,7 +90,7 @@ get_loadable_libraries(void) * http://archives.postgresql.org/pgsql-hackers/2012-03/msg01101.php * http://archives.postgresql.org/pgsql-bugs/2012-05/msg00206.php */ - if (GET_MAJOR_VERSION(old_cluster.major_version) < 901) + if (GET_MAJOR_VERSION(old_cluster.major_version) <= 900) { PGresult *res; @@ -218,7 +218,7 @@ check_loadable_libraries(void) * library name "plpython" in an old PG <= 9.1 cluster must look * for "plpython2" in the new cluster. */ - if (GET_MAJOR_VERSION(old_cluster.major_version) < 901 && + if (GET_MAJOR_VERSION(old_cluster.major_version) <= 900 && strcmp(lib, "$libdir/plpython") == 0) { lib = "$libdir/plpython2"; diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c index 70194eb0964f..1bc86e4205de 100644 --- a/src/bin/pg_upgrade/pg_upgrade.c +++ b/src/bin/pg_upgrade/pg_upgrade.c @@ -407,7 +407,7 @@ create_new_objects(void) * We don't have minmxids for databases or relations in pre-9.3 clusters, * so set those after we have restored the schema. */ - if (GET_MAJOR_VERSION(old_cluster.major_version) < 903) + if (GET_MAJOR_VERSION(old_cluster.major_version) <= 902) set_frozenxids(true); /* update new_cluster info now that we have objects in the databases */ @@ -466,9 +466,9 @@ copy_xact_xlog_xid(void) * Copy old commit logs to new data dir. pg_clog has been renamed to * pg_xact in post-10 clusters. */ - copy_subdir_files(GET_MAJOR_VERSION(old_cluster.major_version) < 1000 ? + copy_subdir_files(GET_MAJOR_VERSION(old_cluster.major_version) <= 906 ? "pg_clog" : "pg_xact", - GET_MAJOR_VERSION(new_cluster.major_version) < 1000 ? + GET_MAJOR_VERSION(new_cluster.major_version) <= 906 ? "pg_clog" : "pg_xact"); /* set the next transaction id and epoch of the new cluster */ diff --git a/src/bin/pg_upgrade/server.c b/src/bin/pg_upgrade/server.c index 7db3c1d51f2e..713509f54062 100644 --- a/src/bin/pg_upgrade/server.c +++ b/src/bin/pg_upgrade/server.c @@ -220,7 +220,7 @@ start_postmaster(ClusterInfo *cluster, bool report_and_exit_on_error) snprintf(socket_string + strlen(socket_string), sizeof(socket_string) - strlen(socket_string), " -c %s='%s'", - (GET_MAJOR_VERSION(cluster->major_version) < 903) ? + (GET_MAJOR_VERSION(cluster->major_version) <= 902) ? "unix_socket_directory" : "unix_socket_directories", cluster->sockdir); #endif From 069179767f79c03381056b23ded0a54156674456 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 6 Oct 2020 14:31:22 -0400 Subject: [PATCH 260/589] pg_upgrade: remove pre-8.4 code and >= 8.4 check We only support upgrading from >= 8.4 so no need for this code or tests. Reported-by: Magnus Hagander Discussion: https://postgr.es/m/CABUevEx-D0PNVe00tkeQRGennZQwDtBJn=493MJt-x6sppbUxA@mail.gmail.com Backpatch-through: 9.5 --- src/bin/pg_upgrade/check.c | 29 +++++++---------------------- src/bin/pg_upgrade/relfilenode.c | 16 ++++++---------- 2 files changed, 13 insertions(+), 32 deletions(-) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 89f3b0d63cc3..168058a873cb 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -234,18 +234,10 @@ void output_completion_banner(char *analyze_script_file_name, char *deletion_script_file_name) { - /* Did we copy the free space files? */ - if (GET_MAJOR_VERSION(old_cluster.major_version) >= 804) - pg_log(PG_REPORT, - "Optimizer statistics are not transferred by pg_upgrade so,\n" - "once you start the new server, consider running:\n" - " %s\n\n", analyze_script_file_name); - else - pg_log(PG_REPORT, - "Optimizer statistics and free space information are not transferred\n" - "by pg_upgrade so, once you start the new server, consider running:\n" - " %s\n\n", analyze_script_file_name); - + pg_log(PG_REPORT, + "Optimizer statistics are not transferred by pg_upgrade so,\n" + "once you start the new server, consider running:\n" + " %s\n\n", analyze_script_file_name); if (deletion_script_file_name) pg_log(PG_REPORT, @@ -275,7 +267,7 @@ check_cluster_versions(void) * upgrades */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 803) + if (GET_MAJOR_VERSION(old_cluster.major_version) < 804) pg_fatal("This utility can only upgrade from PostgreSQL version 8.4 and later.\n"); /* Only current PG version is supported as a target */ @@ -510,19 +502,12 @@ create_script_for_cluster_analyze(char **analyze_script_file_name) ECHO_QUOTE, ECHO_QUOTE); fprintf(script, "echo %sthis script and run:%s\n", ECHO_QUOTE, ECHO_QUOTE); - fprintf(script, "echo %s \"%s/vacuumdb\" %s--all %s%s\n", ECHO_QUOTE, - new_cluster.bindir, user_specification.data, - /* Did we copy the free space files? */ - (GET_MAJOR_VERSION(old_cluster.major_version) >= 804) ? - "--analyze-only" : "--analyze", ECHO_QUOTE); + fprintf(script, "echo %s \"%s/vacuumdb\" %s--all --analyze-only%s\n", ECHO_QUOTE, + new_cluster.bindir, user_specification.data, ECHO_QUOTE); fprintf(script, "echo%s\n\n", ECHO_BLANK); fprintf(script, "\"%s/vacuumdb\" %s--all --analyze-in-stages\n", new_cluster.bindir, user_specification.data); - /* Did we copy the free space files? */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 803) - fprintf(script, "\"%s/vacuumdb\" %s--all\n", new_cluster.bindir, - user_specification.data); fprintf(script, "echo%s\n\n", ECHO_BLANK); fprintf(script, "echo %sDone%s\n", diff --git a/src/bin/pg_upgrade/relfilenode.c b/src/bin/pg_upgrade/relfilenode.c index af9a021400a8..f76ddaaf3a16 100644 --- a/src/bin/pg_upgrade/relfilenode.c +++ b/src/bin/pg_upgrade/relfilenode.c @@ -163,16 +163,12 @@ transfer_single_new_db(FileNameMap *maps, int size, char *old_tablespace) /* transfer primary file */ transfer_relfile(&maps[mapnum], "", vm_must_add_frozenbit); - /* fsm/vm files added in PG 8.4 */ - if (GET_MAJOR_VERSION(old_cluster.major_version) >= 804) - { - /* - * Copy/link any fsm and vm files, if they exist - */ - transfer_relfile(&maps[mapnum], "_fsm", vm_must_add_frozenbit); - if (vm_crashsafe_match) - transfer_relfile(&maps[mapnum], "_vm", vm_must_add_frozenbit); - } + /* + * Copy/link any fsm and vm files, if they exist + */ + transfer_relfile(&maps[mapnum], "_fsm", vm_must_add_frozenbit); + if (vm_crashsafe_match) + transfer_relfile(&maps[mapnum], "_vm", vm_must_add_frozenbit); } } } From f07707099c17e7ff66eac7d38cbd1148672f7ee4 Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Wed, 7 Oct 2020 08:14:19 +0530 Subject: [PATCH 261/589] Display the names of missing columns in error during logical replication. In logical replication when a subscriber is missing some columns, it currently emits an error message that says "some" columns are missing, but it doesn't specify the missing column names. Change that to display missing column names which makes an error to be more informative to the user. We have decided not to backpatch this commit as this is a minor usability improvement and no user has reported this. Reported-by: Bharath Rupireddy Author: Bharath Rupireddy Reviewed-by: Kyotaro Horiguchi and Amit Kapila Discussion: https://postgr.es/m/CALj2ACVkW-EXH_4pmBK8tNeHRz5ksUC4WddGactuCjPiBch-cg@mail.gmail.com --- src/backend/replication/logical/relation.c | 55 ++++++++++++++++++---- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/src/backend/replication/logical/relation.c b/src/backend/replication/logical/relation.c index 2bb8e7d57b36..a6596f79a66a 100644 --- a/src/backend/replication/logical/relation.c +++ b/src/backend/replication/logical/relation.c @@ -228,6 +228,43 @@ logicalrep_rel_att_by_name(LogicalRepRelation *remoterel, const char *attname) return -1; } +/* + * Report error with names of the missing local relation column(s), if any. + */ +static void +logicalrep_report_missing_attrs(LogicalRepRelation *remoterel, + Bitmapset *missingatts) +{ + if (!bms_is_empty(missingatts)) + { + StringInfoData missingattsbuf; + int missingattcnt = 0; + int i; + + initStringInfo(&missingattsbuf); + + while ((i = bms_first_member(missingatts)) >= 0) + { + missingattcnt++; + if (missingattcnt == 1) + appendStringInfo(&missingattsbuf, _("\"%s\""), + remoterel->attnames[i]); + else + appendStringInfo(&missingattsbuf, _(", \"%s\""), + remoterel->attnames[i]); + } + + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg_plural("logical replication target relation \"%s.%s\" is missing replicated column: %s", + "logical replication target relation \"%s.%s\" is missing replicated columns: %s", + missingattcnt, + remoterel->nspname, + remoterel->relname, + missingattsbuf.data))); + } +} + /* * Open the local relation associated with the remote one. * @@ -286,11 +323,11 @@ logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode) if (!entry->localrelvalid) { Oid relid; - int found; Bitmapset *idkey; TupleDesc desc; MemoryContext oldctx; int i; + Bitmapset *missingatts; /* Try to find and lock the relation by name. */ relid = RangeVarGetRelid(makeRangeVar(remoterel->nspname, @@ -318,7 +355,8 @@ logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode) entry->attrmap = make_attrmap(desc->natts); MemoryContextSwitchTo(oldctx); - found = 0; + /* check and report missing attrs, if any */ + missingatts = bms_add_range(NULL, 0, remoterel->natts - 1); for (i = 0; i < desc->natts; i++) { int attnum; @@ -335,16 +373,13 @@ logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode) entry->attrmap->attnums[i] = attnum; if (attnum >= 0) - found++; + missingatts = bms_del_member(missingatts, attnum); } - /* TODO, detail message with names of missing columns */ - if (found < remoterel->natts) - ereport(ERROR, - (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("logical replication target relation \"%s.%s\" is missing " - "some replicated columns", - remoterel->nspname, remoterel->relname))); + logicalrep_report_missing_attrs(remoterel, missingatts); + + /* be tidy */ + bms_free(missingatts); /* * Check that replica identity matches. We allow for stricter replica From 9e5f1f21ad6a10662bc043046d1a2adea05007f6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 7 Oct 2020 12:50:54 -0400 Subject: [PATCH 262/589] Rethink recent fix for pg_dump's handling of extension config tables. Commit 3eb3d3e78 was a few bricks shy of a load: while it correctly set the table's "interesting" flag when deciding to dump the data of an extension config table, it was not correct to clear that flag if we concluded we shouldn't dump the data. This led to the crash reported in bug #16655, because in fact we'll traverse dumpTableSchema anyway for all extension tables (to see if they have user-added seclabels or RLS policies). The right thing to do is to force "interesting" true in makeTableDataInfo, and otherwise leave the flag alone. (Doing it there is more future-proof in case additional calls are added, and it also avoids setting the flag unnecessarily if that function decides the table is non-dumpable.) This investigation also showed that while only the --inserts code path had an obvious failure in the case considered by 3eb3d3e78, the COPY code path also has a problem with not having loaded table subsidiary data. That causes fmtCopyColumnList to silently return an empty string instead of the correct column list. That accidentally mostly works, which perhaps is why we didn't notice this before. It would only fail if the restore column order is different from the dump column order, which only happens in weird inheritance cases, so it's not surprising nobody had hit the case with an extension config table. Nonetheless, it's a bug, and it goes a long way back, not just to v12 where the --inserts code path started to have a problem with this. In hopes of catching such cases a bit sooner in future, add some Asserts that "interesting" has been set in both dumpTableData and dumpTableSchema. Adjust the test case added by 3eb3d3e78 so that it checks the COPY rather than INSERT form of that bug, allowing it to detect the longer-standing symptom. Per bug #16655 from Cameron Daniel. Back-patch to all supported branches. Discussion: https://postgr.es/m/16655-5c92d6b3a9438137@postgresql.org Discussion: https://postgr.es/m/18048b44-3414-b983-8c7c-9165b177900d@2ndQuadrant.com --- src/bin/pg_dump/pg_dump.c | 16 ++++--- src/test/modules/test_pg_dump/t/001_base.pl | 44 +++++++++++++------ .../test_pg_dump/test_pg_dump--1.0.sql | 6 +-- 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 76320468baa2..88bbbd9a9e1c 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -2098,8 +2098,6 @@ dumpTableData_insert(Archive *fout, void *dcontext) if (nfields == 0) continue; - Assert(tbinfo->attgenerated); - /* Emit a row heading */ if (rows_per_statement == 1) archputs(" (", fout); @@ -2260,6 +2258,9 @@ dumpTableData(Archive *fout, TableDataInfo *tdinfo) char *copyStmt; const char *copyFrom; + /* We had better have loaded per-column details about this table */ + Assert(tbinfo->interesting); + if (dopt->dump_inserts == 0) { /* Dump/restore using COPY */ @@ -2451,6 +2452,9 @@ makeTableDataInfo(DumpOptions *dopt, TableInfo *tbinfo) addObjectDependency(&tdinfo->dobj, tbinfo->dobj.dumpId); tbinfo->dataObj = tdinfo; + + /* Make sure that we'll collect per-column info for this table. */ + tbinfo->interesting = true; } /* @@ -8728,7 +8732,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) * Get info about table CHECK constraints. This is skipped for a * data-only dump, as it is only needed for table schemas. */ - if (!dopt->dataOnly && tbinfo->ncheck > 0) + if (tbinfo->ncheck > 0 && !dopt->dataOnly) { ConstraintInfo *constrs; int numConstrs; @@ -15538,10 +15542,12 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo) int j, k; + /* We had better have loaded per-column details about this table */ + Assert(tbinfo->interesting); + qrelname = pg_strdup(fmtId(tbinfo->dobj.name)); qualrelname = pg_strdup(fmtQualifiedDumpable(tbinfo)); - if (tbinfo->hasoids) pg_log_warning("WITH OIDS is not supported anymore (table \"%s\")", qrelname); @@ -17912,8 +17918,6 @@ processExtensionTables(Archive *fout, ExtensionInfo extinfo[], configtbl->dataObj->filtercond = pg_strdup(extconditionarray[j]); } } - - configtbl->interesting = dumpobj; } } if (extconfigarray) diff --git a/src/test/modules/test_pg_dump/t/001_base.pl b/src/test/modules/test_pg_dump/t/001_base.pl index 78aa07ce511a..b3227b855c0a 100644 --- a/src/test/modules/test_pg_dump/t/001_base.pl +++ b/src/test/modules/test_pg_dump/t/001_base.pl @@ -135,9 +135,17 @@ "$tempdir/defaults_tar_format.tar", ], }, + exclude_table => { + dump_cmd => [ + 'pg_dump', + '--exclude-table=regress_table_dumpable', + "--file=$tempdir/exclude_table.sql", + 'postgres', + ], + }, extension_schema => { dump_cmd => [ - 'pg_dump', '--schema=public', '--inserts', + 'pg_dump', '--schema=public', "--file=$tempdir/extension_schema.sql", 'postgres', ], }, @@ -225,6 +233,7 @@ clean_if_exists => 1, createdb => 1, defaults => 1, + exclude_table => 1, no_privs => 1, no_owner => 1,); @@ -317,11 +326,28 @@ regexp => qr/^ \QCREATE TABLE public.regress_pg_dump_table (\E \n\s+\Qcol1 integer NOT NULL,\E - \n\s+\Qcol2 integer\E + \n\s+\Qcol2 integer,\E + \n\s+\QCONSTRAINT regress_pg_dump_table_col2_check CHECK ((col2 > 0))\E \n\);\n/xm, like => { binary_upgrade => 1, }, }, + 'COPY public.regress_table_dumpable (col1)' => { + regexp => qr/^ + \QCOPY public.regress_table_dumpable (col1) FROM stdin;\E + \n/xm, + like => { + %full_runs, + data_only => 1, + section_data => 1, + extension_schema => 1, + }, + unlike => { + binary_upgrade => 1, + exclude_table => 1, + }, + }, + 'CREATE ACCESS METHOD regress_test_am' => { regexp => qr/^ \QCREATE ACCESS METHOD regress_test_am TYPE INDEX HANDLER bthandler;\E @@ -443,7 +469,8 @@ regexp => qr/^ \QCREATE TABLE regress_pg_dump_schema.test_table (\E \n\s+\Qcol1 integer,\E - \n\s+\Qcol2 integer\E + \n\s+\Qcol2 integer,\E + \n\s+\QCONSTRAINT test_table_col2_check CHECK ((col2 > 0))\E \n\);\n/xm, like => { binary_upgrade => 1, }, }, @@ -578,17 +605,6 @@ schema_only => 1, section_pre_data => 1, }, - }, - - # Dumpable object inside specific schema - 'INSERT INTO public.regress_table_dumpable VALUES (1);' => { - create_sql => 'INSERT INTO public.regress_table_dumpable VALUES (1);', - regexp => qr/^ - \QINSERT INTO public.regress_table_dumpable VALUES (1);\E - \n/xm, - like => { - extension_schema => 1, - }, },); ######################################### diff --git a/src/test/modules/test_pg_dump/test_pg_dump--1.0.sql b/src/test/modules/test_pg_dump/test_pg_dump--1.0.sql index 90e461ed3573..c7a35c3afa0f 100644 --- a/src/test/modules/test_pg_dump/test_pg_dump--1.0.sql +++ b/src/test/modules/test_pg_dump/test_pg_dump--1.0.sql @@ -5,7 +5,7 @@ CREATE TABLE regress_pg_dump_table ( col1 serial, - col2 int + col2 int check (col2 > 0) ); CREATE SEQUENCE regress_pg_dump_seq; @@ -14,7 +14,7 @@ CREATE SEQUENCE regress_seq_dumpable; SELECT pg_catalog.pg_extension_config_dump('regress_seq_dumpable', ''); CREATE TABLE regress_table_dumpable ( - col1 int + col1 int check (col1 > 0) ); SELECT pg_catalog.pg_extension_config_dump('regress_table_dumpable', ''); @@ -34,7 +34,7 @@ CREATE ACCESS METHOD regress_test_am TYPE INDEX HANDLER bthandler; -- this extension. CREATE TABLE regress_pg_dump_schema.test_table ( col1 int, - col2 int + col2 int check (col2 > 0) ); GRANT SELECT ON regress_pg_dump_schema.test_table TO regress_dump_test_role; From 6c05e5b77471dfadebe50ad4a8bdedef02ad0078 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 7 Oct 2020 13:27:33 -0400 Subject: [PATCH 263/589] Clean up after newly-added tests for pg_test_fsync and pg_test_timing. Oversight in 4d29e6dbd. --- src/bin/pg_test_fsync/Makefile | 1 + src/bin/pg_test_timing/Makefile | 1 + 2 files changed, 2 insertions(+) diff --git a/src/bin/pg_test_fsync/Makefile b/src/bin/pg_test_fsync/Makefile index c4f9ae066484..631d0f38a8e0 100644 --- a/src/bin/pg_test_fsync/Makefile +++ b/src/bin/pg_test_fsync/Makefile @@ -33,3 +33,4 @@ uninstall: clean distclean maintainer-clean: rm -f pg_test_fsync$(X) $(OBJS) + rm -rf tmp_check diff --git a/src/bin/pg_test_timing/Makefile b/src/bin/pg_test_timing/Makefile index 52994b4103c3..84d84c38aa86 100644 --- a/src/bin/pg_test_timing/Makefile +++ b/src/bin/pg_test_timing/Makefile @@ -33,3 +33,4 @@ uninstall: clean distclean maintainer-clean: rm -f pg_test_timing$(X) $(OBJS) + rm -rf tmp_check From 3db322eaab9688d57643b4d2a5f52b7f350ef46f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 7 Oct 2020 17:10:26 -0400 Subject: [PATCH 264/589] Prevent internal overflows in date-vs-timestamp and related comparisons. The date-vs-timestamp, date-vs-timestamptz, and timestamp-vs-timestamptz comparators all worked by promoting the first type to the second and then doing a simple same-type comparison. This works fine, except when the conversion result is out of range, in which case we throw an entirely avoidable error. The sources of such failures are (a) type date can represent dates much farther in the future than the timestamp types can; (b) timezone rotation might cause a just-in-range timestamp value to become a just-out-of-range timestamptz value. Up to now we just ignored these corner-case issues, but now we have an actual user complaint (bug #16657 from Huss EL-Sheikh), so let's do something about it. It turns out that commit 52ad1e659 already built all the necessary infrastructure to support error-free comparisons, but neglected to actually use it in the main-line code paths. Fix that, do a little bit of code style review, and remove the now-duplicate logic in jsonpath_exec.c. Back-patch to v13 where 52ad1e659 came in. We could take this back further by back-patching said infrastructure, but given the small number of complaints so far, I don't feel a great need to. Discussion: https://postgr.es/m/16657-cde2f876d8cc7971@postgresql.org --- src/backend/utils/adt/date.c | 222 +++++++++++-------------- src/backend/utils/adt/jsonpath_exec.c | 71 +------- src/backend/utils/adt/timestamp.c | 115 ++++++------- src/include/utils/date.h | 3 + src/include/utils/timestamp.h | 2 + src/test/regress/expected/horology.out | 72 ++++++++ src/test/regress/sql/horology.sql | 27 +++ 7 files changed, 260 insertions(+), 252 deletions(-) diff --git a/src/backend/utils/adt/date.c b/src/backend/utils/adt/date.c index 057051fa8550..a470cf890a20 100644 --- a/src/backend/utils/adt/date.c +++ b/src/backend/utils/adt/date.c @@ -554,15 +554,24 @@ date_mii(PG_FUNCTION_ARGS) /* * Promote date to timestamp. * - * On overflow error is thrown if 'overflow' is NULL. Otherwise, '*overflow' - * is set to -1 (+1) when result value exceed lower (upper) boundary and zero - * returned. + * On successful conversion, *overflow is set to zero if it's not NULL. + * + * If the date is finite but out of the valid range for timestamp, then: + * if overflow is NULL, we throw an out-of-range error. + * if overflow is not NULL, we store +1 or -1 there to indicate the sign + * of the overflow, and return the appropriate timestamp infinity. + * + * Note: *overflow = -1 is actually not possible currently, since both + * datatypes have the same lower bound, Julian day zero. */ Timestamp date2timestamp_opt_overflow(DateADT dateVal, int *overflow) { Timestamp result; + if (overflow) + *overflow = 0; + if (DATE_IS_NOBEGIN(dateVal)) TIMESTAMP_NOBEGIN(result); else if (DATE_IS_NOEND(dateVal)) @@ -570,7 +579,6 @@ date2timestamp_opt_overflow(DateADT dateVal, int *overflow) else { /* - * Date's range is wider than timestamp's, so check for boundaries. * Since dates have the same minimum values as timestamps, only upper * boundary need be checked for overflow. */ @@ -579,7 +587,8 @@ date2timestamp_opt_overflow(DateADT dateVal, int *overflow) if (overflow) { *overflow = 1; - return (Timestamp) 0; + TIMESTAMP_NOEND(result); + return result; } else { @@ -597,7 +606,7 @@ date2timestamp_opt_overflow(DateADT dateVal, int *overflow) } /* - * Single-argument version of date2timestamp_opt_overflow(). + * Promote date to timestamp, throwing error for overflow. */ static TimestampTz date2timestamp(DateADT dateVal) @@ -608,9 +617,12 @@ date2timestamp(DateADT dateVal) /* * Promote date to timestamp with time zone. * - * On overflow error is thrown if 'overflow' is NULL. Otherwise, '*overflow' - * is set to -1 (+1) when result value exceed lower (upper) boundary and zero - * returned. + * On successful conversion, *overflow is set to zero if it's not NULL. + * + * If the date is finite but out of the valid range for timestamptz, then: + * if overflow is NULL, we throw an out-of-range error. + * if overflow is not NULL, we store +1 or -1 there to indicate the sign + * of the overflow, and return the appropriate timestamptz infinity. */ TimestampTz date2timestamptz_opt_overflow(DateADT dateVal, int *overflow) @@ -620,6 +632,9 @@ date2timestamptz_opt_overflow(DateADT dateVal, int *overflow) *tm = &tt; int tz; + if (overflow) + *overflow = 0; + if (DATE_IS_NOBEGIN(dateVal)) TIMESTAMP_NOBEGIN(result); else if (DATE_IS_NOEND(dateVal)) @@ -627,7 +642,6 @@ date2timestamptz_opt_overflow(DateADT dateVal, int *overflow) else { /* - * Date's range is wider than timestamp's, so check for boundaries. * Since dates have the same minimum values as timestamps, only upper * boundary need be checked for overflow. */ @@ -636,7 +650,8 @@ date2timestamptz_opt_overflow(DateADT dateVal, int *overflow) if (overflow) { *overflow = 1; - return (TimestampTz) 0; + TIMESTAMP_NOEND(result); + return result; } else { @@ -664,13 +679,15 @@ date2timestamptz_opt_overflow(DateADT dateVal, int *overflow) if (overflow) { if (result < MIN_TIMESTAMP) + { *overflow = -1; + TIMESTAMP_NOBEGIN(result); + } else { - Assert(result >= END_TIMESTAMP); *overflow = 1; + TIMESTAMP_NOEND(result); } - return (TimestampTz) 0; } else { @@ -685,7 +702,7 @@ date2timestamptz_opt_overflow(DateADT dateVal, int *overflow) } /* - * Single-argument version of date2timestamptz_opt_overflow(). + * Promote date to timestamptz, throwing error for overflow. */ static TimestampTz date2timestamptz(DateADT dateVal) @@ -726,16 +743,30 @@ date2timestamp_no_overflow(DateADT dateVal) * Crosstype comparison functions for dates */ +int32 +date_cmp_timestamp_internal(DateADT dateVal, Timestamp dt2) +{ + Timestamp dt1; + int overflow; + + dt1 = date2timestamp_opt_overflow(dateVal, &overflow); + if (overflow > 0) + { + /* dt1 is larger than any finite timestamp, but less than infinity */ + return TIMESTAMP_IS_NOEND(dt2) ? -1 : +1; + } + Assert(overflow == 0); /* -1 case cannot occur */ + + return timestamp_cmp_internal(dt1, dt2); +} + Datum date_eq_timestamp(PG_FUNCTION_ARGS) { DateADT dateVal = PG_GETARG_DATEADT(0); Timestamp dt2 = PG_GETARG_TIMESTAMP(1); - Timestamp dt1; - - dt1 = date2timestamp(dateVal); - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) == 0); + PG_RETURN_BOOL(date_cmp_timestamp_internal(dateVal, dt2) == 0); } Datum @@ -743,11 +774,8 @@ date_ne_timestamp(PG_FUNCTION_ARGS) { DateADT dateVal = PG_GETARG_DATEADT(0); Timestamp dt2 = PG_GETARG_TIMESTAMP(1); - Timestamp dt1; - dt1 = date2timestamp(dateVal); - - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) != 0); + PG_RETURN_BOOL(date_cmp_timestamp_internal(dateVal, dt2) != 0); } Datum @@ -755,11 +783,8 @@ date_lt_timestamp(PG_FUNCTION_ARGS) { DateADT dateVal = PG_GETARG_DATEADT(0); Timestamp dt2 = PG_GETARG_TIMESTAMP(1); - Timestamp dt1; - dt1 = date2timestamp(dateVal); - - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) < 0); + PG_RETURN_BOOL(date_cmp_timestamp_internal(dateVal, dt2) < 0); } Datum @@ -767,11 +792,8 @@ date_gt_timestamp(PG_FUNCTION_ARGS) { DateADT dateVal = PG_GETARG_DATEADT(0); Timestamp dt2 = PG_GETARG_TIMESTAMP(1); - Timestamp dt1; - dt1 = date2timestamp(dateVal); - - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) > 0); + PG_RETURN_BOOL(date_cmp_timestamp_internal(dateVal, dt2) > 0); } Datum @@ -779,11 +801,8 @@ date_le_timestamp(PG_FUNCTION_ARGS) { DateADT dateVal = PG_GETARG_DATEADT(0); Timestamp dt2 = PG_GETARG_TIMESTAMP(1); - Timestamp dt1; - dt1 = date2timestamp(dateVal); - - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) <= 0); + PG_RETURN_BOOL(date_cmp_timestamp_internal(dateVal, dt2) <= 0); } Datum @@ -791,11 +810,8 @@ date_ge_timestamp(PG_FUNCTION_ARGS) { DateADT dateVal = PG_GETARG_DATEADT(0); Timestamp dt2 = PG_GETARG_TIMESTAMP(1); - Timestamp dt1; - dt1 = date2timestamp(dateVal); - - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) >= 0); + PG_RETURN_BOOL(date_cmp_timestamp_internal(dateVal, dt2) >= 0); } Datum @@ -803,11 +819,29 @@ date_cmp_timestamp(PG_FUNCTION_ARGS) { DateADT dateVal = PG_GETARG_DATEADT(0); Timestamp dt2 = PG_GETARG_TIMESTAMP(1); - Timestamp dt1; - dt1 = date2timestamp(dateVal); + PG_RETURN_INT32(date_cmp_timestamp_internal(dateVal, dt2)); +} - PG_RETURN_INT32(timestamp_cmp_internal(dt1, dt2)); +int32 +date_cmp_timestamptz_internal(DateADT dateVal, TimestampTz dt2) +{ + TimestampTz dt1; + int overflow; + + dt1 = date2timestamptz_opt_overflow(dateVal, &overflow); + if (overflow > 0) + { + /* dt1 is larger than any finite timestamp, but less than infinity */ + return TIMESTAMP_IS_NOEND(dt2) ? -1 : +1; + } + if (overflow < 0) + { + /* dt1 is less than any finite timestamp, but more than -infinity */ + return TIMESTAMP_IS_NOBEGIN(dt2) ? +1 : -1; + } + + return timestamptz_cmp_internal(dt1, dt2); } Datum @@ -815,11 +849,8 @@ date_eq_timestamptz(PG_FUNCTION_ARGS) { DateADT dateVal = PG_GETARG_DATEADT(0); TimestampTz dt2 = PG_GETARG_TIMESTAMPTZ(1); - TimestampTz dt1; - dt1 = date2timestamptz(dateVal); - - PG_RETURN_BOOL(timestamptz_cmp_internal(dt1, dt2) == 0); + PG_RETURN_BOOL(date_cmp_timestamptz_internal(dateVal, dt2) == 0); } Datum @@ -827,11 +858,8 @@ date_ne_timestamptz(PG_FUNCTION_ARGS) { DateADT dateVal = PG_GETARG_DATEADT(0); TimestampTz dt2 = PG_GETARG_TIMESTAMPTZ(1); - TimestampTz dt1; - dt1 = date2timestamptz(dateVal); - - PG_RETURN_BOOL(timestamptz_cmp_internal(dt1, dt2) != 0); + PG_RETURN_BOOL(date_cmp_timestamptz_internal(dateVal, dt2) != 0); } Datum @@ -839,11 +867,8 @@ date_lt_timestamptz(PG_FUNCTION_ARGS) { DateADT dateVal = PG_GETARG_DATEADT(0); TimestampTz dt2 = PG_GETARG_TIMESTAMPTZ(1); - TimestampTz dt1; - dt1 = date2timestamptz(dateVal); - - PG_RETURN_BOOL(timestamptz_cmp_internal(dt1, dt2) < 0); + PG_RETURN_BOOL(date_cmp_timestamptz_internal(dateVal, dt2) < 0); } Datum @@ -851,11 +876,8 @@ date_gt_timestamptz(PG_FUNCTION_ARGS) { DateADT dateVal = PG_GETARG_DATEADT(0); TimestampTz dt2 = PG_GETARG_TIMESTAMPTZ(1); - TimestampTz dt1; - dt1 = date2timestamptz(dateVal); - - PG_RETURN_BOOL(timestamptz_cmp_internal(dt1, dt2) > 0); + PG_RETURN_BOOL(date_cmp_timestamptz_internal(dateVal, dt2) > 0); } Datum @@ -863,11 +885,8 @@ date_le_timestamptz(PG_FUNCTION_ARGS) { DateADT dateVal = PG_GETARG_DATEADT(0); TimestampTz dt2 = PG_GETARG_TIMESTAMPTZ(1); - TimestampTz dt1; - dt1 = date2timestamptz(dateVal); - - PG_RETURN_BOOL(timestamptz_cmp_internal(dt1, dt2) <= 0); + PG_RETURN_BOOL(date_cmp_timestamptz_internal(dateVal, dt2) <= 0); } Datum @@ -875,11 +894,8 @@ date_ge_timestamptz(PG_FUNCTION_ARGS) { DateADT dateVal = PG_GETARG_DATEADT(0); TimestampTz dt2 = PG_GETARG_TIMESTAMPTZ(1); - TimestampTz dt1; - dt1 = date2timestamptz(dateVal); - - PG_RETURN_BOOL(timestamptz_cmp_internal(dt1, dt2) >= 0); + PG_RETURN_BOOL(date_cmp_timestamptz_internal(dateVal, dt2) >= 0); } Datum @@ -887,11 +903,8 @@ date_cmp_timestamptz(PG_FUNCTION_ARGS) { DateADT dateVal = PG_GETARG_DATEADT(0); TimestampTz dt2 = PG_GETARG_TIMESTAMPTZ(1); - TimestampTz dt1; - - dt1 = date2timestamptz(dateVal); - PG_RETURN_INT32(timestamptz_cmp_internal(dt1, dt2)); + PG_RETURN_INT32(date_cmp_timestamptz_internal(dateVal, dt2)); } Datum @@ -899,11 +912,8 @@ timestamp_eq_date(PG_FUNCTION_ARGS) { Timestamp dt1 = PG_GETARG_TIMESTAMP(0); DateADT dateVal = PG_GETARG_DATEADT(1); - Timestamp dt2; - dt2 = date2timestamp(dateVal); - - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) == 0); + PG_RETURN_BOOL(date_cmp_timestamp_internal(dateVal, dt1) == 0); } Datum @@ -911,11 +921,8 @@ timestamp_ne_date(PG_FUNCTION_ARGS) { Timestamp dt1 = PG_GETARG_TIMESTAMP(0); DateADT dateVal = PG_GETARG_DATEADT(1); - Timestamp dt2; - - dt2 = date2timestamp(dateVal); - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) != 0); + PG_RETURN_BOOL(date_cmp_timestamp_internal(dateVal, dt1) != 0); } Datum @@ -923,11 +930,8 @@ timestamp_lt_date(PG_FUNCTION_ARGS) { Timestamp dt1 = PG_GETARG_TIMESTAMP(0); DateADT dateVal = PG_GETARG_DATEADT(1); - Timestamp dt2; - dt2 = date2timestamp(dateVal); - - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) < 0); + PG_RETURN_BOOL(date_cmp_timestamp_internal(dateVal, dt1) > 0); } Datum @@ -935,11 +939,8 @@ timestamp_gt_date(PG_FUNCTION_ARGS) { Timestamp dt1 = PG_GETARG_TIMESTAMP(0); DateADT dateVal = PG_GETARG_DATEADT(1); - Timestamp dt2; - - dt2 = date2timestamp(dateVal); - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) > 0); + PG_RETURN_BOOL(date_cmp_timestamp_internal(dateVal, dt1) < 0); } Datum @@ -947,11 +948,8 @@ timestamp_le_date(PG_FUNCTION_ARGS) { Timestamp dt1 = PG_GETARG_TIMESTAMP(0); DateADT dateVal = PG_GETARG_DATEADT(1); - Timestamp dt2; - dt2 = date2timestamp(dateVal); - - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) <= 0); + PG_RETURN_BOOL(date_cmp_timestamp_internal(dateVal, dt1) >= 0); } Datum @@ -959,11 +957,8 @@ timestamp_ge_date(PG_FUNCTION_ARGS) { Timestamp dt1 = PG_GETARG_TIMESTAMP(0); DateADT dateVal = PG_GETARG_DATEADT(1); - Timestamp dt2; - - dt2 = date2timestamp(dateVal); - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) >= 0); + PG_RETURN_BOOL(date_cmp_timestamp_internal(dateVal, dt1) <= 0); } Datum @@ -971,11 +966,8 @@ timestamp_cmp_date(PG_FUNCTION_ARGS) { Timestamp dt1 = PG_GETARG_TIMESTAMP(0); DateADT dateVal = PG_GETARG_DATEADT(1); - Timestamp dt2; - dt2 = date2timestamp(dateVal); - - PG_RETURN_INT32(timestamp_cmp_internal(dt1, dt2)); + PG_RETURN_INT32(-date_cmp_timestamp_internal(dateVal, dt1)); } Datum @@ -983,11 +975,8 @@ timestamptz_eq_date(PG_FUNCTION_ARGS) { TimestampTz dt1 = PG_GETARG_TIMESTAMPTZ(0); DateADT dateVal = PG_GETARG_DATEADT(1); - TimestampTz dt2; - - dt2 = date2timestamptz(dateVal); - PG_RETURN_BOOL(timestamptz_cmp_internal(dt1, dt2) == 0); + PG_RETURN_BOOL(date_cmp_timestamptz_internal(dateVal, dt1) == 0); } Datum @@ -995,11 +984,8 @@ timestamptz_ne_date(PG_FUNCTION_ARGS) { TimestampTz dt1 = PG_GETARG_TIMESTAMPTZ(0); DateADT dateVal = PG_GETARG_DATEADT(1); - TimestampTz dt2; - dt2 = date2timestamptz(dateVal); - - PG_RETURN_BOOL(timestamptz_cmp_internal(dt1, dt2) != 0); + PG_RETURN_BOOL(date_cmp_timestamptz_internal(dateVal, dt1) != 0); } Datum @@ -1007,11 +993,8 @@ timestamptz_lt_date(PG_FUNCTION_ARGS) { TimestampTz dt1 = PG_GETARG_TIMESTAMPTZ(0); DateADT dateVal = PG_GETARG_DATEADT(1); - TimestampTz dt2; - - dt2 = date2timestamptz(dateVal); - PG_RETURN_BOOL(timestamptz_cmp_internal(dt1, dt2) < 0); + PG_RETURN_BOOL(date_cmp_timestamptz_internal(dateVal, dt1) > 0); } Datum @@ -1019,11 +1002,8 @@ timestamptz_gt_date(PG_FUNCTION_ARGS) { TimestampTz dt1 = PG_GETARG_TIMESTAMPTZ(0); DateADT dateVal = PG_GETARG_DATEADT(1); - TimestampTz dt2; - dt2 = date2timestamptz(dateVal); - - PG_RETURN_BOOL(timestamptz_cmp_internal(dt1, dt2) > 0); + PG_RETURN_BOOL(date_cmp_timestamptz_internal(dateVal, dt1) < 0); } Datum @@ -1031,11 +1011,8 @@ timestamptz_le_date(PG_FUNCTION_ARGS) { TimestampTz dt1 = PG_GETARG_TIMESTAMPTZ(0); DateADT dateVal = PG_GETARG_DATEADT(1); - TimestampTz dt2; - - dt2 = date2timestamptz(dateVal); - PG_RETURN_BOOL(timestamptz_cmp_internal(dt1, dt2) <= 0); + PG_RETURN_BOOL(date_cmp_timestamptz_internal(dateVal, dt1) >= 0); } Datum @@ -1043,11 +1020,8 @@ timestamptz_ge_date(PG_FUNCTION_ARGS) { TimestampTz dt1 = PG_GETARG_TIMESTAMPTZ(0); DateADT dateVal = PG_GETARG_DATEADT(1); - TimestampTz dt2; - dt2 = date2timestamptz(dateVal); - - PG_RETURN_BOOL(timestamptz_cmp_internal(dt1, dt2) >= 0); + PG_RETURN_BOOL(date_cmp_timestamptz_internal(dateVal, dt1) <= 0); } Datum @@ -1055,11 +1029,8 @@ timestamptz_cmp_date(PG_FUNCTION_ARGS) { TimestampTz dt1 = PG_GETARG_TIMESTAMPTZ(0); DateADT dateVal = PG_GETARG_DATEADT(1); - TimestampTz dt2; - - dt2 = date2timestamptz(dateVal); - PG_RETURN_INT32(timestamptz_cmp_internal(dt1, dt2)); + PG_RETURN_INT32(-date_cmp_timestamptz_internal(dateVal, dt1)); } /* @@ -1079,6 +1050,7 @@ in_range_date_interval(PG_FUNCTION_ARGS) Timestamp valStamp; Timestamp baseStamp; + /* XXX we could support out-of-range cases here, perhaps */ valStamp = date2timestamp(val); baseStamp = date2timestamp(base); diff --git a/src/backend/utils/adt/jsonpath_exec.c b/src/backend/utils/adt/jsonpath_exec.c index 28be845770a6..1059f34130ae 100644 --- a/src/backend/utils/adt/jsonpath_exec.c +++ b/src/backend/utils/adt/jsonpath_exec.c @@ -2602,93 +2602,36 @@ castTimeToTimeTz(Datum time, bool useTz) return DirectFunctionCall1(time_timetz, time); } -/*--- - * Compares 'ts1' and 'ts2' timestamp, assuming that ts1 might be overflowed - * during cast from another datatype. - * - * 'overflow1' specifies overflow of 'ts1' value: - * 0 - no overflow, - * -1 - exceed lower boundary, - * 1 - exceed upper boundary. - */ -static int -cmpTimestampWithOverflow(Timestamp ts1, int overflow1, Timestamp ts2) -{ - /* - * All the timestamps we deal with in jsonpath are produced by - * to_datetime() method. So, they should be valid. - */ - Assert(IS_VALID_TIMESTAMP(ts2)); - - /* - * Timestamp, which exceed lower (upper) bound, is always lower (higher) - * than any valid timestamp except minus (plus) infinity. - */ - if (overflow1) - { - if (overflow1 < 0) - { - if (TIMESTAMP_IS_NOBEGIN(ts2)) - return 1; - else - return -1; - } - if (overflow1 > 0) - { - if (TIMESTAMP_IS_NOEND(ts2)) - return -1; - else - return 1; - } - } - - return timestamp_cmp_internal(ts1, ts2); -} - /* - * Compare date to timestamptz without throwing overflow error during cast. + * Compare date to timestamp. + * Note that this doesn't involve any timezone considerations. */ static int cmpDateToTimestamp(DateADT date1, Timestamp ts2, bool useTz) { - TimestampTz ts1; - int overflow = 0; - - ts1 = date2timestamp_opt_overflow(date1, &overflow); - - return cmpTimestampWithOverflow(ts1, overflow, ts2); + return date_cmp_timestamp_internal(date1, ts2); } /* - * Compare date to timestamptz without throwing overflow error during cast. + * Compare date to timestamptz. */ static int cmpDateToTimestampTz(DateADT date1, TimestampTz tstz2, bool useTz) { - TimestampTz tstz1; - int overflow = 0; - checkTimezoneIsUsedForCast(useTz, "date", "timestamptz"); - tstz1 = date2timestamptz_opt_overflow(date1, &overflow); - - return cmpTimestampWithOverflow(tstz1, overflow, tstz2); + return date_cmp_timestamptz_internal(date1, tstz2); } /* - * Compare timestamp to timestamptz without throwing overflow error during cast. + * Compare timestamp to timestamptz. */ static int cmpTimestampToTimestampTz(Timestamp ts1, TimestampTz tstz2, bool useTz) { - TimestampTz tstz1; - int overflow = 0; - checkTimezoneIsUsedForCast(useTz, "timestamp", "timestamptz"); - tstz1 = timestamp2timestamptz_opt_overflow(ts1, &overflow); - - return cmpTimestampWithOverflow(tstz1, overflow, tstz2); + return timestamp_cmp_timestamptz_internal(ts1, tstz2); } /* diff --git a/src/backend/utils/adt/timestamp.c b/src/backend/utils/adt/timestamp.c index 4128e3a73925..ea0ada704f20 100644 --- a/src/backend/utils/adt/timestamp.c +++ b/src/backend/utils/adt/timestamp.c @@ -2156,16 +2156,34 @@ timestamp_hash_extended(PG_FUNCTION_ARGS) * Cross-type comparison functions for timestamp vs timestamptz */ +int32 +timestamp_cmp_timestamptz_internal(Timestamp timestampVal, TimestampTz dt2) +{ + TimestampTz dt1; + int overflow; + + dt1 = timestamp2timestamptz_opt_overflow(timestampVal, &overflow); + if (overflow > 0) + { + /* dt1 is larger than any finite timestamp, but less than infinity */ + return TIMESTAMP_IS_NOEND(dt2) ? -1 : +1; + } + if (overflow < 0) + { + /* dt1 is less than any finite timestamp, but more than -infinity */ + return TIMESTAMP_IS_NOBEGIN(dt2) ? +1 : -1; + } + + return timestamptz_cmp_internal(dt1, dt2); +} + Datum timestamp_eq_timestamptz(PG_FUNCTION_ARGS) { Timestamp timestampVal = PG_GETARG_TIMESTAMP(0); TimestampTz dt2 = PG_GETARG_TIMESTAMPTZ(1); - TimestampTz dt1; - - dt1 = timestamp2timestamptz(timestampVal); - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) == 0); + PG_RETURN_BOOL(timestamp_cmp_timestamptz_internal(timestampVal, dt2) == 0); } Datum @@ -2173,11 +2191,8 @@ timestamp_ne_timestamptz(PG_FUNCTION_ARGS) { Timestamp timestampVal = PG_GETARG_TIMESTAMP(0); TimestampTz dt2 = PG_GETARG_TIMESTAMPTZ(1); - TimestampTz dt1; - dt1 = timestamp2timestamptz(timestampVal); - - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) != 0); + PG_RETURN_BOOL(timestamp_cmp_timestamptz_internal(timestampVal, dt2) != 0); } Datum @@ -2185,11 +2200,8 @@ timestamp_lt_timestamptz(PG_FUNCTION_ARGS) { Timestamp timestampVal = PG_GETARG_TIMESTAMP(0); TimestampTz dt2 = PG_GETARG_TIMESTAMPTZ(1); - TimestampTz dt1; - - dt1 = timestamp2timestamptz(timestampVal); - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) < 0); + PG_RETURN_BOOL(timestamp_cmp_timestamptz_internal(timestampVal, dt2) < 0); } Datum @@ -2197,11 +2209,8 @@ timestamp_gt_timestamptz(PG_FUNCTION_ARGS) { Timestamp timestampVal = PG_GETARG_TIMESTAMP(0); TimestampTz dt2 = PG_GETARG_TIMESTAMPTZ(1); - TimestampTz dt1; - - dt1 = timestamp2timestamptz(timestampVal); - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) > 0); + PG_RETURN_BOOL(timestamp_cmp_timestamptz_internal(timestampVal, dt2) > 0); } Datum @@ -2209,11 +2218,8 @@ timestamp_le_timestamptz(PG_FUNCTION_ARGS) { Timestamp timestampVal = PG_GETARG_TIMESTAMP(0); TimestampTz dt2 = PG_GETARG_TIMESTAMPTZ(1); - TimestampTz dt1; - - dt1 = timestamp2timestamptz(timestampVal); - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) <= 0); + PG_RETURN_BOOL(timestamp_cmp_timestamptz_internal(timestampVal, dt2) <= 0); } Datum @@ -2221,11 +2227,8 @@ timestamp_ge_timestamptz(PG_FUNCTION_ARGS) { Timestamp timestampVal = PG_GETARG_TIMESTAMP(0); TimestampTz dt2 = PG_GETARG_TIMESTAMPTZ(1); - TimestampTz dt1; - - dt1 = timestamp2timestamptz(timestampVal); - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) >= 0); + PG_RETURN_BOOL(timestamp_cmp_timestamptz_internal(timestampVal, dt2) >= 0); } Datum @@ -2233,11 +2236,8 @@ timestamp_cmp_timestamptz(PG_FUNCTION_ARGS) { Timestamp timestampVal = PG_GETARG_TIMESTAMP(0); TimestampTz dt2 = PG_GETARG_TIMESTAMPTZ(1); - TimestampTz dt1; - dt1 = timestamp2timestamptz(timestampVal); - - PG_RETURN_INT32(timestamp_cmp_internal(dt1, dt2)); + PG_RETURN_INT32(timestamp_cmp_timestamptz_internal(timestampVal, dt2)); } Datum @@ -2245,11 +2245,8 @@ timestamptz_eq_timestamp(PG_FUNCTION_ARGS) { TimestampTz dt1 = PG_GETARG_TIMESTAMPTZ(0); Timestamp timestampVal = PG_GETARG_TIMESTAMP(1); - TimestampTz dt2; - - dt2 = timestamp2timestamptz(timestampVal); - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) == 0); + PG_RETURN_BOOL(timestamp_cmp_timestamptz_internal(timestampVal, dt1) == 0); } Datum @@ -2257,11 +2254,8 @@ timestamptz_ne_timestamp(PG_FUNCTION_ARGS) { TimestampTz dt1 = PG_GETARG_TIMESTAMPTZ(0); Timestamp timestampVal = PG_GETARG_TIMESTAMP(1); - TimestampTz dt2; - - dt2 = timestamp2timestamptz(timestampVal); - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) != 0); + PG_RETURN_BOOL(timestamp_cmp_timestamptz_internal(timestampVal, dt1) != 0); } Datum @@ -2269,11 +2263,8 @@ timestamptz_lt_timestamp(PG_FUNCTION_ARGS) { TimestampTz dt1 = PG_GETARG_TIMESTAMPTZ(0); Timestamp timestampVal = PG_GETARG_TIMESTAMP(1); - TimestampTz dt2; - dt2 = timestamp2timestamptz(timestampVal); - - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) < 0); + PG_RETURN_BOOL(timestamp_cmp_timestamptz_internal(timestampVal, dt1) > 0); } Datum @@ -2281,11 +2272,8 @@ timestamptz_gt_timestamp(PG_FUNCTION_ARGS) { TimestampTz dt1 = PG_GETARG_TIMESTAMPTZ(0); Timestamp timestampVal = PG_GETARG_TIMESTAMP(1); - TimestampTz dt2; - - dt2 = timestamp2timestamptz(timestampVal); - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) > 0); + PG_RETURN_BOOL(timestamp_cmp_timestamptz_internal(timestampVal, dt1) < 0); } Datum @@ -2293,11 +2281,8 @@ timestamptz_le_timestamp(PG_FUNCTION_ARGS) { TimestampTz dt1 = PG_GETARG_TIMESTAMPTZ(0); Timestamp timestampVal = PG_GETARG_TIMESTAMP(1); - TimestampTz dt2; - - dt2 = timestamp2timestamptz(timestampVal); - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) <= 0); + PG_RETURN_BOOL(timestamp_cmp_timestamptz_internal(timestampVal, dt1) >= 0); } Datum @@ -2305,11 +2290,8 @@ timestamptz_ge_timestamp(PG_FUNCTION_ARGS) { TimestampTz dt1 = PG_GETARG_TIMESTAMPTZ(0); Timestamp timestampVal = PG_GETARG_TIMESTAMP(1); - TimestampTz dt2; - dt2 = timestamp2timestamptz(timestampVal); - - PG_RETURN_BOOL(timestamp_cmp_internal(dt1, dt2) >= 0); + PG_RETURN_BOOL(timestamp_cmp_timestamptz_internal(timestampVal, dt1) <= 0); } Datum @@ -2317,11 +2299,8 @@ timestamptz_cmp_timestamp(PG_FUNCTION_ARGS) { TimestampTz dt1 = PG_GETARG_TIMESTAMPTZ(0); Timestamp timestampVal = PG_GETARG_TIMESTAMP(1); - TimestampTz dt2; - - dt2 = timestamp2timestamptz(timestampVal); - PG_RETURN_INT32(timestamp_cmp_internal(dt1, dt2)); + PG_RETURN_INT32(-timestamp_cmp_timestamptz_internal(timestampVal, dt1)); } @@ -5178,9 +5157,12 @@ timestamp_timestamptz(PG_FUNCTION_ARGS) /* * Convert timestamp to timestamp with time zone. * - * On overflow error is thrown if 'overflow' is NULL. Otherwise, '*overflow' - * is set to -1 (+1) when result value exceed lower (upper) boundary and zero - * returned. + * On successful conversion, *overflow is set to zero if it's not NULL. + * + * If the timestamp is finite but out of the valid range for timestamptz, then: + * if overflow is NULL, we throw an out-of-range error. + * if overflow is not NULL, we store +1 or -1 there to indicate the sign + * of the overflow, and return the appropriate timestamptz infinity. */ TimestampTz timestamp2timestamptz_opt_overflow(Timestamp timestamp, int *overflow) @@ -5191,10 +5173,14 @@ timestamp2timestamptz_opt_overflow(Timestamp timestamp, int *overflow) fsec_t fsec; int tz; + if (overflow) + *overflow = 0; + if (TIMESTAMP_NOT_FINITE(timestamp)) return timestamp; - if (!timestamp2tm(timestamp, NULL, tm, &fsec, NULL, NULL)) + /* We don't expect this to fail, but check it pro forma */ + if (timestamp2tm(timestamp, NULL, tm, &fsec, NULL, NULL) == 0) { tz = DetermineTimeZoneOffset(tm, session_timezone); @@ -5207,13 +5193,16 @@ timestamp2timestamptz_opt_overflow(Timestamp timestamp, int *overflow) else if (overflow) { if (result < MIN_TIMESTAMP) + { *overflow = -1; + TIMESTAMP_NOBEGIN(result); + } else { - Assert(result >= END_TIMESTAMP); *overflow = 1; + TIMESTAMP_NOEND(result); } - return (TimestampTz) 0; + return result; } } @@ -5225,7 +5214,7 @@ timestamp2timestamptz_opt_overflow(Timestamp timestamp, int *overflow) } /* - * Single-argument version of timestamp2timestamptz_opt_overflow(). + * Promote timestamp to timestamptz, throwing error for overflow. */ static TimestampTz timestamp2timestamptz(Timestamp timestamp) diff --git a/src/include/utils/date.h b/src/include/utils/date.h index 4cdb1f97cc8c..6fc491e6a6d7 100644 --- a/src/include/utils/date.h +++ b/src/include/utils/date.h @@ -72,6 +72,9 @@ extern int32 anytime_typmod_check(bool istz, int32 typmod); extern double date2timestamp_no_overflow(DateADT dateVal); extern Timestamp date2timestamp_opt_overflow(DateADT dateVal, int *overflow); extern TimestampTz date2timestamptz_opt_overflow(DateADT dateVal, int *overflow); +extern int32 date_cmp_timestamp_internal(DateADT dateVal, Timestamp dt2); +extern int32 date_cmp_timestamptz_internal(DateADT dateVal, TimestampTz dt2); + extern void EncodeSpecialDate(DateADT dt, char *str); extern DateADT GetSQLCurrentDate(void); extern TimeTzADT *GetSQLCurrentTime(int32 typmod); diff --git a/src/include/utils/timestamp.h b/src/include/utils/timestamp.h index 03a1de569f06..16c3fd8ec977 100644 --- a/src/include/utils/timestamp.h +++ b/src/include/utils/timestamp.h @@ -99,6 +99,8 @@ extern int timestamp_cmp_internal(Timestamp dt1, Timestamp dt2); extern TimestampTz timestamp2timestamptz_opt_overflow(Timestamp timestamp, int *overflow); +extern int32 timestamp_cmp_timestamptz_internal(Timestamp timestampVal, + TimestampTz dt2); extern int isoweek2j(int year, int week); extern void isoweek2date(int woy, int *year, int *mon, int *mday); diff --git a/src/test/regress/expected/horology.out b/src/test/regress/expected/horology.out index 7f82dcfbfef3..d56decd99400 100644 --- a/src/test/regress/expected/horology.out +++ b/src/test/regress/expected/horology.out @@ -2,6 +2,12 @@ -- HOROLOGY -- SET DateStyle = 'Postgres, MDY'; +SHOW TimeZone; -- Many of these tests depend on the prevailing setting + TimeZone +---------- + PST8PDT +(1 row) + -- -- Test various input formats -- @@ -2076,6 +2082,72 @@ SELECT '' AS "16", f1 AS "timestamp", date(f1) AS date DROP TABLE TEMP_TIMESTAMP; -- +-- Comparisons between datetime types, especially overflow cases +--- +SELECT '2202020-10-05'::date::timestamp; -- fail +ERROR: date out of range for timestamp +SELECT '2202020-10-05'::date > '2020-10-05'::timestamp as t; + t +--- + t +(1 row) + +SELECT '2020-10-05'::timestamp > '2202020-10-05'::date as f; + f +--- + f +(1 row) + +SELECT '2202020-10-05'::date::timestamptz; -- fail +ERROR: date out of range for timestamp +SELECT '2202020-10-05'::date > '2020-10-05'::timestamptz as t; + t +--- + t +(1 row) + +SELECT '2020-10-05'::timestamptz > '2202020-10-05'::date as f; + f +--- + f +(1 row) + +-- This conversion may work depending on timezone +SELECT '4714-11-24 BC'::date::timestamptz; + timestamptz +--------------------------------- + Mon Nov 24 00:00:00 4714 PST BC +(1 row) + +SET TimeZone = 'UTC-2'; +SELECT '4714-11-24 BC'::date::timestamptz; -- fail +ERROR: date out of range for timestamp +SELECT '4714-11-24 BC'::date < '2020-10-05'::timestamptz as t; + t +--- + t +(1 row) + +SELECT '2020-10-05'::timestamptz >= '4714-11-24 BC'::date as t; + t +--- + t +(1 row) + +SELECT '4714-11-24 BC'::timestamp < '2020-10-05'::timestamptz as t; + t +--- + t +(1 row) + +SELECT '2020-10-05'::timestamptz >= '4714-11-24 BC'::timestamp as t; + t +--- + t +(1 row) + +RESET TimeZone; +-- -- Formats -- SET DateStyle TO 'US,Postgres'; diff --git a/src/test/regress/sql/horology.sql b/src/test/regress/sql/horology.sql index fed21a53c867..fa92a80d0e66 100644 --- a/src/test/regress/sql/horology.sql +++ b/src/test/regress/sql/horology.sql @@ -3,6 +3,8 @@ -- SET DateStyle = 'Postgres, MDY'; +SHOW TimeZone; -- Many of these tests depend on the prevailing setting + -- -- Test various input formats -- @@ -279,6 +281,31 @@ SELECT '' AS "16", f1 AS "timestamp", date(f1) AS date DROP TABLE TEMP_TIMESTAMP; +-- +-- Comparisons between datetime types, especially overflow cases +--- + +SELECT '2202020-10-05'::date::timestamp; -- fail +SELECT '2202020-10-05'::date > '2020-10-05'::timestamp as t; +SELECT '2020-10-05'::timestamp > '2202020-10-05'::date as f; + +SELECT '2202020-10-05'::date::timestamptz; -- fail +SELECT '2202020-10-05'::date > '2020-10-05'::timestamptz as t; +SELECT '2020-10-05'::timestamptz > '2202020-10-05'::date as f; + +-- This conversion may work depending on timezone +SELECT '4714-11-24 BC'::date::timestamptz; +SET TimeZone = 'UTC-2'; +SELECT '4714-11-24 BC'::date::timestamptz; -- fail + +SELECT '4714-11-24 BC'::date < '2020-10-05'::timestamptz as t; +SELECT '2020-10-05'::timestamptz >= '4714-11-24 BC'::date as t; + +SELECT '4714-11-24 BC'::timestamp < '2020-10-05'::timestamptz as t; +SELECT '2020-10-05'::timestamptz >= '4714-11-24 BC'::timestamp as t; + +RESET TimeZone; + -- -- Formats -- From 8d2a01ae12cd657b33ffd50eace86a341636c586 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 7 Oct 2020 18:41:39 -0400 Subject: [PATCH 265/589] Fix optimization hazard in gram.y's makeOrderedSetArgs(), redux. It appears that commit cf63c641c, which intended to prevent misoptimization of the result-building step in makeOrderedSetArgs, didn't go far enough: buildfarm member hornet's version of xlc is now optimizing back to the old, broken behavior in which list_length(directargs) is fetched only after list_concat() has changed that value. I'm not entirely convinced whether that's an undeniable compiler bug or whether it can be justified by a sufficiently aggressive interpretation of C sequence points. So let's just change the code to make it harder to misinterpret. Back-patch to all supported versions, just in case. Discussion: https://postgr.es/m/1830491.1601944935@sss.pgh.pa.us --- src/backend/parser/gram.y | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 0d101d817154..480d16834687 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -16291,7 +16291,7 @@ makeOrderedSetArgs(List *directargs, List *orderedargs, core_yyscan_t yyscanner) { FunctionParameter *lastd = (FunctionParameter *) llast(directargs); - int ndirectargs; + Value *ndirectargs; /* No restriction unless last direct arg is VARIADIC */ if (lastd->mode == FUNC_PARAM_VARIADIC) @@ -16315,10 +16315,10 @@ makeOrderedSetArgs(List *directargs, List *orderedargs, } /* don't merge into the next line, as list_concat changes directargs */ - ndirectargs = list_length(directargs); + ndirectargs = makeInteger(list_length(directargs)); return list_make2(list_concat(directargs, orderedargs), - makeInteger(ndirectargs)); + ndirectargs); } /* insertSelectOptions() From 98681675002d852d926a49d7bc4d4b4856b2fc4a Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Thu, 8 Oct 2020 09:09:08 +0530 Subject: [PATCH 266/589] Track statistics for spilling of changes from ReorderBuffer. This adds the statistics about transactions spilled to disk from ReorderBuffer. Users can query the pg_stat_replication_slots view to check these stats and call pg_stat_reset_replication_slot to reset the stats of a particular slot. Users can pass NULL in pg_stat_reset_replication_slot to reset stats of all the slots. This commit extends the statistics collector to track this information about slots. Author: Sawada Masahiko and Amit Kapila Reviewed-by: Amit Kapila and Dilip Kumar Discussion: https://postgr.es/m/CA+fd4k5_pPAYRTDrO2PbtTOe0eHQpBvuqmCr8ic39uTNmR49Eg@mail.gmail.com --- doc/src/sgml/monitoring.sgml | 112 ++++++ src/backend/catalog/system_views.sql | 10 + src/backend/postmaster/pgstat.c | 320 ++++++++++++++++++ src/backend/replication/logical/decode.c | 9 + src/backend/replication/logical/logical.c | 29 ++ .../replication/logical/reorderbuffer.c | 22 ++ src/backend/replication/slot.c | 25 +- src/backend/utils/adt/pgstatfuncs.c | 80 +++++ src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.dat | 12 + src/include/pgstat.h | 52 ++- src/include/replication/logical.h | 1 + src/include/replication/reorderbuffer.h | 24 +- src/include/replication/slot.h | 1 + src/test/regress/expected/rules.out | 6 + src/tools/pgindent/typedefs.list | 3 + 16 files changed, 700 insertions(+), 8 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 171ba7049c7b..66566765f0c9 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -314,6 +314,15 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser + + pg_stat_replication_slotspg_stat_replication_slots + One row per replication slot, showing statistics about + replication slot usage. + See + pg_stat_replication_slots for details. + + + pg_stat_wal_receiverpg_stat_wal_receiver Only one row, showing statistics about the WAL receiver from @@ -2552,6 +2561,88 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i + + <structname>pg_stat_replication_slots</structname> + + + pg_stat_replication_slots + + + + The pg_stat_replication_slots view will contain + one row per logical replication slot, showing statistics about its usage. + + +
+ <structname>pg_stat_replication_slots</structname> View + + + + + Column Type + + + Description + + + + + + + + name text + + + A unique, cluster-wide identifier for the replication slot + + + + + + spill_txns bigint + + + Number of transactions spilled to disk after the memory used by + logical decoding exceeds logical_decoding_work_mem. The + counter gets incremented both for toplevel transactions and + subtransactions. + + + + + + spill_count bigint + + + Number of times transactions were spilled to disk. Transactions + may get spilled repeatedly, and this counter gets incremented on every + such invocation. + + + + + + spill_bytes bigint + + + Amount of decoded transaction data spilled to disk. + + + + + + stats_reset timestamp with time zone + + + Time at which these statistics were last reset + + + + +
+ + + <structname>pg_stat_wal_receiver</structname> @@ -4802,6 +4893,27 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i can be granted EXECUTE to run the function. + + + + + pg_stat_reset_replication_slot + + pg_stat_reset_replication_slot ( text ) + void + + + Resets statistics to zero for a single replication slot, or for all + replication slots in the cluster. The argument can be either the name + of the slot to reset the stats or NULL. If the argument is NULL, all + counters shown in the pg_stat_replication_slots + view for all replication slots are reset. + + + This function is restricted to superusers by default, but other users + can be granted EXECUTE to run the function. + + diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 923c2e2be1f6..c29390760fe4 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -796,6 +796,15 @@ CREATE VIEW pg_stat_replication AS JOIN pg_stat_get_wal_senders() AS W ON (S.pid = W.pid) LEFT JOIN pg_authid AS U ON (S.usesysid = U.oid); +CREATE VIEW pg_stat_replication_slots AS + SELECT + s.name, + s.spill_txns, + s.spill_count, + s.spill_bytes, + s.stats_reset + FROM pg_stat_get_replication_slots() AS s; + CREATE VIEW pg_stat_slru AS SELECT s.name, @@ -1453,6 +1462,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_reset_shared(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_slru(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_table_counters(oid) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_function_counters(oid) FROM public; +REVOKE EXECUTE ON FUNCTION pg_stat_reset_replication_slot(text) FROM public; REVOKE EXECUTE ON FUNCTION lo_import(text) FROM public; REVOKE EXECUTE ON FUNCTION lo_import(text, oid) FROM public; diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 5294c7854942..822f0ebc6285 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -51,6 +51,7 @@ #include "postmaster/fork_process.h" #include "postmaster/interrupt.h" #include "postmaster/postmaster.h" +#include "replication/slot.h" #include "replication/walsender.h" #include "storage/backendid.h" #include "storage/dsm.h" @@ -284,6 +285,8 @@ static PgStat_ArchiverStats archiverStats; static PgStat_GlobalStats globalStats; static PgStat_WalStats walStats; static PgStat_SLRUStats slruStats[SLRU_NUM_ELEMENTS]; +static PgStat_ReplSlotStats *replSlotStats; +static int nReplSlotStats; /* * List of OIDs of databases we need to write out. If an entry is InvalidOid, @@ -324,6 +327,9 @@ static void pgstat_read_current_status(void); static bool pgstat_write_statsfile_needed(void); static bool pgstat_db_requested(Oid databaseid); +static int pgstat_replslot_index(const char *name, bool create_it); +static void pgstat_reset_replslot(int i, TimestampTz ts); + static void pgstat_send_tabstat(PgStat_MsgTabstat *tsmsg); static void pgstat_send_funcstats(void); static void pgstat_send_slru(void); @@ -350,6 +356,7 @@ static void pgstat_recv_resetcounter(PgStat_MsgResetcounter *msg, int len); static void pgstat_recv_resetsharedcounter(PgStat_MsgResetsharedcounter *msg, int len); static void pgstat_recv_resetsinglecounter(PgStat_MsgResetsinglecounter *msg, int len); static void pgstat_recv_resetslrucounter(PgStat_MsgResetslrucounter *msg, int len); +static void pgstat_recv_resetreplslotcounter(PgStat_MsgResetreplslotcounter *msg, int len); static void pgstat_recv_autovac(PgStat_MsgAutovacStart *msg, int len); static void pgstat_recv_vacuum(PgStat_MsgVacuum *msg, int len); static void pgstat_recv_analyze(PgStat_MsgAnalyze *msg, int len); @@ -362,6 +369,7 @@ static void pgstat_recv_funcpurge(PgStat_MsgFuncpurge *msg, int len); static void pgstat_recv_recoveryconflict(PgStat_MsgRecoveryConflict *msg, int len); static void pgstat_recv_deadlock(PgStat_MsgDeadlock *msg, int len); static void pgstat_recv_checksum_failure(PgStat_MsgChecksumFailure *msg, int len); +static void pgstat_recv_replslot(PgStat_MsgReplSlot *msg, int len); static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); /* ------------------------------------------------------------ @@ -1437,6 +1445,61 @@ pgstat_reset_slru_counter(const char *name) pgstat_send(&msg, sizeof(msg)); } +/* ---------- + * pgstat_reset_replslot_counter() - + * + * Tell the statistics collector to reset a single replication slot + * counter, or all replication slots counters (when name is null). + * + * Permission checking for this function is managed through the normal + * GRANT system. + * ---------- + */ +void +pgstat_reset_replslot_counter(const char *name) +{ + PgStat_MsgResetreplslotcounter msg; + + if (pgStatSock == PGINVALID_SOCKET) + return; + + if (name) + { + ReplicationSlot *slot; + + /* + * Check if the slot exits with the given name. It is possible that by + * the time this message is executed the slot is dropped but at least + * this check will ensure that the given name is for a valid slot. + */ + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + slot = SearchNamedReplicationSlot(name); + LWLockRelease(ReplicationSlotControlLock); + + if (!slot) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("replication slot \"%s\" does not exist", + name))); + + /* + * Nothing to do for physical slots as we collect stats only for + * logical slots. + */ + if (SlotIsPhysical(slot)) + return; + + memcpy(&msg.m_slotname, name, NAMEDATALEN); + msg.clearall = false; + } + else + msg.clearall = true; + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_RESETREPLSLOTCOUNTER); + + pgstat_send(&msg, sizeof(msg)); +} + /* ---------- * pgstat_report_autovac() - * @@ -1637,6 +1700,46 @@ pgstat_report_tempfile(size_t filesize) pgstat_send(&msg, sizeof(msg)); } +/* ---------- + * pgstat_report_replslot() - + * + * Tell the collector about replication slot statistics. + * ---------- + */ +void +pgstat_report_replslot(const char *slotname, int spilltxns, int spillcount, + int spillbytes) +{ + PgStat_MsgReplSlot msg; + + /* + * Prepare and send the message + */ + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_REPLSLOT); + memcpy(&msg.m_slotname, slotname, NAMEDATALEN); + msg.m_drop = false; + msg.m_spill_txns = spilltxns; + msg.m_spill_count = spillcount; + msg.m_spill_bytes = spillbytes; + pgstat_send(&msg, sizeof(PgStat_MsgReplSlot)); +} + +/* ---------- + * pgstat_report_replslot_drop() - + * + * Tell the collector about dropping the replication slot. + * ---------- + */ +void +pgstat_report_replslot_drop(const char *slotname) +{ + PgStat_MsgReplSlot msg; + + pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_REPLSLOT); + memcpy(&msg.m_slotname, slotname, NAMEDATALEN); + msg.m_drop = true; + pgstat_send(&msg, sizeof(PgStat_MsgReplSlot)); +} /* ---------- * pgstat_ping() - @@ -2714,6 +2817,23 @@ pgstat_fetch_slru(void) return slruStats; } +/* + * --------- + * pgstat_fetch_replslot() - + * + * Support function for the SQL-callable pgstat* functions. Returns + * a pointer to the replication slot statistics struct and sets the + * number of entries in nslots_p. + * --------- + */ +PgStat_ReplSlotStats * +pgstat_fetch_replslot(int *nslots_p) +{ + backend_read_statsfile(); + + *nslots_p = nReplSlotStats; + return replSlotStats; +} /* ------------------------------------------------------------ * Functions for management of the shared-memory PgBackendStatus array @@ -4693,6 +4813,11 @@ PgstatCollectorMain(int argc, char *argv[]) len); break; + case PGSTAT_MTYPE_RESETREPLSLOTCOUNTER: + pgstat_recv_resetreplslotcounter(&msg.msg_resetreplslotcounter, + len); + break; + case PGSTAT_MTYPE_AUTOVAC_START: pgstat_recv_autovac(&msg.msg_autovacuum_start, len); break; @@ -4747,6 +4872,10 @@ PgstatCollectorMain(int argc, char *argv[]) len); break; + case PGSTAT_MTYPE_REPLSLOT: + pgstat_recv_replslot(&msg.msg_replslot, len); + break; + default: break; } @@ -4946,6 +5075,7 @@ pgstat_write_statsfiles(bool permanent, bool allDbs) const char *tmpfile = permanent ? PGSTAT_STAT_PERMANENT_TMPFILE : pgstat_stat_tmpname; const char *statfile = permanent ? PGSTAT_STAT_PERMANENT_FILENAME : pgstat_stat_filename; int rc; + int i; elog(DEBUG2, "writing stats file \"%s\"", statfile); @@ -5025,6 +5155,16 @@ pgstat_write_statsfiles(bool permanent, bool allDbs) (void) rc; /* we'll check for error with ferror */ } + /* + * Write replication slot stats struct + */ + for (i = 0; i < nReplSlotStats; i++) + { + fputc('R', fpout); + rc = fwrite(&replSlotStats[i], sizeof(PgStat_ReplSlotStats), 1, fpout); + (void) rc; /* we'll check for error with ferror */ + } + /* * No more output to be done. Close the temp file and replace the old * pgstat.stat with it. The ferror() check replaces testing for error @@ -5250,6 +5390,10 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) dbhash = hash_create("Databases hash", PGSTAT_DB_HASH_SIZE, &hash_ctl, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + /* Allocate the space for replication slot statistics */ + replSlotStats = palloc0(max_replication_slots * sizeof(PgStat_ReplSlotStats)); + nReplSlotStats = 0; + /* * Clear out global, archiver, WAL and SLRU statistics so they start from * zero in case we can't load an existing statsfile. @@ -5273,6 +5417,12 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) for (i = 0; i < SLRU_NUM_ELEMENTS; i++) slruStats[i].stat_reset_timestamp = globalStats.stat_reset_timestamp; + /* + * Set the same reset timestamp for all replication slots too. + */ + for (i = 0; i < max_replication_slots; i++) + replSlotStats[i].stat_reset_timestamp = globalStats.stat_reset_timestamp; + /* * Try to open the stats file. If it doesn't exist, the backends simply * return zero for anything and the collector simply starts from scratch @@ -5447,6 +5597,23 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) break; + /* + * 'R' A PgStat_ReplSlotStats struct describing a replication + * slot follows. + */ + case 'R': + if (fread(&replSlotStats[nReplSlotStats], 1, sizeof(PgStat_ReplSlotStats), fpin) + != sizeof(PgStat_ReplSlotStats)) + { + ereport(pgStatRunningInCollector ? LOG : WARNING, + (errmsg("corrupted statistics file \"%s\"", + statfile))); + memset(&replSlotStats[nReplSlotStats], 0, sizeof(PgStat_ReplSlotStats)); + goto done; + } + nReplSlotStats++; + break; + case 'E': goto done; @@ -5658,6 +5825,7 @@ pgstat_read_db_statsfile_timestamp(Oid databaseid, bool permanent, PgStat_ArchiverStats myArchiverStats; PgStat_WalStats myWalStats; PgStat_SLRUStats mySLRUStats[SLRU_NUM_ELEMENTS]; + PgStat_ReplSlotStats myReplSlotStats; FILE *fpin; int32 format_id; const char *statfile = permanent ? PGSTAT_STAT_PERMANENT_FILENAME : pgstat_stat_filename; @@ -5772,6 +5940,22 @@ pgstat_read_db_statsfile_timestamp(Oid databaseid, bool permanent, break; + /* + * 'R' A PgStat_ReplSlotStats struct describing a replication + * slot follows. + */ + case 'R': + if (fread(&myReplSlotStats, 1, sizeof(PgStat_ReplSlotStats), fpin) + != sizeof(PgStat_ReplSlotStats)) + { + ereport(pgStatRunningInCollector ? LOG : WARNING, + (errmsg("corrupted statistics file \"%s\"", + statfile))); + FreeFile(fpin); + return false; + } + break; + case 'E': goto done; @@ -6367,6 +6551,46 @@ pgstat_recv_resetslrucounter(PgStat_MsgResetslrucounter *msg, int len) } } +/* ---------- + * pgstat_recv_resetreplslotcounter() - + * + * Reset some replication slot statistics of the cluster. + * ---------- + */ +static void +pgstat_recv_resetreplslotcounter(PgStat_MsgResetreplslotcounter *msg, + int len) +{ + int i; + int idx = -1; + TimestampTz ts; + + ts = GetCurrentTimestamp(); + if (msg->clearall) + { + for (i = 0; i < nReplSlotStats; i++) + pgstat_reset_replslot(i, ts); + } + else + { + /* Get the index of replication slot statistics to reset */ + idx = pgstat_replslot_index(msg->m_slotname, false); + + /* + * Nothing to do if the given slot entry is not found. This could + * happen when the slot with the given name is removed and the + * corresponding statistics entry is also removed before receiving the + * reset message. + */ + if (idx < 0) + return; + + /* Reset the stats for the requested replication slot */ + pgstat_reset_replslot(idx, ts); + } +} + + /* ---------- * pgstat_recv_autovac() - * @@ -6626,6 +6850,51 @@ pgstat_recv_checksum_failure(PgStat_MsgChecksumFailure *msg, int len) dbentry->last_checksum_failure = msg->m_failure_time; } +/* ---------- + * pgstat_recv_replslot() - + * + * Process a REPLSLOT message. + * ---------- + */ +static void +pgstat_recv_replslot(PgStat_MsgReplSlot *msg, int len) +{ + int idx; + + /* + * Get the index of replication slot statistics. On dropping, we don't + * create the new statistics. + */ + idx = pgstat_replslot_index(msg->m_slotname, !msg->m_drop); + + /* + * The slot entry is not found or there is no space to accommodate the new + * entry. This could happen when the message for the creation of a slot + * reached before the drop message even though the actual operations + * happen in reverse order. In such a case, the next update of the + * statistics for the same slot will create the required entry. + */ + if (idx < 0) + return; + + Assert(idx >= 0 && idx <= max_replication_slots); + if (msg->m_drop) + { + /* Remove the replication slot statistics with the given name */ + memcpy(&replSlotStats[idx], &replSlotStats[nReplSlotStats - 1], + sizeof(PgStat_ReplSlotStats)); + nReplSlotStats--; + Assert(nReplSlotStats >= 0); + } + else + { + /* Update the replication slot statistics */ + replSlotStats[idx].spill_txns += msg->m_spill_txns; + replSlotStats[idx].spill_count += msg->m_spill_count; + replSlotStats[idx].spill_bytes += msg->m_spill_bytes; + } +} + /* ---------- * pgstat_recv_tempfile() - * @@ -6808,6 +7077,57 @@ pgstat_clip_activity(const char *raw_activity) return activity; } +/* ---------- + * pgstat_replslot_index + * + * Return the index of entry of a replication slot with the given name, or + * -1 if the slot is not found. + * + * create_it tells whether to create the new slot entry if it is not found. + * ---------- + */ +static int +pgstat_replslot_index(const char *name, bool create_it) +{ + int i; + + Assert(nReplSlotStats <= max_replication_slots); + for (i = 0; i < nReplSlotStats; i++) + { + if (strcmp(replSlotStats[i].slotname, name) == 0) + return i; /* found */ + } + + /* + * The slot is not found. We don't want to register the new statistics if + * the list is already full or the caller didn't request. + */ + if (i == max_replication_slots || !create_it) + return -1; + + /* Register new slot */ + memset(&replSlotStats[nReplSlotStats], 0, sizeof(PgStat_ReplSlotStats)); + memcpy(&replSlotStats[nReplSlotStats].slotname, name, NAMEDATALEN); + + return nReplSlotStats++; +} + +/* ---------- + * pgstat_reset_replslot + * + * Reset the replication slot stats at index 'i'. + * ---------- + */ +static void +pgstat_reset_replslot(int i, TimestampTz ts) +{ + /* reset only counters. Don't clear slot name */ + replSlotStats[i].spill_txns = 0; + replSlotStats[i].spill_count = 0; + replSlotStats[i].spill_bytes = 0; + replSlotStats[i].stat_reset_timestamp = ts; +} + /* * pgstat_slru_index * diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c index f21f61d5e10b..3f84ee99b863 100644 --- a/src/backend/replication/logical/decode.c +++ b/src/backend/replication/logical/decode.c @@ -650,6 +650,12 @@ DecodeCommit(LogicalDecodingContext *ctx, XLogRecordBuffer *buf, /* replay actions of all transaction + subtransactions in order */ ReorderBufferCommit(ctx->reorder, xid, buf->origptr, buf->endptr, commit_time, origin_id, origin_lsn); + + /* + * Update the decoding stats at transaction commit/abort. It is not clear + * that sending more or less frequently than this would be better. + */ + UpdateDecodingStats(ctx); } /* @@ -669,6 +675,9 @@ DecodeAbort(LogicalDecodingContext *ctx, XLogRecordBuffer *buf, } ReorderBufferAbort(ctx->reorder, xid, buf->record->EndRecPtr); + + /* update the decoding stats */ + UpdateDecodingStats(ctx); } /* diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 0f6af952f939..3346df32d3b4 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -32,6 +32,7 @@ #include "access/xlog_internal.h" #include "fmgr.h" #include "miscadmin.h" +#include "pgstat.h" #include "replication/decode.h" #include "replication/logical.h" #include "replication/origin.h" @@ -1460,3 +1461,31 @@ ResetLogicalStreamingState(void) CheckXidAlive = InvalidTransactionId; bsysscan = false; } + +/* + * Report stats for a slot. + */ +void +UpdateDecodingStats(LogicalDecodingContext *ctx) +{ + ReorderBuffer *rb = ctx->reorder; + + /* + * Nothing to do if we haven't spilled anything since the last time the + * stats has been sent. + */ + if (rb->spillBytes <= 0) + return; + + elog(DEBUG2, "UpdateSpillStats: updating stats %p %lld %lld %lld", + rb, + (long long) rb->spillTxns, + (long long) rb->spillCount, + (long long) rb->spillBytes); + + pgstat_report_replslot(NameStr(ctx->slot->data.name), + rb->spillTxns, rb->spillCount, rb->spillBytes); + rb->spillTxns = 0; + rb->spillCount = 0; + rb->spillBytes = 0; +} diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index 1975d629a6e2..189641bbf5b1 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -343,6 +343,10 @@ ReorderBufferAllocate(void) buffer->outbufsize = 0; buffer->size = 0; + buffer->spillTxns = 0; + buffer->spillCount = 0; + buffer->spillBytes = 0; + buffer->current_restart_decoding_lsn = InvalidXLogRecPtr; dlist_init(&buffer->toplevel_by_lsn); @@ -1579,6 +1583,13 @@ ReorderBufferTruncateTXN(ReorderBuffer *rb, ReorderBufferTXN *txn) { ReorderBufferRestoreCleanup(rb, txn); txn->txn_flags &= ~RBTXN_IS_SERIALIZED; + + /* + * We set this flag to indicate if the transaction is ever serialized. + * We need this to accurately update the stats as otherwise the same + * transaction can be counted as serialized multiple times. + */ + txn->txn_flags |= RBTXN_IS_SERIALIZED_CLEAR; } /* also reset the number of entries in the transaction */ @@ -3112,6 +3123,7 @@ ReorderBufferSerializeTXN(ReorderBuffer *rb, ReorderBufferTXN *txn) int fd = -1; XLogSegNo curOpenSegNo = 0; Size spilled = 0; + Size size = txn->size; elog(DEBUG2, "spill %u changes in XID %u to disk", (uint32) txn->nentries_mem, txn->xid); @@ -3170,6 +3182,16 @@ ReorderBufferSerializeTXN(ReorderBuffer *rb, ReorderBufferTXN *txn) spilled++; } + /* update the statistics iff we have spilled anything */ + if (spilled) + { + rb->spillCount += 1; + rb->spillBytes += size; + + /* don't consider already serialized transactions */ + rb->spillTxns += (rbtxn_is_serialized(txn) || rbtxn_is_serialized_clear(txn)) ? 0 : 1; + } + Assert(spilled == txn->nentries_mem); Assert(dlist_is_empty(&txn->changes)); txn->nentries_mem = 0; diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 42c78eabd4eb..220b4cd6e99c 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -99,7 +99,6 @@ ReplicationSlot *MyReplicationSlot = NULL; int max_replication_slots = 0; /* the maximum number of replication * slots */ -static ReplicationSlot *SearchNamedReplicationSlot(const char *name); static int ReplicationSlotAcquireInternal(ReplicationSlot *slot, const char *name, SlotAcquireBehavior behavior); static void ReplicationSlotDropAcquired(void); @@ -314,6 +313,15 @@ ReplicationSlotCreate(const char *name, bool db_specific, LWLockRelease(ReplicationSlotControlLock); + /* + * Create statistics entry for the new logical slot. We don't collect any + * stats for physical slots, so no need to create an entry for the same. + * See ReplicationSlotDropPtr for why we need to do this before releasing + * ReplicationSlotAllocationLock. + */ + if (SlotIsLogical(slot)) + pgstat_report_replslot(NameStr(slot->data.name), 0, 0, 0); + /* * Now that the slot has been marked as in_use and active, it's safe to * let somebody else try to allocate a slot. @@ -331,7 +339,7 @@ ReplicationSlotCreate(const char *name, bool db_specific, * * The caller must hold ReplicationSlotControlLock in shared mode. */ -static ReplicationSlot * +ReplicationSlot * SearchNamedReplicationSlot(const char *name) { int i; @@ -683,6 +691,19 @@ ReplicationSlotDropPtr(ReplicationSlot *slot) ereport(WARNING, (errmsg("could not remove directory \"%s\"", tmppath))); + /* + * Send a message to drop the replication slot to the stats collector. + * Since there is no guarantee of the order of message transfer on a UDP + * connection, it's possible that a message for creating a new slot + * reaches before a message for removing the old slot. We send the drop + * and create messages while holding ReplicationSlotAllocationLock to + * reduce that possibility. If the messages reached in reverse, we would + * lose one statistics update message. But the next update message will + * create the statistics for the replication slot. + */ + if (SlotIsLogical(slot)) + pgstat_report_replslot_drop(NameStr(slot->data.name)); + /* * We release this at the very end, so that nobody starts trying to create * a slot while we're still cleaning up the detritus of the old one. diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 24e191ea30b6..0d0d2e6d2bb2 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2069,6 +2069,20 @@ pg_stat_reset_slru(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +/* Reset replication slots stats (a specific one or all of them). */ +Datum +pg_stat_reset_replication_slot(PG_FUNCTION_ARGS) +{ + char *target = NULL; + + if (!PG_ARGISNULL(0)) + target = text_to_cstring(PG_GETARG_TEXT_PP(0)); + + pgstat_reset_replslot_counter(target); + + PG_RETURN_VOID(); +} + Datum pg_stat_get_archiver(PG_FUNCTION_ARGS) { @@ -2134,3 +2148,69 @@ pg_stat_get_archiver(PG_FUNCTION_ARGS) /* Returns the record as Datum */ PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); } + +/* Get the statistics for the replication slots */ +Datum +pg_stat_get_replication_slots(PG_FUNCTION_ARGS) +{ +#define PG_STAT_GET_REPLICATION_SLOT_CLOS 5 + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + TupleDesc tupdesc; + Tuplestorestate *tupstore; + MemoryContext per_query_ctx; + MemoryContext oldcontext; + PgStat_ReplSlotStats *slotstats; + int nstats; + int i; + + /* check to see if caller supports us returning a tuplestore */ + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that cannot accept a set"))); + if (!(rsinfo->allowedModes & SFRM_Materialize)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("materialize mode required, but it is not allowed in this context"))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; + oldcontext = MemoryContextSwitchTo(per_query_ctx); + + tupstore = tuplestore_begin_heap(true, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; + + MemoryContextSwitchTo(oldcontext); + + slotstats = pgstat_fetch_replslot(&nstats); + for (i = 0; i < nstats; i++) + { + Datum values[PG_STAT_GET_REPLICATION_SLOT_CLOS]; + bool nulls[PG_STAT_GET_REPLICATION_SLOT_CLOS]; + PgStat_ReplSlotStats *s = &(slotstats[i]); + + MemSet(values, 0, sizeof(values)); + MemSet(nulls, 0, sizeof(nulls)); + + values[0] = PointerGetDatum(cstring_to_text(s->slotname)); + values[1] = Int64GetDatum(s->spill_txns); + values[2] = Int64GetDatum(s->spill_count); + values[3] = Int64GetDatum(s->spill_bytes); + + if (s->stat_reset_timestamp == 0) + nulls[4] = true; + else + values[4] = TimestampTzGetDatum(s->stat_reset_timestamp); + + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } + + tuplestore_donestoring(tupstore); + + return (Datum) 0; +} diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 1be017972438..584c61728439 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 202010021 +#define CATALOG_VERSION_NO 202010081 #endif diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index d6f3e2d286b4..22340baf1c67 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5257,6 +5257,14 @@ proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', proargnames => '{pid,status,receive_start_lsn,receive_start_tli,written_lsn,flushed_lsn,received_tli,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time,slot_name,sender_host,sender_port,conninfo}', prosrc => 'pg_stat_get_wal_receiver' }, +{ oid => '8595', descr => 'statistics: information about replication slots', + proname => 'pg_stat_get_replication_slots', prorows => '10', proisstrict => 'f', + proretset => 't', provolatile => 's', proparallel => 'r', + prorettype => 'record', proargtypes => '', + proallargtypes => '{text,int8,int8,int8,timestamptz}', + proargmodes => '{o,o,o,o,o}', + proargnames => '{name,spill_txns,spill_count,spill_bytes,stats_reset}', + prosrc => 'pg_stat_get_replication_slots' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', proisstrict => 'f', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'oid', @@ -5606,6 +5614,10 @@ descr => 'statistics: reset collected statistics for a single SLRU', proname => 'pg_stat_reset_slru', proisstrict => 'f', provolatile => 'v', prorettype => 'void', proargtypes => 'text', prosrc => 'pg_stat_reset_slru' }, +{ oid => '8596', + descr => 'statistics: reset collected statistics for a single replication slot', + proname => 'pg_stat_reset_replication_slot', proisstrict => 'f', provolatile => 'v', + prorettype => 'void', proargtypes => 'text', prosrc => 'pg_stat_reset_replication_slot' }, { oid => '3163', descr => 'current trigger depth', proname => 'pg_trigger_depth', provolatile => 's', proparallel => 'r', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 343eef507eab..a821ff4f158f 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -56,6 +56,7 @@ typedef enum StatMsgType PGSTAT_MTYPE_RESETSHAREDCOUNTER, PGSTAT_MTYPE_RESETSINGLECOUNTER, PGSTAT_MTYPE_RESETSLRUCOUNTER, + PGSTAT_MTYPE_RESETREPLSLOTCOUNTER, PGSTAT_MTYPE_AUTOVAC_START, PGSTAT_MTYPE_VACUUM, PGSTAT_MTYPE_ANALYZE, @@ -68,7 +69,8 @@ typedef enum StatMsgType PGSTAT_MTYPE_RECOVERYCONFLICT, PGSTAT_MTYPE_TEMPFILE, PGSTAT_MTYPE_DEADLOCK, - PGSTAT_MTYPE_CHECKSUMFAILURE + PGSTAT_MTYPE_CHECKSUMFAILURE, + PGSTAT_MTYPE_REPLSLOT, } StatMsgType; /* ---------- @@ -358,6 +360,18 @@ typedef struct PgStat_MsgResetslrucounter int m_index; } PgStat_MsgResetslrucounter; +/* ---------- + * PgStat_MsgResetreplslotcounter Sent by the backend to tell the collector + * to reset replication slot counter(s) + * ---------- + */ +typedef struct PgStat_MsgResetreplslotcounter +{ + PgStat_MsgHdr m_hdr; + char m_slotname[NAMEDATALEN]; + bool clearall; +} PgStat_MsgResetreplslotcounter; + /* ---------- * PgStat_MsgAutovacStart Sent by the autovacuum daemon to signal * that a database is going to be processed @@ -465,6 +479,22 @@ typedef struct PgStat_MsgSLRU PgStat_Counter m_truncate; } PgStat_MsgSLRU; +/* ---------- + * PgStat_MsgReplSlot Sent by a backend or a wal sender to update replication + * slot statistics. + * ---------- + */ +typedef struct PgStat_MsgReplSlot +{ + PgStat_MsgHdr m_hdr; + char m_slotname[NAMEDATALEN]; + bool m_drop; + PgStat_Counter m_spill_txns; + PgStat_Counter m_spill_count; + PgStat_Counter m_spill_bytes; +} PgStat_MsgReplSlot; + + /* ---------- * PgStat_MsgRecoveryConflict Sent by the backend upon recovery conflict * ---------- @@ -603,6 +633,7 @@ typedef union PgStat_Msg PgStat_MsgResetsharedcounter msg_resetsharedcounter; PgStat_MsgResetsinglecounter msg_resetsinglecounter; PgStat_MsgResetslrucounter msg_resetslrucounter; + PgStat_MsgResetreplslotcounter msg_resetreplslotcounter; PgStat_MsgAutovacStart msg_autovacuum_start; PgStat_MsgVacuum msg_vacuum; PgStat_MsgAnalyze msg_analyze; @@ -616,6 +647,7 @@ typedef union PgStat_Msg PgStat_MsgDeadlock msg_deadlock; PgStat_MsgTempFile msg_tempfile; PgStat_MsgChecksumFailure msg_checksumfailure; + PgStat_MsgReplSlot msg_replslot; } PgStat_Msg; @@ -627,7 +659,7 @@ typedef union PgStat_Msg * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BC9E +#define PGSTAT_FILE_FORMAT_ID 0x01A5BC9F /* ---------- * PgStat_StatDBEntry The collector's data per database @@ -782,6 +814,17 @@ typedef struct PgStat_SLRUStats TimestampTz stat_reset_timestamp; } PgStat_SLRUStats; +/* + * Replication slot statistics kept in the stats collector + */ +typedef struct PgStat_ReplSlotStats +{ + char slotname[NAMEDATALEN]; + PgStat_Counter spill_txns; + PgStat_Counter spill_count; + PgStat_Counter spill_bytes; + TimestampTz stat_reset_timestamp; +} PgStat_ReplSlotStats; /* ---------- * Backend states @@ -1330,6 +1373,7 @@ extern void pgstat_reset_counters(void); extern void pgstat_reset_shared_counters(const char *); extern void pgstat_reset_single_counter(Oid objectid, PgStat_Single_Reset_Type type); extern void pgstat_reset_slru_counter(const char *); +extern void pgstat_reset_replslot_counter(const char *name); extern void pgstat_report_autovac(Oid dboid); extern void pgstat_report_vacuum(Oid tableoid, bool shared, @@ -1342,6 +1386,9 @@ extern void pgstat_report_recovery_conflict(int reason); extern void pgstat_report_deadlock(void); extern void pgstat_report_checksum_failures_in_db(Oid dboid, int failurecount); extern void pgstat_report_checksum_failure(void); +extern void pgstat_report_replslot(const char *slotname, int spilltxns, int spillcount, + int spillbytes); +extern void pgstat_report_replslot_drop(const char *slotname); extern void pgstat_initialize(void); extern void pgstat_bestart(void); @@ -1508,6 +1555,7 @@ extern PgStat_ArchiverStats *pgstat_fetch_stat_archiver(void); extern PgStat_GlobalStats *pgstat_fetch_global(void); extern PgStat_WalStats *pgstat_fetch_stat_wal(void); extern PgStat_SLRUStats *pgstat_fetch_slru(void); +extern PgStat_ReplSlotStats *pgstat_fetch_replslot(int *nslots_p); extern void pgstat_count_slru_page_zeroed(int slru_idx); extern void pgstat_count_slru_page_hit(int slru_idx); diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h index 45abc444b7a5..40bab7ee02df 100644 --- a/src/include/replication/logical.h +++ b/src/include/replication/logical.h @@ -122,5 +122,6 @@ extern void LogicalConfirmReceivedLocation(XLogRecPtr lsn); extern bool filter_by_origin_cb_wrapper(LogicalDecodingContext *ctx, RepOriginId origin_id); extern void ResetLogicalStreamingState(void); +extern void UpdateDecodingStats(LogicalDecodingContext *ctx); #endif diff --git a/src/include/replication/reorderbuffer.h b/src/include/replication/reorderbuffer.h index 1ae17d5f11fd..0cc3aebb1113 100644 --- a/src/include/replication/reorderbuffer.h +++ b/src/include/replication/reorderbuffer.h @@ -162,9 +162,10 @@ typedef struct ReorderBufferChange #define RBTXN_HAS_CATALOG_CHANGES 0x0001 #define RBTXN_IS_SUBXACT 0x0002 #define RBTXN_IS_SERIALIZED 0x0004 -#define RBTXN_IS_STREAMED 0x0008 -#define RBTXN_HAS_TOAST_INSERT 0x0010 -#define RBTXN_HAS_SPEC_INSERT 0x0020 +#define RBTXN_IS_SERIALIZED_CLEAR 0x0008 +#define RBTXN_IS_STREAMED 0x0010 +#define RBTXN_HAS_TOAST_INSERT 0x0020 +#define RBTXN_HAS_SPEC_INSERT 0x0040 /* Does the transaction have catalog changes? */ #define rbtxn_has_catalog_changes(txn) \ @@ -184,6 +185,12 @@ typedef struct ReorderBufferChange ((txn)->txn_flags & RBTXN_IS_SERIALIZED) != 0 \ ) +/* Has this transaction ever been spilled to disk? */ +#define rbtxn_is_serialized_clear(txn) \ +( \ + ((txn)->txn_flags & RBTXN_IS_SERIALIZED_CLEAR) != 0 \ +) + /* This transaction's changes has toast insert, without main table insert. */ #define rbtxn_has_toast_insert(txn) \ ( \ @@ -525,6 +532,17 @@ struct ReorderBuffer /* memory accounting */ Size size; + + /* + * Statistics about transactions spilled to disk. + * + * A single transaction may be spilled repeatedly, which is why we keep + * two different counters. For spilling, the transaction counter includes + * both toplevel transactions and subtransactions. + */ + int64 spillTxns; /* number of transactions spilled to disk */ + int64 spillCount; /* spill-to-disk invocation counter */ + int64 spillBytes; /* amount of data spilled to disk */ }; diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index 31362585ecb1..63bab6967fb5 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -210,6 +210,7 @@ extern XLogRecPtr ReplicationSlotsComputeLogicalRestartLSN(void); extern bool ReplicationSlotsCountDBSlots(Oid dboid, int *nslots, int *nactive); extern void ReplicationSlotsDropDBSlots(Oid dboid); extern void InvalidateObsoleteReplicationSlots(XLogSegNo oldestSegno); +extern ReplicationSlot *SearchNamedReplicationSlot(const char *name); extern void StartupReplicationSlots(void); extern void CheckPointReplicationSlots(void); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index af4192f9a872..cf2a9b44082e 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2018,6 +2018,12 @@ pg_stat_replication| SELECT s.pid, FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, sslcompression, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid) JOIN pg_stat_get_wal_senders() w(pid, state, sent_lsn, write_lsn, flush_lsn, replay_lsn, write_lag, flush_lag, replay_lag, sync_priority, sync_state, reply_time) ON ((s.pid = w.pid))) LEFT JOIN pg_authid u ON ((s.usesysid = u.oid))); +pg_stat_replication_slots| SELECT s.name, + s.spill_txns, + s.spill_count, + s.spill_bytes, + s.stats_reset + FROM pg_stat_get_replication_slots() s(name, spill_txns, spill_count, spill_bytes, stats_reset); pg_stat_slru| SELECT s.name, s.blks_zeroed, s.blks_hit, diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 4191f9486963..be570329ea73 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1832,7 +1832,9 @@ PgStat_MsgFuncstat PgStat_MsgHdr PgStat_MsgInquiry PgStat_MsgRecoveryConflict +PgStat_MsgReplSlot PgStat_MsgResetcounter +PgStat_MsgResetreplslotcounter PgStat_MsgResetsharedcounter PgStat_MsgResetsinglecounter PgStat_MsgResetslrucounter @@ -1842,6 +1844,7 @@ PgStat_MsgTabstat PgStat_MsgTempFile PgStat_MsgVacuum PgStat_MsgWal +PgStat_ReplSlotStats PgStat_SLRUStats PgStat_Shared_Reset_Target PgStat_Single_Reset_Type From 2a7316458164369436e252e5e60a5957b17103c3 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 8 Oct 2020 13:16:43 +0900 Subject: [PATCH 267/589] Improve set of candidate multipliers for perfect hash function generation The previous set of multipliers was not adapted for large sets of short keys, and this new set of multipliers allows to generate perfect hash functions for larger sets without having an impact for existing callers of those functions, as experimentation has showed. A future commit will make use of that to improve the performance of unicode normalization. All multipliers compile to shift-and-add instructions on most platforms. This has been tested as far back as gcc 4.1 and clang 3.8. Author: John Naylor Reviewed-by: Mark Dilger, Michael Paquier Discussion: https://postgr.es/m/CACPNZCt4fbJ0_bGrN5QPt34N4whv=mszM0LMVQdoa2rC9UMRXA@mail.gmail.com --- src/tools/PerfectHash.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tools/PerfectHash.pm b/src/tools/PerfectHash.pm index 74fb1f2ef628..d6841589a39b 100644 --- a/src/tools/PerfectHash.pm +++ b/src/tools/PerfectHash.pm @@ -81,13 +81,13 @@ sub generate_hash_function # to calculate via shift-and-add, so don't change them without care. # (Commonly, random seeds are tried, but we want reproducible results # from this program so we don't do that.) - my $hash_mult1 = 31; + my $hash_mult1 = 257; my $hash_mult2; my $hash_seed1; my $hash_seed2; my @subresult; FIND_PARAMS: - foreach (127, 257, 521, 1033, 2053) + foreach (17, 31, 127, 8191) { $hash_mult2 = $_; # "foreach $hash_mult2" doesn't work for ($hash_seed1 = 0; $hash_seed1 < 10; $hash_seed1++) From b90b79e1409b7cbffcadf89ae2e85c7ba1332818 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 8 Oct 2020 14:06:12 +0900 Subject: [PATCH 268/589] Fix typo in multixact.c AtEOXact_MultiXact() was referenced in two places with an incorrect routine name. Author: Hou Zhijie Discussion: https://postgr.es/m/1b41e9311e8f474cb5a360292f0b3cb1@G08CNEXMBPEKD05.g08.fujitsu.local --- src/backend/access/transam/multixact.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/access/transam/multixact.c b/src/backend/access/transam/multixact.c index a2ce617c8ce2..6ccdc5b58cba 100644 --- a/src/backend/access/transam/multixact.c +++ b/src/backend/access/transam/multixact.c @@ -1742,7 +1742,7 @@ PostPrepare_MultiXact(TransactionId xid) OldestVisibleMXactId[MyBackendId] = InvalidMultiXactId; /* - * Discard the local MultiXactId cache like in AtEOX_MultiXact + * Discard the local MultiXactId cache like in AtEOXact_MultiXact. */ MXactContext = NULL; dlist_init(&MXactCache); @@ -1772,7 +1772,7 @@ multixact_twophase_recover(TransactionId xid, uint16 info, /* * multixact_twophase_postcommit - * Similar to AtEOX_MultiXact but for COMMIT PREPARED + * Similar to AtEOXact_MultiXact but for COMMIT PREPARED */ void multixact_twophase_postcommit(TransactionId xid, uint16 info, From 8ce423b1912b8303dbec5dc3ec78a7a725acf6c2 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 8 Oct 2020 12:37:59 -0400 Subject: [PATCH 269/589] Fix numeric width_bucket() to allow its first argument to be infinite. While the calculation is not well-defined if the bounds arguments are infinite, there is a perfectly sane outcome if the test operand is infinite: it's just like any other value that's before the first bucket or after the last one. width_bucket_float8() got this right, but I was too hasty about the case when adding infinities to numerics (commit a57d312a7), so that width_bucket_numeric() just rejected it. Fix that, and sync the relevant error message strings. No back-patch needed, since infinities-in-numeric haven't shipped yet. Discussion: https://postgr.es/m/2465409.1602170063@sss.pgh.pa.us --- src/backend/utils/adt/numeric.c | 5 +++-- src/test/regress/expected/numeric.out | 17 +++++++++++++---- src/test/regress/sql/numeric.sql | 7 +++++-- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c index 27d65557dfa9..a4692d8cfc05 100644 --- a/src/backend/utils/adt/numeric.c +++ b/src/backend/utils/adt/numeric.c @@ -1724,10 +1724,11 @@ width_bucket_numeric(PG_FUNCTION_ARGS) ereport(ERROR, (errcode(ERRCODE_INVALID_ARGUMENT_FOR_WIDTH_BUCKET_FUNCTION), errmsg("operand, lower bound, and upper bound cannot be NaN"))); - else + /* We allow "operand" to be infinite; cmp_numerics will cope */ + if (NUMERIC_IS_INF(bound1) || NUMERIC_IS_INF(bound2)) ereport(ERROR, (errcode(ERRCODE_INVALID_ARGUMENT_FOR_WIDTH_BUCKET_FUNCTION), - errmsg("operand, lower bound, and upper bound cannot be infinity"))); + errmsg("lower and upper bounds must be finite"))); } init_var(&result_var); diff --git a/src/test/regress/expected/numeric.out b/src/test/regress/expected/numeric.out index 823f3fbccf51..70d6cfe4232c 100644 --- a/src/test/regress/expected/numeric.out +++ b/src/test/regress/expected/numeric.out @@ -1316,10 +1316,8 @@ SELECT width_bucket('NaN', 3.0, 4.0, 888); ERROR: operand, lower bound, and upper bound cannot be NaN SELECT width_bucket(0::float8, 'NaN', 4.0::float8, 888); ERROR: operand, lower bound, and upper bound cannot be NaN -SELECT width_bucket('inf', 3.0, 4.0, 888); -ERROR: operand, lower bound, and upper bound cannot be infinity SELECT width_bucket(2.0, 3.0, '-inf', 888); -ERROR: operand, lower bound, and upper bound cannot be infinity +ERROR: lower and upper bounds must be finite SELECT width_bucket(0::float8, '-inf', 4.0::float8, 888); ERROR: lower and upper bounds must be finite -- normal operation @@ -1362,8 +1360,19 @@ SELECT 10.0000000000001 | 6 | 6 | 0 | 0 | 5 | 5 | 21 | 21 | 8 | 8 (19 rows) --- for float8 only, check positive and negative infinity: we require +-- Check positive and negative infinity: we require -- finite bucket bounds, but allow an infinite operand +SELECT width_bucket(0.0::numeric, 'Infinity'::numeric, 5, 10); -- error +ERROR: lower and upper bounds must be finite +SELECT width_bucket(0.0::numeric, 5, '-Infinity'::numeric, 20); -- error +ERROR: lower and upper bounds must be finite +SELECT width_bucket('Infinity'::numeric, 1, 10, 10), + width_bucket('-Infinity'::numeric, 1, 10, 10); + width_bucket | width_bucket +--------------+-------------- + 11 | 0 +(1 row) + SELECT width_bucket(0.0::float8, 'Infinity'::float8, 5, 10); -- error ERROR: lower and upper bounds must be finite SELECT width_bucket(0.0::float8, 5, '-Infinity'::float8, 20); -- error diff --git a/src/test/regress/sql/numeric.sql b/src/test/regress/sql/numeric.sql index 5eac895e86e1..520e23a9f554 100644 --- a/src/test/regress/sql/numeric.sql +++ b/src/test/regress/sql/numeric.sql @@ -831,7 +831,6 @@ SELECT width_bucket(5.0::float8, 3.0::float8, 4.0::float8, -5); SELECT width_bucket(3.5::float8, 3.0::float8, 3.0::float8, 888); SELECT width_bucket('NaN', 3.0, 4.0, 888); SELECT width_bucket(0::float8, 'NaN', 4.0::float8, 888); -SELECT width_bucket('inf', 3.0, 4.0, 888); SELECT width_bucket(2.0, 3.0, '-inf', 888); SELECT width_bucket(0::float8, '-inf', 4.0::float8, 888); @@ -876,8 +875,12 @@ SELECT width_bucket(operand_f8, -25, 25, 10) AS wb_5f FROM width_bucket_test; --- for float8 only, check positive and negative infinity: we require +-- Check positive and negative infinity: we require -- finite bucket bounds, but allow an infinite operand +SELECT width_bucket(0.0::numeric, 'Infinity'::numeric, 5, 10); -- error +SELECT width_bucket(0.0::numeric, 5, '-Infinity'::numeric, 20); -- error +SELECT width_bucket('Infinity'::numeric, 1, 10, 10), + width_bucket('-Infinity'::numeric, 1, 10, 10); SELECT width_bucket(0.0::float8, 'Infinity'::float8, 5, 10); -- error SELECT width_bucket(0.0::float8, 5, '-Infinity'::float8, 20); -- error SELECT width_bucket('Infinity'::float8, 1, 10, 10), From 7538708394e7a70105a4e601e253adf80f47cca8 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 8 Oct 2020 13:06:27 -0400 Subject: [PATCH 270/589] Avoid gratuitous inaccuracy in numeric width_bucket(). Multiply before dividing, not the reverse, so that cases that should produce exact results do produce exact results. (width_bucket_float8 got this right already.) Even when the result is inexact, this avoids making it more inexact, since only the division step introduces any imprecision. While at it, fix compute_bucket() to not uselessly repeat the sign check already done by its caller, and avoid duplicating the multiply/divide steps by adjusting variable usage. Per complaint from Martin Visser. Although this seems like a bug fix, I'm hesitant to risk changing width_bucket()'s results in stable branches, so no back-patch. Discussion: https://postgr.es/m/6FA5117D-6AED-4656-8FEF-B74AC18FAD85@brytlyt.com --- src/backend/utils/adt/numeric.c | 31 +++++++++++---------- src/test/regress/expected/numeric.out | 40 +++++++++++++++++++++++++++ src/test/regress/sql/numeric.sql | 9 ++++++ 3 files changed, 65 insertions(+), 15 deletions(-) diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c index a4692d8cfc05..20c9cac2fa2e 100644 --- a/src/backend/utils/adt/numeric.c +++ b/src/backend/utils/adt/numeric.c @@ -592,7 +592,8 @@ static void round_var(NumericVar *var, int rscale); static void trunc_var(NumericVar *var, int rscale); static void strip_var(NumericVar *var); static void compute_bucket(Numeric operand, Numeric bound1, Numeric bound2, - const NumericVar *count_var, NumericVar *result_var); + const NumericVar *count_var, bool reversed_bounds, + NumericVar *result_var); static void accum_sum_add(NumericSumAccum *accum, const NumericVar *var1); static void accum_sum_rescale(NumericSumAccum *accum, const NumericVar *val); @@ -1752,8 +1753,8 @@ width_bucket_numeric(PG_FUNCTION_ARGS) else if (cmp_numerics(operand, bound2) >= 0) add_var(&count_var, &const_one, &result_var); else - compute_bucket(operand, bound1, bound2, - &count_var, &result_var); + compute_bucket(operand, bound1, bound2, &count_var, false, + &result_var); break; /* bound1 > bound2 */ @@ -1763,8 +1764,8 @@ width_bucket_numeric(PG_FUNCTION_ARGS) else if (cmp_numerics(operand, bound2) <= 0) add_var(&count_var, &const_one, &result_var); else - compute_bucket(operand, bound1, bound2, - &count_var, &result_var); + compute_bucket(operand, bound1, bound2, &count_var, true, + &result_var); break; } @@ -1783,11 +1784,13 @@ width_bucket_numeric(PG_FUNCTION_ARGS) /* * If 'operand' is not outside the bucket range, determine the correct * bucket for it to go. The calculations performed by this function - * are derived directly from the SQL2003 spec. + * are derived directly from the SQL2003 spec. Note however that we + * multiply by count before dividing, to avoid unnecessary roundoff error. */ static void compute_bucket(Numeric operand, Numeric bound1, Numeric bound2, - const NumericVar *count_var, NumericVar *result_var) + const NumericVar *count_var, bool reversed_bounds, + NumericVar *result_var) { NumericVar bound1_var; NumericVar bound2_var; @@ -1797,23 +1800,21 @@ compute_bucket(Numeric operand, Numeric bound1, Numeric bound2, init_var_from_num(bound2, &bound2_var); init_var_from_num(operand, &operand_var); - if (cmp_var(&bound1_var, &bound2_var) < 0) + if (!reversed_bounds) { sub_var(&operand_var, &bound1_var, &operand_var); sub_var(&bound2_var, &bound1_var, &bound2_var); - div_var(&operand_var, &bound2_var, result_var, - select_div_scale(&operand_var, &bound2_var), true); } else { sub_var(&bound1_var, &operand_var, &operand_var); - sub_var(&bound1_var, &bound2_var, &bound1_var); - div_var(&operand_var, &bound1_var, result_var, - select_div_scale(&operand_var, &bound1_var), true); + sub_var(&bound1_var, &bound2_var, &bound2_var); } - mul_var(result_var, count_var, result_var, - result_var->dscale + count_var->dscale); + mul_var(&operand_var, count_var, &operand_var, + operand_var.dscale + count_var->dscale); + div_var(&operand_var, &bound2_var, result_var, + select_div_scale(&operand_var, &bound2_var), true); add_var(result_var, &const_one, result_var); floor_var(result_var, result_var); diff --git a/src/test/regress/expected/numeric.out b/src/test/regress/expected/numeric.out index 70d6cfe4232c..cb782d0e2a8f 100644 --- a/src/test/regress/expected/numeric.out +++ b/src/test/regress/expected/numeric.out @@ -1385,6 +1385,46 @@ SELECT width_bucket('Infinity'::float8, 1, 10, 10), (1 row) DROP TABLE width_bucket_test; +-- Simple test for roundoff error when results should be exact +SELECT x, width_bucket(x::float8, 10, 100, 9) as flt, + width_bucket(x::numeric, 10, 100, 9) as num +FROM generate_series(0, 110, 10) x; + x | flt | num +-----+-----+----- + 0 | 0 | 0 + 10 | 1 | 1 + 20 | 2 | 2 + 30 | 3 | 3 + 40 | 4 | 4 + 50 | 5 | 5 + 60 | 6 | 6 + 70 | 7 | 7 + 80 | 8 | 8 + 90 | 9 | 9 + 100 | 10 | 10 + 110 | 10 | 10 +(12 rows) + +SELECT x, width_bucket(x::float8, 100, 10, 9) as flt, + width_bucket(x::numeric, 100, 10, 9) as num +FROM generate_series(0, 110, 10) x; + x | flt | num +-----+-----+----- + 0 | 10 | 10 + 10 | 10 | 10 + 20 | 9 | 9 + 30 | 8 | 8 + 40 | 7 | 7 + 50 | 6 | 6 + 60 | 5 | 5 + 70 | 4 | 4 + 80 | 3 | 3 + 90 | 2 | 2 + 100 | 1 | 1 + 110 | 0 | 0 +(12 rows) + +-- -- TO_CHAR() -- SELECT '' AS to_char_1, to_char(val, '9G999G999G999G999G999') diff --git a/src/test/regress/sql/numeric.sql b/src/test/regress/sql/numeric.sql index 520e23a9f554..76969db22a7d 100644 --- a/src/test/regress/sql/numeric.sql +++ b/src/test/regress/sql/numeric.sql @@ -888,6 +888,15 @@ SELECT width_bucket('Infinity'::float8, 1, 10, 10), DROP TABLE width_bucket_test; +-- Simple test for roundoff error when results should be exact +SELECT x, width_bucket(x::float8, 10, 100, 9) as flt, + width_bucket(x::numeric, 10, 100, 9) as num +FROM generate_series(0, 110, 10) x; +SELECT x, width_bucket(x::float8, 100, 10, 9) as flt, + width_bucket(x::numeric, 100, 10, 9) as num +FROM generate_series(0, 110, 10) x; + +-- -- TO_CHAR() -- SELECT '' AS to_char_1, to_char(val, '9G999G999G999G999G999') From f13f2e484172a1c865cd067796cee3568467dd51 Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Fri, 9 Oct 2020 08:15:53 +0530 Subject: [PATCH 271/589] Fix typos in logical.c and reorderbuffer.c. Reviewed-by: Sawada Masahiko Discussion: https://postgr.es/m/CAA4eK1K6zTpuqf_d7wXCBjo_EF0_B6Fz3Ecp71Vq18t=wG-nzg@mail.gmail.com --- src/backend/replication/logical/logical.c | 2 +- src/backend/replication/logical/reorderbuffer.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 3346df32d3b4..8675832f4d6e 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -1477,7 +1477,7 @@ UpdateDecodingStats(LogicalDecodingContext *ctx) if (rb->spillBytes <= 0) return; - elog(DEBUG2, "UpdateSpillStats: updating stats %p %lld %lld %lld", + elog(DEBUG2, "UpdateDecodingStats: updating stats %p %lld %lld %lld", rb, (long long) rb->spillTxns, (long long) rb->spillCount, diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index 189641bbf5b1..4cb27f222445 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -1432,7 +1432,7 @@ ReorderBufferCleanupTXN(ReorderBuffer *rb, ReorderBufferTXN *txn) ReorderBufferCleanupTXN(rb, subtxn); } - /* cleanup changes in the toplevel txn */ + /* cleanup changes in the txn */ dlist_foreach_modify(iter, &txn->changes) { ReorderBufferChange *change; @@ -1533,7 +1533,7 @@ ReorderBufferTruncateTXN(ReorderBuffer *rb, ReorderBufferTXN *txn) ReorderBufferTruncateTXN(rb, subtxn); } - /* cleanup changes in the toplevel txn */ + /* cleanup changes in the txn */ dlist_foreach_modify(iter, &txn->changes) { ReorderBufferChange *change; From bed90759fcbcd72d4d06969eebab81e47326f9a2 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 9 Oct 2020 16:20:12 -0400 Subject: [PATCH 272/589] Fix our Windows stat() emulation to handle file sizes > 4GB. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hack things so that our idea of "struct stat" is equivalent to Windows' struct __stat64, allowing it to have a wide enough st_size field. Instead of relying on native stat(), use GetFileInformationByHandle(). This avoids a number of issues with Microsoft's multiple and rather slipshod emulations of stat(). We still need to jump through hoops to deal with ERROR_DELETE_PENDING, though :-( Pull the relevant support code out of dirmod.c and put it into its own file, win32stat.c. Still TODO: do we need to do something different with lstat(), rather than treating it identically to stat()? Juan José Santamaría Flecha, reviewed by Emil Iggland; based on prior work by Michael Paquier, Sergey Zubkovsky, and others Discussion: https://postgr.es/m/1803D792815FC24D871C00D17AE95905CF5099@g01jpexmbkw24 Discussion: https://postgr.es/m/15858-9572469fd3b73263@postgresql.org --- configure | 6 + configure.ac | 1 + src/include/port/win32_port.h | 44 +++-- src/port/dirmod.c | 52 ------ src/port/win32stat.c | 299 ++++++++++++++++++++++++++++++++++ src/tools/msvc/Mkvcbuild.pm | 2 +- 6 files changed, 339 insertions(+), 65 deletions(-) create mode 100644 src/port/win32stat.c diff --git a/configure b/configure index 19a3cd09a0a3..071d050ef009 100755 --- a/configure +++ b/configure @@ -16137,6 +16137,12 @@ esac ;; esac + case " $LIBOBJS " in + *" win32stat.$ac_objext "* ) ;; + *) LIBOBJS="$LIBOBJS win32stat.$ac_objext" + ;; +esac + $as_echo "#define HAVE_SYMLINK 1" >>confdefs.h diff --git a/configure.ac b/configure.ac index 6b9d0487a8db..e9ce611d23cb 100644 --- a/configure.ac +++ b/configure.ac @@ -1807,6 +1807,7 @@ if test "$PORTNAME" = "win32"; then AC_LIBOBJ(win32error) AC_LIBOBJ(win32security) AC_LIBOBJ(win32setlocale) + AC_LIBOBJ(win32stat) AC_DEFINE([HAVE_SYMLINK], 1, [Define to 1 if you have the `symlink' function.]) AC_CHECK_TYPES(MINIDUMP_TYPE, [pgac_minidump_type=yes], [pgac_minidump_type=no], [ diff --git a/src/include/port/win32_port.h b/src/include/port/win32_port.h index 8b6576b23dc8..f65f426cdbd9 100644 --- a/src/include/port/win32_port.h +++ b/src/include/port/win32_port.h @@ -51,7 +51,13 @@ #include #include #undef near -#include /* needed before sys/stat hacking below */ + +/* needed before sys/stat hacking below: */ +#define fstat microsoft_native_fstat +#define stat microsoft_native_stat +#include +#undef fstat +#undef stat /* Must be here to avoid conflicting with prototype in windows.h */ #define mkdir(a,b) mkdir(a) @@ -240,20 +246,34 @@ typedef int pid_t; * Supplement to . * * We must pull in sys/stat.h before this part, else our overrides lose. - */ -#define lstat(path, sb) stat(path, sb) - -/* + * * stat() is not guaranteed to set the st_size field on win32, so we - * redefine it to our own implementation that is. + * redefine it to our own implementation. See src/port/win32stat.c. * - * Some frontends don't need the size from stat, so if UNSAFE_STAT_OK - * is defined we don't bother with this. + * The struct stat is 32 bit in MSVC, so we redefine it as a copy of + * struct __stat64. This also fixes the struct size for MINGW builds. */ -#ifndef UNSAFE_STAT_OK -extern int pgwin32_safestat(const char *path, struct stat *buf); -#define stat(a,b) pgwin32_safestat(a,b) -#endif +struct stat /* This should match struct __stat64 */ +{ + _dev_t st_dev; + _ino_t st_ino; + unsigned short st_mode; + short st_nlink; + short st_uid; + short st_gid; + _dev_t st_rdev; + __int64 st_size; + __time64_t st_atime; + __time64_t st_mtime; + __time64_t st_ctime; +}; + +extern int _pgfstat64(int fileno, struct stat *buf); +extern int _pgstat64(const char *name, struct stat *buf); + +#define fstat(fileno, sb) _pgfstat64(fileno, sb) +#define stat(path, sb) _pgstat64(path, sb) +#define lstat(path, sb) _pgstat64(path, sb) /* These macros are not provided by older MinGW, nor by MSVC */ #ifndef S_IRUSR diff --git a/src/port/dirmod.c b/src/port/dirmod.c index e22a41c77e1e..8979f100803b 100644 --- a/src/port/dirmod.c +++ b/src/port/dirmod.c @@ -353,55 +353,3 @@ pgwin32_is_junction(const char *path) return ((attr & FILE_ATTRIBUTE_REPARSE_POINT) == FILE_ATTRIBUTE_REPARSE_POINT); } #endif /* defined(WIN32) && !defined(__CYGWIN__) */ - - -#if defined(WIN32) && !defined(__CYGWIN__) - -#undef stat - -/* - * The stat() function in win32 is not guaranteed to update the st_size - * field when run. So we define our own version that uses the Win32 API - * to update this field. - */ -int -pgwin32_safestat(const char *path, struct stat *buf) -{ - int r; - WIN32_FILE_ATTRIBUTE_DATA attr; - - r = stat(path, buf); - if (r < 0) - { - if (GetLastError() == ERROR_DELETE_PENDING) - { - /* - * File has been deleted, but is not gone from the filesystem yet. - * This can happen when some process with FILE_SHARE_DELETE has it - * open and it will be fully removed once that handle is closed. - * Meanwhile, we can't open it, so indicate that the file just - * doesn't exist. - */ - errno = ENOENT; - return -1; - } - - return r; - } - - if (!GetFileAttributesEx(path, GetFileExInfoStandard, &attr)) - { - _dosmaperr(GetLastError()); - return -1; - } - - /* - * XXX no support for large files here, but we don't do that in general on - * Win32 yet. - */ - buf->st_size = attr.nFileSizeLow; - - return 0; -} - -#endif diff --git a/src/port/win32stat.c b/src/port/win32stat.c new file mode 100644 index 000000000000..a70df3a5900a --- /dev/null +++ b/src/port/win32stat.c @@ -0,0 +1,299 @@ +/*------------------------------------------------------------------------- + * + * win32stat.c + * Replacements for functions using GetFileInformationByHandle + * + * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/port/win32stat.c + * + *------------------------------------------------------------------------- + */ + +#ifdef WIN32 + +#include "c.h" +#include + +/* + * In order to support MinGW and MSVC2013 we use NtQueryInformationFile as an + * alternative for GetFileInformationByHandleEx. It is loaded from the ntdll + * library. + */ +#if _WIN32_WINNT < 0x0600 +#include + +#if !defined(__MINGW32__) && !defined(__MINGW64__) +/* MinGW includes this in , but it is missing in MSVC */ +typedef struct _FILE_STANDARD_INFORMATION +{ + LARGE_INTEGER AllocationSize; + LARGE_INTEGER EndOfFile; + ULONG NumberOfLinks; + BOOLEAN DeletePending; + BOOLEAN Directory; +} FILE_STANDARD_INFORMATION; +#define FileStandardInformation 5 +#endif /* !defined(__MINGW32__) && + * !defined(__MINGW64__) */ + +typedef NTSTATUS(NTAPI * PFN_NTQUERYINFORMATIONFILE) +( + IN HANDLE FileHandle, + OUT PIO_STATUS_BLOCK IoStatusBlock, + OUT PVOID FileInformation, + IN ULONG Length, + IN FILE_INFORMATION_CLASS FileInformationClass +); + +static PFN_NTQUERYINFORMATIONFILE _NtQueryInformationFile = NULL; + +static HMODULE ntdll = NULL; + +/* + * Load DLL file just once regardless of how many functions we load/call in it. + */ +static void +LoadNtdll(void) +{ + if (ntdll != NULL) + return; + ntdll = LoadLibraryEx("ntdll.dll", NULL, 0); +} + +#endif /* _WIN32_WINNT < 0x0600 */ + + +/* + * Convert a FILETIME struct into a 64 bit time_t. + */ +static __time64_t +filetime_to_time(const FILETIME *ft) +{ + ULARGE_INTEGER unified_ft = {0}; + static const uint64 EpochShift = UINT64CONST(116444736000000000); + + unified_ft.LowPart = ft->dwLowDateTime; + unified_ft.HighPart = ft->dwHighDateTime; + + if (unified_ft.QuadPart < EpochShift) + return -1; + + unified_ft.QuadPart -= EpochShift; + unified_ft.QuadPart /= 10 * 1000 * 1000; + + return unified_ft.QuadPart; +} + +/* + * Convert WIN32 file attributes to a Unix-style mode. + * + * Only owner permissions are set. + */ +static unsigned short +fileattr_to_unixmode(int attr) +{ + unsigned short uxmode = 0; + + uxmode |= (unsigned short) ((attr & FILE_ATTRIBUTE_DIRECTORY) ? + (_S_IFDIR) : (_S_IFREG)); + + uxmode |= (unsigned short) (attr & FILE_ATTRIBUTE_READONLY) ? + (_S_IREAD) : (_S_IREAD | _S_IWRITE); + + /* there is no need to simulate _S_IEXEC using CMD's PATHEXT extensions */ + uxmode |= _S_IEXEC; + + return uxmode; +} + +/* + * Convert WIN32 file information (from a HANDLE) to a struct stat. + */ +static int +fileinfo_to_stat(HANDLE hFile, struct stat *buf) +{ + BY_HANDLE_FILE_INFORMATION fiData; + + memset(buf, 0, sizeof(*buf)); + + /* + * GetFileInformationByHandle minimum supported version: Windows XP and + * Windows Server 2003, so it exists everywhere we care about. + */ + if (!GetFileInformationByHandle(hFile, &fiData)) + { + _dosmaperr(GetLastError()); + return -1; + } + + if (fiData.ftLastWriteTime.dwLowDateTime || + fiData.ftLastWriteTime.dwHighDateTime) + buf->st_mtime = filetime_to_time(&fiData.ftLastWriteTime); + + if (fiData.ftLastAccessTime.dwLowDateTime || + fiData.ftLastAccessTime.dwHighDateTime) + buf->st_atime = filetime_to_time(&fiData.ftLastAccessTime); + else + buf->st_atime = buf->st_mtime; + + if (fiData.ftCreationTime.dwLowDateTime || + fiData.ftCreationTime.dwHighDateTime) + buf->st_ctime = filetime_to_time(&fiData.ftCreationTime); + else + buf->st_ctime = buf->st_mtime; + + buf->st_mode = fileattr_to_unixmode(fiData.dwFileAttributes); + buf->st_nlink = fiData.nNumberOfLinks; + + buf->st_size = (((uint64) fiData.nFileSizeHigh) << 32) | + (uint64) fiData.nFileSizeLow; + + return 0; +} + +/* + * Windows implementation of stat(). + * + * This currently also implements lstat(), though perhaps that should change. + */ +int +_pgstat64(const char *name, struct stat *buf) +{ + /* + * We must use a handle so lstat() returns the information of the target + * file. To have a reliable test for ERROR_DELETE_PENDING, we use + * NtQueryInformationFile from Windows 2000 or + * GetFileInformationByHandleEx from Server 2008 / Vista. + */ + SECURITY_ATTRIBUTES sa; + HANDLE hFile; + int ret; +#if _WIN32_WINNT < 0x0600 + IO_STATUS_BLOCK ioStatus; + FILE_STANDARD_INFORMATION standardInfo; +#else + FILE_STANDARD_INFO standardInfo; +#endif + + if (name == NULL || buf == NULL) + { + errno = EINVAL; + return -1; + } + + /* fast not-exists check */ + if (GetFileAttributes(name) == INVALID_FILE_ATTRIBUTES) + { + _dosmaperr(GetLastError()); + return -1; + } + + /* get a file handle as lightweight as we can */ + sa.nLength = sizeof(SECURITY_ATTRIBUTES); + sa.bInheritHandle = TRUE; + sa.lpSecurityDescriptor = NULL; + hFile = CreateFile(name, + GENERIC_READ, + (FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE), + &sa, + OPEN_EXISTING, + (FILE_FLAG_NO_BUFFERING | FILE_FLAG_BACKUP_SEMANTICS | + FILE_FLAG_OVERLAPPED), + NULL); + if (hFile == INVALID_HANDLE_VALUE) + { + CloseHandle(hFile); + errno = ENOENT; + return -1; + } + + memset(&standardInfo, 0, sizeof(standardInfo)); + +#if _WIN32_WINNT < 0x0600 + if (_NtQueryInformationFile == NULL) + { + /* First time through: load ntdll.dll and find NtQueryInformationFile */ + LoadNtdll(); + if (ntdll == NULL) + { + _dosmaperr(GetLastError()); + CloseHandle(hFile); + return -1; + } + + _NtQueryInformationFile = (PFN_NTQUERYINFORMATIONFILE) + GetProcAddress(ntdll, "NtQueryInformationFile"); + if (_NtQueryInformationFile == NULL) + { + _dosmaperr(GetLastError()); + CloseHandle(hFile); + return -1; + } + } + + if (!NT_SUCCESS(_NtQueryInformationFile(hFile, &ioStatus, &standardInfo, + sizeof(standardInfo), + FileStandardInformation))) + { + _dosmaperr(GetLastError()); + CloseHandle(hFile); + return -1; + } +#else + if (!GetFileInformationByHandleEx(hFile, FileStandardInfo, &standardInfo, + sizeof(standardInfo))) + { + _dosmaperr(GetLastError()); + CloseHandle(hFile); + return -1; + } +#endif /* _WIN32_WINNT < 0x0600 */ + + if (standardInfo.DeletePending) + { + /* + * File has been deleted, but is not gone from the filesystem yet. + * This can happen when some process with FILE_SHARE_DELETE has it + * open, and it will be fully removed once that handle is closed. + * Meanwhile, we can't open it, so indicate that the file just doesn't + * exist. + */ + CloseHandle(hFile); + errno = ENOENT; + return -1; + } + + /* At last we can invoke fileinfo_to_stat */ + ret = fileinfo_to_stat(hFile, buf); + + CloseHandle(hFile); + return ret; +} + +/* + * Windows implementation of fstat(). + */ +int +_pgfstat64(int fileno, struct stat *buf) +{ + HANDLE hFile = (HANDLE) _get_osfhandle(fileno); + + if (hFile == INVALID_HANDLE_VALUE || buf == NULL) + { + errno = EINVAL; + return -1; + } + + /* + * Since we already have a file handle there is no need to check for + * ERROR_DELETE_PENDING. + */ + + return fileinfo_to_stat(hFile, buf); +} + +#endif /* WIN32 */ diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 89e1b3903656..90594bd41bac 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -103,7 +103,7 @@ sub mkvcbuild pg_strong_random.c pgcheckdir.c pgmkdirp.c pgsleep.c pgstrcasecmp.c pqsignal.c mkdtemp.c qsort.c qsort_arg.c quotes.c system.c strerror.c tar.c thread.c - win32env.c win32error.c win32security.c win32setlocale.c); + win32env.c win32error.c win32security.c win32setlocale.c win32stat.c); push(@pgportfiles, 'strtof.c') if ($vsVersion < '14.00'); From ed30b1a60dadf2b7cc58bce5009ad8676b8fe479 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 9 Oct 2020 17:54:34 -0400 Subject: [PATCH 273/589] plperl.h should #undef fstat along with stat and lstat. Needed now that commit bed90759f caused win32_port.h to provide a #define for that too. Per buildfarm. --- src/pl/plperl/plperl.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pl/plperl/plperl.h b/src/pl/plperl/plperl.h index a9449d9d8a4f..619e7121a1bf 100644 --- a/src/pl/plperl/plperl.h +++ b/src/pl/plperl/plperl.h @@ -92,6 +92,7 @@ #undef bind #undef connect #undef fopen +#undef fstat #undef kill #undef listen #undef lstat From fe27009cbb5fff53f87969786ca8dac82ed7deba Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 10 Oct 2020 13:28:12 -0400 Subject: [PATCH 274/589] Recognize network-failure errnos as indicating hard connection loss. Up to now, only ECONNRESET (and EPIPE, in most but not quite all places) received special treatment in our error handling logic. This patch changes things so that related error codes such as ECONNABORTED are also recognized as indicating that the connection's dead and unlikely to come back. We continue to think, however, that only ECONNRESET and EPIPE should be reported as probable server crashes; the other cases indicate network connectivity problems but prove little about the server's state. Thus, there's no change in the error message texts that are output for such cases. The key practical effect is that errcode_for_socket_access() will report ERRCODE_CONNECTION_FAILURE rather than ERRCODE_INTERNAL_ERROR for a network failure. It's expected that this will fix buildfarm member lorikeet's failures since commit 32a9c0bdf, as that seems to be due to not treating ECONNABORTED equivalently to ECONNRESET. The set of errnos treated this way now includes ECONNABORTED, EHOSTDOWN, EHOSTUNREACH, ENETDOWN, ENETRESET, and ENETUNREACH. Several of these were second-class citizens in terms of their handling in places like get_errno_symbol(), so upgrade the infrastructure where necessary. As committed, this patch assumes that all these symbols are defined everywhere. POSIX specifies all of them except EHOSTDOWN, but that seems to exist on all platforms of interest; we'll see what the buildfarm says about that. Probably this should be back-patched, but let's see what the buildfarm thinks of it first. Fujii Masao and Tom Lane Discussion: https://postgr.es/m/2621622.1602184554@sss.pgh.pa.us --- src/backend/port/win32/socket.c | 12 +++++- src/backend/utils/error/elog.c | 5 +-- src/bin/pg_dump/parallel.c | 11 ++++-- src/include/port.h | 22 +++++++++++ src/include/port/win32_port.h | 8 ++++ src/interfaces/libpq/fe-misc.c | 66 ++++++++++++++++++-------------- src/interfaces/libpq/fe-secure.c | 5 +-- src/interfaces/libpq/win32.h | 11 ------ src/port/strerror.c | 14 ++++--- 9 files changed, 96 insertions(+), 58 deletions(-) diff --git a/src/backend/port/win32/socket.c b/src/backend/port/win32/socket.c index 6fbd1ed6fb49..7c7611a01e23 100644 --- a/src/backend/port/win32/socket.c +++ b/src/backend/port/win32/socket.c @@ -120,13 +120,21 @@ TranslateSocketError(void) case WSAEADDRNOTAVAIL: errno = EADDRNOTAVAIL; break; - case WSAEHOSTUNREACH: case WSAEHOSTDOWN: + errno = EHOSTDOWN; + break; + case WSAEHOSTUNREACH: case WSAHOST_NOT_FOUND: + errno = EHOSTUNREACH; + break; case WSAENETDOWN: + errno = ENETDOWN; + break; case WSAENETUNREACH: + errno = ENETUNREACH; + break; case WSAENETRESET: - errno = EHOSTUNREACH; + errno = ENETRESET; break; case WSAENOTCONN: case WSAESHUTDOWN: diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index d0b368530e7e..1ba47c194b2f 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -711,10 +711,7 @@ errcode_for_socket_access(void) switch (edata->saved_errno) { /* Loss of connection */ - case EPIPE: -#ifdef ECONNRESET - case ECONNRESET: -#endif + case ALL_CONNECTION_FAILURE_ERRNOS: edata->sqlerrcode = ERRCODE_CONNECTION_FAILURE; break; diff --git a/src/bin/pg_dump/parallel.c b/src/bin/pg_dump/parallel.c index f0587f41e492..4b38ed6c5a91 100644 --- a/src/bin/pg_dump/parallel.c +++ b/src/bin/pg_dump/parallel.c @@ -1825,10 +1825,15 @@ piperead(int s, char *buf, int len) { int ret = recv(s, buf, len, 0); - if (ret < 0 && WSAGetLastError() == WSAECONNRESET) + if (ret < 0) { - /* EOF on the pipe! */ - ret = 0; + switch (TranslateSocketError()) + { + case ALL_CONNECTION_FAILURE_ERRNOS: + /* Treat connection loss as EOF on the pipe */ + ret = 0; + break; + } } return ret; } diff --git a/src/include/port.h b/src/include/port.h index 84bf2c363f55..d25716bf7f83 100644 --- a/src/include/port.h +++ b/src/include/port.h @@ -99,6 +99,28 @@ extern void pgfnames_cleanup(char **filenames); ) #endif +/* + * This macro provides a centralized list of all errnos that identify + * hard failure of a previously-established network connection. + * The macro is intended to be used in a switch statement, in the form + * "case ALL_CONNECTION_FAILURE_ERRNOS:". + * + * Note: this groups EPIPE and ECONNRESET, which we take to indicate a + * probable server crash, with other errors that indicate loss of network + * connectivity without proving much about the server's state. Places that + * are actually reporting errors typically single out EPIPE and ECONNRESET, + * while allowing the network failures to be reported generically. + */ +#define ALL_CONNECTION_FAILURE_ERRNOS \ + EPIPE: \ + case ECONNRESET: \ + case ECONNABORTED: \ + case EHOSTDOWN: \ + case EHOSTUNREACH: \ + case ENETDOWN: \ + case ENETRESET: \ + case ENETUNREACH + /* Portable locale initialization (in exec.c) */ extern void set_pglocale_pgservice(const char *argv0, const char *app); diff --git a/src/include/port/win32_port.h b/src/include/port/win32_port.h index f65f426cdbd9..59c7f35e3dfa 100644 --- a/src/include/port/win32_port.h +++ b/src/include/port/win32_port.h @@ -369,8 +369,16 @@ extern int _pgstat64(const char *name, struct stat *buf); #define EADDRINUSE WSAEADDRINUSE #undef EADDRNOTAVAIL #define EADDRNOTAVAIL WSAEADDRNOTAVAIL +#undef EHOSTDOWN +#define EHOSTDOWN WSAEHOSTDOWN #undef EHOSTUNREACH #define EHOSTUNREACH WSAEHOSTUNREACH +#undef ENETDOWN +#define ENETDOWN WSAENETDOWN +#undef ENETRESET +#define ENETRESET WSAENETRESET +#undef ENETUNREACH +#define ENETUNREACH WSAENETUNREACH #undef ENOTCONN #define ENOTCONN WSAENOTCONN diff --git a/src/interfaces/libpq/fe-misc.c b/src/interfaces/libpq/fe-misc.c index ff840b7730d8..4ffc7f33fb5e 100644 --- a/src/interfaces/libpq/fe-misc.c +++ b/src/interfaces/libpq/fe-misc.c @@ -668,24 +668,29 @@ pqReadData(PGconn *conn) conn->inBufSize - conn->inEnd); if (nread < 0) { - if (SOCK_ERRNO == EINTR) - goto retry3; - /* Some systems return EAGAIN/EWOULDBLOCK for no data */ + switch (SOCK_ERRNO) + { + case EINTR: + goto retry3; + + /* Some systems return EAGAIN/EWOULDBLOCK for no data */ #ifdef EAGAIN - if (SOCK_ERRNO == EAGAIN) - return someread; + case EAGAIN: + return someread; #endif #if defined(EWOULDBLOCK) && (!defined(EAGAIN) || (EWOULDBLOCK != EAGAIN)) - if (SOCK_ERRNO == EWOULDBLOCK) - return someread; + case EWOULDBLOCK: + return someread; #endif - /* We might get ECONNRESET here if using TCP and backend died */ -#ifdef ECONNRESET - if (SOCK_ERRNO == ECONNRESET) - goto definitelyFailed; -#endif - /* pqsecure_read set the error message for us */ - return -1; + + /* We might get ECONNRESET etc here if connection failed */ + case ALL_CONNECTION_FAILURE_ERRNOS: + goto definitelyFailed; + + default: + /* pqsecure_read set the error message for us */ + return -1; + } } if (nread > 0) { @@ -758,24 +763,29 @@ pqReadData(PGconn *conn) conn->inBufSize - conn->inEnd); if (nread < 0) { - if (SOCK_ERRNO == EINTR) - goto retry4; - /* Some systems return EAGAIN/EWOULDBLOCK for no data */ + switch (SOCK_ERRNO) + { + case EINTR: + goto retry4; + + /* Some systems return EAGAIN/EWOULDBLOCK for no data */ #ifdef EAGAIN - if (SOCK_ERRNO == EAGAIN) - return 0; + case EAGAIN: + return 0; #endif #if defined(EWOULDBLOCK) && (!defined(EAGAIN) || (EWOULDBLOCK != EAGAIN)) - if (SOCK_ERRNO == EWOULDBLOCK) - return 0; + case EWOULDBLOCK: + return 0; #endif - /* We might get ECONNRESET here if using TCP and backend died */ -#ifdef ECONNRESET - if (SOCK_ERRNO == ECONNRESET) - goto definitelyFailed; -#endif - /* pqsecure_read set the error message for us */ - return -1; + + /* We might get ECONNRESET etc here if connection failed */ + case ALL_CONNECTION_FAILURE_ERRNOS: + goto definitelyFailed; + + default: + /* pqsecure_read set the error message for us */ + return -1; + } } if (nread > 0) { diff --git a/src/interfaces/libpq/fe-secure.c b/src/interfaces/libpq/fe-secure.c index 3311fd7a5bda..97c3805303f5 100644 --- a/src/interfaces/libpq/fe-secure.c +++ b/src/interfaces/libpq/fe-secure.c @@ -261,14 +261,13 @@ pqsecure_raw_read(PGconn *conn, void *ptr, size_t len) /* no error message, caller is expected to retry */ break; -#ifdef ECONNRESET + case EPIPE: case ECONNRESET: printfPQExpBuffer(&conn->errorMessage, libpq_gettext("server closed the connection unexpectedly\n" "\tThis probably means the server terminated abnormally\n" "\tbefore or while processing the request.\n")); break; -#endif default: printfPQExpBuffer(&conn->errorMessage, @@ -374,11 +373,9 @@ pqsecure_raw_write(PGconn *conn, const void *ptr, size_t len) /* Set flag for EPIPE */ REMEMBER_EPIPE(spinfo, true); -#ifdef ECONNRESET /* FALL THRU */ case ECONNRESET: -#endif printfPQExpBuffer(&conn->errorMessage, libpq_gettext("server closed the connection unexpectedly\n" "\tThis probably means the server terminated abnormally\n" diff --git a/src/interfaces/libpq/win32.h b/src/interfaces/libpq/win32.h index c42d7abfe30a..fcce1e0544a7 100644 --- a/src/interfaces/libpq/win32.h +++ b/src/interfaces/libpq/win32.h @@ -14,17 +14,6 @@ #define write(a,b,c) _write(a,b,c) #undef EAGAIN /* doesn't apply on sockets */ -#undef EINTR -#define EINTR WSAEINTR -#ifndef EWOULDBLOCK -#define EWOULDBLOCK WSAEWOULDBLOCK -#endif -#ifndef ECONNRESET -#define ECONNRESET WSAECONNRESET -#endif -#ifndef EINPROGRESS -#define EINPROGRESS WSAEINPROGRESS -#endif /* * support for handling Windows Socket errors diff --git a/src/port/strerror.c b/src/port/strerror.c index 375edb0f5abd..43a9761d90bd 100644 --- a/src/port/strerror.c +++ b/src/port/strerror.c @@ -146,16 +146,12 @@ get_errno_symbol(int errnum) return "EBUSY"; case ECHILD: return "ECHILD"; -#ifdef ECONNABORTED case ECONNABORTED: return "ECONNABORTED"; -#endif case ECONNREFUSED: return "ECONNREFUSED"; -#ifdef ECONNRESET case ECONNRESET: return "ECONNRESET"; -#endif case EDEADLK: return "EDEADLK"; case EDOM: @@ -166,10 +162,10 @@ get_errno_symbol(int errnum) return "EFAULT"; case EFBIG: return "EFBIG"; -#ifdef EHOSTUNREACH + case EHOSTDOWN: + return "EHOSTDOWN"; case EHOSTUNREACH: return "EHOSTUNREACH"; -#endif case EIDRM: return "EIDRM"; case EINPROGRESS: @@ -198,6 +194,12 @@ get_errno_symbol(int errnum) return "EMSGSIZE"; case ENAMETOOLONG: return "ENAMETOOLONG"; + case ENETDOWN: + return "ENETDOWN"; + case ENETRESET: + return "ENETRESET"; + case ENETUNREACH: + return "ENETUNREACH"; case ENFILE: return "ENFILE"; case ENOBUFS: From c94cfb38c32a68ef9b021a6851fff25f9ecdf297 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 10 Oct 2020 13:39:21 -0400 Subject: [PATCH 275/589] Minor cleanup for win32stat.c. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ensure that CloseHandle() can't clobber the errno we set for failure exits, and make a couple of tweaks for pgindent. Juan José Santamaría Flecha Discussion: https://postgr.es/m/CAC+AXB0g44SbvSpC86o_1HWh8TAU2pZrMRW6tJT-dkijotx5Qg@mail.gmail.com --- src/port/win32stat.c | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/port/win32stat.c b/src/port/win32stat.c index a70df3a5900a..c111f0ef353c 100644 --- a/src/port/win32stat.c +++ b/src/port/win32stat.c @@ -40,14 +40,12 @@ typedef struct _FILE_STANDARD_INFORMATION #endif /* !defined(__MINGW32__) && * !defined(__MINGW64__) */ -typedef NTSTATUS(NTAPI * PFN_NTQUERYINFORMATIONFILE) -( - IN HANDLE FileHandle, - OUT PIO_STATUS_BLOCK IoStatusBlock, - OUT PVOID FileInformation, - IN ULONG Length, - IN FILE_INFORMATION_CLASS FileInformationClass -); +typedef NTSTATUS (NTAPI * PFN_NTQUERYINFORMATIONFILE) + (IN HANDLE FileHandle, + OUT PIO_STATUS_BLOCK IoStatusBlock, + OUT PVOID FileInformation, + IN ULONG Length, + IN FILE_INFORMATION_CLASS FileInformationClass); static PFN_NTQUERYINFORMATIONFILE _NtQueryInformationFile = NULL; @@ -101,8 +99,8 @@ fileattr_to_unixmode(int attr) uxmode |= (unsigned short) ((attr & FILE_ATTRIBUTE_DIRECTORY) ? (_S_IFDIR) : (_S_IFREG)); - uxmode |= (unsigned short) (attr & FILE_ATTRIBUTE_READONLY) ? - (_S_IREAD) : (_S_IREAD | _S_IWRITE); + uxmode |= (unsigned short) ((attr & FILE_ATTRIBUTE_READONLY) ? + (_S_IREAD) : (_S_IREAD | _S_IWRITE)); /* there is no need to simulate _S_IEXEC using CMD's PATHEXT extensions */ uxmode |= _S_IEXEC; @@ -149,8 +147,8 @@ fileinfo_to_stat(HANDLE hFile, struct stat *buf) buf->st_mode = fileattr_to_unixmode(fiData.dwFileAttributes); buf->st_nlink = fiData.nNumberOfLinks; - buf->st_size = (((uint64) fiData.nFileSizeHigh) << 32) | - (uint64) fiData.nFileSizeLow; + buf->st_size = ((((uint64) fiData.nFileSizeHigh) << 32) | + fiData.nFileSizeLowi); return 0; } @@ -220,8 +218,10 @@ _pgstat64(const char *name, struct stat *buf) LoadNtdll(); if (ntdll == NULL) { - _dosmaperr(GetLastError()); + DWORD err = GetLastError(); + CloseHandle(hFile); + _dosmaperr(err); return -1; } @@ -229,8 +229,10 @@ _pgstat64(const char *name, struct stat *buf) GetProcAddress(ntdll, "NtQueryInformationFile"); if (_NtQueryInformationFile == NULL) { - _dosmaperr(GetLastError()); + DWORD err = GetLastError(); + CloseHandle(hFile); + _dosmaperr(err); return -1; } } @@ -239,16 +241,20 @@ _pgstat64(const char *name, struct stat *buf) sizeof(standardInfo), FileStandardInformation))) { - _dosmaperr(GetLastError()); + DWORD err = GetLastError(); + CloseHandle(hFile); + _dosmaperr(err); return -1; } #else if (!GetFileInformationByHandleEx(hFile, FileStandardInfo, &standardInfo, sizeof(standardInfo))) { - _dosmaperr(GetLastError()); + DWORD err = GetLastError(); + CloseHandle(hFile); + _dosmaperr(err); return -1; } #endif /* _WIN32_WINNT < 0x0600 */ From 961e07b8ccb56cb3979185b066b503b3d4f7c036 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 10 Oct 2020 14:53:23 -0400 Subject: [PATCH 276/589] Minor cleanup for win32stat.c. Fix silly typo in previous commit. Discussion: https://postgr.es/m/CAC+AXB0g44SbvSpC86o_1HWh8TAU2pZrMRW6tJT-dkijotx5Qg@mail.gmail.com --- src/port/win32stat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/port/win32stat.c b/src/port/win32stat.c index c111f0ef353c..d4315c44f4d8 100644 --- a/src/port/win32stat.c +++ b/src/port/win32stat.c @@ -148,7 +148,7 @@ fileinfo_to_stat(HANDLE hFile, struct stat *buf) buf->st_nlink = fiData.nNumberOfLinks; buf->st_size = ((((uint64) fiData.nFileSizeHigh) << 32) | - fiData.nFileSizeLowi); + fiData.nFileSizeLow); return 0; } From eeb01eb1f560d90625ac4c8ee210f29421ba4f0d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 10 Oct 2020 15:33:54 -0400 Subject: [PATCH 277/589] Remove pointless error-code checking in pg_dump/parallel.c. Commit fe27009cb tried to make parallel.c's Windows implementation of piperead() translate Windows socket errors to Unix, but that didn't actually work because TranslateSocketError() is backend-internal code (and not even public there). But on closer inspection, the sole caller of this function doesn't actually care whether the result is zero or negative, much less inspect the errno. So the whole exercise is totally useless, and has been since this code was introduced. Rip it out and just call recv() directly. Per buildfarm. Discussion: https://postgr.es/m/2621622.1602184554@sss.pgh.pa.us --- src/bin/pg_dump/parallel.c | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/src/bin/pg_dump/parallel.c b/src/bin/pg_dump/parallel.c index 4b38ed6c5a91..a967e1137824 100644 --- a/src/bin/pg_dump/parallel.c +++ b/src/bin/pg_dump/parallel.c @@ -130,7 +130,7 @@ typedef struct /* Windows implementation of pipe access */ static int pgpipe(int handles[2]); -static int piperead(int s, char *buf, int len); +#define piperead(a,b,c) recv(a,b,c,0) #define pipewrite(a,b,c) send(a,b,c,0) #else /* !WIN32 */ @@ -1817,25 +1817,4 @@ pgpipe(int handles[2]) return 0; } -/* - * Windows implementation of reading from a pipe. - */ -static int -piperead(int s, char *buf, int len) -{ - int ret = recv(s, buf, len, 0); - - if (ret < 0) - { - switch (TranslateSocketError()) - { - case ALL_CONNECTION_FAILURE_ERRNOS: - /* Treat connection loss as EOF on the pipe */ - ret = 0; - break; - } - } - return ret; -} - #endif /* WIN32 */ From 85d08b8b721fb3b9359bca9325bc425cc95c30b1 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 10 Oct 2020 19:57:25 -0400 Subject: [PATCH 278/589] Band-aid new postgres_fdw test case to remove error text dependency. Buildfarm member lorikeet is still failing the test from commit 32a9c0bdf, but now it's down to the should-have-foreseen-it problem that the error message isn't what the expected-output file expects. Let's see if we can get stable results by printing just the SQLSTATE. I believe we'll reliably see ERRCODE_CONNECTION_FAILURE, since pgfdw_report_error() will report that for any libpq-originated error. There may be a better way to do this, but I'd like to get the buildfarm back to green before we discuss further improvements. Discussion: https://postgr.es/m/E1kPc9v-0005L4-2l@gemulon.postgresql.org Discussion: https://postgr.es/m/2621622.1602184554@sss.pgh.pa.us --- contrib/postgres_fdw/expected/postgres_fdw.out | 8 ++++---- contrib/postgres_fdw/sql/postgres_fdw.sql | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 2c5614073f94..2d88d0635837 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -9025,13 +9025,13 @@ SELECT 1 FROM ft1 LIMIT 1; -- If the query detects the broken connection when starting new remote -- subtransaction, it doesn't reestablish new connection and should fail. +-- The text of the error might vary across platforms, so don't show it. CALL terminate_backend_and_wait('fdw_retry_check'); SAVEPOINT s; +\set VERBOSITY sqlstate SELECT 1 FROM ft1 LIMIT 1; -- should fail -ERROR: server closed the connection unexpectedly - This probably means the server terminated abnormally - before or while processing the request. -CONTEXT: remote SQL command: SAVEPOINT s2 +ERROR: 08006 +\set VERBOSITY default COMMIT; -- Clean up DROP PROCEDURE terminate_backend_and_wait(text); diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index 4da1f78956e5..7581c5417b90 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -2687,9 +2687,12 @@ SELECT 1 FROM ft1 LIMIT 1; -- If the query detects the broken connection when starting new remote -- subtransaction, it doesn't reestablish new connection and should fail. +-- The text of the error might vary across platforms, so don't show it. CALL terminate_backend_and_wait('fdw_retry_check'); SAVEPOINT s; +\set VERBOSITY sqlstate SELECT 1 FROM ft1 LIMIT 1; -- should fail +\set VERBOSITY default COMMIT; -- Clean up From 80f8eb79e24d9b7963eaf17ce846667e2c6b6e6f Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Sun, 11 Oct 2020 19:09:01 +0900 Subject: [PATCH 279/589] Use perfect hash for NFC and NFKC Unicode Normalization quick check This makes the normalization quick check about 30% faster for NFC and 50% faster for NFKC than the binary search used previously. The hash lookup reuses the existing array of bit fields used for the binary search to get the quick check property and is generated as part of "make update-unicode" in src/common/unicode/. Author: John Naylor Reviewed-by: Mark Dilger, Michael Paquier Discussion: https://postgr.es/m/CACPNZCt4fbJ0_bGrN5QPt34N4whv=mszM0LMVQdoa2rC9UMRXA@mail.gmail.com --- .../generate-unicode_normprops_table.pl | 41 +- src/common/unicode_norm.c | 48 +- src/include/common/unicode_normprops_table.h | 1610 ++++++++++++++++- src/tools/pgindent/exclude_file_patterns | 5 + src/tools/pgindent/typedefs.list | 1 + 5 files changed, 1681 insertions(+), 24 deletions(-) diff --git a/src/common/unicode/generate-unicode_normprops_table.pl b/src/common/unicode/generate-unicode_normprops_table.pl index e8e5097c094b..d652b95965dc 100644 --- a/src/common/unicode/generate-unicode_normprops_table.pl +++ b/src/common/unicode/generate-unicode_normprops_table.pl @@ -9,6 +9,10 @@ use strict; use warnings; +use FindBin; +use lib "$FindBin::RealBin/../../tools/"; +use PerfectHash; + my %data; print @@ -18,13 +22,25 @@ #include "common/unicode_norm.h" /* - * We use a bit field here to save space. + * Normalization quick check entry for codepoint. We use a bit field + * here to save space. */ typedef struct { unsigned int codepoint:21; signed int quickcheck:4; /* really UnicodeNormalizationQC */ -} pg_unicode_normprops; +} pg_unicode_normprops; + +/* Typedef for hash function on quick check table */ +typedef int (*qc_hash_func) (const void *key); + +/* Information for quick check lookup with perfect hash function */ +typedef struct +{ + const pg_unicode_normprops *normprops; + qc_hash_func hash; + int num_normprops; +} pg_unicode_norminfo; EOS foreach my $line () @@ -66,6 +82,7 @@ "static const pg_unicode_normprops UnicodeNormProps_${prop}[] = {\n"; my %subdata = %{ $data{$prop} }; + my @cp_packed; foreach my $cp (sort { $a <=> $b } keys %subdata) { my $qc; @@ -82,7 +99,27 @@ die; } printf "\t{0x%04X, %s},\n", $cp, $qc; + + # Save the bytes as a string in network order. + push @cp_packed, pack('N', $cp); } print "};\n"; + + # Emit the definition of the perfect hash function. + my $funcname = $prop . '_hash_func'; + my $f = PerfectHash::generate_hash_function(\@cp_packed, $funcname, + fixed_key_length => 4); + printf "\n/* Perfect hash function for %s */", $prop; + print "\nstatic $f\n"; + + # Emit the structure that wraps the hash lookup information into + # one variable. + printf "/* Hash lookup information for %s */", $prop; + printf "\nstatic const pg_unicode_norminfo "; + printf "UnicodeNormInfo_%s = {\n", $prop; + printf "\tUnicodeNormProps_%s,\n", $prop; + printf "\t%s,\n", $funcname; + printf "\t%d\n", scalar @cp_packed; + printf "};\n"; } diff --git a/src/common/unicode_norm.c b/src/common/unicode_norm.c index ab5ce5934569..626645ac8705 100644 --- a/src/common/unicode_norm.c +++ b/src/common/unicode_norm.c @@ -465,15 +465,32 @@ get_canonical_class(pg_wchar ch) return entry->comb_class; } -static int -qc_compare(const void *p1, const void *p2) +static const pg_unicode_normprops * +qc_hash_lookup(pg_wchar ch, const pg_unicode_norminfo *norminfo) { - uint32 v1, - v2; + int h; + uint32 hashkey; - v1 = ((const pg_unicode_normprops *) p1)->codepoint; - v2 = ((const pg_unicode_normprops *) p2)->codepoint; - return (v1 - v2); + /* + * Compute the hash function. The hash key is the codepoint with the bytes + * in network order. + */ + hashkey = htonl(ch); + h = norminfo->hash(&hashkey); + + /* An out-of-range result implies no match */ + if (h < 0 || h >= norminfo->num_normprops) + return NULL; + + /* + * Since it's a perfect hash, we need only match to the specific codepoint + * it identifies. + */ + if (ch != norminfo->normprops[h].codepoint) + return NULL; + + /* Success! */ + return &norminfo->normprops[h]; } /* @@ -482,26 +499,15 @@ qc_compare(const void *p1, const void *p2) static UnicodeNormalizationQC qc_is_allowed(UnicodeNormalizationForm form, pg_wchar ch) { - pg_unicode_normprops key; - pg_unicode_normprops *found = NULL; - - key.codepoint = ch; + const pg_unicode_normprops *found = NULL; switch (form) { case UNICODE_NFC: - found = bsearch(&key, - UnicodeNormProps_NFC_QC, - lengthof(UnicodeNormProps_NFC_QC), - sizeof(pg_unicode_normprops), - qc_compare); + found = qc_hash_lookup(ch, &UnicodeNormInfo_NFC_QC); break; case UNICODE_NFKC: - found = bsearch(&key, - UnicodeNormProps_NFKC_QC, - lengthof(UnicodeNormProps_NFKC_QC), - sizeof(pg_unicode_normprops), - qc_compare); + found = qc_hash_lookup(ch, &UnicodeNormInfo_NFKC_QC); break; default: Assert(false); diff --git a/src/include/common/unicode_normprops_table.h b/src/include/common/unicode_normprops_table.h index 93a2e55b7583..2ae13d847f2a 100644 --- a/src/include/common/unicode_normprops_table.h +++ b/src/include/common/unicode_normprops_table.h @@ -3,7 +3,8 @@ #include "common/unicode_norm.h" /* - * We use a bit field here to save space. + * Normalization quick check entry for codepoint. We use a bit field + * here to save space. */ typedef struct { @@ -11,6 +12,17 @@ typedef struct signed int quickcheck:4; /* really UnicodeNormalizationQC */ } pg_unicode_normprops; +/* Typedef for hash function on quick check table */ +typedef int (*qc_hash_func) (const void *key); + +/* Information for quick check lookup with perfect hash function */ +typedef struct +{ + const pg_unicode_normprops *normprops; + qc_hash_func hash; + int num_normprops; +} pg_unicode_norminfo; + static const pg_unicode_normprops UnicodeNormProps_NFC_QC[] = { {0x0300, UNICODE_NORM_QC_MAYBE}, {0x0301, UNICODE_NORM_QC_MAYBE}, @@ -1245,6 +1257,343 @@ static const pg_unicode_normprops UnicodeNormProps_NFC_QC[] = { {0x2FA1D, UNICODE_NORM_QC_NO}, }; +/* Perfect hash function for NFC_QC */ +static int +NFC_QC_hash_func(const void *key) +{ + static const int16 h[2463] = { + 0, -2717, 0, 221, 1293, 223, 1295, 225, + 226, 241, 0, 229, 230, 231, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + -386, 0, 0, 0, 0, 0, 0, 0, + -163, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + -246, -175, 1260, 0, 0, 0, -174, -173, + 0, -172, 0, 0, 0, 0, 0, 0, + 1049, 0, 300, 301, 1071, 0, 1071, 0, + 1071, 1071, 1057, 0, 0, 0, 0, 1061, + 0, -1053, 1664, 0, 2956, 0, 0, -13, + 0, 0, 0, 0, 2156, 0, 0, 0, + 0, 0, 0, 0, 71, 0, 1082, 0, + 1083, 1083, 0, 1084, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 359, 360, 361, + -1091, 363, -762, -130, -129, -128, -127, -126, + 137, -124, -708, -707, -706, -120, -185, -705, + -117, -184, -1307, -114, -113, -112, -111, 0, + 386, 387, 388, 389, -90, 391, 171, 172, + 394, -94, -183, 397, 398, 399, -98, -225, + 402, -1019, -636, -1019, -225, 407, 408, 409, + 410, 411, 674, 413, -171, -170, -169, 417, + 352, -168, 420, 353, -770, 423, 424, 425, + 426, 427, 428, 32767, 239, 239, 239, 239, + 239, 239, 239, 239, 239, 239, 239, 239, + 239, 239, 32767, 32767, 237, 32767, 236, 32767, + 32767, 234, 234, 234, 234, 617, 234, 234, + 234, -2483, 234, -1430, 1526, -1430, 1527, 47, + 48, 471, 230, 32767, 32767, 32767, 227, 227, + 227, 227, 227, 227, 227, 227, 227, 227, + 227, 227, 227, 227, 227, 227, 227, 227, + -159, 227, 227, 227, 227, 227, 227, 227, + 64, 227, 227, 227, 227, 227, 227, 227, + 227, 227, 227, 227, 227, 227, 227, 227, + 227, 227, 227, 227, 227, 227, 227, 227, + -19, 52, 1487, 227, 227, 227, 53, 54, + 227, 55, 227, 227, 227, 227, 227, 227, + 1276, 227, -989, 32767, 1296, 225, 1296, 225, + 1296, 1296, 1282, 225, 225, 225, 225, 1286, + 225, -828, 1889, 225, 3181, 225, 225, 212, + 225, 225, 225, 225, 2381, 225, 225, 225, + 225, 225, 225, 225, 296, 225, 1307, 225, + 1308, 1308, 225, 1309, 225, 225, 225, 225, + 225, 225, 225, 225, 225, 225, 225, 225, + 225, 225, 225, 225, 225, 584, 585, 586, + -866, 588, -537, 95, 96, 97, 98, 99, + 362, 101, -483, -482, -481, 105, 40, -480, + 108, 41, -1082, 111, 112, 113, 114, 225, + 611, 612, 613, 614, 135, 616, 396, 397, + 619, 131, 42, 622, 623, 624, 127, 0, + 627, -794, -411, -794, 0, 632, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + -272, 32767, 32767, 32767, 0, 32767, 32767, 32767, + 32767, 32767, -166, -165, 32767, 32767, 32767, 32767, + -164, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 397, 32767, 396, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 386, + 0, 386, 386, 386, 386, 386, 386, 386, + 223, 386, 386, 386, 32767, 385, 385, 385, + 385, 385, 32767, 384, 32767, 383, 383, 32767, + 382, 382, 32767, 381, 381, 381, 381, 381, + 135, 206, 1641, 381, 32767, 32767, 32767, 32767, + 32767, 32767, -160, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 1148, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 0, + 32767, 32767, 32767, 0, 0, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, -257, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, -910, -910, 32767, 32767, + 0, 32767, 0, 32767, 0, 32767, 0, 32767, + 147, 32767, 0, 32767, 0, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 0, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 143, 32767, 144, 32767, 145, + 32767, 146, 32767, 0, 32767, 148, 32767, 149, + 32767, 32767, 32767, -160, 32767, 32767, 32767, 32767, + 32767, 32767, 15, 32767, 32767, 0, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 145, 32767, 144, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 0, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 0, -148, 32767, 32767, 32767, 32767, + 32767, 32767, 2009, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 0, 32767, 32767, 135, -918, 32767, + 151, 32767, 32767, 0, 1, 2, 3, 4, + 133, 5, 6, 7, 8, 9, 10, 11, + 32767, 32767, -1248, 32767, 13, 154, 188, 188, + 32767, 32767, 32767, 32767, 32767, 155, 16, 32767, + 32767, 32767, 32767, 32767, 32767, -1853, -1054, 18, + -1052, -1051, -1036, 22, 32767, 157, 32767, 28, + 23, 1077, 673, 25, -2930, 0, 32767, 32767, + 32767, 32767, 32767, 27, 32767, 155, 32767, 154, + 32767, 32767, -62, 28, -42, 30, -1051, 32, + -1050, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 34, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 129, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 672, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 0, 32767, + 32767, 32767, 32767, 32767, -156, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, -155, 32767, 32767, + 32767, 0, 0, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 73, 32767, 32767, 32767, 32767, 74, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 675, + 32767, 32767, 32767, 32767, 32767, 75, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 165, 32767, 32767, 32767, 166, 167, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 170, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 689, 690, 691, 692, 693, 694, 695, + 696, 697, 698, 699, 700, 701, 702, 703, + 704, 705, 706, 707, 708, 709, 710, 711, + 712, 713, 714, 715, 716, 717, 718, 719, + 720, 721, 722, -304, -303, -302, -301, -300, + -299, -298, -297, 930, -295, -294, -293, -292, + -291, -290, -289, -288, -287, -286, -285, -284, + -283, -282, -281, -280, -279, -278, -277, -276, + -275, 753, 754, 755, 646, 757, -712, -1765, + 952, -712, 2244, -712, 2245, 765, 766, 767, + 768, 125, 770, 771, 772, 773, 774, 775, + 603, 777, 778, 779, 780, 781, 782, 783, + 784, 2011, 786, 787, 788, 789, 790, 791, + 792, 793, 794, 795, 796, 797, 798, 799, + 800, 801, 802, 803, 804, 805, 806, 603, + 603, 809, 603, 811, 603, 603, 814, 815, + 816, 817, 435, 819, 820, 821, 3539, 823, + 603, -468, 603, -468, 603, 603, 589, 831, + 603, 603, 603, 835, 836, 837, 838, 839, + 840, 841, 842, 843, 844, 845, 846, 847, + 848, 849, 850, 851, 852, 1239, 854, 855, + 856, 857, 858, 859, 860, 1024, 862, 863, + 864, 865, 866, 867, 868, 869, 870, 871, + 872, 873, 874, 875, 876, 877, 878, 879, + 880, 881, 882, 883, 884, 1131, 1061, -373, + 888, 889, 890, 1065, 1065, 893, 1066, 895, + 896, 897, 898, 899, 900, -148, 902, 603, + 603, -166, 906, -164, 908, -162, -161, -146, + 912, 913, 914, 915, -145, 917, 1971, -745, + 920, -2035, 922, 923, 937, 925, 926, 927, + 928, -1227, 930, 931, 932, 933, 934, 935, + 936, 866, 938, -143, 940, -142, -141, 943, + -140, 32767, 945, 946, 947, 948, 949, 950, + 951, 952, 953, 954, 955, 956, 957, 958, + 959, 960, 961, -65, -64, -63, -62, -61, + -60, -59, -58, 1169, -56, -55, -54, -53, + -52, -51, -50, -49, -48, -47, -46, -45, + -44, -43, -42, -41, -40, -39, -38, -37, + -36, 992, 993, 994, 885, 996, -473, -1526, + 1191, -473, 2483, -473, 2484, 1004, 1005, 1006, + 1007, 364, 1009, 1010, 1011, 1012, 1013, 1014, + 842, 1016, 1017, 1018, 1019, 1020, 1021, 1022, + 1023, 2250, 1025, 1026, 1027, 1028, 1029, 1030, + 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, + 1039, 1040, 1041, 1042, 1043, 1044, 1045, 842, + 842, 1048, 842, 1050, 842, 842, 1053, 1054, + 1055, 1056, 674, 1058, 1059, 1060, 3778, 1062, + 842, -229, 842, -229, 842, 842, 828, 1070, + 842, 842, 842, 1074, 1075, 1076, 1077, 1078, + 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, + 1087, 1088, 1089, 1090, 1091, 1478, 1093, 1094, + 1095, 1096, 1097, 1098, 1099, 1263, 1101, 1102, + 1103, 1104, 1105, 1106, 1107, 1108, 1109, 1110, + 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, + 1119, 1120, 1121, 1122, 1123, 1370, 1300, -134, + 1127, 1128, 1129, 1304, 1304, 1132, 1305, 1134, + 1135, 1136, 1137, 1138, 1139, 91, 1141, 842, + 842, 73, 1145, 75, 1147, 77, 78, 93, + 1151, 1152, 1153, 1154, 94, 1156, 2210, -506, + 1159, -1796, 1161, 1162, 1176, 1164, 1165, 1166, + 1167, -988, 1169, 1170, 1171, 1172, 1173, 1174, + 1175, 1105, 1177, 96, 1179, 97, 98, 1182, + 99, 1184, 1185, 1186, 1187, 1188, 1189, 1190, + 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, + 1199, 1200, 0, 174, 175, 176, 177, 178, + 179, 180, 181, 1408, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 198, 199, 200, 201, 202, + 203, 0, 0, 206, 0, 208, 0, 0, + 211, 212, 213, 214, -168, 216, 217, 218, + 2936, 220, 0, -1071, 0, -1071, 0, 0, + -14, 228, 0, 0, 0, 232, 233, 234, + 235, 236, 237, 238, 239, 240, 241, 242, + 243, 244, 245, 246, 247, 248, 249, 636, + 251, 252, 253, 254, 255, 256, 257, 421, + 259, 260, 261, 262, 263, 264, 265, 266, + 267, 268, 269, 270, 271, 272, 273, 274, + 275, 276, 277, 278, 279, 280, 281, 528, + 458, -976, 285, 286, 287, 462, 462, 290, + 463, 292, 293, 294, 295, 296, 297, -751, + 299, 0, 0, -769, 303, -767, 305, -765, + -764, -749, 309, 310, 311, 312, -748, 314, + 1368, -1348, 317, -2638, 319, 320, 334, 322, + 323, 324, 325, -1830, 327, 328, 329, 330, + 331, 332, 333, 263, 335, -746, 337, -745, + -744, 340, -743, 342, 343, 344, 345, 346, + 347, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 0, 0, 0, 1453, + 0, 1126, 495, 495, 495, 495, 495, 233, + 495, 1080, 1080, 1080, 495, 561, 1082, 495, + 563, 1687, 495, 495, 495, 495, 385, 0, + 0, 0, 0, 480, 0, 221, 221, 0, + 489, 579, 0, 0, 0, 498, 626, 0, + 1422, 1040, 1424, 631, 0, 0, 0, 0, + 0, -262, 0, 585, 585, 585, 0, 66, + 587, 0, 68, 1192, 0, 0, 0, 0, + 0, 0, 32767, 32767, 32767, 32767, 669, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 670, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 142, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 115, 116, 117, 118, 119, 120, + 121, 122, 123, 124, 125, 126, 127, 128, + 129, 130, 131, 132, 133, 134, 135, 136, + 137, 138, 139, 140, 141, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1027, 1027, 1027, + 1027, 1027, 1027, 1027, 1027, -199, 1027, 1027, + 1027, 1027, 1027, 1027, 1027, 1027, 1027, 1027, + 1027, 1027, 1027, 1027, 1027, 1027, 1027, 1027, + 1027, 1027, 1027, 0, 0, 0, 110, 0, + 1470, 2524, -192, 1473, -1482, 1475, -1481, 0, + 0, 0, 0, 644, 0, 0, 0, 0, + 0, 0, 173, 0, 0, 0, 0, 0, + 0, 0, 0, -1226, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 204, 205, 0, 207, 0, 209, 210, + 0, 0, 0, 0, 383, 0, 0, + }; + + const unsigned char *k = (const unsigned char *) key; + size_t keylen = 4; + uint32 a = 0; + uint32 b = 0; + + while (keylen--) + { + unsigned char c = *k++; + + a = a * 257 + c; + b = b * 17 + c; + } + return h[a % 2463] + h[b % 2463]; +} + +/* Hash lookup information for NFC_QC */ +static const pg_unicode_norminfo UnicodeNormInfo_NFC_QC = { + UnicodeNormProps_NFC_QC, + NFC_QC_hash_func, + 1231 +}; + static const pg_unicode_normprops UnicodeNormProps_NFKC_QC[] = { {0x00A0, UNICODE_NORM_QC_NO}, {0x00A8, UNICODE_NORM_QC_NO}, @@ -6165,3 +6514,1262 @@ static const pg_unicode_normprops UnicodeNormProps_NFKC_QC[] = { {0x2FA1C, UNICODE_NORM_QC_NO}, {0x2FA1D, UNICODE_NORM_QC_NO}, }; + +/* Perfect hash function for NFKC_QC */ +static int +NFKC_QC_hash_func(const void *key) +{ + static const int16 h[9837] = { + -2472, -2472, -2472, -2472, -2472, -2472, -2472, -2472, + -2472, -2472, -2472, -2472, -2472, -2472, -2472, -2472, + -2472, -2472, -2472, -2472, -2472, -2472, -2472, -2472, + -2472, -2472, -2472, -2472, -2472, 32767, 32767, 32767, + -2475, -2475, -2475, -2475, -2475, -2475, -2475, -2475, + -2475, -2475, -2475, -2475, -2475, -2475, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 865, 865, 865, 865, 865, 865, 865, + 865, 865, 865, 865, -2255, 32767, -5207, 32767, + -5207, 860, 860, 860, 860, 860, 860, 860, + 860, 860, 4250, 861, 861, 861, 3339, 3339, + 3339, 3339, 3339, 3339, 3339, 3339, 3339, 3339, + 3339, 3339, 3339, 3339, 3339, 3339, 3339, 3339, + 32767, 3338, 3338, 3338, 3338, 3338, 3338, 3338, + 3338, 3338, 3338, 3338, 3338, 3338, 3338, 3338, + 3338, 3338, 3338, 3338, 3338, 3338, 3338, 3338, + 3338, 3338, 3338, 3338, 3338, 3338, 3338, 3338, + 3338, 9, 10, 32767, 11, 12, 0, 32767, + 0, 2913, 2914, 2915, 2916, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 2917, 32767, 2918, -100, + 2919, 2920, 2921, 840, 840, 840, 2922, 0, + 0, 0, 0, 0, 2206, 0, 2923, 0, + 2924, 2925, 2926, 0, 0, 0, -2590, 0, + 0, 0, 0, 0, 0, 0, 2934, 0, + 2474, 2931, 2932, 0, 0, 0, 0, 0, + 14, 805, 0, 0, 2933, 0, 2934, 0, + 2935, 2936, 0, 0, 0, 16, 17, 0, + 0, 0, 0, 0, 0, 0, 0, 18, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, -790, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, -1675, 0, 0, 19, 0, -1679, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, -1694, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 29, 30, 31, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 724, 2668, 724, 4350, -2633, -2633, + 2533, 2534, 2535, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 2518, 2519, 2520, 1431, 45, 46, + 32767, 32767, 47, 48, 49, 50, 51, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, -3011, 53, -1125, -3010, -3010, + 32767, -3334, -1123, -3011, 60, 61, 62, 63, + 32767, 32767, 64, 32767, 65, 32767, 66, 67, + 32767, 32767, 32767, 32767, 32767, 32767, 2268, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 69, 70, + 71, 72, 73, 74, 32767, 32767, 32767, 32767, + 75, 76, 32767, 77, 281, 32767, 32767, 32767, + 32767, 32767, 32767, 811, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 1341, 1342, 1343, 1344, 1345, + 1346, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 86, + 32767, 32767, 32767, 32767, 32767, 4550, 32767, 32767, + 32767, 1135, 32767, 32767, 32767, 32767, 32767, 1130, + 3016, 32767, 3017, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 677, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 2858, 2859, 651, 2861, -438, + 2863, 2864, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, -5305, -5305, -5305, 32767, -5306, + -5306, 32767, 32767, 32767, 2871, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 3022, 3023, 680, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, -272, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 4308, 4309, 4310, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 4311, 4312, 4313, + 4314, 4315, 4316, 4317, 4318, 4319, 4320, 4321, + 4322, 4323, 4324, 4325, 4326, 4307, 4307, 4307, + 4307, 4307, 4307, 4307, 4307, 4307, 4336, 4337, + 4338, 4339, 4340, 4341, 4342, 4343, 4344, 4345, + 4346, 4347, 4348, 4349, 4350, 4351, 4352, 4353, + 4354, 32767, 32767, 32767, 32767, 4355, 4356, 4357, + 4358, 4359, 4360, 4361, 4362, 4363, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 4364, 4365, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 2202, 0, 0, 0, 59, 0, + 0, 35, 0, 0, 0, 3549, 0, 0, + 0, 0, 0, 3394, 0, 0, 3399, 0, + 0, 0, 0, 0, 0, 0, 0, 2012, + 0, 0, 0, 0, 87, 2022, 0, 7490, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 2255, 0, 2256, 2256, 2256, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 32767, 0, 0, + 0, 0, 0, 0, -1759, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 4767, 0, 0, 4772, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 32767, 5977, 0, + 892, 32767, 0, 32767, 32767, 0, 0, 32767, + 32767, 2344, 4834, 4835, 4836, 32767, 0, 4840, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 32767, 0, 32767, 0, 0, 0, + 0, 0, 0, 0, 32767, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 32767, 32767, 0, 32767, 0, 0, 0, 32767, + 32767, 32767, 32767, 3261, 3262, 32767, 3007, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 106, 107, 108, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 109, 110, 111, 112, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 0, 0, -2344, + -2344, 0, 32767, 0, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, -1642, 1469, -1641, 1469, -1640, 1469, + 1469, 1457, 1469, 1469, 1469, -4254, -4254, -4254, + -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4254, + -4254, -4254, -4254, -4254, -4254, -4254, -3359, -4254, + -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4254, + -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4254, + -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4254, + -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4254, + -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4103, + -1478, 0, -4254, -4254, -4254, -4254, -4254, -4254, + -4254, -4254, -4254, -2433, -4254, -4254, -4254, -3658, + -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4254, + -4254, -4254, 0, -4253, -4253, -4253, -4253, -4253, + -4253, -4253, -4253, -4253, -678, -677, -676, -675, + -674, -673, -672, -4253, 314, -4253, -4253, -4253, + -4253, -4253, -4253, -4253, -4253, -4253, -4253, -4253, + -4253, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 1464, 1465, 1466, 1467, + 1468, 1469, 0, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 0, + 0, 0, 0, 0, 32767, 32767, 32767, 32767, + 32767, 0, 32767, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 827, 828, 829, -2469, -2469, -260, 0, + 0, 32767, 0, 32767, 0, 0, 32767, 0, + 0, 32767, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 3575, 3576, 3577, 3578, 3579, 3580, 3581, 0, + 4567, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 2201, 4411, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, -3338, 0, 0, 0, + 0, 0, 0, 0, -3337, 0, -3336, 0, + 0, 0, 0, -3335, 0, 0, -3334, -3333, + -3332, -3331, 0, 0, -3330, 0, 0, 32767, + 0, 0, 13, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 3073, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + -2556, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 3074, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 2355, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, -488, -488, -488, -302, -3067, -3067, + -3067, -3067, -488, -488, -488, -488, 2999, -488, + 2999, -488, -488, -488, -3067, -3067, -3067, -488, + -488, -3067, -3067, -3067, -488, -488, -488, 2463, + -488, -488, -488, -301, 2465, -488, 2466, 2467, + -3600, -493, -3599, -488, -3598, -488, -3597, -488, + -488, -500, -488, -488, -488, -488, -488, 2470, + 2471, 2472, -488, -488, -254, -488, -488, -488, + -488, -488, -104, -488, -488, -488, -102, -101, + -100, -99, -98, -97, -96, -95, -94, -93, + -92, -488, -488, -488, -488, -488, -488, -488, + -488, -488, -2194, -2194, -2194, -2194, -2194, -2194, + -2194, -2194, -2194, -2194, 5211, 3269, 5213, 3269, + 6895, -88, -88, 5078, 5079, 5080, 1773, -92, + -92, 1773, 1773, 1773, 1773, 1773, 1773, 5072, + 5073, 2865, 5075, 1776, 5077, 5078, 1778, 1778, + 6942, 6943, 1778, 1778, 1778, 5086, 6952, 6953, + 5089, 5090, 5091, 5092, 5093, 5094, 5095, 5096, + 4007, 5098, 2333, 2334, 2335, 2336, 2337, -3066, + -3066, -3066, 2341, -3066, -3066, 2344, 2345, 2346, + 5114, 317, 2349, 848, 849, 850, 2353, 852, + 853, 854, 855, 856, 857, 858, 859, 860, + 861, 692, 692, 692, 692, 692, 692, 692, + 692, 692, 692, 692, 692, 692, 692, 692, + 692, 692, 692, 692, 692, 692, 692, 692, + 692, 692, 692, 692, 692, 692, 692, 692, + 692, 692, 692, 692, 692, 692, 692, 692, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 3093, 3094, 3095, 3096, 3097, 3098, 3099, + 3100, 3101, 3102, 901, 3104, 3105, 3106, 3048, + 3108, 3109, 3075, 3111, 3112, 3113, -435, 3115, + 3116, 3117, 3118, 3119, -274, 3121, 3122, -276, + 3124, 3125, 3126, 3127, 3128, 3129, 3130, 3131, + 1120, 3133, 3134, 3135, 3136, 3050, 1116, 3139, + -4350, 3141, 3142, 3143, 3144, 3145, 3146, 3147, + 3148, 3149, 3150, 3151, 3152, 3153, 3154, 3155, + 3156, 902, 3158, 903, 904, 905, 3162, 3163, + 3164, 3165, 3166, 3167, 3168, 3169, 3170, 3171, + 3172, 3173, 3174, 3175, 3176, 3177, 32767, 3178, + 3179, 3180, 3181, 3182, 3183, 4943, 3185, 3186, + 3187, 3188, 3189, 3190, 3191, 3192, 3193, 3194, + 3195, 3196, 3197, 3198, 3199, 3200, 3201, 3202, + 3203, 3204, 3205, 3206, 3207, 3208, 3209, 3210, + 3211, 3212, 3213, 3214, 3215, 3216, 3217, 3218, + 3219, 3220, 3221, 3222, 3223, -1543, 3225, 3226, + -1545, 3228, 3229, 3230, 3231, 3232, 3233, 3234, + 3235, 3236, 3237, 3238, 3239, 3240, 3241, 3242, + 3243, 3244, 3245, 3246, 3247, 3248, -1251, -2728, + 3250, 32767, 32767, 3251, 906, 907, 3252, 3253, + 32767, 32767, 910, -1579, -1579, -1579, 32767, 3258, + -1581, 3260, 3261, 3262, 3263, 3264, 3265, 3266, + 3267, 3268, 3269, 32767, 3270, 32767, 3271, 3272, + 3273, 3274, 3275, 3276, 3277, 32767, 3278, 3279, + 3280, 3281, 3282, 3283, 3284, 3285, 3286, 3287, + 3288, 3289, 3290, 3291, 3292, 3293, 3294, 3295, + 3296, 3297, 3298, 3299, 3300, 3301, 3302, 3303, + 3304, 3305, 3306, 3307, 3308, 3309, 3310, 3311, + 3312, 3313, 3314, 3315, 3316, 3317, 3318, 3319, + 3320, 3321, 3322, 3323, 3324, 3325, 3326, 3327, + 3328, 3329, 3330, 3331, 3332, 3333, 3334, 3335, + 3336, 32767, 3337, 3338, 3339, 3340, 3341, 3342, + 0, 3343, 3344, 3345, 3346, 32767, 32767, 3347, + 3348, 3349, 3350, 3351, 3352, 3353, 3354, 32767, + 3355, 3356, 3357, 3358, 3359, 3360, 3361, 32767, + 3362, 3363, 3364, 3365, 3366, 3367, 3368, 3369, + 3370, 3371, 3372, 3373, 3374, 3375, 3376, 3377, + 3378, 3379, 3380, 3381, 3382, 3383, 3384, 3385, + 3386, 3387, 3388, 3389, 0, 3390, 3391, 3392, + 915, 916, 917, 918, 919, 920, 921, 922, + 923, 924, 925, 926, 927, 928, 929, 930, + 931, 932, 933, 934, 935, 936, 937, 938, + 939, 940, 941, 942, 943, 944, 945, 946, + 947, 948, 949, 950, 951, 952, 953, 954, + 955, 956, 957, 958, 959, 960, 961, 962, + 963, 964, 965, 966, 967, 968, 969, 970, + 971, 972, 973, 974, 975, 976, 3449, 3450, + 3451, 3452, 3453, 3454, 3455, 3456, 3457, 3458, + 3459, 3460, 3461, 3462, 3463, 3464, 3465, 3466, + 3467, 3468, 3469, 3470, 3471, 3472, 3473, 3474, + 3475, 3476, 3477, 3478, 3479, 3480, 3481, 3482, + 3483, 3484, 3485, 3486, 3487, 3488, 3489, 3490, + 3491, 3492, 3493, 3494, 3495, 3496, 3497, 3498, + 3499, 3500, 3501, 3502, 3503, 3504, 3505, 3506, + 3507, 3508, 3509, 3510, 3511, 3512, 3513, 3514, + 3515, 3516, 3517, 3518, 3519, 3520, 3521, 3522, + 3523, 3524, 3525, 3526, 3527, 3528, 3529, 3530, + 3531, 3532, 3533, 3534, 3535, 3536, 3537, 3538, + 3539, 3540, 3541, 3542, 3543, 3544, 3545, 3546, + 3547, 3548, 3549, 3550, 3551, 3552, 3553, 3554, + 3555, 3556, 3557, 3558, 3559, 3560, 3561, 3562, + 3563, 3564, 3565, 3566, 3567, 3568, 3569, 3570, + 3571, 3572, 3573, 3574, 3575, 3576, 3577, 6056, + 6057, 6058, 32767, 3581, 3582, 3583, 3584, 3585, + 4157, 4158, 4159, 3589, 4162, -4510, -1558, -1557, + -1556, -1742, -4507, -1553, -4506, -4506, 1562, -1544, + 1563, -1547, 1564, -1545, 1565, -1543, -1542, -1529, + -1540, -1539, -1538, -1537, -1536, -4493, -4493, -4493, + -1532, -1531, -1764, -1529, 3622, -1528, -1527, -1526, + -1909, -1524, -1523, -1522, -1907, -1907, -1907, -1907, + -1907, -1907, -1907, -1907, -1907, -1907, -1907, -1510, + -1509, 1071, 1072, 1073, 1074, 1075, 1076, 1077, + 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, + 1086, 1087, 1088, 1089, 1090, 3663, 3664, 3665, + 3666, 3667, 3668, 3669, 3670, 3671, 3672, 3673, + 3674, 1095, 1096, 1097, 1098, 1099, 1100, 1101, + 3682, 1103, 3684, 1105, 3686, 3687, 3688, 1109, + 1110, 1111, 3692, 1113, 1114, 1115, 1116, 1117, + 1118, 1119, 3700, 1121, 3702, 3703, 3704, 1125, + 1126, 1127, -1809, -1809, -1809, -1809, -1809, -1809, + 3720, 3721, 3722, 3717, 3718, 3719, 3720, 1140, + 1141, 1142, 1143, -1802, 1145, 1146, 1147, 1148, + 3730, -1797, 3732, 1152, 3734, 3735, 1155, 1156, + 3738, 3739, 3740, 3741, 3742, 3743, -1785, -1785, + -1785, -1779, -1324, 1168, 1169, 1170, 1171, 1172, + 3752, 3753, 1175, 1176, 1177, 992, 3758, 3759, + 3760, 3761, 1183, 1184, 1185, 1186, -2300, 1188, + -2298, 1190, 1191, 1192, 3772, 3773, 3774, 1196, + 1197, 3777, 3778, 3779, 1201, 1202, 1203, -1747, + 1205, 1206, 1207, 1021, -1744, 1210, -1743, -1743, + 4325, 1219, 4326, 1216, 4327, 1218, 4328, 1220, + 1221, 1234, 1223, 1224, 1225, 1226, 1227, -1730, + -1730, -1730, 1231, 1232, 999, 1234, 1235, 1236, + 1237, 1238, 855, 1240, 1241, 1242, 857, 857, + 857, 857, 857, 857, 857, 857, 857, 857, + 857, 1254, 1255, 1256, 1257, 1258, 1259, 1260, + 1261, 1262, 2969, 2970, 2971, 2972, 2973, 2974, + 2975, 2976, 2977, 2978, -4426, -2483, -4426, -2481, + -6106, 878, 879, -4286, -4286, -4286, -978, 888, + 889, -975, -974, -973, -972, -971, -970, -4268, + -4268, -2059, -4268, -968, -4268, -4268, -967, -966, + -6129, -6129, -963, -962, -961, -4268, -6133, -6133, + -4268, -4268, -4268, -4268, -4268, -4268, -4268, -4268, + -3178, -4268, -1502, -1502, -1502, -1502, -1502, 3902, + 3903, 3904, -1502, 3906, 3907, -1502, -1502, -1502, + -4269, 529, -1502, 0, 0, 0, -1502, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 170, 171, 172, 173, 174, 175, 176, + 177, 178, 179, 180, 181, 182, 183, 184, + 185, 186, 187, 188, 189, 190, 191, 192, + 193, 194, 195, 196, 197, 198, 199, 200, + 201, 202, 203, 204, 205, 206, 207, 208, + 209, 210, 211, 212, 213, 214, 215, 216, + 217, 218, 219, -3194, 221, 222, 223, 224, + -1657, 226, 227, -1657, 229, 230, -1655, 555, + -1655, 234, 235, 236, 732, 238, 239, 240, + 241, 242, 243, -1655, 245, 246, 247, 248, + -1655, 250, -1655, 252, -1655, -1655, -1655, -1655, + -1655, -1655, 259, -1655, -1655, -1655, -1655, 264, + -1655, 266, -1655, 268, -1655, -3620, 271, 272, + -1655, 274, 275, -1655, 277, -1655, -1655, 280, + -1655, 282, 5746, 5747, 5748, 5749, -1655, 288, + -1655, 290, -3335, 3649, 3650, -1515, -1515, -1515, + 1793, 3659, 3660, 1796, 1797, 1798, 1799, 1800, + 1801, -1497, -1497, 712, -1497, 1803, -1497, -1497, + 1804, 1805, -3358, -3358, 1808, 1809, 1810, -1497, + -3362, -3362, -1497, -1497, -1497, -1497, -1497, -1497, + -1497, -1497, -407, -1497, -1497, -1497, -1497, -1497, + -1497, 3667, 3668, -1497, -1497, -1497, 1811, 3677, + 3678, 32767, 1814, 32767, 1815, 32767, 32767, 1816, + 1817, 32767, 32767, 32767, 1818, 1819, 1820, 1821, + -3342, -3342, 1824, 1825, 1826, 1827, 1828, 1829, + 1830, 1831, 1832, 1833, 1834, 1835, 1836, 1837, + 1838, 1839, 1840, 1841, 1842, 1843, 1844, 1845, + 1846, 1847, 1848, 1849, 1850, 1851, 1852, 1853, + 1854, 1855, 1856, 1857, 1858, 1859, 1860, 1861, + 1862, 1863, 1864, 1865, 1866, 1867, 1868, 1869, + 1870, 1871, 1872, 1873, 1874, 1875, 1876, -1537, + 1878, 1879, 1880, 1881, 0, 1883, 1884, 0, + 529, 0, 0, 2210, 0, 1889, 1890, 1891, + 2387, 1893, 1894, 1895, 1896, 1897, 1898, 0, + 1900, 1901, 1902, 1903, 0, 1905, 0, 1907, + 0, 0, 0, 0, 0, 0, 1914, 0, + 0, 0, 0, 1919, 0, 1921, 0, 1923, + 0, -1965, 1926, 1927, 0, 1929, 1930, 0, + 1932, 0, 0, 1935, 0, 1937, 7401, 7402, + 7403, 7404, 0, 1943, 0, 1945, 1946, 0, + 1948, 0, 0, 1951, 1952, 1953, 1954, 0, + 1956, 1957, 1958, 1959, 1960, 1961, 1962, 0, + 1964, 1965, 1966, 1967, 0, 1969, 1970, 1971, + 1972, 0, 1974, 0, 1976, 1977, 1978, 1979, + 1980, 1981, 1982, 1983, 1984, 1985, 0, 1987, + 1988, 1989, 1990, 1991, 566, 566, 566, 5141, + 5142, 566, 566, 566, 566, 566, 566, 566, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 8673, 5722, 5722, 5722, 0, 8676, + 5723, 8677, 8678, 2611, 5718, 2612, 5723, 2613, + 5723, 2614, 5723, 5723, 5711, 5723, 5723, 5723, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 895, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 151, 2776, 4254, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1821, 0, + 0, 0, 596, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, -2856, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, -2901, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, -1025, 32767, 32767, 32767, + 32767, -2910, 32767, 32767, 32767, 32767, 157, 32767, + 32767, 32767, 32767, 158, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 2359, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 160, 32767, 161, 162, 163, 164, + 165, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 898, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 1428, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 1254, 32767, 32767, 32767, + 32767, 1250, 32767, 32767, 32767, 32767, 1246, 32767, + 32767, 32767, 32767, 1243, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 1231, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 1842, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 3177, 1235, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, -4323, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 0, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 0, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 174, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 1830, -112, 1832, -112, 3514, -3469, + -3469, 1697, 1698, 1699, -1608, -3473, -3473, -1608, + -1608, -1608, -1608, -1608, -1608, 1691, 1692, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, -1623, -1623, -1623, 3541, 3542, -1623, -1623, + -1623, -1623, -1623, -1623, -1623, -1623, -1623, -1623, + -1623, -1623, -1623, -1623, -1623, -1623, -1623, -1623, + -1623, -1623, -1623, -1623, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, -766, 2253, 2254, 2255, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 1531, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 0, 0, 32767, 0, 0, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, -173, -173, -173, -173, -173, + -173, -173, -173, -173, -173, -173, -173, 3241, + -173, -173, -173, -173, 1709, -173, -173, 1712, + -173, -173, 1713, -496, 1715, -173, -173, -173, + -668, -173, -173, -173, -173, -173, -173, 1726, + -173, -173, -173, -173, 1731, -173, 1733, -173, + 1735, 1736, 1737, 1738, 1739, 1740, -173, 1742, + 1743, 1744, 1745, -173, 1747, -173, 1749, -173, + 1751, 3717, -173, -173, 1755, -173, -173, 1758, + -173, 1760, 1761, -173, 1763, -173, -5636, -5636, + -5636, -5636, 1769, -173, 1771, -173, 3453, -3530, + -3530, 1636, 1637, 1638, -1669, -3534, -3534, -1669, + -1669, -1669, -1669, -1669, -1669, 1630, 1631, -577, + 1633, -1666, 1635, 1636, -1664, -1664, 3500, 3501, + -1664, -1664, -1664, 1644, 3510, 3511, 1647, 1648, + 1649, 1650, 1651, 1652, 1653, 1654, 565, 1656, + 1657, 1658, 1659, 1660, 1661, -3502, -3502, 1664, + 1665, 1666, 1667, 1668, 1669, 1670, 1671, 1672, + 1673, 1674, 1675, 1676, 1677, 1678, 1679, 1680, + 1681, 1682, 1683, 1684, 1685, 1686, 1687, 1688, + 1689, 1690, 1691, 1692, 1693, 1694, 1695, 1696, + 1697, 1698, 1699, 1700, 1701, 1702, 1703, 1704, + 1705, 1706, 1707, 1708, 1709, 1710, 1711, 1712, + 1713, 1714, 1715, 1716, -1697, 1718, 1719, 1720, + 1721, -160, 1723, 1724, -160, 1726, 1727, -158, + 2052, -158, 1731, 1732, 1733, 2229, 1735, 1736, + 1737, 1738, 1739, 1740, -158, 1742, 1743, 1744, + 1745, -158, 1747, -158, 1749, -158, -158, -158, + -158, -158, -158, 1756, -158, -158, -158, -158, + 1761, -158, 1763, -158, 1765, -158, -2123, 1768, + 1769, -158, 1771, 1772, -158, 1774, -158, -158, + 1777, -158, 1779, 7243, 7244, 7245, 7246, -158, + 1785, -158, 1787, -1838, 5146, 5147, -18, -18, + -18, 3290, 5156, 5157, 3293, 3294, 3295, 3296, + 3297, 3298, 0, 0, 2209, 0, 3300, 0, + 0, 3301, 3302, -1861, -1861, 3305, 3306, 3307, + 0, -1865, -1865, 0, 0, 0, 0, 0, + 0, 0, 0, 1090, 0, 0, 0, 0, + 0, 0, 5164, 5165, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 3414, 0, 0, 0, 0, 1882, 0, + 0, 1885, 0, 0, 1886, -323, 1888, 0, + 0, 0, -495, 0, 0, 0, 0, 0, + 0, 1899, 0, 0, 0, 0, 1904, 0, + 1906, 0, 1908, 1909, 1910, 1911, 1912, 1913, + 0, 1915, 1916, 1917, 1918, 0, 1920, 0, + 1922, 0, 1924, 3890, 0, 0, 1928, 0, + 0, 1931, 0, 1933, 1934, 0, 1936, 0, + -5463, -5463, -5463, -5463, 1942, 0, 1944, 0, + 0, 1947, 0, 1949, 1950, 0, 0, 0, + 0, 1955, 0, 0, 0, 0, 0, 0, + 0, 1963, 0, 0, 0, 0, 1968, 0, + 0, 0, 0, 1973, 0, 1975, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1986, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 527, 527, 527, 527, 0, + 528, 528, 528, 528, 528, 528, 528, 528, + 528, 528, 528, 1998, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 1999, 2000, 2001, 2002, 2003, 32767, 32767, 32767, + 32767, 32767, 2004, 32767, 2005, 2006, 2007, 2008, + 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, + 2025, 2026, 1200, 1200, 32767, 4498, 4499, 2291, + 2032, 2033, 32767, 2034, 32767, 2035, 2036, 32767, + 2037, 2038, 32767, 2039, 2040, 2041, 2042, 2043, + 2044, 2045, 2046, 2047, 2048, 2049, 2050, 2051, + 2052, 2053, 2054, 2055, 2056, 2057, 2058, 2059, + 2060, 2061, 2062, 2063, 2064, 2065, 2066, 2067, + 2068, -1506, -1506, -1506, -1506, -1506, -1506, -1506, + 2076, -2490, 2078, 2079, 2080, 2081, 2082, 2083, + 2084, 2085, 2086, 2087, 2088, 2089, 2090, 2091, + 2092, 2093, 2094, 2095, -105, -2314, 2098, 2099, + 2100, 2101, 2102, 2103, 2104, 2105, 2106, 2107, + 2108, 2109, 2110, 2111, 2112, 2113, 2114, 2115, + 2116, 2117, 2118, 2119, 2120, 5459, 2122, 2123, + 2124, 2125, 2126, 2127, 2128, 5466, 2130, 5467, + 2132, 2133, 2134, 2135, 5471, 2137, 2138, 5473, + 5473, 5473, 5473, 2143, 2144, 5475, 2146, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 2147, 2148, 2149, 2150, 2151, 2152, 2153, 2154, + 2155, 2156, 2157, 2158, 2159, 2160, 2161, 2162, + 2163, 2164, 2165, 2166, 2167, 2168, 2169, 2170, + 2171, 2172, 2173, 2174, 2175, 2176, 2177, 2178, + 2179, 2180, 2181, 2182, 2183, 2184, 2185, 2186, + 2187, 2188, 2189, 2190, 2191, 32767, -726, 2293, + -725, -725, -725, 1357, 1358, 1359, -722, 2201, + 2202, 2203, 2204, 2205, 0, 2207, -715, 2209, + -714, -714, -714, 2213, 2214, 2215, 4806, 2217, + 2218, 2219, 2220, 2221, 2222, 2223, -710, 2225, + -248, -704, -704, 2229, 2230, 2231, 2232, 2233, + 2220, 1430, 2236, 2237, -695, 2239, -694, 2241, + -693, -693, 2244, 2245, 2246, 2231, 2231, 2249, + 2250, 2251, 2252, 2253, 2254, 2255, 2256, 2239, + 2258, 2259, 2260, 2261, 2262, 2263, 2264, 2265, + 2266, 2267, 2268, 2269, 2270, 2271, 2272, 2273, + 2274, 2275, 2276, 2277, 2278, 2279, 2280, 2281, + 2282, 2283, 2284, 2285, 2286, 2287, 2288, 2289, + 2290, 2291, 2292, 2293, 3084, 2295, 2296, 2297, + 2298, 2299, 2300, 2301, 2302, 2303, 2304, 2305, + 2306, 2307, 3983, 2309, 2310, 2292, 2312, 3992, + 2314, 2315, 2316, 2317, 2318, 2319, 2320, 2321, + 2322, 2323, 2324, 2325, 2326, 2327, 2328, 4023, + 2330, 2331, 2332, 2333, 2334, 2335, 2336, 2337, + 2338, 2339, 2340, 2341, 2342, 2343, 2344, 2345, + 2346, 2347, 2348, 2349, 2350, 2351, 2352, 2353, + 2354, 2355, 2356, 2357, 2358, 2359, 2360, 2361, + 2362, 2363, 2364, 2365, 2366, 2367, 2368, 2369, + 2370, 2371, 2372, 2373, 2374, 2375, 2376, 2377, + 2378, 2379, 2360, 2360, 2360, 2360, 2360, 2360, + 2360, 2360, 2360, 2389, 2390, 2391, 2392, 2393, + 2394, 2395, 2396, 2397, 2398, 2399, 2400, 2401, + 2402, 2403, 2404, 2405, 2406, 2407, 2408, 2409, + 2410, 2411, 2412, 2413, 2414, 2415, 2416, 2417, + 2418, 2419, 2420, 2421, 2422, 2423, 2424, 2425, + 2426, 2427, 2428, 2429, 2430, 2431, 2432, 2433, + 2434, 2435, 2436, 2437, 2438, 2439, 2440, 2441, + 2442, 2443, 2444, 2445, 2446, 2447, 32767, 2448, + 2449, 2450, 2451, 2452, 2453, 2454, 2455, 2456, + 2457, 2458, 2459, 2460, 2461, 2462, 2463, 2464, + 2465, 2466, 2467, 2468, 2469, 2470, 2471, 2472, + 2473, 2474, 2475, 2476, 2477, 2478, 2479, 2480, + 2481, 2482, 2483, 2484, 2485, 2486, 2487, 2488, + 2489, 2490, 2491, 2492, 2493, 2494, 2495, 2496, + 2497, 2498, 2499, 2500, 2501, 2502, 2503, 2504, + 2505, 2506, 2507, 2508, 2509, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 2510, + 2511, 2512, 2513, 3266, 3266, 3266, 3266, 2518, + 3267, 3267, 3267, 2522, 3268, 3268, 3268, 3268, + 3268, 3268, 3268, 6682, 3268, 3268, 3268, 2534, + 5151, 3269, 2537, 2538, 3271, 3271, 5157, 2948, + 5159, 2544, 2545, 3273, 2778, 3273, 2549, 3274, + 2551, 3275, 2553, 5175, 2555, 3277, 3277, 3277, + 5181, 2560, 5184, 3278, 5186, 2564, 5189, 5190, + 5191, 5192, 3279, 5194, 5195, 2572, 5198, 32767, + 32767, 3278, 5200, 3278, 2577, 2578, 2579, 2580, + 5210, 3282, 3282, 5213, 3282, 2586, 2587, 2588, + 2589, 2590, 2591, -2175, -2175, -2175, 5230, 3288, + 5232, 3288, 6914, -69, -69, 5097, 5098, 5099, + 1792, -73, -73, 1792, 1792, 1792, 1792, 1792, + 1792, 5091, 5092, 2884, 5094, 1795, 5096, 5097, + 1797, 1797, 6961, 6962, 1797, 1797, 1797, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 2578, 2578, 2578, 2578, 2578, + 2578, 872, 872, 872, 872, 872, 872, 872, + 872, 872, 872, 8277, 6335, 8279, 6335, 9961, + 2978, 2978, 8144, 8145, 8146, 4839, 2974, 2974, + 4839, 4839, 4839, 4839, 4839, 4839, 8138, 8139, + 5931, 8141, 4842, 8143, 8144, 4844, 4844, 10008, + 10009, 4844, 4844, 4844, 8152, 10018, 10019, 8155, + 8156, 8157, 8158, 8159, 8160, 8161, 8162, 7073, + 8164, 5399, 5400, 5401, 5402, 5403, 0, 0, + 0, 5407, 0, 0, 5410, 5411, 5412, 8180, + 3383, 5415, 3914, 3915, 3916, 5419, 3918, 3919, + 3920, 3921, 3922, 3923, 3924, 3925, 3926, 3927, + 3758, 3758, 3758, 3758, 3758, 3758, 3758, 3758, + 3758, 3758, 3758, 3758, 3758, 3758, 3758, 3758, + 3758, 3758, 3758, 3758, 3758, 3758, 3758, 3758, + 3758, 3758, 3758, 3758, 3758, 3758, 3758, 3758, + 3758, 3758, 3758, 3758, 3758, 3758, 3758, 3758, + 3758, 3758, 3758, 3758, 3758, 3758, 3758, 3758, + 3758, 3758, 7172, 3758, 3758, 3758, 3758, 5640, + 3758, 3758, 5643, 3758, 3758, 5644, 3435, 5646, + 3758, 3758, 3758, 3263, 3758, 3758, 3758, 3758, + 3758, 3758, 5657, 3758, 3758, 3758, 3758, 5662, + 3758, 5664, 3758, 5666, 5667, 5668, 5669, 5670, + 5671, 3758, 5673, 5674, 5675, 5676, 3758, 5678, + 3758, 5680, 3758, 5682, 7648, 3758, 3758, 5686, + 3758, 3758, 5689, 3758, 5691, 5692, 3758, -1707, + -1707, -1707, -1707, -1707, -1707, 5698, 3756, 5700, + 3756, 7382, 399, 399, 5565, 5566, 5567, 2260, + 395, 395, 2260, 2260, 2260, 2260, 2260, 2260, + 5559, 5560, 3352, 5562, 2263, 5564, 5565, 2265, + 2265, 7429, 7430, 2265, 2265, 2265, 5573, 7439, + 7440, 5576, 5577, 5578, 5579, 5580, 5581, 5582, + 5583, 4494, 5585, 2820, 2821, 2822, 2823, 2824, + -2579, -2579, -2579, 2828, -2579, -2579, 2831, 2832, + 2833, 5601, 804, 2836, 1335, 1336, 1337, 2840, + 1339, 1340, 1341, 1342, 1343, 1344, 1345, 1346, + 1347, 1348, 1179, 1179, 1179, 1179, 1179, 1179, + 1179, 1179, 1179, 1179, 1179, 1179, 1179, 1179, + 1179, 1179, 1179, 1179, 1179, 1179, 1179, 1179, + 1179, 1179, 1179, 1179, 1179, 1179, 1179, 1179, + 1179, 1179, 1179, 1179, 1179, 1179, 1179, 1179, + 1179, 1179, 1179, 1179, 1179, 1179, 1179, 1179, + 1179, 1179, 1179, 1179, 4593, 1179, 1179, 1179, + 1179, 3061, 1179, 1179, 3064, 1179, 1179, 3065, + 856, 3067, 1179, 1179, 1179, 684, 1179, 1179, + 1179, 1179, 1179, 1179, 3078, 1179, 1179, 1179, + 1179, 3083, 1179, 3085, 1179, 3087, 3088, 3089, + 3090, 3091, 3092, 1179, 3094, 3095, 3096, 3097, + 1179, 3099, 1179, 3101, 1179, 3103, 5069, 1179, + 1179, 3107, 1179, 1179, 3110, 1179, 3112, 3113, + 1179, 3115, 1179, -4284, -4284, -4284, -4284, 3121, + 1179, 3123, 1179, 4805, -2178, -2178, 2988, 2989, + 2990, -317, -2182, -2182, -317, -317, -317, -317, + -317, -317, 2982, 2983, 775, 2985, -314, 2987, + 2988, -312, -312, 4852, 4853, -312, -312, -312, + 2996, 4862, 4863, 2999, 3000, 3001, 3002, 3003, + 3004, 3005, 3006, 1917, 3008, 3009, 3010, 3011, + 3012, 3013, -2150, -2150, 3016, 3017, 3018, 3019, + 3020, 3021, 3022, 3023, 3024, 3025, 3026, 3027, + 3028, 3029, 3030, 3031, 3032, 3033, 3034, 3035, + 32767, 32767, 32767, 3036, 3037, 3038, 3039, 3040, + 3041, 32767, 32767, 3042, 3043, 3044, 3045, 3046, + 3047, 32767, 32767, 3048, 3049, 3050, 3051, 3052, + 3053, 32767, 32767, 3054, 3055, 3056, 32767, 32767, + 32767, -357, 3058, 3059, 3060, 3061, 1180, 3063, + 0, 1179, 3065, 3066, 1181, 3391, 1181, 3070, + 0, 0, 0, 0, 32767, 0, 0, 32767, + 0, 32767, 0, 0, -4973, 32767, 32767, -7368, + -2202, -2201, -2200, -5507, -7372, -7372, -5507, -5507, + -5507, 32767, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 32767, 0, 0, 32767, 0, + -203, -2234, -732, -732, -732, -2234, -732, -732, + -2763, -1261, -1261, -1261, -2763, -1261, -1261, -1261, + -1261, -1261, -1261, -1261, -1261, -1261, -1261, -1091, + -1090, -1089, -1088, -1087, 32767, 32767, -1086, -1085, + -1084, -1083, -1082, -1081, -1080, -1079, -1078, -1077, + -1076, -1075, 32767, -1074, -1073, -1072, -1071, -1070, + -1069, -1068, -1067, -1066, -1065, -1064, -1063, -1062, + -1061, -1060, -1059, -1058, -1057, -1056, 32767, -1055, + -1054, -1053, -1052, 0, 32767, 32767, 32767, -1051, + -1050, -4463, 32767, -1048, 32767, -1047, -2928, -1045, + -1044, -2928, -1042, -1041, -2926, -716, -2926, -1037, + -1036, -1035, -539, -1033, -1032, -1031, -1030, -1029, + -1028, -2926, -1026, -1025, -1024, -1023, -2926, -1021, + -2926, -1019, -2926, -2926, -2926, -2926, -2926, -2926, + -1012, -2926, -2926, -2926, -2926, -1007, -2926, -1005, + -2926, -1003, -2926, -4891, -1000, -999, -2926, -997, + -996, -2926, -994, -2926, -2926, -991, 4475, 4476, + 4477, 4478, 4479, 4480, -2924, -981, -2924, -979, + -4604, 2380, 2381, -2784, -2784, -2784, 524, 2390, + 2391, 527, 528, 529, 530, 531, 532, -2766, + -2766, -557, -2766, 534, -2766, -2766, 535, 536, + -4627, -4627, 539, 540, 541, -2766, -4631, -4631, + -2766, -2766, -2766, -2766, -2766, -2766, -2766, -2766, + -1676, -2766, 0, 0, 0, 0, 0, 5404, + 5405, 5406, 0, 5408, 5409, 0, 0, 0, + -2767, 2031, 0, 1502, 1502, 1502, 0, 1502, + 1502, 1502, 1502, 1502, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 224, 225, 226, 32767, 227, 228, 229, + 230, 231, 232, 233, 234, 235, 236, 67, + 32767, 66, 66, 66, 66, 66, 66, 66, + 66, 66, 66, 66, 66, 66, 66, 66, + 66, 66, 66, 32767, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 65, 65, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, -271, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 1940, 18, 1942, 3908, 18, 18, 1946, 18, + 18, 1949, 18, 1951, 1952, 18, 1954, 18, + -5445, -5445, -5445, -5445, 1960, 18, 1962, 18, + 3644, -3339, -3339, 1827, 1828, 1829, -1478, -3343, + -3343, -1478, -1478, -1478, -1478, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 0, 0, 0, + 0, 32767, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1340, 1341, + 1342, 1343, 1344, 1345, 1346, 1347, 1348, 1349, + -2064, 1351, 1352, 1353, 1354, 32767, 1355, 1356, + 32767, 0, 32767, 32767, 1679, 32767, 1357, 1358, + 1359, 1855, 1361, 1362, 1363, 1364, 1365, 1366, + 32767, 1367, 1368, 1369, 1370, 32767, 1371, 32767, + 1372, 32767, 32767, 32767, 32767, 32767, 32767, 1373, + 32767, 32767, 32767, 32767, 1374, 32767, 1375, 32767, + 1376, 32767, -2513, 1378, 1379, 32767, 1380, 1381, + 32767, 1382, 32767, 32767, 1383, 32767, 1384, 32767, + 6848, 32767, 6849, 32767, 1387, 32767, 1388, 1389, + 32767, 1390, 32767, 32767, 1391, 1392, 1393, 1394, + 32767, 1395, 1396, 1397, 1398, 1399, 1400, 1401, + 32767, 1402, 1403, 1404, 1405, 32767, 1406, 1407, + 1408, 1409, 32767, 1410, 32767, 1411, 1412, 1413, + 1414, 1415, 1416, 1417, 1418, 1419, 1420, 32767, + 1421, 1422, 1423, 1424, 1425, 0, 0, 0, + 4575, 4576, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, -571, -571, + -571, 0, -572, 8101, 5150, 5150, 5150, 5337, + 8103, 5150, 8104, 8105, 2038, 5145, 2039, 5150, + 2040, 5150, 2041, 5150, 5150, 5138, 5150, 5150, + 5150, 5150, 5150, 8108, 8109, 8110, 5150, 5150, + 5384, 5150, 0, 5151, 5151, 5151, 5535, 5151, + 5151, 5151, 5537, 5538, 5539, 5540, 5541, 5542, + 5543, 5544, 5545, 5546, 5547, 5151, 5151, 2572, + 2572, 2572, 2572, 2572, 2572, 2572, 2572, 2572, + 2572, 2572, 2572, 2572, 2572, 2572, 2572, 2572, + 2572, 2572, 2572, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 2580, + 2580, 2580, 2580, 2580, 2580, 2580, 0, 2580, + 0, 2580, 0, 0, 0, 2580, 2580, 2580, + 0, 2580, 2580, 2580, 2580, 2580, 2580, 2580, + 0, 2580, 0, 0, 0, 2580, 2580, 2580, + 5517, 5518, 5519, 5520, 5521, 5522, -6, -6, + -6, 0, 0, 0, 0, 2581, 2581, 2581, + 2581, 5527, 2581, 2581, 2581, 2581, 0, 5528, + 0, 2581, 0, 0, 2581, 2581, 0, 0, + 0, 0, 0, 0, 5529, 5530, 5531, 32767, + 32767, 2579, 2579, 2579, 2579, 2579, 0, 0, + 2579, 2579, 2579, 2765, 0, 0, 0, 0, + 2579, 2579, 2579, 2579, 6066, 2579, 6066, 2579, + 2579, 2579, 0, 0, 0, 2579, 2579, 0, + 0, 0, 2579, 2579, 2579, 5530, 2579, 2579, + 2579, 2766, 5532, 2579, 5533, 5534, -533, 2574, + -532, 2579, -531, 2579, -530, 2579, 2579, 2567, + 2579, 2579, 2579, 2579, 2579, 5537, 5538, 5539, + 2579, 2579, 2813, 2579, 2579, 2579, 2579, 2579, + 2963, 2579, 2579, 2579, 2965, 2966, 2967, 2968, + 2969, 2970, 2971, 2972, 2973, 2974, 2975, 2579, + 2579, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 32767, 32767, 32767, + 32767, 32767, 331, 32767, 332, -2580, -2580, -2580, + -2580, 0, 0, 0, 0, 0, 0, 0, + -2580, 0, -2580, 0, -2580, -2580, -2580, 0, + 0, 0, -2580, 0, 0, 0, 0, 0, + 0, 0, -2580, 0, -2580, -2580, -2580, 0, + 0, 0, 2937, 2938, 2939, 2940, 2941, 2942, + -2586, -2586, -2586, -2580, -2125, -2581, -2581, 0, + 0, 0, 0, 2946, 0, 0, 0, 0, + -2581, 2947, -2581, 0, -2581, -2581, 0, 0, + -2581, -2581, -2581, -2581, -2581, -2581, 2948, 2949, + 2950, 2945, 2491, 0, 0, 0, 0, 0, + -2579, -2579, 0, 0, 0, 186, -2579, -2579, + -2579, -2579, 0, 0, 0, 0, 3487, 0, + 3487, 0, 0, 0, -2579, -2579, -2579, 0, + 0, -2579, -2579, -2579, 0, 0, 0, 2951, + 0, 0, 0, 187, 2953, 0, 2954, 2955, + -3112, -5, -3111, 0, -3110, 0, -3109, 0, + 0, -12, 0, 0, 0, 0, 0, 2958, + 2959, 2960, 0, 0, 234, 0, 0, 0, + 0, 0, 384, 0, 0, 0, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, + 396, 0, 0, 0, 0, 0, 0, 0, + 0, 0, -1706, -1706, -1706, 0, 0, 0, + 0, 385, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 397, + 398, 399, 400, 401, 402, 403, 404, 405, + 2112, 2113, 2114, 409, 410, 411, 412, 32767, + 413, 414, 415, 416, 417, 418, 419, 420, + 421, 422, 423, 424, 425, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + -1688, 32767, 32767, 32767, 32767, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 0, 0, 0, + 0, -752, -751, -750, -749, 0, -748, -747, + -746, 0, -745, -744, -743, -742, -741, -740, + -739, -4152, -737, -736, -735, 0, -2616, -733, + 0, 0, -732, -731, -2616, -406, -2616, 0, + 0, -727, -231, -725, 0, -724, 0, -723, + 0, -2621, 0, -721, -720, -719, -2622, 0, + -2623, -716, -2623, 0, -2624, -2624, -2624, -2624, + -710, -2624, -2624, 0, -2625, -706, -2625, -704, + -2625, -702, 0, 0, 0, 0, -2629, -700, + -699, -2629, -697, 0, 0, 0, 0, 0, + 0, 4767, 4768, 4769, -2635, -692, -2635, -690, + -4315, 2669, 2670, -2495, -2495, -2495, 813, 2679, + 2680, 816, 817, 818, 819, 820, 821, -2477, + -2477, -268, -2477, 823, -2477, -2477, 824, 825, + -4338, -4338, 828, 829, 830, -2477, -4342, -4342, + -2477, -2477, -2477, -2477, -2477, -2477, -2477, -2477, + -1387, 0, 0, 32767, 32767, 0, 0, 0, + 0, 0, -2486, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 1756, 1757, 1758, + 1759, -5645, -3702, -5645, -3700, -7325, -341, -340, + -5505, -5505, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 532, 533, + 32767, 534, 535, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, -781, 1084, 1084, 1084, 1084, + 1084, 1084, 4383, 4384, 2176, 4386, 1087, 4388, + 4389, 1089, 1089, 6253, 6254, 1089, 1089, 1089, + 4397, 6263, 6264, 4400, 4401, 4402, 4403, 4404, + 4405, 4406, 4407, 3318, 4409, 4410, 4411, 4412, + 4413, 4414, -749, -749, 4417, 4418, 4419, 4420, + 4421, 4422, 4423, 4424, 4425, 4426, 4427, 4428, + 4429, 4430, 4431, 4432, 4433, 4434, 4435, 4436, + 4437, 4438, 4439, 4440, 4441, 4442, 4443, 4444, + 4445, 4446, 4447, 4448, 4449, 4450, 4451, 4452, + 4453, 4454, 4455, 4456, 4457, 4458, 4459, 4460, + 4461, 4462, 4463, 4464, 4465, 4466, 4467, 4468, + 4469, 1056, 4471, 4472, 4473, 4474, 2593, 4476, + 4477, 2593, 4479, 4480, 2595, 4805, 2595, 4484, + 4485, 4486, 4982, 4488, 4489, 4490, 4491, 4492, + 4493, 2595, 4495, 4496, 4497, 4498, 2595, 4500, + 2595, 4502, 2595, 2595, 2595, 2595, 2595, 2595, + 4509, 2595, 2595, 2595, 2595, 4514, 2595, 4516, + 2595, 4518, 2595, 630, 4521, 4522, 2595, 4524, + 4525, 2595, 4527, 2595, 2595, 4530, 2595, 4532, + 9996, 9997, 9998, 9999, 2595, 4538, 2595, 4540, + 4541, 2595, 4543, 2595, 2595, 4546, 4547, 4548, + 4549, 2595, 4551, 4552, 4553, 4554, 4555, 4556, + 4557, 2595, 4559, 4560, 4561, 4562, 2595, 4564, + 4565, 4566, 4567, 2595, 4569, 2595, 4571, 4572, + 4573, 4574, 4575, 4576, 4577, 4578, 4579, 4580, + 2595, 4582, 4583, 4584, 4585, 4586, 4587, 4588, + 4589, 4590, 4591, 4592, 4593, 4594, 4595, 4596, + 4597, 4598, 4599, 4600, 4601, 4602, 4603, 4604, + 4605, 4606, 4607, 4608, 4609, 4610, 4611, 4612, + 4613, 4614, 4615, 4089, 4090, 4091, 4092, 4620, + 4093, 4094, 4095, 4096, 4097, 4098, 4099, 4100, + 4101, 4102, 4103, 4104, 2765, 2765, 2765, 2765, + 2765, 2765, 2765, 2765, 2765, 2765, 6179, 2765, + 2765, 2765, 2765, 4647, 2765, 2765, 4650, 4122, + 4652, 4653, 2444, 4655, 2767, 2767, 2767, 2272, + 2767, 2767, 2767, 2767, 2767, 2767, 4666, 2767, + 2767, 2767, 2767, 4671, 2767, 4673, 2767, 4675, + 4676, 4677, 4678, 4679, 4680, 2767, 4682, 4683, + 4684, 4685, 2767, 4687, 2767, 4689, 2767, 4691, + 6657, 2767, 2767, 4695, 2767, 2767, 4698, 2767, + 4700, 4701, 2767, 4703, 2767, -2696, -2696, -2696, + -2696, 4709, 2767, 4711, 2767, 2767, 4714, 2767, + 4716, 4717, 2767, 2767, 2767, 2767, 4722, 2767, + 2767, 2767, 2767, 2767, 2767, 2767, 4730, 2767, + 2767, 2767, 2767, 4735, 2767, 2767, 2767, 2767, + 4740, 2767, 4742, 2767, 2767, 2767, 2767, 2767, + 2767, 2767, 2767, 2767, 2767, 4753, 2767, 2767, + 2767, 2767, 2767, 4193, 4194, 4195, -379, -379, + 4198, 4199, 4200, 4201, 4202, 4203, 4204, 4771, + 4772, 4773, 4774, 4775, 4776, 4777, 4778, 4779, + 4780, -3892, -940, -939, -938, 4785, -3890, -936, + -3889, -3889, 2179, -927, 2180, -930, 2181, -928, + 2182, -926, -925, -912, -923, -922, -921, 4803, + 4804, 4805, 4806, 4807, 4808, 4809, 4810, 4811, + 4812, 4813, 4814, 4815, 4816, 4817, 4818, 4819, + 3925, 4821, 4822, 4823, 4824, 4825, 4826, 4827, + 4828, 4829, 4830, 4831, 4832, 4833, 4834, 4835, + 4836, 4837, 4838, 4839, 4840, 4841, 4842, 4843, + 4844, 4845, 4846, 4847, 4848, 4849, 4850, 4851, + 4852, 4853, 4854, 4855, 4856, 4857, 4858, 4859, + 4860, 4710, 2086, 609, 4864, 4865, 4866, 4867, + 4868, 4869, 4870, 4871, 4872, 3052, 4874, 4875, + 4876, 4281, 4878, 4879, 4880, 4881, 4882, 4883, + 4884, 4885, 4886, 4887, 634, 4888, 4889, 4890, + 4891, 4892, 4893, 4894, 4895, 4896, 1322, 1322, + 1322, 1322, 1322, 1322, 1322, 4904, 338, 4906, + 4907, 4908, 4909, 4910, 4911, 4912, 4913, 4914, + 4915, 4916, 4917, 665, 666, 667, 668, 669, + 670, 671, 672, 673, 674, 675, 676, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 0, + 0, 0, 0, 0, 0, 32767, 0, 0, + 0, 0, 32767, 32767, 0, 0, 0, 0, + 0, 0, 0, 0, 32767, 0, 0, 0, + 0, 0, 0, 0, 32767, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 32767, 0, 0, 0, 2478, 32767, 2477, + 2477, 2477, 2477, 2477, 32767, 2476, 32767, 32767, + 32767, 2473, 2473, 2473, 2473, 2473, 2473, 2473, + 32767, 2472, 2472, 2472, 2472, 2472, 2472, 2472, + 2472, 2472, 2472, 2472, 2472, 2472, 2472, 2472, + 2472, 2472, 2472, 2472, 2472, 2472, 2472, 2472, + 2472, 2472, 2472, 2472, 2472, 2472, 2472, 2472, + 2472, 2472, 2472, 2472, 2472, 2472, 2472, 2472, + 2472, 2472, 2472, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, -2478, -2478, -2478, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + }; + + const unsigned char *k = (const unsigned char *) key; + size_t keylen = 4; + uint32 a = 0; + uint32 b = 1; + + while (keylen--) + { + unsigned char c = *k++; + + a = a * 257 + c; + b = b * 8191 + c; + } + return h[a % 9837] + h[b % 9837]; +} + +/* Hash lookup information for NFKC_QC */ +static const pg_unicode_norminfo UnicodeNormInfo_NFKC_QC = { + UnicodeNormProps_NFKC_QC, + NFKC_QC_hash_func, + 4918 +}; diff --git a/src/tools/pgindent/exclude_file_patterns b/src/tools/pgindent/exclude_file_patterns index bfe103f1955a..86bdd9d6dcbf 100644 --- a/src/tools/pgindent/exclude_file_patterns +++ b/src/tools/pgindent/exclude_file_patterns @@ -18,6 +18,11 @@ src/backend/utils/fmgrprotos\.h$ # they match pgindent style, they'd look worse not better, so exclude them. kwlist_d\.h$ # +# This is generated by the scripts from src/common/unicode/. It uses +# hash functions generated by PerfectHash.pm whose format looks worse with +# pgindent. +src/include/common/unicode_normprops_table\.h$ +# # Exclude ecpg test files to avoid breaking the ecpg regression tests # (but include files at the top level of the ecpg/test/ directory). src/interfaces/ecpg/test/.*/ diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index be570329ea73..c52f20d4ba4d 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -3191,6 +3191,7 @@ pg_tz pg_tz_cache pg_tzenum pg_unicode_decomposition +pg_unicode_norminfo pg_unicode_normprops pg_utf_to_local_combined pg_uuid_t From f5c1167b173d6e7a5d4c938fe716f0d29ae7228d Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Sun, 11 Oct 2020 21:31:37 -0700 Subject: [PATCH 280/589] For ppc gcc, implement 64-bit compare_exchange and fetch_add with asm. While xlc defines __64BIT__, gcc does not. Due to this oversight in commit 30ee5d17c20dbb282a9952b3048d6ad52d56c371, gcc builds continued implementing 64-bit atomics by way of intrinsics. Back-patch to v13, where that commit first appeared. Reviewed by Tom Lane. Discussion: https://postgr.es/m/20201011051043.GA1724101@rfd.leadboat.com --- src/include/port/atomics/arch-ppc.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/include/port/atomics/arch-ppc.h b/src/include/port/atomics/arch-ppc.h index fdfe0d0cd5f4..68e66033ad7a 100644 --- a/src/include/port/atomics/arch-ppc.h +++ b/src/include/port/atomics/arch-ppc.h @@ -32,14 +32,14 @@ typedef struct pg_atomic_uint32 } pg_atomic_uint32; /* 64bit atomics are only supported in 64bit mode */ -#ifdef __64BIT__ +#if SIZEOF_VOID_P >= 8 #define PG_HAVE_ATOMIC_U64_SUPPORT typedef struct pg_atomic_uint64 { volatile uint64 value pg_attribute_aligned(8); } pg_atomic_uint64; -#endif /* __64BIT__ */ +#endif /* * This mimics gcc __atomic_compare_exchange_n(..., __ATOMIC_SEQ_CST), but From 88ea7a1188d1afb25695124045e0ff81870f55e3 Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Sun, 11 Oct 2020 21:31:37 -0700 Subject: [PATCH 281/589] Choose ppc compare_exchange constant path for more operand values. The implementation uses smaller code when the "expected" operand is a small constant, but the implementation needlessly defined the set of acceptable constants more narrowly than the ABI does. Core PostgreSQL and PGXN don't use the constant path at all, so this is future-proofing. Back-patch to v13, where commit 30ee5d17c20dbb282a9952b3048d6ad52d56c371 introduced this code. Reviewed by Tom Lane. Reported by Christoph Berg. Discussion: https://postgr.es/m/20201009092825.GD889580@msg.df7cb.de --- src/include/port/atomics/arch-ppc.h | 14 ++++---------- src/test/regress/regress.c | 8 ++++++++ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/include/port/atomics/arch-ppc.h b/src/include/port/atomics/arch-ppc.h index 68e66033ad7a..a82ae38c1d6e 100644 --- a/src/include/port/atomics/arch-ppc.h +++ b/src/include/port/atomics/arch-ppc.h @@ -72,14 +72,6 @@ typedef struct pg_atomic_uint64 * the __asm__. (That would remove the freedom to eliminate dead stores when * the caller ignores "expected", but few callers do.) * - * The cmpwi variant may be dead code. In gcc 7.2.0, - * __builtin_constant_p(*expected) always reports false. - * __atomic_compare_exchange_n() does use cmpwi when its second argument - * points to a constant. Hence, using this instead of - * __atomic_compare_exchange_n() nominally penalizes the generic.h - * pg_atomic_test_set_flag_impl(). Modern GCC will use the generic-gcc.h - * version, making the penalty theoretical only. - * * Recognizing constant "newval" would be superfluous, because there's no * immediate-operand version of stwcx. */ @@ -94,7 +86,8 @@ pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, #ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P if (__builtin_constant_p(*expected) && - *expected <= PG_INT16_MAX && *expected >= PG_INT16_MIN) + (int32) *expected <= PG_INT16_MAX && + (int32) *expected >= PG_INT16_MIN) __asm__ __volatile__( " sync \n" " lwarx %0,0,%5 \n" @@ -183,7 +176,8 @@ pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, /* Like u32, but s/lwarx/ldarx/; s/stwcx/stdcx/; s/cmpw/cmpd/ */ #ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P if (__builtin_constant_p(*expected) && - *expected <= PG_INT16_MAX && *expected >= PG_INT16_MIN) + (int64) *expected <= PG_INT16_MAX && + (int64) *expected >= PG_INT16_MIN) __asm__ __volatile__( " sync \n" " ldarx %0,0,%5 \n" diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 02397f2eb104..09bc42a8c0f2 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -720,6 +720,14 @@ test_atomic_uint32(void) EXPECT_EQ_U32(pg_atomic_read_u32(&var), (uint32) INT_MAX + 1); EXPECT_EQ_U32(pg_atomic_sub_fetch_u32(&var, INT_MAX), 1); pg_atomic_sub_fetch_u32(&var, 1); + expected = PG_INT16_MAX; + EXPECT_TRUE(!pg_atomic_compare_exchange_u32(&var, &expected, 1)); + expected = PG_INT16_MAX + 1; + EXPECT_TRUE(!pg_atomic_compare_exchange_u32(&var, &expected, 1)); + expected = PG_INT16_MIN; + EXPECT_TRUE(!pg_atomic_compare_exchange_u32(&var, &expected, 1)); + expected = PG_INT16_MIN - 1; + EXPECT_TRUE(!pg_atomic_compare_exchange_u32(&var, &expected, 1)); /* fail exchange because of old expected */ expected = 10; From 3fb676504da9c019540c7384423c7e3d7d394110 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 12 Oct 2020 07:46:20 +0200 Subject: [PATCH 282/589] Adjust cycle detection examples and tests Adjust the existing cycle detection example and test queries to put the cycle column before the path column. This is mainly because the SQL-standard CYCLE clause puts them in that order, and so if we added that feature that would make the sequence of examples more consistent and easier to follow. Discussion: https://www.postgresql.org/message-id/c5603982-0088-7f14-0caa-fdcd0c837b57@2ndquadrant.com --- doc/src/sgml/queries.sgml | 26 +++--- src/test/regress/expected/with.out | 124 ++++++++++++++--------------- src/test/regress/sql/with.sql | 16 ++-- 3 files changed, 83 insertions(+), 83 deletions(-) diff --git a/doc/src/sgml/queries.sgml b/doc/src/sgml/queries.sgml index 77fb1991aebd..bad97a75b275 100644 --- a/doc/src/sgml/queries.sgml +++ b/doc/src/sgml/queries.sgml @@ -2144,20 +2144,20 @@ SELECT * FROM search_graph; UNION ALL to UNION would not eliminate the looping. Instead we need to recognize whether we have reached the same row again while following a particular path of links. We add two columns - path and cycle to the loop-prone query: + is_cycle and path to the loop-prone query: -WITH RECURSIVE search_graph(id, link, data, depth, path, cycle) AS ( +WITH RECURSIVE search_graph(id, link, data, depth, is_cycle, path) AS ( SELECT g.id, g.link, g.data, 1, - ARRAY[g.id], - false + false, + ARRAY[g.id] FROM graph g UNION ALL SELECT g.id, g.link, g.data, sg.depth + 1, - path || g.id, - g.id = ANY(path) + g.id = ANY(path), + path || g.id FROM graph g, search_graph sg - WHERE g.id = sg.link AND NOT cycle + WHERE g.id = sg.link AND NOT is_cycle ) SELECT * FROM search_graph; @@ -2172,17 +2172,17 @@ SELECT * FROM search_graph; compare fields f1 and f2: -WITH RECURSIVE search_graph(id, link, data, depth, path, cycle) AS ( +WITH RECURSIVE search_graph(id, link, data, depth, is_cycle, path) AS ( SELECT g.id, g.link, g.data, 1, - ARRAY[ROW(g.f1, g.f2)], - false + false, + ARRAY[ROW(g.f1, g.f2)] FROM graph g UNION ALL SELECT g.id, g.link, g.data, sg.depth + 1, - path || ROW(g.f1, g.f2), - ROW(g.f1, g.f2) = ANY(path) + ROW(g.f1, g.f2) = ANY(path), + path || ROW(g.f1, g.f2) FROM graph g, search_graph sg - WHERE g.id = sg.link AND NOT cycle + WHERE g.id = sg.link AND NOT is_cycle ) SELECT * FROM search_graph; diff --git a/src/test/regress/expected/with.out b/src/test/regress/expected/with.out index 67eaeb4f3ed4..457f3bf04fa8 100644 --- a/src/test/regress/expected/with.out +++ b/src/test/regress/expected/with.out @@ -579,79 +579,79 @@ insert into graph values (1, 4, 'arc 1 -> 4'), (4, 5, 'arc 4 -> 5'), (5, 1, 'arc 5 -> 1'); -with recursive search_graph(f, t, label, path, cycle) as ( - select *, array[row(g.f, g.t)], false from graph g +with recursive search_graph(f, t, label, is_cycle, path) as ( + select *, false, array[row(g.f, g.t)] from graph g union all - select g.*, path || row(g.f, g.t), row(g.f, g.t) = any(path) + select g.*, row(g.f, g.t) = any(path), path || row(g.f, g.t) from graph g, search_graph sg - where g.f = sg.t and not cycle + where g.f = sg.t and not is_cycle ) select * from search_graph; - f | t | label | path | cycle ----+---+------------+-------------------------------------------+------- - 1 | 2 | arc 1 -> 2 | {"(1,2)"} | f - 1 | 3 | arc 1 -> 3 | {"(1,3)"} | f - 2 | 3 | arc 2 -> 3 | {"(2,3)"} | f - 1 | 4 | arc 1 -> 4 | {"(1,4)"} | f - 4 | 5 | arc 4 -> 5 | {"(4,5)"} | f - 5 | 1 | arc 5 -> 1 | {"(5,1)"} | f - 1 | 2 | arc 1 -> 2 | {"(5,1)","(1,2)"} | f - 1 | 3 | arc 1 -> 3 | {"(5,1)","(1,3)"} | f - 1 | 4 | arc 1 -> 4 | {"(5,1)","(1,4)"} | f - 2 | 3 | arc 2 -> 3 | {"(1,2)","(2,3)"} | f - 4 | 5 | arc 4 -> 5 | {"(1,4)","(4,5)"} | f - 5 | 1 | arc 5 -> 1 | {"(4,5)","(5,1)"} | f - 1 | 2 | arc 1 -> 2 | {"(4,5)","(5,1)","(1,2)"} | f - 1 | 3 | arc 1 -> 3 | {"(4,5)","(5,1)","(1,3)"} | f - 1 | 4 | arc 1 -> 4 | {"(4,5)","(5,1)","(1,4)"} | f - 2 | 3 | arc 2 -> 3 | {"(5,1)","(1,2)","(2,3)"} | f - 4 | 5 | arc 4 -> 5 | {"(5,1)","(1,4)","(4,5)"} | f - 5 | 1 | arc 5 -> 1 | {"(1,4)","(4,5)","(5,1)"} | f - 1 | 2 | arc 1 -> 2 | {"(1,4)","(4,5)","(5,1)","(1,2)"} | f - 1 | 3 | arc 1 -> 3 | {"(1,4)","(4,5)","(5,1)","(1,3)"} | f - 1 | 4 | arc 1 -> 4 | {"(1,4)","(4,5)","(5,1)","(1,4)"} | t - 2 | 3 | arc 2 -> 3 | {"(4,5)","(5,1)","(1,2)","(2,3)"} | f - 4 | 5 | arc 4 -> 5 | {"(4,5)","(5,1)","(1,4)","(4,5)"} | t - 5 | 1 | arc 5 -> 1 | {"(5,1)","(1,4)","(4,5)","(5,1)"} | t - 2 | 3 | arc 2 -> 3 | {"(1,4)","(4,5)","(5,1)","(1,2)","(2,3)"} | f + f | t | label | is_cycle | path +---+---+------------+----------+------------------------------------------- + 1 | 2 | arc 1 -> 2 | f | {"(1,2)"} + 1 | 3 | arc 1 -> 3 | f | {"(1,3)"} + 2 | 3 | arc 2 -> 3 | f | {"(2,3)"} + 1 | 4 | arc 1 -> 4 | f | {"(1,4)"} + 4 | 5 | arc 4 -> 5 | f | {"(4,5)"} + 5 | 1 | arc 5 -> 1 | f | {"(5,1)"} + 1 | 2 | arc 1 -> 2 | f | {"(5,1)","(1,2)"} + 1 | 3 | arc 1 -> 3 | f | {"(5,1)","(1,3)"} + 1 | 4 | arc 1 -> 4 | f | {"(5,1)","(1,4)"} + 2 | 3 | arc 2 -> 3 | f | {"(1,2)","(2,3)"} + 4 | 5 | arc 4 -> 5 | f | {"(1,4)","(4,5)"} + 5 | 1 | arc 5 -> 1 | f | {"(4,5)","(5,1)"} + 1 | 2 | arc 1 -> 2 | f | {"(4,5)","(5,1)","(1,2)"} + 1 | 3 | arc 1 -> 3 | f | {"(4,5)","(5,1)","(1,3)"} + 1 | 4 | arc 1 -> 4 | f | {"(4,5)","(5,1)","(1,4)"} + 2 | 3 | arc 2 -> 3 | f | {"(5,1)","(1,2)","(2,3)"} + 4 | 5 | arc 4 -> 5 | f | {"(5,1)","(1,4)","(4,5)"} + 5 | 1 | arc 5 -> 1 | f | {"(1,4)","(4,5)","(5,1)"} + 1 | 2 | arc 1 -> 2 | f | {"(1,4)","(4,5)","(5,1)","(1,2)"} + 1 | 3 | arc 1 -> 3 | f | {"(1,4)","(4,5)","(5,1)","(1,3)"} + 1 | 4 | arc 1 -> 4 | t | {"(1,4)","(4,5)","(5,1)","(1,4)"} + 2 | 3 | arc 2 -> 3 | f | {"(4,5)","(5,1)","(1,2)","(2,3)"} + 4 | 5 | arc 4 -> 5 | t | {"(4,5)","(5,1)","(1,4)","(4,5)"} + 5 | 1 | arc 5 -> 1 | t | {"(5,1)","(1,4)","(4,5)","(5,1)"} + 2 | 3 | arc 2 -> 3 | f | {"(1,4)","(4,5)","(5,1)","(1,2)","(2,3)"} (25 rows) -- ordering by the path column has same effect as SEARCH DEPTH FIRST -with recursive search_graph(f, t, label, path, cycle) as ( - select *, array[row(g.f, g.t)], false from graph g +with recursive search_graph(f, t, label, is_cycle, path) as ( + select *, false, array[row(g.f, g.t)] from graph g union all - select g.*, path || row(g.f, g.t), row(g.f, g.t) = any(path) + select g.*, row(g.f, g.t) = any(path), path || row(g.f, g.t) from graph g, search_graph sg - where g.f = sg.t and not cycle + where g.f = sg.t and not is_cycle ) select * from search_graph order by path; - f | t | label | path | cycle ----+---+------------+-------------------------------------------+------- - 1 | 2 | arc 1 -> 2 | {"(1,2)"} | f - 2 | 3 | arc 2 -> 3 | {"(1,2)","(2,3)"} | f - 1 | 3 | arc 1 -> 3 | {"(1,3)"} | f - 1 | 4 | arc 1 -> 4 | {"(1,4)"} | f - 4 | 5 | arc 4 -> 5 | {"(1,4)","(4,5)"} | f - 5 | 1 | arc 5 -> 1 | {"(1,4)","(4,5)","(5,1)"} | f - 1 | 2 | arc 1 -> 2 | {"(1,4)","(4,5)","(5,1)","(1,2)"} | f - 2 | 3 | arc 2 -> 3 | {"(1,4)","(4,5)","(5,1)","(1,2)","(2,3)"} | f - 1 | 3 | arc 1 -> 3 | {"(1,4)","(4,5)","(5,1)","(1,3)"} | f - 1 | 4 | arc 1 -> 4 | {"(1,4)","(4,5)","(5,1)","(1,4)"} | t - 2 | 3 | arc 2 -> 3 | {"(2,3)"} | f - 4 | 5 | arc 4 -> 5 | {"(4,5)"} | f - 5 | 1 | arc 5 -> 1 | {"(4,5)","(5,1)"} | f - 1 | 2 | arc 1 -> 2 | {"(4,5)","(5,1)","(1,2)"} | f - 2 | 3 | arc 2 -> 3 | {"(4,5)","(5,1)","(1,2)","(2,3)"} | f - 1 | 3 | arc 1 -> 3 | {"(4,5)","(5,1)","(1,3)"} | f - 1 | 4 | arc 1 -> 4 | {"(4,5)","(5,1)","(1,4)"} | f - 4 | 5 | arc 4 -> 5 | {"(4,5)","(5,1)","(1,4)","(4,5)"} | t - 5 | 1 | arc 5 -> 1 | {"(5,1)"} | f - 1 | 2 | arc 1 -> 2 | {"(5,1)","(1,2)"} | f - 2 | 3 | arc 2 -> 3 | {"(5,1)","(1,2)","(2,3)"} | f - 1 | 3 | arc 1 -> 3 | {"(5,1)","(1,3)"} | f - 1 | 4 | arc 1 -> 4 | {"(5,1)","(1,4)"} | f - 4 | 5 | arc 4 -> 5 | {"(5,1)","(1,4)","(4,5)"} | f - 5 | 1 | arc 5 -> 1 | {"(5,1)","(1,4)","(4,5)","(5,1)"} | t + f | t | label | is_cycle | path +---+---+------------+----------+------------------------------------------- + 1 | 2 | arc 1 -> 2 | f | {"(1,2)"} + 2 | 3 | arc 2 -> 3 | f | {"(1,2)","(2,3)"} + 1 | 3 | arc 1 -> 3 | f | {"(1,3)"} + 1 | 4 | arc 1 -> 4 | f | {"(1,4)"} + 4 | 5 | arc 4 -> 5 | f | {"(1,4)","(4,5)"} + 5 | 1 | arc 5 -> 1 | f | {"(1,4)","(4,5)","(5,1)"} + 1 | 2 | arc 1 -> 2 | f | {"(1,4)","(4,5)","(5,1)","(1,2)"} + 2 | 3 | arc 2 -> 3 | f | {"(1,4)","(4,5)","(5,1)","(1,2)","(2,3)"} + 1 | 3 | arc 1 -> 3 | f | {"(1,4)","(4,5)","(5,1)","(1,3)"} + 1 | 4 | arc 1 -> 4 | t | {"(1,4)","(4,5)","(5,1)","(1,4)"} + 2 | 3 | arc 2 -> 3 | f | {"(2,3)"} + 4 | 5 | arc 4 -> 5 | f | {"(4,5)"} + 5 | 1 | arc 5 -> 1 | f | {"(4,5)","(5,1)"} + 1 | 2 | arc 1 -> 2 | f | {"(4,5)","(5,1)","(1,2)"} + 2 | 3 | arc 2 -> 3 | f | {"(4,5)","(5,1)","(1,2)","(2,3)"} + 1 | 3 | arc 1 -> 3 | f | {"(4,5)","(5,1)","(1,3)"} + 1 | 4 | arc 1 -> 4 | f | {"(4,5)","(5,1)","(1,4)"} + 4 | 5 | arc 4 -> 5 | t | {"(4,5)","(5,1)","(1,4)","(4,5)"} + 5 | 1 | arc 5 -> 1 | f | {"(5,1)"} + 1 | 2 | arc 1 -> 2 | f | {"(5,1)","(1,2)"} + 2 | 3 | arc 2 -> 3 | f | {"(5,1)","(1,2)","(2,3)"} + 1 | 3 | arc 1 -> 3 | f | {"(5,1)","(1,3)"} + 1 | 4 | arc 1 -> 4 | f | {"(5,1)","(1,4)"} + 4 | 5 | arc 4 -> 5 | f | {"(5,1)","(1,4)","(4,5)"} + 5 | 1 | arc 5 -> 1 | t | {"(5,1)","(1,4)","(4,5)","(5,1)"} (25 rows) -- diff --git a/src/test/regress/sql/with.sql b/src/test/regress/sql/with.sql index f85645efdee6..2eea297a7198 100644 --- a/src/test/regress/sql/with.sql +++ b/src/test/regress/sql/with.sql @@ -308,22 +308,22 @@ insert into graph values (4, 5, 'arc 4 -> 5'), (5, 1, 'arc 5 -> 1'); -with recursive search_graph(f, t, label, path, cycle) as ( - select *, array[row(g.f, g.t)], false from graph g +with recursive search_graph(f, t, label, is_cycle, path) as ( + select *, false, array[row(g.f, g.t)] from graph g union all - select g.*, path || row(g.f, g.t), row(g.f, g.t) = any(path) + select g.*, row(g.f, g.t) = any(path), path || row(g.f, g.t) from graph g, search_graph sg - where g.f = sg.t and not cycle + where g.f = sg.t and not is_cycle ) select * from search_graph; -- ordering by the path column has same effect as SEARCH DEPTH FIRST -with recursive search_graph(f, t, label, path, cycle) as ( - select *, array[row(g.f, g.t)], false from graph g +with recursive search_graph(f, t, label, is_cycle, path) as ( + select *, false, array[row(g.f, g.t)] from graph g union all - select g.*, path || row(g.f, g.t), row(g.f, g.t) = any(path) + select g.*, row(g.f, g.t) = any(path), path || row(g.f, g.t) from graph g, search_graph sg - where g.f = sg.t and not cycle + where g.f = sg.t and not is_cycle ) select * from search_graph order by path; From f0f13a3a08b2757997410f3a1c38bdc22973c525 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Mon, 12 Oct 2020 20:41:16 +1300 Subject: [PATCH 283/589] Fix estimates for ModifyTable paths without RETURNING. In the past, we always estimated that a ModifyTable node would emit the same number of rows as its subpaths. Without a RETURNING clause, the correct estimate is zero. Fix, in preparation for a proposed parallel write patch that is sensitive to that number. A remaining problem is that for RETURNING queries, the estimated width is based on subpath output rather than the RETURNING tlist. Reviewed-by: Greg Nancarrow Discussion: https://postgr.es/m/CAJcOf-cXnB5cnMKqWEp2E2z7Mvcd04iLVmV%3DqpFJrR3AcrTS3g%40mail.gmail.com --- src/backend/optimizer/util/pathnode.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index c1fc866cbf91..5281a2f9983f 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -3583,15 +3583,18 @@ create_modifytable_path(PlannerInfo *root, RelOptInfo *rel, if (lc == list_head(subpaths)) /* first node? */ pathnode->path.startup_cost = subpath->startup_cost; pathnode->path.total_cost += subpath->total_cost; - pathnode->path.rows += subpath->rows; - total_size += subpath->pathtarget->width * subpath->rows; + if (returningLists != NIL) + { + pathnode->path.rows += subpath->rows; + total_size += subpath->pathtarget->width * subpath->rows; + } } /* * Set width to the average width of the subpath outputs. XXX this is - * totally wrong: we should report zero if no RETURNING, else an average - * of the RETURNING tlist widths. But it's what happened historically, - * and improving it is a task for another day. + * totally wrong: we should return an average of the RETURNING tlist + * widths. But it's what happened historically, and improving it is a task + * for another day. */ if (pathnode->path.rows > 0) total_size /= pathnode->path.rows; From e578c17d81662b4f4f63a2797bc1be64af3c8f93 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 12 Oct 2020 20:34:55 +0900 Subject: [PATCH 284/589] Fix compilation warning in unicode_norm.c 80f8eb7 has introduced in unicode_norm.c some new code that uses htonl(). On at least some FreeBSD environments, it is possible to find that this function is undeclared, causing a compilation warning. It is worth noting that no buildfarm members have reported this issue. Instead of adding a new inclusion to arpa/inet.h, switch to use the equivalent defined in pg_bswap.h, to benefit from any built-in function if the compiler has one. Reported-by: Masahiko Sawada Discussion: https://postgr.es/m/CA+fd4k7D4b12ShywWj=AbcHZzV1-OqMjNe7RZAu+tgz5rd_11A@mail.gmail.com --- src/common/unicode_norm.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/common/unicode_norm.c b/src/common/unicode_norm.c index 626645ac8705..4bb6a0f58738 100644 --- a/src/common/unicode_norm.c +++ b/src/common/unicode_norm.c @@ -23,6 +23,7 @@ #ifndef FRONTEND #include "common/unicode_normprops_table.h" #endif +#include "port/pg_bswap.h" #ifndef FRONTEND #define ALLOC(size) palloc(size) @@ -475,7 +476,7 @@ qc_hash_lookup(pg_wchar ch, const pg_unicode_norminfo *norminfo) * Compute the hash function. The hash key is the codepoint with the bytes * in network order. */ - hashkey = htonl(ch); + hashkey = pg_hton32(ch); h = norminfo->hash(&hashkey); /* An out-of-range result implies no match */ From fcd11329db5bca9993207f099a642a7d1cd59ff8 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 12 Oct 2020 11:13:02 -0400 Subject: [PATCH 285/589] Minor cleanup for win32stat.c. Use GetLastError(), rather than assuming that CreateFile() failure must map to ENOENT. Noted by Michael Paquier. Discussion: https://postgr.es/m/CAC+AXB0g44SbvSpC86o_1HWh8TAU2pZrMRW6tJT-dkijotx5Qg@mail.gmail.com --- src/port/win32stat.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/port/win32stat.c b/src/port/win32stat.c index d4315c44f4d8..9051c713e7bd 100644 --- a/src/port/win32stat.c +++ b/src/port/win32stat.c @@ -204,8 +204,10 @@ _pgstat64(const char *name, struct stat *buf) NULL); if (hFile == INVALID_HANDLE_VALUE) { + DWORD err = GetLastError(); + CloseHandle(hFile); - errno = ENOENT; + _dosmaperr(err); return -1; } From 397ea901e85b83e6381a0edeba7a45d794063569 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 12 Oct 2020 13:31:24 -0400 Subject: [PATCH 286/589] Fix memory leak when guc.c decides a setting can't be applied now. The prohibitValueChange code paths in set_config_option(), which are executed whenever we re-read a PGC_POSTMASTER variable from postgresql.conf, neglected to free anything before exiting. Thus we'd leak the proposed new value of a PGC_STRING variable, as noted by BoChen in bug #16666. For all variable types, if the check hook creates an "extra" chunk, we'd also leak that. These are malloc not palloc chunks, so there is no mechanism for recovering the leaks before process exit. Fortunately, the values are typically not very large, meaning you'd have to go through an awful lot of SIGHUP configuration-reload cycles to make the leakage amount to anything. Still, for a long-lived postmaster process it could potentially be a problem. Oversight in commit 2594cf0e8. Back-patch to all supported branches. Discussion: https://postgr.es/m/16666-2c41a4eec61b03e1@postgresql.org --- src/backend/utils/misc/guc.c | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 596bcb7b842e..a62d64eaa479 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -7222,6 +7222,10 @@ set_config_option(const char *name, const char *value, if (prohibitValueChange) { + /* Release newextra, unless it's reset_extra */ + if (newextra && !extra_field_used(&conf->gen, newextra)) + free(newextra); + if (*conf->variable != newval) { record->status |= GUC_PENDING_RESTART; @@ -7312,6 +7316,10 @@ set_config_option(const char *name, const char *value, if (prohibitValueChange) { + /* Release newextra, unless it's reset_extra */ + if (newextra && !extra_field_used(&conf->gen, newextra)) + free(newextra); + if (*conf->variable != newval) { record->status |= GUC_PENDING_RESTART; @@ -7402,6 +7410,10 @@ set_config_option(const char *name, const char *value, if (prohibitValueChange) { + /* Release newextra, unless it's reset_extra */ + if (newextra && !extra_field_used(&conf->gen, newextra)) + free(newextra); + if (*conf->variable != newval) { record->status |= GUC_PENDING_RESTART; @@ -7508,9 +7520,21 @@ set_config_option(const char *name, const char *value, if (prohibitValueChange) { + bool newval_different; + /* newval shouldn't be NULL, so we're a bit sloppy here */ - if (*conf->variable == NULL || newval == NULL || - strcmp(*conf->variable, newval) != 0) + newval_different = (*conf->variable == NULL || + newval == NULL || + strcmp(*conf->variable, newval) != 0); + + /* Release newval, unless it's reset_val */ + if (newval && !string_field_used(conf, newval)) + free(newval); + /* Release newextra, unless it's reset_extra */ + if (newextra && !extra_field_used(&conf->gen, newextra)) + free(newextra); + + if (newval_different) { record->status |= GUC_PENDING_RESTART; ereport(elevel, @@ -7605,6 +7629,10 @@ set_config_option(const char *name, const char *value, if (prohibitValueChange) { + /* Release newextra, unless it's reset_extra */ + if (newextra && !extra_field_used(&conf->gen, newextra)) + free(newextra); + if (*conf->variable != newval) { record->status |= GUC_PENDING_RESTART; From 78c0b6ed273a1262f96efe94004bc92d99865005 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 12 Oct 2020 17:09:50 -0400 Subject: [PATCH 287/589] Re-allow testing of GiST buffered builds. Commit 16fa9b2b3 broke the ability to reliably test GiST buffered builds, because it caused sorted builds to be done instead if sortsupport is available, regardless of any attempt to override that. While a would-be test case could try to work around that by choosing an opclass that has no sortsupport function, coverage would be silently lost the moment someone decides it'd be a good idea to add a sortsupport function. Hence, rearrange the logic in gistbuild() so that if "buffering = on" is specified in CREATE INDEX, we will use that method, sortsupport or no. Also document the interaction between sorting and the buffering parameter, as 16fa9b2b3 failed to do. (Note that in fact we still lack any test coverage of buffered builds, but this is a prerequisite to adding a non-fragile test.) Discussion: https://postgr.es/m/3249980.1602532990@sss.pgh.pa.us --- doc/src/sgml/gist.sgml | 58 ++++++++++++++++++---------- doc/src/sgml/ref/create_index.sgml | 12 ++++-- src/backend/access/gist/gistbuild.c | 60 +++++++++++++++-------------- 3 files changed, 77 insertions(+), 53 deletions(-) diff --git a/doc/src/sgml/gist.sgml b/doc/src/sgml/gist.sgml index 192338be8810..1bf5f0965913 100644 --- a/doc/src/sgml/gist.sgml +++ b/doc/src/sgml/gist.sgml @@ -975,7 +975,7 @@ static char *str_param_default = "default"; /* * Sample validator: checks that string is not longer than 8 bytes. */ -static void +static void validate_my_string_relopt(const char *value) { if (strlen(value) > 8) @@ -987,7 +987,7 @@ validate_my_string_relopt(const char *value) /* * Sample filler: switches characters to lower case. */ -static Size +static Size fill_my_string_relopt(const char *value, void *ptr) { char *tmp = str_tolower(value, strlen(value), DEFAULT_COLLATION_OID); @@ -1157,23 +1157,38 @@ my_sortsupport(PG_FUNCTION_ARGS) Implementation - GiST Buffering Build + GiST Index Build Methods + + + The simplest way to build a GiST index is just to insert all the entries, + one by one. This tends to be slow for large indexes, because if the + index tuples are scattered across the index and the index is large enough + to not fit in cache, a lot of random I/O will be + needed. PostgreSQL supports two alternative + methods for initial build of a GiST index: sorted + and buffered modes. + + + + The sorted method is only available if each of the opclasses used by the + index provides a sortsupport function, as described + in . If they do, this method is + usually the best, so it is used by default. + + - Building large GiST indexes by simply inserting all the tuples tends to be - slow, because if the index tuples are scattered across the index and the - index is large enough to not fit in cache, the insertions need to perform - a lot of random I/O. Beginning in version 9.2, PostgreSQL supports a more - efficient method to build GiST indexes based on buffering, which can - dramatically reduce the number of random I/Os needed for non-ordered data - sets. For well-ordered data sets the benefit is smaller or non-existent, - because only a small number of pages receive new tuples at a time, and - those pages fit in cache even if the index as whole does not. + The buffered method works by not inserting tuples directly into the index + right away. It can dramatically reduce the amount of random I/O needed + for non-ordered data sets. For well-ordered data sets the benefit is + smaller or non-existent, because only a small number of pages receive new + tuples at a time, and those pages fit in cache even if the index as a + whole does not. - However, buffering index build needs to call the penalty - function more often, which consumes some extra CPU resources. Also, the - buffers used in the buffering build need temporary disk space, up to + The buffered method needs to call the penalty + function more often than the simple method does, which consumes some + extra CPU resources. Also, the buffers need temporary disk space, up to the size of the resulting index. Buffering can also influence the quality of the resulting index, in both positive and negative directions. That influence depends on various factors, like the distribution of the input @@ -1181,12 +1196,13 @@ my_sortsupport(PG_FUNCTION_ARGS) - By default, a GiST index build switches to the buffering method when the - index size reaches . It can - be manually turned on or off by the buffering parameter - to the CREATE INDEX command. The default behavior is good for most cases, - but turning buffering off might speed up the build somewhat if the input - data is ordered. + If sorting is not possible, then by default a GiST index build switches + to the buffering method when the index size reaches + . Buffering can be manually + forced or prevented by the buffering parameter to the + CREATE INDEX command. The default behavior is good for most cases, but + turning buffering off might speed up the build somewhat if the input data + is ordered. diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 847b8efcf4d8..749db2845e8c 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -463,11 +463,15 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] - Determines whether the buffering build technique described in + Determines whether the buffered build technique described in is used to build the index. With - OFF it is disabled, with ON it is enabled, and - with AUTO it is initially disabled, but turned on - on-the-fly once the index size reaches . The default is AUTO. + OFF buffering is disabled, with ON + it is enabled, and with AUTO it is initially disabled, + but is turned on on-the-fly once the index size reaches + . The default + is AUTO. + Note that if sorted build is possible, it will be used instead of + buffered build unless buffering=ON is specified. diff --git a/src/backend/access/gist/gistbuild.c b/src/backend/access/gist/gistbuild.c index 28bc5855ad9f..9d3fa9c3b75b 100644 --- a/src/backend/access/gist/gistbuild.c +++ b/src/backend/access/gist/gistbuild.c @@ -180,9 +180,7 @@ gistbuild(Relation heap, Relation index, IndexInfo *indexInfo) MemoryContext oldcxt = CurrentMemoryContext; int fillfactor; Oid SortSupportFnOids[INDEX_MAX_KEYS]; - bool hasallsortsupports; - int keyscount = IndexRelationGetNumberOfKeyAttributes(index); - GiSTOptions *options = NULL; + GiSTOptions *options = (GiSTOptions *) index->rd_options; /* * We expect to be called exactly once for any index relation. If that's @@ -192,9 +190,6 @@ gistbuild(Relation heap, Relation index, IndexInfo *indexInfo) elog(ERROR, "index \"%s\" already contains data", RelationGetRelationName(index)); - if (index->rd_options) - options = (GiSTOptions *) index->rd_options; - buildstate.indexrel = index; buildstate.heaprel = heap; buildstate.sortstate = NULL; @@ -208,33 +203,17 @@ gistbuild(Relation heap, Relation index, IndexInfo *indexInfo) buildstate.giststate->tempCxt = createTempGistContext(); /* - * Choose build strategy. If all keys support sorting, do that. Otherwise - * the default strategy is switch to buffering mode when the index grows - * too large to fit in cache. + * Choose build strategy. First check whether the user specified to use + * buffering mode. (The use-case for that in the field is somewhat + * questionable perhaps, but it's important for testing purposes.) */ - hasallsortsupports = true; - for (int i = 0; i < keyscount; i++) - { - SortSupportFnOids[i] = index_getprocid(index, i + 1, - GIST_SORTSUPPORT_PROC); - if (!OidIsValid(SortSupportFnOids[i])) - { - hasallsortsupports = false; - break; - } - } - - if (hasallsortsupports) - { - buildstate.buildMode = GIST_SORTED_BUILD; - } - else if (options) + if (options) { if (options->buffering_mode == GIST_OPTION_BUFFERING_ON) buildstate.buildMode = GIST_BUFFERING_STATS; else if (options->buffering_mode == GIST_OPTION_BUFFERING_OFF) buildstate.buildMode = GIST_BUFFERING_DISABLED; - else + else /* must be "auto" */ buildstate.buildMode = GIST_BUFFERING_AUTO; } else @@ -242,6 +221,28 @@ gistbuild(Relation heap, Relation index, IndexInfo *indexInfo) buildstate.buildMode = GIST_BUFFERING_AUTO; } + /* + * Unless buffering mode was forced, see if we can use sorting instead. + */ + if (buildstate.buildMode != GIST_BUFFERING_STATS) + { + bool hasallsortsupports = true; + int keyscount = IndexRelationGetNumberOfKeyAttributes(index); + + for (int i = 0; i < keyscount; i++) + { + SortSupportFnOids[i] = index_getprocid(index, i + 1, + GIST_SORTSUPPORT_PROC); + if (!OidIsValid(SortSupportFnOids[i])) + { + hasallsortsupports = false; + break; + } + } + if (hasallsortsupports) + buildstate.buildMode = GIST_SORTED_BUILD; + } + /* * Calculate target amount of free space to leave on pages. */ @@ -852,7 +853,10 @@ gistBuildCallback(Relation index, * and switch to buffering mode if it has. * * To avoid excessive calls to smgrnblocks(), only check this every - * BUFFERING_MODE_SWITCH_CHECK_STEP index tuples + * BUFFERING_MODE_SWITCH_CHECK_STEP index tuples. + * + * In 'stats' state, switch as soon as we have seen enough tuples to have + * some idea of the average tuple size. */ if ((buildstate->buildMode == GIST_BUFFERING_AUTO && buildstate->indtuples % BUFFERING_MODE_SWITCH_CHECK_STEP == 0 && From 371668a8389d792279bd7aff8ca796e284f595df Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 12 Oct 2020 18:01:34 -0400 Subject: [PATCH 288/589] Fix GiST buffering build to work when there are included columns. gistRelocateBuildBuffersOnSplit did not get the memo about which attribute count to use. This could lead to a crash if there were included columns and buffering build was chosen. (Because there are random page-split decisions elsewhere in GiST index build, the crashes are not entirely deterministic.) Back-patch to v12 where GiST gained support for included columns. Pavel Borisov Discussion: https://postgr.es/m/CALT9ZEECCV5m7wvxg46PC-7x-EybUmnpupBGhSFMoAAay+r6HQ@mail.gmail.com --- src/backend/access/gist/gistbuildbuffers.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/access/gist/gistbuildbuffers.c b/src/backend/access/gist/gistbuildbuffers.c index 4eab9bb83ac0..4ad67c88b4e5 100644 --- a/src/backend/access/gist/gistbuildbuffers.c +++ b/src/backend/access/gist/gistbuildbuffers.c @@ -666,7 +666,7 @@ gistRelocateBuildBuffersOnSplit(GISTBuildBuffers *gfbb, GISTSTATE *giststate, zero_penalty = true; /* Loop over index attributes. */ - for (j = 0; j < r->rd_att->natts; j++) + for (j = 0; j < IndexRelationGetNumberOfKeyAttributes(r); j++) { float usize; @@ -692,7 +692,7 @@ gistRelocateBuildBuffersOnSplit(GISTBuildBuffers *gfbb, GISTSTATE *giststate, which = i; best_penalty[j] = usize; - if (j < r->rd_att->natts - 1) + if (j < IndexRelationGetNumberOfKeyAttributes(r) - 1) best_penalty[j + 1] = -1; } else if (best_penalty[j] == usize) From 8fccf75834c11798e3b13a4cebedfb8ae683857a Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Tue, 13 Oct 2020 08:30:35 +0530 Subject: [PATCH 289/589] Add tests for logical replication spilled stats. Commit 9868167500 added a mechanism to track statistics corresponding to the spilling of changes from ReorderBuffer but didn't add any tests. Author: Amit Kapila and Sawada Masahiko Discussion: https://postgr.es/m/CA+fd4k5_pPAYRTDrO2PbtTOe0eHQpBvuqmCr8ic39uTNmR49Eg@mail.gmail.com --- contrib/test_decoding/Makefile | 2 +- contrib/test_decoding/expected/stats.out | 109 +++++++++++++++++++++++ contrib/test_decoding/sql/stats.sql | 62 +++++++++++++ 3 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 contrib/test_decoding/expected/stats.out create mode 100644 contrib/test_decoding/sql/stats.sql diff --git a/contrib/test_decoding/Makefile b/contrib/test_decoding/Makefile index f23f15b04d4a..9a4c76f01364 100644 --- a/contrib/test_decoding/Makefile +++ b/contrib/test_decoding/Makefile @@ -5,7 +5,7 @@ PGFILEDESC = "test_decoding - example of a logical decoding output plugin" REGRESS = ddl xact rewrite toast permissions decoding_in_xact \ decoding_into_rel binary prepared replorigin time messages \ - spill slot truncate stream + spill slot truncate stream stats ISOLATION = mxact delayed_startup ondisk_startup concurrent_ddl_dml \ oldest_xmin snapshot_transfer subxact_without_top concurrent_stream diff --git a/contrib/test_decoding/expected/stats.out b/contrib/test_decoding/expected/stats.out new file mode 100644 index 000000000000..c4464f49dada --- /dev/null +++ b/contrib/test_decoding/expected/stats.out @@ -0,0 +1,109 @@ +-- predictability +SET synchronous_commit = on; +SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_decoding'); + ?column? +---------- + init +(1 row) + +CREATE TABLE stats_test(data text); +-- function to wait for counters to advance +CREATE FUNCTION wait_for_decode_stats(check_reset bool) RETURNS void AS $$ +DECLARE + start_time timestamptz := clock_timestamp(); + updated bool; +BEGIN + -- we don't want to wait forever; loop will exit after 30 seconds + FOR i IN 1 .. 300 LOOP + + -- check to see if all updates have been reset/updated + SELECT CASE WHEN check_reset THEN (spill_txns = 0) + ELSE (spill_txns > 0) + END + INTO updated + FROM pg_stat_replication_slots WHERE name='regression_slot'; + + exit WHEN updated; + + -- wait a little + perform pg_sleep_for('100 milliseconds'); + + -- reset stats snapshot so we can test again + perform pg_stat_clear_snapshot(); + + END LOOP; + + -- report time waited in postmaster log (where it won't change test output) + RAISE LOG 'wait_for_decode_stats delayed % seconds', + extract(epoch from clock_timestamp() - start_time); +END +$$ LANGUAGE plpgsql; +-- spilling the xact +BEGIN; +INSERT INTO stats_test SELECT 'serialize-topbig--1:'||g.i FROM generate_series(1, 5000) g(i); +COMMIT; +SELECT count(*) FROM pg_logical_slot_peek_changes('regression_slot', NULL,NULL); + count +------- + 5006 +(1 row) + +-- check stats, wait for stats collector to update. +SELECT wait_for_decode_stats(false); + wait_for_decode_stats +----------------------- + +(1 row) + +SELECT name, spill_txns, spill_count FROM pg_stat_replication_slots; + name | spill_txns | spill_count +-----------------+------------+------------- + regression_slot | 1 | 12 +(1 row) + +-- reset the slot stats, and wait for stats collector to reset +SELECT pg_stat_reset_replication_slot('regression_slot'); + pg_stat_reset_replication_slot +-------------------------------- + +(1 row) + +SELECT wait_for_decode_stats(true); + wait_for_decode_stats +----------------------- + +(1 row) + +SELECT name, spill_txns, spill_count FROM pg_stat_replication_slots; + name | spill_txns | spill_count +-----------------+------------+------------- + regression_slot | 0 | 0 +(1 row) + +-- decode and check stats again. +SELECT count(*) FROM pg_logical_slot_peek_changes('regression_slot', NULL,NULL); + count +------- + 5006 +(1 row) + +SELECT wait_for_decode_stats(false); + wait_for_decode_stats +----------------------- + +(1 row) + +SELECT name, spill_txns, spill_count FROM pg_stat_replication_slots; + name | spill_txns | spill_count +-----------------+------------+------------- + regression_slot | 1 | 12 +(1 row) + +DROP FUNCTION wait_for_decode_stats(bool); +DROP TABLE stats_test; +SELECT pg_drop_replication_slot('regression_slot'); + pg_drop_replication_slot +-------------------------- + +(1 row) + diff --git a/contrib/test_decoding/sql/stats.sql b/contrib/test_decoding/sql/stats.sql new file mode 100644 index 000000000000..7d406f0b1139 --- /dev/null +++ b/contrib/test_decoding/sql/stats.sql @@ -0,0 +1,62 @@ +-- predictability +SET synchronous_commit = on; + +SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_decoding'); + +CREATE TABLE stats_test(data text); + +-- function to wait for counters to advance +CREATE FUNCTION wait_for_decode_stats(check_reset bool) RETURNS void AS $$ +DECLARE + start_time timestamptz := clock_timestamp(); + updated bool; +BEGIN + -- we don't want to wait forever; loop will exit after 30 seconds + FOR i IN 1 .. 300 LOOP + + -- check to see if all updates have been reset/updated + SELECT CASE WHEN check_reset THEN (spill_txns = 0) + ELSE (spill_txns > 0) + END + INTO updated + FROM pg_stat_replication_slots WHERE name='regression_slot'; + + exit WHEN updated; + + -- wait a little + perform pg_sleep_for('100 milliseconds'); + + -- reset stats snapshot so we can test again + perform pg_stat_clear_snapshot(); + + END LOOP; + + -- report time waited in postmaster log (where it won't change test output) + RAISE LOG 'wait_for_decode_stats delayed % seconds', + extract(epoch from clock_timestamp() - start_time); +END +$$ LANGUAGE plpgsql; + +-- spilling the xact +BEGIN; +INSERT INTO stats_test SELECT 'serialize-topbig--1:'||g.i FROM generate_series(1, 5000) g(i); +COMMIT; +SELECT count(*) FROM pg_logical_slot_peek_changes('regression_slot', NULL,NULL); + +-- check stats, wait for stats collector to update. +SELECT wait_for_decode_stats(false); +SELECT name, spill_txns, spill_count FROM pg_stat_replication_slots; + +-- reset the slot stats, and wait for stats collector to reset +SELECT pg_stat_reset_replication_slot('regression_slot'); +SELECT wait_for_decode_stats(true); +SELECT name, spill_txns, spill_count FROM pg_stat_replication_slots; + +-- decode and check stats again. +SELECT count(*) FROM pg_logical_slot_peek_changes('regression_slot', NULL,NULL); +SELECT wait_for_decode_stats(false); +SELECT name, spill_txns, spill_count FROM pg_stat_replication_slots; + +DROP FUNCTION wait_for_decode_stats(bool); +DROP TABLE stats_test; +SELECT pg_drop_replication_slot('regression_slot'); From 323ae003e464cac35f86790f3708b383c18df9e2 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 13 Oct 2020 06:29:06 +0200 Subject: [PATCH 290/589] doc: Expand recursive query documentation Break the section up with subsection headings. Add examples for depth- and breadth-first search ordering. For consistency with the SQL search clause, start the depth counting at 0 instead of 1 in the examples. Discussion: https://www.postgresql.org/message-id/c5603982-0088-7f14-0caa-fdcd0c837b57@2ndquadrant.com --- doc/src/sgml/queries.sgml | 163 +++++++++++++++++++++++++++++++++----- 1 file changed, 142 insertions(+), 21 deletions(-) diff --git a/doc/src/sgml/queries.sgml b/doc/src/sgml/queries.sgml index bad97a75b275..f06afe2c3fb1 100644 --- a/doc/src/sgml/queries.sgml +++ b/doc/src/sgml/queries.sgml @@ -2011,6 +2011,10 @@ GROUP BY region, product; but we'd have needed two levels of nested sub-SELECTs. It's a bit easier to follow this way. + + + + Recursive Queries @@ -2114,6 +2118,120 @@ GROUP BY sub_part + + Search Order + + + When computing a tree traversal using a recursive query, you might want to + order the results in either depth-first or breadth-first order. This can + be done by computing an ordering column alongside the other data columns + and using that to sort the results at the end. Note that this does not + actually control in which order the query evaluation visits the rows; that + is as always in SQL implementation-dependent. This approach merely + provides a convenient way to order the results afterwards. + + + + To create a depth-first order, we compute for each result row an array of + rows that we have visited so far. For example, consider the following + query that searches a table tree using a + link field: + + +WITH RECURSIVE search_tree(id, link, data) AS ( + SELECT t.id, t.link, t.data + FROM tree t + UNION ALL + SELECT t.id, t.link, t.data + FROM tree t, search_tree st + WHERE t.id = st.link +) +SELECT * FROM search_tree; + + + To add depth-first ordering information, you can write this: + + +WITH RECURSIVE search_tree(id, link, data, path) AS ( + SELECT t.id, t.link, t.data, ARRAY[t.id] + FROM tree t + UNION ALL + SELECT t.id, t.link, t.data, path || t.id + FROM tree t, search_tree st + WHERE t.id = st.link +) +SELECT * FROM search_tree ORDER BY path; + + + + + In the general case where more than one field needs to be used to identify + a row, use an array of rows. For example, if we needed to track fields + f1 and f2: + + +WITH RECURSIVE search_tree(id, link, data, path) AS ( + SELECT t.id, t.link, t.data, ARRAY[ROW(t.f1, t.f2)] + FROM tree t + UNION ALL + SELECT t.id, t.link, t.data, path || ROW(t.f1, t.f2) + FROM tree t, search_tree st + WHERE t.id = st.link +) +SELECT * FROM search_tree ORDER BY path; + + + + + + The queries shown in this and the following section involving + ROW constructors in the target list only support + UNION ALL (not plain UNION) in the + current implementation. + + + + + + Omit the ROW() syntax in the common case where only one + field needs to be tracked. This allows a simple array rather than a + composite-type array to be used, gaining efficiency. + + + + + To create a breadth-first order, you can add a column that tracks the depth + of the search, for example: + + +WITH RECURSIVE search_tree(id, link, data, depth) AS ( + SELECT t.id, t.link, t.data, 0 + FROM tree t + UNION ALL + SELECT t.id, t.link, t.data, depth + 1 + FROM tree t, search_tree st + WHERE t.id = st.link +) +SELECT * FROM search_tree ORDER BY depth; + + + To get a stable sort, add data columns as secondary sorting columns. + + + + + The recursive query evaluation algorithm produces its output in + breadth-first search order. However, this is an implementation detail and + it is perhaps unsound to rely on it. The order of the rows within each + level is certainly undefined, so some explicit ordering might be desired + in any case. + + + + + + Cycle Detection + When working with recursive queries it is important to be sure that the recursive part of the query will eventually return no tuples, @@ -2123,13 +2241,13 @@ GROUP BY sub_part cycle does not involve output rows that are completely duplicate: it may be necessary to check just one or a few fields to see if the same point has been reached before. The standard method for handling such situations is - to compute an array of the already-visited values. For example, consider + to compute an array of the already-visited values. For example, consider again the following query that searches a table graph using a link field: WITH RECURSIVE search_graph(id, link, data, depth) AS ( - SELECT g.id, g.link, g.data, 1 + SELECT g.id, g.link, g.data, 0 FROM graph g UNION ALL SELECT g.id, g.link, g.data, sg.depth + 1 @@ -2147,17 +2265,17 @@ SELECT * FROM search_graph; is_cycle and path to the loop-prone query: -WITH RECURSIVE search_graph(id, link, data, depth, is_cycle, path) AS ( - SELECT g.id, g.link, g.data, 1, - false, - ARRAY[g.id] +WITH RECURSIVE search_graph(id, link, data, depth, is_cycle, path) AS ( + SELECT g.id, g.link, g.data, 0, + false, + ARRAY[g.id] FROM graph g UNION ALL SELECT g.id, g.link, g.data, sg.depth + 1, - g.id = ANY(path), - path || g.id + g.id = ANY(path), + path || g.id FROM graph g, search_graph sg - WHERE g.id = sg.link AND NOT is_cycle + WHERE g.id = sg.link AND NOT is_cycle ) SELECT * FROM search_graph; @@ -2172,17 +2290,17 @@ SELECT * FROM search_graph; compare fields f1 and f2: -WITH RECURSIVE search_graph(id, link, data, depth, is_cycle, path) AS ( - SELECT g.id, g.link, g.data, 1, - false, - ARRAY[ROW(g.f1, g.f2)] +WITH RECURSIVE search_graph(id, link, data, depth, is_cycle, path) AS ( + SELECT g.id, g.link, g.data, 0, + false, + ARRAY[ROW(g.f1, g.f2)] FROM graph g UNION ALL SELECT g.id, g.link, g.data, sg.depth + 1, - ROW(g.f1, g.f2) = ANY(path), - path || ROW(g.f1, g.f2) + ROW(g.f1, g.f2) = ANY(path), + path || ROW(g.f1, g.f2) FROM graph g, search_graph sg - WHERE g.id = sg.link AND NOT is_cycle + WHERE g.id = sg.link AND NOT is_cycle ) SELECT * FROM search_graph; @@ -2198,10 +2316,8 @@ SELECT * FROM search_graph; - The recursive query evaluation algorithm produces its output in - breadth-first search order. You can display the results in depth-first - search order by making the outer query ORDER BY a - path column constructed in this way. + The cycle path column is computed in the same way as the depth-first + ordering column show in the previous section. @@ -2217,7 +2333,7 @@ WITH RECURSIVE t(n) AS ( UNION ALL SELECT n+1 FROM t ) -SELECT n FROM t LIMIT 100; +SELECT n FROM t LIMIT 100; This works because PostgreSQL's implementation @@ -2229,6 +2345,11 @@ SELECT n FROM t LIMIT 100; outer query will usually try to fetch all of the WITH query's output anyway. + + + + + Common Table Expression Materialization A useful property of WITH queries is that they are From 2050832d0d637358a376a99514071941aa93ed31 Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Tue, 13 Oct 2020 12:46:38 +0530 Subject: [PATCH 291/589] Fix the unstable output of tests added by commit 8fccf75834. The test cases added by that commit were trying to test the exact number of times a particular transaction has spilled. However, that number can vary if any background transaction (say by autovacuum) happens in parallel to the main transaction. So let's not try to verify the exact count. Author: Amit Kapila Reviewed-by: Sawada Masahiko Discussion: https://postgr.es/m/CA+fd4k5_pPAYRTDrO2PbtTOe0eHQpBvuqmCr8ic39uTNmR49Eg@mail.gmail.com --- contrib/test_decoding/expected/stats.out | 20 +++++++++++--------- contrib/test_decoding/sql/stats.sql | 12 +++++++----- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/contrib/test_decoding/expected/stats.out b/contrib/test_decoding/expected/stats.out index c4464f49dada..bfffd1ac21df 100644 --- a/contrib/test_decoding/expected/stats.out +++ b/contrib/test_decoding/expected/stats.out @@ -42,23 +42,25 @@ $$ LANGUAGE plpgsql; BEGIN; INSERT INTO stats_test SELECT 'serialize-topbig--1:'||g.i FROM generate_series(1, 5000) g(i); COMMIT; -SELECT count(*) FROM pg_logical_slot_peek_changes('regression_slot', NULL,NULL); +SELECT count(*) FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'skip-empty-xacts', '1'); count ------- - 5006 + 5002 (1 row) --- check stats, wait for stats collector to update. +-- Check stats, wait for the stats collector to update. We can't test the +-- exact stats count as that can vary if any background transaction (say by +-- autovacuum) happens in parallel to the main transaction. SELECT wait_for_decode_stats(false); wait_for_decode_stats ----------------------- (1 row) -SELECT name, spill_txns, spill_count FROM pg_stat_replication_slots; +SELECT name, spill_txns > 0 AS spill_txns, spill_count > 0 AS spill_count FROM pg_stat_replication_slots; name | spill_txns | spill_count -----------------+------------+------------- - regression_slot | 1 | 12 + regression_slot | t | t (1 row) -- reset the slot stats, and wait for stats collector to reset @@ -81,10 +83,10 @@ SELECT name, spill_txns, spill_count FROM pg_stat_replication_slots; (1 row) -- decode and check stats again. -SELECT count(*) FROM pg_logical_slot_peek_changes('regression_slot', NULL,NULL); +SELECT count(*) FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'skip-empty-xacts', '1'); count ------- - 5006 + 5002 (1 row) SELECT wait_for_decode_stats(false); @@ -93,10 +95,10 @@ SELECT wait_for_decode_stats(false); (1 row) -SELECT name, spill_txns, spill_count FROM pg_stat_replication_slots; +SELECT name, spill_txns > 0 AS spill_txns, spill_count > 0 AS spill_count FROM pg_stat_replication_slots; name | spill_txns | spill_count -----------------+------------+------------- - regression_slot | 1 | 12 + regression_slot | t | t (1 row) DROP FUNCTION wait_for_decode_stats(bool); diff --git a/contrib/test_decoding/sql/stats.sql b/contrib/test_decoding/sql/stats.sql index 7d406f0b1139..b95adb16fa71 100644 --- a/contrib/test_decoding/sql/stats.sql +++ b/contrib/test_decoding/sql/stats.sql @@ -41,11 +41,13 @@ $$ LANGUAGE plpgsql; BEGIN; INSERT INTO stats_test SELECT 'serialize-topbig--1:'||g.i FROM generate_series(1, 5000) g(i); COMMIT; -SELECT count(*) FROM pg_logical_slot_peek_changes('regression_slot', NULL,NULL); +SELECT count(*) FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'skip-empty-xacts', '1'); --- check stats, wait for stats collector to update. +-- Check stats, wait for the stats collector to update. We can't test the +-- exact stats count as that can vary if any background transaction (say by +-- autovacuum) happens in parallel to the main transaction. SELECT wait_for_decode_stats(false); -SELECT name, spill_txns, spill_count FROM pg_stat_replication_slots; +SELECT name, spill_txns > 0 AS spill_txns, spill_count > 0 AS spill_count FROM pg_stat_replication_slots; -- reset the slot stats, and wait for stats collector to reset SELECT pg_stat_reset_replication_slot('regression_slot'); @@ -53,9 +55,9 @@ SELECT wait_for_decode_stats(true); SELECT name, spill_txns, spill_count FROM pg_stat_replication_slots; -- decode and check stats again. -SELECT count(*) FROM pg_logical_slot_peek_changes('regression_slot', NULL,NULL); +SELECT count(*) FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'skip-empty-xacts', '1'); SELECT wait_for_decode_stats(false); -SELECT name, spill_txns, spill_count FROM pg_stat_replication_slots; +SELECT name, spill_txns > 0 AS spill_txns, spill_count > 0 AS spill_count FROM pg_stat_replication_slots; DROP FUNCTION wait_for_decode_stats(bool); DROP TABLE stats_test; From 1375422c7826a2bf387be29895e961614f69de4b Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 13 Oct 2020 12:57:02 +0300 Subject: [PATCH 292/589] Create ResultRelInfos later in InitPlan, index them by RT index. Instead of allocating all the ResultRelInfos upfront in one big array, allocate them in ExecInitModifyTable(). es_result_relations is now an array of ResultRelInfo pointers, rather than an array of structs, and it is indexed by the RT index. This simplifies things: we get rid of the separate concept of a "result rel index", and don't need to set it in setrefs.c anymore. This also allows follow-up optimizations (not included in this commit yet) to skip initializing ResultRelInfos for target relations that were not needed at runtime, and removal of the es_result_relation_info pointer. The EState arrays of regular result rels and root result rels are merged into one array. Similarly, the resultRelations and rootResultRelations lists in PlannedStmt are merged into one. It's not actually clear to me why they were kept separate in the first place, but now that the es_result_relations array is indexed by RT index, it certainly seems pointless. The PlannedStmt->resultRelations list is now only needed for ExecRelationIsTargetRelation(). One visible effect of this change is that ExecRelationIsTargetRelation() will now return 'true' also for the partition root, if a partitioned table is updated. That seems like a good thing, although the function isn't used in core code, and I don't see any reason for an FDW to call it on a partition root. Author: Amit Langote Discussion: https://www.postgresql.org/message-id/CA%2BHiwqGEmiib8FLiHMhKB%2BCH5dRgHSLc5N5wnvc4kym%2BZYpQEQ%40mail.gmail.com --- src/backend/commands/copy.c | 24 +-- src/backend/commands/explain.c | 17 +- src/backend/commands/tablecmds.c | 9 +- src/backend/commands/trigger.c | 2 +- src/backend/executor/execMain.c | 256 +++++++---------------- src/backend/executor/execParallel.c | 1 - src/backend/executor/execUtils.c | 57 +++-- src/backend/executor/nodeModifyTable.c | 24 ++- src/backend/nodes/copyfuncs.c | 3 - src/backend/nodes/outfuncs.c | 4 - src/backend/nodes/readfuncs.c | 3 - src/backend/optimizer/plan/createplan.c | 2 - src/backend/optimizer/plan/planner.c | 3 - src/backend/optimizer/plan/setrefs.c | 17 +- src/backend/replication/logical/worker.c | 32 +-- src/include/executor/executor.h | 5 +- src/include/nodes/execnodes.h | 18 +- src/include/nodes/pathnodes.h | 2 - src/include/nodes/plannodes.h | 8 - 19 files changed, 184 insertions(+), 303 deletions(-) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 3c7dbad27a21..71d48d45743b 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -2727,6 +2727,7 @@ CopyFrom(CopyState cstate) bool leafpart_use_multi_insert = false; Assert(cstate->rel); + Assert(list_length(cstate->range_table) == 1); /* * The target must be a plain, foreign, or partitioned relation, or have @@ -2829,25 +2830,17 @@ CopyFrom(CopyState cstate) * index-entry-making machinery. (There used to be a huge amount of code * here that basically duplicated execUtils.c ...) */ - resultRelInfo = makeNode(ResultRelInfo); - InitResultRelInfo(resultRelInfo, - cstate->rel, - 1, /* must match rel's position in range_table */ - NULL, - 0); - target_resultRelInfo = resultRelInfo; + ExecInitRangeTable(estate, cstate->range_table); + resultRelInfo = target_resultRelInfo = makeNode(ResultRelInfo); + ExecInitResultRelation(estate, resultRelInfo, 1); /* Verify the named relation is a valid target for INSERT */ CheckValidResultRel(resultRelInfo, CMD_INSERT); ExecOpenIndices(resultRelInfo, false); - estate->es_result_relations = resultRelInfo; - estate->es_num_result_relations = 1; estate->es_result_relation_info = resultRelInfo; - ExecInitRangeTable(estate, cstate->range_table); - /* * Set up a ModifyTableState so we can let FDW(s) init themselves for * foreign-table result relation(s). @@ -2856,7 +2849,7 @@ CopyFrom(CopyState cstate) mtstate->ps.plan = NULL; mtstate->ps.state = estate; mtstate->operation = CMD_INSERT; - mtstate->resultRelInfo = estate->es_result_relations; + mtstate->resultRelInfo = resultRelInfo; if (resultRelInfo->ri_FdwRoutine != NULL && resultRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL) @@ -3359,14 +3352,13 @@ CopyFrom(CopyState cstate) if (insertMethod != CIM_SINGLE) CopyMultiInsertInfoCleanup(&multiInsertInfo); - ExecCloseIndices(target_resultRelInfo); - /* Close all the partitioned tables, leaf partitions, and their indices */ if (proute) ExecCleanupTupleRouting(mtstate, proute); - /* Close any trigger target relations */ - ExecCleanUpTriggerState(estate); + /* Close the result relations, including any trigger target relations */ + ExecCloseResultRelations(estate); + ExecCloseRangeTableRelations(estate); FreeExecutorState(estate); diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index c98c9b5547c5..c8e292adfa63 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -769,27 +769,24 @@ ExplainPrintTriggers(ExplainState *es, QueryDesc *queryDesc) { ResultRelInfo *rInfo; bool show_relname; - int numrels = queryDesc->estate->es_num_result_relations; - int numrootrels = queryDesc->estate->es_num_root_result_relations; + List *resultrels; List *routerels; List *targrels; - int nr; ListCell *l; + resultrels = queryDesc->estate->es_opened_result_relations; routerels = queryDesc->estate->es_tuple_routing_result_relations; targrels = queryDesc->estate->es_trig_target_relations; ExplainOpenGroup("Triggers", "Triggers", false, es); - show_relname = (numrels > 1 || numrootrels > 0 || + show_relname = (list_length(resultrels) > 1 || routerels != NIL || targrels != NIL); - rInfo = queryDesc->estate->es_result_relations; - for (nr = 0; nr < numrels; rInfo++, nr++) - report_triggers(rInfo, show_relname, es); - - rInfo = queryDesc->estate->es_root_result_relations; - for (nr = 0; nr < numrootrels; rInfo++, nr++) + foreach(l, resultrels) + { + rInfo = (ResultRelInfo *) lfirst(l); report_triggers(rInfo, show_relname, es); + } foreach(l, routerels) { diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index e0ac4e05e5f5..80fedad5e045 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -1787,6 +1787,11 @@ ExecuteTruncateGuts(List *explicit_rels, List *relids, List *relids_logged, /* * To fire triggers, we'll need an EState as well as a ResultRelInfo for * each relation. We don't need to call ExecOpenIndices, though. + * + * We put the ResultRelInfos in the es_opened_result_relations list, even + * though we don't have a range table and don't populate the + * es_result_relations array. That's a big bogus, but it's enough to make + * ExecGetTriggerResultRel() find them. */ estate = CreateExecutorState(); resultRelInfos = (ResultRelInfo *) @@ -1801,10 +1806,10 @@ ExecuteTruncateGuts(List *explicit_rels, List *relids, List *relids_logged, 0, /* dummy rangetable index */ NULL, 0); + estate->es_opened_result_relations = + lappend(estate->es_opened_result_relations, resultRelInfo); resultRelInfo++; } - estate->es_result_relations = resultRelInfos; - estate->es_num_result_relations = list_length(rels); /* * Process all BEFORE STATEMENT TRUNCATE triggers before we begin diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c index 672fccff5bd1..3b4fbdadf4f3 100644 --- a/src/backend/commands/trigger.c +++ b/src/backend/commands/trigger.c @@ -4227,7 +4227,7 @@ afterTriggerInvokeEvents(AfterTriggerEventList *events, if (local_estate) { - ExecCleanUpTriggerState(estate); + ExecCloseResultRelations(estate); ExecResetTupleTable(estate->es_tupleTable, false); FreeExecutorState(estate); } diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index 2e27e26ba446..783eecbc1337 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -827,86 +827,8 @@ InitPlan(QueryDesc *queryDesc, int eflags) estate->es_plannedstmt = plannedstmt; - /* - * Initialize ResultRelInfo data structures, and open the result rels. - */ - if (plannedstmt->resultRelations) - { - List *resultRelations = plannedstmt->resultRelations; - int numResultRelations = list_length(resultRelations); - ResultRelInfo *resultRelInfos; - ResultRelInfo *resultRelInfo; - - resultRelInfos = (ResultRelInfo *) - palloc(numResultRelations * sizeof(ResultRelInfo)); - resultRelInfo = resultRelInfos; - foreach(l, resultRelations) - { - Index resultRelationIndex = lfirst_int(l); - Relation resultRelation; - - resultRelation = ExecGetRangeTableRelation(estate, - resultRelationIndex); - InitResultRelInfo(resultRelInfo, - resultRelation, - resultRelationIndex, - NULL, - estate->es_instrument); - resultRelInfo++; - } - estate->es_result_relations = resultRelInfos; - estate->es_num_result_relations = numResultRelations; - - /* es_result_relation_info is NULL except when within ModifyTable */ - estate->es_result_relation_info = NULL; - - /* - * In the partitioned result relation case, also build ResultRelInfos - * for all the partitioned table roots, because we will need them to - * fire statement-level triggers, if any. - */ - if (plannedstmt->rootResultRelations) - { - int num_roots = list_length(plannedstmt->rootResultRelations); - - resultRelInfos = (ResultRelInfo *) - palloc(num_roots * sizeof(ResultRelInfo)); - resultRelInfo = resultRelInfos; - foreach(l, plannedstmt->rootResultRelations) - { - Index resultRelIndex = lfirst_int(l); - Relation resultRelDesc; - - resultRelDesc = ExecGetRangeTableRelation(estate, - resultRelIndex); - InitResultRelInfo(resultRelInfo, - resultRelDesc, - resultRelIndex, - NULL, - estate->es_instrument); - resultRelInfo++; - } - - estate->es_root_result_relations = resultRelInfos; - estate->es_num_root_result_relations = num_roots; - } - else - { - estate->es_root_result_relations = NULL; - estate->es_num_root_result_relations = 0; - } - } - else - { - /* - * if no result relation, then set state appropriately - */ - estate->es_result_relations = NULL; - estate->es_num_result_relations = 0; - estate->es_result_relation_info = NULL; - estate->es_root_result_relations = NULL; - estate->es_num_root_result_relations = 0; - } + /* es_result_relation_info is NULL except when within ModifyTable */ + estate->es_result_relation_info = NULL; /* * Next, build the ExecRowMark array from the PlanRowMark(s), if any. @@ -1334,8 +1256,7 @@ InitResultRelInfo(ResultRelInfo *resultRelInfo, * * Most of the time, triggers are fired on one of the result relations of the * query, and so we can just return a member of the es_result_relations array, - * or the es_root_result_relations array (if any), or the - * es_tuple_routing_result_relations list (if any). (Note: in self-join + * or the es_tuple_routing_result_relations list (if any). (Note: in self-join * situations there might be multiple members with the same OID; if so it * doesn't matter which one we pick.) * @@ -1352,30 +1273,16 @@ ResultRelInfo * ExecGetTriggerResultRel(EState *estate, Oid relid) { ResultRelInfo *rInfo; - int nr; ListCell *l; Relation rel; MemoryContext oldcontext; /* First, search through the query result relations */ - rInfo = estate->es_result_relations; - nr = estate->es_num_result_relations; - while (nr > 0) + foreach(l, estate->es_opened_result_relations) { + rInfo = lfirst(l); if (RelationGetRelid(rInfo->ri_RelationDesc) == relid) return rInfo; - rInfo++; - nr--; - } - /* Second, search through the root result relations, if any */ - rInfo = estate->es_root_result_relations; - nr = estate->es_num_root_result_relations; - while (nr > 0) - { - if (RelationGetRelid(rInfo->ri_RelationDesc) == relid) - return rInfo; - rInfo++; - nr--; } /* @@ -1428,35 +1335,6 @@ ExecGetTriggerResultRel(EState *estate, Oid relid) return rInfo; } -/* - * Close any relations that have been opened by ExecGetTriggerResultRel(). - */ -void -ExecCleanUpTriggerState(EState *estate) -{ - ListCell *l; - - foreach(l, estate->es_trig_target_relations) - { - ResultRelInfo *resultRelInfo = (ResultRelInfo *) lfirst(l); - - /* - * Assert this is a "dummy" ResultRelInfo, see above. Otherwise we - * might be issuing a duplicate close against a Relation opened by - * ExecGetRangeTableRelation. - */ - Assert(resultRelInfo->ri_RangeTableIndex == 0); - - /* - * Since ExecGetTriggerResultRel doesn't call ExecOpenIndices for - * these rels, we needn't call ExecCloseIndices either. - */ - Assert(resultRelInfo->ri_NumIndices == 0); - - table_close(resultRelInfo->ri_RelationDesc, NoLock); - } -} - /* ---------------------------------------------------------------- * ExecPostprocessPlan * @@ -1512,9 +1390,6 @@ ExecPostprocessPlan(EState *estate) static void ExecEndPlan(PlanState *planstate, EState *estate) { - ResultRelInfo *resultRelInfo; - Index num_relations; - Index i; ListCell *l; /* @@ -1541,29 +1416,69 @@ ExecEndPlan(PlanState *planstate, EState *estate) ExecResetTupleTable(estate->es_tupleTable, false); /* - * close indexes of result relation(s) if any. (Rels themselves get - * closed next.) + * Close any Relations that have been opened for range table entries or + * result relations. */ - resultRelInfo = estate->es_result_relations; - for (i = estate->es_num_result_relations; i > 0; i--) + ExecCloseResultRelations(estate); + ExecCloseRangeTableRelations(estate); +} + +/* + * Close any relations that have been opened for ResultRelInfos. + */ +void +ExecCloseResultRelations(EState *estate) +{ + ListCell *l; + + /* + * close indexes of result relation(s) if any. (Rels themselves are + * closed in ExecCloseRangeTableRelations()) + */ + foreach(l, estate->es_opened_result_relations) { + ResultRelInfo *resultRelInfo = lfirst(l); + ExecCloseIndices(resultRelInfo); - resultRelInfo++; } - /* - * close whatever rangetable Relations have been opened. We do not - * release any locks we might hold on those rels. - */ - num_relations = estate->es_range_table_size; - for (i = 0; i < num_relations; i++) + /* Close any relations that have been opened by ExecGetTriggerResultRel(). */ + foreach(l, estate->es_trig_target_relations) + { + ResultRelInfo *resultRelInfo = (ResultRelInfo *) lfirst(l); + + /* + * Assert this is a "dummy" ResultRelInfo, see above. Otherwise we + * might be issuing a duplicate close against a Relation opened by + * ExecGetRangeTableRelation. + */ + Assert(resultRelInfo->ri_RangeTableIndex == 0); + + /* + * Since ExecGetTriggerResultRel doesn't call ExecOpenIndices for + * these rels, we needn't call ExecCloseIndices either. + */ + Assert(resultRelInfo->ri_NumIndices == 0); + + table_close(resultRelInfo->ri_RelationDesc, NoLock); + } +} + +/* + * Close all relations opened by ExecGetRangeTableRelation(). + * + * We do not release any locks we might hold on those rels. + */ +void +ExecCloseRangeTableRelations(EState *estate) +{ + int i; + + for (i = 0; i < estate->es_range_table_size; i++) { if (estate->es_relations[i]) table_close(estate->es_relations[i], NoLock); } - - /* likewise close any trigger target relations */ - ExecCleanUpTriggerState(estate); } /* ---------------------------------------------------------------- @@ -2758,17 +2673,9 @@ EvalPlanQualStart(EPQState *epqstate, Plan *planTree) /* * Child EPQ EStates share the parent's copy of unchanging state such as - * the snapshot, rangetable, result-rel info, and external Param info. - * They need their own copies of local state, including a tuple table, - * es_param_exec_vals, etc. - * - * The ResultRelInfo array management is trickier than it looks. We - * create fresh arrays for the child but copy all the content from the - * parent. This is because it's okay for the child to share any - * per-relation state the parent has already created --- but if the child - * sets up any ResultRelInfo fields, such as its own junkfilter, that - * state must *not* propagate back to the parent. (For one thing, the - * pointed-to data is in a memory context that won't last long enough.) + * the snapshot, rangetable, and external Param info. They need their own + * copies of local state, including a tuple table, es_param_exec_vals, + * result-rel info, etc. */ rcestate->es_direction = ForwardScanDirection; rcestate->es_snapshot = parentestate->es_snapshot; @@ -2781,30 +2688,12 @@ EvalPlanQualStart(EPQState *epqstate, Plan *planTree) rcestate->es_plannedstmt = parentestate->es_plannedstmt; rcestate->es_junkFilter = parentestate->es_junkFilter; rcestate->es_output_cid = parentestate->es_output_cid; - if (parentestate->es_num_result_relations > 0) - { - int numResultRelations = parentestate->es_num_result_relations; - int numRootResultRels = parentestate->es_num_root_result_relations; - ResultRelInfo *resultRelInfos; - - resultRelInfos = (ResultRelInfo *) - palloc(numResultRelations * sizeof(ResultRelInfo)); - memcpy(resultRelInfos, parentestate->es_result_relations, - numResultRelations * sizeof(ResultRelInfo)); - rcestate->es_result_relations = resultRelInfos; - rcestate->es_num_result_relations = numResultRelations; - - /* Also transfer partitioned root result relations. */ - if (numRootResultRels > 0) - { - resultRelInfos = (ResultRelInfo *) - palloc(numRootResultRels * sizeof(ResultRelInfo)); - memcpy(resultRelInfos, parentestate->es_root_result_relations, - numRootResultRels * sizeof(ResultRelInfo)); - rcestate->es_root_result_relations = resultRelInfos; - rcestate->es_num_root_result_relations = numRootResultRels; - } - } + + /* + * ResultRelInfos needed by subplans are initialized from scratch when the + * subplans themselves are initialized. + */ + parentestate->es_result_relations = NULL; /* es_result_relation_info must NOT be copied */ /* es_trig_target_relations must NOT be copied */ rcestate->es_top_eflags = parentestate->es_top_eflags; @@ -2914,8 +2803,9 @@ EvalPlanQualStart(EPQState *epqstate, Plan *planTree) * This is a cut-down version of ExecutorEnd(); basically we want to do most * of the normal cleanup, but *not* close result relations (which we are * just sharing from the outer query). We do, however, have to close any - * trigger target relations that got opened, since those are not shared. - * (There probably shouldn't be any of the latter, but just in case...) + * result and trigger target relations that got opened, since those are not + * shared. (There probably shouldn't be any of the latter, but just in + * case...) */ void EvalPlanQualEnd(EPQState *epqstate) @@ -2957,8 +2847,8 @@ EvalPlanQualEnd(EPQState *epqstate) /* throw away the per-estate tuple table, some node may have used it */ ExecResetTupleTable(estate->es_tupleTable, false); - /* close any trigger target relations attached to this EState */ - ExecCleanUpTriggerState(estate); + /* Close any result and trigger target relations attached to this EState */ + ExecCloseResultRelations(estate); MemoryContextSwitchTo(oldcontext); diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c index 382e78fb7fed..befde526910a 100644 --- a/src/backend/executor/execParallel.c +++ b/src/backend/executor/execParallel.c @@ -184,7 +184,6 @@ ExecSerializePlan(Plan *plan, EState *estate) pstmt->planTree = plan; pstmt->rtable = estate->es_range_table; pstmt->resultRelations = NIL; - pstmt->rootResultRelations = NIL; pstmt->appendRelations = NIL; /* diff --git a/src/backend/executor/execUtils.c b/src/backend/executor/execUtils.c index d0e65b86473d..6d8c112e2fe4 100644 --- a/src/backend/executor/execUtils.c +++ b/src/backend/executor/execUtils.c @@ -124,14 +124,9 @@ CreateExecutorState(void) estate->es_output_cid = (CommandId) 0; estate->es_result_relations = NULL; - estate->es_num_result_relations = 0; + estate->es_opened_result_relations = NIL; estate->es_result_relation_info = NULL; - - estate->es_root_result_relations = NULL; - estate->es_num_root_result_relations = 0; - estate->es_tuple_routing_result_relations = NIL; - estate->es_trig_target_relations = NIL; estate->es_param_list_info = NULL; @@ -711,16 +706,7 @@ ExecCreateScanSlotFromOuterPlan(EState *estate, bool ExecRelationIsTargetRelation(EState *estate, Index scanrelid) { - ResultRelInfo *resultRelInfos; - int i; - - resultRelInfos = estate->es_result_relations; - for (i = 0; i < estate->es_num_result_relations; i++) - { - if (resultRelInfos[i].ri_RangeTableIndex == scanrelid) - return true; - } - return false; + return list_member_int(estate->es_plannedstmt->resultRelations, scanrelid); } /* ---------------------------------------------------------------- @@ -779,9 +765,10 @@ ExecInitRangeTable(EState *estate, List *rangeTable) palloc0(estate->es_range_table_size * sizeof(Relation)); /* - * es_rowmarks is also parallel to the es_range_table, but it's allocated - * only if needed. + * es_result_relations and es_rowmarks are also parallel to + * es_range_table, but are allocated only if needed. */ + estate->es_result_relations = NULL; estate->es_rowmarks = NULL; } @@ -835,6 +822,40 @@ ExecGetRangeTableRelation(EState *estate, Index rti) return rel; } +/* + * ExecInitResultRelation + * Open relation given by the passed-in RT index and fill its + * ResultRelInfo node + * + * Here, we also save the ResultRelInfo in estate->es_result_relations array + * such that it can be accessed later using the RT index. + */ +void +ExecInitResultRelation(EState *estate, ResultRelInfo *resultRelInfo, + Index rti) +{ + Relation resultRelationDesc; + + resultRelationDesc = ExecGetRangeTableRelation(estate, rti); + InitResultRelInfo(resultRelInfo, + resultRelationDesc, + rti, + NULL, + estate->es_instrument); + + if (estate->es_result_relations == NULL) + estate->es_result_relations = (ResultRelInfo **) + palloc0(estate->es_range_table_size * sizeof(ResultRelInfo *)); + estate->es_result_relations[rti - 1] = resultRelInfo; + + /* + * Saving in the list allows to avoid needlessly traversing the whole + * array when only a few of its entries are possibly non-NULL. + */ + estate->es_opened_result_relations = + lappend(estate->es_opened_result_relations, resultRelInfo); +} + /* * UpdateChangedParamSet * Add changed parameters to a plan node's chgParam set diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index 98120891619b..b3f7012e3869 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -2301,7 +2301,8 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) ResultRelInfo *saved_resultRelInfo; ResultRelInfo *resultRelInfo; Plan *subplan; - ListCell *l; + ListCell *l, + *l1; int i; Relation rel; bool update_tuple_routing_needed = node->partColsUpdated; @@ -2322,13 +2323,17 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) mtstate->mt_done = false; mtstate->mt_plans = (PlanState **) palloc0(sizeof(PlanState *) * nplans); - mtstate->resultRelInfo = estate->es_result_relations + node->resultRelIndex; + mtstate->resultRelInfo = (ResultRelInfo *) + palloc(nplans * sizeof(ResultRelInfo)); mtstate->mt_scans = (TupleTableSlot **) palloc0(sizeof(TupleTableSlot *) * nplans); /* If modifying a partitioned table, initialize the root table info */ - if (node->rootResultRelIndex >= 0) - mtstate->rootResultRelInfo = estate->es_root_result_relations + - node->rootResultRelIndex; + if (node->rootRelation > 0) + { + mtstate->rootResultRelInfo = makeNode(ResultRelInfo); + ExecInitResultRelation(estate, mtstate->rootResultRelInfo, + node->rootRelation); + } mtstate->mt_arowmarks = (List **) palloc0(sizeof(List *) * nplans); mtstate->mt_nplans = nplans; @@ -2351,9 +2356,14 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) resultRelInfo = mtstate->resultRelInfo; i = 0; - foreach(l, node->plans) + forboth(l, node->resultRelations, l1, node->plans) { - subplan = (Plan *) lfirst(l); + Index resultRelation = lfirst_int(l); + + subplan = (Plan *) lfirst(l1); + + /* This opens the relation and fills ResultRelInfo. */ + ExecInitResultRelation(estate, resultRelInfo, resultRelation); /* Initialize the usesFdwDirectModify flag */ resultRelInfo->ri_usesFdwDirectModify = bms_is_member(i, diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 0409a40b82a8..4d79f70950b7 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -91,7 +91,6 @@ _copyPlannedStmt(const PlannedStmt *from) COPY_NODE_FIELD(planTree); COPY_NODE_FIELD(rtable); COPY_NODE_FIELD(resultRelations); - COPY_NODE_FIELD(rootResultRelations); COPY_NODE_FIELD(appendRelations); COPY_NODE_FIELD(subplans); COPY_BITMAPSET_FIELD(rewindPlanIDs); @@ -207,8 +206,6 @@ _copyModifyTable(const ModifyTable *from) COPY_SCALAR_FIELD(rootRelation); COPY_SCALAR_FIELD(partColsUpdated); COPY_NODE_FIELD(resultRelations); - COPY_SCALAR_FIELD(resultRelIndex); - COPY_SCALAR_FIELD(rootResultRelIndex); COPY_NODE_FIELD(plans); COPY_NODE_FIELD(withCheckOptionLists); COPY_NODE_FIELD(returningLists); diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index f0386480ab89..f441ae3c5192 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -309,7 +309,6 @@ _outPlannedStmt(StringInfo str, const PlannedStmt *node) WRITE_NODE_FIELD(planTree); WRITE_NODE_FIELD(rtable); WRITE_NODE_FIELD(resultRelations); - WRITE_NODE_FIELD(rootResultRelations); WRITE_NODE_FIELD(appendRelations); WRITE_NODE_FIELD(subplans); WRITE_BITMAPSET_FIELD(rewindPlanIDs); @@ -408,8 +407,6 @@ _outModifyTable(StringInfo str, const ModifyTable *node) WRITE_UINT_FIELD(rootRelation); WRITE_BOOL_FIELD(partColsUpdated); WRITE_NODE_FIELD(resultRelations); - WRITE_INT_FIELD(resultRelIndex); - WRITE_INT_FIELD(rootResultRelIndex); WRITE_NODE_FIELD(plans); WRITE_NODE_FIELD(withCheckOptionLists); WRITE_NODE_FIELD(returningLists); @@ -2194,7 +2191,6 @@ _outPlannerGlobal(StringInfo str, const PlannerGlobal *node) WRITE_NODE_FIELD(finalrtable); WRITE_NODE_FIELD(finalrowmarks); WRITE_NODE_FIELD(resultRelations); - WRITE_NODE_FIELD(rootResultRelations); WRITE_NODE_FIELD(appendRelations); WRITE_NODE_FIELD(relationOids); WRITE_NODE_FIELD(invalItems); diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 42050ab71955..3a54765f5ca1 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -1542,7 +1542,6 @@ _readPlannedStmt(void) READ_NODE_FIELD(planTree); READ_NODE_FIELD(rtable); READ_NODE_FIELD(resultRelations); - READ_NODE_FIELD(rootResultRelations); READ_NODE_FIELD(appendRelations); READ_NODE_FIELD(subplans); READ_BITMAPSET_FIELD(rewindPlanIDs); @@ -1639,8 +1638,6 @@ _readModifyTable(void) READ_UINT_FIELD(rootRelation); READ_BOOL_FIELD(partColsUpdated); READ_NODE_FIELD(resultRelations); - READ_INT_FIELD(resultRelIndex); - READ_INT_FIELD(rootResultRelIndex); READ_NODE_FIELD(plans); READ_NODE_FIELD(withCheckOptionLists); READ_NODE_FIELD(returningLists); diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 3d7a4e373fb5..881eaf481339 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -6808,8 +6808,6 @@ make_modifytable(PlannerInfo *root, node->rootRelation = rootRelation; node->partColsUpdated = partColsUpdated; node->resultRelations = resultRelations; - node->resultRelIndex = -1; /* will be set correctly in setrefs.c */ - node->rootResultRelIndex = -1; /* will be set correctly in setrefs.c */ node->plans = subplans; if (!onconflict) { diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index f331f82a6c20..986d7a52e32c 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -305,7 +305,6 @@ standard_planner(Query *parse, const char *query_string, int cursorOptions, glob->finalrtable = NIL; glob->finalrowmarks = NIL; glob->resultRelations = NIL; - glob->rootResultRelations = NIL; glob->appendRelations = NIL; glob->relationOids = NIL; glob->invalItems = NIL; @@ -493,7 +492,6 @@ standard_planner(Query *parse, const char *query_string, int cursorOptions, Assert(glob->finalrtable == NIL); Assert(glob->finalrowmarks == NIL); Assert(glob->resultRelations == NIL); - Assert(glob->rootResultRelations == NIL); Assert(glob->appendRelations == NIL); top_plan = set_plan_references(root, top_plan); /* ... and the subplans (both regular subplans and initplans) */ @@ -520,7 +518,6 @@ standard_planner(Query *parse, const char *query_string, int cursorOptions, result->planTree = top_plan; result->rtable = glob->finalrtable; result->resultRelations = glob->resultRelations; - result->rootResultRelations = glob->rootResultRelations; result->appendRelations = glob->appendRelations; result->subplans = glob->subplans; result->rewindPlanIDs = glob->rewindPlanIDs; diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index dd8e2e966ddd..6847ff6f4473 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -975,26 +975,15 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) /* * Append this ModifyTable node's final result relation RT - * index(es) to the global list for the plan, and set its - * resultRelIndex to reflect their starting position in the - * global list. + * index(es) to the global list for the plan. */ - splan->resultRelIndex = list_length(root->glob->resultRelations); root->glob->resultRelations = list_concat(root->glob->resultRelations, splan->resultRelations); - - /* - * If the main target relation is a partitioned table, also - * add the partition root's RT index to rootResultRelations, - * and remember its index in that list in rootResultRelIndex. - */ if (splan->rootRelation) { - splan->rootResultRelIndex = - list_length(root->glob->rootResultRelations); - root->glob->rootResultRelations = - lappend_int(root->glob->rootResultRelations, + root->glob->resultRelations = + lappend_int(root->glob->resultRelations, splan->rootRelation); } } diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 9c6fdeeb56c4..8d5d9e05b3c0 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -344,7 +344,6 @@ static EState * create_estate_for_relation(LogicalRepRelMapEntry *rel) { EState *estate; - ResultRelInfo *resultRelInfo; RangeTblEntry *rte; estate = CreateExecutorState(); @@ -356,13 +355,6 @@ create_estate_for_relation(LogicalRepRelMapEntry *rel) rte->rellockmode = AccessShareLock; ExecInitRangeTable(estate, list_make1(rte)); - resultRelInfo = makeNode(ResultRelInfo); - InitResultRelInfo(resultRelInfo, rel->localrel, 1, NULL, 0); - - estate->es_result_relations = resultRelInfo; - estate->es_num_result_relations = 1; - estate->es_result_relation_info = resultRelInfo; - estate->es_output_cid = GetCurrentCommandId(true); /* Prepare to catch AFTER triggers. */ @@ -1150,6 +1142,7 @@ GetRelationIdentityOrPK(Relation rel) static void apply_handle_insert(StringInfo s) { + ResultRelInfo *resultRelInfo; LogicalRepRelMapEntry *rel; LogicalRepTupleData newtup; LogicalRepRelId relid; @@ -1179,6 +1172,9 @@ apply_handle_insert(StringInfo s) remoteslot = ExecInitExtraTupleSlot(estate, RelationGetDescr(rel->localrel), &TTSOpsVirtual); + resultRelInfo = makeNode(ResultRelInfo); + InitResultRelInfo(resultRelInfo, rel->localrel, 1, NULL, 0); + estate->es_result_relation_info = resultRelInfo; /* Input functions may need an active snapshot, so get one */ PushActiveSnapshot(GetTransactionSnapshot()); @@ -1191,10 +1187,10 @@ apply_handle_insert(StringInfo s) /* For a partitioned table, insert the tuple into a partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) - apply_handle_tuple_routing(estate->es_result_relation_info, estate, + apply_handle_tuple_routing(resultRelInfo, estate, remoteslot, NULL, rel, CMD_INSERT); else - apply_handle_insert_internal(estate->es_result_relation_info, estate, + apply_handle_insert_internal(resultRelInfo, estate, remoteslot); PopActiveSnapshot(); @@ -1265,6 +1261,7 @@ check_relation_updatable(LogicalRepRelMapEntry *rel) static void apply_handle_update(StringInfo s) { + ResultRelInfo *resultRelInfo; LogicalRepRelMapEntry *rel; LogicalRepRelId relid; EState *estate; @@ -1301,6 +1298,9 @@ apply_handle_update(StringInfo s) remoteslot = ExecInitExtraTupleSlot(estate, RelationGetDescr(rel->localrel), &TTSOpsVirtual); + resultRelInfo = makeNode(ResultRelInfo); + InitResultRelInfo(resultRelInfo, rel->localrel, 1, NULL, 0); + estate->es_result_relation_info = resultRelInfo; /* * Populate updatedCols so that per-column triggers can fire. This could @@ -1337,10 +1337,10 @@ apply_handle_update(StringInfo s) /* For a partitioned table, apply update to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) - apply_handle_tuple_routing(estate->es_result_relation_info, estate, + apply_handle_tuple_routing(resultRelInfo, estate, remoteslot, &newtup, rel, CMD_UPDATE); else - apply_handle_update_internal(estate->es_result_relation_info, estate, + apply_handle_update_internal(resultRelInfo, estate, remoteslot, &newtup, rel); PopActiveSnapshot(); @@ -1420,6 +1420,7 @@ apply_handle_update_internal(ResultRelInfo *relinfo, static void apply_handle_delete(StringInfo s) { + ResultRelInfo *resultRelInfo; LogicalRepRelMapEntry *rel; LogicalRepTupleData oldtup; LogicalRepRelId relid; @@ -1452,6 +1453,9 @@ apply_handle_delete(StringInfo s) remoteslot = ExecInitExtraTupleSlot(estate, RelationGetDescr(rel->localrel), &TTSOpsVirtual); + resultRelInfo = makeNode(ResultRelInfo); + InitResultRelInfo(resultRelInfo, rel->localrel, 1, NULL, 0); + estate->es_result_relation_info = resultRelInfo; PushActiveSnapshot(GetTransactionSnapshot()); @@ -1462,10 +1466,10 @@ apply_handle_delete(StringInfo s) /* For a partitioned table, apply delete to correct partition. */ if (rel->localrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) - apply_handle_tuple_routing(estate->es_result_relation_info, estate, + apply_handle_tuple_routing(resultRelInfo, estate, remoteslot, NULL, rel, CMD_DELETE); else - apply_handle_delete_internal(estate->es_result_relation_info, estate, + apply_handle_delete_internal(resultRelInfo, estate, remoteslot, &rel->remoterel); PopActiveSnapshot(); diff --git a/src/include/executor/executor.h b/src/include/executor/executor.h index 415e117407c9..c283bf14541c 100644 --- a/src/include/executor/executor.h +++ b/src/include/executor/executor.h @@ -191,7 +191,6 @@ extern void InitResultRelInfo(ResultRelInfo *resultRelInfo, Relation partition_root, int instrument_options); extern ResultRelInfo *ExecGetTriggerResultRel(EState *estate, Oid relid); -extern void ExecCleanUpTriggerState(EState *estate); extern void ExecConstraints(ResultRelInfo *resultRelInfo, TupleTableSlot *slot, EState *estate); extern bool ExecPartitionCheck(ResultRelInfo *resultRelInfo, @@ -538,6 +537,8 @@ extern bool ExecRelationIsTargetRelation(EState *estate, Index scanrelid); extern Relation ExecOpenScanRelation(EState *estate, Index scanrelid, int eflags); extern void ExecInitRangeTable(EState *estate, List *rangeTable); +extern void ExecCloseRangeTableRelations(EState *estate); +extern void ExecCloseResultRelations(EState *estate); static inline RangeTblEntry * exec_rt_fetch(Index rti, EState *estate) @@ -546,6 +547,8 @@ exec_rt_fetch(Index rti, EState *estate) } extern Relation ExecGetRangeTableRelation(EState *estate, Index rti); +extern void ExecInitResultRelation(EState *estate, ResultRelInfo *resultRelInfo, + Index rti); extern int executor_errposition(EState *estate, int location); diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index ef448d67c77d..a926ff17118b 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -519,23 +519,19 @@ typedef struct EState CommandId es_output_cid; /* Info about target table(s) for insert/update/delete queries: */ - ResultRelInfo *es_result_relations; /* array of ResultRelInfos */ - int es_num_result_relations; /* length of array */ + ResultRelInfo **es_result_relations; /* Array of per-range-table-entry + * ResultRelInfo pointers, or NULL + * if not a target table */ + List *es_opened_result_relations; /* List of non-NULL entries in + * es_result_relations in no + * specific order */ ResultRelInfo *es_result_relation_info; /* currently active array elt */ - /* - * Info about the partition root table(s) for insert/update/delete queries - * targeting partitioned tables. Only leaf partitions are mentioned in - * es_result_relations, but we need access to the roots for firing - * triggers and for runtime tuple routing. - */ - ResultRelInfo *es_root_result_relations; /* array of ResultRelInfos */ - int es_num_root_result_relations; /* length of the array */ PartitionDirectory es_partition_directory; /* for PartitionDesc lookup */ /* * The following list contains ResultRelInfos created by the tuple routing - * code for partitions that don't already have one. + * code for partitions that aren't found in the es_result_relations array. */ List *es_tuple_routing_result_relations; diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index dbe86e7af657..3dd16b9ad534 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -120,8 +120,6 @@ typedef struct PlannerGlobal List *resultRelations; /* "flat" list of integer RT indexes */ - List *rootResultRelations; /* "flat" list of integer RT indexes */ - List *appendRelations; /* "flat" list of AppendRelInfos */ List *relationOids; /* OIDs of relations the plan depends on */ diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index 83e01074ed1d..a7bdf3497e13 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -68,12 +68,6 @@ typedef struct PlannedStmt /* rtable indexes of target relations for INSERT/UPDATE/DELETE */ List *resultRelations; /* integer list of RT indexes, or NIL */ - /* - * rtable indexes of partitioned table roots that are UPDATE/DELETE - * targets; needed for trigger firing. - */ - List *rootResultRelations; - List *appendRelations; /* list of AppendRelInfo nodes */ List *subplans; /* Plan trees for SubPlan expressions; note @@ -224,8 +218,6 @@ typedef struct ModifyTable Index rootRelation; /* Root RT index, if target is partitioned */ bool partColsUpdated; /* some part key in hierarchy updated */ List *resultRelations; /* integer list of RT indexes */ - int resultRelIndex; /* index of first resultRel in plan's list */ - int rootResultRelIndex; /* index of the partitioned table root */ List *plans; /* plan(s) producing source data */ List *withCheckOptionLists; /* per-target-table WCO lists */ List *returningLists; /* per-target-table RETURNING tlists */ From ae0f7b11f143d9748e2201c3647330893c4c1c5a Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 13 Oct 2020 17:44:56 -0400 Subject: [PATCH 293/589] Paper over regression failures in infinite_recurse() on PPC64 Linux. Our infinite_recurse() test to verify sane stack-overrun behavior is affected by a bug of the Linux kernel on PPC64: it will get SIGSEGV if it receives a signal when the stack depth is (a) over 1MB and (b) within a few kB of filling the current physical stack allocation. See https://bugzilla.kernel.org/show_bug.cgi?id=205183. Since this test is a bit time-consuming and we run it in parallel with test scripts that do a lot of DDL, it can be expected to get an sinval catchup interrupt at some point, leading to failure if the timing is wrong. This has caused more than 100 buildfarm failures over the past year or so. While a fix exists for the kernel bug, it might be years before that propagates into all production kernels, particularly in some of the older distros we have in the buildfarm. For now, let's just back off and not run this test on Linux PPC64; that loses nothing in test coverage so far as our own code is concerned. To do that, split this test into a new script infinite_recurse.sql and skip the test when the platform name is powerpc64...-linux-gnu. Back-patch to v12. Branches before that have not been seen to get this failure. No doubt that's because the "errors" test was not run in parallel with other tests before commit 798070ec0, greatly reducing the odds of an sinval catchup being necessary. I also back-patched 3c8553547 into v12, just so the new regression script would look the same in all branches having it. Discussion: https://postgr.es/m/3479046.1602607848@sss.pgh.pa.us Discussion: https://postgr.es/m/20190723162703.GM22387%40telsasoft.com --- src/test/regress/expected/errors.out | 10 ------- .../regress/expected/infinite_recurse.out | 24 +++++++++++++++ .../regress/expected/infinite_recurse_1.out | 16 ++++++++++ src/test/regress/parallel_schedule | 2 +- src/test/regress/serial_schedule | 1 + src/test/regress/sql/errors.sql | 9 ------ src/test/regress/sql/infinite_recurse.sql | 29 +++++++++++++++++++ 7 files changed, 71 insertions(+), 20 deletions(-) create mode 100644 src/test/regress/expected/infinite_recurse.out create mode 100644 src/test/regress/expected/infinite_recurse_1.out create mode 100644 src/test/regress/sql/infinite_recurse.sql diff --git a/src/test/regress/expected/errors.out b/src/test/regress/expected/errors.out index a525aa2f9377..1e7b5a704619 100644 --- a/src/test/regress/expected/errors.out +++ b/src/test/regress/expected/errors.out @@ -440,13 +440,3 @@ NULL); ERROR: syntax error at or near "NUL" LINE 16: ...L, id2 TEXT NOT NULL PRIMARY KEY, id3 INTEGER NOT NUL, id4 I... ^ --- Check that stack depth detection mechanism works and --- max_stack_depth is not set too high. The full error report is not --- very stable, so show only SQLSTATE and primary error message. -create function infinite_recurse() returns int as -'select infinite_recurse()' language sql; -\set VERBOSITY sqlstate -select infinite_recurse(); -ERROR: 54001 -\echo :LAST_ERROR_MESSAGE -stack depth limit exceeded diff --git a/src/test/regress/expected/infinite_recurse.out b/src/test/regress/expected/infinite_recurse.out new file mode 100644 index 000000000000..aa102fadd839 --- /dev/null +++ b/src/test/regress/expected/infinite_recurse.out @@ -0,0 +1,24 @@ +-- Check that stack depth detection mechanism works and +-- max_stack_depth is not set too high. +create function infinite_recurse() returns int as +'select infinite_recurse()' language sql; +-- Unfortunately, up till mid 2020 the Linux kernel had a bug in PPC64 +-- signal handling that would cause this test to crash if it happened +-- to receive an sinval catchup interrupt while the stack is deep: +-- https://bugzilla.kernel.org/show_bug.cgi?id=205183 +-- It is likely to be many years before that bug disappears from all +-- production kernels, so disable this test on such platforms. +-- (We still create the function, so as not to have a cross-platform +-- difference in the end state of the regression database.) +SELECT version() ~ 'powerpc64[^,]*-linux-gnu' + AS skip_test \gset +\if :skip_test +\quit +\endif +-- The full error report is not very stable, so we show only SQLSTATE +-- and primary error message. +\set VERBOSITY sqlstate +select infinite_recurse(); +ERROR: 54001 +\echo :LAST_ERROR_MESSAGE +stack depth limit exceeded diff --git a/src/test/regress/expected/infinite_recurse_1.out b/src/test/regress/expected/infinite_recurse_1.out new file mode 100644 index 000000000000..b2c99a0d0d41 --- /dev/null +++ b/src/test/regress/expected/infinite_recurse_1.out @@ -0,0 +1,16 @@ +-- Check that stack depth detection mechanism works and +-- max_stack_depth is not set too high. +create function infinite_recurse() returns int as +'select infinite_recurse()' language sql; +-- Unfortunately, up till mid 2020 the Linux kernel had a bug in PPC64 +-- signal handling that would cause this test to crash if it happened +-- to receive an sinval catchup interrupt while the stack is deep: +-- https://bugzilla.kernel.org/show_bug.cgi?id=205183 +-- It is likely to be many years before that bug disappears from all +-- production kernels, so disable this test on such platforms. +-- (We still create the function, so as not to have a cross-platform +-- difference in the end state of the regression database.) +SELECT version() ~ 'powerpc64[^,]*-linux-gnu' + AS skip_test \gset +\if :skip_test +\quit diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 026ea880cde3..ae89ed7f0b40 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -55,7 +55,7 @@ test: create_index create_index_spgist create_view index_including index_includi # ---------- # Another group of parallel tests # ---------- -test: create_aggregate create_function_3 create_cast constraints triggers select inherit typed_table vacuum drop_if_exists updatable_views roleattributes create_am hash_func errors +test: create_aggregate create_function_3 create_cast constraints triggers select inherit typed_table vacuum drop_if_exists updatable_views roleattributes create_am hash_func errors infinite_recurse # ---------- # sanity_check does a vacuum, affecting the sort order of SELECT * diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule index 979d9261197d..525bdc804f61 100644 --- a/src/test/regress/serial_schedule +++ b/src/test/regress/serial_schedule @@ -83,6 +83,7 @@ test: roleattributes test: create_am test: hash_func test: errors +test: infinite_recurse test: sanity_check test: select_into test: select_distinct diff --git a/src/test/regress/sql/errors.sql b/src/test/regress/sql/errors.sql index 86b672538a17..66a56b28f62b 100644 --- a/src/test/regress/sql/errors.sql +++ b/src/test/regress/sql/errors.sql @@ -364,12 +364,3 @@ INT4 UNIQUE NOT NULL); - --- Check that stack depth detection mechanism works and --- max_stack_depth is not set too high. The full error report is not --- very stable, so show only SQLSTATE and primary error message. -create function infinite_recurse() returns int as -'select infinite_recurse()' language sql; -\set VERBOSITY sqlstate -select infinite_recurse(); -\echo :LAST_ERROR_MESSAGE diff --git a/src/test/regress/sql/infinite_recurse.sql b/src/test/regress/sql/infinite_recurse.sql new file mode 100644 index 000000000000..151dba4a7aef --- /dev/null +++ b/src/test/regress/sql/infinite_recurse.sql @@ -0,0 +1,29 @@ +-- Check that stack depth detection mechanism works and +-- max_stack_depth is not set too high. + +create function infinite_recurse() returns int as +'select infinite_recurse()' language sql; + +-- Unfortunately, up till mid 2020 the Linux kernel had a bug in PPC64 +-- signal handling that would cause this test to crash if it happened +-- to receive an sinval catchup interrupt while the stack is deep: +-- https://bugzilla.kernel.org/show_bug.cgi?id=205183 +-- It is likely to be many years before that bug disappears from all +-- production kernels, so disable this test on such platforms. +-- (We still create the function, so as not to have a cross-platform +-- difference in the end state of the regression database.) + +SELECT version() ~ 'powerpc64[^,]*-linux-gnu' + AS skip_test \gset +\if :skip_test +\quit +\endif + +-- The full error report is not very stable, so we show only SQLSTATE +-- and primary error message. + +\set VERBOSITY sqlstate + +select infinite_recurse(); + +\echo :LAST_ERROR_MESSAGE From 4e118fc33e3ca5244c11a81a71bd25cf9ed3d484 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 14 Oct 2020 07:54:14 +0200 Subject: [PATCH 294/589] Correct error message Apparently copy-and-pasted from nearby. --- src/backend/parser/parse_target.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/parser/parse_target.c b/src/backend/parser/parse_target.c index 566c5178373e..9de0cff83389 100644 --- a/src/backend/parser/parse_target.c +++ b/src/backend/parser/parse_target.c @@ -412,7 +412,7 @@ markTargetListOrigin(ParseState *pstate, TargetEntry *tle, ste = get_tle_by_resno(GetCTETargetList(cte), attnum); if (ste == NULL || ste->resjunk) - elog(ERROR, "subquery %s does not have attribute %d", + elog(ERROR, "CTE %s does not have attribute %d", rte->eref->aliasname, attnum); tle->resorigtbl = ste->resorigtbl; tle->resorigcol = ste->resorigcol; @@ -1606,7 +1606,7 @@ expandRecordVariable(ParseState *pstate, Var *var, int levelsup) ste = get_tle_by_resno(GetCTETargetList(cte), attnum); if (ste == NULL || ste->resjunk) - elog(ERROR, "subquery %s does not have attribute %d", + elog(ERROR, "CTE %s does not have attribute %d", rte->eref->aliasname, attnum); expr = (Node *) ste->expr; if (IsA(expr, Var)) From 39b4a951003a6545268e141272e123929d0d710f Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 14 Oct 2020 08:24:54 +0200 Subject: [PATCH 295/589] Use https for gnu.org links Mostly already done, but there were some stragglers. --- doc/src/sgml/installation.sgml | 2 +- src/include/c.h | 3 ++- src/include/port/atomics/generic-gcc.h | 4 ++-- src/interfaces/ecpg/Makefile | 2 +- src/tools/RELEASE_CHANGES | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 3f12acb7189d..3ac588dfb5c9 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -239,7 +239,7 @@ su - postgres class="osname">Linux, NetBSD, Solaris), for other systems you can download an add-on package from . + url="https://www.gnu.org/software/gettext/">. If you are using the Gettext implementation in the GNU C library then you will additionally need the GNU Gettext package for some diff --git a/src/include/c.h b/src/include/c.h index 2c61ca8aa894..aca36c757ec8 100644 --- a/src/include/c.h +++ b/src/include/c.h @@ -1132,7 +1132,8 @@ typedef union PGAlignedXLogBlock * access to the original string and translated string, and for cases where * immediate translation is not possible, like when initializing global * variables. - * http://www.gnu.org/software/autoconf/manual/gettext/Special-cases.html + * + * https://www.gnu.org/software/gettext/manual/html_node/Special-cases.html */ #define gettext_noop(x) (x) diff --git a/src/include/port/atomics/generic-gcc.h b/src/include/port/atomics/generic-gcc.h index 2d84305f26bc..1a3dce34ed37 100644 --- a/src/include/port/atomics/generic-gcc.h +++ b/src/include/port/atomics/generic-gcc.h @@ -10,9 +10,9 @@ * * Documentation: * * Legacy __sync Built-in Functions for Atomic Memory Access - * http://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fsync-Builtins.html + * https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fsync-Builtins.html * * Built-in functions for memory model aware atomic operations - * http://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fatomic-Builtins.html + * https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fatomic-Builtins.html * * src/include/port/atomics/generic-gcc.h * diff --git a/src/interfaces/ecpg/Makefile b/src/interfaces/ecpg/Makefile index 41460a17c964..a8f91e3dc2b3 100644 --- a/src/interfaces/ecpg/Makefile +++ b/src/interfaces/ecpg/Makefile @@ -5,7 +5,7 @@ include $(top_builddir)/src/Makefile.global SUBDIRS = include pgtypeslib ecpglib compatlib preproc # Suppress parallel build of subdirectories to avoid a bug in GNU make 3.82, cf -# http://savannah.gnu.org/bugs/?30653 +# https://savannah.gnu.org/bugs/?30653 # https://bugzilla.redhat.com/show_bug.cgi?id=835424 # (There are some other parallelism bugs in the subdirectory makefiles # themselves, but there's little point in fixing them as long as we have diff --git a/src/tools/RELEASE_CHANGES b/src/tools/RELEASE_CHANGES index 6ba9121e303a..5206640341d5 100644 --- a/src/tools/RELEASE_CHANGES +++ b/src/tools/RELEASE_CHANGES @@ -73,7 +73,7 @@ but there may be reasons to do them at other times as well. to lower numbers, using renumber_oids.pl (see notes in bki.sgml) * Update config.guess and config.sub - (from http://savannah.gnu.org/projects/config) + (from https://savannah.gnu.org/projects/config) * Update inet/cidr data types with newest Bind patches From 178f2d560dde05b356ab2f586b8bc62514f454aa Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 14 Oct 2020 10:58:38 +0300 Subject: [PATCH 296/589] Include result relation info in direct modify ForeignScan nodes. FDWs that can perform an UPDATE/DELETE remotely using the "direct modify" set of APIs need to access the ResultRelInfo of the target table. That's currently available in EState.es_result_relation_info, but the next commit will remove that field. This commit adds a new resultRelation field in ForeignScan, to store the target relation's RT index, and the corresponding ResultRelInfo in ForeignScanState. The FDW's PlanDirectModify callback is expected to set 'resultRelation' along with 'operation'. The core code doesn't need them for anything, they are for the convenience of FDW's Begin- and IterateDirectModify callbacks. Authors: Amit Langote, Etsuro Fujita Discussion: https://www.postgresql.org/message-id/CA%2BHiwqGEmiib8FLiHMhKB%2BCH5dRgHSLc5N5wnvc4kym%2BZYpQEQ%40mail.gmail.com --- contrib/postgres_fdw/postgres_fdw.c | 16 +++++++++------- doc/src/sgml/fdwhandler.sgml | 15 +++++++++------ src/backend/executor/nodeForeignscan.c | 7 +++++++ src/backend/nodes/copyfuncs.c | 1 + src/backend/nodes/outfuncs.c | 1 + src/backend/nodes/readfuncs.c | 1 + src/backend/optimizer/plan/createplan.c | 4 ++++ src/backend/optimizer/plan/setrefs.c | 4 ++++ src/include/nodes/execnodes.h | 1 + src/include/nodes/plannodes.h | 8 ++++++++ 10 files changed, 45 insertions(+), 13 deletions(-) diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index a31abce7c996..78facb8ebfa4 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -451,6 +451,7 @@ static void init_returning_filter(PgFdwDirectModifyState *dmstate, List *fdw_scan_tlist, Index rtindex); static TupleTableSlot *apply_returning_filter(PgFdwDirectModifyState *dmstate, + ResultRelInfo *resultRelInfo, TupleTableSlot *slot, EState *estate); static void prepare_query_params(PlanState *node, @@ -2287,9 +2288,10 @@ postgresPlanDirectModify(PlannerInfo *root, } /* - * Update the operation info. + * Update the operation and target relation info. */ fscan->operation = operation; + fscan->resultRelation = resultRelation; /* * Update the fdw_exprs list that will be available to the executor. @@ -2355,7 +2357,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags) * Identify which user to do the remote access as. This should match what * ExecCheckRTEPerms() does. */ - rtindex = estate->es_result_relation_info->ri_RangeTableIndex; + rtindex = node->resultRelInfo->ri_RangeTableIndex; rte = exec_rt_fetch(rtindex, estate); userid = rte->checkAsUser ? rte->checkAsUser : GetUserId(); @@ -2450,7 +2452,7 @@ postgresIterateDirectModify(ForeignScanState *node) { PgFdwDirectModifyState *dmstate = (PgFdwDirectModifyState *) node->fdw_state; EState *estate = node->ss.ps.state; - ResultRelInfo *resultRelInfo = estate->es_result_relation_info; + ResultRelInfo *resultRelInfo = node->resultRelInfo; /* * If this is the first call after Begin, execute the statement. @@ -4086,7 +4088,7 @@ get_returning_data(ForeignScanState *node) { PgFdwDirectModifyState *dmstate = (PgFdwDirectModifyState *) node->fdw_state; EState *estate = node->ss.ps.state; - ResultRelInfo *resultRelInfo = estate->es_result_relation_info; + ResultRelInfo *resultRelInfo = node->resultRelInfo; TupleTableSlot *slot = node->ss.ss_ScanTupleSlot; TupleTableSlot *resultSlot; @@ -4141,7 +4143,7 @@ get_returning_data(ForeignScanState *node) if (dmstate->rel) resultSlot = slot; else - resultSlot = apply_returning_filter(dmstate, slot, estate); + resultSlot = apply_returning_filter(dmstate, resultRelInfo, slot, estate); } dmstate->next_tuple++; @@ -4230,10 +4232,10 @@ init_returning_filter(PgFdwDirectModifyState *dmstate, */ static TupleTableSlot * apply_returning_filter(PgFdwDirectModifyState *dmstate, + ResultRelInfo *resultRelInfo, TupleTableSlot *slot, EState *estate) { - ResultRelInfo *relInfo = estate->es_result_relation_info; TupleDesc resultTupType = RelationGetDescr(dmstate->resultRel); TupleTableSlot *resultSlot; Datum *values; @@ -4245,7 +4247,7 @@ apply_returning_filter(PgFdwDirectModifyState *dmstate, /* * Use the return tuple slot as a place to store the result tuple. */ - resultSlot = ExecGetReturningSlot(estate, relInfo); + resultSlot = ExecGetReturningSlot(estate, resultRelInfo); /* * Extract all the values of the scan tuple. diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml index 72fa1272120d..9c9293414c58 100644 --- a/doc/src/sgml/fdwhandler.sgml +++ b/doc/src/sgml/fdwhandler.sgml @@ -861,11 +861,15 @@ PlanDirectModify(PlannerInfo *root, To execute the direct modification on the remote server, this function must rewrite the target subplan with a ForeignScan plan node that executes the direct modification on the remote server. The - operation field of the ForeignScan must - be set to the CmdType enumeration appropriately; that is, + operation and resultRelation fields + of the ForeignScan must be set appropriately. + operation must be set to the CmdType + enumeration corresponding to the statement kind (that is, CMD_UPDATE for UPDATE, CMD_INSERT for INSERT, and - CMD_DELETE for DELETE. + CMD_DELETE for DELETE), and the + resultRelation argument must be copied to the + resultRelation field. @@ -925,9 +929,8 @@ IterateDirectModify(ForeignScanState *node); needed for the RETURNING calculation, returning it in a tuple table slot (the node's ScanTupleSlot should be used for this purpose). The data that was actually inserted, updated - or deleted must be stored in the - es_result_relation_info->ri_projectReturning->pi_exprContext->ecxt_scantuple - of the node's EState. + or deleted must be stored in + node->resultRelInfo->ri_projectReturning->pi_exprContext->ecxt_scantuple. Return NULL if no more rows are available. Note that this is called in a short-lived memory context that will be reset between invocations. Create a memory context in diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c index 513471ab9b90..0b20f94035ed 100644 --- a/src/backend/executor/nodeForeignscan.c +++ b/src/backend/executor/nodeForeignscan.c @@ -215,6 +215,13 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags) scanstate->fdwroutine = fdwroutine; scanstate->fdw_state = NULL; + /* + * For the FDW's convenience, look up the modification target relation's. + * ResultRelInfo. + */ + if (node->resultRelation > 0) + scanstate->resultRelInfo = estate->es_result_relations[node->resultRelation - 1]; + /* Initialize any outer plan. */ if (outerPlan(node)) outerPlanState(scanstate) = diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 4d79f70950b7..2b4d7654cc71 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -758,6 +758,7 @@ _copyForeignScan(const ForeignScan *from) COPY_NODE_FIELD(fdw_recheck_quals); COPY_BITMAPSET_FIELD(fs_relids); COPY_SCALAR_FIELD(fsSystemCol); + COPY_SCALAR_FIELD(resultRelation); return newnode; } diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index f441ae3c5192..08a049232e0a 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -695,6 +695,7 @@ _outForeignScan(StringInfo str, const ForeignScan *node) WRITE_NODE_FIELD(fdw_recheck_quals); WRITE_BITMAPSET_FIELD(fs_relids); WRITE_BOOL_FIELD(fsSystemCol); + WRITE_INT_FIELD(resultRelation); } static void diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 3a54765f5ca1..ab7b535caaea 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -2014,6 +2014,7 @@ _readForeignScan(void) READ_NODE_FIELD(fdw_recheck_quals); READ_BITMAPSET_FIELD(fs_relids); READ_BOOL_FIELD(fsSystemCol); + READ_INT_FIELD(resultRelation); READ_DONE(); } diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 881eaf481339..94280a730c4d 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -5530,7 +5530,11 @@ make_foreignscan(List *qptlist, plan->lefttree = outer_plan; plan->righttree = NULL; node->scan.scanrelid = scanrelid; + + /* these may be overridden by the FDW's PlanDirectModify callback. */ node->operation = CMD_SELECT; + node->resultRelation = 0; + /* fs_server will be filled in by create_foreignscan_plan */ node->fs_server = InvalidOid; node->fdw_exprs = fdw_exprs; diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index 6847ff6f4473..8b4337142594 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -1310,6 +1310,10 @@ set_foreignscan_references(PlannerInfo *root, } fscan->fs_relids = offset_relid_set(fscan->fs_relids, rtoffset); + + /* Adjust resultRelation if it's valid */ + if (fscan->resultRelation > 0) + fscan->resultRelation += rtoffset; } /* diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index a926ff17118b..d9b09c592068 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1777,6 +1777,7 @@ typedef struct ForeignScanState ScanState ss; /* its first field is NodeTag */ ExprState *fdw_recheck_quals; /* original quals not in ss.ps.qual */ Size pscan_len; /* size of parallel coordination information */ + ResultRelInfo *resultRelInfo; /* result rel info, if UPDATE or DELETE */ /* use struct pointer to avoid including fdwapi.h here */ struct FdwRoutine *fdwroutine; void *fdw_state; /* foreign-data wrapper can keep state here */ diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index a7bdf3497e13..7e6b10f86b97 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -599,12 +599,20 @@ typedef struct WorkTableScan * When the plan node represents a foreign join, scan.scanrelid is zero and * fs_relids must be consulted to identify the join relation. (fs_relids * is valid for simple scans as well, but will always match scan.scanrelid.) + * + * If the FDW's PlanDirectModify() callback decides to repurpose a ForeignScan + * node to perform the UPDATE or DELETE operation directly in the remote + * server, it sets 'operation' and 'resultRelation' to identify the operation + * type and target relation. Note that these fields are only set if the + * modification is performed *fully* remotely; otherwise, the modification is + * driven by a local ModifyTable node and 'operation' is left to CMD_SELECT. * ---------------- */ typedef struct ForeignScan { Scan scan; CmdType operation; /* SELECT/INSERT/UPDATE/DELETE */ + Index resultRelation; /* direct modification target's RT index */ Oid fs_server; /* OID of foreign server */ List *fdw_exprs; /* expressions that FDW may evaluate */ List *fdw_private; /* private data for FDW */ From a04daa97a4339c38e304cd6164d37da540d665a8 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 14 Oct 2020 11:41:40 +0300 Subject: [PATCH 297/589] Remove es_result_relation_info from EState. Maintaining 'es_result_relation_info' correctly at all times has become cumbersome, especially with partitioning where each partition gets its own result relation info. Having to set and reset it across arbitrary operations has caused bugs in the past. This changes all the places that used 'es_result_relation_info', to receive the currently active ResultRelInfo via function parameters instead. Author: Amit Langote Discussion: https://www.postgresql.org/message-id/CA%2BHiwqGEmiib8FLiHMhKB%2BCH5dRgHSLc5N5wnvc4kym%2BZYpQEQ%40mail.gmail.com --- src/backend/commands/copy.c | 19 +-- src/backend/commands/tablecmds.c | 2 - src/backend/executor/execIndexing.c | 9 +- src/backend/executor/execMain.c | 4 - src/backend/executor/execReplication.c | 24 +-- src/backend/executor/execUtils.c | 1 - src/backend/executor/nodeModifyTable.c | 200 ++++++++++------------- src/backend/replication/logical/worker.c | 17 +- src/include/executor/executor.h | 19 ++- src/include/executor/nodeModifyTable.h | 4 +- src/include/nodes/execnodes.h | 1 - src/test/regress/expected/insert.out | 4 +- src/test/regress/sql/insert.sql | 4 +- 13 files changed, 131 insertions(+), 177 deletions(-) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 71d48d45743b..531bd7c73a52 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -2489,9 +2489,6 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo, ResultRelInfo *resultRelInfo = buffer->resultRelInfo; TupleTableSlot **slots = buffer->slots; - /* Set es_result_relation_info to the ResultRelInfo we're flushing. */ - estate->es_result_relation_info = resultRelInfo; - /* * Print error context information correctly, if one of the operations * below fail. @@ -2524,7 +2521,8 @@ CopyMultiInsertBufferFlush(CopyMultiInsertInfo *miinfo, cstate->cur_lineno = buffer->linenos[i]; recheckIndexes = - ExecInsertIndexTuples(buffer->slots[i], estate, false, NULL, + ExecInsertIndexTuples(resultRelInfo, + buffer->slots[i], estate, false, NULL, NIL); ExecARInsertTriggers(estate, resultRelInfo, slots[i], recheckIndexes, @@ -2839,8 +2837,6 @@ CopyFrom(CopyState cstate) ExecOpenIndices(resultRelInfo, false); - estate->es_result_relation_info = resultRelInfo; - /* * Set up a ModifyTableState so we can let FDW(s) init themselves for * foreign-table result relation(s). @@ -3108,11 +3104,6 @@ CopyFrom(CopyState cstate) prevResultRelInfo = resultRelInfo; } - /* - * For ExecInsertIndexTuples() to work on the partition's indexes - */ - estate->es_result_relation_info = resultRelInfo; - /* * If we're capturing transition tuples, we might need to convert * from the partition rowtype to root rowtype. @@ -3217,7 +3208,8 @@ CopyFrom(CopyState cstate) /* Compute stored generated columns */ if (resultRelInfo->ri_RelationDesc->rd_att->constr && resultRelInfo->ri_RelationDesc->rd_att->constr->has_generated_stored) - ExecComputeStoredGenerated(estate, myslot, CMD_INSERT); + ExecComputeStoredGenerated(resultRelInfo, estate, myslot, + CMD_INSERT); /* * If the target is a plain table, check the constraints of @@ -3288,7 +3280,8 @@ CopyFrom(CopyState cstate) myslot, mycid, ti_options, bistate); if (resultRelInfo->ri_NumIndices > 0) - recheckIndexes = ExecInsertIndexTuples(myslot, + recheckIndexes = ExecInsertIndexTuples(resultRelInfo, + myslot, estate, false, NULL, diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 80fedad5e045..511f015a861c 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -1820,7 +1820,6 @@ ExecuteTruncateGuts(List *explicit_rels, List *relids, List *relids_logged, resultRelInfo = resultRelInfos; foreach(cell, rels) { - estate->es_result_relation_info = resultRelInfo; ExecBSTruncateTriggers(estate, resultRelInfo); resultRelInfo++; } @@ -1950,7 +1949,6 @@ ExecuteTruncateGuts(List *explicit_rels, List *relids, List *relids_logged, resultRelInfo = resultRelInfos; foreach(cell, rels) { - estate->es_result_relation_info = resultRelInfo; ExecASTruncateTriggers(estate, resultRelInfo); resultRelInfo++; } diff --git a/src/backend/executor/execIndexing.c b/src/backend/executor/execIndexing.c index 1862af621be0..c6b5bcba7b47 100644 --- a/src/backend/executor/execIndexing.c +++ b/src/backend/executor/execIndexing.c @@ -270,7 +270,8 @@ ExecCloseIndices(ResultRelInfo *resultRelInfo) * ---------------------------------------------------------------- */ List * -ExecInsertIndexTuples(TupleTableSlot *slot, +ExecInsertIndexTuples(ResultRelInfo *resultRelInfo, + TupleTableSlot *slot, EState *estate, bool noDupErr, bool *specConflict, @@ -278,7 +279,6 @@ ExecInsertIndexTuples(TupleTableSlot *slot, { ItemPointer tupleid = &slot->tts_tid; List *result = NIL; - ResultRelInfo *resultRelInfo; int i; int numIndices; RelationPtr relationDescs; @@ -293,7 +293,6 @@ ExecInsertIndexTuples(TupleTableSlot *slot, /* * Get information from the result relation info structure. */ - resultRelInfo = estate->es_result_relation_info; numIndices = resultRelInfo->ri_NumIndices; relationDescs = resultRelInfo->ri_IndexRelationDescs; indexInfoArray = resultRelInfo->ri_IndexRelationInfo; @@ -479,11 +478,10 @@ ExecInsertIndexTuples(TupleTableSlot *slot, * ---------------------------------------------------------------- */ bool -ExecCheckIndexConstraints(TupleTableSlot *slot, +ExecCheckIndexConstraints(ResultRelInfo *resultRelInfo, TupleTableSlot *slot, EState *estate, ItemPointer conflictTid, List *arbiterIndexes) { - ResultRelInfo *resultRelInfo; int i; int numIndices; RelationPtr relationDescs; @@ -501,7 +499,6 @@ ExecCheckIndexConstraints(TupleTableSlot *slot, /* * Get information from the result relation info structure. */ - resultRelInfo = estate->es_result_relation_info; numIndices = resultRelInfo->ri_NumIndices; relationDescs = resultRelInfo->ri_IndexRelationDescs; indexInfoArray = resultRelInfo->ri_IndexRelationInfo; diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index 783eecbc1337..293f53d07c9d 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -827,9 +827,6 @@ InitPlan(QueryDesc *queryDesc, int eflags) estate->es_plannedstmt = plannedstmt; - /* es_result_relation_info is NULL except when within ModifyTable */ - estate->es_result_relation_info = NULL; - /* * Next, build the ExecRowMark array from the PlanRowMark(s), if any. */ @@ -2694,7 +2691,6 @@ EvalPlanQualStart(EPQState *epqstate, Plan *planTree) * subplans themselves are initialized. */ parentestate->es_result_relations = NULL; - /* es_result_relation_info must NOT be copied */ /* es_trig_target_relations must NOT be copied */ rcestate->es_top_eflags = parentestate->es_top_eflags; rcestate->es_instrument = parentestate->es_instrument; diff --git a/src/backend/executor/execReplication.c b/src/backend/executor/execReplication.c index b29db7bf4f95..01d26881e770 100644 --- a/src/backend/executor/execReplication.c +++ b/src/backend/executor/execReplication.c @@ -404,10 +404,10 @@ RelationFindReplTupleSeq(Relation rel, LockTupleMode lockmode, * Caller is responsible for opening the indexes. */ void -ExecSimpleRelationInsert(EState *estate, TupleTableSlot *slot) +ExecSimpleRelationInsert(ResultRelInfo *resultRelInfo, + EState *estate, TupleTableSlot *slot) { bool skip_tuple = false; - ResultRelInfo *resultRelInfo = estate->es_result_relation_info; Relation rel = resultRelInfo->ri_RelationDesc; /* For now we support only tables. */ @@ -430,7 +430,8 @@ ExecSimpleRelationInsert(EState *estate, TupleTableSlot *slot) /* Compute stored generated columns */ if (rel->rd_att->constr && rel->rd_att->constr->has_generated_stored) - ExecComputeStoredGenerated(estate, slot, CMD_INSERT); + ExecComputeStoredGenerated(resultRelInfo, estate, slot, + CMD_INSERT); /* Check the constraints of the tuple */ if (rel->rd_att->constr) @@ -442,7 +443,8 @@ ExecSimpleRelationInsert(EState *estate, TupleTableSlot *slot) simple_table_tuple_insert(resultRelInfo->ri_RelationDesc, slot); if (resultRelInfo->ri_NumIndices > 0) - recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL, + recheckIndexes = ExecInsertIndexTuples(resultRelInfo, + slot, estate, false, NULL, NIL); /* AFTER ROW INSERT Triggers */ @@ -466,11 +468,11 @@ ExecSimpleRelationInsert(EState *estate, TupleTableSlot *slot) * Caller is responsible for opening the indexes. */ void -ExecSimpleRelationUpdate(EState *estate, EPQState *epqstate, +ExecSimpleRelationUpdate(ResultRelInfo *resultRelInfo, + EState *estate, EPQState *epqstate, TupleTableSlot *searchslot, TupleTableSlot *slot) { bool skip_tuple = false; - ResultRelInfo *resultRelInfo = estate->es_result_relation_info; Relation rel = resultRelInfo->ri_RelationDesc; ItemPointer tid = &(searchslot->tts_tid); @@ -496,7 +498,8 @@ ExecSimpleRelationUpdate(EState *estate, EPQState *epqstate, /* Compute stored generated columns */ if (rel->rd_att->constr && rel->rd_att->constr->has_generated_stored) - ExecComputeStoredGenerated(estate, slot, CMD_UPDATE); + ExecComputeStoredGenerated(resultRelInfo, estate, slot, + CMD_UPDATE); /* Check the constraints of the tuple */ if (rel->rd_att->constr) @@ -508,7 +511,8 @@ ExecSimpleRelationUpdate(EState *estate, EPQState *epqstate, &update_indexes); if (resultRelInfo->ri_NumIndices > 0 && update_indexes) - recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL, + recheckIndexes = ExecInsertIndexTuples(resultRelInfo, + slot, estate, false, NULL, NIL); /* AFTER ROW UPDATE Triggers */ @@ -527,11 +531,11 @@ ExecSimpleRelationUpdate(EState *estate, EPQState *epqstate, * Caller is responsible for opening the indexes. */ void -ExecSimpleRelationDelete(EState *estate, EPQState *epqstate, +ExecSimpleRelationDelete(ResultRelInfo *resultRelInfo, + EState *estate, EPQState *epqstate, TupleTableSlot *searchslot) { bool skip_tuple = false; - ResultRelInfo *resultRelInfo = estate->es_result_relation_info; Relation rel = resultRelInfo->ri_RelationDesc; ItemPointer tid = &searchslot->tts_tid; diff --git a/src/backend/executor/execUtils.c b/src/backend/executor/execUtils.c index 6d8c112e2fe4..071a0007ebcd 100644 --- a/src/backend/executor/execUtils.c +++ b/src/backend/executor/execUtils.c @@ -125,7 +125,6 @@ CreateExecutorState(void) estate->es_result_relations = NULL; estate->es_opened_result_relations = NIL; - estate->es_result_relation_info = NULL; estate->es_tuple_routing_result_relations = NIL; estate->es_trig_target_relations = NIL; diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index b3f7012e3869..6782a2dcd283 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -70,7 +70,8 @@ static TupleTableSlot *ExecPrepareTupleRouting(ModifyTableState *mtstate, EState *estate, PartitionTupleRouting *proute, ResultRelInfo *targetRelInfo, - TupleTableSlot *slot); + TupleTableSlot *slot, + ResultRelInfo **partRelInfo); static ResultRelInfo *getTargetResultRelInfo(ModifyTableState *node); static void ExecSetupChildParentMapForSubplan(ModifyTableState *mtstate); static TupleConversionMap *tupconv_map_for_subplan(ModifyTableState *node, @@ -246,9 +247,10 @@ ExecCheckTIDVisible(EState *estate, * Compute stored generated columns for a tuple */ void -ExecComputeStoredGenerated(EState *estate, TupleTableSlot *slot, CmdType cmdtype) +ExecComputeStoredGenerated(ResultRelInfo *resultRelInfo, + EState *estate, TupleTableSlot *slot, + CmdType cmdtype) { - ResultRelInfo *resultRelInfo = estate->es_result_relation_info; Relation rel = resultRelInfo->ri_RelationDesc; TupleDesc tupdesc = RelationGetDescr(rel); int natts = tupdesc->natts; @@ -366,32 +368,48 @@ ExecComputeStoredGenerated(EState *estate, TupleTableSlot *slot, CmdType cmdtype * ExecInsert * * For INSERT, we have to insert the tuple into the target relation - * and insert appropriate tuples into the index relations. + * (or partition thereof) and insert appropriate tuples into the index + * relations. * * Returns RETURNING result if any, otherwise NULL. + * + * This may change the currently active tuple conversion map in + * mtstate->mt_transition_capture, so the callers must take care to + * save the previous value to avoid losing track of it. * ---------------------------------------------------------------- */ static TupleTableSlot * ExecInsert(ModifyTableState *mtstate, + ResultRelInfo *resultRelInfo, TupleTableSlot *slot, TupleTableSlot *planSlot, EState *estate, bool canSetTag) { - ResultRelInfo *resultRelInfo; Relation resultRelationDesc; List *recheckIndexes = NIL; TupleTableSlot *result = NULL; TransitionCaptureState *ar_insert_trig_tcs; ModifyTable *node = (ModifyTable *) mtstate->ps.plan; OnConflictAction onconflict = node->onConflictAction; - - ExecMaterializeSlot(slot); + PartitionTupleRouting *proute = mtstate->mt_partition_tuple_routing; /* - * get information on the (current) result relation + * If the input result relation is a partitioned table, find the leaf + * partition to insert the tuple into. */ - resultRelInfo = estate->es_result_relation_info; + if (proute) + { + ResultRelInfo *partRelInfo; + + slot = ExecPrepareTupleRouting(mtstate, estate, proute, + resultRelInfo, slot, + &partRelInfo); + resultRelInfo = partRelInfo; + } + + ExecMaterializeSlot(slot); + resultRelationDesc = resultRelInfo->ri_RelationDesc; /* @@ -424,7 +442,8 @@ ExecInsert(ModifyTableState *mtstate, */ if (resultRelationDesc->rd_att->constr && resultRelationDesc->rd_att->constr->has_generated_stored) - ExecComputeStoredGenerated(estate, slot, CMD_INSERT); + ExecComputeStoredGenerated(resultRelInfo, estate, slot, + CMD_INSERT); /* * insert into foreign table: let the FDW do it @@ -459,7 +478,8 @@ ExecInsert(ModifyTableState *mtstate, */ if (resultRelationDesc->rd_att->constr && resultRelationDesc->rd_att->constr->has_generated_stored) - ExecComputeStoredGenerated(estate, slot, CMD_INSERT); + ExecComputeStoredGenerated(resultRelInfo, estate, slot, + CMD_INSERT); /* * Check any RLS WITH CHECK policies. @@ -521,8 +541,8 @@ ExecInsert(ModifyTableState *mtstate, */ vlock: specConflict = false; - if (!ExecCheckIndexConstraints(slot, estate, &conflictTid, - arbiterIndexes)) + if (!ExecCheckIndexConstraints(resultRelInfo, slot, estate, + &conflictTid, arbiterIndexes)) { /* committed conflict tuple found */ if (onconflict == ONCONFLICT_UPDATE) @@ -582,7 +602,8 @@ ExecInsert(ModifyTableState *mtstate, specToken); /* insert index entries for tuple */ - recheckIndexes = ExecInsertIndexTuples(slot, estate, true, + recheckIndexes = ExecInsertIndexTuples(resultRelInfo, + slot, estate, true, &specConflict, arbiterIndexes); @@ -621,8 +642,9 @@ ExecInsert(ModifyTableState *mtstate, /* insert index entries for tuple */ if (resultRelInfo->ri_NumIndices > 0) - recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL, - NIL); + recheckIndexes = ExecInsertIndexTuples(resultRelInfo, + slot, estate, false, + NULL, NIL); } } @@ -707,6 +729,7 @@ ExecInsert(ModifyTableState *mtstate, */ static TupleTableSlot * ExecDelete(ModifyTableState *mtstate, + ResultRelInfo *resultRelInfo, ItemPointer tupleid, HeapTuple oldtuple, TupleTableSlot *planSlot, @@ -718,8 +741,7 @@ ExecDelete(ModifyTableState *mtstate, bool *tupleDeleted, TupleTableSlot **epqreturnslot) { - ResultRelInfo *resultRelInfo; - Relation resultRelationDesc; + Relation resultRelationDesc = resultRelInfo->ri_RelationDesc; TM_Result result; TM_FailureData tmfd; TupleTableSlot *slot = NULL; @@ -728,12 +750,6 @@ ExecDelete(ModifyTableState *mtstate, if (tupleDeleted) *tupleDeleted = false; - /* - * get information on the (current) result relation - */ - resultRelInfo = estate->es_result_relation_info; - resultRelationDesc = resultRelInfo->ri_RelationDesc; - /* BEFORE ROW DELETE Triggers */ if (resultRelInfo->ri_TrigDesc && resultRelInfo->ri_TrigDesc->trig_delete_before_row) @@ -1067,6 +1083,7 @@ ldelete:; */ static TupleTableSlot * ExecUpdate(ModifyTableState *mtstate, + ResultRelInfo *resultRelInfo, ItemPointer tupleid, HeapTuple oldtuple, TupleTableSlot *slot, @@ -1075,12 +1092,10 @@ ExecUpdate(ModifyTableState *mtstate, EState *estate, bool canSetTag) { - ResultRelInfo *resultRelInfo; - Relation resultRelationDesc; + Relation resultRelationDesc = resultRelInfo->ri_RelationDesc; TM_Result result; TM_FailureData tmfd; List *recheckIndexes = NIL; - TupleConversionMap *saved_tcs_map = NULL; /* * abort the operation if not running transactions @@ -1090,12 +1105,6 @@ ExecUpdate(ModifyTableState *mtstate, ExecMaterializeSlot(slot); - /* - * get information on the (current) result relation - */ - resultRelInfo = estate->es_result_relation_info; - resultRelationDesc = resultRelInfo->ri_RelationDesc; - /* BEFORE ROW UPDATE Triggers */ if (resultRelInfo->ri_TrigDesc && resultRelInfo->ri_TrigDesc->trig_update_before_row) @@ -1120,7 +1129,8 @@ ExecUpdate(ModifyTableState *mtstate, */ if (resultRelationDesc->rd_att->constr && resultRelationDesc->rd_att->constr->has_generated_stored) - ExecComputeStoredGenerated(estate, slot, CMD_UPDATE); + ExecComputeStoredGenerated(resultRelInfo, estate, slot, + CMD_UPDATE); /* * update in foreign table: let the FDW do it @@ -1157,7 +1167,8 @@ ExecUpdate(ModifyTableState *mtstate, */ if (resultRelationDesc->rd_att->constr && resultRelationDesc->rd_att->constr->has_generated_stored) - ExecComputeStoredGenerated(estate, slot, CMD_UPDATE); + ExecComputeStoredGenerated(resultRelInfo, estate, slot, + CMD_UPDATE); /* * Check any RLS UPDATE WITH CHECK policies @@ -1207,6 +1218,7 @@ lreplace:; PartitionTupleRouting *proute = mtstate->mt_partition_tuple_routing; int map_index; TupleConversionMap *tupconv_map; + TupleConversionMap *saved_tcs_map = NULL; /* * Disallow an INSERT ON CONFLICT DO UPDATE that causes the @@ -1232,9 +1244,12 @@ lreplace:; * Row movement, part 1. Delete the tuple, but skip RETURNING * processing. We want to return rows from INSERT. */ - ExecDelete(mtstate, tupleid, oldtuple, planSlot, epqstate, - estate, false, false /* canSetTag */ , - true /* changingPart */ , &tuple_deleted, &epqslot); + ExecDelete(mtstate, resultRelInfo, tupleid, oldtuple, planSlot, + epqstate, estate, + false, /* processReturning */ + false, /* canSetTag */ + true, /* changingPart */ + &tuple_deleted, &epqslot); /* * For some reason if DELETE didn't happen (e.g. trigger prevented @@ -1274,16 +1289,6 @@ lreplace:; } } - /* - * Updates set the transition capture map only when a new subplan - * is chosen. But for inserts, it is set for each row. So after - * INSERT, we need to revert back to the map created for UPDATE; - * otherwise the next UPDATE will incorrectly use the one created - * for INSERT. So first save the one created for UPDATE. - */ - if (mtstate->mt_transition_capture) - saved_tcs_map = mtstate->mt_transition_capture->tcs_map; - /* * resultRelInfo is one of the per-subplan resultRelInfos. So we * should convert the tuple into root's tuple descriptor, since @@ -1301,18 +1306,18 @@ lreplace:; mtstate->mt_root_tuple_slot); /* - * Prepare for tuple routing, making it look like we're inserting - * into the root. + * ExecInsert() may scribble on mtstate->mt_transition_capture, so + * save the currently active map. */ - Assert(mtstate->rootResultRelInfo != NULL); - slot = ExecPrepareTupleRouting(mtstate, estate, proute, - mtstate->rootResultRelInfo, slot); + if (mtstate->mt_transition_capture) + saved_tcs_map = mtstate->mt_transition_capture->tcs_map; - ret_slot = ExecInsert(mtstate, slot, planSlot, - estate, canSetTag); + /* Tuple routing starts from the root table. */ + Assert(mtstate->rootResultRelInfo != NULL); + ret_slot = ExecInsert(mtstate, mtstate->rootResultRelInfo, slot, + planSlot, estate, canSetTag); - /* Revert ExecPrepareTupleRouting's node change. */ - estate->es_result_relation_info = resultRelInfo; + /* Clear the INSERT's tuple and restore the saved map. */ if (mtstate->mt_transition_capture) { mtstate->mt_transition_capture->tcs_original_insert_tuple = NULL; @@ -1476,7 +1481,9 @@ lreplace:; /* insert index entries for tuple if necessary */ if (resultRelInfo->ri_NumIndices > 0 && update_indexes) - recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL, NIL); + recheckIndexes = ExecInsertIndexTuples(resultRelInfo, + slot, estate, false, + NULL, NIL); } if (canSetTag) @@ -1715,7 +1722,7 @@ ExecOnConflictUpdate(ModifyTableState *mtstate, */ /* Execute UPDATE with projection */ - *returning = ExecUpdate(mtstate, conflictTid, NULL, + *returning = ExecUpdate(mtstate, resultRelInfo, conflictTid, NULL, resultRelInfo->ri_onConflict->oc_ProjSlot, planSlot, &mtstate->mt_epqstate, mtstate->ps.state, @@ -1872,19 +1879,19 @@ ExecSetupTransitionCaptureState(ModifyTableState *mtstate, EState *estate) * ExecPrepareTupleRouting --- prepare for routing one tuple * * Determine the partition in which the tuple in slot is to be inserted, - * and modify mtstate and estate to prepare for it. - * - * Caller must revert the estate changes after executing the insertion! - * In mtstate, transition capture changes may also need to be reverted. + * and return its ResultRelInfo in *partRelInfo. The return value is + * a slot holding the tuple of the partition rowtype. * - * Returns a slot holding the tuple of the partition rowtype. + * This also sets the transition table information in mtstate based on the + * selected partition. */ static TupleTableSlot * ExecPrepareTupleRouting(ModifyTableState *mtstate, EState *estate, PartitionTupleRouting *proute, ResultRelInfo *targetRelInfo, - TupleTableSlot *slot) + TupleTableSlot *slot, + ResultRelInfo **partRelInfo) { ResultRelInfo *partrel; PartitionRoutingInfo *partrouteinfo; @@ -1901,11 +1908,6 @@ ExecPrepareTupleRouting(ModifyTableState *mtstate, partrouteinfo = partrel->ri_PartitionInfo; Assert(partrouteinfo != NULL); - /* - * Make it look like we are inserting into the partition. - */ - estate->es_result_relation_info = partrel; - /* * If we're capturing transition tuples, we might need to convert from the * partition rowtype to root partitioned table's rowtype. @@ -1950,6 +1952,7 @@ ExecPrepareTupleRouting(ModifyTableState *mtstate, slot = execute_attr_map_slot(map->attrMap, slot, new_slot); } + *partRelInfo = partrel; return slot; } @@ -2016,10 +2019,8 @@ static TupleTableSlot * ExecModifyTable(PlanState *pstate) { ModifyTableState *node = castNode(ModifyTableState, pstate); - PartitionTupleRouting *proute = node->mt_partition_tuple_routing; EState *estate = node->ps.state; CmdType operation = node->operation; - ResultRelInfo *saved_resultRelInfo; ResultRelInfo *resultRelInfo; PlanState *subplanstate; JunkFilter *junkfilter; @@ -2067,17 +2068,6 @@ ExecModifyTable(PlanState *pstate) subplanstate = node->mt_plans[node->mt_whichplan]; junkfilter = resultRelInfo->ri_junkFilter; - /* - * es_result_relation_info must point to the currently active result - * relation while we are within this ModifyTable node. Even though - * ModifyTable nodes can't be nested statically, they can be nested - * dynamically (since our subplan could include a reference to a modifying - * CTE). So we have to save and restore the caller's value. - */ - saved_resultRelInfo = estate->es_result_relation_info; - - estate->es_result_relation_info = resultRelInfo; - /* * Fetch rows from subplan(s), and execute the required table modification * for each row. @@ -2111,7 +2101,6 @@ ExecModifyTable(PlanState *pstate) resultRelInfo++; subplanstate = node->mt_plans[node->mt_whichplan]; junkfilter = resultRelInfo->ri_junkFilter; - estate->es_result_relation_info = resultRelInfo; EvalPlanQualSetPlan(&node->mt_epqstate, subplanstate->plan, node->mt_arowmarks[node->mt_whichplan]); /* Prepare to convert transition tuples from this child. */ @@ -2156,7 +2145,6 @@ ExecModifyTable(PlanState *pstate) */ slot = ExecProcessReturning(resultRelInfo, NULL, planSlot); - estate->es_result_relation_info = saved_resultRelInfo; return slot; } @@ -2239,25 +2227,21 @@ ExecModifyTable(PlanState *pstate) switch (operation) { case CMD_INSERT: - /* Prepare for tuple routing if needed. */ - if (proute) - slot = ExecPrepareTupleRouting(node, estate, proute, - resultRelInfo, slot); - slot = ExecInsert(node, slot, planSlot, + slot = ExecInsert(node, resultRelInfo, slot, planSlot, estate, node->canSetTag); - /* Revert ExecPrepareTupleRouting's state change. */ - if (proute) - estate->es_result_relation_info = resultRelInfo; break; case CMD_UPDATE: - slot = ExecUpdate(node, tupleid, oldtuple, slot, planSlot, - &node->mt_epqstate, estate, node->canSetTag); + slot = ExecUpdate(node, resultRelInfo, tupleid, oldtuple, slot, + planSlot, &node->mt_epqstate, estate, + node->canSetTag); break; case CMD_DELETE: - slot = ExecDelete(node, tupleid, oldtuple, planSlot, - &node->mt_epqstate, estate, - true, node->canSetTag, - false /* changingPart */ , NULL, NULL); + slot = ExecDelete(node, resultRelInfo, tupleid, oldtuple, + planSlot, &node->mt_epqstate, estate, + true, /* processReturning */ + node->canSetTag, + false, /* changingPart */ + NULL, NULL); break; default: elog(ERROR, "unknown operation"); @@ -2269,15 +2253,9 @@ ExecModifyTable(PlanState *pstate) * the work on next call. */ if (slot) - { - estate->es_result_relation_info = saved_resultRelInfo; return slot; - } } - /* Restore es_result_relation_info before exiting */ - estate->es_result_relation_info = saved_resultRelInfo; - /* * We're done, but fire AFTER STATEMENT triggers before exiting. */ @@ -2298,7 +2276,6 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) ModifyTableState *mtstate; CmdType operation = node->operation; int nplans = list_length(node->plans); - ResultRelInfo *saved_resultRelInfo; ResultRelInfo *resultRelInfo; Plan *subplan; ListCell *l, @@ -2346,14 +2323,8 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) * call ExecInitNode on each of the plans to be executed and save the * results into the array "mt_plans". This is also a convenient place to * verify that the proposed target relations are valid and open their - * indexes for insertion of new index entries. Note we *must* set - * estate->es_result_relation_info correctly while we initialize each - * sub-plan; external modules such as FDWs may depend on that (see - * contrib/postgres_fdw/postgres_fdw.c: postgresBeginDirectModify() as one - * example). + * indexes for insertion of new index entries. */ - saved_resultRelInfo = estate->es_result_relation_info; - resultRelInfo = mtstate->resultRelInfo; i = 0; forboth(l, node->resultRelations, l1, node->plans) @@ -2400,7 +2371,6 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) update_tuple_routing_needed = true; /* Now init the plan for this result rel */ - estate->es_result_relation_info = resultRelInfo; mtstate->mt_plans[i] = ExecInitNode(subplan, estate, eflags); mtstate->mt_scans[i] = ExecInitExtraTupleSlot(mtstate->ps.state, ExecGetResultType(mtstate->mt_plans[i]), @@ -2424,8 +2394,6 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) i++; } - estate->es_result_relation_info = saved_resultRelInfo; - /* Get the target relation */ rel = (getTargetResultRelInfo(mtstate))->ri_RelationDesc; diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 8d5d9e05b3c0..4f32dc74c867 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -1174,7 +1174,6 @@ apply_handle_insert(StringInfo s) &TTSOpsVirtual); resultRelInfo = makeNode(ResultRelInfo); InitResultRelInfo(resultRelInfo, rel->localrel, 1, NULL, 0); - estate->es_result_relation_info = resultRelInfo; /* Input functions may need an active snapshot, so get one */ PushActiveSnapshot(GetTransactionSnapshot()); @@ -1214,7 +1213,7 @@ apply_handle_insert_internal(ResultRelInfo *relinfo, ExecOpenIndices(relinfo, false); /* Do the insert. */ - ExecSimpleRelationInsert(estate, remoteslot); + ExecSimpleRelationInsert(relinfo, estate, remoteslot); /* Cleanup. */ ExecCloseIndices(relinfo); @@ -1300,7 +1299,6 @@ apply_handle_update(StringInfo s) &TTSOpsVirtual); resultRelInfo = makeNode(ResultRelInfo); InitResultRelInfo(resultRelInfo, rel->localrel, 1, NULL, 0); - estate->es_result_relation_info = resultRelInfo; /* * Populate updatedCols so that per-column triggers can fire. This could @@ -1392,7 +1390,8 @@ apply_handle_update_internal(ResultRelInfo *relinfo, EvalPlanQualSetSlot(&epqstate, remoteslot); /* Do the actual update. */ - ExecSimpleRelationUpdate(estate, &epqstate, localslot, remoteslot); + ExecSimpleRelationUpdate(relinfo, estate, &epqstate, localslot, + remoteslot); } else { @@ -1455,7 +1454,6 @@ apply_handle_delete(StringInfo s) &TTSOpsVirtual); resultRelInfo = makeNode(ResultRelInfo); InitResultRelInfo(resultRelInfo, rel->localrel, 1, NULL, 0); - estate->es_result_relation_info = resultRelInfo; PushActiveSnapshot(GetTransactionSnapshot()); @@ -1508,7 +1506,7 @@ apply_handle_delete_internal(ResultRelInfo *relinfo, EState *estate, EvalPlanQualSetSlot(&epqstate, localslot); /* Do the actual delete. */ - ExecSimpleRelationDelete(estate, &epqstate, localslot); + ExecSimpleRelationDelete(relinfo, estate, &epqstate, localslot); } else { @@ -1616,7 +1614,6 @@ apply_handle_tuple_routing(ResultRelInfo *relinfo, } MemoryContextSwitchTo(oldctx); - estate->es_result_relation_info = partrelinfo; switch (operation) { case CMD_INSERT: @@ -1697,8 +1694,8 @@ apply_handle_tuple_routing(ResultRelInfo *relinfo, ExecOpenIndices(partrelinfo, false); EvalPlanQualSetSlot(&epqstate, remoteslot_part); - ExecSimpleRelationUpdate(estate, &epqstate, localslot, - remoteslot_part); + ExecSimpleRelationUpdate(partrelinfo, estate, &epqstate, + localslot, remoteslot_part); ExecCloseIndices(partrelinfo); EvalPlanQualEnd(&epqstate); } @@ -1739,7 +1736,6 @@ apply_handle_tuple_routing(ResultRelInfo *relinfo, Assert(partrelinfo_new != partrelinfo); /* DELETE old tuple found in the old partition. */ - estate->es_result_relation_info = partrelinfo; apply_handle_delete_internal(partrelinfo, estate, localslot, &relmapentry->remoterel); @@ -1771,7 +1767,6 @@ apply_handle_tuple_routing(ResultRelInfo *relinfo, slot_getallattrs(remoteslot); } MemoryContextSwitchTo(oldctx); - estate->es_result_relation_info = partrelinfo_new; apply_handle_insert_internal(partrelinfo_new, estate, remoteslot_part); } diff --git a/src/include/executor/executor.h b/src/include/executor/executor.h index c283bf14541c..b7978cd22ebc 100644 --- a/src/include/executor/executor.h +++ b/src/include/executor/executor.h @@ -576,10 +576,14 @@ extern TupleTableSlot *ExecGetReturningSlot(EState *estate, ResultRelInfo *relIn */ extern void ExecOpenIndices(ResultRelInfo *resultRelInfo, bool speculative); extern void ExecCloseIndices(ResultRelInfo *resultRelInfo); -extern List *ExecInsertIndexTuples(TupleTableSlot *slot, EState *estate, bool noDupErr, +extern List *ExecInsertIndexTuples(ResultRelInfo *resultRelInfo, + TupleTableSlot *slot, EState *estate, + bool noDupErr, bool *specConflict, List *arbiterIndexes); -extern bool ExecCheckIndexConstraints(TupleTableSlot *slot, EState *estate, - ItemPointer conflictTid, List *arbiterIndexes); +extern bool ExecCheckIndexConstraints(ResultRelInfo *resultRelInfo, + TupleTableSlot *slot, + EState *estate, ItemPointer conflictTid, + List *arbiterIndexes); extern void check_exclusion_constraint(Relation heap, Relation index, IndexInfo *indexInfo, ItemPointer tupleid, @@ -596,10 +600,13 @@ extern bool RelationFindReplTupleByIndex(Relation rel, Oid idxoid, extern bool RelationFindReplTupleSeq(Relation rel, LockTupleMode lockmode, TupleTableSlot *searchslot, TupleTableSlot *outslot); -extern void ExecSimpleRelationInsert(EState *estate, TupleTableSlot *slot); -extern void ExecSimpleRelationUpdate(EState *estate, EPQState *epqstate, +extern void ExecSimpleRelationInsert(ResultRelInfo *resultRelInfo, + EState *estate, TupleTableSlot *slot); +extern void ExecSimpleRelationUpdate(ResultRelInfo *resultRelInfo, + EState *estate, EPQState *epqstate, TupleTableSlot *searchslot, TupleTableSlot *slot); -extern void ExecSimpleRelationDelete(EState *estate, EPQState *epqstate, +extern void ExecSimpleRelationDelete(ResultRelInfo *resultRelInfo, + EState *estate, EPQState *epqstate, TupleTableSlot *searchslot); extern void CheckCmdReplicaIdentity(Relation rel, CmdType cmd); diff --git a/src/include/executor/nodeModifyTable.h b/src/include/executor/nodeModifyTable.h index 4ec4ebdabc13..46a2dc951188 100644 --- a/src/include/executor/nodeModifyTable.h +++ b/src/include/executor/nodeModifyTable.h @@ -15,7 +15,9 @@ #include "nodes/execnodes.h" -extern void ExecComputeStoredGenerated(EState *estate, TupleTableSlot *slot, CmdType cmdtype); +extern void ExecComputeStoredGenerated(ResultRelInfo *resultRelInfo, + EState *estate, TupleTableSlot *slot, + CmdType cmdtype); extern ModifyTableState *ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags); extern void ExecEndModifyTable(ModifyTableState *node); diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index d9b09c592068..b7e9e5d539de 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -525,7 +525,6 @@ typedef struct EState List *es_opened_result_relations; /* List of non-NULL entries in * es_result_relations in no * specific order */ - ResultRelInfo *es_result_relation_info; /* currently active array elt */ PartitionDirectory es_partition_directory; /* for PartitionDesc lookup */ diff --git a/src/test/regress/expected/insert.out b/src/test/regress/expected/insert.out index eb9d45be5e5f..da50ee3b670a 100644 --- a/src/test/regress/expected/insert.out +++ b/src/test/regress/expected/insert.out @@ -818,9 +818,7 @@ drop role regress_coldesc_role; drop table inserttest3; drop table brtrigpartcon; drop function brtrigpartcon1trigf(); --- check that "do nothing" BR triggers work with tuple-routing (this checks --- that estate->es_result_relation_info is appropriately set/reset for each --- routed tuple) +-- check that "do nothing" BR triggers work with tuple-routing create table donothingbrtrig_test (a int, b text) partition by list (a); create table donothingbrtrig_test1 (b text, a int); create table donothingbrtrig_test2 (c text, b text, a int); diff --git a/src/test/regress/sql/insert.sql b/src/test/regress/sql/insert.sql index ffd4aacbc48b..963faa1614c6 100644 --- a/src/test/regress/sql/insert.sql +++ b/src/test/regress/sql/insert.sql @@ -542,9 +542,7 @@ drop table inserttest3; drop table brtrigpartcon; drop function brtrigpartcon1trigf(); --- check that "do nothing" BR triggers work with tuple-routing (this checks --- that estate->es_result_relation_info is appropriately set/reset for each --- routed tuple) +-- check that "do nothing" BR triggers work with tuple-routing create table donothingbrtrig_test (a int, b text) partition by list (a); create table donothingbrtrig_test1 (b text, a int); create table donothingbrtrig_test2 (c text, b text, a int); From b94109ce375b137f235149bfba3559c69f4573e7 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Thu, 15 Oct 2020 11:31:20 +1300 Subject: [PATCH 298/589] Make WL_POSTMASTER_DEATH level-triggered on kqueue builds. If WaitEventSetWait() reports that the postmaster has gone away, later calls to WaitEventSetWait() should continue to report that. Otherwise further waits that occur in the proc_exit() path after we already noticed the postmaster's demise could block forever. Back-patch to 13, where the kqueue support landed. Reported-by: Tom Lane Discussion: https://postgr.es/m/3624029.1602701929%40sss.pgh.pa.us --- src/backend/storage/ipc/latch.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/backend/storage/ipc/latch.c b/src/backend/storage/ipc/latch.c index 63c6c9753601..eacb8c3c9ee9 100644 --- a/src/backend/storage/ipc/latch.c +++ b/src/backend/storage/ipc/latch.c @@ -1492,7 +1492,10 @@ WaitEventSetWaitBlock(WaitEventSet *set, int cur_timeout, timeout_p = &timeout; } - /* Report events discovered by WaitEventAdjustKqueue(). */ + /* + * Report postmaster events discovered by WaitEventAdjustKqueue() or an + * earlier call to WaitEventSetWait(). + */ if (unlikely(set->report_postmaster_not_running)) { if (set->exit_on_postmaster_death) @@ -1563,6 +1566,13 @@ WaitEventSetWaitBlock(WaitEventSet *set, int cur_timeout, cur_kqueue_event->filter == EVFILT_PROC && (cur_kqueue_event->fflags & NOTE_EXIT) != 0) { + /* + * The kernel will tell this kqueue object only once about the exit + * of the postmaster, so let's remember that for next time so that + * we provide level-triggered semantics. + */ + set->report_postmaster_not_running = true; + if (set->exit_on_postmaster_death) proc_exit(1); occurred_events->fd = PGINVALID_SOCKET; From 4e9821b6fac5042e872d5397f711a67984b165f8 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Wed, 14 Oct 2020 20:12:26 -0300 Subject: [PATCH 299/589] Restore replication protocol's duplicate command tags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I removed the duplicate command tags for START_REPLICATION inadvertently in commit 07082b08cc5d, but the replication protocol requires them. The fact that the replication protocol was broken was not noticed because all our test cases use an optimized code path that exits early, failing to verify that the behavior is correct for non-optimized cases. Put them back. Also document this protocol quirk. Add a test case that shows the failure. It might still succeed even without the patch when run on a fast enough server, but it suffices to show the bug in enough cases that it would be noticed in buildfarm. Author: Álvaro Herrera Reported-by: Henry Hinze Reviewed-by: Petr Jelínek Discussion: https://postgr.es/m/16643-eaadeb2a1a58d28c@postgresql.org --- doc/src/sgml/protocol.sgml | 8 ++-- src/backend/replication/logical/worker.c | 1 - src/backend/replication/walsender.c | 3 +- src/test/subscription/t/100_bugs.pl | 55 +++++++++++++++++++++++- 4 files changed, 61 insertions(+), 6 deletions(-) diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index f5e33181061b..5e06c7523d8e 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2059,8 +2059,9 @@ The commands accepted in replication mode are: the switch position is the end of the WAL that was streamed, but there are corner cases where the server can send some WAL from the old timeline that it has not itself replayed before promoting. Finally, the - server sends CommandComplete message, and is ready to accept a new - command. + server sends two CommandComplete messages (one that ends the CopyData + and the other ends the START_REPLICATION itself), and + is ready to accept a new command. @@ -2382,7 +2383,8 @@ The commands accepted in replication mode are: The messages inside the CopyBothResponse messages are of the same format - documented for START_REPLICATION ... PHYSICAL. + documented for START_REPLICATION ... PHYSICAL, including + two CommandComplete messages. diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 4f32dc74c867..640409b757f8 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -3071,7 +3071,6 @@ ApplyWorkerMain(Datum main_arg) * does some initializations on the upstream so let's still call it. */ (void) walrcv_identify_system(wrconn, &startpointTLI); - } /* diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 7c9d1b67dfbd..df27e8476175 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1656,7 +1656,8 @@ exec_replication_command(const char *cmd_string) else StartLogicalReplication(cmd); - /* callees already sent their own completion message */ + /* dupe, but necessary per libpqrcv_endstreaming */ + EndReplicationCommand(cmdtag); Assert(xlogreader != NULL); break; diff --git a/src/test/subscription/t/100_bugs.pl b/src/test/subscription/t/100_bugs.pl index 366a7a94350a..7dc8983d6aee 100644 --- a/src/test/subscription/t/100_bugs.pl +++ b/src/test/subscription/t/100_bugs.pl @@ -3,7 +3,7 @@ use warnings; use PostgresNode; use TestLib; -use Test::More tests => 3; +use Test::More tests => 5; # Bug #15114 @@ -100,3 +100,56 @@ ); $node_publisher->stop('fast'); + +# Bug #16643 - https://postgr.es/m/16643-eaadeb2a1a58d28c@postgresql.org +# +# Initial sync doesn't complete; the protocol was not being followed per +# expectations after commit 07082b08cc5d. +my $node_twoways = get_new_node('twoways'); +$node_twoways->init(allows_streaming => 'logical'); +$node_twoways->start; +for my $db (qw(d1 d2)) +{ + $node_twoways->safe_psql('postgres', "CREATE DATABASE $db"); + $node_twoways->safe_psql($db, "CREATE TABLE t (f int)"); + $node_twoways->safe_psql($db, "CREATE TABLE t2 (f int)"); +} + +my $rows = 3000; +$node_twoways->safe_psql( + 'd1', qq{ + INSERT INTO t SELECT * FROM generate_series(1, $rows); + INSERT INTO t2 SELECT * FROM generate_series(1, $rows); + CREATE PUBLICATION testpub FOR TABLE t; + SELECT pg_create_logical_replication_slot('testslot', 'pgoutput'); + }); + +$node_twoways->safe_psql('d2', + "CREATE SUBSCRIPTION testsub CONNECTION \$\$" + . $node_twoways->connstr('d1') + . "\$\$ PUBLICATION testpub WITH (create_slot=false, " + . "slot_name='testslot')"); +$node_twoways->safe_psql( + 'd1', qq{ + INSERT INTO t SELECT * FROM generate_series(1, $rows); + INSERT INTO t2 SELECT * FROM generate_series(1, $rows); + }); +$node_twoways->safe_psql( + 'd1', 'ALTER PUBLICATION testpub ADD TABLE t2'); +$node_twoways->safe_psql( + 'd2', 'ALTER SUBSCRIPTION testsub REFRESH PUBLICATION'); + +# We cannot rely solely on wait_for_catchup() here; it isn't sufficient +# when tablesync workers might still be running. So in addition to that, +# we verify that no tablesync workers appear for the subscription. +# XXX maybe this should be integrated in wait_for_catchup() itself. +$node_twoways->wait_for_catchup('testsub'); +$node_twoways->poll_query_until( + 'd2', + "SELECT count(*) FROM pg_stat_subscription WHERE subname = 'testsub' AND relid <> 0", + "0"); + +is($node_twoways->safe_psql('d2', "SELECT count(f) FROM t"), + $rows * 2, "2x$rows rows in t"); +is($node_twoways->safe_psql('d2', "SELECT count(f) FROM t2"), + $rows * 2, "2x$rows rows in t2"); From 564a410c81b4a9d289c948690a3135d0cd25411b Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Thu, 15 Oct 2020 11:04:07 +0900 Subject: [PATCH 300/589] doc: Mention that toast_tuple_target affects also column marked as Main. Previously it was documented that toast_tuple_target affected column marked as only External or Extended. But this description is not correct and toast_tuple_target affects also column marked as Main. Back-patch to v11 where toast_tuple_target reloption was introduced. Author: Shinya Okano Reviewed-by: Tatsuhito Kasahara, Fujii Masao Discussion: https://postgr.es/m/93f46e311a67422e89e770d236059817@oss.nttdata.com --- doc/src/sgml/ref/create_table.sgml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index 28f844071b06..fd6777ae0192 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -1359,10 +1359,11 @@ WITH ( MODULUS numeric_literal, REM The toast_tuple_target specifies the minimum tuple length required before - we try to move long column values into TOAST tables, and is also the - target length we try to reduce the length below once toasting begins. - This only affects columns marked as either External or Extended - and applies only to new tuples; there is no effect on existing rows. + we try to compress and/or move long column values into TOAST tables, and + is also the target length we try to reduce the length below once toasting + begins. This affects columns marked as External (for move), + Main (for compression), or Extended (for both) and applies only to new + tuples. There is no effect on existing rows. By default this parameter is set to allow at least 4 tuples per block, which with the default block size will be 2040 bytes. Valid values are between 128 bytes and the (block size - header), by default 8160 bytes. From d7eb52d7181d83cf2363570f7a205b8eb1008dbc Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Thu, 15 Oct 2020 08:17:51 +0530 Subject: [PATCH 301/589] Execute invalidation messages for each XLOG_XACT_INVALIDATIONS message during logical decoding. Prior to commit c55040ccd0 we have no way of knowing the invalidations before commit. So, while decoding we use to execute all the invalidations at each command end as we had no way of knowing which invalidations happened before that command. Due to this, transactions involving large amounts of DDLs use to take more time and also lead to high CPU usage. But now we know specific invalidations at each command end so we execute only required invalidations. It has been observed that decoding of a transaction containing truncation of a table with 1000 partitions would be finished in 1s whereas before this patch it used to take 4-5 minutes. Author: Dilip Kumar Reviewed-by: Amit Kapila and Keisuke Kuroda Discussion: https://postgr.es/m/CANDwggKYveEtXjXjqHA6RL3AKSHMsQyfRY6bK+NqhAWJyw8psQ@mail.gmail.com --- .../replication/logical/reorderbuffer.c | 103 ++++++++++++++---- src/include/replication/reorderbuffer.h | 12 +- 2 files changed, 92 insertions(+), 23 deletions(-) diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index 4cb27f222445..7a8bf760791c 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -235,7 +235,7 @@ static void ReorderBufferIterTXNInit(ReorderBuffer *rb, ReorderBufferTXN *txn, static ReorderBufferChange *ReorderBufferIterTXNNext(ReorderBuffer *rb, ReorderBufferIterTXNState *state); static void ReorderBufferIterTXNFinish(ReorderBuffer *rb, ReorderBufferIterTXNState *state); -static void ReorderBufferExecuteInvalidations(ReorderBuffer *rb, ReorderBufferTXN *txn); +static void ReorderBufferExecuteInvalidations(uint32 nmsgs, SharedInvalidationMessage *msgs); /* * --------------------------------------- @@ -486,6 +486,11 @@ ReorderBufferReturnChange(ReorderBuffer *rb, ReorderBufferChange *change, pfree(change->data.msg.message); change->data.msg.message = NULL; break; + case REORDER_BUFFER_CHANGE_INVALIDATION: + if (change->data.inval.invalidations) + pfree(change->data.inval.invalidations); + change->data.inval.invalidations = NULL; + break; case REORDER_BUFFER_CHANGE_INTERNAL_SNAPSHOT: if (change->data.snapshot) { @@ -2194,6 +2199,13 @@ ReorderBufferProcessTXN(ReorderBuffer *rb, ReorderBufferTXN *txn, ReorderBufferApplyMessage(rb, txn, change, streaming); break; + case REORDER_BUFFER_CHANGE_INVALIDATION: + /* Execute the invalidation messages locally */ + ReorderBufferExecuteInvalidations( + change->data.inval.ninvalidations, + change->data.inval.invalidations); + break; + case REORDER_BUFFER_CHANGE_INTERNAL_SNAPSHOT: /* get rid of the old */ TeardownHistoricSnapshot(false); @@ -2244,13 +2256,6 @@ ReorderBufferProcessTXN(ReorderBuffer *rb, ReorderBufferTXN *txn, TeardownHistoricSnapshot(false); SetupHistoricSnapshot(snapshot_now, txn->tuplecid_hash); - - /* - * Every time the CommandId is incremented, we could - * see new catalog contents, so execute all - * invalidations. - */ - ReorderBufferExecuteInvalidations(rb, txn); } break; @@ -2317,7 +2322,7 @@ ReorderBufferProcessTXN(ReorderBuffer *rb, ReorderBufferTXN *txn, AbortCurrentTransaction(); /* make sure there's no cache pollution */ - ReorderBufferExecuteInvalidations(rb, txn); + ReorderBufferExecuteInvalidations(txn->ninvalidations, txn->invalidations); if (using_subtxn) RollbackAndReleaseCurrentSubTransaction(); @@ -2356,7 +2361,8 @@ ReorderBufferProcessTXN(ReorderBuffer *rb, ReorderBufferTXN *txn, AbortCurrentTransaction(); /* make sure there's no cache pollution */ - ReorderBufferExecuteInvalidations(rb, txn); + ReorderBufferExecuteInvalidations(txn->ninvalidations, + txn->invalidations); if (using_subtxn) RollbackAndReleaseCurrentSubTransaction(); @@ -2813,10 +2819,13 @@ ReorderBufferAddNewTupleCids(ReorderBuffer *rb, TransactionId xid, * Setup the invalidation of the toplevel transaction. * * This needs to be called for each XLOG_XACT_INVALIDATIONS message and - * accumulates all the invalidation messages in the toplevel transaction. - * This is required because in some cases where we skip processing the - * transaction (see ReorderBufferForget), we need to execute all the - * invalidations together. + * accumulates all the invalidation messages in the toplevel transaction as + * well as in the form of change in reorder buffer. We require to record it in + * form of the change so that we can execute only the required invalidations + * instead of executing all the invalidations on each CommandId increment. We + * also need to accumulate these in the toplevel transaction because in some + * cases we skip processing the transaction (see ReorderBufferForget), we need + * to execute all the invalidations together. */ void ReorderBufferAddInvalidations(ReorderBuffer *rb, TransactionId xid, @@ -2824,12 +2833,16 @@ ReorderBufferAddInvalidations(ReorderBuffer *rb, TransactionId xid, SharedInvalidationMessage *msgs) { ReorderBufferTXN *txn; + MemoryContext oldcontext; + ReorderBufferChange *change; txn = ReorderBufferTXNByXid(rb, xid, true, NULL, lsn, true); + oldcontext = MemoryContextSwitchTo(rb->context); + /* - * We collect all the invalidations under the top transaction so that we - * can execute them all together. + * Collect all the invalidations under the top transaction so that we can + * execute them all together. See comment atop this function */ if (txn->toptxn) txn = txn->toptxn; @@ -2841,8 +2854,7 @@ ReorderBufferAddInvalidations(ReorderBuffer *rb, TransactionId xid, { txn->ninvalidations = nmsgs; txn->invalidations = (SharedInvalidationMessage *) - MemoryContextAlloc(rb->context, - sizeof(SharedInvalidationMessage) * nmsgs); + palloc(sizeof(SharedInvalidationMessage) * nmsgs); memcpy(txn->invalidations, msgs, sizeof(SharedInvalidationMessage) * nmsgs); } @@ -2856,6 +2868,18 @@ ReorderBufferAddInvalidations(ReorderBuffer *rb, TransactionId xid, nmsgs * sizeof(SharedInvalidationMessage)); txn->ninvalidations += nmsgs; } + + change = ReorderBufferGetChange(rb); + change->action = REORDER_BUFFER_CHANGE_INVALIDATION; + change->data.inval.ninvalidations = nmsgs; + change->data.inval.invalidations = (SharedInvalidationMessage *) + palloc(sizeof(SharedInvalidationMessage) * nmsgs); + memcpy(change->data.inval.invalidations, msgs, + sizeof(SharedInvalidationMessage) * nmsgs); + + ReorderBufferQueueChange(rb, xid, lsn, change, false); + + MemoryContextSwitchTo(oldcontext); } /* @@ -2863,12 +2887,12 @@ ReorderBufferAddInvalidations(ReorderBuffer *rb, TransactionId xid, * in the changestream but we don't know which those are. */ static void -ReorderBufferExecuteInvalidations(ReorderBuffer *rb, ReorderBufferTXN *txn) +ReorderBufferExecuteInvalidations(uint32 nmsgs, SharedInvalidationMessage *msgs) { int i; - for (i = 0; i < txn->ninvalidations; i++) - LocalExecuteInvalidationMessage(&txn->invalidations[i]); + for (i = 0; i < nmsgs; i++) + LocalExecuteInvalidationMessage(&msgs[i]); } /* @@ -3301,6 +3325,24 @@ ReorderBufferSerializeChange(ReorderBuffer *rb, ReorderBufferTXN *txn, change->data.msg.message_size); data += change->data.msg.message_size; + break; + } + case REORDER_BUFFER_CHANGE_INVALIDATION: + { + char *data; + Size inval_size = sizeof(SharedInvalidationMessage) * + change->data.inval.ninvalidations; + + sz += inval_size; + + ReorderBufferSerializeReserve(rb, sz); + data = ((char *) rb->outbuf) + sizeof(ReorderBufferDiskChange); + + /* might have been reallocated above */ + ondisk = (ReorderBufferDiskChange *) rb->outbuf; + memcpy(data, change->data.inval.invalidations, inval_size); + data += inval_size; + break; } case REORDER_BUFFER_CHANGE_INTERNAL_SNAPSHOT: @@ -3578,6 +3620,12 @@ ReorderBufferChangeSize(ReorderBufferChange *change) break; } + case REORDER_BUFFER_CHANGE_INVALIDATION: + { + sz += sizeof(SharedInvalidationMessage) * + change->data.inval.ninvalidations; + break; + } case REORDER_BUFFER_CHANGE_INTERNAL_SNAPSHOT: { Snapshot snap; @@ -3844,6 +3892,19 @@ ReorderBufferRestoreChange(ReorderBuffer *rb, ReorderBufferTXN *txn, change->data.msg.message_size); data += change->data.msg.message_size; + break; + } + case REORDER_BUFFER_CHANGE_INVALIDATION: + { + Size inval_size = sizeof(SharedInvalidationMessage) * + change->data.inval.ninvalidations; + + change->data.inval.invalidations = + MemoryContextAlloc(rb->context, inval_size); + + /* read the message */ + memcpy(change->data.inval.invalidations, data, inval_size); + break; } case REORDER_BUFFER_CHANGE_INTERNAL_SNAPSHOT: diff --git a/src/include/replication/reorderbuffer.h b/src/include/replication/reorderbuffer.h index 0cc3aebb1113..1c77819aad25 100644 --- a/src/include/replication/reorderbuffer.h +++ b/src/include/replication/reorderbuffer.h @@ -57,6 +57,7 @@ enum ReorderBufferChangeType REORDER_BUFFER_CHANGE_UPDATE, REORDER_BUFFER_CHANGE_DELETE, REORDER_BUFFER_CHANGE_MESSAGE, + REORDER_BUFFER_CHANGE_INVALIDATION, REORDER_BUFFER_CHANGE_INTERNAL_SNAPSHOT, REORDER_BUFFER_CHANGE_INTERNAL_COMMAND_ID, REORDER_BUFFER_CHANGE_INTERNAL_TUPLECID, @@ -149,6 +150,13 @@ typedef struct ReorderBufferChange CommandId cmax; CommandId combocid; } tuplecid; + + /* Invalidation. */ + struct + { + uint32 ninvalidations; /* Number of messages */ + SharedInvalidationMessage *invalidations; /* invalidation message */ + } inval; } data; /* @@ -313,8 +321,8 @@ typedef struct ReorderBufferTXN uint64 nentries_mem; /* - * List of ReorderBufferChange structs, including new Snapshots and new - * CommandIds + * List of ReorderBufferChange structs, including new Snapshots, new + * CommandIds and command invalidation messages. */ dlist_head changes; From 70516a178ad0fc54d0f0d2a88175af3e1e92f83f Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Thu, 15 Oct 2020 18:23:30 +1300 Subject: [PATCH 302/589] Handle EACCES errors from kevent() better. While registering for postmaster exit events, we have to handle a couple of edge cases where the postmaster is already gone. Commit 815c2f09 missed one: EACCES must surely imply that PostmasterPid no longer belongs to our postmaster process (or alternatively an unexpected permissions model has been imposed on us). Like ESRCH, this should be treated as a WL_POSTMASTER_DEATH event, rather than being raised with ereport(). No known problems reported in the wild. Per code review from Tom Lane. Back-patch to 13. Reported-by: Tom Lane Discussion: https://postgr.es/m/3624029.1602701929%40sss.pgh.pa.us --- src/backend/storage/ipc/latch.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/storage/ipc/latch.c b/src/backend/storage/ipc/latch.c index eacb8c3c9ee9..24d44c982dab 100644 --- a/src/backend/storage/ipc/latch.c +++ b/src/backend/storage/ipc/latch.c @@ -1148,7 +1148,8 @@ WaitEventAdjustKqueue(WaitEventSet *set, WaitEvent *event, int old_events) if (rc < 0) { - if (event->events == WL_POSTMASTER_DEATH && errno == ESRCH) + if (event->events == WL_POSTMASTER_DEATH && + (errno == ESRCH || errno == EACCES)) set->report_postmaster_not_running = true; else ereport(ERROR, From 73c381cee71e7dd8a78a394731958b0bbb4d8991 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 15 Oct 2020 08:54:16 +0200 Subject: [PATCH 303/589] Add documentation link to attributes supported by Clang --- src/include/c.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/include/c.h b/src/include/c.h index aca36c757ec8..9cd67f8f7655 100644 --- a/src/include/c.h +++ b/src/include/c.h @@ -98,6 +98,7 @@ * * GCC: https://gcc.gnu.org/onlinedocs/gcc/Function-Attributes.html * GCC: https://gcc.gnu.org/onlinedocs/gcc/Type-Attributes.html + * Clang: https://clang.llvm.org/docs/AttributeReference.html * Sunpro: https://docs.oracle.com/cd/E18659_01/html/821-1384/gjzke.html * XLC: https://www.ibm.com/support/knowledgecenter/SSGH2K_13.1.2/com.ibm.xlc131.aix.doc/language_ref/function_attributes.html * XLC: https://www.ibm.com/support/knowledgecenter/SSGH2K_13.1.2/com.ibm.xlc131.aix.doc/language_ref/type_attrib.html From 110d81728a0a006abcf654543fc15346f8043dc0 Mon Sep 17 00:00:00 2001 From: David Rowley Date: Thu, 15 Oct 2020 20:35:17 +1300 Subject: [PATCH 304/589] Fixup some appendStringInfo and appendPQExpBuffer calls A number of places were using appendStringInfo() when they could have been using appendStringInfoString() instead. While there's no functionality change there, it's just more efficient to use appendStringInfoString() when no formatting is required. Likewise for some appendStringInfoString() calls which were just appending a single char. We can just use appendStringInfoChar() for that. Additionally, many places were using appendPQExpBuffer() when they could have used appendPQExpBufferStr(). Change those too. Patch by Zhijie Hou, but further searching by me found significantly more places that deserved the same treatment. Author: Zhijie Hou, David Rowley Discussion: https://postgr.es/m/cb172cf4361e4c7ba7167429070979d4@G08CNEXMBPEKD05.g08.fujitsu.local --- contrib/postgres_fdw/postgres_fdw.c | 4 +- contrib/test_decoding/test_decoding.c | 12 +- src/backend/access/rmgrdesc/dbasedesc.c | 2 +- src/backend/commands/explain.c | 16 +- src/backend/replication/backup_manifest.c | 6 +- .../libpqwalreceiver/libpqwalreceiver.c | 2 +- src/backend/replication/logical/tablesync.c | 2 +- src/backend/utils/adt/jsonpath.c | 2 +- src/backend/utils/adt/ri_triggers.c | 2 +- src/backend/utils/adt/ruleutils.c | 4 +- src/bin/pg_dump/pg_dump.c | 228 +++++++++--------- src/bin/pg_upgrade/version.c | 44 ++-- src/bin/psql/describe.c | 56 +++-- src/bin/scripts/reindexdb.c | 44 ++-- src/pl/plpython/plpy_elog.c | 2 +- 15 files changed, 210 insertions(+), 216 deletions(-) diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index 78facb8ebfa4..9c5aaacc5156 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -2591,8 +2591,8 @@ postgresExplainForeignScan(ForeignScanState *node, ExplainState *es) quote_identifier(relname)); } else - appendStringInfo(relations, "%s", - quote_identifier(relname)); + appendStringInfoString(relations, + quote_identifier(relname)); refname = (char *) list_nth(es->rtable_names, rti - 1); if (refname == NULL) refname = rte->eref->aliasname; diff --git a/contrib/test_decoding/test_decoding.c b/contrib/test_decoding/test_decoding.c index e60ab34a5a7c..8e33614f1442 100644 --- a/contrib/test_decoding/test_decoding.c +++ b/contrib/test_decoding/test_decoding.c @@ -606,7 +606,7 @@ pg_output_stream_start(LogicalDecodingContext *ctx, TestDecodingData *data, Reor if (data->include_xids) appendStringInfo(ctx->out, "opening a streamed block for transaction TXN %u", txn->xid); else - appendStringInfo(ctx->out, "opening a streamed block for transaction"); + appendStringInfoString(ctx->out, "opening a streamed block for transaction"); OutputPluginWrite(ctx, last_write); } @@ -623,7 +623,7 @@ pg_decode_stream_stop(LogicalDecodingContext *ctx, if (data->include_xids) appendStringInfo(ctx->out, "closing a streamed block for transaction TXN %u", txn->xid); else - appendStringInfo(ctx->out, "closing a streamed block for transaction"); + appendStringInfoString(ctx->out, "closing a streamed block for transaction"); OutputPluginWrite(ctx, true); } @@ -641,7 +641,7 @@ pg_decode_stream_abort(LogicalDecodingContext *ctx, if (data->include_xids) appendStringInfo(ctx->out, "aborting streamed (sub)transaction TXN %u", txn->xid); else - appendStringInfo(ctx->out, "aborting streamed (sub)transaction"); + appendStringInfoString(ctx->out, "aborting streamed (sub)transaction"); OutputPluginWrite(ctx, true); } @@ -660,7 +660,7 @@ pg_decode_stream_commit(LogicalDecodingContext *ctx, if (data->include_xids) appendStringInfo(ctx->out, "committing streamed transaction TXN %u", txn->xid); else - appendStringInfo(ctx->out, "committing streamed transaction"); + appendStringInfoString(ctx->out, "committing streamed transaction"); if (data->include_timestamp) appendStringInfo(ctx->out, " (at %s)", @@ -693,7 +693,7 @@ pg_decode_stream_change(LogicalDecodingContext *ctx, if (data->include_xids) appendStringInfo(ctx->out, "streaming change for TXN %u", txn->xid); else - appendStringInfo(ctx->out, "streaming change for transaction"); + appendStringInfoString(ctx->out, "streaming change for transaction"); OutputPluginWrite(ctx, true); } @@ -745,6 +745,6 @@ pg_decode_stream_truncate(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, if (data->include_xids) appendStringInfo(ctx->out, "streaming truncate for TXN %u", txn->xid); else - appendStringInfo(ctx->out, "streaming truncate for transaction"); + appendStringInfoString(ctx->out, "streaming truncate for transaction"); OutputPluginWrite(ctx, true); } diff --git a/src/backend/access/rmgrdesc/dbasedesc.c b/src/backend/access/rmgrdesc/dbasedesc.c index d82484b9db40..47580feaeae4 100644 --- a/src/backend/access/rmgrdesc/dbasedesc.c +++ b/src/backend/access/rmgrdesc/dbasedesc.c @@ -37,7 +37,7 @@ dbase_desc(StringInfo buf, XLogReaderState *record) xl_dbase_drop_rec *xlrec = (xl_dbase_drop_rec *) rec; int i; - appendStringInfo(buf, "dir"); + appendStringInfoString(buf, "dir"); for (i = 0; i < xlrec->ntablespaces; i++) appendStringInfo(buf, " %u/%u", xlrec->tablespace_ids[i], xlrec->db_id); diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index c8e292adfa63..41317f183743 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -2768,14 +2768,14 @@ show_incremental_sort_group_info(IncrementalSortGroupInfo *groupInfo, groupInfo->groupCount); /* plural/singular based on methodNames size */ if (list_length(methodNames) > 1) - appendStringInfo(es->str, "s: "); + appendStringInfoString(es->str, "s: "); else - appendStringInfo(es->str, ": "); + appendStringInfoString(es->str, ": "); foreach(methodCell, methodNames) { - appendStringInfo(es->str, "%s", (char *) methodCell->ptr_value); + appendStringInfoString(es->str, (char *) methodCell->ptr_value); if (foreach_current_index(methodCell) < list_length(methodNames) - 1) - appendStringInfo(es->str, ", "); + appendStringInfoString(es->str, ", "); } if (groupInfo->maxMemorySpaceUsed > 0) @@ -2882,11 +2882,11 @@ show_incremental_sort_info(IncrementalSortState *incrsortstate, if (prefixsortGroupInfo->groupCount > 0) { if (es->format == EXPLAIN_FORMAT_TEXT) - appendStringInfo(es->str, "\n"); + appendStringInfoChar(es->str, '\n'); show_incremental_sort_group_info(prefixsortGroupInfo, "Pre-sorted", true, es); } if (es->format == EXPLAIN_FORMAT_TEXT) - appendStringInfo(es->str, "\n"); + appendStringInfoChar(es->str, '\n'); } if (incrsortstate->shared_info != NULL) @@ -2925,11 +2925,11 @@ show_incremental_sort_info(IncrementalSortState *incrsortstate, if (prefixsortGroupInfo->groupCount > 0) { if (es->format == EXPLAIN_FORMAT_TEXT) - appendStringInfo(es->str, "\n"); + appendStringInfoChar(es->str, '\n'); show_incremental_sort_group_info(prefixsortGroupInfo, "Pre-sorted", true, es); } if (es->format == EXPLAIN_FORMAT_TEXT) - appendStringInfo(es->str, "\n"); + appendStringInfoChar(es->str, '\n'); if (es->workers_state) ExplainCloseWorker(n, es); diff --git a/src/backend/replication/backup_manifest.c b/src/backend/replication/backup_manifest.c index a43c793e2892..556e6b504085 100644 --- a/src/backend/replication/backup_manifest.c +++ b/src/backend/replication/backup_manifest.c @@ -112,7 +112,7 @@ AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid, initStringInfo(&buf); if (manifest->first_file) { - appendStringInfoString(&buf, "\n"); + appendStringInfoChar(&buf, '\n'); manifest->first_file = false; } else @@ -152,7 +152,7 @@ AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid, enlargeStringInfo(&buf, 128); buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z", pg_gmtime(&mtime)); - appendStringInfoString(&buf, "\""); + appendStringInfoChar(&buf, '"'); /* Add checksum information. */ if (checksum_ctx->type != CHECKSUM_TYPE_NONE) @@ -168,7 +168,7 @@ AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid, enlargeStringInfo(&buf, 2 * checksumlen); buf.len += hex_encode((char *) checksumbuf, checksumlen, &buf.data[buf.len]); - appendStringInfoString(&buf, "\""); + appendStringInfoChar(&buf, '"'); } /* Close out the object. */ diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index ad574099ff70..24f8b3e42ece 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -427,7 +427,7 @@ libpqrcv_startstreaming(WalReceiverConn *conn, if (options->proto.logical.streaming && PQserverVersion(conn->streamConn) >= 140000) - appendStringInfo(&cmd, ", streaming 'on'"); + appendStringInfoString(&cmd, ", streaming 'on'"); pubnames = options->proto.logical.publication_names; pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames); diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index c27d97058955..843c9285d59d 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -774,7 +774,7 @@ copy_table(Relation rel) * For non-tables, we need to do COPY (SELECT ...), but we can't just * do SELECT * because we need to not copy generated columns. */ - appendStringInfo(&cmd, "COPY (SELECT "); + appendStringInfoString(&cmd, "COPY (SELECT "); for (int i = 0; i < lrel.natts; i++) { appendStringInfoString(&cmd, quote_identifier(lrel.attnames[i])); diff --git a/src/backend/utils/adt/jsonpath.c b/src/backend/utils/adt/jsonpath.c index 3c0dc38a7f84..31d9d92d14ed 100644 --- a/src/backend/utils/adt/jsonpath.c +++ b/src/backend/utils/adt/jsonpath.c @@ -660,7 +660,7 @@ printJsonPathItem(StringInfo buf, JsonPathItem *v, bool inKey, else if (v->content.anybounds.first == v->content.anybounds.last) { if (v->content.anybounds.first == PG_UINT32_MAX) - appendStringInfo(buf, "**{last}"); + appendStringInfoString(buf, "**{last}"); else appendStringInfo(buf, "**{%u}", v->content.anybounds.first); diff --git a/src/backend/utils/adt/ri_triggers.c b/src/backend/utils/adt/ri_triggers.c index 06cf16d9d716..7e2b2e3dd646 100644 --- a/src/backend/utils/adt/ri_triggers.c +++ b/src/backend/utils/adt/ri_triggers.c @@ -1663,7 +1663,7 @@ RI_PartitionRemove_Check(Trigger *trigger, Relation fk_rel, Relation pk_rel) appendStringInfo(&querybuf, ") WHERE %s AND (", constraintDef); else - appendStringInfo(&querybuf, ") WHERE ("); + appendStringInfoString(&querybuf, ") WHERE ("); sep = ""; for (i = 0; i < riinfo->nkeys; i++) diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index 62023c20b21e..6c656586e857 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -5250,7 +5250,7 @@ get_select_query_def(Query *query, deparse_context *context, appendContextKeyword(context, " FETCH FIRST ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); get_rule_expr(query->limitCount, context, false); - appendStringInfo(buf, " ROWS WITH TIES"); + appendStringInfoString(buf, " ROWS WITH TIES"); } else { @@ -11362,7 +11362,7 @@ get_range_partbound_string(List *bound_datums) memset(&context, 0, sizeof(deparse_context)); context.buf = buf; - appendStringInfoString(buf, "("); + appendStringInfoChar(buf, '('); sep = ""; foreach(cell, bound_datums) { diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 88bbbd9a9e1c..ff45e3fb8c37 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -1375,8 +1375,8 @@ expand_foreign_server_name_patterns(Archive *fout, for (cell = patterns->head; cell; cell = cell->next) { - appendPQExpBuffer(query, - "SELECT oid FROM pg_catalog.pg_foreign_server s\n"); + appendPQExpBufferStr(query, + "SELECT oid FROM pg_catalog.pg_foreign_server s\n"); processSQLNamePattern(GetConnection(fout), query, cell->val, false, false, NULL, "s.srvname", NULL, NULL); @@ -4250,23 +4250,19 @@ getSubscriptions(Archive *fout) username_subquery); if (fout->remoteVersion >= 140000) - appendPQExpBuffer(query, - " s.subbinary,\n"); + appendPQExpBufferStr(query, " s.subbinary,\n"); else - appendPQExpBuffer(query, - " false AS subbinary,\n"); + appendPQExpBufferStr(query, " false AS subbinary,\n"); if (fout->remoteVersion >= 140000) - appendPQExpBuffer(query, - " s.substream\n"); + appendPQExpBufferStr(query, " s.substream\n"); else - appendPQExpBuffer(query, - " false AS substream\n"); + appendPQExpBufferStr(query, " false AS substream\n"); - appendPQExpBuffer(query, - "FROM pg_subscription s\n" - "WHERE s.subdbid = (SELECT oid FROM pg_database\n" - " WHERE datname = current_database())"); + appendPQExpBufferStr(query, + "FROM pg_subscription s\n" + "WHERE s.subdbid = (SELECT oid FROM pg_database\n" + " WHERE datname = current_database())"); res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -4376,10 +4372,10 @@ dumpSubscription(Archive *fout, SubscriptionInfo *subinfo) appendPQExpBufferStr(query, "NONE"); if (strcmp(subinfo->subbinary, "t") == 0) - appendPQExpBuffer(query, ", binary = true"); + appendPQExpBufferStr(query, ", binary = true"); if (strcmp(subinfo->substream, "f") != 0) - appendPQExpBuffer(query, ", streaming = on"); + appendPQExpBufferStr(query, ", streaming = on"); if (strcmp(subinfo->subsynccommit, "off") != 0) appendPQExpBuffer(query, ", synchronous_commit = %s", fmtId(subinfo->subsynccommit)); @@ -11845,26 +11841,26 @@ dumpFunc(Archive *fout, FuncInfo *finfo) asPart = createPQExpBuffer(); /* Fetch function-specific details */ - appendPQExpBuffer(query, - "SELECT\n" - "proretset,\n" - "prosrc,\n" - "probin,\n" - "provolatile,\n" - "proisstrict,\n" - "prosecdef,\n" - "(SELECT lanname FROM pg_catalog.pg_language WHERE oid = prolang) AS lanname,\n"); + appendPQExpBufferStr(query, + "SELECT\n" + "proretset,\n" + "prosrc,\n" + "probin,\n" + "provolatile,\n" + "proisstrict,\n" + "prosecdef,\n" + "(SELECT lanname FROM pg_catalog.pg_language WHERE oid = prolang) AS lanname,\n"); if (fout->remoteVersion >= 80300) - appendPQExpBuffer(query, - "proconfig,\n" - "procost,\n" - "prorows,\n"); + appendPQExpBufferStr(query, + "proconfig,\n" + "procost,\n" + "prorows,\n"); else - appendPQExpBuffer(query, - "null AS proconfig,\n" - "0 AS procost,\n" - "0 AS prorows,\n"); + appendPQExpBufferStr(query, + "null AS proconfig,\n" + "0 AS procost,\n" + "0 AS prorows,\n"); if (fout->remoteVersion >= 80400) { @@ -11872,56 +11868,56 @@ dumpFunc(Archive *fout, FuncInfo *finfo) * In 8.4 and up we rely on pg_get_function_arguments and * pg_get_function_result instead of examining proallargtypes etc. */ - appendPQExpBuffer(query, - "pg_catalog.pg_get_function_arguments(oid) AS funcargs,\n" - "pg_catalog.pg_get_function_identity_arguments(oid) AS funciargs,\n" - "pg_catalog.pg_get_function_result(oid) AS funcresult,\n"); + appendPQExpBufferStr(query, + "pg_catalog.pg_get_function_arguments(oid) AS funcargs,\n" + "pg_catalog.pg_get_function_identity_arguments(oid) AS funciargs,\n" + "pg_catalog.pg_get_function_result(oid) AS funcresult,\n"); } else if (fout->remoteVersion >= 80100) - appendPQExpBuffer(query, - "proallargtypes,\n" - "proargmodes,\n" - "proargnames,\n"); + appendPQExpBufferStr(query, + "proallargtypes,\n" + "proargmodes,\n" + "proargnames,\n"); else - appendPQExpBuffer(query, - "null AS proallargtypes,\n" - "null AS proargmodes,\n" - "proargnames,\n"); + appendPQExpBufferStr(query, + "null AS proallargtypes,\n" + "null AS proargmodes,\n" + "proargnames,\n"); if (fout->remoteVersion >= 90200) - appendPQExpBuffer(query, - "proleakproof,\n"); + appendPQExpBufferStr(query, + "proleakproof,\n"); else - appendPQExpBuffer(query, - "false AS proleakproof,\n"); + appendPQExpBufferStr(query, + "false AS proleakproof,\n"); if (fout->remoteVersion >= 90500) - appendPQExpBuffer(query, - "array_to_string(protrftypes, ' ') AS protrftypes,\n"); + appendPQExpBufferStr(query, + "array_to_string(protrftypes, ' ') AS protrftypes,\n"); if (fout->remoteVersion >= 90600) - appendPQExpBuffer(query, - "proparallel,\n"); + appendPQExpBufferStr(query, + "proparallel,\n"); else - appendPQExpBuffer(query, - "'u' AS proparallel,\n"); + appendPQExpBufferStr(query, + "'u' AS proparallel,\n"); if (fout->remoteVersion >= 110000) - appendPQExpBuffer(query, - "prokind,\n"); + appendPQExpBufferStr(query, + "prokind,\n"); else if (fout->remoteVersion >= 80400) - appendPQExpBuffer(query, - "CASE WHEN proiswindow THEN 'w' ELSE 'f' END AS prokind,\n"); + appendPQExpBufferStr(query, + "CASE WHEN proiswindow THEN 'w' ELSE 'f' END AS prokind,\n"); else - appendPQExpBuffer(query, - "'f' AS prokind,\n"); + appendPQExpBufferStr(query, + "'f' AS prokind,\n"); if (fout->remoteVersion >= 120000) - appendPQExpBuffer(query, - "prosupport\n"); + appendPQExpBufferStr(query, + "prosupport\n"); else - appendPQExpBuffer(query, - "'-' AS prosupport\n"); + appendPQExpBufferStr(query, + "'-' AS prosupport\n"); appendPQExpBuffer(query, "FROM pg_catalog.pg_proc " @@ -13891,71 +13887,71 @@ dumpAgg(Archive *fout, AggInfo *agginfo) details = createPQExpBuffer(); /* Get aggregate-specific details */ - appendPQExpBuffer(query, - "SELECT\n" - "aggtransfn,\n" - "aggfinalfn,\n" - "aggtranstype::pg_catalog.regtype,\n" - "agginitval,\n"); + appendPQExpBufferStr(query, + "SELECT\n" + "aggtransfn,\n" + "aggfinalfn,\n" + "aggtranstype::pg_catalog.regtype,\n" + "agginitval,\n"); if (fout->remoteVersion >= 80100) - appendPQExpBuffer(query, - "aggsortop,\n"); + appendPQExpBufferStr(query, + "aggsortop,\n"); else - appendPQExpBuffer(query, - "0 AS aggsortop,\n"); + appendPQExpBufferStr(query, + "0 AS aggsortop,\n"); if (fout->remoteVersion >= 80400) - appendPQExpBuffer(query, - "pg_catalog.pg_get_function_arguments(p.oid) AS funcargs,\n" - "pg_catalog.pg_get_function_identity_arguments(p.oid) AS funciargs,\n"); + appendPQExpBufferStr(query, + "pg_catalog.pg_get_function_arguments(p.oid) AS funcargs,\n" + "pg_catalog.pg_get_function_identity_arguments(p.oid) AS funciargs,\n"); if (fout->remoteVersion >= 90400) - appendPQExpBuffer(query, - "aggkind,\n" - "aggmtransfn,\n" - "aggminvtransfn,\n" - "aggmfinalfn,\n" - "aggmtranstype::pg_catalog.regtype,\n" - "aggfinalextra,\n" - "aggmfinalextra,\n" - "aggtransspace,\n" - "aggmtransspace,\n" - "aggminitval,\n"); + appendPQExpBufferStr(query, + "aggkind,\n" + "aggmtransfn,\n" + "aggminvtransfn,\n" + "aggmfinalfn,\n" + "aggmtranstype::pg_catalog.regtype,\n" + "aggfinalextra,\n" + "aggmfinalextra,\n" + "aggtransspace,\n" + "aggmtransspace,\n" + "aggminitval,\n"); else - appendPQExpBuffer(query, - "'n' AS aggkind,\n" - "'-' AS aggmtransfn,\n" - "'-' AS aggminvtransfn,\n" - "'-' AS aggmfinalfn,\n" - "0 AS aggmtranstype,\n" - "false AS aggfinalextra,\n" - "false AS aggmfinalextra,\n" - "0 AS aggtransspace,\n" - "0 AS aggmtransspace,\n" - "NULL AS aggminitval,\n"); + appendPQExpBufferStr(query, + "'n' AS aggkind,\n" + "'-' AS aggmtransfn,\n" + "'-' AS aggminvtransfn,\n" + "'-' AS aggmfinalfn,\n" + "0 AS aggmtranstype,\n" + "false AS aggfinalextra,\n" + "false AS aggmfinalextra,\n" + "0 AS aggtransspace,\n" + "0 AS aggmtransspace,\n" + "NULL AS aggminitval,\n"); if (fout->remoteVersion >= 90600) - appendPQExpBuffer(query, - "aggcombinefn,\n" - "aggserialfn,\n" - "aggdeserialfn,\n" - "proparallel,\n"); + appendPQExpBufferStr(query, + "aggcombinefn,\n" + "aggserialfn,\n" + "aggdeserialfn,\n" + "proparallel,\n"); else - appendPQExpBuffer(query, - "'-' AS aggcombinefn,\n" - "'-' AS aggserialfn,\n" - "'-' AS aggdeserialfn,\n" - "'u' AS proparallel,\n"); + appendPQExpBufferStr(query, + "'-' AS aggcombinefn,\n" + "'-' AS aggserialfn,\n" + "'-' AS aggdeserialfn,\n" + "'u' AS proparallel,\n"); if (fout->remoteVersion >= 110000) - appendPQExpBuffer(query, - "aggfinalmodify,\n" - "aggmfinalmodify\n"); + appendPQExpBufferStr(query, + "aggfinalmodify,\n" + "aggmfinalmodify\n"); else - appendPQExpBuffer(query, - "'0' AS aggfinalmodify,\n" - "'0' AS aggmfinalmodify\n"); + appendPQExpBufferStr(query, + "'0' AS aggfinalmodify,\n" + "'0' AS aggmfinalmodify\n"); appendPQExpBuffer(query, "FROM pg_catalog.pg_aggregate a, pg_catalog.pg_proc p " diff --git a/src/bin/pg_upgrade/version.c b/src/bin/pg_upgrade/version.c index 4e5d27f76eb2..db1934124ee3 100644 --- a/src/bin/pg_upgrade/version.c +++ b/src/bin/pg_upgrade/version.c @@ -158,33 +158,33 @@ check_for_data_type_usage(ClusterInfo *cluster, const char *typename, /* Ranges were introduced in 9.2 */ if (GET_MAJOR_VERSION(cluster->major_version) >= 902) - appendPQExpBuffer(&querybuf, - " UNION ALL " + appendPQExpBufferStr(&querybuf, + " UNION ALL " /* ranges containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " - " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid"); + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " + " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid"); - appendPQExpBuffer(&querybuf, - " ) foo " - ") " + appendPQExpBufferStr(&querybuf, + " ) foo " + ") " /* now look for stored columns of any such type */ - "SELECT n.nspname, c.relname, a.attname " - "FROM pg_catalog.pg_class c, " - " pg_catalog.pg_namespace n, " - " pg_catalog.pg_attribute a " - "WHERE c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid IN (SELECT oid FROM oids) AND " - " c.relkind IN (" - CppAsString2(RELKIND_RELATION) ", " - CppAsString2(RELKIND_MATVIEW) ", " - CppAsString2(RELKIND_INDEX) ") AND " - " c.relnamespace = n.oid AND " + "SELECT n.nspname, c.relname, a.attname " + "FROM pg_catalog.pg_class c, " + " pg_catalog.pg_namespace n, " + " pg_catalog.pg_attribute a " + "WHERE c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid IN (SELECT oid FROM oids) AND " + " c.relkind IN (" + CppAsString2(RELKIND_RELATION) ", " + CppAsString2(RELKIND_MATVIEW) ", " + CppAsString2(RELKIND_INDEX) ") AND " + " c.relnamespace = n.oid AND " /* exclude possible orphaned temp tables */ - " n.nspname !~ '^pg_temp_' AND " - " n.nspname !~ '^pg_toast_temp_' AND " + " n.nspname !~ '^pg_temp_' AND " + " n.nspname !~ '^pg_toast_temp_' AND " /* exclude system catalogs, too */ - " n.nspname NOT IN ('pg_catalog', 'information_schema')"); + " n.nspname NOT IN ('pg_catalog', 'information_schema')"); res = executeQueryOrDie(conn, "%s", querybuf.data); diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 58de433fd302..6bb0316bd98b 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -6137,17 +6137,16 @@ listOperatorClasses(const char *access_method_pattern, " pg_catalog.pg_get_userbyid(c.opcowner) AS \"%s\"\n", gettext_noop("Operator family"), gettext_noop("Owner")); - appendPQExpBuffer(&buf, - "\nFROM pg_catalog.pg_opclass c\n" - " LEFT JOIN pg_catalog.pg_am am on am.oid = c.opcmethod\n" - " LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.opcnamespace\n" - " LEFT JOIN pg_catalog.pg_type t ON t.oid = c.opcintype\n" - " LEFT JOIN pg_catalog.pg_namespace tn ON tn.oid = t.typnamespace\n" - ); + appendPQExpBufferStr(&buf, + "\nFROM pg_catalog.pg_opclass c\n" + " LEFT JOIN pg_catalog.pg_am am on am.oid = c.opcmethod\n" + " LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.opcnamespace\n" + " LEFT JOIN pg_catalog.pg_type t ON t.oid = c.opcintype\n" + " LEFT JOIN pg_catalog.pg_namespace tn ON tn.oid = t.typnamespace\n"); if (verbose) - appendPQExpBuffer(&buf, - " LEFT JOIN pg_catalog.pg_opfamily of ON of.oid = c.opcfamily\n" - " LEFT JOIN pg_catalog.pg_namespace ofn ON ofn.oid = of.opfnamespace\n"); + appendPQExpBufferStr(&buf, + " LEFT JOIN pg_catalog.pg_opfamily of ON of.oid = c.opcfamily\n" + " LEFT JOIN pg_catalog.pg_namespace ofn ON ofn.oid = of.opfnamespace\n"); if (access_method_pattern) have_where = processSQLNamePattern(pset.db, &buf, access_method_pattern, @@ -6216,11 +6215,10 @@ listOperatorFamilies(const char *access_method_pattern, appendPQExpBuffer(&buf, ",\n pg_catalog.pg_get_userbyid(f.opfowner) AS \"%s\"\n", gettext_noop("Owner")); - appendPQExpBuffer(&buf, - "\nFROM pg_catalog.pg_opfamily f\n" - " LEFT JOIN pg_catalog.pg_am am on am.oid = f.opfmethod\n" - " LEFT JOIN pg_catalog.pg_namespace n ON n.oid = f.opfnamespace\n" - ); + appendPQExpBufferStr(&buf, + "\nFROM pg_catalog.pg_opfamily f\n" + " LEFT JOIN pg_catalog.pg_am am on am.oid = f.opfmethod\n" + " LEFT JOIN pg_catalog.pg_namespace n ON n.oid = f.opfnamespace\n"); if (access_method_pattern) have_where = processSQLNamePattern(pset.db, &buf, access_method_pattern, @@ -6240,7 +6238,7 @@ listOperatorFamilies(const char *access_method_pattern, "tn.nspname", "t.typname", "pg_catalog.format_type(t.oid, NULL)", "pg_catalog.pg_type_is_visible(t.oid)"); - appendPQExpBuffer(&buf, " )\n"); + appendPQExpBufferStr(&buf, " )\n"); } appendPQExpBufferStr(&buf, "ORDER BY 1, 2;"); @@ -6307,14 +6305,14 @@ listOpFamilyOperators(const char *access_method_pattern, appendPQExpBuffer(&buf, ", ofs.opfname AS \"%s\"\n", gettext_noop("Sort opfamily")); - appendPQExpBuffer(&buf, - "FROM pg_catalog.pg_amop o\n" - " LEFT JOIN pg_catalog.pg_opfamily of ON of.oid = o.amopfamily\n" - " LEFT JOIN pg_catalog.pg_am am ON am.oid = of.opfmethod AND am.oid = o.amopmethod\n" - " LEFT JOIN pg_catalog.pg_namespace nsf ON of.opfnamespace = nsf.oid\n"); + appendPQExpBufferStr(&buf, + "FROM pg_catalog.pg_amop o\n" + " LEFT JOIN pg_catalog.pg_opfamily of ON of.oid = o.amopfamily\n" + " LEFT JOIN pg_catalog.pg_am am ON am.oid = of.opfmethod AND am.oid = o.amopmethod\n" + " LEFT JOIN pg_catalog.pg_namespace nsf ON of.opfnamespace = nsf.oid\n"); if (verbose) - appendPQExpBuffer(&buf, - " LEFT JOIN pg_catalog.pg_opfamily ofs ON ofs.oid = o.amopsortfamily\n"); + appendPQExpBufferStr(&buf, + " LEFT JOIN pg_catalog.pg_opfamily ofs ON ofs.oid = o.amopsortfamily\n"); if (access_method_pattern) have_where = processSQLNamePattern(pset.db, &buf, access_method_pattern, @@ -6393,12 +6391,12 @@ listOpFamilyFunctions(const char *access_method_pattern, ", ap.amproc::pg_catalog.regprocedure AS \"%s\"\n", gettext_noop("Function")); - appendPQExpBuffer(&buf, - "FROM pg_catalog.pg_amproc ap\n" - " LEFT JOIN pg_catalog.pg_opfamily of ON of.oid = ap.amprocfamily\n" - " LEFT JOIN pg_catalog.pg_am am ON am.oid = of.opfmethod\n" - " LEFT JOIN pg_catalog.pg_namespace ns ON of.opfnamespace = ns.oid\n" - " LEFT JOIN pg_catalog.pg_proc p ON ap.amproc = p.oid\n"); + appendPQExpBufferStr(&buf, + "FROM pg_catalog.pg_amproc ap\n" + " LEFT JOIN pg_catalog.pg_opfamily of ON of.oid = ap.amprocfamily\n" + " LEFT JOIN pg_catalog.pg_am am ON am.oid = of.opfmethod\n" + " LEFT JOIN pg_catalog.pg_namespace ns ON of.opfnamespace = ns.oid\n" + " LEFT JOIN pg_catalog.pg_proc p ON ap.amproc = p.oid\n"); if (access_method_pattern) have_where = processSQLNamePattern(pset.db, &buf, access_method_pattern, diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c index 40dcbc928332..1efb53110efa 100644 --- a/src/bin/scripts/reindexdb.c +++ b/src/bin/scripts/reindexdb.c @@ -614,16 +614,16 @@ get_parallel_object_list(PGconn *conn, ReindexType type, { case REINDEX_DATABASE: Assert(user_list == NULL); - appendPQExpBuffer(&catalog_query, - "SELECT c.relname, ns.nspname\n" - " FROM pg_catalog.pg_class c\n" - " JOIN pg_catalog.pg_namespace ns" - " ON c.relnamespace = ns.oid\n" - " WHERE ns.nspname != 'pg_catalog'\n" - " AND c.relkind IN (" - CppAsString2(RELKIND_RELATION) ", " - CppAsString2(RELKIND_MATVIEW) ")\n" - " ORDER BY c.relpages DESC;"); + appendPQExpBufferStr(&catalog_query, + "SELECT c.relname, ns.nspname\n" + " FROM pg_catalog.pg_class c\n" + " JOIN pg_catalog.pg_namespace ns" + " ON c.relnamespace = ns.oid\n" + " WHERE ns.nspname != 'pg_catalog'\n" + " AND c.relkind IN (" + CppAsString2(RELKIND_RELATION) ", " + CppAsString2(RELKIND_MATVIEW) ")\n" + " ORDER BY c.relpages DESC;"); break; case REINDEX_SCHEMA: @@ -637,30 +637,30 @@ get_parallel_object_list(PGconn *conn, ReindexType type, * All the tables from all the listed schemas are grabbed at * once. */ - appendPQExpBuffer(&catalog_query, - "SELECT c.relname, ns.nspname\n" - " FROM pg_catalog.pg_class c\n" - " JOIN pg_catalog.pg_namespace ns" - " ON c.relnamespace = ns.oid\n" - " WHERE c.relkind IN (" - CppAsString2(RELKIND_RELATION) ", " - CppAsString2(RELKIND_MATVIEW) ")\n" - " AND ns.nspname IN ("); + appendPQExpBufferStr(&catalog_query, + "SELECT c.relname, ns.nspname\n" + " FROM pg_catalog.pg_class c\n" + " JOIN pg_catalog.pg_namespace ns" + " ON c.relnamespace = ns.oid\n" + " WHERE c.relkind IN (" + CppAsString2(RELKIND_RELATION) ", " + CppAsString2(RELKIND_MATVIEW) ")\n" + " AND ns.nspname IN ("); for (cell = user_list->head; cell; cell = cell->next) { const char *nspname = cell->val; if (nsp_listed) - appendPQExpBuffer(&catalog_query, ", "); + appendPQExpBufferStr(&catalog_query, ", "); else nsp_listed = true; appendStringLiteralConn(&catalog_query, nspname, conn); } - appendPQExpBuffer(&catalog_query, ")\n" - " ORDER BY c.relpages DESC;"); + appendPQExpBufferStr(&catalog_query, ")\n" + " ORDER BY c.relpages DESC;"); } break; diff --git a/src/pl/plpython/plpy_elog.c b/src/pl/plpython/plpy_elog.c index ae0b97c85d3f..224b8836fba9 100644 --- a/src/pl/plpython/plpy_elog.c +++ b/src/pl/plpython/plpy_elog.c @@ -216,7 +216,7 @@ PLy_traceback(PyObject *e, PyObject *v, PyObject *tb, else if (strcmp(e_module_s, "builtins") == 0 || strcmp(e_module_s, "__main__") == 0 || strcmp(e_module_s, "exceptions") == 0) - appendStringInfo(&xstr, "%s", e_type_s); + appendStringInfoString(&xstr, e_type_s); else appendStringInfo(&xstr, "%s.%s", e_module_s, e_type_s); appendStringInfo(&xstr, ": %s", vstr); From 8176afd8b764e54fc6e34240928effc661fc40cd Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Thu, 15 Oct 2020 16:50:57 +0900 Subject: [PATCH 305/589] Improve tab-completion for FETCH/MOVE. Author: Naoki Nakamichi Reviewed-by: Fujii Masao Discussion: https://postgr.es/m/d05a46b599634ca0d94144387507f4b4@oss.nttdata.com --- src/bin/psql/tab-complete.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 24c7b414cf3a..561fe1dff93b 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -3077,19 +3077,27 @@ psql_completion(const char *text, int start, int end) COMPLETE_WITH("SELECT", "INSERT", "DELETE", "UPDATE", "DECLARE"); /* FETCH && MOVE */ - /* Complete FETCH with one of FORWARD, BACKWARD, RELATIVE */ + + /* + * Complete FETCH with one of ABSOLUTE, BACKWARD, FORWARD, RELATIVE, ALL, + * NEXT, PRIOR, FIRST, LAST + */ else if (Matches("FETCH|MOVE")) - COMPLETE_WITH("ABSOLUTE", "BACKWARD", "FORWARD", "RELATIVE"); - /* Complete FETCH with one of ALL, NEXT, PRIOR */ - else if (Matches("FETCH|MOVE", MatchAny)) - COMPLETE_WITH("ALL", "NEXT", "PRIOR"); + COMPLETE_WITH("ABSOLUTE", "BACKWARD", "FORWARD", "RELATIVE", + "ALL", "NEXT", "PRIOR", "FIRST", "LAST"); + + /* Complete FETCH BACKWARD or FORWARD with one of ALL, FROM, IN */ + else if (Matches("FETCH|MOVE", "BACKWARD|FORWARD")) + COMPLETE_WITH("ALL", "FROM", "IN"); /* - * Complete FETCH with "FROM" or "IN". These are equivalent, + * Complete FETCH with "FROM" or "IN". These are equivalent, * but we may as well tab-complete both: perhaps some users prefer one * variant or the other. */ - else if (Matches("FETCH|MOVE", MatchAny, MatchAny)) + else if (Matches("FETCH|MOVE", "ABSOLUTE|BACKWARD|FORWARD|RELATIVE", + MatchAnyExcept("FROM|IN")) || + Matches("FETCH|MOVE", "ALL|NEXT|PRIOR|FIRST|LAST")) COMPLETE_WITH("FROM", "IN"); /* FOREIGN DATA WRAPPER */ From 86dba33217bb0b48fcf59da76912c3382b4418cd Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 15 Oct 2020 17:03:56 +0900 Subject: [PATCH 306/589] Replace calls of htonl()/ntohl() with pg_bswap.h for GSSAPI encryption The in-core equivalents can make use of built-in functions if the compiler supports this option, making optimizations possible. 0ba99c8 replaced all existing calls in the code base at this time, but b0b39f7 (GSSAPI encryption) has forgotten to do the switch. Discussion: https://postgr.es/m/20201014055303.GG3349@paquier.xyz --- src/backend/libpq/be-secure-gssapi.c | 8 ++++---- src/interfaces/libpq/fe-secure-gssapi.c | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/backend/libpq/be-secure-gssapi.c b/src/backend/libpq/be-secure-gssapi.c index 64427f185bb7..5a73302b7b9e 100644 --- a/src/backend/libpq/be-secure-gssapi.c +++ b/src/backend/libpq/be-secure-gssapi.c @@ -209,7 +209,7 @@ be_gssapi_write(Port *port, void *ptr, size_t len) PqGSSSendConsumed += input.length; /* 4 network-order bytes of length, then payload */ - netlen = htonl(output.length); + netlen = pg_hton32(output.length); memcpy(PqGSSSendBuffer + PqGSSSendLength, &netlen, sizeof(uint32)); PqGSSSendLength += sizeof(uint32); @@ -323,7 +323,7 @@ be_gssapi_read(Port *port, void *ptr, size_t len) } /* Decode the packet length and check for overlength packet */ - input.length = ntohl(*(uint32 *) PqGSSRecvBuffer); + input.length = pg_ntoh32(*(uint32 *) PqGSSRecvBuffer); if (input.length > PQ_GSS_RECV_BUFFER_SIZE - sizeof(uint32)) ereport(FATAL, @@ -509,7 +509,7 @@ secure_open_gssapi(Port *port) /* * Get the length for this packet from the length header. */ - input.length = ntohl(*(uint32 *) PqGSSRecvBuffer); + input.length = pg_ntoh32(*(uint32 *) PqGSSRecvBuffer); /* Done with the length, reset our buffer */ PqGSSRecvLength = 0; @@ -567,7 +567,7 @@ secure_open_gssapi(Port *port) */ if (output.length > 0) { - uint32 netlen = htonl(output.length); + uint32 netlen = pg_hton32(output.length); if (output.length > PQ_GSS_SEND_BUFFER_SIZE - sizeof(uint32)) ereport(FATAL, diff --git a/src/interfaces/libpq/fe-secure-gssapi.c b/src/interfaces/libpq/fe-secure-gssapi.c index 1994e9f61506..bfc0f5521468 100644 --- a/src/interfaces/libpq/fe-secure-gssapi.c +++ b/src/interfaces/libpq/fe-secure-gssapi.c @@ -226,7 +226,7 @@ pg_GSS_write(PGconn *conn, const void *ptr, size_t len) PqGSSSendConsumed += input.length; /* 4 network-order bytes of length, then payload */ - netlen = htonl(output.length); + netlen = pg_hton32(output.length); memcpy(PqGSSSendBuffer + PqGSSSendLength, &netlen, sizeof(uint32)); PqGSSSendLength += sizeof(uint32); @@ -346,7 +346,7 @@ pg_GSS_read(PGconn *conn, void *ptr, size_t len) } /* Decode the packet length and check for overlength packet */ - input.length = ntohl(*(uint32 *) PqGSSRecvBuffer); + input.length = pg_ntoh32(*(uint32 *) PqGSSRecvBuffer); if (input.length > PQ_GSS_RECV_BUFFER_SIZE - sizeof(uint32)) { @@ -589,7 +589,7 @@ pqsecure_open_gss(PGconn *conn) */ /* Get the length and check for over-length packet */ - input.length = ntohl(*(uint32 *) PqGSSRecvBuffer); + input.length = pg_ntoh32(*(uint32 *) PqGSSRecvBuffer); if (input.length > PQ_GSS_RECV_BUFFER_SIZE - sizeof(uint32)) { printfPQExpBuffer(&conn->errorMessage, @@ -688,7 +688,7 @@ pqsecure_open_gss(PGconn *conn) } /* Queue the token for writing */ - netlen = htonl(output.length); + netlen = pg_hton32(output.length); memcpy(PqGSSSendBuffer, (char *) &netlen, sizeof(uint32)); PqGSSSendLength += sizeof(uint32); From 7f4708818317f59a0452f549dedf4ce35182c13f Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Thu, 15 Oct 2020 09:48:36 -0300 Subject: [PATCH 307/589] Fix query in new test to check tables are synced Rather than looking for tablesync workers, it is more reliable to see the sync state of the tables. Per note from Amit Kapila. Discussion: https://postgr.es/m/CAA4eK1JSSD7FVwq+_rOme86jUZTQFzjsNU06hQ4-LiRt1xFmSg@mail.gmail.com --- src/test/subscription/t/100_bugs.pl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/subscription/t/100_bugs.pl b/src/test/subscription/t/100_bugs.pl index 7dc8983d6aee..d1e407aacb32 100644 --- a/src/test/subscription/t/100_bugs.pl +++ b/src/test/subscription/t/100_bugs.pl @@ -141,13 +141,13 @@ # We cannot rely solely on wait_for_catchup() here; it isn't sufficient # when tablesync workers might still be running. So in addition to that, -# we verify that no tablesync workers appear for the subscription. +# verify that tables are synced. # XXX maybe this should be integrated in wait_for_catchup() itself. $node_twoways->wait_for_catchup('testsub'); -$node_twoways->poll_query_until( - 'd2', - "SELECT count(*) FROM pg_stat_subscription WHERE subname = 'testsub' AND relid <> 0", - "0"); +my $synced_query = + "SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('r', 's');"; +$node_twoways->poll_query_until('d2', $synced_query) + or die "Timed out while waiting for subscriber to synchronize data"; is($node_twoways->safe_psql('d2', "SELECT count(f) FROM t"), $rows * 2, "2x$rows rows in t"); From c5b097f8fa69861ab83470c0bfe732ab90df9e62 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 15 Oct 2020 17:08:10 +0300 Subject: [PATCH 308/589] Refactor code for cross-partition updates to a separate function. ExecUpdate() is very long, so extract the part of it that deals with cross-partition updates to a separate function to make it more readable. Per Andres Freund's suggestion. Author: Amit Langote Discussion: https://www.postgresql.org/message-id/CA%2BHiwqEUgb5RdUgxR7Sqco4S09jzJstHiaT2vnCRPGR4JCAPqA%40mail.gmail.com --- src/backend/executor/nodeModifyTable.c | 265 +++++++++++++++---------- 1 file changed, 158 insertions(+), 107 deletions(-) diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index 6782a2dcd283..0c055ed40805 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -1059,6 +1059,148 @@ ldelete:; return NULL; } +/* + * ExecCrossPartitionUpdate --- Move an updated tuple to another partition. + * + * This works by first deleting the old tuple from the current partition, + * followed by inserting the new tuple into the root parent table, that is, + * mtstate->rootResultRelInfo. It will be re-routed from there to the + * correct partition. + * + * Returns true if the tuple has been successfully moved, or if it's found + * that the tuple was concurrently deleted so there's nothing more to do + * for the caller. + * + * False is returned if the tuple we're trying to move is found to have been + * concurrently updated. In that case, the caller must to check if the + * updated tuple that's returned in *retry_slot still needs to be re-routed, + * and call this function again or perform a regular update accordingly. + */ +static bool +ExecCrossPartitionUpdate(ModifyTableState *mtstate, + ResultRelInfo *resultRelInfo, + ItemPointer tupleid, HeapTuple oldtuple, + TupleTableSlot *slot, TupleTableSlot *planSlot, + EPQState *epqstate, bool canSetTag, + TupleTableSlot **retry_slot, + TupleTableSlot **inserted_tuple) +{ + EState *estate = mtstate->ps.state; + PartitionTupleRouting *proute = mtstate->mt_partition_tuple_routing; + int map_index; + TupleConversionMap *tupconv_map; + TupleConversionMap *saved_tcs_map = NULL; + bool tuple_deleted; + TupleTableSlot *epqslot = NULL; + + *inserted_tuple = NULL; + *retry_slot = NULL; + + /* + * Disallow an INSERT ON CONFLICT DO UPDATE that causes the original row + * to migrate to a different partition. Maybe this can be implemented + * some day, but it seems a fringe feature with little redeeming value. + */ + if (((ModifyTable *) mtstate->ps.plan)->onConflictAction == ONCONFLICT_UPDATE) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("invalid ON UPDATE specification"), + errdetail("The result tuple would appear in a different partition than the original tuple."))); + + /* + * When an UPDATE is run on a leaf partition, we will not have partition + * tuple routing set up. In that case, fail with partition constraint + * violation error. + */ + if (proute == NULL) + ExecPartitionCheckEmitError(resultRelInfo, slot, estate); + + /* + * Row movement, part 1. Delete the tuple, but skip RETURNING processing. + * We want to return rows from INSERT. + */ + ExecDelete(mtstate, resultRelInfo, tupleid, oldtuple, planSlot, + epqstate, estate, + false, /* processReturning */ + false, /* canSetTag */ + true, /* changingPart */ + &tuple_deleted, &epqslot); + + /* + * For some reason if DELETE didn't happen (e.g. trigger prevented it, or + * it was already deleted by self, or it was concurrently deleted by + * another transaction), then we should skip the insert as well; + * otherwise, an UPDATE could cause an increase in the total number of + * rows across all partitions, which is clearly wrong. + * + * For a normal UPDATE, the case where the tuple has been the subject of a + * concurrent UPDATE or DELETE would be handled by the EvalPlanQual + * machinery, but for an UPDATE that we've translated into a DELETE from + * this partition and an INSERT into some other partition, that's not + * available, because CTID chains can't span relation boundaries. We + * mimic the semantics to a limited extent by skipping the INSERT if the + * DELETE fails to find a tuple. This ensures that two concurrent + * attempts to UPDATE the same tuple at the same time can't turn one tuple + * into two, and that an UPDATE of a just-deleted tuple can't resurrect + * it. + */ + if (!tuple_deleted) + { + /* + * epqslot will be typically NULL. But when ExecDelete() finds that + * another transaction has concurrently updated the same row, it + * re-fetches the row, skips the delete, and epqslot is set to the + * re-fetched tuple slot. In that case, we need to do all the checks + * again. + */ + if (TupIsNull(epqslot)) + return true; + else + { + *retry_slot = ExecFilterJunk(resultRelInfo->ri_junkFilter, epqslot); + return false; + } + } + + /* + * resultRelInfo is one of the per-subplan resultRelInfos. So we should + * convert the tuple into root's tuple descriptor, since ExecInsert() + * starts the search from root. The tuple conversion map list is in the + * order of mtstate->resultRelInfo[], so to retrieve the one for this + * resultRel, we need to know the position of the resultRel in + * mtstate->resultRelInfo[]. + */ + map_index = resultRelInfo - mtstate->resultRelInfo; + Assert(map_index >= 0 && map_index < mtstate->mt_nplans); + tupconv_map = tupconv_map_for_subplan(mtstate, map_index); + if (tupconv_map != NULL) + slot = execute_attr_map_slot(tupconv_map->attrMap, + slot, + mtstate->mt_root_tuple_slot); + + /* + * ExecInsert() may scribble on mtstate->mt_transition_capture, so save + * the currently active map. + */ + if (mtstate->mt_transition_capture) + saved_tcs_map = mtstate->mt_transition_capture->tcs_map; + + /* Tuple routing starts from the root table. */ + Assert(mtstate->rootResultRelInfo != NULL); + *inserted_tuple = ExecInsert(mtstate, mtstate->rootResultRelInfo, slot, + planSlot, estate, canSetTag); + + /* Clear the INSERT's tuple and restore the saved map. */ + if (mtstate->mt_transition_capture) + { + mtstate->mt_transition_capture->tcs_original_insert_tuple = NULL; + mtstate->mt_transition_capture->tcs_map = saved_tcs_map; + } + + /* We're done moving. */ + return true; +} + /* ---------------------------------------------------------------- * ExecUpdate * @@ -1212,119 +1354,28 @@ lreplace:; */ if (partition_constraint_failed) { - bool tuple_deleted; - TupleTableSlot *ret_slot; - TupleTableSlot *epqslot = NULL; - PartitionTupleRouting *proute = mtstate->mt_partition_tuple_routing; - int map_index; - TupleConversionMap *tupconv_map; - TupleConversionMap *saved_tcs_map = NULL; - - /* - * Disallow an INSERT ON CONFLICT DO UPDATE that causes the - * original row to migrate to a different partition. Maybe this - * can be implemented some day, but it seems a fringe feature with - * little redeeming value. - */ - if (((ModifyTable *) mtstate->ps.plan)->onConflictAction == ONCONFLICT_UPDATE) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("invalid ON UPDATE specification"), - errdetail("The result tuple would appear in a different partition than the original tuple."))); - - /* - * When an UPDATE is run on a leaf partition, we will not have - * partition tuple routing set up. In that case, fail with - * partition constraint violation error. - */ - if (proute == NULL) - ExecPartitionCheckEmitError(resultRelInfo, slot, estate); - - /* - * Row movement, part 1. Delete the tuple, but skip RETURNING - * processing. We want to return rows from INSERT. - */ - ExecDelete(mtstate, resultRelInfo, tupleid, oldtuple, planSlot, - epqstate, estate, - false, /* processReturning */ - false, /* canSetTag */ - true, /* changingPart */ - &tuple_deleted, &epqslot); - - /* - * For some reason if DELETE didn't happen (e.g. trigger prevented - * it, or it was already deleted by self, or it was concurrently - * deleted by another transaction), then we should skip the insert - * as well; otherwise, an UPDATE could cause an increase in the - * total number of rows across all partitions, which is clearly - * wrong. - * - * For a normal UPDATE, the case where the tuple has been the - * subject of a concurrent UPDATE or DELETE would be handled by - * the EvalPlanQual machinery, but for an UPDATE that we've - * translated into a DELETE from this partition and an INSERT into - * some other partition, that's not available, because CTID chains - * can't span relation boundaries. We mimic the semantics to a - * limited extent by skipping the INSERT if the DELETE fails to - * find a tuple. This ensures that two concurrent attempts to - * UPDATE the same tuple at the same time can't turn one tuple - * into two, and that an UPDATE of a just-deleted tuple can't - * resurrect it. - */ - if (!tuple_deleted) - { - /* - * epqslot will be typically NULL. But when ExecDelete() - * finds that another transaction has concurrently updated the - * same row, it re-fetches the row, skips the delete, and - * epqslot is set to the re-fetched tuple slot. In that case, - * we need to do all the checks again. - */ - if (TupIsNull(epqslot)) - return NULL; - else - { - slot = ExecFilterJunk(resultRelInfo->ri_junkFilter, epqslot); - goto lreplace; - } - } + TupleTableSlot *inserted_tuple, + *retry_slot; + bool retry; /* - * resultRelInfo is one of the per-subplan resultRelInfos. So we - * should convert the tuple into root's tuple descriptor, since - * ExecInsert() starts the search from root. The tuple conversion - * map list is in the order of mtstate->resultRelInfo[], so to - * retrieve the one for this resultRel, we need to know the - * position of the resultRel in mtstate->resultRelInfo[]. + * ExecCrossPartitionUpdate will first DELETE the row from the + * partition it's currently in and then insert it back into the + * root table, which will re-route it to the correct partition. + * The first part may have to be repeated if it is detected that + * the tuple we're trying to move has been concurrently updated. */ - map_index = resultRelInfo - mtstate->resultRelInfo; - Assert(map_index >= 0 && map_index < mtstate->mt_nplans); - tupconv_map = tupconv_map_for_subplan(mtstate, map_index); - if (tupconv_map != NULL) - slot = execute_attr_map_slot(tupconv_map->attrMap, - slot, - mtstate->mt_root_tuple_slot); - - /* - * ExecInsert() may scribble on mtstate->mt_transition_capture, so - * save the currently active map. - */ - if (mtstate->mt_transition_capture) - saved_tcs_map = mtstate->mt_transition_capture->tcs_map; - - /* Tuple routing starts from the root table. */ - Assert(mtstate->rootResultRelInfo != NULL); - ret_slot = ExecInsert(mtstate, mtstate->rootResultRelInfo, slot, - planSlot, estate, canSetTag); - - /* Clear the INSERT's tuple and restore the saved map. */ - if (mtstate->mt_transition_capture) + retry = !ExecCrossPartitionUpdate(mtstate, resultRelInfo, tupleid, + oldtuple, slot, planSlot, + epqstate, canSetTag, + &retry_slot, &inserted_tuple); + if (retry) { - mtstate->mt_transition_capture->tcs_original_insert_tuple = NULL; - mtstate->mt_transition_capture->tcs_map = saved_tcs_map; + slot = retry_slot; + goto lreplace; } - return ret_slot; + return inserted_tuple; } /* From b05fe7b442fd185b9705c537adb602b3b9d49518 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Thu, 15 Oct 2020 11:33:48 -0300 Subject: [PATCH 309/589] Review logical replication tablesync code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Most importantly, remove optimization in LogicalRepSyncTableStart that skips the normal walrcv_startstreaming/endstreaming dance. The optimization is not critically important for production uses anyway, since it only fires in cases with no activity, and saves an uninteresting amount of work even then. Critically, it obscures bugs by hiding the interesting code path from test cases. Also: in GetSubscriptionRelState, remove pointless relation open; access pg_subscription_rel->srsubstate with GETSTRUCT as is typical rather than SysCacheGetAttr; remove unused 'missing_ok' argument. In wait_for_relation_state_change, use explicit catalog snapshot invalidation rather than obscurely (and expensively) through GetLatestSnapshot. In various places: sprinkle comments more liberally and rewrite a number of them. Other cosmetic code improvements. No backpatch, since no bug is being fixed here. Author: Álvaro Herrera Reviewed-by: Petr Jelínek Discussion: https://postgr.es/m/20201010190637.GA5774@alvherre.pgsql --- src/backend/catalog/pg_subscription.c | 27 +- src/backend/replication/logical/relation.c | 3 +- src/backend/replication/logical/tablesync.c | 261 +++++++++----------- src/backend/replication/logical/worker.c | 15 +- src/include/catalog/pg_subscription_rel.h | 3 +- 5 files changed, 131 insertions(+), 178 deletions(-) diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c index 311d46225adf..ca78d395181a 100644 --- a/src/backend/catalog/pg_subscription.c +++ b/src/backend/catalog/pg_subscription.c @@ -328,20 +328,16 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state, /* * Get state of subscription table. * - * Returns SUBREL_STATE_UNKNOWN when not found and missing_ok is true. + * Returns SUBREL_STATE_UNKNOWN when the table is not in the subscription. */ char -GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn, - bool missing_ok) +GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn) { - Relation rel; HeapTuple tup; char substate; bool isnull; Datum d; - rel = table_open(SubscriptionRelRelationId, AccessShareLock); - /* Try finding the mapping. */ tup = SearchSysCache2(SUBSCRIPTIONRELMAP, ObjectIdGetDatum(relid), @@ -349,22 +345,14 @@ GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn, if (!HeapTupleIsValid(tup)) { - if (missing_ok) - { - table_close(rel, AccessShareLock); - *sublsn = InvalidXLogRecPtr; - return SUBREL_STATE_UNKNOWN; - } - - elog(ERROR, "subscription table %u in subscription %u does not exist", - relid, subid); + *sublsn = InvalidXLogRecPtr; + return SUBREL_STATE_UNKNOWN; } /* Get the state. */ - d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup, - Anum_pg_subscription_rel_srsubstate, &isnull); - Assert(!isnull); - substate = DatumGetChar(d); + substate = ((Form_pg_subscription_rel) GETSTRUCT(tup))->srsubstate; + + /* Get the LSN */ d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup, Anum_pg_subscription_rel_srsublsn, &isnull); if (isnull) @@ -374,7 +362,6 @@ GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn, /* Cleanup */ ReleaseSysCache(tup); - table_close(rel, AccessShareLock); return substate; } diff --git a/src/backend/replication/logical/relation.c b/src/backend/replication/logical/relation.c index a6596f79a66a..07aa52977f96 100644 --- a/src/backend/replication/logical/relation.c +++ b/src/backend/replication/logical/relation.c @@ -437,8 +437,7 @@ logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode) if (entry->state != SUBREL_STATE_READY) entry->state = GetSubscriptionRelState(MySubscription->oid, entry->localreloid, - &entry->statelsn, - true); + &entry->statelsn); return entry; } diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 843c9285d59d..a91b00ed4bcf 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -1,6 +1,6 @@ /*------------------------------------------------------------------------- * tablesync.c - * PostgreSQL logical replication + * PostgreSQL logical replication: initial table data synchronization * * Copyright (c) 2012-2020, PostgreSQL Global Development Group * @@ -26,26 +26,30 @@ * - It allows us to synchronize any tables added after the initial * synchronization has finished. * - * The stream position synchronization works in multiple steps. - * - Sync finishes copy and sets worker state as SYNCWAIT and waits for - * state to change in a loop. - * - Apply periodically checks tables that are synchronizing for SYNCWAIT. - * When the desired state appears, it will set the worker state to - * CATCHUP and starts loop-waiting until either the table state is set - * to SYNCDONE or the sync worker exits. + * The stream position synchronization works in multiple steps: + * - Apply worker requests a tablesync worker to start, setting the new + * table state to INIT. + * - Tablesync worker starts; changes table state from INIT to DATASYNC while + * copying. + * - Tablesync worker finishes the copy and sets table state to SYNCWAIT; + * waits for state change. + * - Apply worker periodically checks for tables in SYNCWAIT state. When + * any appear, it sets the table state to CATCHUP and starts loop-waiting + * until either the table state is set to SYNCDONE or the sync worker + * exits. * - After the sync worker has seen the state change to CATCHUP, it will * read the stream and apply changes (acting like an apply worker) until * it catches up to the specified stream position. Then it sets the * state to SYNCDONE. There might be zero changes applied between * CATCHUP and SYNCDONE, because the sync worker might be ahead of the * apply worker. - * - Once the state was set to SYNCDONE, the apply will continue tracking + * - Once the state is set to SYNCDONE, the apply will continue tracking * the table until it reaches the SYNCDONE stream position, at which * point it sets state to READY and stops tracking. Again, there might * be zero changes in between. * - * So the state progression is always: INIT -> DATASYNC -> SYNCWAIT -> CATCHUP -> - * SYNCDONE -> READY. + * So the state progression is always: INIT -> DATASYNC -> SYNCWAIT -> + * CATCHUP -> SYNCDONE -> READY. * * The catalog pg_subscription_rel is used to keep information about * subscribed tables and their state. Some transient state during data @@ -67,7 +71,8 @@ * -> continue rep * apply:11 * -> set in catalog READY - * - Sync in front: + * + * - Sync is in front: * sync:10 * -> set in memory SYNCWAIT * apply:8 @@ -142,13 +147,14 @@ finish_sync_worker(void) } /* - * Wait until the relation synchronization state is set in the catalog to the - * expected one. + * Wait until the relation sync state is set in the catalog to the expected + * one; return true when it happens. * - * Used when transitioning from CATCHUP state to SYNCDONE. + * Returns false if the table sync worker or the table itself have + * disappeared, or the table state has been reset. * - * Returns false if the synchronization worker has disappeared or the table state - * has been reset. + * Currently, this is used in the apply worker when transitioning from + * CATCHUP state to SYNCDONE. */ static bool wait_for_relation_state_change(Oid relid, char expected_state) @@ -162,28 +168,23 @@ wait_for_relation_state_change(Oid relid, char expected_state) CHECK_FOR_INTERRUPTS(); - /* XXX use cache invalidation here to improve performance? */ - PushActiveSnapshot(GetLatestSnapshot()); + InvalidateCatalogSnapshot(); state = GetSubscriptionRelState(MyLogicalRepWorker->subid, - relid, &statelsn, true); - PopActiveSnapshot(); + relid, &statelsn); if (state == SUBREL_STATE_UNKNOWN) - return false; + break; if (state == expected_state) return true; /* Check if the sync worker is still running and bail if not. */ LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); - - /* Check if the opposite worker is still running and bail if not. */ - worker = logicalrep_worker_find(MyLogicalRepWorker->subid, - am_tablesync_worker() ? InvalidOid : relid, + worker = logicalrep_worker_find(MyLogicalRepWorker->subid, relid, false); LWLockRelease(LogicalRepWorkerLock); if (!worker) - return false; + break; (void) WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, @@ -810,6 +811,9 @@ copy_table(Relation rel) /* * Start syncing the table in the sync worker. * + * If nothing needs to be done to sync the table, we exit the worker without + * any further action. + * * The returned slot name is palloc'ed in current memory context. */ char * @@ -819,12 +823,14 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos) char *err; char relstate; XLogRecPtr relstate_lsn; + Relation rel; + WalRcvExecResult *res; /* Check the state of the table synchronization. */ StartTransactionCommand(); relstate = GetSubscriptionRelState(MyLogicalRepWorker->subid, MyLogicalRepWorker->relid, - &relstate_lsn, true); + &relstate_lsn); CommitTransactionCommand(); SpinLockAcquire(&MyLogicalRepWorker->relmutex); @@ -832,6 +838,18 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos) MyLogicalRepWorker->relstate_lsn = relstate_lsn; SpinLockRelease(&MyLogicalRepWorker->relmutex); + /* + * If synchronization is already done or no longer necessary, exit now + * that we've updated shared memory state. + */ + switch (relstate) + { + case SUBREL_STATE_SYNCDONE: + case SUBREL_STATE_READY: + case SUBREL_STATE_UNKNOWN: + finish_sync_worker(); /* doesn't return */ + } + /* * To build a slot name for the sync work, we are limited to NAMEDATALEN - * 1 characters. We cut the original slot name to NAMEDATALEN - 28 chars @@ -856,134 +874,87 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos) ereport(ERROR, (errmsg("could not connect to the publisher: %s", err))); - switch (MyLogicalRepWorker->relstate) - { - case SUBREL_STATE_INIT: - case SUBREL_STATE_DATASYNC: - { - Relation rel; - WalRcvExecResult *res; + Assert(MyLogicalRepWorker->relstate == SUBREL_STATE_INIT || + MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC); - SpinLockAcquire(&MyLogicalRepWorker->relmutex); - MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC; - MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr; - SpinLockRelease(&MyLogicalRepWorker->relmutex); - - /* Update the state and make it visible to others. */ - StartTransactionCommand(); - UpdateSubscriptionRelState(MyLogicalRepWorker->subid, - MyLogicalRepWorker->relid, - MyLogicalRepWorker->relstate, - MyLogicalRepWorker->relstate_lsn); - CommitTransactionCommand(); - pgstat_report_stat(false); + SpinLockAcquire(&MyLogicalRepWorker->relmutex); + MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC; + MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr; + SpinLockRelease(&MyLogicalRepWorker->relmutex); - /* - * We want to do the table data sync in a single transaction. - */ - StartTransactionCommand(); + /* Update the state and make it visible to others. */ + StartTransactionCommand(); + UpdateSubscriptionRelState(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + MyLogicalRepWorker->relstate, + MyLogicalRepWorker->relstate_lsn); + CommitTransactionCommand(); + pgstat_report_stat(false); - /* - * Use a standard write lock here. It might be better to - * disallow access to the table while it's being synchronized. - * But we don't want to block the main apply process from - * working and it has to open the relation in RowExclusiveLock - * when remapping remote relation id to local one. - */ - rel = table_open(MyLogicalRepWorker->relid, RowExclusiveLock); + /* + * We want to do the table data sync in a single transaction. + */ + StartTransactionCommand(); - /* - * Create a temporary slot for the sync process. We do this - * inside the transaction so that we can use the snapshot made - * by the slot to get existing data. - */ - res = walrcv_exec(wrconn, - "BEGIN READ ONLY ISOLATION LEVEL " - "REPEATABLE READ", 0, NULL); - if (res->status != WALRCV_OK_COMMAND) - ereport(ERROR, - (errmsg("table copy could not start transaction on publisher"), - errdetail("The error was: %s", res->err))); - walrcv_clear_result(res); + /* + * Use a standard write lock here. It might be better to disallow access + * to the table while it's being synchronized. But we don't want to block + * the main apply process from working and it has to open the relation in + * RowExclusiveLock when remapping remote relation id to local one. + */ + rel = table_open(MyLogicalRepWorker->relid, RowExclusiveLock); - /* - * Create new temporary logical decoding slot. - * - * We'll use slot for data copy so make sure the snapshot is - * used for the transaction; that way the COPY will get data - * that is consistent with the lsn used by the slot to start - * decoding. - */ - walrcv_create_slot(wrconn, slotname, true, - CRS_USE_SNAPSHOT, origin_startpos); + /* + * Start a transaction in the remote node in REPEATABLE READ mode. This + * ensures that both the replication slot we create (see below) and the + * COPY are consistent with each other. + */ + res = walrcv_exec(wrconn, + "BEGIN READ ONLY ISOLATION LEVEL REPEATABLE READ", + 0, NULL); + if (res->status != WALRCV_OK_COMMAND) + ereport(ERROR, + (errmsg("table copy could not start transaction on publisher"), + errdetail("The error was: %s", res->err))); + walrcv_clear_result(res); - PushActiveSnapshot(GetTransactionSnapshot()); - copy_table(rel); - PopActiveSnapshot(); + /* + * Create a new temporary logical decoding slot. This slot will be used + * for the catchup phase after COPY is done, so tell it to use the + * snapshot to make the final data consistent. + */ + walrcv_create_slot(wrconn, slotname, true, + CRS_USE_SNAPSHOT, origin_startpos); - res = walrcv_exec(wrconn, "COMMIT", 0, NULL); - if (res->status != WALRCV_OK_COMMAND) - ereport(ERROR, - (errmsg("table copy could not finish transaction on publisher"), - errdetail("The error was: %s", res->err))); - walrcv_clear_result(res); + /* Now do the initial data copy */ + PushActiveSnapshot(GetTransactionSnapshot()); + copy_table(rel); + PopActiveSnapshot(); - table_close(rel, NoLock); + res = walrcv_exec(wrconn, "COMMIT", 0, NULL); + if (res->status != WALRCV_OK_COMMAND) + ereport(ERROR, + (errmsg("table copy could not finish transaction on publisher"), + errdetail("The error was: %s", res->err))); + walrcv_clear_result(res); - /* Make the copy visible. */ - CommandCounterIncrement(); + table_close(rel, NoLock); - /* - * We are done with the initial data synchronization, update - * the state. - */ - SpinLockAcquire(&MyLogicalRepWorker->relmutex); - MyLogicalRepWorker->relstate = SUBREL_STATE_SYNCWAIT; - MyLogicalRepWorker->relstate_lsn = *origin_startpos; - SpinLockRelease(&MyLogicalRepWorker->relmutex); - - /* Wait for main apply worker to tell us to catchup. */ - wait_for_worker_state_change(SUBREL_STATE_CATCHUP); - - /*---------- - * There are now two possible states here: - * a) Sync is behind the apply. If that's the case we need to - * catch up with it by consuming the logical replication - * stream up to the relstate_lsn. For that, we exit this - * function and continue in ApplyWorkerMain(). - * b) Sync is caught up with the apply. So it can just set - * the state to SYNCDONE and finish. - *---------- - */ - if (*origin_startpos >= MyLogicalRepWorker->relstate_lsn) - { - /* - * Update the new state in catalog. No need to bother - * with the shmem state as we are exiting for good. - */ - UpdateSubscriptionRelState(MyLogicalRepWorker->subid, - MyLogicalRepWorker->relid, - SUBREL_STATE_SYNCDONE, - *origin_startpos); - finish_sync_worker(); - } - break; - } - case SUBREL_STATE_SYNCDONE: - case SUBREL_STATE_READY: - case SUBREL_STATE_UNKNOWN: + /* Make the copy visible. */ + CommandCounterIncrement(); - /* - * Nothing to do here but finish. (UNKNOWN means the relation was - * removed from pg_subscription_rel before the sync worker could - * start.) - */ - finish_sync_worker(); - break; - default: - elog(ERROR, "unknown relation state \"%c\"", - MyLogicalRepWorker->relstate); - } + /* + * We are done with the initial data synchronization, update the state. + */ + SpinLockAcquire(&MyLogicalRepWorker->relmutex); + MyLogicalRepWorker->relstate = SUBREL_STATE_SYNCWAIT; + MyLogicalRepWorker->relstate_lsn = *origin_startpos; + SpinLockRelease(&MyLogicalRepWorker->relmutex); + /* + * Finally, wait until the main apply worker tells us to catch up and then + * return to let LogicalRepApplyLoop do it. + */ + wait_for_worker_state_change(SUBREL_STATE_CATCHUP); return slotname; } diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 640409b757f8..b8e297c5d34e 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -2060,6 +2060,7 @@ LogicalRepApplyLoop(XLogRecPtr last_received) { TimestampTz last_recv_timestamp = GetCurrentTimestamp(); bool ping_sent = false; + TimeLineID tli; /* * Init the ApplyMessageContext which we clean up after each replication @@ -2201,12 +2202,7 @@ LogicalRepApplyLoop(XLogRecPtr last_received) /* Check if we need to exit the streaming loop. */ if (endofstream) - { - TimeLineID tli; - - walrcv_endstreaming(wrconn, &tli); break; - } /* * Wait for more data or latch. If we have unflushed transactions, @@ -2283,6 +2279,9 @@ LogicalRepApplyLoop(XLogRecPtr last_received) send_feedback(last_received, requestReply, requestReply); } } + + /* All done */ + walrcv_endstreaming(wrconn, &tli); } /* @@ -3024,10 +3023,8 @@ ApplyWorkerMain(Datum main_arg) /* This is table synchronization worker, call initial sync. */ syncslotname = LogicalRepSyncTableStart(&origin_startpos); - /* The slot name needs to be allocated in permanent memory context. */ - oldctx = MemoryContextSwitchTo(ApplyContext); - myslotname = pstrdup(syncslotname); - MemoryContextSwitchTo(oldctx); + /* allocate slot name in long-lived context */ + myslotname = MemoryContextStrdup(ApplyContext, syncslotname); pfree(syncslotname); } diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h index f384f4e7fa65..ff5c8d7ff91c 100644 --- a/src/include/catalog/pg_subscription_rel.h +++ b/src/include/catalog/pg_subscription_rel.h @@ -80,8 +80,7 @@ extern void AddSubscriptionRelState(Oid subid, Oid relid, char state, XLogRecPtr sublsn); extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state, XLogRecPtr sublsn); -extern char GetSubscriptionRelState(Oid subid, Oid relid, - XLogRecPtr *sublsn, bool missing_ok); +extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn); extern void RemoveSubscriptionRel(Oid subid, Oid relid); extern List *GetSubscriptionRelations(Oid subid); From 2203ede9ae85b6423f850466122606275ea09b17 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Thu, 15 Oct 2020 13:09:29 -0300 Subject: [PATCH 310/589] Install pg_isolation_regress and isolationtester MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We already install assorted tools for testing extensions, but these two were missing. Having them installed, and after ISOLATION support was added to PGXS's makefiles by d3c09b9b1307, helps third-party modules usefully include isolation tests. Compare c3a0818460a8. Author: Craig Ringer Reviewed-by: Álvaro Herrera Discussion: https://postgr.es/m/CAMsr+YFsCMH3B4uOPFE+2qWM6k=o=hf9LGiPNCfwqKdUPz_BsQ@mail.gmail.com --- src/test/isolation/Makefile | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/test/isolation/Makefile b/src/test/isolation/Makefile index da5e088bdde7..d23e2cec6400 100644 --- a/src/test/isolation/Makefile +++ b/src/test/isolation/Makefile @@ -18,12 +18,16 @@ OBJS = \ all: isolationtester$(X) pg_isolation_regress$(X) -# Though we don't install these binaries, build them during installation -# (including temp-install). Otherwise, "make -j check-world" and "make -j -# installcheck-world" would spawn multiple, concurrent builds in this -# directory. Later builds would overwrite files while earlier builds are -# reading them, causing occasional failures. -install: | all +install: all installdirs + $(INSTALL_PROGRAM) pg_isolation_regress$(X) '$(DESTDIR)$(pgxsdir)/$(subdir)/pg_isolation_regress$(X)' + $(INSTALL_PROGRAM) isolationtester$(X) '$(DESTDIR)$(pgxsdir)/$(subdir)/isolationtester$(X)' + +installdirs: + $(MKDIR_P) '$(DESTDIR)$(pgxsdir)/$(subdir)' + +uninstall: + rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/pg_isolation_regress$(X)' + rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/isolationtester$(X)' submake-regress: $(MAKE) -C $(top_builddir)/src/test/regress pg_regress.o From 85adb5e91ec2f45a388bef7a92a3d988c7e45501 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Thu, 15 Oct 2020 14:32:34 -0300 Subject: [PATCH 311/589] Remove pointless HeapTupleHeaderIndicatesMovedPartitions calls Pavan Deolasee recently noted that a few of the HeapTupleHeaderIndicatesMovedPartitions calls added by commit 5db6df0c0117 are useless, since they are done after comparing t_self with t_ctid. But because t_self can never be set to the magical values that indicate that the tuple moved partition, this can never succeed: if the first test fails (so we know t_self equals t_ctid), necessarily the second test will also fail. So these checks can be removed and no harm is done. Discussion: https://postgr.es/m/20200929164411.GA15497@alvherre.pgsql --- src/backend/access/heap/heapam.c | 12 ++++-------- src/backend/access/heap/heapam_visibility.c | 9 +++------ 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 1585861a021d..868ff134539e 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -2618,8 +2618,7 @@ heap_delete(Relation relation, ItemPointer tid, HEAP_XMAX_IS_LOCKED_ONLY(tp.t_data->t_infomask) || HeapTupleHeaderIsOnlyLocked(tp.t_data)) result = TM_Ok; - else if (!ItemPointerEquals(&tp.t_self, &tp.t_data->t_ctid) || - HeapTupleHeaderIndicatesMovedPartitions(tp.t_data)) + else if (!ItemPointerEquals(&tp.t_self, &tp.t_data->t_ctid)) result = TM_Updated; else result = TM_Deleted; @@ -3248,8 +3247,7 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, if (can_continue) result = TM_Ok; - else if (!ItemPointerEquals(&oldtup.t_self, &oldtup.t_data->t_ctid) || - HeapTupleHeaderIndicatesMovedPartitions(oldtup.t_data)) + else if (!ItemPointerEquals(&oldtup.t_self, &oldtup.t_data->t_ctid)) result = TM_Updated; else result = TM_Deleted; @@ -4485,8 +4483,7 @@ heap_lock_tuple(Relation relation, HeapTuple tuple, HEAP_XMAX_IS_LOCKED_ONLY(tuple->t_data->t_infomask) || HeapTupleHeaderIsOnlyLocked(tuple->t_data)) result = TM_Ok; - else if (!ItemPointerEquals(&tuple->t_self, &tuple->t_data->t_ctid) || - HeapTupleHeaderIndicatesMovedPartitions(tuple->t_data)) + else if (!ItemPointerEquals(&tuple->t_self, &tuple->t_data->t_ctid)) result = TM_Updated; else result = TM_Deleted; @@ -5059,8 +5056,7 @@ test_lockmode_for_conflict(MultiXactStatus status, TransactionId xid, LOCKMODE_from_mxstatus(wantedstatus))) { /* bummer */ - if (!ItemPointerEquals(&tup->t_self, &tup->t_data->t_ctid) || - HeapTupleHeaderIndicatesMovedPartitions(tup->t_data)) + if (!ItemPointerEquals(&tup->t_self, &tup->t_data->t_ctid)) return TM_Updated; else return TM_Deleted; diff --git a/src/backend/access/heap/heapam_visibility.c b/src/backend/access/heap/heapam_visibility.c index 80bd4940769c..cab6a48a5dae 100644 --- a/src/backend/access/heap/heapam_visibility.c +++ b/src/backend/access/heap/heapam_visibility.c @@ -607,8 +607,7 @@ HeapTupleSatisfiesUpdate(HeapTuple htup, CommandId curcid, { if (HEAP_XMAX_IS_LOCKED_ONLY(tuple->t_infomask)) return TM_Ok; - if (!ItemPointerEquals(&htup->t_self, &tuple->t_ctid) || - HeapTupleHeaderIndicatesMovedPartitions(tuple)) + if (!ItemPointerEquals(&htup->t_self, &tuple->t_ctid)) return TM_Updated; /* updated by other */ else return TM_Deleted; /* deleted by other */ @@ -653,8 +652,7 @@ HeapTupleSatisfiesUpdate(HeapTuple htup, CommandId curcid, if (TransactionIdDidCommit(xmax)) { - if (!ItemPointerEquals(&htup->t_self, &tuple->t_ctid) || - HeapTupleHeaderIndicatesMovedPartitions(tuple)) + if (!ItemPointerEquals(&htup->t_self, &tuple->t_ctid)) return TM_Updated; else return TM_Deleted; @@ -714,8 +712,7 @@ HeapTupleSatisfiesUpdate(HeapTuple htup, CommandId curcid, SetHintBits(tuple, buffer, HEAP_XMAX_COMMITTED, HeapTupleHeaderGetRawXmax(tuple)); - if (!ItemPointerEquals(&htup->t_self, &tuple->t_ctid) || - HeapTupleHeaderIndicatesMovedPartitions(tuple)) + if (!ItemPointerEquals(&htup->t_self, &tuple->t_ctid)) return TM_Updated; /* updated by other */ else return TM_Deleted; /* deleted by other */ From 93f84d59f80d11a3d8ade9ae71560162d6f3ecb2 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Thu, 15 Oct 2020 15:16:11 -0300 Subject: [PATCH 312/589] Revert "Remove pointless HeapTupleHeaderIndicatesMovedPartitions calls" This reverts commit 85adb5e91ec2. It was not intended for commit just yet. --- src/backend/access/heap/heapam.c | 12 ++++++++---- src/backend/access/heap/heapam_visibility.c | 9 ++++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 868ff134539e..1585861a021d 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -2618,7 +2618,8 @@ heap_delete(Relation relation, ItemPointer tid, HEAP_XMAX_IS_LOCKED_ONLY(tp.t_data->t_infomask) || HeapTupleHeaderIsOnlyLocked(tp.t_data)) result = TM_Ok; - else if (!ItemPointerEquals(&tp.t_self, &tp.t_data->t_ctid)) + else if (!ItemPointerEquals(&tp.t_self, &tp.t_data->t_ctid) || + HeapTupleHeaderIndicatesMovedPartitions(tp.t_data)) result = TM_Updated; else result = TM_Deleted; @@ -3247,7 +3248,8 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, if (can_continue) result = TM_Ok; - else if (!ItemPointerEquals(&oldtup.t_self, &oldtup.t_data->t_ctid)) + else if (!ItemPointerEquals(&oldtup.t_self, &oldtup.t_data->t_ctid) || + HeapTupleHeaderIndicatesMovedPartitions(oldtup.t_data)) result = TM_Updated; else result = TM_Deleted; @@ -4483,7 +4485,8 @@ heap_lock_tuple(Relation relation, HeapTuple tuple, HEAP_XMAX_IS_LOCKED_ONLY(tuple->t_data->t_infomask) || HeapTupleHeaderIsOnlyLocked(tuple->t_data)) result = TM_Ok; - else if (!ItemPointerEquals(&tuple->t_self, &tuple->t_data->t_ctid)) + else if (!ItemPointerEquals(&tuple->t_self, &tuple->t_data->t_ctid) || + HeapTupleHeaderIndicatesMovedPartitions(tuple->t_data)) result = TM_Updated; else result = TM_Deleted; @@ -5056,7 +5059,8 @@ test_lockmode_for_conflict(MultiXactStatus status, TransactionId xid, LOCKMODE_from_mxstatus(wantedstatus))) { /* bummer */ - if (!ItemPointerEquals(&tup->t_self, &tup->t_data->t_ctid)) + if (!ItemPointerEquals(&tup->t_self, &tup->t_data->t_ctid) || + HeapTupleHeaderIndicatesMovedPartitions(tup->t_data)) return TM_Updated; else return TM_Deleted; diff --git a/src/backend/access/heap/heapam_visibility.c b/src/backend/access/heap/heapam_visibility.c index cab6a48a5dae..80bd4940769c 100644 --- a/src/backend/access/heap/heapam_visibility.c +++ b/src/backend/access/heap/heapam_visibility.c @@ -607,7 +607,8 @@ HeapTupleSatisfiesUpdate(HeapTuple htup, CommandId curcid, { if (HEAP_XMAX_IS_LOCKED_ONLY(tuple->t_infomask)) return TM_Ok; - if (!ItemPointerEquals(&htup->t_self, &tuple->t_ctid)) + if (!ItemPointerEquals(&htup->t_self, &tuple->t_ctid) || + HeapTupleHeaderIndicatesMovedPartitions(tuple)) return TM_Updated; /* updated by other */ else return TM_Deleted; /* deleted by other */ @@ -652,7 +653,8 @@ HeapTupleSatisfiesUpdate(HeapTuple htup, CommandId curcid, if (TransactionIdDidCommit(xmax)) { - if (!ItemPointerEquals(&htup->t_self, &tuple->t_ctid)) + if (!ItemPointerEquals(&htup->t_self, &tuple->t_ctid) || + HeapTupleHeaderIndicatesMovedPartitions(tuple)) return TM_Updated; else return TM_Deleted; @@ -712,7 +714,8 @@ HeapTupleSatisfiesUpdate(HeapTuple htup, CommandId curcid, SetHintBits(tuple, buffer, HEAP_XMAX_COMMITTED, HeapTupleHeaderGetRawXmax(tuple)); - if (!ItemPointerEquals(&htup->t_self, &tuple->t_ctid)) + if (!ItemPointerEquals(&htup->t_self, &tuple->t_ctid) || + HeapTupleHeaderIndicatesMovedPartitions(tuple)) return TM_Updated; /* updated by other */ else return TM_Deleted; /* deleted by other */ From a97e85f2be400a08d863c1d65f172eb48af3e9b5 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Thu, 15 Oct 2020 15:15:29 -0400 Subject: [PATCH 313/589] doc: improve description of synchronous_commit modes Previously it wasn't clear exactly what each of the synchronous_commit modes accomplished. This clarifies that, and adds a table describing it. Only backpatched through 9.6 since 9.5 doesn't have all the options. Reported-by: kghost0@gmail.com Discussion: https://postgr.es/m/159741195522.14321.13812604195366728976@wrigleys.postgresql.org Backpatch-through: 9.6 --- doc/src/sgml/config.sgml | 144 ++++++++++++++++++++++++++++++-------- src/include/access/xact.h | 3 +- 2 files changed, 116 insertions(+), 31 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index ee914740cc4b..2768c85a96fb 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -2703,14 +2703,26 @@ include_dir 'conf.d' - Specifies whether transaction commit will wait for WAL records - to be written to disk before the command returns a success - indication to the client. Valid values are on, - remote_apply, remote_write, local, - and off. The default, and safe, setting - is on. When off, there can be a delay between - when success is reported to the client and when the transaction is - really guaranteed to be safe against a server crash. (The maximum + Specifies how much WAL processing must complete before + the database server returns a success + indication to the client. Valid values are + remote_apply, on + (the default), remote_write, + local, and off. + + + + If synchronous_standby_names is empty, + the only meaningful settings are on and + off; remote_apply, + remote_write and local + all provide the same local synchronization level + as on. The local behavior of all + non-off modes is to wait for local flush of WAL + to disk. In off mode, there is no waiting, + so there can be a delay between when success is reported to the + client and when the transaction is later guaranteed to be safe + against a server crash. (The maximum delay is three times .) Unlike , setting this parameter to off does not create any risk of database inconsistency: an operating @@ -2722,38 +2734,40 @@ include_dir 'conf.d' exact certainty about the durability of a transaction. For more discussion see . + - If is non-empty, this - parameter also controls whether or not transaction commits will wait - for their WAL records to be replicated to the standby server(s). - When set to on, commits will wait until replies + If is non-empty, + synchronous_commit also controls whether + transaction commits will wait for their WAL records to be + processed on the standby server(s). + + + + When set to remote_apply, commits will wait + until replies from the current synchronous standby(s) indicate they + have received the commit record of the transaction and applied + it, so that it has become visible to queries on the standby(s), + and also written to durable storage on the standbys. This will + cause much larger commit delays than previous settings since + it waits for WAL replay. When set to on, + commits wait until replies from the current synchronous standby(s) indicate they have received - the commit record of the transaction and flushed it to disk. This + the commit record of the transaction and flushed it to durable storage. This ensures the transaction will not be lost unless both the primary and all synchronous standbys suffer corruption of their database storage. - When set to remote_apply, commits will wait until replies - from the current synchronous standby(s) indicate they have received the - commit record of the transaction and applied it, so that it has become - visible to queries on the standby(s). When set to remote_write, commits will wait until replies from the current synchronous standby(s) indicate they have - received the commit record of the transaction and written it out to - their operating system. This setting is sufficient to - ensure data preservation even if a standby instance of - PostgreSQL were to crash, but not if the standby - suffers an operating-system-level crash, since the data has not + received the commit record of the transaction and written it to + their file systems. This setting ensures data preservation if a standby instance of + PostgreSQL crashes, but not if the standby + suffers an operating-system-level crash because the data has not necessarily reached durable storage on the standby. - Finally, the setting local causes commits to wait for - local flush to disk, but not for replication. This is not usually + The setting local causes commits to wait for + local flush to disk, but not for replication. This is usually not desirable when synchronous replication is in use, but is provided for completeness. - - If synchronous_standby_names is empty, the settings - on, remote_apply, remote_write - and local all provide the same synchronization level: - transaction commits only wait for local flush to disk. - + This parameter can be changed at any time; the behavior for any one transaction is determined by the setting in effect when it @@ -2763,6 +2777,76 @@ include_dir 'conf.d' asynchronously when the default is the opposite, issue SET LOCAL synchronous_commit TO OFF within the transaction. + + + summarizes the + capabilities of the synchronous_commit settings. + + + + synchronous_commit Modes + + + + + + + + + synchronous_commit setting + local durable commit + standby durable commit after PG crash + standby durable commit after OS crash + standby query consistency + + + + + + + remote_apply + • + • + • + • + + + + on + • + • + • + + + + + remote_write + • + • + + + + + + local + • + + + + + + + off + + + + + + + + +
+ diff --git a/src/include/access/xact.h b/src/include/access/xact.h index df1b43a932e3..7320de345c99 100644 --- a/src/include/access/xact.h +++ b/src/include/access/xact.h @@ -72,7 +72,8 @@ typedef enum SYNCHRONOUS_COMMIT_REMOTE_WRITE, /* wait for local flush and remote * write */ SYNCHRONOUS_COMMIT_REMOTE_FLUSH, /* wait for local and remote flush */ - SYNCHRONOUS_COMMIT_REMOTE_APPLY /* wait for local flush and remote apply */ + SYNCHRONOUS_COMMIT_REMOTE_APPLY /* wait for local and remote flush + and remote apply */ } SyncCommitLevel; /* Define the default setting for synchronous_commit */ From 72559438f92f9a7d9205192bef601be953c7bd2a Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Thu, 15 Oct 2020 13:39:41 -0700 Subject: [PATCH 314/589] llvmjit: Also copy parameter / return value attributes from template functions. Previously we only copied the function attributes. That caused problems at least on s390x: Because we didn't copy the 'zeroext' attribute for ExecAggTransReparent()'s *IsNull parameters, expressions invoking it didn't ensure that the upper bytes of the registers were zeroed. In the - relatively rare - cases where not, ExecAggTransReparent() wrongly ended up in the newValueIsNull branch due to the register not being zero. Subsequently causing a crash. It's quite possible that this would cause problems on other platforms, and in other places than just ExecAggTransReparent() on s390x. Thanks to Christoph (and the Debian project) for providing me with access to a s390x machine, allowing me to debug this. Reported-By: Christoph Berg Author: Andres Freund Discussion: https://postgr.es/m/20201015083246.kie5726xerdt3ael@alap3.anarazel.de Backpatch: 11-, where JIT was added --- src/backend/jit/llvm/llvmjit.c | 44 ++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c index 43bed78a5299..7965b61165e3 100644 --- a/src/backend/jit/llvm/llvmjit.c +++ b/src/backend/jit/llvm/llvmjit.c @@ -324,26 +324,46 @@ llvm_pg_func(LLVMModuleRef mod, const char *funcname) } /* - * Copy attributes from one function to another. + * Copy attributes from one function to another, for a specific index (an + * index can reference return value, function and parameter attributes). */ -void -llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to) +static void +llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index) { int num_attributes; - int attno; LLVMAttributeRef *attrs; - num_attributes = - LLVMGetAttributeCountAtIndex(v_from, LLVMAttributeFunctionIndex); + num_attributes = LLVMGetAttributeCountAtIndex(v_from, index); attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes); - LLVMGetAttributesAtIndex(v_from, LLVMAttributeFunctionIndex, attrs); + LLVMGetAttributesAtIndex(v_from, index, attrs); - for (attno = 0; attno < num_attributes; attno++) - { - LLVMAddAttributeAtIndex(v_to, LLVMAttributeFunctionIndex, - attrs[attno]); - } + for (int attno = 0; attno < num_attributes; attno++) + LLVMAddAttributeAtIndex(v_to, index, attrs[attno]); + + pfree(attrs); +} + +/* + * Copy all attributes from one function to another. I.e. function, return and + * parameters will be copied. + */ +void +llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to) +{ + uint32 param_count; + + /* copy function attributes */ + llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeFunctionIndex); + + /* and the return value attributes */ + llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeReturnIndex); + + /* and each function parameter's attribute */ + param_count = LLVMCountParams(v_from); + + for (int paramidx = 1; paramidx <= param_count; paramidx++) + llvm_copy_attributes_at_index(v_from, v_to, paramidx); } /* From 3c0471b5fdd847febb30ec805a3e44f3eac5b66f Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Thu, 15 Oct 2020 19:33:36 -0400 Subject: [PATCH 315/589] pg_upgrade: generate check error for left-over new tablespace Previously, if pg_upgrade failed, and the user recreated the cluster but did not remove the new cluster tablespace directory, a later pg_upgrade would fail since the new tablespace directory would already exists. This adds error reporting for this during check. Reported-by: Justin Pryzby Discussion: https://postgr.es/m/20200925005531.GJ23631@telsasoft.com Backpatch-through: 9.5 --- src/bin/pg_upgrade/check.c | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 168058a873cb..c24ca9b141d0 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -27,6 +27,7 @@ static void check_for_tables_with_oids(ClusterInfo *cluster); static void check_for_reg_data_type_usage(ClusterInfo *cluster); static void check_for_jsonb_9_4_usage(ClusterInfo *cluster); static void check_for_pg_role_prefix(ClusterInfo *cluster); +static void check_for_new_tablespace_dir(ClusterInfo *new_cluster); static char *get_canonical_locale_name(int category, const char *locale); @@ -187,6 +188,8 @@ check_new_cluster(void) check_is_install_user(&new_cluster); check_for_prepared_transactions(&new_cluster); + + check_for_new_tablespace_dir(&new_cluster); } @@ -527,6 +530,43 @@ create_script_for_cluster_analyze(char **analyze_script_file_name) } +/* + * A previous run of pg_upgrade might have failed and the new cluster + * directory recreated, but they might have forgotten to remove + * the new cluster's tablespace directories. Therefore, check that + * new cluster tablespace directories do not already exist. If + * they do, it would cause an error while restoring global objects. + * This allows the failure to be detected at check time, rather than + * during schema restore. + * + * Note, v8.4 has no tablespace_suffix, which is fine so long as the + * version being upgraded *to* has a suffix, since it's not allowed + * to pg_upgrade from a version to the same version if tablespaces are + * in use. + */ +static void +check_for_new_tablespace_dir(ClusterInfo *new_cluster) +{ + char new_tablespace_dir[MAXPGPATH]; + + prep_status("Checking for new cluster tablespace directories"); + + for (int tblnum = 0; tblnum < os_info.num_old_tablespaces; tblnum++) + { + struct stat statbuf; + + snprintf(new_tablespace_dir, MAXPGPATH, "%s%s", + os_info.old_tablespaces[tblnum], + new_cluster->tablespace_suffix); + + if (stat(new_tablespace_dir, &statbuf) == 0 || errno != ENOENT) + pg_fatal("new cluster tablespace directory already exists: \"%s\"\n", + new_tablespace_dir); + } + + check_ok(); +} + /* * create_script_for_old_cluster_deletion() * From 536de14e2bdf467aefc5e25f85056fe1adf4846d Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Thu, 15 Oct 2020 20:37:20 -0400 Subject: [PATCH 316/589] pg_upgrade: remove C99 compiler req. from commit 3c0471b5fd This commit required support for inline variable definition, which is not a requirement. RELEASE NOTE AUTHOR: the author of commit 3c0471b5fd (pg_upgrade/tablespaces) was Justin Pryzby, not me. Reported-by: Andres Freund Discussion: https://postgr.es/m/20201016001959.h24fkywfubkv2pc5@alap3.anarazel.de Backpatch-through: 9.5 --- src/bin/pg_upgrade/check.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index c24ca9b141d0..05e6bf7f2c34 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -547,11 +547,12 @@ create_script_for_cluster_analyze(char **analyze_script_file_name) static void check_for_new_tablespace_dir(ClusterInfo *new_cluster) { + int tblnum; char new_tablespace_dir[MAXPGPATH]; prep_status("Checking for new cluster tablespace directories"); - for (int tblnum = 0; tblnum < os_info.num_old_tablespaces; tblnum++) + for (tblnum = 0; tblnum < os_info.num_old_tablespaces; tblnum++) { struct stat statbuf; From fe2a16d8b3e6fde07a8a990057e983854ad50616 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Thu, 15 Oct 2020 17:38:00 -0700 Subject: [PATCH 317/589] llvmjit: Work around bug in LLVM 3.9 causing crashes after 72559438f92. Unfortunately in LLVM 3.9 LLVMGetAttributeCountAtIndex(func, index) crashes when called with an index that has 0 attributes. Since there's no way to work around this in the C API, add a small C++ wrapper doing so. The only reason this didn't fail before 72559438f92 is that there always are function attributes... Author: Andres Freund Discussion: https://postgr.es/m/20201016001254.w2nfj7gd74jmb5in@alap3.anarazel.de Backpatch: 11-, like 72559438f92 --- src/backend/jit/llvm/llvmjit.c | 9 +++++++- src/backend/jit/llvm/llvmjit_wrap.cpp | 32 +++++++++++++++++++++++++++ src/include/jit/llvmjit.h | 2 ++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c index 7965b61165e3..4c026d37d0dd 100644 --- a/src/backend/jit/llvm/llvmjit.c +++ b/src/backend/jit/llvm/llvmjit.c @@ -333,7 +333,14 @@ llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 ind int num_attributes; LLVMAttributeRef *attrs; - num_attributes = LLVMGetAttributeCountAtIndex(v_from, index); + num_attributes = LLVMGetAttributeCountAtIndexPG(v_from, index); + + /* + * Not just for efficiency: LLVM <= 3.9 crashes when + * LLVMGetAttributesAtIndex() is called for an index with 0 attributes. + */ + if (num_attributes == 0) + return; attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes); LLVMGetAttributesAtIndex(v_from, index, attrs); diff --git a/src/backend/jit/llvm/llvmjit_wrap.cpp b/src/backend/jit/llvm/llvmjit_wrap.cpp index e8a7380325a8..37c006a1ff50 100644 --- a/src/backend/jit/llvm/llvmjit_wrap.cpp +++ b/src/backend/jit/llvm/llvmjit_wrap.cpp @@ -16,6 +16,13 @@ extern "C" #include "postgres.h" } +#include + +/* Avoid macro clash with LLVM's C++ headers */ +#undef Min + +#include +#include #include #include @@ -44,3 +51,28 @@ char *LLVMGetHostCPUFeatures(void) { return strdup(Features.getString().c_str()); } #endif + +/* + * Like LLVM's LLVMGetAttributeCountAtIndex(), works around a bug in LLVM 3.9. + * + * In LLVM <= 3.9, LLVMGetAttributeCountAtIndex() segfaults if there are no + * attributes at an index (fixed in LLVM commit ce9bb1097dc2). + */ +unsigned +LLVMGetAttributeCountAtIndexPG(LLVMValueRef F, uint32 Idx) +{ + /* + * This is more expensive, so only do when using a problematic LLVM + * version. + */ +#if LLVM_VERSION_MAJOR < 4 + if (!llvm::unwrap(F)->getAttributes().hasAttributes(Idx)) + return 0; +#endif + + /* + * There is no nice public API to determine the count nicely, so just + * always fall back to LLVM's C API. + */ + return LLVMGetAttributeCountAtIndex(F, Idx); +} diff --git a/src/include/jit/llvmjit.h b/src/include/jit/llvmjit.h index 706906c1cc83..325409acd5c5 100644 --- a/src/include/jit/llvmjit.h +++ b/src/include/jit/llvmjit.h @@ -129,6 +129,8 @@ extern char *LLVMGetHostCPUName(void); extern char *LLVMGetHostCPUFeatures(void); #endif +extern unsigned LLVMGetAttributeCountAtIndexPG(LLVMValueRef F, uint32 Idx); + #ifdef __cplusplus } /* extern "C" */ #endif From 7fc1a81e4982b6722405b032e21be38dbbef5b80 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Fri, 16 Oct 2020 13:58:45 +0900 Subject: [PATCH 318/589] postgres_fdw: Restructure connection retry logic. Commit 32a9c0bdf introduced connection retry logic into postgres_fdw. Previously it used goto statement for retry. This commit gets rid of that goto from the logic to make the code simpler and easier-to-read. When getting out of PG_CATCH() for the retry, the error state should be cleaned up and the memory context should be reset. But commit 32a9c0bdf forgot to do that. This commit also fixes this bug. Previously only PQstatus()==CONNECTION_BAD was verified to detect connection failure. But this could cause false detection in the case where any error other than connection failure (e.g., out-of-memory) was thrown after a broken connection was detected in libpq and CONNECTION_BAD is set. To fix this issue, this commit changes the logic so that it also checks the error's sqlstate is ERRCODE_CONNECTION_FAILURE. Author: Fujii Masao Reviewed-by: Tom Lane Discussion: https://postgr.es/m/2943611.1602375376@sss.pgh.pa.us --- contrib/postgres_fdw/connection.c | 136 ++++++++++++++++++++---------- 1 file changed, 90 insertions(+), 46 deletions(-) diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c index 76994f3820fc..2f411cf2f7b1 100644 --- a/contrib/postgres_fdw/connection.c +++ b/contrib/postgres_fdw/connection.c @@ -74,6 +74,7 @@ static unsigned int prep_stmt_number = 0; static bool xact_got_connection = false; /* prototypes of private functions */ +static void make_new_connection(ConnCacheEntry *entry, UserMapping *user); static PGconn *connect_pg_server(ForeignServer *server, UserMapping *user); static void disconnect_pg_server(ConnCacheEntry *entry); static void check_conn_params(const char **keywords, const char **values, UserMapping *user); @@ -108,9 +109,10 @@ PGconn * GetConnection(UserMapping *user, bool will_prep_stmt) { bool found; - volatile bool retry_conn = false; + bool retry = false; ConnCacheEntry *entry; ConnCacheKey key; + MemoryContext ccxt = CurrentMemoryContext; /* First time through, initialize connection cache hashtable */ if (ConnectionHash == NULL) @@ -160,23 +162,14 @@ GetConnection(UserMapping *user, bool will_prep_stmt) /* Reject further use of connections which failed abort cleanup. */ pgfdw_reject_incomplete_xact_state_change(entry); -retry: - /* * If the connection needs to be remade due to invalidation, disconnect as - * soon as we're out of all transactions. Also, if previous attempt to - * start new remote transaction failed on the cached connection, - * disconnect it to retry a new connection. + * soon as we're out of all transactions. */ - if ((entry->conn != NULL && entry->invalidated && - entry->xact_depth == 0) || retry_conn) + if (entry->conn != NULL && entry->invalidated && entry->xact_depth == 0) { - if (retry_conn) - elog(DEBUG3, "closing connection %p to reestablish a new one", - entry->conn); - else - elog(DEBUG3, "closing connection %p for option changes to take effect", - entry->conn); + elog(DEBUG3, "closing connection %p for option changes to take effect", + entry->conn); disconnect_pg_server(entry); } @@ -186,58 +179,78 @@ GetConnection(UserMapping *user, bool will_prep_stmt) * will remain in a valid empty state, ie conn == NULL.) */ if (entry->conn == NULL) - { - ForeignServer *server = GetForeignServer(user->serverid); - - /* Reset all transient state fields, to be sure all are clean */ - entry->xact_depth = 0; - entry->have_prep_stmt = false; - entry->have_error = false; - entry->changing_xact_state = false; - entry->invalidated = false; - entry->server_hashvalue = - GetSysCacheHashValue1(FOREIGNSERVEROID, - ObjectIdGetDatum(server->serverid)); - entry->mapping_hashvalue = - GetSysCacheHashValue1(USERMAPPINGOID, - ObjectIdGetDatum(user->umid)); - - /* Now try to make the connection */ - entry->conn = connect_pg_server(server, user); - - elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)", - entry->conn, server->servername, user->umid, user->userid); - } + make_new_connection(entry, user); /* * We check the health of the cached connection here when starting a new - * remote transaction. If a broken connection is detected in the first - * attempt, we try to reestablish a new connection. If broken connection - * is detected again here, we give up getting a connection. + * remote transaction. If a broken connection is detected, we try to + * reestablish a new connection later. */ PG_TRY(); { /* Start a new transaction or subtransaction if needed. */ begin_remote_xact(entry); - retry_conn = false; } PG_CATCH(); { - if (PQstatus(entry->conn) != CONNECTION_BAD || - entry->xact_depth > 0 || - retry_conn) + MemoryContext ecxt = MemoryContextSwitchTo(ccxt); + ErrorData *errdata = CopyErrorData(); + + /* + * If connection failure is reported when starting a new remote + * transaction (not subtransaction), new connection will be + * reestablished later. + * + * After a broken connection is detected in libpq, any error other + * than connection failure (e.g., out-of-memory) can be thrown + * somewhere between return from libpq and the expected ereport() call + * in pgfdw_report_error(). In this case, since PQstatus() indicates + * CONNECTION_BAD, checking only PQstatus() causes the false detection + * of connection failure. To avoid this, we also verify that the + * error's sqlstate is ERRCODE_CONNECTION_FAILURE. Note that also + * checking only the sqlstate can cause another false detection + * because pgfdw_report_error() may report ERRCODE_CONNECTION_FAILURE + * for any libpq-originated error condition. + */ + if (errdata->sqlerrcode != ERRCODE_CONNECTION_FAILURE || + PQstatus(entry->conn) != CONNECTION_BAD || + entry->xact_depth > 0) + { + MemoryContextSwitchTo(ecxt); PG_RE_THROW(); - retry_conn = true; + } + + /* Clean up the error state */ + FlushErrorState(); + FreeErrorData(errdata); + errdata = NULL; + + retry = true; } PG_END_TRY(); - if (retry_conn) + /* + * If a broken connection is detected, disconnect it, reestablish a new + * connection and retry a new remote transaction. If connection failure is + * reported again, we give up getting a connection. + */ + if (retry) { + Assert(entry->xact_depth == 0); + ereport(DEBUG3, (errmsg_internal("could not start remote transaction on connection %p", entry->conn)), errdetail_internal("%s", pchomp(PQerrorMessage(entry->conn)))); - goto retry; + + elog(DEBUG3, "closing connection %p to reestablish a new one", + entry->conn); + disconnect_pg_server(entry); + + if (entry->conn == NULL) + make_new_connection(entry, user); + + begin_remote_xact(entry); } /* Remember if caller will prepare statements */ @@ -246,6 +259,37 @@ GetConnection(UserMapping *user, bool will_prep_stmt) return entry->conn; } +/* + * Reset all transient state fields in the cached connection entry and + * establish new connection to the remote server. + */ +static void +make_new_connection(ConnCacheEntry *entry, UserMapping *user) +{ + ForeignServer *server = GetForeignServer(user->serverid); + + Assert(entry->conn == NULL); + + /* Reset all transient state fields, to be sure all are clean */ + entry->xact_depth = 0; + entry->have_prep_stmt = false; + entry->have_error = false; + entry->changing_xact_state = false; + entry->invalidated = false; + entry->server_hashvalue = + GetSysCacheHashValue1(FOREIGNSERVEROID, + ObjectIdGetDatum(server->serverid)); + entry->mapping_hashvalue = + GetSysCacheHashValue1(USERMAPPINGOID, + ObjectIdGetDatum(user->umid)); + + /* Now try to make the connection */ + entry->conn = connect_pg_server(server, user); + + elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)", + entry->conn, server->servername, user->umid, user->userid); +} + /* * Connect to remote server using specified server and user mapping properties. */ From bc49f8780b361591141e35c03ffb17d4035847cb Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 16 Oct 2020 11:36:34 -0400 Subject: [PATCH 319/589] Doc: tweak column widths in synchronous-commit-matrix table. Commit a97e85f2b caused "exceed the available area" warnings in PDF builds. Fine-tune colwidth values to avoid that. Back-patch to 9.6, like the prior patch. (This is of dubious value before v13, since we were far from free of such warnings in older branches. But we might as well keep the SGML looking the same in all branches.) Per buildfarm. --- doc/src/sgml/config.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 2768c85a96fb..1ef880cda59c 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -2786,7 +2786,7 @@ include_dir 'conf.d' synchronous_commit Modes - + From 02a75f8369b1b14b709349ee60b29f79b70320f6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 16 Oct 2020 11:59:13 -0400 Subject: [PATCH 320/589] Add missing error check in pgcrypto/crypt-md5.c. In theory, the second px_find_digest call in px_crypt_md5 could fail even though the first one succeeded, since resource allocation is required. Don't skip testing for a failure. (If one did happen, the likely result would be a crash rather than clean recovery from an OOM failure.) The code's been like this all along, so back-patch to all supported branches. Daniel Gustafsson Discussion: https://postgr.es/m/AA8D6FE9-4AB2-41B4-98CB-AE64BA668C03@yesql.se --- contrib/pgcrypto/crypt-md5.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/contrib/pgcrypto/crypt-md5.c b/contrib/pgcrypto/crypt-md5.c index b6466d3e3178..d38721a1010a 100644 --- a/contrib/pgcrypto/crypt-md5.c +++ b/contrib/pgcrypto/crypt-md5.c @@ -65,11 +65,17 @@ px_crypt_md5(const char *pw, const char *salt, char *passwd, unsigned dstlen) /* get the length of the true salt */ sl = ep - sp; - /* */ + /* we need two PX_MD objects */ err = px_find_digest("md5", &ctx); if (err) return NULL; err = px_find_digest("md5", &ctx1); + if (err) + { + /* this path is possible under low-memory circumstances */ + px_md_free(ctx); + return NULL; + } /* The password first, since that is what is most unknown */ px_md_update(ctx, (const uint8 *) pw, strlen(pw)); From ce0e97f808683dc1732880ec3196443875efa262 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 16 Oct 2020 21:40:16 -0400 Subject: [PATCH 321/589] Sync our copy of the timezone library with IANA release tzcode2020c. This changes zic's default output format from "-b fat" to "-b slim". We were already using "slim" in v13/HEAD, so those branches drop the explicit -b switch in the Makefiles. Instead, add an explicit "-b fat" in v12 and before, so that we don't change the output file format in those branches. (This is perhaps excessively conservative, but we decided not to do so in a12079109, and I'll stick with that.) Other non-cosmetic changes are to drop support for zic's long-obsolete "-y" switch, and to ensure that strftime() does not change errno unless it fails. As usual with tzcode changes, back-patch to all supported branches. --- src/timezone/Makefile | 2 +- src/timezone/README | 2 +- src/timezone/strftime.c | 10 ++ src/timezone/zic.c | 205 +++++++++++++------------------------- src/tools/msvc/Install.pm | 2 +- 5 files changed, 80 insertions(+), 141 deletions(-) diff --git a/src/timezone/Makefile b/src/timezone/Makefile index 715b63cee0cd..2b5d8ecbef81 100644 --- a/src/timezone/Makefile +++ b/src/timezone/Makefile @@ -56,7 +56,7 @@ zic: $(ZICOBJS) | submake-libpgport install: all installdirs ifeq (,$(with_system_tzdata)) - $(ZIC) -d '$(DESTDIR)$(datadir)/timezone' -b slim $(ZIC_OPTIONS) $(TZDATAFILES) + $(ZIC) -d '$(DESTDIR)$(datadir)/timezone' $(ZIC_OPTIONS) $(TZDATAFILES) endif $(MAKE) -C tznames $@ diff --git a/src/timezone/README b/src/timezone/README index 8af44449329a..10aeedce92a0 100644 --- a/src/timezone/README +++ b/src/timezone/README @@ -55,7 +55,7 @@ match properly on the old version. Time Zone code ============== -The code in this directory is currently synced with tzcode release 2020a. +The code in this directory is currently synced with tzcode release 2020c. There are many cosmetic (and not so cosmetic) differences from the original tzcode library, but diffs in the upstream version should usually be propagated to our version. Here are some notes about that. diff --git a/src/timezone/strftime.c b/src/timezone/strftime.c index 4b942c393a34..dd6c7db86958 100644 --- a/src/timezone/strftime.c +++ b/src/timezone/strftime.c @@ -128,12 +128,22 @@ size_t pg_strftime(char *s, size_t maxsize, const char *format, const struct pg_tm *t) { char *p; + int saved_errno = errno; enum warn warn = IN_NONE; p = _fmt(format, t, s, s + maxsize, &warn); + if (!p) + { + errno = EOVERFLOW; + return 0; + } if (p == s + maxsize) + { + errno = ERANGE; return 0; + } *p = '\0'; + errno = saved_errno; return p - s; } diff --git a/src/timezone/zic.c b/src/timezone/zic.c index 10c5b4bfb5b5..a1c64051f788 100644 --- a/src/timezone/zic.c +++ b/src/timezone/zic.c @@ -37,10 +37,6 @@ typedef int64 zic_t; #define MKDIR_UMASK 0755 #endif #endif -#ifndef AT_SYMLINK_FOLLOW -#define linkat(fromdir, from, todir, to, flag) \ - (itssymlink(from) ? (errno = ENOTSUP, -1) : link(from, to)) -#endif /* Port to native MS-Windows and to ancient UNIX. */ #if !defined S_ISDIR && defined S_IFDIR && defined S_IFMT #define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) @@ -66,7 +62,6 @@ struct rule zic_t r_loyear; /* for example, 1986 */ zic_t r_hiyear; /* for example, 1986 */ - const char *r_yrtype; bool r_lowasnum; bool r_hiwasnum; @@ -116,7 +111,11 @@ struct zone zic_t z_untiltime; }; -extern int link(const char *fromname, const char *toname); +extern int link(const char *target, const char *linkname); +#ifndef AT_SYMLINK_FOLLOW +#define linkat(targetdir, target, linknamedir, linkname, flag) \ + (itssymlink(target) ? (errno = ENOTSUP, -1) : link(target, linkname)) +#endif static void memory_exhausted(const char *msg) pg_attribute_noreturn(); static void verror(const char *string, va_list args) pg_attribute_printf(1, 0); @@ -154,7 +153,6 @@ static void rulesub(struct rule *rp, const char *typep, const char *monthp, const char *dayp, const char *timep); static zic_t tadd(zic_t t1, zic_t t2); -static bool yearistype(zic_t year, const char *type); /* Bound on length of what %z can expand to. */ enum @@ -253,8 +251,8 @@ static int typecnt; * Which fields are which on a Link line. */ -#define LF_FROM 1 -#define LF_TO 2 +#define LF_TARGET 1 +#define LF_LINKNAME 2 #define LINK_FIELDS 3 /* @@ -292,8 +290,8 @@ struct link { const char *l_filename; lineno_t l_linenum; - const char *l_from; - const char *l_to; + const char *l_target; + const char *l_linkname; }; static struct link *links; @@ -634,11 +632,9 @@ static const char *lcltime; static const char *directory; static const char *leapsec; static const char *tzdefault; -static const char *yitcommand; /* -1 if the TZif output file should be slim, 0 if default, 1 if the - output should be fat for backward compatibility. Currently the - default is fat, although this may change. */ + output should be fat for backward compatibility. The default is slim. */ static int bloat; static bool @@ -648,7 +644,7 @@ want_bloat(void) } #ifndef ZIC_BLOAT_DEFAULT -#define ZIC_BLOAT_DEFAULT "fat" +#define ZIC_BLOAT_DEFAULT "slim" #endif int @@ -747,18 +743,7 @@ main(int argc, char **argv) tzdefault = optarg; break; case 'y': - if (yitcommand == NULL) - { - warning(_("-y is obsolescent")); - yitcommand = strdup(optarg); - } - else - { - fprintf(stderr, - _("%s: More than one -y option specified\n"), - progname); - return EXIT_FAILURE; - } + warning(_("-y ignored")); break; case 'L': if (leapsec == NULL) @@ -807,8 +792,6 @@ main(int argc, char **argv) directory = "data"; if (tzdefault == NULL) tzdefault = TZDEFAULT; - if (yitcommand == NULL) - yitcommand = "yearistype"; if (optind < argc && leapsec != NULL) { @@ -838,11 +821,11 @@ main(int argc, char **argv) for (i = 0; i < nlinks; ++i) { eat(links[i].l_filename, links[i].l_linenum); - dolink(links[i].l_from, links[i].l_to, false); + dolink(links[i].l_target, links[i].l_linkname, false); if (noise) for (j = 0; j < nlinks; ++j) - if (strcmp(links[i].l_to, - links[j].l_from) == 0) + if (strcmp(links[i].l_linkname, + links[j].l_target) == 0) warning(_("link to link")); } if (lcltime != NULL) @@ -953,7 +936,7 @@ namecheck(const char *name) */ #ifdef HAVE_SYMLINK static char * -relname(char const *from, char const *to) +relname(char const *target, char const *linkname) { size_t i, taillen, @@ -961,26 +944,26 @@ relname(char const *from, char const *to) size_t dir_len = 0, dotdots = 0, linksize = SIZE_MAX; - char const *f = from; + char const *f = target; char *result = NULL; - if (*to == '/') + if (*linkname == '/') { /* Make F absolute too. */ size_t len = strlen(directory); bool needslash = len && directory[len - 1] != '/'; - linksize = len + needslash + strlen(from) + 1; + linksize = len + needslash + strlen(target) + 1; f = result = emalloc(linksize); strcpy(result, directory); result[len] = '/'; - strcpy(result + len + needslash, from); + strcpy(result + len + needslash, target); } - for (i = 0; f[i] && f[i] == to[i]; i++) + for (i = 0; f[i] && f[i] == linkname[i]; i++) if (f[i] == '/') dir_len = i + 1; - for (; to[i]; i++) - dotdots += to[i] == '/' && to[i - 1] != '/'; + for (; linkname[i]; i++) + dotdots += linkname[i] == '/' && linkname[i - 1] != '/'; taillen = strlen(f + dir_len); dotdotetcsize = 3 * dotdots + taillen + 1; if (dotdotetcsize <= linksize) @@ -998,62 +981,65 @@ relname(char const *from, char const *to) /* Hard link FROM to TO, following any symbolic links. Return 0 if successful, an error number otherwise. */ static int -hardlinkerr(char const *from, char const *to) +hardlinkerr(char const *target, char const *linkname) { - int r = linkat(AT_FDCWD, from, AT_FDCWD, to, AT_SYMLINK_FOLLOW); + int r = linkat(AT_FDCWD, target, AT_FDCWD, linkname, AT_SYMLINK_FOLLOW); return r == 0 ? 0 : errno; } static void -dolink(char const *fromfield, char const *tofield, bool staysymlink) +dolink(char const *target, char const *linkname, bool staysymlink) { - bool todirs_made = false; + bool remove_only = strcmp(target, "-") == 0; + bool linkdirs_made = false; int link_errno; /* * We get to be careful here since there's a fair chance of root running * us. */ - if (itsdir(fromfield)) + if (!remove_only && itsdir(target)) { - fprintf(stderr, _("%s: link from %s/%s failed: %s\n"), - progname, directory, fromfield, strerror(EPERM)); + fprintf(stderr, _("%s: linking target %s/%s failed: %s\n"), + progname, directory, target, strerror(EPERM)); exit(EXIT_FAILURE); } if (staysymlink) - staysymlink = itssymlink(tofield); - if (remove(tofield) == 0) - todirs_made = true; + staysymlink = itssymlink(linkname); + if (remove(linkname) == 0) + linkdirs_made = true; else if (errno != ENOENT) { char const *e = strerror(errno); fprintf(stderr, _("%s: Can't remove %s/%s: %s\n"), - progname, directory, tofield, e); + progname, directory, linkname, e); exit(EXIT_FAILURE); } - link_errno = staysymlink ? ENOTSUP : hardlinkerr(fromfield, tofield); - if (link_errno == ENOENT && !todirs_made) + if (remove_only) + return; + link_errno = staysymlink ? ENOTSUP : hardlinkerr(target, linkname); + if (link_errno == ENOENT && !linkdirs_made) { - mkdirs(tofield, true); - todirs_made = true; - link_errno = hardlinkerr(fromfield, tofield); + mkdirs(linkname, true); + linkdirs_made = true; + link_errno = hardlinkerr(target, linkname); } if (link_errno != 0) { #ifdef HAVE_SYMLINK - bool absolute = *fromfield == '/'; - char *linkalloc = absolute ? NULL : relname(fromfield, tofield); - char const *contents = absolute ? fromfield : linkalloc; - int symlink_errno = symlink(contents, tofield) == 0 ? 0 : errno; + bool absolute = *target == '/'; + char *linkalloc = absolute ? NULL : relname(target, linkname); + char const *contents = absolute ? target : linkalloc; + int symlink_errno = symlink(contents, linkname) == 0 ? 0 : errno; - if (!todirs_made + if (!linkdirs_made && (symlink_errno == ENOENT || symlink_errno == ENOTSUP)) { - mkdirs(tofield, true); + mkdirs(linkname, true); if (symlink_errno == ENOENT) - symlink_errno = symlink(contents, tofield) == 0 ? 0 : errno; + symlink_errno = symlink(contents, linkname) == 0 ? 0 : errno; } free(linkalloc); if (symlink_errno == 0) @@ -1069,28 +1055,28 @@ dolink(char const *fromfield, char const *tofield, bool staysymlink) *tp; int c; - fp = fopen(fromfield, "rb"); + fp = fopen(target, "rb"); if (!fp) { char const *e = strerror(errno); fprintf(stderr, _("%s: Can't read %s/%s: %s\n"), - progname, directory, fromfield, e); + progname, directory, target, e); exit(EXIT_FAILURE); } - tp = fopen(tofield, "wb"); + tp = fopen(linkname, "wb"); if (!tp) { char const *e = strerror(errno); fprintf(stderr, _("%s: Can't create %s/%s: %s\n"), - progname, directory, tofield, e); + progname, directory, linkname, e); exit(EXIT_FAILURE); } while ((c = getc(fp)) != EOF) putc(c, tp); - close_file(fp, directory, fromfield); - close_file(tp, directory, tofield); + close_file(fp, directory, target); + close_file(tp, directory, linkname); if (link_errno != ENOTSUP) warning(_("copy used because hard link failed: %s"), strerror(link_errno)); @@ -1806,17 +1792,17 @@ inlink(char **fields, int nfields) error(_("wrong number of fields on Link line")); return; } - if (*fields[LF_FROM] == '\0') + if (*fields[LF_TARGET] == '\0') { - error(_("blank FROM field on Link line")); + error(_("blank TARGET field on Link line")); return; } - if (!namecheck(fields[LF_TO])) + if (!namecheck(fields[LF_LINKNAME])) return; l.l_filename = filename; l.l_linenum = linenum; - l.l_from = ecpyalloc(fields[LF_FROM]); - l.l_to = ecpyalloc(fields[LF_TO]); + l.l_target = ecpyalloc(fields[LF_TARGET]); + l.l_linkname = ecpyalloc(fields[LF_LINKNAME]); links = growalloc(links, sizeof *links, nlinks, &nlinks_alloc); links[nlinks++] = l; } @@ -1932,18 +1918,11 @@ rulesub(struct rule *rp, const char *loyearp, const char *hiyearp, error(_("starting year greater than ending year")); return; } - if (*typep == '\0') - rp->r_yrtype = NULL; - else + if (*typep != '\0') { - if (rp->r_loyear == rp->r_hiyear) - { - error(_("typed single year")); - return; - } - warning(_("year type \"%s\" is obsolete; use \"-\" instead"), - typep); - rp->r_yrtype = ecpyalloc(typep); + error(_("year type \"%s\" is unsupported; use \"-\" instead"), + typep); + return; } /* @@ -2848,8 +2827,6 @@ stringzone(char *result, struct zone const *zpfirst, ptrdiff_t zonecount) rp = &zp->z_rules[i]; if (rp->r_hiwasnum || rp->r_hiyear != ZIC_MAX) continue; - if (rp->r_yrtype != NULL) - continue; if (!rp->r_isdst) { if (stdrp == NULL) @@ -3145,7 +3122,8 @@ outzone(const struct zone *zpfirst, ptrdiff_t zonecount) /* * Mark which rules to do in the current year. For those to - * do, calculate rpytime(rp, year); + * do, calculate rpytime(rp, year); The former TYPE field was + * also considered here. */ for (j = 0; j < zp->z_nrules; ++j) { @@ -3153,8 +3131,7 @@ outzone(const struct zone *zpfirst, ptrdiff_t zonecount) eats(zp->z_filename, zp->z_linenum, rp->r_filename, rp->r_linenum); rp->r_todo = year >= rp->r_loyear && - year <= rp->r_hiyear && - yearistype(year, rp->r_yrtype); + year <= rp->r_hiyear; if (rp->r_todo) { rp->r_temp = rpytime(rp, year); @@ -3474,54 +3451,6 @@ adjleap(void) } } -static char * -shellquote(char *b, char const *s) -{ - *b++ = '\''; - while (*s) - { - if (*s == '\'') - *b++ = '\'', *b++ = '\\', *b++ = '\''; - *b++ = *s++; - } - *b++ = '\''; - return b; -} - -static bool -yearistype(zic_t year, const char *type) -{ - char *buf; - char *b; - int result; - - if (type == NULL || *type == '\0') - return true; - buf = emalloc(1 + 4 * strlen(yitcommand) + 2 - + INT_STRLEN_MAXIMUM(zic_t) + 2 + 4 * strlen(type) + 2); - b = shellquote(buf, yitcommand); - *b++ = ' '; - b += sprintf(b, INT64_FORMAT, year); - *b++ = ' '; - b = shellquote(b, type); - *b = '\0'; - result = system(buf); - if (WIFEXITED(result)) - { - int status = WEXITSTATUS(result); - - if (status <= 1) - { - free(buf); - return status == 0; - } - } - error(_("Wild result from command execution")); - fprintf(stderr, _("%s: command was '%s', result was %d\n"), - progname, buf, result); - exit(EXIT_FAILURE); -} - /* Is A a space character in the C locale? */ static bool is_space(char a) diff --git a/src/tools/msvc/Install.pm b/src/tools/msvc/Install.pm index b6d0cfd39b49..ea3af4877754 100644 --- a/src/tools/msvc/Install.pm +++ b/src/tools/msvc/Install.pm @@ -369,7 +369,7 @@ sub GenerateTimezoneFiles print "Generating timezone files..."; my @args = ( - "$conf/zic/zic", '-d', "$target/share/timezone", '-b', 'slim'); + "$conf/zic/zic", '-d', "$target/share/timezone"); foreach (@tzfiles) { my $tzfile = $_; From c4a803ac7e7539d5f4538e0e677bb543f2a5eaa0 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 16 Oct 2020 21:53:33 -0400 Subject: [PATCH 322/589] Update time zone data files to tzdata release 2020c. DST law changes in Morocco, Canadian Yukon, Fiji, Macquarie Island, Casey Station (Antarctica). Historical corrections for France, Hungary, Monaco. --- src/timezone/data/tzdata.zi | 89 ++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 40 deletions(-) diff --git a/src/timezone/data/tzdata.zi b/src/timezone/data/tzdata.zi index e7c31b68c867..df175e61cd8d 100644 --- a/src/timezone/data/tzdata.zi +++ b/src/timezone/data/tzdata.zi @@ -1,4 +1,4 @@ -# version 2020a +# version 2020c # This zic input file is in the public domain. R d 1916 o - Jun 14 23s 1 S R d 1916 1919 - O Su>=1 23s 0 - @@ -22,7 +22,7 @@ R d 1978 o - Mar 24 1 1 S R d 1978 o - S 22 3 0 - R d 1980 o - Ap 25 0 1 S R d 1980 o - O 31 2 0 - -Z Africa/Algiers 0:12:12 - LMT 1891 Mar 15 0:1 +Z Africa/Algiers 0:12:12 - LMT 1891 Mar 16 0:9:21 - PMT 1911 Mar 11 0 d WE%sT 1940 F 25 2 1 d CE%sT 1946 O 7 @@ -193,7 +193,7 @@ R M 2021 o - May 16 2 0 - R M 2022 o - Mar 27 3 -1 - R M 2022 o - May 8 2 0 - R M 2023 o - Mar 19 3 -1 - -R M 2023 o - Ap 23 2 0 - +R M 2023 o - Ap 30 2 0 - R M 2024 o - Mar 10 3 -1 - R M 2024 o - Ap 14 2 0 - R M 2025 o - F 23 3 -1 - @@ -209,7 +209,7 @@ R M 2029 o - F 18 2 0 - R M 2029 o - D 30 3 -1 - R M 2030 o - F 10 2 0 - R M 2030 o - D 22 3 -1 - -R M 2031 o - Ja 26 2 0 - +R M 2031 o - F 2 2 0 - R M 2031 o - D 14 3 -1 - R M 2032 o - Ja 18 2 0 - R M 2032 o - N 28 3 -1 - @@ -225,7 +225,7 @@ R M 2036 o - N 23 2 0 - R M 2037 o - O 4 3 -1 - R M 2037 o - N 15 2 0 - R M 2038 o - S 26 3 -1 - -R M 2038 o - O 31 2 0 - +R M 2038 o - N 7 2 0 - R M 2039 o - S 18 3 -1 - R M 2039 o - O 23 2 0 - R M 2040 o - S 2 3 -1 - @@ -241,7 +241,7 @@ R M 2044 o - Au 28 2 0 - R M 2045 o - Jul 9 3 -1 - R M 2045 o - Au 20 2 0 - R M 2046 o - Jul 1 3 -1 - -R M 2046 o - Au 5 2 0 - +R M 2046 o - Au 12 2 0 - R M 2047 o - Jun 23 3 -1 - R M 2047 o - Jul 28 2 0 - R M 2048 o - Jun 7 3 -1 - @@ -257,7 +257,7 @@ R M 2052 o - Jun 2 2 0 - R M 2053 o - Ap 13 3 -1 - R M 2053 o - May 25 2 0 - R M 2054 o - Ap 5 3 -1 - -R M 2054 o - May 10 2 0 - +R M 2054 o - May 17 2 0 - R M 2055 o - Mar 28 3 -1 - R M 2055 o - May 2 2 0 - R M 2056 o - Mar 12 3 -1 - @@ -273,7 +273,7 @@ R M 2060 o - Mar 7 2 0 - R M 2061 o - Ja 16 3 -1 - R M 2061 o - F 27 2 0 - R M 2062 o - Ja 8 3 -1 - -R M 2062 o - F 12 2 0 - +R M 2062 o - F 19 2 0 - R M 2062 o - D 31 3 -1 - R M 2063 o - F 4 2 0 - R M 2063 o - D 16 3 -1 - @@ -289,7 +289,7 @@ R M 2067 o - D 11 2 0 - R M 2068 o - O 21 3 -1 - R M 2068 o - D 2 2 0 - R M 2069 o - O 13 3 -1 - -R M 2069 o - N 17 2 0 - +R M 2069 o - N 24 2 0 - R M 2070 o - O 5 3 -1 - R M 2070 o - N 9 2 0 - R M 2071 o - S 20 3 -1 - @@ -305,7 +305,7 @@ R M 2075 o - S 15 2 0 - R M 2076 o - Jul 26 3 -1 - R M 2076 o - S 6 2 0 - R M 2077 o - Jul 18 3 -1 - -R M 2077 o - Au 22 2 0 - +R M 2077 o - Au 29 2 0 - R M 2078 o - Jul 10 3 -1 - R M 2078 o - Au 14 2 0 - R M 2079 o - Jun 25 3 -1 - @@ -315,13 +315,13 @@ R M 2080 o - Jul 21 2 0 - R M 2081 o - Jun 1 3 -1 - R M 2081 o - Jul 13 2 0 - R M 2082 o - May 24 3 -1 - -R M 2082 o - Jun 28 2 0 - +R M 2082 o - Jul 5 2 0 - R M 2083 o - May 16 3 -1 - R M 2083 o - Jun 20 2 0 - R M 2084 o - Ap 30 3 -1 - R M 2084 o - Jun 11 2 0 - R M 2085 o - Ap 22 3 -1 - -R M 2085 o - May 27 2 0 - +R M 2085 o - Jun 3 2 0 - R M 2086 o - Ap 14 3 -1 - R M 2086 o - May 19 2 0 - R M 2087 o - Mar 30 3 -1 - @@ -426,7 +426,12 @@ Z Antarctica/Casey 0 - -00 1969 11 - +11 2012 F 21 17u 8 - +08 2016 O 22 11 - +11 2018 Mar 11 4 -8 - +08 +8 - +08 2018 O 7 4 +11 - +11 2019 Mar 17 3 +8 - +08 2019 O 4 3 +11 - +11 2020 Mar 8 3 +8 - +08 2020 O 4 0:1 +11 - +11 Z Antarctica/Davis 0 - -00 1957 Ja 13 7 - +07 1964 N 0 - -00 1969 F @@ -1399,8 +1404,9 @@ Z Antarctica/Macquarie 0 - -00 1899 N 10 AU AE%sT 1919 Ap 1 0s 0 - -00 1948 Mar 25 10 AU AE%sT 1967 -10 AT AE%sT 2010 Ap 4 3 -11 - +11 +10 AT AE%sT 2010 +10 1 AEDT 2011 +10 AT AE%sT Z Indian/Christmas 7:2:52 - LMT 1895 F 7 - +07 Z Indian/Cocos 6:27:40 - LMT 1900 @@ -1415,7 +1421,9 @@ R FJ 2012 2013 - Ja Su>=18 3 0 - R FJ 2014 o - Ja Su>=18 2 0 - R FJ 2014 2018 - N Su>=1 2 1 - R FJ 2015 ma - Ja Su>=12 3 0 - -R FJ 2019 ma - N Su>=8 2 1 - +R FJ 2019 o - N Su>=8 2 1 - +R FJ 2020 o - D 20 2 1 - +R FJ 2021 ma - N Su>=8 2 1 - Z Pacific/Fiji 11:55:44 - LMT 1915 O 26 12 FJ +12/+13 Z Pacific/Gambier -8:59:48 - LMT 1912 O @@ -1992,8 +2000,8 @@ R F 1945 o - Ap 2 2 2 M R F 1945 o - S 16 3 0 - R F 1976 o - Mar 28 1 1 S R F 1976 o - S 26 1 0 - -Z Europe/Paris 0:9:21 - LMT 1891 Mar 15 0:1 -0:9:21 - PMT 1911 Mar 11 0:1 +Z Europe/Paris 0:9:21 - LMT 1891 Mar 16 +0:9:21 - PMT 1911 Mar 11 0 F WE%sT 1940 Jun 14 23 1 c CE%sT 1944 Au 25 0 F WE%sT 1945 S 16 3 @@ -2045,29 +2053,30 @@ Z Europe/Athens 1:34:52 - LMT 1895 S 14 1 g CE%sT 1944 Ap 4 2 g EE%sT 1981 2 E EE%sT -R h 1918 o - Ap 1 3 1 S -R h 1918 o - S 16 3 0 - -R h 1919 o - Ap 15 3 1 S -R h 1919 o - N 24 3 0 - +R h 1918 1919 - Ap 15 2 1 S +R h 1918 1920 - S M>=15 3 0 - +R h 1920 o - Ap 5 2 1 S R h 1945 o - May 1 23 1 S -R h 1945 o - N 1 0 0 - +R h 1945 o - N 1 1 0 - R h 1946 o - Mar 31 2s 1 S -R h 1946 1949 - O Su>=1 2s 0 - +R h 1946 o - O 7 2 0 - R h 1947 1949 - Ap Su>=4 2s 1 S -R h 1950 o - Ap 17 2s 1 S -R h 1950 o - O 23 2s 0 - -R h 1954 1955 - May 23 0 1 S -R h 1954 1955 - O 3 0 0 - -R h 1956 o - Jun Su>=1 0 1 S -R h 1956 o - S lastSu 0 0 - -R h 1957 o - Jun Su>=1 1 1 S -R h 1957 o - S lastSu 3 0 - -R h 1980 o - Ap 6 1 1 S -Z Europe/Budapest 1:16:20 - LMT 1890 O +R h 1947 1949 - O Su>=1 2s 0 - +R h 1954 o - May 23 0 1 S +R h 1954 o - O 3 0 0 - +R h 1955 o - May 22 2 1 S +R h 1955 o - O 2 3 0 - +R h 1956 1957 - Jun Su>=1 2 1 S +R h 1956 1957 - S lastSu 3 0 - +R h 1980 o - Ap 6 0 1 S +R h 1980 o - S 28 1 0 - +R h 1981 1983 - Mar lastSu 0 1 S +R h 1981 1983 - S lastSu 1 0 - +Z Europe/Budapest 1:16:20 - LMT 1890 N 1 c CE%sT 1918 -1 h CE%sT 1941 Ap 8 +1 h CE%sT 1941 Ap 7 23 1 c CE%sT 1945 -1 h CE%sT 1980 S 28 2s +1 h CE%sT 1984 1 E CE%sT R w 1917 1919 - F 19 23 1 - R w 1917 o - O 21 1 0 - @@ -2223,8 +2232,8 @@ Z Europe/Chisinau 1:55:20 - LMT 1880 2 R EE%sT 1992 2 e EE%sT 1997 2 MD EE%sT -Z Europe/Monaco 0:29:32 - LMT 1891 Mar 15 -0:9:21 - PMT 1911 Mar 11 +Z Europe/Monaco 0:29:32 - LMT 1892 Jun +0:9:21 - PMT 1911 Mar 29 0 F WE%sT 1945 S 16 3 1 F CE%sT 1977 1 E CE%sT @@ -3413,12 +3422,12 @@ Z America/Inuvik 0 - -00 1953 Z America/Whitehorse -9:0:12 - LMT 1900 Au 20 -9 Y Y%sT 1967 May 28 -8 Y P%sT 1980 --8 C P%sT 2020 Mar 8 2 +-8 C P%sT 2020 N -7 - MST Z America/Dawson -9:17:40 - LMT 1900 Au 20 -9 Y Y%sT 1973 O 28 -8 Y P%sT 1980 --8 C P%sT 2020 Mar 8 2 +-8 C P%sT 2020 N -7 - MST R m 1939 o - F 5 0 1 D R m 1939 o - Jun 25 0 0 S From 540849814cdc22ea025777d374ff6705b4d64a0f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 17 Oct 2020 16:02:47 -0400 Subject: [PATCH 323/589] Doc: caution against misuse of 'now' and related datetime literals. Section 8.5.1.4, which defines these literals, made only a vague reference to the fact that they might be evaluated too soon to be safe in non-interactive contexts. Provide a more explicit caution against misuse. Also, generalize the wording in the related tip in section 9.9.4: while it clearly described this problem, it implied (or really, stated outright) that the problem only applies to table DEFAULT clauses. Per gripe from Tijs van Dam. Back-patch to all supported branches. Discussion: https://postgr.es/m/c2LuRv9BiRT3bqIo5mMQiVraEXey_25B4vUn0kDqVqilwOEu_iVF1tbtvLnyQK7yDG3PFaz_GxLLPil2SDkj1MCObNRVaac-7j1dVdFERk8=@thalex.com --- doc/src/sgml/datatype.sgml | 20 +++++++++++++++++--- doc/src/sgml/func.sgml | 8 +++++--- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/doc/src/sgml/datatype.sgml b/doc/src/sgml/datatype.sgml index 49ea0003aad8..c2951854b5ce 100644 --- a/doc/src/sgml/datatype.sgml +++ b/doc/src/sgml/datatype.sgml @@ -2197,7 +2197,7 @@ TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54+02' - + Special Values @@ -2285,12 +2285,26 @@ TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54+02' type: CURRENT_DATE, CURRENT_TIME, CURRENT_TIMESTAMP, LOCALTIME, - LOCALTIMESTAMP. The latter four accept an - optional subsecond precision specification. (See LOCALTIMESTAMP. (See .) Note that these are SQL functions and are not recognized in data input strings. + + + While the input strings now, + today, tomorrow, + and yesterday are fine to use in interactive SQL + commands, they can have surprising behavior when the command is + saved to be executed later, for example in prepared statements, + views, and function definitions. The string can be converted to a + specific time value that continues to be used long after it becomes + stale. Use one of the SQL functions instead in such contexts. + For example, CURRENT_DATE + 1 is safer than + 'tomorrow'::date. + + + diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index e7cff980ddf9..2762a647edce 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -10044,20 +10044,22 @@ now() SELECT CURRENT_TIMESTAMP; SELECT now(); -SELECT TIMESTAMP 'now'; -- incorrect for use with DEFAULT +SELECT TIMESTAMP 'now'; -- but see tip below - You do not want to use the third form when specifying a DEFAULT - clause while creating a table. The system will convert now + Do not use the third form when specifying a value to be evaluated later, + for example in a DEFAULT clause for a table column. + The system will convert now to a timestamp as soon as the constant is parsed, so that when the default value is needed, the time of the table creation would be used! The first two forms will not be evaluated until the default value is used, because they are function calls. Thus they will give the desired behavior of defaulting to the time of row insertion. + (See also .) From 7d00a6b2de8ab1e95e052663064defb0bc3022f6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 17 Oct 2020 16:53:48 -0400 Subject: [PATCH 324/589] In libpq for Windows, call WSAStartup once and WSACleanup not at all. The Windows documentation insists that every WSAStartup call should have a matching WSACleanup call. However, if that ever had actual relevance, it wasn't in this century. Every remotely-modern Windows kernel is capable of cleaning up when a process exits without doing that, and must be so to avoid resource leaks in case of a process crash. Moreover, Postgres backends have done WSAStartup without WSACleanup since commit 4cdf51e64 in 2004, and we've never seen any indication of a problem with that. libpq's habit of doing WSAStartup during connection start and WSACleanup during shutdown is also rather inefficient, since a series of non-overlapping connection requests leads to repeated, quite expensive DLL unload/reload cycles. We document a workaround for that (having the application call WSAStartup for itself), but that's just a kluge. It's also worth noting that it's far from uncommon for applications to exit without doing PQfinish, and we've not heard reports of trouble from that either. However, the real reason for acting on this is that recent experiments by Alexander Lakhin suggest that calling WSACleanup during PQfinish might be triggering the symptom we occasionally see that a process using libpq fails to emit expected stdio output. Therefore, let's change libpq so that it calls WSAStartup only once per process, during the first connection attempt, and never calls WSACleanup at all. While at it, get rid of the only other WSACleanup call in our code tree, in pg_dump/parallel.c; that presumably is equally useless. If this proves to suppress the fairly-common ecpg test failures we see on Windows, I'll back-patch, but for now let's just do it in HEAD and see what happens. Discussion: https://postgr.es/m/ac976d8c-03df-d6b8-025c-15a2de8d9af1@postgrespro.ru --- doc/src/sgml/libpq.sgml | 15 --------------- src/bin/pg_dump/parallel.c | 16 +--------------- src/interfaces/libpq/fe-connect.c | 31 +++++++++++++++++-------------- 3 files changed, 18 insertions(+), 44 deletions(-) diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index 3315f1dd05b5..de60281fcb49 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -91,21 +91,6 @@ - - - On Windows, there is a way to improve performance if a single - database connection is repeatedly started and shutdown. Internally, - libpq calls WSAStartup() and WSACleanup() for connection startup - and shutdown, respectively. WSAStartup() increments an internal - Windows library reference count which is decremented by WSACleanup(). - When the reference count is just one, calling WSACleanup() frees - all resources and all DLLs are unloaded. This is an expensive - operation. To avoid this, an application can manually call - WSAStartup() so resources will not be freed when the last database - connection is closed. - - - PQconnectdbParamsPQconnectdbParams diff --git a/src/bin/pg_dump/parallel.c b/src/bin/pg_dump/parallel.c index a967e1137824..b51cc76c7dc2 100644 --- a/src/bin/pg_dump/parallel.c +++ b/src/bin/pg_dump/parallel.c @@ -229,19 +229,6 @@ static char *readMessageFromPipe(int fd); (strncmp(msg, prefix, strlen(prefix)) == 0) -/* - * Shutdown callback to clean up socket access - */ -#ifdef WIN32 -static void -shutdown_parallel_dump_utils(int code, void *unused) -{ - /* Call the cleanup function only from the main thread */ - if (mainThreadId == GetCurrentThreadId()) - WSACleanup(); -} -#endif - /* * Initialize parallel dump support --- should be called early in process * startup. (Currently, this is called whether or not we intend parallel @@ -267,8 +254,7 @@ init_parallel_dump_utils(void) pg_log_error("WSAStartup failed: %d", err); exit_nicely(1); } - /* ... and arrange to shut it down at exit */ - on_exit_nicely(shutdown_parallel_dump_utils, NULL); + parallel_init_done = true; } #endif diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index af27fee6b51e..704c9e2f79f8 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -3871,23 +3871,30 @@ makeEmptyPGconn(void) #ifdef WIN32 /* - * Make sure socket support is up and running. + * Make sure socket support is up and running in this process. + * + * Note: the Windows documentation says that we should eventually do a + * matching WSACleanup() call, but experience suggests that that is at + * least as likely to cause problems as fix them. So we don't. */ - WSADATA wsaData; + static bool wsastartup_done = false; - if (WSAStartup(MAKEWORD(1, 1), &wsaData)) - return NULL; + if (!wsastartup_done) + { + WSADATA wsaData; + + if (WSAStartup(MAKEWORD(1, 1), &wsaData) != 0) + return NULL; + wsastartup_done = true; + } + + /* Forget any earlier error */ WSASetLastError(0); -#endif +#endif /* WIN32 */ conn = (PGconn *) malloc(sizeof(PGconn)); if (conn == NULL) - { -#ifdef WIN32 - WSACleanup(); -#endif return conn; - } /* Zero all pointers and booleans */ MemSet(conn, 0, sizeof(PGconn)); @@ -4080,10 +4087,6 @@ freePGconn(PGconn *conn) termPQExpBuffer(&conn->workBuffer); free(conn); - -#ifdef WIN32 - WSACleanup(); -#endif } /* From 929c69aa1970b3ae30bbb5a159b9dc530ec34d5c Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 18 Oct 2020 12:26:02 -0400 Subject: [PATCH 325/589] In pg_restore's dump_lo_buf(), work a little harder on error handling. Failure to write data to a large object during restore led to an ugly and uninformative error message. To add insult to injury, it then fatal'd out, where other SQL-level errors usually result in pressing on. Report the underlying error condition, rather than just giving not-very- useful byte counts, and use warn_or_exit_horribly() so as to adhere to pg_restore's general policy about whether to continue or not. Also recognize that lo_write() returns int not size_t. Per report from Justin Pryzby, though I didn't use his patch. Given the lack of comparable complaints, I'm not sure this is worth back-patching. Discussion: https://postgr.es/m/20201018010232.GF9241@telsasoft.com --- src/bin/pg_dump/pg_backup_archiver.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index d61b290d2a62..b961a24b36dd 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -1637,16 +1637,17 @@ dump_lo_buf(ArchiveHandle *AH) { if (AH->connection) { - size_t res; + int res; res = lo_write(AH->connection, AH->loFd, AH->lo_buf, AH->lo_buf_used); - pg_log_debug(ngettext("wrote %lu byte of large object data (result = %lu)", - "wrote %lu bytes of large object data (result = %lu)", + pg_log_debug(ngettext("wrote %zu byte of large object data (result = %d)", + "wrote %zu bytes of large object data (result = %d)", AH->lo_buf_used), - (unsigned long) AH->lo_buf_used, (unsigned long) res); + AH->lo_buf_used, res); + /* We assume there are no short writes, only errors */ if (res != AH->lo_buf_used) - fatal("could not write to large object (result: %lu, expected: %lu)", - (unsigned long) res, (unsigned long) AH->lo_buf_used); + warn_or_exit_horribly(AH, "could not write to large object: %s", + PQerrorMessage(AH->connection)); } else { From d5a9a661fcd2f5db037274157f931863a52004fd Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 18 Oct 2020 12:56:43 -0400 Subject: [PATCH 326/589] Update the Winsock API version requested by libpq. According to Microsoft's documentation, 2.2 has been the current version since Windows 98 or so. Moreover, that's what the Postgres backend has been requesting since 2004 (cf commit 4cdf51e64). So there seems no reason for libpq to keep asking for 1.1. Bring thread_test along, too, so that we're uniformly asking for 2.2 in all our WSAStartup calls. It's not clear whether there's any point in back-patching this, so for now I didn't. Discussion: https://postgr.es/m/132799.1602960277@sss.pgh.pa.us --- src/interfaces/libpq/fe-connect.c | 2 +- src/test/thread/thread_test.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 704c9e2f79f8..b0ca37c2ed81 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -3883,7 +3883,7 @@ makeEmptyPGconn(void) { WSADATA wsaData; - if (WSAStartup(MAKEWORD(1, 1), &wsaData) != 0) + if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0) return NULL; wsastartup_done = true; } diff --git a/src/test/thread/thread_test.c b/src/test/thread/thread_test.c index e1bec01b81ad..09603c95dd68 100644 --- a/src/test/thread/thread_test.c +++ b/src/test/thread/thread_test.c @@ -126,7 +126,7 @@ main(int argc, char *argv[]) #endif #ifdef WIN32 - err = WSAStartup(MAKEWORD(1, 1), &wsaData); + err = WSAStartup(MAKEWORD(2, 2), &wsaData); if (err != 0) { fprintf(stderr, "Cannot start the network subsystem - %d**\nexiting\n", err); From a90c950fc7fd8796daa8c7948e7046bceb272894 Mon Sep 17 00:00:00 2001 From: David Rowley Date: Mon, 19 Oct 2020 10:53:52 +1300 Subject: [PATCH 327/589] Prevent overly large and NaN row estimates in relations Given a query with enough joins, it was possible that the query planner, after multiplying the row estimates with the join selectivity that the estimated number of rows would exceed the limits of the double data type and become infinite. To give an indication on how extreme a case is required to hit this, the particular example case reported required 379 joins to a table without any statistics, which resulted in the 1.0/DEFAULT_NUM_DISTINCT being used for the join selectivity. This eventually caused the row estimates to go infinite and resulted in an assert failure in initial_cost_mergejoin() where the infinite row estimated was multiplied by an outerstartsel of 0.0 resulting in NaN. The failing assert verified that NaN <= Inf, which is false. To get around this we use clamp_row_est() to cap row estimates at a maximum of 1e100. This value is thought to be low enough that costs derived from it would remain within the bounds of what the double type can represent. Aside from fixing the failing Assert, this also has the added benefit of making it so add_path() will still receive proper numerical values as costs which will allow it to make more sane choices when determining the cheaper path in extreme cases such as the one described above. Additionally, we also get rid of the isnan() checks in the join costing functions. The actual case which originally triggered those checks to be added in the first place never made it to the mailing lists. It seems likely that the new code being added to clamp_row_est() will result in those becoming checks redundant, so just remove them. The fairly harmless assert failure problem does also exist in the backbranches, however, a more minimalistic fix will be applied there. Reported-by: Onder Kalaci Reviewed-by: Tom Lane Discussion: https://postgr.es/m/DM6PR21MB1211FF360183BCA901B27F04D80B0@DM6PR21MB1211.namprd21.prod.outlook.com --- src/backend/optimizer/path/costsize.c | 35 +++++++++++++++++---------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index cd3716d494f0..733f7ea54328 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -107,6 +107,13 @@ */ #define APPEND_CPU_COST_MULTIPLIER 0.5 +/* + * Maximum value for row estimates. We cap row estimates to this to help + * ensure that costs based on these estimates remain within the range of what + * double can represent. add_path() wouldn't act sanely given infinite or NaN + * cost values. + */ +#define MAXIMUM_ROWCOUNT 1e100 double seq_page_cost = DEFAULT_SEQ_PAGE_COST; double random_page_cost = DEFAULT_RANDOM_PAGE_COST; @@ -189,11 +196,14 @@ double clamp_row_est(double nrows) { /* - * Force estimate to be at least one row, to make explain output look - * better and to avoid possible divide-by-zero when interpolating costs. - * Make it an integer, too. + * Avoid infinite and NaN row estimates. Costs derived from such values + * are going to be useless. Also force the estimate to be at least one + * row, to make explain output look better and to avoid possible + * divide-by-zero when interpolating costs. Make it an integer, too. */ - if (nrows <= 1.0) + if (nrows > MAXIMUM_ROWCOUNT || isnan(nrows)) + nrows = MAXIMUM_ROWCOUNT; + else if (nrows <= 1.0) nrows = 1.0; else nrows = rint(nrows); @@ -2737,12 +2747,11 @@ final_cost_nestloop(PlannerInfo *root, NestPath *path, QualCost restrict_qual_cost; double ntuples; - /* Protect some assumptions below that rowcounts aren't zero or NaN */ - if (outer_path_rows <= 0 || isnan(outer_path_rows)) + /* Protect some assumptions below that rowcounts aren't zero */ + if (outer_path_rows <= 0) outer_path_rows = 1; - if (inner_path_rows <= 0 || isnan(inner_path_rows)) + if (inner_path_rows <= 0) inner_path_rows = 1; - /* Mark the path with the correct row estimate */ if (path->path.param_info) path->path.rows = path->path.param_info->ppi_rows; @@ -2952,10 +2961,10 @@ initial_cost_mergejoin(PlannerInfo *root, JoinCostWorkspace *workspace, innerendsel; Path sort_path; /* dummy for result of cost_sort */ - /* Protect some assumptions below that rowcounts aren't zero or NaN */ - if (outer_path_rows <= 0 || isnan(outer_path_rows)) + /* Protect some assumptions below that rowcounts aren't zero */ + if (outer_path_rows <= 0) outer_path_rows = 1; - if (inner_path_rows <= 0 || isnan(inner_path_rows)) + if (inner_path_rows <= 0) inner_path_rows = 1; /* @@ -3185,8 +3194,8 @@ final_cost_mergejoin(PlannerInfo *root, MergePath *path, rescannedtuples; double rescanratio; - /* Protect some assumptions below that rowcounts aren't zero or NaN */ - if (inner_path_rows <= 0 || isnan(inner_path_rows)) + /* Protect some assumptions below that rowcounts aren't zero */ + if (inner_path_rows <= 0) inner_path_rows = 1; /* Mark the path with the correct row estimate */ From ca2a12c935f75fb56c3b14527d6f2ff6f549ea85 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 19 Oct 2020 09:36:56 +0900 Subject: [PATCH 328/589] Fix potential memory leak in pgcrypto When allocating a EVP context, it would have been possible to leak some memory allocated directly by OpenSSL, that PostgreSQL lost track of if the initialization of the context allocated failed. The cleanup can be done with EVP_MD_CTX_destroy(). Note that EVP APIs exist since OpenSSL 0.9.7 and we have in the tree equivalent implementations for older versions since ce9b75d (code removed with 9b7cd59a as of 10~). However, in 9.5 and 9.6, the existing code makes use of EVP_MD_CTX_destroy() and EVP_MD_CTX_create() without an equivalent implementation when building the tree with OpenSSL 0.9.6 or older, meaning that this code is in reality broken with such versions since it got introduced in e2838c5. As we have heard no complains about that, it does not seem worth bothering with in 9.5 and 9.6, so I have left that out for simplicity. Author: Michael Paquier Discussion: https://postgr.es/m/20201015072212.GC2305@paquier.xyz Backpatch-through: 9.5 --- contrib/pgcrypto/openssl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/pgcrypto/openssl.c b/contrib/pgcrypto/openssl.c index 90951a8ae7b0..ed96e4ce5359 100644 --- a/contrib/pgcrypto/openssl.c +++ b/contrib/pgcrypto/openssl.c @@ -202,6 +202,7 @@ px_find_digest(const char *name, PX_MD **res) } if (EVP_DigestInit_ex(ctx, md, NULL) == 0) { + EVP_MD_CTX_destroy(ctx); pfree(digest); return -1; } From 560d260d7852dc54a8c587c1b388843e8c433bc8 Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Mon, 19 Oct 2020 09:13:17 +0530 Subject: [PATCH 329/589] Change the docs for PARALLEL option of Vacuum. The rules to choose the number of parallel workers to perform parallel vacuum operation were not clearly specified. Reported-by: Peter Eisentraut Author: Amit Kapila Backpatch-through: 13, where it was introduced Discussion: https://postgr.es/m/36aa8aea-61b7-eb3c-263b-648e0cb117b7@2ndquadrant.com --- doc/src/sgml/ref/vacuum.sgml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index 26ede69bb31d..21ab57d88048 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -235,22 +235,22 @@ VACUUM [ FULL ] [ FREEZE ] [ VERBOSE ] [ ANALYZE ] [ integer background workers (for the details of each vacuum phase, please - refer to ). In plain VACUUM - (without FULL), if the PARALLEL option - is omitted, then the number of workers is determined based on the number of - indexes on the relation that support parallel vacuum operation and is further - limited by . An index - can participate in parallel vacuum if and only if the size of the index is - more than . Please note - that it is not guaranteed that the number of parallel workers specified in - integer will be used during - execution. It is possible for a vacuum to run with fewer workers than - specified, or even with no workers at all. Only one worker can be used per - index. So parallel workers are launched only when there are at least - 2 indexes in the table. Workers for vacuum are launched - before the start of each phase and exit at the end of the phase. These - behaviors might change in a future release. This option can't be used with - the FULL option. + refer to ). The number of workers used + to perform the operation is equal to the number of indexes on the + relation that support parallel vacuum which is limited by the number of + workers specified with PARALLEL option if any which is + further limited by . + An index can participate in parallel vacuum if and only if the size of the + index is more than . + Please note that it is not guaranteed that the number of parallel workers + specified in integer will be + used during execution. It is possible for a vacuum to run with fewer + workers than specified, or even with no workers at all. Only one worker + can be used per index. So parallel workers are launched only when there + are at least 2 indexes in the table. Workers for + vacuum are launched before the start of each phase and exit at the end of + the phase. These behaviors might change in a future release. This + option can't be used with the FULL option. From afa0d53d4de6d2f43fbc0a76c88dceb22ba65e86 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 19 Oct 2020 08:32:53 +0200 Subject: [PATCH 330/589] Improve whitespace The SQL file was using a mix of tabs and spaces inconsistently. Convert all to spaces and adjust indentation accordingly. --- src/test/regress/expected/hash_func.out | 134 ++++++++++++------------ src/test/regress/sql/hash_func.sql | 134 ++++++++++++------------ 2 files changed, 134 insertions(+), 134 deletions(-) diff --git a/src/test/regress/expected/hash_func.out b/src/test/regress/expected/hash_func.out index da0948e95a93..e6e3410aaa18 100644 --- a/src/test/regress/expected/hash_func.out +++ b/src/test/regress/expected/hash_func.out @@ -16,8 +16,8 @@ WHERE hashint2(v)::bit(32) != hashint2extended(v, 0)::bit(32) (0 rows) SELECT v as value, hashint4(v)::bit(32) as standard, - hashint4extended(v, 0)::bit(32) as extended0, - hashint4extended(v, 1)::bit(32) as extended1 + hashint4extended(v, 0)::bit(32) as extended0, + hashint4extended(v, 1)::bit(32) as extended1 FROM (VALUES (0), (1), (17), (42), (550273), (207112489)) x(v) WHERE hashint4(v)::bit(32) != hashint4extended(v, 0)::bit(32) OR hashint4(v)::bit(32) = hashint4extended(v, 1)::bit(32); @@ -26,8 +26,8 @@ WHERE hashint4(v)::bit(32) != hashint4extended(v, 0)::bit(32) (0 rows) SELECT v as value, hashint8(v)::bit(32) as standard, - hashint8extended(v, 0)::bit(32) as extended0, - hashint8extended(v, 1)::bit(32) as extended1 + hashint8extended(v, 0)::bit(32) as extended0, + hashint8extended(v, 1)::bit(32) as extended1 FROM (VALUES (0), (1), (17), (42), (550273), (207112489)) x(v) WHERE hashint8(v)::bit(32) != hashint8extended(v, 0)::bit(32) OR hashint8(v)::bit(32) = hashint8extended(v, 1)::bit(32); @@ -36,8 +36,8 @@ WHERE hashint8(v)::bit(32) != hashint8extended(v, 0)::bit(32) (0 rows) SELECT v as value, hashfloat4(v)::bit(32) as standard, - hashfloat4extended(v, 0)::bit(32) as extended0, - hashfloat4extended(v, 1)::bit(32) as extended1 + hashfloat4extended(v, 0)::bit(32) as extended0, + hashfloat4extended(v, 1)::bit(32) as extended1 FROM (VALUES (0), (1), (17), (42), (550273), (207112489)) x(v) WHERE hashfloat4(v)::bit(32) != hashfloat4extended(v, 0)::bit(32) OR hashfloat4(v)::bit(32) = hashfloat4extended(v, 1)::bit(32); @@ -46,8 +46,8 @@ WHERE hashfloat4(v)::bit(32) != hashfloat4extended(v, 0)::bit(32) (0 rows) SELECT v as value, hashfloat8(v)::bit(32) as standard, - hashfloat8extended(v, 0)::bit(32) as extended0, - hashfloat8extended(v, 1)::bit(32) as extended1 + hashfloat8extended(v, 0)::bit(32) as extended0, + hashfloat8extended(v, 1)::bit(32) as extended1 FROM (VALUES (0), (1), (17), (42), (550273), (207112489)) x(v) WHERE hashfloat8(v)::bit(32) != hashfloat8extended(v, 0)::bit(32) OR hashfloat8(v)::bit(32) = hashfloat8extended(v, 1)::bit(32); @@ -56,8 +56,8 @@ WHERE hashfloat8(v)::bit(32) != hashfloat8extended(v, 0)::bit(32) (0 rows) SELECT v as value, hashoid(v)::bit(32) as standard, - hashoidextended(v, 0)::bit(32) as extended0, - hashoidextended(v, 1)::bit(32) as extended1 + hashoidextended(v, 0)::bit(32) as extended0, + hashoidextended(v, 1)::bit(32) as extended1 FROM (VALUES (0), (1), (17), (42), (550273), (207112489)) x(v) WHERE hashoid(v)::bit(32) != hashoidextended(v, 0)::bit(32) OR hashoid(v)::bit(32) = hashoidextended(v, 1)::bit(32); @@ -66,8 +66,8 @@ WHERE hashoid(v)::bit(32) != hashoidextended(v, 0)::bit(32) (0 rows) SELECT v as value, hashchar(v)::bit(32) as standard, - hashcharextended(v, 0)::bit(32) as extended0, - hashcharextended(v, 1)::bit(32) as extended1 + hashcharextended(v, 0)::bit(32) as extended0, + hashcharextended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::"char"), ('1'), ('x'), ('X'), ('p'), ('N')) x(v) WHERE hashchar(v)::bit(32) != hashcharextended(v, 0)::bit(32) OR hashchar(v)::bit(32) = hashcharextended(v, 1)::bit(32); @@ -76,10 +76,10 @@ WHERE hashchar(v)::bit(32) != hashcharextended(v, 0)::bit(32) (0 rows) SELECT v as value, hashname(v)::bit(32) as standard, - hashnameextended(v, 0)::bit(32) as extended0, - hashnameextended(v, 1)::bit(32) as extended1 + hashnameextended(v, 0)::bit(32) as extended0, + hashnameextended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL), ('PostgreSQL'), ('eIpUEtqmY89'), ('AXKEJBTK'), - ('muop28x03'), ('yi3nm0d73')) x(v) + ('muop28x03'), ('yi3nm0d73')) x(v) WHERE hashname(v)::bit(32) != hashnameextended(v, 0)::bit(32) OR hashname(v)::bit(32) = hashnameextended(v, 1)::bit(32); value | standard | extended0 | extended1 @@ -87,10 +87,10 @@ WHERE hashname(v)::bit(32) != hashnameextended(v, 0)::bit(32) (0 rows) SELECT v as value, hashtext(v)::bit(32) as standard, - hashtextextended(v, 0)::bit(32) as extended0, - hashtextextended(v, 1)::bit(32) as extended1 + hashtextextended(v, 0)::bit(32) as extended0, + hashtextextended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL), ('PostgreSQL'), ('eIpUEtqmY89'), ('AXKEJBTK'), - ('muop28x03'), ('yi3nm0d73')) x(v) + ('muop28x03'), ('yi3nm0d73')) x(v) WHERE hashtext(v)::bit(32) != hashtextextended(v, 0)::bit(32) OR hashtext(v)::bit(32) = hashtextextended(v, 1)::bit(32); value | standard | extended0 | extended1 @@ -98,8 +98,8 @@ WHERE hashtext(v)::bit(32) != hashtextextended(v, 0)::bit(32) (0 rows) SELECT v as value, hashoidvector(v)::bit(32) as standard, - hashoidvectorextended(v, 0)::bit(32) as extended0, - hashoidvectorextended(v, 1)::bit(32) as extended1 + hashoidvectorextended(v, 0)::bit(32) as extended0, + hashoidvectorextended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::oidvector), ('0 1 2 3 4'), ('17 18 19 20'), ('42 43 42 45'), ('550273 550273 570274'), ('207112489 207112499 21512 2155 372325 1363252')) x(v) @@ -110,8 +110,8 @@ WHERE hashoidvector(v)::bit(32) != hashoidvectorextended(v, 0)::bit(32) (0 rows) SELECT v as value, hash_aclitem(v)::bit(32) as standard, - hash_aclitem_extended(v, 0)::bit(32) as extended0, - hash_aclitem_extended(v, 1)::bit(32) as extended1 + hash_aclitem_extended(v, 0)::bit(32) as extended0, + hash_aclitem_extended(v, 1)::bit(32) as extended1 FROM (SELECT DISTINCT(relacl[1]) FROM pg_class LIMIT 10) x(v) WHERE hash_aclitem(v)::bit(32) != hash_aclitem_extended(v, 0)::bit(32) OR hash_aclitem(v)::bit(32) = hash_aclitem_extended(v, 1)::bit(32); @@ -120,10 +120,10 @@ WHERE hash_aclitem(v)::bit(32) != hash_aclitem_extended(v, 0)::bit(32) (0 rows) SELECT v as value, hashmacaddr(v)::bit(32) as standard, - hashmacaddrextended(v, 0)::bit(32) as extended0, - hashmacaddrextended(v, 1)::bit(32) as extended1 + hashmacaddrextended(v, 0)::bit(32) as extended0, + hashmacaddrextended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::macaddr), ('08:00:2b:01:02:04'), ('08:00:2b:01:02:04'), - ('e2:7f:51:3e:70:49'), ('d6:a9:4a:78:1c:d5'), + ('e2:7f:51:3e:70:49'), ('d6:a9:4a:78:1c:d5'), ('ea:29:b1:5e:1f:a5')) x(v) WHERE hashmacaddr(v)::bit(32) != hashmacaddrextended(v, 0)::bit(32) OR hashmacaddr(v)::bit(32) = hashmacaddrextended(v, 1)::bit(32); @@ -132,10 +132,10 @@ WHERE hashmacaddr(v)::bit(32) != hashmacaddrextended(v, 0)::bit(32) (0 rows) SELECT v as value, hashinet(v)::bit(32) as standard, - hashinetextended(v, 0)::bit(32) as extended0, - hashinetextended(v, 1)::bit(32) as extended1 + hashinetextended(v, 0)::bit(32) as extended0, + hashinetextended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::inet), ('192.168.100.128/25'), ('192.168.100.0/8'), - ('172.168.10.126/16'), ('172.18.103.126/24'), ('192.188.13.16/32')) x(v) + ('172.168.10.126/16'), ('172.18.103.126/24'), ('192.188.13.16/32')) x(v) WHERE hashinet(v)::bit(32) != hashinetextended(v, 0)::bit(32) OR hashinet(v)::bit(32) = hashinetextended(v, 1)::bit(32); value | standard | extended0 | extended1 @@ -143,8 +143,8 @@ WHERE hashinet(v)::bit(32) != hashinetextended(v, 0)::bit(32) (0 rows) SELECT v as value, hash_numeric(v)::bit(32) as standard, - hash_numeric_extended(v, 0)::bit(32) as extended0, - hash_numeric_extended(v, 1)::bit(32) as extended1 + hash_numeric_extended(v, 0)::bit(32) as extended0, + hash_numeric_extended(v, 1)::bit(32) as extended1 FROM (VALUES (0), (1.149484958), (17.149484958), (42.149484958), (149484958.550273), (2071124898672)) x(v) WHERE hash_numeric(v)::bit(32) != hash_numeric_extended(v, 0)::bit(32) @@ -154,8 +154,8 @@ WHERE hash_numeric(v)::bit(32) != hash_numeric_extended(v, 0)::bit(32) (0 rows) SELECT v as value, hashmacaddr8(v)::bit(32) as standard, - hashmacaddr8extended(v, 0)::bit(32) as extended0, - hashmacaddr8extended(v, 1)::bit(32) as extended1 + hashmacaddr8extended(v, 0)::bit(32) as extended0, + hashmacaddr8extended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::macaddr8), ('08:00:2b:01:02:04:36:49'), ('08:00:2b:01:02:04:f0:e8'), ('e2:7f:51:3e:70:49:16:29'), ('d6:a9:4a:78:1c:d5:47:32'), ('ea:29:b1:5e:1f:a5')) x(v) @@ -166,8 +166,8 @@ WHERE hashmacaddr8(v)::bit(32) != hashmacaddr8extended(v, 0)::bit(32) (0 rows) SELECT v as value, hash_array(v)::bit(32) as standard, - hash_array_extended(v, 0)::bit(32) as extended0, - hash_array_extended(v, 1)::bit(32) as extended1 + hash_array_extended(v, 0)::bit(32) as extended0, + hash_array_extended(v, 1)::bit(32) as extended1 FROM (VALUES ('{0}'::int4[]), ('{0,1,2,3,4}'), ('{17,18,19,20}'), ('{42,34,65,98}'), ('{550273,590027, 870273}'), ('{207112489, 807112489}')) x(v) @@ -178,10 +178,10 @@ WHERE hash_array(v)::bit(32) != hash_array_extended(v, 0)::bit(32) (0 rows) SELECT v as value, hashbpchar(v)::bit(32) as standard, - hashbpcharextended(v, 0)::bit(32) as extended0, - hashbpcharextended(v, 1)::bit(32) as extended1 + hashbpcharextended(v, 0)::bit(32) as extended0, + hashbpcharextended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL), ('PostgreSQL'), ('eIpUEtqmY89'), ('AXKEJBTK'), - ('muop28x03'), ('yi3nm0d73')) x(v) + ('muop28x03'), ('yi3nm0d73')) x(v) WHERE hashbpchar(v)::bit(32) != hashbpcharextended(v, 0)::bit(32) OR hashbpchar(v)::bit(32) = hashbpcharextended(v, 1)::bit(32); value | standard | extended0 | extended1 @@ -189,8 +189,8 @@ WHERE hashbpchar(v)::bit(32) != hashbpcharextended(v, 0)::bit(32) (0 rows) SELECT v as value, time_hash(v)::bit(32) as standard, - time_hash_extended(v, 0)::bit(32) as extended0, - time_hash_extended(v, 1)::bit(32) as extended1 + time_hash_extended(v, 0)::bit(32) as extended0, + time_hash_extended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::time), ('11:09:59'), ('1:09:59'), ('11:59:59'), ('7:9:59'), ('5:15:59')) x(v) WHERE time_hash(v)::bit(32) != time_hash_extended(v, 0)::bit(32) @@ -200,10 +200,10 @@ WHERE time_hash(v)::bit(32) != time_hash_extended(v, 0)::bit(32) (0 rows) SELECT v as value, timetz_hash(v)::bit(32) as standard, - timetz_hash_extended(v, 0)::bit(32) as extended0, - timetz_hash_extended(v, 1)::bit(32) as extended1 + timetz_hash_extended(v, 0)::bit(32) as extended0, + timetz_hash_extended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::timetz), ('00:11:52.518762-07'), ('00:11:52.51762-08'), - ('00:11:52.62-01'), ('00:11:52.62+01'), ('11:59:59+04')) x(v) + ('00:11:52.62-01'), ('00:11:52.62+01'), ('11:59:59+04')) x(v) WHERE timetz_hash(v)::bit(32) != timetz_hash_extended(v, 0)::bit(32) OR timetz_hash(v)::bit(32) = timetz_hash_extended(v, 1)::bit(32); value | standard | extended0 | extended1 @@ -211,12 +211,12 @@ WHERE timetz_hash(v)::bit(32) != timetz_hash_extended(v, 0)::bit(32) (0 rows) SELECT v as value, interval_hash(v)::bit(32) as standard, - interval_hash_extended(v, 0)::bit(32) as extended0, - interval_hash_extended(v, 1)::bit(32) as extended1 + interval_hash_extended(v, 0)::bit(32) as extended0, + interval_hash_extended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::interval), ('5 month 7 day 46 minutes'), ('1 year 7 day 46 minutes'), - ('1 year 7 month 20 day 46 minutes'), ('5 month'), - ('17 year 11 month 7 day 9 hours 46 minutes 5 seconds')) x(v) + ('1 year 7 month 20 day 46 minutes'), ('5 month'), + ('17 year 11 month 7 day 9 hours 46 minutes 5 seconds')) x(v) WHERE interval_hash(v)::bit(32) != interval_hash_extended(v, 0)::bit(32) OR interval_hash(v)::bit(32) = interval_hash_extended(v, 1)::bit(32); value | standard | extended0 | extended1 @@ -224,11 +224,11 @@ WHERE interval_hash(v)::bit(32) != interval_hash_extended(v, 0)::bit(32) (0 rows) SELECT v as value, timestamp_hash(v)::bit(32) as standard, - timestamp_hash_extended(v, 0)::bit(32) as extended0, - timestamp_hash_extended(v, 1)::bit(32) as extended1 + timestamp_hash_extended(v, 0)::bit(32) as extended0, + timestamp_hash_extended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::timestamp), ('2017-08-22 00:09:59.518762'), ('2015-08-20 00:11:52.51762-08'), - ('2017-05-22 00:11:52.62-01'), + ('2017-05-22 00:11:52.62-01'), ('2013-08-22 00:11:52.62+01'), ('2013-08-22 11:59:59+04')) x(v) WHERE timestamp_hash(v)::bit(32) != timestamp_hash_extended(v, 0)::bit(32) OR timestamp_hash(v)::bit(32) = timestamp_hash_extended(v, 1)::bit(32); @@ -237,12 +237,12 @@ WHERE timestamp_hash(v)::bit(32) != timestamp_hash_extended(v, 0)::bit(32) (0 rows) SELECT v as value, uuid_hash(v)::bit(32) as standard, - uuid_hash_extended(v, 0)::bit(32) as extended0, - uuid_hash_extended(v, 1)::bit(32) as extended1 + uuid_hash_extended(v, 0)::bit(32) as extended0, + uuid_hash_extended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::uuid), ('a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'), - ('5a9ba4ac-8d6f-11e7-bb31-be2e44b06b34'), + ('5a9ba4ac-8d6f-11e7-bb31-be2e44b06b34'), ('99c6705c-d939-461c-a3c9-1690ad64ed7b'), - ('7deed3ca-8d6f-11e7-bb31-be2e44b06b34'), + ('7deed3ca-8d6f-11e7-bb31-be2e44b06b34'), ('9ad46d4f-6f2a-4edd-aadb-745993928e1e')) x(v) WHERE uuid_hash(v)::bit(32) != uuid_hash_extended(v, 0)::bit(32) OR uuid_hash(v)::bit(32) = uuid_hash_extended(v, 1)::bit(32); @@ -251,10 +251,10 @@ WHERE uuid_hash(v)::bit(32) != uuid_hash_extended(v, 0)::bit(32) (0 rows) SELECT v as value, pg_lsn_hash(v)::bit(32) as standard, - pg_lsn_hash_extended(v, 0)::bit(32) as extended0, - pg_lsn_hash_extended(v, 1)::bit(32) as extended1 + pg_lsn_hash_extended(v, 0)::bit(32) as extended0, + pg_lsn_hash_extended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::pg_lsn), ('16/B374D84'), ('30/B374D84'), - ('255/B374D84'), ('25/B379D90'), ('900/F37FD90')) x(v) + ('255/B374D84'), ('25/B379D90'), ('900/F37FD90')) x(v) WHERE pg_lsn_hash(v)::bit(32) != pg_lsn_hash_extended(v, 0)::bit(32) OR pg_lsn_hash(v)::bit(32) = pg_lsn_hash_extended(v, 1)::bit(32); value | standard | extended0 | extended1 @@ -263,8 +263,8 @@ WHERE pg_lsn_hash(v)::bit(32) != pg_lsn_hash_extended(v, 0)::bit(32) CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy'); SELECT v as value, hashenum(v)::bit(32) as standard, - hashenumextended(v, 0)::bit(32) as extended0, - hashenumextended(v, 1)::bit(32) as extended1 + hashenumextended(v, 0)::bit(32) as extended0, + hashenumextended(v, 1)::bit(32) as extended1 FROM (VALUES ('sad'::mood), ('ok'), ('happy')) x(v) WHERE hashenum(v)::bit(32) != hashenumextended(v, 0)::bit(32) OR hashenum(v)::bit(32) = hashenumextended(v, 1)::bit(32); @@ -274,12 +274,12 @@ WHERE hashenum(v)::bit(32) != hashenumextended(v, 0)::bit(32) DROP TYPE mood; SELECT v as value, jsonb_hash(v)::bit(32) as standard, - jsonb_hash_extended(v, 0)::bit(32) as extended0, - jsonb_hash_extended(v, 1)::bit(32) as extended1 + jsonb_hash_extended(v, 0)::bit(32) as extended0, + jsonb_hash_extended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::jsonb), - ('{"a": "aaa bbb ddd ccc", "b": ["eee fff ggg"], "c": {"d": "hhh iii"}}'), - ('{"foo": [true, "bar"], "tags": {"e": 1, "f": null}}'), - ('{"g": {"h": "value"}}')) x(v) + ('{"a": "aaa bbb ddd ccc", "b": ["eee fff ggg"], "c": {"d": "hhh iii"}}'), + ('{"foo": [true, "bar"], "tags": {"e": 1, "f": null}}'), + ('{"g": {"h": "value"}}')) x(v) WHERE jsonb_hash(v)::bit(32) != jsonb_hash_extended(v, 0)::bit(32) OR jsonb_hash(v)::bit(32) = jsonb_hash_extended(v, 1)::bit(32); value | standard | extended0 | extended1 @@ -287,11 +287,11 @@ WHERE jsonb_hash(v)::bit(32) != jsonb_hash_extended(v, 0)::bit(32) (0 rows) SELECT v as value, hash_range(v)::bit(32) as standard, - hash_range_extended(v, 0)::bit(32) as extended0, - hash_range_extended(v, 1)::bit(32) as extended1 + hash_range_extended(v, 0)::bit(32) as extended0, + hash_range_extended(v, 1)::bit(32) as extended1 FROM (VALUES (int4range(10, 20)), (int4range(23, 43)), - (int4range(5675, 550273)), - (int4range(550274, 1550274)), (int4range(1550275, 208112489))) x(v) + (int4range(5675, 550273)), + (int4range(550274, 1550274)), (int4range(1550275, 208112489))) x(v) WHERE hash_range(v)::bit(32) != hash_range_extended(v, 0)::bit(32) OR hash_range(v)::bit(32) = hash_range_extended(v, 1)::bit(32); value | standard | extended0 | extended1 diff --git a/src/test/regress/sql/hash_func.sql b/src/test/regress/sql/hash_func.sql index b7ce8b21a3a0..a3e2decc2c4b 100644 --- a/src/test/regress/sql/hash_func.sql +++ b/src/test/regress/sql/hash_func.sql @@ -14,66 +14,66 @@ WHERE hashint2(v)::bit(32) != hashint2extended(v, 0)::bit(32) OR hashint2(v)::bit(32) = hashint2extended(v, 1)::bit(32); SELECT v as value, hashint4(v)::bit(32) as standard, - hashint4extended(v, 0)::bit(32) as extended0, - hashint4extended(v, 1)::bit(32) as extended1 + hashint4extended(v, 0)::bit(32) as extended0, + hashint4extended(v, 1)::bit(32) as extended1 FROM (VALUES (0), (1), (17), (42), (550273), (207112489)) x(v) WHERE hashint4(v)::bit(32) != hashint4extended(v, 0)::bit(32) OR hashint4(v)::bit(32) = hashint4extended(v, 1)::bit(32); SELECT v as value, hashint8(v)::bit(32) as standard, - hashint8extended(v, 0)::bit(32) as extended0, - hashint8extended(v, 1)::bit(32) as extended1 + hashint8extended(v, 0)::bit(32) as extended0, + hashint8extended(v, 1)::bit(32) as extended1 FROM (VALUES (0), (1), (17), (42), (550273), (207112489)) x(v) WHERE hashint8(v)::bit(32) != hashint8extended(v, 0)::bit(32) OR hashint8(v)::bit(32) = hashint8extended(v, 1)::bit(32); SELECT v as value, hashfloat4(v)::bit(32) as standard, - hashfloat4extended(v, 0)::bit(32) as extended0, - hashfloat4extended(v, 1)::bit(32) as extended1 + hashfloat4extended(v, 0)::bit(32) as extended0, + hashfloat4extended(v, 1)::bit(32) as extended1 FROM (VALUES (0), (1), (17), (42), (550273), (207112489)) x(v) WHERE hashfloat4(v)::bit(32) != hashfloat4extended(v, 0)::bit(32) OR hashfloat4(v)::bit(32) = hashfloat4extended(v, 1)::bit(32); SELECT v as value, hashfloat8(v)::bit(32) as standard, - hashfloat8extended(v, 0)::bit(32) as extended0, - hashfloat8extended(v, 1)::bit(32) as extended1 + hashfloat8extended(v, 0)::bit(32) as extended0, + hashfloat8extended(v, 1)::bit(32) as extended1 FROM (VALUES (0), (1), (17), (42), (550273), (207112489)) x(v) WHERE hashfloat8(v)::bit(32) != hashfloat8extended(v, 0)::bit(32) OR hashfloat8(v)::bit(32) = hashfloat8extended(v, 1)::bit(32); SELECT v as value, hashoid(v)::bit(32) as standard, - hashoidextended(v, 0)::bit(32) as extended0, - hashoidextended(v, 1)::bit(32) as extended1 + hashoidextended(v, 0)::bit(32) as extended0, + hashoidextended(v, 1)::bit(32) as extended1 FROM (VALUES (0), (1), (17), (42), (550273), (207112489)) x(v) WHERE hashoid(v)::bit(32) != hashoidextended(v, 0)::bit(32) OR hashoid(v)::bit(32) = hashoidextended(v, 1)::bit(32); SELECT v as value, hashchar(v)::bit(32) as standard, - hashcharextended(v, 0)::bit(32) as extended0, - hashcharextended(v, 1)::bit(32) as extended1 + hashcharextended(v, 0)::bit(32) as extended0, + hashcharextended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::"char"), ('1'), ('x'), ('X'), ('p'), ('N')) x(v) WHERE hashchar(v)::bit(32) != hashcharextended(v, 0)::bit(32) OR hashchar(v)::bit(32) = hashcharextended(v, 1)::bit(32); SELECT v as value, hashname(v)::bit(32) as standard, - hashnameextended(v, 0)::bit(32) as extended0, - hashnameextended(v, 1)::bit(32) as extended1 + hashnameextended(v, 0)::bit(32) as extended0, + hashnameextended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL), ('PostgreSQL'), ('eIpUEtqmY89'), ('AXKEJBTK'), - ('muop28x03'), ('yi3nm0d73')) x(v) + ('muop28x03'), ('yi3nm0d73')) x(v) WHERE hashname(v)::bit(32) != hashnameextended(v, 0)::bit(32) OR hashname(v)::bit(32) = hashnameextended(v, 1)::bit(32); SELECT v as value, hashtext(v)::bit(32) as standard, - hashtextextended(v, 0)::bit(32) as extended0, - hashtextextended(v, 1)::bit(32) as extended1 + hashtextextended(v, 0)::bit(32) as extended0, + hashtextextended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL), ('PostgreSQL'), ('eIpUEtqmY89'), ('AXKEJBTK'), - ('muop28x03'), ('yi3nm0d73')) x(v) + ('muop28x03'), ('yi3nm0d73')) x(v) WHERE hashtext(v)::bit(32) != hashtextextended(v, 0)::bit(32) OR hashtext(v)::bit(32) = hashtextextended(v, 1)::bit(32); SELECT v as value, hashoidvector(v)::bit(32) as standard, - hashoidvectorextended(v, 0)::bit(32) as extended0, - hashoidvectorextended(v, 1)::bit(32) as extended1 + hashoidvectorextended(v, 0)::bit(32) as extended0, + hashoidvectorextended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::oidvector), ('0 1 2 3 4'), ('17 18 19 20'), ('42 43 42 45'), ('550273 550273 570274'), ('207112489 207112499 21512 2155 372325 1363252')) x(v) @@ -81,40 +81,40 @@ WHERE hashoidvector(v)::bit(32) != hashoidvectorextended(v, 0)::bit(32) OR hashoidvector(v)::bit(32) = hashoidvectorextended(v, 1)::bit(32); SELECT v as value, hash_aclitem(v)::bit(32) as standard, - hash_aclitem_extended(v, 0)::bit(32) as extended0, - hash_aclitem_extended(v, 1)::bit(32) as extended1 + hash_aclitem_extended(v, 0)::bit(32) as extended0, + hash_aclitem_extended(v, 1)::bit(32) as extended1 FROM (SELECT DISTINCT(relacl[1]) FROM pg_class LIMIT 10) x(v) WHERE hash_aclitem(v)::bit(32) != hash_aclitem_extended(v, 0)::bit(32) OR hash_aclitem(v)::bit(32) = hash_aclitem_extended(v, 1)::bit(32); SELECT v as value, hashmacaddr(v)::bit(32) as standard, - hashmacaddrextended(v, 0)::bit(32) as extended0, - hashmacaddrextended(v, 1)::bit(32) as extended1 + hashmacaddrextended(v, 0)::bit(32) as extended0, + hashmacaddrextended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::macaddr), ('08:00:2b:01:02:04'), ('08:00:2b:01:02:04'), - ('e2:7f:51:3e:70:49'), ('d6:a9:4a:78:1c:d5'), + ('e2:7f:51:3e:70:49'), ('d6:a9:4a:78:1c:d5'), ('ea:29:b1:5e:1f:a5')) x(v) WHERE hashmacaddr(v)::bit(32) != hashmacaddrextended(v, 0)::bit(32) OR hashmacaddr(v)::bit(32) = hashmacaddrextended(v, 1)::bit(32); SELECT v as value, hashinet(v)::bit(32) as standard, - hashinetextended(v, 0)::bit(32) as extended0, - hashinetextended(v, 1)::bit(32) as extended1 + hashinetextended(v, 0)::bit(32) as extended0, + hashinetextended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::inet), ('192.168.100.128/25'), ('192.168.100.0/8'), - ('172.168.10.126/16'), ('172.18.103.126/24'), ('192.188.13.16/32')) x(v) + ('172.168.10.126/16'), ('172.18.103.126/24'), ('192.188.13.16/32')) x(v) WHERE hashinet(v)::bit(32) != hashinetextended(v, 0)::bit(32) OR hashinet(v)::bit(32) = hashinetextended(v, 1)::bit(32); SELECT v as value, hash_numeric(v)::bit(32) as standard, - hash_numeric_extended(v, 0)::bit(32) as extended0, - hash_numeric_extended(v, 1)::bit(32) as extended1 + hash_numeric_extended(v, 0)::bit(32) as extended0, + hash_numeric_extended(v, 1)::bit(32) as extended1 FROM (VALUES (0), (1.149484958), (17.149484958), (42.149484958), (149484958.550273), (2071124898672)) x(v) WHERE hash_numeric(v)::bit(32) != hash_numeric_extended(v, 0)::bit(32) OR hash_numeric(v)::bit(32) = hash_numeric_extended(v, 1)::bit(32); SELECT v as value, hashmacaddr8(v)::bit(32) as standard, - hashmacaddr8extended(v, 0)::bit(32) as extended0, - hashmacaddr8extended(v, 1)::bit(32) as extended1 + hashmacaddr8extended(v, 0)::bit(32) as extended0, + hashmacaddr8extended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::macaddr8), ('08:00:2b:01:02:04:36:49'), ('08:00:2b:01:02:04:f0:e8'), ('e2:7f:51:3e:70:49:16:29'), ('d6:a9:4a:78:1c:d5:47:32'), ('ea:29:b1:5e:1f:a5')) x(v) @@ -122,8 +122,8 @@ WHERE hashmacaddr8(v)::bit(32) != hashmacaddr8extended(v, 0)::bit(32) OR hashmacaddr8(v)::bit(32) = hashmacaddr8extended(v, 1)::bit(32); SELECT v as value, hash_array(v)::bit(32) as standard, - hash_array_extended(v, 0)::bit(32) as extended0, - hash_array_extended(v, 1)::bit(32) as extended1 + hash_array_extended(v, 0)::bit(32) as extended0, + hash_array_extended(v, 1)::bit(32) as extended1 FROM (VALUES ('{0}'::int4[]), ('{0,1,2,3,4}'), ('{17,18,19,20}'), ('{42,34,65,98}'), ('{550273,590027, 870273}'), ('{207112489, 807112489}')) x(v) @@ -131,92 +131,92 @@ WHERE hash_array(v)::bit(32) != hash_array_extended(v, 0)::bit(32) OR hash_array(v)::bit(32) = hash_array_extended(v, 1)::bit(32); SELECT v as value, hashbpchar(v)::bit(32) as standard, - hashbpcharextended(v, 0)::bit(32) as extended0, - hashbpcharextended(v, 1)::bit(32) as extended1 + hashbpcharextended(v, 0)::bit(32) as extended0, + hashbpcharextended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL), ('PostgreSQL'), ('eIpUEtqmY89'), ('AXKEJBTK'), - ('muop28x03'), ('yi3nm0d73')) x(v) + ('muop28x03'), ('yi3nm0d73')) x(v) WHERE hashbpchar(v)::bit(32) != hashbpcharextended(v, 0)::bit(32) OR hashbpchar(v)::bit(32) = hashbpcharextended(v, 1)::bit(32); SELECT v as value, time_hash(v)::bit(32) as standard, - time_hash_extended(v, 0)::bit(32) as extended0, - time_hash_extended(v, 1)::bit(32) as extended1 + time_hash_extended(v, 0)::bit(32) as extended0, + time_hash_extended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::time), ('11:09:59'), ('1:09:59'), ('11:59:59'), ('7:9:59'), ('5:15:59')) x(v) WHERE time_hash(v)::bit(32) != time_hash_extended(v, 0)::bit(32) OR time_hash(v)::bit(32) = time_hash_extended(v, 1)::bit(32); SELECT v as value, timetz_hash(v)::bit(32) as standard, - timetz_hash_extended(v, 0)::bit(32) as extended0, - timetz_hash_extended(v, 1)::bit(32) as extended1 + timetz_hash_extended(v, 0)::bit(32) as extended0, + timetz_hash_extended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::timetz), ('00:11:52.518762-07'), ('00:11:52.51762-08'), - ('00:11:52.62-01'), ('00:11:52.62+01'), ('11:59:59+04')) x(v) + ('00:11:52.62-01'), ('00:11:52.62+01'), ('11:59:59+04')) x(v) WHERE timetz_hash(v)::bit(32) != timetz_hash_extended(v, 0)::bit(32) OR timetz_hash(v)::bit(32) = timetz_hash_extended(v, 1)::bit(32); SELECT v as value, interval_hash(v)::bit(32) as standard, - interval_hash_extended(v, 0)::bit(32) as extended0, - interval_hash_extended(v, 1)::bit(32) as extended1 + interval_hash_extended(v, 0)::bit(32) as extended0, + interval_hash_extended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::interval), ('5 month 7 day 46 minutes'), ('1 year 7 day 46 minutes'), - ('1 year 7 month 20 day 46 minutes'), ('5 month'), - ('17 year 11 month 7 day 9 hours 46 minutes 5 seconds')) x(v) + ('1 year 7 month 20 day 46 minutes'), ('5 month'), + ('17 year 11 month 7 day 9 hours 46 minutes 5 seconds')) x(v) WHERE interval_hash(v)::bit(32) != interval_hash_extended(v, 0)::bit(32) OR interval_hash(v)::bit(32) = interval_hash_extended(v, 1)::bit(32); SELECT v as value, timestamp_hash(v)::bit(32) as standard, - timestamp_hash_extended(v, 0)::bit(32) as extended0, - timestamp_hash_extended(v, 1)::bit(32) as extended1 + timestamp_hash_extended(v, 0)::bit(32) as extended0, + timestamp_hash_extended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::timestamp), ('2017-08-22 00:09:59.518762'), ('2015-08-20 00:11:52.51762-08'), - ('2017-05-22 00:11:52.62-01'), + ('2017-05-22 00:11:52.62-01'), ('2013-08-22 00:11:52.62+01'), ('2013-08-22 11:59:59+04')) x(v) WHERE timestamp_hash(v)::bit(32) != timestamp_hash_extended(v, 0)::bit(32) OR timestamp_hash(v)::bit(32) = timestamp_hash_extended(v, 1)::bit(32); SELECT v as value, uuid_hash(v)::bit(32) as standard, - uuid_hash_extended(v, 0)::bit(32) as extended0, - uuid_hash_extended(v, 1)::bit(32) as extended1 + uuid_hash_extended(v, 0)::bit(32) as extended0, + uuid_hash_extended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::uuid), ('a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'), - ('5a9ba4ac-8d6f-11e7-bb31-be2e44b06b34'), + ('5a9ba4ac-8d6f-11e7-bb31-be2e44b06b34'), ('99c6705c-d939-461c-a3c9-1690ad64ed7b'), - ('7deed3ca-8d6f-11e7-bb31-be2e44b06b34'), + ('7deed3ca-8d6f-11e7-bb31-be2e44b06b34'), ('9ad46d4f-6f2a-4edd-aadb-745993928e1e')) x(v) WHERE uuid_hash(v)::bit(32) != uuid_hash_extended(v, 0)::bit(32) OR uuid_hash(v)::bit(32) = uuid_hash_extended(v, 1)::bit(32); SELECT v as value, pg_lsn_hash(v)::bit(32) as standard, - pg_lsn_hash_extended(v, 0)::bit(32) as extended0, - pg_lsn_hash_extended(v, 1)::bit(32) as extended1 + pg_lsn_hash_extended(v, 0)::bit(32) as extended0, + pg_lsn_hash_extended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::pg_lsn), ('16/B374D84'), ('30/B374D84'), - ('255/B374D84'), ('25/B379D90'), ('900/F37FD90')) x(v) + ('255/B374D84'), ('25/B379D90'), ('900/F37FD90')) x(v) WHERE pg_lsn_hash(v)::bit(32) != pg_lsn_hash_extended(v, 0)::bit(32) OR pg_lsn_hash(v)::bit(32) = pg_lsn_hash_extended(v, 1)::bit(32); CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy'); SELECT v as value, hashenum(v)::bit(32) as standard, - hashenumextended(v, 0)::bit(32) as extended0, - hashenumextended(v, 1)::bit(32) as extended1 + hashenumextended(v, 0)::bit(32) as extended0, + hashenumextended(v, 1)::bit(32) as extended1 FROM (VALUES ('sad'::mood), ('ok'), ('happy')) x(v) WHERE hashenum(v)::bit(32) != hashenumextended(v, 0)::bit(32) OR hashenum(v)::bit(32) = hashenumextended(v, 1)::bit(32); DROP TYPE mood; SELECT v as value, jsonb_hash(v)::bit(32) as standard, - jsonb_hash_extended(v, 0)::bit(32) as extended0, - jsonb_hash_extended(v, 1)::bit(32) as extended1 + jsonb_hash_extended(v, 0)::bit(32) as extended0, + jsonb_hash_extended(v, 1)::bit(32) as extended1 FROM (VALUES (NULL::jsonb), - ('{"a": "aaa bbb ddd ccc", "b": ["eee fff ggg"], "c": {"d": "hhh iii"}}'), - ('{"foo": [true, "bar"], "tags": {"e": 1, "f": null}}'), - ('{"g": {"h": "value"}}')) x(v) + ('{"a": "aaa bbb ddd ccc", "b": ["eee fff ggg"], "c": {"d": "hhh iii"}}'), + ('{"foo": [true, "bar"], "tags": {"e": 1, "f": null}}'), + ('{"g": {"h": "value"}}')) x(v) WHERE jsonb_hash(v)::bit(32) != jsonb_hash_extended(v, 0)::bit(32) OR jsonb_hash(v)::bit(32) = jsonb_hash_extended(v, 1)::bit(32); SELECT v as value, hash_range(v)::bit(32) as standard, - hash_range_extended(v, 0)::bit(32) as extended0, - hash_range_extended(v, 1)::bit(32) as extended1 + hash_range_extended(v, 0)::bit(32) as extended0, + hash_range_extended(v, 1)::bit(32) as extended1 FROM (VALUES (int4range(10, 20)), (int4range(23, 43)), - (int4range(5675, 550273)), - (int4range(550274, 1550274)), (int4range(1550275, 208112489))) x(v) + (int4range(5675, 550273)), + (int4range(550274, 1550274)), (int4range(1550275, 208112489))) x(v) WHERE hash_range(v)::bit(32) != hash_range_extended(v, 0)::bit(32) OR hash_range(v)::bit(32) = hash_range_extended(v, 1)::bit(32); From 26ec6b5948a73d0e07ed9435ee4554594acdf34f Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 19 Oct 2020 08:52:25 +0200 Subject: [PATCH 331/589] Avoid invalid alloc size error in shm_mq In shm_mq_receive(), a huge payload could trigger an unjustified "invalid memory alloc request size" error due to the way the buffer size is increased. Add error checks (documenting the upper limit) and avoid the error by limiting the allocation size to MaxAllocSize. Author: Markus Wanner Discussion: https://www.postgresql.org/message-id/flat/3bb363e7-ac04-0ac4-9fe8-db1148755bfa%402ndquadrant.com --- src/backend/storage/ipc/shm_mq.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/backend/storage/ipc/shm_mq.c b/src/backend/storage/ipc/shm_mq.c index 6f40fb165d28..ac9d23a3403a 100644 --- a/src/backend/storage/ipc/shm_mq.c +++ b/src/backend/storage/ipc/shm_mq.c @@ -24,6 +24,7 @@ #include "storage/procsignal.h" #include "storage/shm_mq.h" #include "storage/spin.h" +#include "utils/memutils.h" /* * This structure represents the actual queue, stored in shared memory. @@ -360,6 +361,13 @@ shm_mq_sendv(shm_mq_handle *mqh, shm_mq_iovec *iov, int iovcnt, bool nowait) for (i = 0; i < iovcnt; ++i) nbytes += iov[i].len; + /* Prevent writing messages overwhelming the receiver. */ + if (nbytes > MaxAllocSize) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("cannot send a message of size %zu via shared memory queue", + nbytes))); + /* Try to write, or finish writing, the length word into the buffer. */ while (!mqh->mqh_length_word_complete) { @@ -675,6 +683,17 @@ shm_mq_receive(shm_mq_handle *mqh, Size *nbytesp, void **datap, bool nowait) } nbytes = mqh->mqh_expected_bytes; + /* + * Should be disallowed on the sending side already, but better check and + * error out on the receiver side as well rather than trying to read a + * prohibitively large message. + */ + if (nbytes > MaxAllocSize) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("invalid message size %zu in shared memory queue", + nbytes))); + if (mqh->mqh_partial_bytes == 0) { /* @@ -703,8 +722,13 @@ shm_mq_receive(shm_mq_handle *mqh, Size *nbytesp, void **datap, bool nowait) { Size newbuflen = Max(mqh->mqh_buflen, MQH_INITIAL_BUFSIZE); + /* + * Double the buffer size until the payload fits, but limit to + * MaxAllocSize. + */ while (newbuflen < nbytes) newbuflen *= 2; + newbuflen = Min(newbuflen, MaxAllocSize); if (mqh->mqh_buffer != NULL) { From f49b85d783f6781138f33bbe5f6e98da86907d84 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 19 Oct 2020 14:11:44 +0300 Subject: [PATCH 332/589] Clean up code to resolve the "root target relation" in nodeModifyTable.c When executing DDL on a partitioned table or on a table with inheritance children, statement-level triggers must be fired against the table given in the original statement. The code to look that up was a bit messy and duplicative. Commit 501ed02cf6 added a helper function, getASTriggerResultRelInfo() (later renamed to getTargetResultRelInfo()) for it, but for some reason it was only used when firing AFTER STATEMENT triggers and the code to fire BEFORE STATEMENT triggers duplicated the logic. Determine the target relation in ExecInitModifyTable(), and set it always in ModifyTableState. Code that used to call getTargetResultRelInfo() can now use ModifyTableState->rootResultRelInfo directly. Discussion: https://www.postgresql.org/message-id/CA%2BHiwqFViT47Zbr_ASBejiK7iDG8%3DQ1swQ-tjM6caRPQ67pT%3Dw%40mail.gmail.com --- src/backend/executor/nodeModifyTable.c | 62 ++++++++++---------------- src/include/nodes/execnodes.h | 9 +++- 2 files changed, 31 insertions(+), 40 deletions(-) diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index 0c055ed40805..ae209bc774a3 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -72,7 +72,6 @@ static TupleTableSlot *ExecPrepareTupleRouting(ModifyTableState *mtstate, ResultRelInfo *targetRelInfo, TupleTableSlot *slot, ResultRelInfo **partRelInfo); -static ResultRelInfo *getTargetResultRelInfo(ModifyTableState *node); static void ExecSetupChildParentMapForSubplan(ModifyTableState *mtstate); static TupleConversionMap *tupconv_map_for_subplan(ModifyTableState *node, int whichplan); @@ -1186,7 +1185,6 @@ ExecCrossPartitionUpdate(ModifyTableState *mtstate, saved_tcs_map = mtstate->mt_transition_capture->tcs_map; /* Tuple routing starts from the root table. */ - Assert(mtstate->rootResultRelInfo != NULL); *inserted_tuple = ExecInsert(mtstate, mtstate->rootResultRelInfo, slot, planSlot, estate, canSetTag); @@ -1796,15 +1794,7 @@ static void fireBSTriggers(ModifyTableState *node) { ModifyTable *plan = (ModifyTable *) node->ps.plan; - ResultRelInfo *resultRelInfo = node->resultRelInfo; - - /* - * If the node modifies a partitioned table, we must fire its triggers. - * Note that in that case, node->resultRelInfo points to the first leaf - * partition, not the root table. - */ - if (node->rootResultRelInfo != NULL) - resultRelInfo = node->rootResultRelInfo; + ResultRelInfo *resultRelInfo = node->rootResultRelInfo; switch (node->operation) { @@ -1826,28 +1816,6 @@ fireBSTriggers(ModifyTableState *node) } } -/* - * Return the target rel ResultRelInfo. - * - * This relation is the same as : - * - the relation for which we will fire AFTER STATEMENT triggers. - * - the relation into whose tuple format all captured transition tuples must - * be converted. - * - the root partitioned table. - */ -static ResultRelInfo * -getTargetResultRelInfo(ModifyTableState *node) -{ - /* - * Note that if the node modifies a partitioned table, node->resultRelInfo - * points to the first leaf partition, not the root table. - */ - if (node->rootResultRelInfo != NULL) - return node->rootResultRelInfo; - else - return node->resultRelInfo; -} - /* * Process AFTER EACH STATEMENT triggers */ @@ -1855,7 +1823,7 @@ static void fireASTriggers(ModifyTableState *node) { ModifyTable *plan = (ModifyTable *) node->ps.plan; - ResultRelInfo *resultRelInfo = getTargetResultRelInfo(node); + ResultRelInfo *resultRelInfo = node->rootResultRelInfo; switch (node->operation) { @@ -1889,7 +1857,7 @@ static void ExecSetupTransitionCaptureState(ModifyTableState *mtstate, EState *estate) { ModifyTable *plan = (ModifyTable *) mtstate->ps.plan; - ResultRelInfo *targetRelInfo = getTargetResultRelInfo(mtstate); + ResultRelInfo *targetRelInfo = mtstate->rootResultRelInfo; /* Check for transition tables on the directly targeted relation. */ mtstate->mt_transition_capture = @@ -2019,7 +1987,7 @@ ExecPrepareTupleRouting(ModifyTableState *mtstate, static void ExecSetupChildParentMapForSubplan(ModifyTableState *mtstate) { - ResultRelInfo *targetRelInfo = getTargetResultRelInfo(mtstate); + ResultRelInfo *targetRelInfo = mtstate->rootResultRelInfo; ResultRelInfo *resultRelInfos = mtstate->resultRelInfo; TupleDesc outdesc; int numResultRelInfos = mtstate->mt_nplans; @@ -2355,13 +2323,31 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) palloc(nplans * sizeof(ResultRelInfo)); mtstate->mt_scans = (TupleTableSlot **) palloc0(sizeof(TupleTableSlot *) * nplans); - /* If modifying a partitioned table, initialize the root table info */ + /*---------- + * Resolve the target relation. This is the same as: + * + * - the relation for which we will fire FOR STATEMENT triggers, + * - the relation into whose tuple format all captured transition tuples + * must be converted, and + * - the root partitioned table used for tuple routing. + * + * If it's a partitioned table, the root partition doesn't appear + * elsewhere in the plan and its RT index is given explicitly in + * node->rootRelation. Otherwise (i.e. table inheritance) the target + * relation is the first relation in the node->resultRelations list, and + * we will initialize it in the loop below. + *---------- + */ if (node->rootRelation > 0) { mtstate->rootResultRelInfo = makeNode(ResultRelInfo); ExecInitResultRelation(estate, mtstate->rootResultRelInfo, node->rootRelation); } + else + { + mtstate->rootResultRelInfo = mtstate->resultRelInfo; + } mtstate->mt_arowmarks = (List **) palloc0(sizeof(List *) * nplans); mtstate->mt_nplans = nplans; @@ -2446,7 +2432,7 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) } /* Get the target relation */ - rel = (getTargetResultRelInfo(mtstate))->ri_RelationDesc; + rel = mtstate->rootResultRelInfo->ri_RelationDesc; /* * If it's not a partitioned table after all, UPDATE tuple routing should diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index b7e9e5d539de..dff34fbc14e2 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1154,8 +1154,13 @@ typedef struct ModifyTableState TupleTableSlot **mt_scans; /* input tuple corresponding to underlying * plans */ ResultRelInfo *resultRelInfo; /* per-subplan target relations */ - ResultRelInfo *rootResultRelInfo; /* root target relation (partitioned - * table root) */ + + /* + * Target relation mentioned in the original statement, used to fire + * statement-level triggers and as the root for tuple routing. + */ + ResultRelInfo *rootResultRelInfo; + List **mt_arowmarks; /* per-subplan ExecAuxRowMark lists */ EPQState mt_epqstate; /* for evaluating EvalPlanQual rechecks */ bool fireBSTriggers; /* do we need to fire stmt triggers? */ From 6973533650c04653d9018f61c1ac99ecb11094bd Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 19 Oct 2020 14:11:54 +0300 Subject: [PATCH 333/589] Revise child-to-root tuple conversion map management. Store the tuple conversion map to convert a tuple from a child table's format to the root format in a new ri_ChildToRootMap field in ResultRelInfo. It is initialized if transition tuple capture for FOR STATEMENT triggers or INSERT tuple routing on a partitioned table is needed. Previously, ModifyTable kept the maps in the per-subplan ModifyTableState->mt_per_subplan_tupconv_maps array, or when tuple routing was used, in ResultRelInfo->ri_Partitioninfo->pi_PartitionToRootMap. The new field replaces both of those. Now that the child-to-root tuple conversion map is always available in ResultRelInfo (when needed), remove the TransitionCaptureState.tcs_map field. The callers of Exec*Trigger() functions no longer need to set or save it, which is much less confusing and bug-prone. Also, as a future optimization, this will allow us to delay creating the map for a given result relation until the relation is actually processed during execution. Author: Amit Langote Discussion: https://www.postgresql.org/message-id/CA%2BHiwqHtCWLdK-LO%3DNEsvOdHx%2B7yv4mE_zYK0i3BH7dXb-wxog%40mail.gmail.com --- src/backend/commands/copy.c | 29 +--- src/backend/commands/trigger.c | 10 +- src/backend/executor/execMain.c | 1 + src/backend/executor/execPartition.c | 23 ++- src/backend/executor/nodeModifyTable.c | 210 +++++++------------------ src/include/commands/trigger.h | 10 +- src/include/executor/execPartition.h | 6 - src/include/nodes/execnodes.h | 10 +- 8 files changed, 84 insertions(+), 215 deletions(-) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 531bd7c73a52..b4ccd35ec180 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -3106,31 +3106,14 @@ CopyFrom(CopyState cstate) /* * If we're capturing transition tuples, we might need to convert - * from the partition rowtype to root rowtype. + * from the partition rowtype to root rowtype. But if there are no + * BEFORE triggers on the partition that could change the tuple, + * we can just remember the original unconverted tuple to avoid a + * needless round trip conversion. */ if (cstate->transition_capture != NULL) - { - if (has_before_insert_row_trig) - { - /* - * If there are any BEFORE triggers on the partition, - * we'll have to be ready to convert their result back to - * tuplestore format. - */ - cstate->transition_capture->tcs_original_insert_tuple = NULL; - cstate->transition_capture->tcs_map = - resultRelInfo->ri_PartitionInfo->pi_PartitionToRootMap; - } - else - { - /* - * Otherwise, just remember the original unconverted - * tuple, to avoid a needless round trip conversion. - */ - cstate->transition_capture->tcs_original_insert_tuple = myslot; - cstate->transition_capture->tcs_map = NULL; - } - } + cstate->transition_capture->tcs_original_insert_tuple = + !has_before_insert_row_trig ? myslot : NULL; /* * We might need to convert from the root rowtype to the partition diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c index 3b4fbdadf4f3..5719672f4ce1 100644 --- a/src/backend/commands/trigger.c +++ b/src/backend/commands/trigger.c @@ -35,6 +35,7 @@ #include "commands/defrem.h" #include "commands/trigger.h" #include "executor/executor.h" +#include "executor/execPartition.h" #include "miscadmin.h" #include "nodes/bitmapset.h" #include "nodes/makefuncs.h" @@ -4292,9 +4293,10 @@ GetAfterTriggersTableData(Oid relid, CmdType cmdType) * If there are no triggers in 'trigdesc' that request relevant transition * tables, then return NULL. * - * The resulting object can be passed to the ExecAR* functions. The caller - * should set tcs_map or tcs_original_insert_tuple as appropriate when dealing - * with child tables. + * The resulting object can be passed to the ExecAR* functions. When + * dealing with child tables, the caller can set tcs_original_insert_tuple + * to avoid having to reconstruct the original tuple in the root table's + * format. * * Note that we copy the flags from a parent table into this struct (rather * than subsequently using the relation's TriggerDesc directly) so that we can @@ -5389,7 +5391,7 @@ AfterTriggerSaveEvent(EState *estate, ResultRelInfo *relinfo, if (row_trigger && transition_capture != NULL) { TupleTableSlot *original_insert_tuple = transition_capture->tcs_original_insert_tuple; - TupleConversionMap *map = transition_capture->tcs_map; + TupleConversionMap *map = relinfo->ri_ChildToRootMap; bool delete_old_table = transition_capture->tcs_delete_old_table; bool update_old_table = transition_capture->tcs_update_old_table; bool update_new_table = transition_capture->tcs_update_new_table; diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index 293f53d07c9d..fcdd4b356741 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -1244,6 +1244,7 @@ InitResultRelInfo(ResultRelInfo *resultRelInfo, resultRelInfo->ri_TrigNewSlot = NULL; resultRelInfo->ri_PartitionRoot = partition_root; resultRelInfo->ri_PartitionInfo = NULL; /* may be set later */ + resultRelInfo->ri_ChildToRootMap = NULL; resultRelInfo->ri_CopyMultiInsertBuffer = NULL; } diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 33d2c6f63de4..08f91e59a7ac 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -907,6 +907,15 @@ ExecInitPartitionInfo(ModifyTableState *mtstate, EState *estate, } } + /* + * Also, if transition capture is required, store a map to convert tuples + * from partition's rowtype to the root partition table's. + */ + if (mtstate->mt_transition_capture || mtstate->mt_oc_transition_capture) + leaf_part_rri->ri_ChildToRootMap = + convert_tuples_by_name(RelationGetDescr(leaf_part_rri->ri_RelationDesc), + RelationGetDescr(leaf_part_rri->ri_PartitionRoot)); + /* * Since we've just initialized this ResultRelInfo, it's not in any list * attached to the estate as yet. Add it, so that it can be found later. @@ -976,20 +985,6 @@ ExecInitRoutingInfo(ModifyTableState *mtstate, else partrouteinfo->pi_PartitionTupleSlot = NULL; - /* - * Also, if transition capture is required, store a map to convert tuples - * from partition's rowtype to the root partition table's. - */ - if (mtstate && - (mtstate->mt_transition_capture || mtstate->mt_oc_transition_capture)) - { - partrouteinfo->pi_PartitionToRootMap = - convert_tuples_by_name(RelationGetDescr(partRelInfo->ri_RelationDesc), - RelationGetDescr(partRelInfo->ri_PartitionRoot)); - } - else - partrouteinfo->pi_PartitionToRootMap = NULL; - /* * If the partition is a foreign table, let the FDW init itself for * routing tuples to the partition. diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index ae209bc774a3..5f85fd7cd88f 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -72,9 +72,6 @@ static TupleTableSlot *ExecPrepareTupleRouting(ModifyTableState *mtstate, ResultRelInfo *targetRelInfo, TupleTableSlot *slot, ResultRelInfo **partRelInfo); -static void ExecSetupChildParentMapForSubplan(ModifyTableState *mtstate); -static TupleConversionMap *tupconv_map_for_subplan(ModifyTableState *node, - int whichplan); /* * Verify that the tuples to be produced by INSERT or UPDATE match the @@ -1086,9 +1083,7 @@ ExecCrossPartitionUpdate(ModifyTableState *mtstate, { EState *estate = mtstate->ps.state; PartitionTupleRouting *proute = mtstate->mt_partition_tuple_routing; - int map_index; TupleConversionMap *tupconv_map; - TupleConversionMap *saved_tcs_map = NULL; bool tuple_deleted; TupleTableSlot *epqslot = NULL; @@ -1163,37 +1158,25 @@ ExecCrossPartitionUpdate(ModifyTableState *mtstate, /* * resultRelInfo is one of the per-subplan resultRelInfos. So we should - * convert the tuple into root's tuple descriptor, since ExecInsert() - * starts the search from root. The tuple conversion map list is in the - * order of mtstate->resultRelInfo[], so to retrieve the one for this - * resultRel, we need to know the position of the resultRel in - * mtstate->resultRelInfo[]. + * convert the tuple into root's tuple descriptor if needed, since + * ExecInsert() starts the search from root. */ - map_index = resultRelInfo - mtstate->resultRelInfo; - Assert(map_index >= 0 && map_index < mtstate->mt_nplans); - tupconv_map = tupconv_map_for_subplan(mtstate, map_index); + tupconv_map = resultRelInfo->ri_ChildToRootMap; if (tupconv_map != NULL) slot = execute_attr_map_slot(tupconv_map->attrMap, slot, mtstate->mt_root_tuple_slot); - /* - * ExecInsert() may scribble on mtstate->mt_transition_capture, so save - * the currently active map. - */ - if (mtstate->mt_transition_capture) - saved_tcs_map = mtstate->mt_transition_capture->tcs_map; - /* Tuple routing starts from the root table. */ *inserted_tuple = ExecInsert(mtstate, mtstate->rootResultRelInfo, slot, planSlot, estate, canSetTag); - /* Clear the INSERT's tuple and restore the saved map. */ + /* + * Reset the transition state that may possibly have been written + * by INSERT. + */ if (mtstate->mt_transition_capture) - { mtstate->mt_transition_capture->tcs_original_insert_tuple = NULL; - mtstate->mt_transition_capture->tcs_map = saved_tcs_map; - } /* We're done moving. */ return true; @@ -1870,28 +1853,6 @@ ExecSetupTransitionCaptureState(ModifyTableState *mtstate, EState *estate) MakeTransitionCaptureState(targetRelInfo->ri_TrigDesc, RelationGetRelid(targetRelInfo->ri_RelationDesc), CMD_UPDATE); - - /* - * If we found that we need to collect transition tuples then we may also - * need tuple conversion maps for any children that have TupleDescs that - * aren't compatible with the tuplestores. (We can share these maps - * between the regular and ON CONFLICT cases.) - */ - if (mtstate->mt_transition_capture != NULL || - mtstate->mt_oc_transition_capture != NULL) - { - ExecSetupChildParentMapForSubplan(mtstate); - - /* - * Install the conversion map for the first plan for UPDATE and DELETE - * operations. It will be advanced each time we switch to the next - * plan. (INSERT operations set it every time, so we need not update - * mtstate->mt_oc_transition_capture here.) - */ - if (mtstate->mt_transition_capture && mtstate->operation != CMD_INSERT) - mtstate->mt_transition_capture->tcs_map = - tupconv_map_for_subplan(mtstate, 0); - } } /* @@ -1929,35 +1890,20 @@ ExecPrepareTupleRouting(ModifyTableState *mtstate, /* * If we're capturing transition tuples, we might need to convert from the - * partition rowtype to root partitioned table's rowtype. + * partition rowtype to root partitioned table's rowtype. But if there + * are no BEFORE triggers on the partition that could change the tuple, we + * can just remember the original unconverted tuple to avoid a needless + * round trip conversion. */ if (mtstate->mt_transition_capture != NULL) { - if (partrel->ri_TrigDesc && - partrel->ri_TrigDesc->trig_insert_before_row) - { - /* - * If there are any BEFORE triggers on the partition, we'll have - * to be ready to convert their result back to tuplestore format. - */ - mtstate->mt_transition_capture->tcs_original_insert_tuple = NULL; - mtstate->mt_transition_capture->tcs_map = - partrouteinfo->pi_PartitionToRootMap; - } - else - { - /* - * Otherwise, just remember the original unconverted tuple, to - * avoid a needless round trip conversion. - */ - mtstate->mt_transition_capture->tcs_original_insert_tuple = slot; - mtstate->mt_transition_capture->tcs_map = NULL; - } - } - if (mtstate->mt_oc_transition_capture != NULL) - { - mtstate->mt_oc_transition_capture->tcs_map = - partrouteinfo->pi_PartitionToRootMap; + bool has_before_insert_row_trig; + + has_before_insert_row_trig = (partrel->ri_TrigDesc && + partrel->ri_TrigDesc->trig_insert_before_row); + + mtstate->mt_transition_capture->tcs_original_insert_tuple = + !has_before_insert_row_trig ? slot : NULL; } /* @@ -1975,58 +1921,6 @@ ExecPrepareTupleRouting(ModifyTableState *mtstate, return slot; } -/* - * Initialize the child-to-root tuple conversion map array for UPDATE subplans. - * - * This map array is required to convert the tuple from the subplan result rel - * to the target table descriptor. This requirement arises for two independent - * scenarios: - * 1. For update-tuple-routing. - * 2. For capturing tuples in transition tables. - */ -static void -ExecSetupChildParentMapForSubplan(ModifyTableState *mtstate) -{ - ResultRelInfo *targetRelInfo = mtstate->rootResultRelInfo; - ResultRelInfo *resultRelInfos = mtstate->resultRelInfo; - TupleDesc outdesc; - int numResultRelInfos = mtstate->mt_nplans; - int i; - - /* - * Build array of conversion maps from each child's TupleDesc to the one - * used in the target relation. The map pointers may be NULL when no - * conversion is necessary, which is hopefully a common case. - */ - - /* Get tuple descriptor of the target rel. */ - outdesc = RelationGetDescr(targetRelInfo->ri_RelationDesc); - - mtstate->mt_per_subplan_tupconv_maps = (TupleConversionMap **) - palloc(sizeof(TupleConversionMap *) * numResultRelInfos); - - for (i = 0; i < numResultRelInfos; ++i) - { - mtstate->mt_per_subplan_tupconv_maps[i] = - convert_tuples_by_name(RelationGetDescr(resultRelInfos[i].ri_RelationDesc), - outdesc); - } -} - -/* - * For a given subplan index, get the tuple conversion map. - */ -static TupleConversionMap * -tupconv_map_for_subplan(ModifyTableState *mtstate, int whichplan) -{ - /* If nobody else set the per-subplan array of maps, do so ourselves. */ - if (mtstate->mt_per_subplan_tupconv_maps == NULL) - ExecSetupChildParentMapForSubplan(mtstate); - - Assert(whichplan >= 0 && whichplan < mtstate->mt_nplans); - return mtstate->mt_per_subplan_tupconv_maps[whichplan]; -} - /* ---------------------------------------------------------------- * ExecModifyTable * @@ -2122,17 +2016,6 @@ ExecModifyTable(PlanState *pstate) junkfilter = resultRelInfo->ri_junkFilter; EvalPlanQualSetPlan(&node->mt_epqstate, subplanstate->plan, node->mt_arowmarks[node->mt_whichplan]); - /* Prepare to convert transition tuples from this child. */ - if (node->mt_transition_capture != NULL) - { - node->mt_transition_capture->tcs_map = - tupconv_map_for_subplan(node, node->mt_whichplan); - } - if (node->mt_oc_transition_capture != NULL) - { - node->mt_oc_transition_capture->tcs_map = - tupconv_map_for_subplan(node, node->mt_whichplan); - } continue; } else @@ -2334,8 +2217,7 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) * If it's a partitioned table, the root partition doesn't appear * elsewhere in the plan and its RT index is given explicitly in * node->rootRelation. Otherwise (i.e. table inheritance) the target - * relation is the first relation in the node->resultRelations list, and - * we will initialize it in the loop below. + * relation is the first relation in the node->resultRelations list. *---------- */ if (node->rootRelation > 0) @@ -2347,6 +2229,8 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) else { mtstate->rootResultRelInfo = mtstate->resultRelInfo; + ExecInitResultRelation(estate, mtstate->resultRelInfo, + linitial_int(node->resultRelations)); } mtstate->mt_arowmarks = (List **) palloc0(sizeof(List *) * nplans); @@ -2356,6 +2240,13 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) EvalPlanQualInit(&mtstate->mt_epqstate, estate, NULL, NIL, node->epqParam); mtstate->fireBSTriggers = true; + /* + * Build state for collecting transition tuples. This requires having a + * valid trigger query context, so skip it in explain-only mode. + */ + if (!(eflags & EXEC_FLAG_EXPLAIN_ONLY)) + ExecSetupTransitionCaptureState(mtstate, estate); + /* * call ExecInitNode on each of the plans to be executed and save the * results into the array "mt_plans". This is also a convenient place to @@ -2370,8 +2261,12 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) subplan = (Plan *) lfirst(l1); - /* This opens the relation and fills ResultRelInfo. */ - ExecInitResultRelation(estate, resultRelInfo, resultRelation); + /* + * This opens result relation and fills ResultRelInfo. (root relation + * was initialized already.) + */ + if (resultRelInfo != mtstate->rootResultRelInfo) + ExecInitResultRelation(estate, resultRelInfo, resultRelation); /* Initialize the usesFdwDirectModify flag */ resultRelInfo->ri_usesFdwDirectModify = bms_is_member(i, @@ -2427,6 +2322,23 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) eflags); } + /* + * If needed, initialize a map to convert tuples in the child format + * to the format of the table mentioned in the query (root relation). + * It's needed for update tuple routing, because the routing starts + * from the root relation. It's also needed for capturing transition + * tuples, because the transition tuple store can only store tuples + * in the root table format. + * + * For INSERT, the map is only initialized for a given partition when + * the partition itself is first initialized by ExecFindPartition(). + */ + if (update_tuple_routing_needed || + (mtstate->mt_transition_capture && + mtstate->operation != CMD_INSERT)) + resultRelInfo->ri_ChildToRootMap = + convert_tuples_by_name(RelationGetDescr(resultRelInfo->ri_RelationDesc), + RelationGetDescr(mtstate->rootResultRelInfo->ri_RelationDesc)); resultRelInfo++; i++; } @@ -2451,26 +2363,12 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) ExecSetupPartitionTupleRouting(estate, mtstate, rel); /* - * Build state for collecting transition tuples. This requires having a - * valid trigger query context, so skip it in explain-only mode. - */ - if (!(eflags & EXEC_FLAG_EXPLAIN_ONLY)) - ExecSetupTransitionCaptureState(mtstate, estate); - - /* - * Construct mapping from each of the per-subplan partition attnos to the - * root attno. This is required when during update row movement the tuple - * descriptor of a source partition does not match the root partitioned - * table descriptor. In such a case we need to convert tuples to the root - * tuple descriptor, because the search for destination partition starts - * from the root. We'll also need a slot to store these converted tuples. - * We can skip this setup if it's not a partition key update. + * For update row movement we'll need a dedicated slot to store the + * tuples that have been converted from partition format to the root + * table format. */ if (update_tuple_routing_needed) - { - ExecSetupChildParentMapForSubplan(mtstate); mtstate->mt_root_tuple_slot = table_slot_create(rel, NULL); - } /* * Initialize any WITH CHECK OPTION constraints if needed. diff --git a/src/include/commands/trigger.h b/src/include/commands/trigger.h index a40ddf5db52c..e38d732ed477 100644 --- a/src/include/commands/trigger.h +++ b/src/include/commands/trigger.h @@ -46,7 +46,7 @@ typedef struct TriggerData * The state for capturing old and new tuples into transition tables for a * single ModifyTable node (or other operation source, e.g. copy.c). * - * This is per-caller to avoid conflicts in setting tcs_map or + * This is per-caller to avoid conflicts in setting * tcs_original_insert_tuple. Note, however, that the pointed-to * private data may be shared across multiple callers. */ @@ -65,14 +65,6 @@ typedef struct TransitionCaptureState bool tcs_update_new_table; bool tcs_insert_new_table; - /* - * For UPDATE and DELETE, AfterTriggerSaveEvent may need to convert the - * new and old tuples from a child table's format to the format of the - * relation named in a query so that it is compatible with the transition - * tuplestores. The caller must store the conversion map here if so. - */ - TupleConversionMap *tcs_map; - /* * For INSERT and COPY, it would be wasteful to convert tuples from child * format to parent format after they have already been converted in the diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index 6d1b72219872..74c39911b2a2 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -36,12 +36,6 @@ typedef struct PartitionRoutingInfo */ TupleConversionMap *pi_RootToPartitionMap; - /* - * Map for converting tuples in partition format into the root partitioned - * table format, or NULL if no conversion is required. - */ - TupleConversionMap *pi_PartitionToRootMap; - /* * Slot to store tuples in partition format, or NULL when no translation * is required between root and partition. diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index dff34fbc14e2..46789cb00703 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -486,6 +486,13 @@ typedef struct ResultRelInfo /* info for partition tuple routing (NULL if not set up yet) */ struct PartitionRoutingInfo *ri_PartitionInfo; + /* + * Map to convert child result relation tuples to the format of the table + * actually mentioned in the query (called "root"). Set only if + * transition tuple capture or update partition row movement is active. + */ + TupleConversionMap *ri_ChildToRootMap; + /* for use by copy.c when performing multi-inserts */ struct CopyMultiInsertBuffer *ri_CopyMultiInsertBuffer; } ResultRelInfo; @@ -1179,9 +1186,6 @@ typedef struct ModifyTableState /* controls transition table population for INSERT...ON CONFLICT UPDATE */ struct TransitionCaptureState *mt_oc_transition_capture; - - /* Per plan map for tuple conversion from child to root */ - TupleConversionMap **mt_per_subplan_tupconv_maps; } ModifyTableState; /* ---------------- From fb5883da86154c3126264bfd97b0cd6f293bcebd Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 19 Oct 2020 14:11:57 +0300 Subject: [PATCH 334/589] Remove PartitionRoutingInfo struct. The extra indirection neeeded to access its members via its enclosing ResultRelInfo seems pointless. Move all the fields from PartitionRoutingInfo to ResultRelInfo. Author: Amit Langote Reviewed-by: Alvaro Herrera Discussion: https://www.postgresql.org/message-id/CA%2BHiwqFViT47Zbr_ASBejiK7iDG8%3DQ1swQ-tjM6caRPQ67pT%3Dw%40mail.gmail.com --- src/backend/commands/copy.c | 4 +-- src/backend/executor/execMain.c | 4 ++- src/backend/executor/execPartition.c | 34 ++++++++++++------------ src/backend/executor/nodeModifyTable.c | 21 +++++++-------- src/backend/replication/logical/worker.c | 11 +++----- src/include/executor/execPartition.h | 21 --------------- src/include/nodes/execnodes.h | 15 +++++++---- 7 files changed, 45 insertions(+), 65 deletions(-) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index b4ccd35ec180..36ddcdccdb83 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -3119,7 +3119,7 @@ CopyFrom(CopyState cstate) * We might need to convert from the root rowtype to the partition * rowtype. */ - map = resultRelInfo->ri_PartitionInfo->pi_RootToPartitionMap; + map = resultRelInfo->ri_RootToPartitionMap; if (insertMethod == CIM_SINGLE || !leafpart_use_multi_insert) { /* non batch insert */ @@ -3127,7 +3127,7 @@ CopyFrom(CopyState cstate) { TupleTableSlot *new_slot; - new_slot = resultRelInfo->ri_PartitionInfo->pi_PartitionTupleSlot; + new_slot = resultRelInfo->ri_PartitionTupleSlot; myslot = execute_attr_map_slot(map->attrMap, myslot, new_slot); } } diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index fcdd4b356741..aea04794487f 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -1243,7 +1243,9 @@ InitResultRelInfo(ResultRelInfo *resultRelInfo, resultRelInfo->ri_TrigOldSlot = NULL; resultRelInfo->ri_TrigNewSlot = NULL; resultRelInfo->ri_PartitionRoot = partition_root; - resultRelInfo->ri_PartitionInfo = NULL; /* may be set later */ + resultRelInfo->ri_RootToPartitionMap = NULL; /* set by + * ExecInitRoutingInfo */ + resultRelInfo->ri_PartitionTupleSlot = NULL; /* ditto */ resultRelInfo->ri_ChildToRootMap = NULL; resultRelInfo->ri_CopyMultiInsertBuffer = NULL; } diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 08f91e59a7ac..86594bd0565b 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -261,7 +261,7 @@ ExecSetupPartitionTupleRouting(EState *estate, ModifyTableState *mtstate, * If the partition's ResultRelInfo does not yet exist in 'proute' then we set * one up or reuse one from mtstate's resultRelInfo array. When reusing a * ResultRelInfo from the mtstate we verify that the relation is a valid - * target for INSERTs and then set up a PartitionRoutingInfo for it. + * target for INSERTs and initialize tuple routing information. * * rootResultRelInfo is the relation named in the query. * @@ -307,6 +307,7 @@ ExecFindPartition(ModifyTableState *mtstate, while (dispatch != NULL) { int partidx = -1; + bool is_leaf; CHECK_FOR_INTERRUPTS(); @@ -346,8 +347,10 @@ ExecFindPartition(ModifyTableState *mtstate, errtable(rel))); } - if (partdesc->is_leaf[partidx]) + is_leaf = partdesc->is_leaf[partidx]; + if (is_leaf) { + /* * We've reached the leaf -- hurray, we're done. Look to see if * we've already got a ResultRelInfo for this partition. @@ -382,7 +385,10 @@ ExecFindPartition(ModifyTableState *mtstate, /* Verify this ResultRelInfo allows INSERTs */ CheckValidResultRel(rri, CMD_INSERT); - /* Set up the PartitionRoutingInfo for it */ + /* + * Initialize information needed to insert this and + * subsequent tuples routed to this partition. + */ ExecInitRoutingInfo(mtstate, estate, proute, dispatch, rri, partidx); } @@ -464,8 +470,6 @@ ExecFindPartition(ModifyTableState *mtstate, */ if (partidx == partdesc->boundinfo->default_index) { - PartitionRoutingInfo *partrouteinfo = rri->ri_PartitionInfo; - /* * The tuple must match the partition's layout for the constraint * expression to be evaluated successfully. If the partition is @@ -478,13 +482,13 @@ ExecFindPartition(ModifyTableState *mtstate, * So if we have to convert, do it from the root slot; if not, use * the root slot as-is. */ - if (partrouteinfo) + if (is_leaf) { - TupleConversionMap *map = partrouteinfo->pi_RootToPartitionMap; + TupleConversionMap *map = rri->ri_RootToPartitionMap; if (map) slot = execute_attr_map_slot(map->attrMap, rootslot, - partrouteinfo->pi_PartitionTupleSlot); + rri->ri_PartitionTupleSlot); else slot = rootslot; } @@ -788,7 +792,7 @@ ExecInitPartitionInfo(ModifyTableState *mtstate, EState *estate, { TupleConversionMap *map; - map = leaf_part_rri->ri_PartitionInfo->pi_RootToPartitionMap; + map = leaf_part_rri->ri_RootToPartitionMap; Assert(node->onConflictSet != NIL); Assert(rootResultRelInfo->ri_onConflict != NULL); @@ -949,18 +953,15 @@ ExecInitRoutingInfo(ModifyTableState *mtstate, int partidx) { MemoryContext oldcxt; - PartitionRoutingInfo *partrouteinfo; int rri_index; oldcxt = MemoryContextSwitchTo(proute->memcxt); - partrouteinfo = palloc(sizeof(PartitionRoutingInfo)); - /* * Set up a tuple conversion map to convert a tuple routed to the * partition from the parent's type to the partition's. */ - partrouteinfo->pi_RootToPartitionMap = + partRelInfo->ri_RootToPartitionMap = convert_tuples_by_name(RelationGetDescr(partRelInfo->ri_PartitionRoot), RelationGetDescr(partRelInfo->ri_RelationDesc)); @@ -970,7 +971,7 @@ ExecInitRoutingInfo(ModifyTableState *mtstate, * for various operations that are applied to tuples after routing, such * as checking constraints. */ - if (partrouteinfo->pi_RootToPartitionMap != NULL) + if (partRelInfo->ri_RootToPartitionMap != NULL) { Relation partrel = partRelInfo->ri_RelationDesc; @@ -979,11 +980,11 @@ ExecInitRoutingInfo(ModifyTableState *mtstate, * partition's TupleDesc; TupleDesc reference will be released at the * end of the command. */ - partrouteinfo->pi_PartitionTupleSlot = + partRelInfo->ri_PartitionTupleSlot = table_slot_create(partrel, &estate->es_tupleTable); } else - partrouteinfo->pi_PartitionTupleSlot = NULL; + partRelInfo->ri_PartitionTupleSlot = NULL; /* * If the partition is a foreign table, let the FDW init itself for @@ -993,7 +994,6 @@ ExecInitRoutingInfo(ModifyTableState *mtstate, partRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL) partRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate, partRelInfo); - partRelInfo->ri_PartitionInfo = partrouteinfo; partRelInfo->ri_CopyMultiInsertBuffer = NULL; /* diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index 5f85fd7cd88f..a33423c896e3 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -1172,8 +1172,8 @@ ExecCrossPartitionUpdate(ModifyTableState *mtstate, planSlot, estate, canSetTag); /* - * Reset the transition state that may possibly have been written - * by INSERT. + * Reset the transition state that may possibly have been written by + * INSERT. */ if (mtstate->mt_transition_capture) mtstate->mt_transition_capture->tcs_original_insert_tuple = NULL; @@ -1874,7 +1874,6 @@ ExecPrepareTupleRouting(ModifyTableState *mtstate, ResultRelInfo **partRelInfo) { ResultRelInfo *partrel; - PartitionRoutingInfo *partrouteinfo; TupleConversionMap *map; /* @@ -1885,8 +1884,6 @@ ExecPrepareTupleRouting(ModifyTableState *mtstate, * UPDATE to another partition becomes a DELETE+INSERT. */ partrel = ExecFindPartition(mtstate, targetRelInfo, proute, slot, estate); - partrouteinfo = partrel->ri_PartitionInfo; - Assert(partrouteinfo != NULL); /* * If we're capturing transition tuples, we might need to convert from the @@ -1909,10 +1906,10 @@ ExecPrepareTupleRouting(ModifyTableState *mtstate, /* * Convert the tuple, if necessary. */ - map = partrouteinfo->pi_RootToPartitionMap; + map = partrel->ri_RootToPartitionMap; if (map != NULL) { - TupleTableSlot *new_slot = partrouteinfo->pi_PartitionTupleSlot; + TupleTableSlot *new_slot = partrel->ri_PartitionTupleSlot; slot = execute_attr_map_slot(map->attrMap, slot, new_slot); } @@ -2327,8 +2324,8 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) * to the format of the table mentioned in the query (root relation). * It's needed for update tuple routing, because the routing starts * from the root relation. It's also needed for capturing transition - * tuples, because the transition tuple store can only store tuples - * in the root table format. + * tuples, because the transition tuple store can only store tuples in + * the root table format. * * For INSERT, the map is only initialized for a given partition when * the partition itself is first initialized by ExecFindPartition(). @@ -2363,9 +2360,9 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) ExecSetupPartitionTupleRouting(estate, mtstate, rel); /* - * For update row movement we'll need a dedicated slot to store the - * tuples that have been converted from partition format to the root - * table format. + * For update row movement we'll need a dedicated slot to store the tuples + * that have been converted from partition format to the root table + * format. */ if (update_tuple_routing_needed) mtstate->mt_root_tuple_slot = table_slot_create(rel, NULL); diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index b8e297c5d34e..3a5b733ee38c 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -1572,7 +1572,6 @@ apply_handle_tuple_routing(ResultRelInfo *relinfo, ResultRelInfo *partrelinfo; Relation partrel; TupleTableSlot *remoteslot_part; - PartitionRoutingInfo *partinfo; TupleConversionMap *map; MemoryContext oldctx; @@ -1599,11 +1598,10 @@ apply_handle_tuple_routing(ResultRelInfo *relinfo, * partition's rowtype. Convert if needed or just copy, using a dedicated * slot to store the tuple in any case. */ - partinfo = partrelinfo->ri_PartitionInfo; - remoteslot_part = partinfo->pi_PartitionTupleSlot; + remoteslot_part = partrelinfo->ri_PartitionTupleSlot; if (remoteslot_part == NULL) remoteslot_part = table_slot_create(partrel, &estate->es_tupleTable); - map = partinfo->pi_RootToPartitionMap; + map = partrelinfo->ri_RootToPartitionMap; if (map != NULL) remoteslot_part = execute_attr_map_slot(map->attrMap, remoteslot, remoteslot_part); @@ -1748,12 +1746,11 @@ apply_handle_tuple_routing(ResultRelInfo *relinfo, */ oldctx = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate)); partrel = partrelinfo_new->ri_RelationDesc; - partinfo = partrelinfo_new->ri_PartitionInfo; - remoteslot_part = partinfo->pi_PartitionTupleSlot; + remoteslot_part = partrelinfo_new->ri_PartitionTupleSlot; if (remoteslot_part == NULL) remoteslot_part = table_slot_create(partrel, &estate->es_tupleTable); - map = partinfo->pi_RootToPartitionMap; + map = partrelinfo_new->ri_RootToPartitionMap; if (map != NULL) { remoteslot_part = execute_attr_map_slot(map->attrMap, diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index 74c39911b2a2..473c4cd84fc6 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -22,27 +22,6 @@ typedef struct PartitionDispatchData *PartitionDispatch; typedef struct PartitionTupleRouting PartitionTupleRouting; -/* - * PartitionRoutingInfo - * - * Additional result relation information specific to routing tuples to a - * table partition. - */ -typedef struct PartitionRoutingInfo -{ - /* - * Map for converting tuples in root partitioned table format into - * partition format, or NULL if no conversion is required. - */ - TupleConversionMap *pi_RootToPartitionMap; - - /* - * Slot to store tuples in partition format, or NULL when no translation - * is required between root and partition. - */ - TupleTableSlot *pi_PartitionTupleSlot; -} PartitionRoutingInfo; - /* * PartitionedRelPruningData - Per-partitioned-table data for run-time pruning * of partitions. For a multilevel partitioned table, we have one of these diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 46789cb00703..6c0a7d68d615 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -33,7 +33,6 @@ #include "utils/tuplestore.h" struct PlanState; /* forward references in this file */ -struct PartitionRoutingInfo; struct ParallelHashJoinState; struct ExecRowMark; struct ExprState; @@ -480,11 +479,17 @@ typedef struct ResultRelInfo /* partition check expression state (NULL if not set up yet) */ ExprState *ri_PartitionCheckExpr; - /* relation descriptor for partitioned table's root, if any */ + /* + * Information needed by tuple routing target relations + * + * PartitionRoot gives the target relation mentioned in the query. + * RootToPartitionMap and PartitionTupleSlot, initialized by + * ExecInitRoutingInfo, are non-NULL if partition has a different tuple + * format than the root table. + */ Relation ri_PartitionRoot; - - /* info for partition tuple routing (NULL if not set up yet) */ - struct PartitionRoutingInfo *ri_PartitionInfo; + TupleConversionMap *ri_RootToPartitionMap; + TupleTableSlot *ri_PartitionTupleSlot; /* * Map to convert child result relation tuples to the format of the table From b4d5b458e6fb4c861f92888753deea6a5005a68d Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Mon, 19 Oct 2020 13:47:09 +0200 Subject: [PATCH 335/589] Update link for pllua Author: Daniel Gustafsson Discussion: https://postgr.es/m/A05874AE-8771-4C61-A24E-0B6249B8F3C2@yesql.se --- doc/src/sgml/external-projects.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/external-projects.sgml b/doc/src/sgml/external-projects.sgml index 4627adc18fcb..bf590aba5d9d 100644 --- a/doc/src/sgml/external-projects.sgml +++ b/doc/src/sgml/external-projects.sgml @@ -199,7 +199,7 @@ PL/Lua Lua - + From 1a64c7636f24096155f7aa15d7164eba3cdef075 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 19 Oct 2020 17:58:38 +0300 Subject: [PATCH 336/589] Fix doc for full text search distance operator. Commit 028350f619 changed its behavior from "at most" to "exactly", but forgot to update the documentation. Backpatch to 9.6. Patch by Justin Pryzby, per Yaroslav Schekin's report. Discussion: https://www.postgresql.org/message-id/20201005191922.GE17626%40telsasoft.com --- doc/src/sgml/textsearch.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/textsearch.sgml b/doc/src/sgml/textsearch.sgml index 2ebdf02bfaf0..8af6ac01d302 100644 --- a/doc/src/sgml/textsearch.sgml +++ b/doc/src/sgml/textsearch.sgml @@ -1645,7 +1645,7 @@ SELECT to_tsquery('fat') <-> to_tsquery('cat | rat'); Returns a query that searches for a match to the first given query - followed by a match to the second given query at a distance of at + followed by a match to the second given query at a distance of exactly distance lexemes, using the <N> tsquery operator. For example: From c0bc4c682ee2a84dee2b562db3d17d173a6abc06 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 19 Oct 2020 18:50:33 +0300 Subject: [PATCH 337/589] Fix output of tsquery example in docs. The output for this query changed in commit 4e2477b7b8. Backport to 9.6 like that commit. Patch by Justin Pryzby, per Yaroslav Schekin's report. Discussion: https://www.postgresql.org/message-id/20201005191922.GE17626%40telsasoft.com --- doc/src/sgml/textsearch.sgml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/src/sgml/textsearch.sgml b/doc/src/sgml/textsearch.sgml index 8af6ac01d302..7daf292468a5 100644 --- a/doc/src/sgml/textsearch.sgml +++ b/doc/src/sgml/textsearch.sgml @@ -1623,9 +1623,9 @@ occurrences to display in the result.', SELECT to_tsquery('fat') <-> to_tsquery('cat | rat'); - ?column? ------------------------------------ - 'fat' <-> 'cat' | 'fat' <-> 'rat' + ?column? +---------------------------- + 'fat' <-> ( 'cat' | 'rat' ) From 2a972e0165c9b34da4cff3001a2dad8e5a9a5a2b Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 19 Oct 2020 19:02:25 +0300 Subject: [PATCH 338/589] Fix TRUNCATE doc: ALTER SEQUENCE RESTART is now transactional. ALTER SEQUENCE RESTART was made transactional in commit 3d79013b97. Backpatch to v10, where that was introduced. Patch by Justin Pryzby, per Yaroslav Schekin's report. Discussion: https://www.postgresql.org/message-id/20201005191922.GE17626%40telsasoft.com --- doc/src/sgml/ref/truncate.sgml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/src/sgml/ref/truncate.sgml b/doc/src/sgml/ref/truncate.sgml index 5922ee579e11..91cdac556233 100644 --- a/doc/src/sgml/ref/truncate.sgml +++ b/doc/src/sgml/ref/truncate.sgml @@ -160,8 +160,7 @@ TRUNCATE [ TABLE ] [ ONLY ] name [ When RESTART IDENTITY is specified, the implied ALTER SEQUENCE RESTART operations are also done transactionally; that is, they will be rolled back if the surrounding - transaction does not commit. This is unlike the normal behavior of - ALTER SEQUENCE RESTART. Be aware that if any additional + transaction does not commit. Be aware that if any additional sequence operations are done on the restarted sequences before the transaction rolls back, the effects of these operations on the sequences will be rolled back, but not their effects on currval(); From c5f42daa6077a4c309c5280a47d0e114c12dc572 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 19 Oct 2020 19:28:54 +0300 Subject: [PATCH 339/589] Misc documentation fixes. - Misc grammar and punctuation fixes. - Stylistic cleanup: use spaces between function arguments and JSON fields in examples. For example "foo(a,b)" -> "foo(a, b)". Add semicolon after last END in a few PL/pgSQL examples that were missing them. - Make sentence that talked about "..." and ".." operators more clear, by avoiding to end the sentence with "..". That makes it look the same as "..." - Fix syntax description for HAVING: HAVING conditions cannot be repeated Patch by Justin Pryzby, per Yaroslav Schekin's report. Backpatch to all supported versions, to the extent that the patch applies easily. Discussion: https://www.postgresql.org/message-id/20201005191922.GE17626%40telsasoft.com --- doc/src/sgml/catalogs.sgml | 2 +- doc/src/sgml/config.sgml | 4 ++-- doc/src/sgml/dblink.sgml | 8 ++++---- doc/src/sgml/func.sgml | 32 ++++++++++++++--------------- doc/src/sgml/gin.sgml | 2 +- doc/src/sgml/high-availability.sgml | 2 +- doc/src/sgml/hstore.sgml | 10 ++++----- doc/src/sgml/indexam.sgml | 2 +- doc/src/sgml/isn.sgml | 2 +- doc/src/sgml/ltree.sgml | 12 +++++------ doc/src/sgml/mvcc.sgml | 2 +- doc/src/sgml/parallel.sgml | 2 +- doc/src/sgml/plpgsql.sgml | 12 +++++------ doc/src/sgml/protocol.sgml | 2 +- doc/src/sgml/ref/select.sgml | 2 +- doc/src/sgml/ref/select_into.sgml | 2 +- doc/src/sgml/rules.sgml | 6 +++--- doc/src/sgml/seg.sgml | 4 ++-- doc/src/sgml/textsearch.sgml | 30 +++++++++++++-------------- 19 files changed, 69 insertions(+), 69 deletions(-) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 3927b1030df4..5bd54cb21832 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -1525,7 +1525,7 @@ Role can log in. That is, this role can be given as the initial - session authorization identifier + session authorization identifier. diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 1ef880cda59c..f043433e3185 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -10168,8 +10168,8 @@ LOG: CleanUpLock: deleting: lock(0xb7acd844) id(24688,24696,0,0,0,1) - If set, do not trace locks for tables below this OID. (use to avoid - output on system tables) + If set, do not trace locks for tables below this OID (used to avoid + output on system tables). This parameter is only available if the LOCK_DEBUG diff --git a/doc/src/sgml/dblink.sgml b/doc/src/sgml/dblink.sgml index eba7fcfb989f..bcf623117c4b 100644 --- a/doc/src/sgml/dblink.sgml +++ b/doc/src/sgml/dblink.sgml @@ -166,7 +166,7 @@ SELECT dblink_connect('myconn', 'fdtest'); OK (1 row) -SELECT * FROM dblink('myconn','SELECT * FROM foo') AS t(a int, b text, c text[]); +SELECT * FROM dblink('myconn', 'SELECT * FROM foo') AS t(a int, b text, c text[]); a | b | c ----+---+--------------- 0 | a | {a0,b0,c0} @@ -615,7 +615,7 @@ dblink_exec(text sql [, bool fail_on_error]) returns text The SQL command that you wish to execute in the remote database, for example - insert into foo values(0,'a','{"a0","b0","c0"}'). + insert into foo values(0, 'a', '{"a0","b0","c0"}'). @@ -652,7 +652,7 @@ SELECT dblink_connect('dbname=dblink_test_standby'); OK (1 row) -SELECT dblink_exec('insert into foo values(21,''z'',''{"a0","b0","c0"}'');'); +SELECT dblink_exec('insert into foo values(21, ''z'', ''{"a0","b0","c0"}'');'); dblink_exec ----------------- INSERT 943366 1 @@ -664,7 +664,7 @@ SELECT dblink_connect('myconn', 'dbname=regression'); OK (1 row) -SELECT dblink_exec('myconn', 'insert into foo values(21,''z'',''{"a0","b0","c0"}'');'); +SELECT dblink_exec('myconn', 'insert into foo values(21, ''z'', ''{"a0","b0","c0"}'');'); dblink_exec ------------------ INSERT 6432584 1 diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 2762a647edce..c99499e52bdb 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -15034,7 +15034,7 @@ table2-mapping per to_json or to_jsonb. - json_build_array(1,2,'foo',4,5) + json_build_array(1, 2, 'foo', 4, 5) [1, 2, "foo", 4, 5] @@ -15061,7 +15061,7 @@ table2-mapping per to_json or to_jsonb. - json_build_object('foo',1,2,row(3,'bar')) + json_build_object('foo', 1, 2, row(3,'bar')) {"foo" : 1, "2" : {"f1":3,"f2":"bar"}} @@ -15093,7 +15093,7 @@ table2-mapping json_object('{a, 1, b, "def", c, 3.5}') {"a" : "1", "b" : "def", "c" : "3.5"} - json_object('{{a, 1},{b, "def"},{c, 3.5}}') + json_object('{{a, 1}, {b, "def"}, {c, 3.5}}') {"a" : "1", "b" : "def", "c" : "3.5"} @@ -15113,7 +15113,7 @@ table2-mapping the one-argument form. - json_object('{a, b}', '{1,2}') + json_object('{a,b}', '{1,2}') {"a": "1", "b": "2"} @@ -15504,7 +15504,7 @@ table2-mapping create type twoints as (a int, b int); - select * from json_populate_recordset(null::twoints, '[{"a":1,"b":2},{"a":3,"b":4}]') + select * from json_populate_recordset(null::twoints, '[{"a":1,"b":2}, {"a":3,"b":4}]') a | b @@ -15579,7 +15579,7 @@ table2-mapping for json[b]_populate_record. - select * from json_to_recordset('[{"a":1,"b":"foo"},{"a":"2","c":"bar"}]') as x(a int, b text) + select * from json_to_recordset('[{"a":1,"b":"foo"}, {"a":"2","c":"bar"}]') as x(a int, b text) a | b @@ -15617,11 +15617,11 @@ table2-mapping or at the end of the array if it is positive. - jsonb_set('[{"f1":1,"f2":null},2,null,3]', '{0,f1}','[2,3,4]', false) + jsonb_set('[{"f1":1,"f2":null},2,null,3]', '{0,f1}', '[2,3,4]', false) [{"f1": [2, 3, 4], "f2": null}, 2, null, 3] - jsonb_set('[{"f1":1,"f2":null},2]', '{0,f3}','[2,3,4]') + jsonb_set('[{"f1":1,"f2":null},2]', '{0,f3}', '[2,3,4]') [{"f1": 1, "f2": null, "f3": [2, 3, 4]}, 2] @@ -15713,7 +15713,7 @@ table2-mapping untouched. - json_strip_nulls('[{"f1":1,"f2":null},2,null,3]') + json_strip_nulls('[{"f1":1, "f2":null}, 2, null, 3]') [{"f1":1},2,null,3] @@ -15737,7 +15737,7 @@ table2-mapping as the @? and @@ operators do. - jsonb_path_exists('{"a":[1,2,3,4,5]}', '$.a[*] ? (@ >= $min && @ <= $max)', '{"min":2,"max":4}') + jsonb_path_exists('{"a":[1,2,3,4,5]}', '$.a[*] ? (@ >= $min && @ <= $max)', '{"min":2, "max":4}') t @@ -15759,7 +15759,7 @@ table2-mapping for jsonb_path_exists. - jsonb_path_match('{"a":[1,2,3,4,5]}', 'exists($.a[*] ? (@ >= $min && @ <= $max))', '{"min":2,"max":4}') + jsonb_path_match('{"a":[1,2,3,4,5]}', 'exists($.a[*] ? (@ >= $min && @ <= $max))', '{"min":2, "max":4}') t @@ -15780,7 +15780,7 @@ table2-mapping for jsonb_path_exists. - select * from jsonb_path_query('{"a":[1,2,3,4,5]}', '$.a[*] ? (@ >= $min && @ <= $max)', '{"min":2,"max":4}') + select * from jsonb_path_query('{"a":[1,2,3,4,5]}', '$.a[*] ? (@ >= $min && @ <= $max)', '{"min":2, "max":4}') jsonb_path_query @@ -15808,7 +15808,7 @@ table2-mapping for jsonb_path_exists. - jsonb_path_query_array('{"a":[1,2,3,4,5]}', '$.a[*] ? (@ >= $min && @ <= $max)', '{"min":2,"max":4}') + jsonb_path_query_array('{"a":[1,2,3,4,5]}', '$.a[*] ? (@ >= $min && @ <= $max)', '{"min":2, "max":4}') [2, 3, 4] @@ -15830,7 +15830,7 @@ table2-mapping for jsonb_path_exists. - jsonb_path_query_first('{"a":[1,2,3,4,5]}', '$.a[*] ? (@ >= $min && @ <= $max)', '{"min":2,"max":4}') + jsonb_path_query_first('{"a":[1,2,3,4,5]}', '$.a[*] ? (@ >= $min && @ <= $max)', '{"min":2, "max":4}') 2 @@ -15902,7 +15902,7 @@ table2-mapping Converts the given JSON value to pretty-printed, indented text. - jsonb_pretty('[{"f1":1,"f2":null},2]') + jsonb_pretty('[{"f1":1,"f2":null}, 2]') [ @@ -26599,7 +26599,7 @@ BEGIN obj.object_name, obj.object_identity; END LOOP; -END +END; $$; CREATE EVENT TRIGGER test_event_trigger_for_drops ON sql_drop diff --git a/doc/src/sgml/gin.sgml b/doc/src/sgml/gin.sgml index 5c8d4d52757c..67754f52f649 100644 --- a/doc/src/sgml/gin.sgml +++ b/doc/src/sgml/gin.sgml @@ -612,7 +612,7 @@ gin_pending_list_limit can be overridden for individual - GIN indexes by changing storage parameters, and which allows each + GIN indexes by changing storage parameters, which allows each GIN index to have its own cleanup threshold. For example, it's possible to increase the threshold only for the GIN index which can be updated heavily, and decrease it otherwise. diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index 86da84fce709..339ed38d42c8 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -1502,7 +1502,7 @@ synchronous_standby_names = 'ANY 2 (s1, s2, s3)' Note that in this mode, the server will apply WAL one file at a time, so if you use the standby server for queries (see Hot Standby), there is a delay between an action in the primary and when the - action becomes visible in the standby, corresponding the time it takes + action becomes visible in the standby, corresponding to the time it takes to fill up the WAL file. archive_timeout can be used to make that delay shorter. Also note that you can't combine streaming replication with this method. diff --git a/doc/src/sgml/hstore.sgml b/doc/src/sgml/hstore.sgml index 8a1caa357613..14a36ade00a0 100644 --- a/doc/src/sgml/hstore.sgml +++ b/doc/src/sgml/hstore.sgml @@ -631,7 +631,7 @@ b Does hstore contain key? - exist('a=>1','a') + exist('a=>1', 'a') t @@ -647,7 +647,7 @@ b for key? - defined('a=>NULL','a') + defined('a=>NULL', 'a') f @@ -662,7 +662,7 @@ b Deletes pair with matching key. - delete('a=>1,b=>2','b') + delete('a=>1,b=>2', 'b') "a"=>"1" @@ -676,7 +676,7 @@ b Deletes pairs with matching keys. - delete('a=>1,b=>2,c=>3',ARRAY['a','b']) + delete('a=>1,b=>2,c=>3', ARRAY['a','b']) "c"=>"3" @@ -690,7 +690,7 @@ b Deletes pairs matching those in the second argument. - delete('a=>1,b=>2','a=>4,b=>2'::hstore) + delete('a=>1,b=>2', 'a=>4,b=>2'::hstore) "a"=>"1" diff --git a/doc/src/sgml/indexam.sgml b/doc/src/sgml/indexam.sgml index 390c49eb6aba..649020b7daa2 100644 --- a/doc/src/sgml/indexam.sgml +++ b/doc/src/sgml/indexam.sgml @@ -612,7 +612,7 @@ amgettuple (IndexScanDesc scan, will pass the caller's snapshot test. On success, amgettuple must also set scan->xs_recheck to true or false. False means it is certain that the index entry matches the scan keys. - true means this is not certain, and the conditions represented by the + True means this is not certain, and the conditions represented by the scan keys must be rechecked against the heap tuple after fetching it. This provision supports lossy index operators. Note that rechecking will extend only to the scan conditions; a partial diff --git a/doc/src/sgml/isn.sgml b/doc/src/sgml/isn.sgml index e55ed073120c..709bc8345c7e 100644 --- a/doc/src/sgml/isn.sgml +++ b/doc/src/sgml/isn.sgml @@ -14,7 +14,7 @@ hard-coded list of prefixes; this list of prefixes is also used to hyphenate numbers on output. Since new prefixes are assigned from time to time, the list of prefixes may be out of date. It is hoped that a future version of - this module will obtained the prefix list from one or more tables that + this module will obtain the prefix list from one or more tables that can be easily updated by users as needed; however, at present, the list can only be updated by modifying the source code and recompiling. Alternatively, prefix validation and hyphenation support may be diff --git a/doc/src/sgml/ltree.sgml b/doc/src/sgml/ltree.sgml index 36aa2b5fad86..06a983c075be 100644 --- a/doc/src/sgml/ltree.sgml +++ b/doc/src/sgml/ltree.sgml @@ -460,7 +460,7 @@ Europe & Russia*@ & !Transportation position end-1 (counting from 0). - subltree('Top.Child1.Child2',1,2) + subltree('Top.Child1.Child2', 1, 2) Child1 @@ -480,7 +480,7 @@ Europe & Russia*@ & !Transportation the end of the path. - subpath('Top.Child1.Child2',0,2) + subpath('Top.Child1.Child2', 0, 2) Top.Child1 @@ -497,7 +497,7 @@ Europe & Russia*@ & !Transportation from the end of the path. - subpath('Top.Child1.Child2',1) + subpath('Top.Child1.Child2', 1) Child1.Child2 @@ -528,7 +528,7 @@ Europe & Russia*@ & !Transportation a, or -1 if not found. - index('0.1.2.3.5.4.5.6.8.5.6.8','5.6') + index('0.1.2.3.5.4.5.6.8.5.6.8', '5.6') 6 @@ -546,7 +546,7 @@ Europe & Russia*@ & !Transportation start -offset labels from the end of the path. - index('0.1.2.3.5.4.5.6.8.5.6.8','5.6',-4) + index('0.1.2.3.5.4.5.6.8.5.6.8', '5.6', -4) 9 @@ -584,7 +584,7 @@ Europe & Russia*@ & !Transportation (up to 8 arguments are supported). - lca('1.2.3','1.2.3.4.5.6') + lca('1.2.3', '1.2.3.4.5.6') 1.2 diff --git a/doc/src/sgml/mvcc.sgml b/doc/src/sgml/mvcc.sgml index 1d406176568d..71dd4033372d 100644 --- a/doc/src/sgml/mvcc.sgml +++ b/doc/src/sgml/mvcc.sgml @@ -1245,7 +1245,7 @@ ERROR: could not serialize access due to read/write dependencies among transact The FOR UPDATE lock mode is also acquired by any DELETE on a row, and also by an - UPDATE that modifies the values on certain columns. Currently, + UPDATE that modifies the values of certain columns. Currently, the set of columns considered for the UPDATE case are those that have a unique index on them that can be used in a foreign key (so partial indexes and expressional indexes are not considered), but this may change diff --git a/doc/src/sgml/parallel.sgml b/doc/src/sgml/parallel.sgml index e31bd9d3cebd..c81abff48d37 100644 --- a/doc/src/sgml/parallel.sgml +++ b/doc/src/sgml/parallel.sgml @@ -471,7 +471,7 @@ EXPLAIN SELECT * FROM pgbench_accounts WHERE filler LIKE '%x%'; - The following operations are always parallel restricted. + The following operations are always parallel restricted: diff --git a/doc/src/sgml/plpgsql.sgml b/doc/src/sgml/plpgsql.sgml index 74b6b2587809..94a3b36458df 100644 --- a/doc/src/sgml/plpgsql.sgml +++ b/doc/src/sgml/plpgsql.sgml @@ -1181,7 +1181,7 @@ BEGIN SELECT users.userid INTO STRICT userid FROM users WHERE users.username = get_userid.username; RETURN userid; -END +END; $$ LANGUAGE plpgsql; On failure, this function might produce an error message such as @@ -1854,7 +1854,7 @@ BEGIN RETURN NEXT r; -- return current row of SELECT END LOOP; RETURN; -END +END; $BODY$ LANGUAGE plpgsql; @@ -1882,7 +1882,7 @@ BEGIN END IF; RETURN; - END + END; $BODY$ LANGUAGE plpgsql; @@ -1956,7 +1956,7 @@ DECLARE myvar int := 5; BEGIN CALL triple(myvar); RAISE NOTICE 'myvar = %', myvar; -- prints 15 -END +END; $$; @@ -3559,7 +3559,7 @@ BEGIN ROLLBACK; END IF; END LOOP; -END +END; $$; CALL transaction_test1(); @@ -5213,7 +5213,7 @@ DECLARE f1 int; BEGIN RETURN f1; -END +END; $$ LANGUAGE plpgsql; WARNING: variable "f1" shadows a previously defined variable LINE 3: f1 int; diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 5e06c7523d8e..a46cb728b79d 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2839,7 +2839,7 @@ The commands accepted in replication mode are: Every sent transaction contains zero or more DML messages (Insert, Update, Delete). In case of a cascaded setup it can also contain Origin - messages. The origin message indicated that the transaction originated on + messages. The origin message indicates that the transaction originated on different replication node. Since a replication node in the scope of logical replication protocol can be pretty much anything, the only identifier is the origin name. It's downstream's responsibility to handle this as diff --git a/doc/src/sgml/ref/select.sgml b/doc/src/sgml/ref/select.sgml index b4dea9b6acf4..472b7cae812b 100644 --- a/doc/src/sgml/ref/select.sgml +++ b/doc/src/sgml/ref/select.sgml @@ -38,7 +38,7 @@ SELECT [ ALL | DISTINCT [ ON ( expressionfrom_item [, ...] ] [ WHERE condition ] [ GROUP BY grouping_element [, ...] ] - [ HAVING condition [, ...] ] + [ HAVING condition ] [ WINDOW window_name AS ( window_definition ) [, ...] ] [ { UNION | INTERSECT | EXCEPT } [ ALL | DISTINCT ] select ] [ ORDER BY expression [ ASC | DESC | USING operator ] [ NULLS { FIRST | LAST } ] [, ...] ] diff --git a/doc/src/sgml/ref/select_into.sgml b/doc/src/sgml/ref/select_into.sgml index 6cfa706b5752..7b327d9eeef3 100644 --- a/doc/src/sgml/ref/select_into.sgml +++ b/doc/src/sgml/ref/select_into.sgml @@ -28,7 +28,7 @@ SELECT [ ALL | DISTINCT [ ON ( expressionfrom_item [, ...] ] [ WHERE condition ] [ GROUP BY expression [, ...] ] - [ HAVING condition [, ...] ] + [ HAVING condition ] [ WINDOW window_name AS ( window_definition ) [, ...] ] [ { UNION | INTERSECT | EXCEPT } [ ALL | DISTINCT ] select ] [ ORDER BY expression [ ASC | DESC | USING operator ] [ NULLS { FIRST | LAST } ] [, ...] ] diff --git a/doc/src/sgml/rules.sgml b/doc/src/sgml/rules.sgml index bcf860b68b9c..e81addcfa9bb 100644 --- a/doc/src/sgml/rules.sgml +++ b/doc/src/sgml/rules.sgml @@ -769,7 +769,7 @@ SELECT t1.a, t2.b, t1.ctid FROM t1, t2 WHERE t1.a = t2.a; - The benefit of implementing views with the rule system is, + The benefit of implementing views with the rule system is that the planner has all the information about which tables have to be scanned plus the relationships between these tables plus the restrictive @@ -781,7 +781,7 @@ SELECT t1.a, t2.b, t1.ctid FROM t1, t2 WHERE t1.a = t2.a; the best path to execute the query, and the more information the planner has, the better this decision can be. And the rule system as implemented in PostgreSQL - ensures, that this is all information available about the query + ensures that this is all information available about the query up to that point. @@ -2087,7 +2087,7 @@ CREATE FUNCTION tricky(text, text) RETURNS bool AS $$ BEGIN RAISE NOTICE '% => %', $1, $2; RETURN true; -END +END; $$ LANGUAGE plpgsql COST 0.0000000000000000000001; SELECT * FROM phone_number WHERE tricky(person, phone); diff --git a/doc/src/sgml/seg.sgml b/doc/src/sgml/seg.sgml index e86142d885a5..e0dfbc76cf35 100644 --- a/doc/src/sgml/seg.sgml +++ b/doc/src/sgml/seg.sgml @@ -205,8 +205,8 @@ test=> select '6.25 .. 6.50'::seg as "pH";
- Because ... is widely used in data sources, it is allowed - as an alternative spelling of ... Unfortunately, this + Because the ... operator is widely used in data sources, it is allowed + as an alternative spelling of the .. operator. Unfortunately, this creates a parsing ambiguity: it is not clear whether the upper bound in 0...23 is meant to be 23 or 0.23. This is resolved by requiring at least one digit before the decimal diff --git a/doc/src/sgml/textsearch.sgml b/doc/src/sgml/textsearch.sgml index 7daf292468a5..7bd8d53dc4e0 100644 --- a/doc/src/sgml/textsearch.sgml +++ b/doc/src/sgml/textsearch.sgml @@ -2419,7 +2419,7 @@ ALTER TEXT SEARCH CONFIGURATION astro_en positions in tsvector, which in turn affect ranking: -SELECT to_tsvector('english','in the list of stop words'); +SELECT to_tsvector('english', 'in the list of stop words'); to_tsvector ---------------------------- 'list':3 'stop':5 'word':6 @@ -2429,12 +2429,12 @@ SELECT to_tsvector('english','in the list of stop words'); calculated for documents with and without stop words are quite different: -SELECT ts_rank_cd (to_tsvector('english','in the list of stop words'), to_tsquery('list & stop')); +SELECT ts_rank_cd (to_tsvector('english', 'in the list of stop words'), to_tsquery('list & stop')); ts_rank_cd ------------ 0.05 -SELECT ts_rank_cd (to_tsvector('english','list stop words'), to_tsquery('list & stop')); +SELECT ts_rank_cd (to_tsvector('english', 'list stop words'), to_tsquery('list & stop')); ts_rank_cd ------------ 0.1 @@ -2493,12 +2493,12 @@ CREATE TEXT SEARCH DICTIONARY public.simple_dict ( Now we can test our dictionary: -SELECT ts_lexize('public.simple_dict','YeS'); +SELECT ts_lexize('public.simple_dict', 'YeS'); ts_lexize ----------- {yes} -SELECT ts_lexize('public.simple_dict','The'); +SELECT ts_lexize('public.simple_dict', 'The'); ts_lexize ----------- {} @@ -2514,12 +2514,12 @@ SELECT ts_lexize('public.simple_dict','The'); ALTER TEXT SEARCH DICTIONARY public.simple_dict ( Accept = false ); -SELECT ts_lexize('public.simple_dict','YeS'); +SELECT ts_lexize('public.simple_dict', 'YeS'); ts_lexize ----------- -SELECT ts_lexize('public.simple_dict','The'); +SELECT ts_lexize('public.simple_dict', 'The'); ts_lexize ----------- {} @@ -2633,7 +2633,7 @@ indices index* Then we will get these results: mydb=# CREATE TEXT SEARCH DICTIONARY syn (template=synonym, synonyms='synonym_sample'); -mydb=# SELECT ts_lexize('syn','indices'); +mydb=# SELECT ts_lexize('syn', 'indices'); ts_lexize ----------- {index} @@ -2641,13 +2641,13 @@ mydb=# SELECT ts_lexize('syn','indices'); mydb=# CREATE TEXT SEARCH CONFIGURATION tst (copy=simple); mydb=# ALTER TEXT SEARCH CONFIGURATION tst ALTER MAPPING FOR asciiword WITH syn; -mydb=# SELECT to_tsvector('tst','indices'); +mydb=# SELECT to_tsvector('tst', 'indices'); to_tsvector ------------- 'index':1 (1 row) -mydb=# SELECT to_tsquery('tst','indices'); +mydb=# SELECT to_tsquery('tst', 'indices'); to_tsquery ------------ 'index':* @@ -2659,7 +2659,7 @@ mydb=# SELECT 'indexes are very useful'::tsvector; 'are' 'indexes' 'useful' 'very' (1 row) -mydb=# SELECT 'indexes are very useful'::tsvector @@ to_tsquery('tst','indices'); +mydb=# SELECT 'indexes are very useful'::tsvector @@ to_tsquery('tst', 'indices'); ?column? ---------- t @@ -3354,7 +3354,7 @@ ts_debug( config re Here is a simple example: -SELECT * FROM ts_debug('english','a fat cat sat on a mat - it ate a fat rats'); +SELECT * FROM ts_debug('english', 'a fat cat sat on a mat - it ate a fat rats'); alias | description | token | dictionaries | dictionary | lexemes -----------+-----------------+-------+----------------+--------------+--------- asciiword | Word, all ASCII | a | {english_stem} | english_stem | {} @@ -3405,7 +3405,7 @@ ALTER TEXT SEARCH CONFIGURATION public.english -SELECT * FROM ts_debug('public.english','The Brightest supernovaes'); +SELECT * FROM ts_debug('public.english', 'The Brightest supernovaes'); alias | description | token | dictionaries | dictionary | lexemes -----------+-----------------+-------------+-------------------------------+----------------+------------- asciiword | Word, all ASCII | The | {english_ispell,english_stem} | english_ispell | {} @@ -3444,7 +3444,7 @@ SELECT * FROM ts_debug('public.english','The Brightest supernovaes'); SELECT alias, token, dictionary, lexemes -FROM ts_debug('public.english','The Brightest supernovaes'); +FROM ts_debug('public.english', 'The Brightest supernovaes'); alias | token | dictionary | lexemes -----------+-------------+----------------+------------- asciiword | The | english_ispell | {} @@ -3592,7 +3592,7 @@ SELECT ts_lexize('english_stem', 'a'); where this can be confusing: -SELECT ts_lexize('thesaurus_astro','supernovae stars') is null; +SELECT ts_lexize('thesaurus_astro', 'supernovae stars') is null; ?column? ---------- t From c8ab9701791e22f7a8e1badf362654db179c9703 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 19 Oct 2020 14:33:01 -0400 Subject: [PATCH 340/589] Fix list-munging bug that broke SQL function result coercions. Since commit 913bbd88d, check_sql_fn_retval() can either insert type coercion steps in-line in the Query that produces the SQL function's results, or generate a new top-level Query to perform the coercions, if modifying the Query's output in-place wouldn't be safe. However, it appears that the latter case has never actually worked, because the code tried to inject the new Query back into the query list it was passed ... which is not the list that will be used for later processing when we execute the SQL function "normally" (without inlining it). So we ended up with no coercion happening at run-time, leading to wrong results or crashes depending on the datatypes involved. While the regression tests look like they cover this area well enough, through a huge bit of bad luck all the test cases that exercise the separate-Query path were checking either inline-able cases (which accidentally didn't have the bug) or cases that are no-ops at runtime (e.g., varchar to text), so that the failure to perform the coercion wasn't obvious. The fact that the cases that don't work weren't allowed at all before v13 probably contributed to not noticing the problem sooner, too. To fix, get rid of the separate "flat" list of Query nodes and instead pass the real two-level list that is going to be used later. I chose to make the same change in check_sql_fn_statements(), although that has no actual bug, just so that we don't need that data structure at all. This is an API change, as evidenced by the adjustments needed to callers outside functions.c. That's a bit scary to be doing in a released branch, but so far as I can tell from a quick search, there are no outside callers of these functions (and they are sufficiently specific to our semantics for SQL-language functions that it's not apparent why any extension would need to call them). In any case, v13 already changed the API of check_sql_fn_retval() compared to prior branches. Per report from pinker. Back-patch to v13 where this code came in. Discussion: https://postgr.es/m/1603050466566-0.post@n3.nabble.com --- src/backend/catalog/pg_proc.c | 4 +- src/backend/executor/functions.c | 122 +++++++++++++---------- src/backend/optimizer/util/clauses.c | 7 +- src/include/executor/functions.h | 4 +- src/test/regress/expected/rangefuncs.out | 44 ++++++++ src/test/regress/sql/rangefuncs.sql | 11 ++ 6 files changed, 132 insertions(+), 60 deletions(-) diff --git a/src/backend/catalog/pg_proc.c b/src/backend/catalog/pg_proc.c index f7dab9925b9e..1dd9ecc0634a 100644 --- a/src/backend/catalog/pg_proc.c +++ b/src/backend/catalog/pg_proc.c @@ -913,8 +913,8 @@ fmgr_sql_validator(PG_FUNCTION_ARGS) (ParserSetupHook) sql_fn_parser_setup, pinfo, NULL); - querytree_list = list_concat(querytree_list, - querytree_sublist); + querytree_list = lappend(querytree_list, + querytree_sublist); } check_sql_fn_statements(querytree_list); diff --git a/src/backend/executor/functions.c b/src/backend/executor/functions.c index bf00a9c1e8da..459a33375b14 100644 --- a/src/backend/executor/functions.c +++ b/src/backend/executor/functions.c @@ -609,7 +609,6 @@ init_sql_fcache(FunctionCallInfo fcinfo, Oid collation, bool lazyEvalOK) SQLFunctionCachePtr fcache; List *raw_parsetree_list; List *queryTree_list; - List *flat_query_list; List *resulttlist; ListCell *lc; Datum tmp; @@ -689,13 +688,7 @@ init_sql_fcache(FunctionCallInfo fcinfo, Oid collation, bool lazyEvalOK) /* * Parse and rewrite the queries in the function text. Use sublists to - * keep track of the original query boundaries. But we also build a - * "flat" list of the rewritten queries to pass to check_sql_fn_retval. - * This is because the last canSetTag query determines the result type - * independently of query boundaries --- and it might not be in the last - * sublist, for example if the last query rewrites to DO INSTEAD NOTHING. - * (It might not be unreasonable to throw an error in such a case, but - * this is the historical behavior and it doesn't seem worth changing.) + * keep track of the original query boundaries. * * Note: since parsing and planning is done in fcontext, we will generate * a lot of cruft that lives as long as the fcache does. This is annoying @@ -705,7 +698,6 @@ init_sql_fcache(FunctionCallInfo fcinfo, Oid collation, bool lazyEvalOK) raw_parsetree_list = pg_parse_query(fcache->src); queryTree_list = NIL; - flat_query_list = NIL; foreach(lc, raw_parsetree_list) { RawStmt *parsetree = lfirst_node(RawStmt, lc); @@ -717,10 +709,12 @@ init_sql_fcache(FunctionCallInfo fcinfo, Oid collation, bool lazyEvalOK) fcache->pinfo, NULL); queryTree_list = lappend(queryTree_list, queryTree_sublist); - flat_query_list = list_concat(flat_query_list, queryTree_sublist); } - check_sql_fn_statements(flat_query_list); + /* + * Check that there are no statements we don't want to allow. + */ + check_sql_fn_statements(queryTree_list); /* * Check that the function returns the type it claims to. Although in @@ -740,7 +734,7 @@ init_sql_fcache(FunctionCallInfo fcinfo, Oid collation, bool lazyEvalOK) * the rowtype column into multiple columns, since we have no way to * notify the caller that it should do that.) */ - fcache->returnsTuple = check_sql_fn_retval(flat_query_list, + fcache->returnsTuple = check_sql_fn_retval(queryTree_list, rettype, rettupdesc, false, @@ -1510,51 +1504,63 @@ ShutdownSQLFunction(Datum arg) * is not acceptable. */ void -check_sql_fn_statements(List *queryTreeList) +check_sql_fn_statements(List *queryTreeLists) { ListCell *lc; - foreach(lc, queryTreeList) + /* We are given a list of sublists of Queries */ + foreach(lc, queryTreeLists) { - Query *query = lfirst_node(Query, lc); + List *sublist = lfirst_node(List, lc); + ListCell *lc2; - /* - * Disallow procedures with output arguments. The current - * implementation would just throw the output values away, unless the - * statement is the last one. Per SQL standard, we should assign the - * output values by name. By disallowing this here, we preserve an - * opportunity for future improvement. - */ - if (query->commandType == CMD_UTILITY && - IsA(query->utilityStmt, CallStmt)) + foreach(lc2, sublist) { - CallStmt *stmt = castNode(CallStmt, query->utilityStmt); - HeapTuple tuple; - int numargs; - Oid *argtypes; - char **argnames; - char *argmodes; - int i; - - tuple = SearchSysCache1(PROCOID, ObjectIdGetDatum(stmt->funcexpr->funcid)); - if (!HeapTupleIsValid(tuple)) - elog(ERROR, "cache lookup failed for function %u", stmt->funcexpr->funcid); - numargs = get_func_arg_info(tuple, &argtypes, &argnames, &argmodes); - ReleaseSysCache(tuple); - - for (i = 0; i < numargs; i++) + Query *query = lfirst_node(Query, lc2); + + /* + * Disallow procedures with output arguments. The current + * implementation would just throw the output values away, unless + * the statement is the last one. Per SQL standard, we should + * assign the output values by name. By disallowing this here, we + * preserve an opportunity for future improvement. + */ + if (query->commandType == CMD_UTILITY && + IsA(query->utilityStmt, CallStmt)) { - if (argmodes && (argmodes[i] == PROARGMODE_INOUT || argmodes[i] == PROARGMODE_OUT)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("calling procedures with output arguments is not supported in SQL functions"))); + CallStmt *stmt = castNode(CallStmt, query->utilityStmt); + HeapTuple tuple; + int numargs; + Oid *argtypes; + char **argnames; + char *argmodes; + int i; + + tuple = SearchSysCache1(PROCOID, + ObjectIdGetDatum(stmt->funcexpr->funcid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for function %u", + stmt->funcexpr->funcid); + numargs = get_func_arg_info(tuple, + &argtypes, &argnames, &argmodes); + ReleaseSysCache(tuple); + + for (i = 0; i < numargs; i++) + { + if (argmodes && (argmodes[i] == PROARGMODE_INOUT || + argmodes[i] == PROARGMODE_OUT)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("calling procedures with output arguments is not supported in SQL functions"))); + } } } } } /* - * check_sql_fn_retval() -- check return value of a list of sql parse trees. + * check_sql_fn_retval() + * Check return value of a list of lists of sql parse trees. * * The return value of a sql function is the value returned by the last * canSetTag query in the function. We do some ad-hoc type checking and @@ -1592,7 +1598,7 @@ check_sql_fn_statements(List *queryTreeList) * function is defined to return VOID then *resultTargetList is set to NIL. */ bool -check_sql_fn_retval(List *queryTreeList, +check_sql_fn_retval(List *queryTreeLists, Oid rettype, TupleDesc rettupdesc, bool insertDroppedCols, List **resultTargetList) @@ -1619,20 +1625,30 @@ check_sql_fn_retval(List *queryTreeList, return false; /* - * Find the last canSetTag query in the list. This isn't necessarily the - * last parsetree, because rule rewriting can insert queries after what - * the user wrote. + * Find the last canSetTag query in the function body (which is presented + * to us as a list of sublists of Query nodes). This isn't necessarily + * the last parsetree, because rule rewriting can insert queries after + * what the user wrote. Note that it might not even be in the last + * sublist, for example if the last query rewrites to DO INSTEAD NOTHING. + * (It might not be unreasonable to throw an error in such a case, but + * this is the historical behavior and it doesn't seem worth changing.) */ parse = NULL; parse_cell = NULL; - foreach(lc, queryTreeList) + foreach(lc, queryTreeLists) { - Query *q = lfirst_node(Query, lc); + List *sublist = lfirst_node(List, lc); + ListCell *lc2; - if (q->canSetTag) + foreach(lc2, sublist) { - parse = q; - parse_cell = lc; + Query *q = lfirst_node(Query, lc2); + + if (q->canSetTag) + { + parse = q; + parse_cell = lc2; + } } } diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c index 750586fceb74..e7d814651b18 100644 --- a/src/backend/optimizer/util/clauses.c +++ b/src/backend/optimizer/util/clauses.c @@ -4522,7 +4522,8 @@ inline_function(Oid funcid, Oid result_type, Oid result_collid, * needed; that's probably not important, but let's be careful. */ querytree_list = list_make1(querytree); - if (check_sql_fn_retval(querytree_list, result_type, rettupdesc, + if (check_sql_fn_retval(list_make1(querytree_list), + result_type, rettupdesc, false, NULL)) goto fail; /* reject whole-tuple-result cases */ @@ -5040,7 +5041,7 @@ inline_set_returning_function(PlannerInfo *root, RangeTblEntry *rte) * shows it's returning a whole tuple result; otherwise what it's * returning is a single composite column which is not what we need. */ - if (!check_sql_fn_retval(querytree_list, + if (!check_sql_fn_retval(list_make1(querytree_list), fexpr->funcresulttype, rettupdesc, true, NULL) && (functypclass == TYPEFUNC_COMPOSITE || @@ -5052,7 +5053,7 @@ inline_set_returning_function(PlannerInfo *root, RangeTblEntry *rte) * check_sql_fn_retval might've inserted a projection step, but that's * fine; just make sure we use the upper Query. */ - querytree = linitial(querytree_list); + querytree = linitial_node(Query, querytree_list); /* * Looks good --- substitute parameters into the query. diff --git a/src/include/executor/functions.h b/src/include/executor/functions.h index cb13428a5a88..a0db24bde699 100644 --- a/src/include/executor/functions.h +++ b/src/include/executor/functions.h @@ -29,9 +29,9 @@ extern SQLFunctionParseInfoPtr prepare_sql_fn_parse_info(HeapTuple procedureTupl extern void sql_fn_parser_setup(struct ParseState *pstate, SQLFunctionParseInfoPtr pinfo); -extern void check_sql_fn_statements(List *queryTreeList); +extern void check_sql_fn_statements(List *queryTreeLists); -extern bool check_sql_fn_retval(List *queryTreeList, +extern bool check_sql_fn_retval(List *queryTreeLists, Oid rettype, TupleDesc rettupdesc, bool insertDroppedCols, List **resultTargetList); diff --git a/src/test/regress/expected/rangefuncs.out b/src/test/regress/expected/rangefuncs.out index e618aec2ebca..06bd129fd228 100644 --- a/src/test/regress/expected/rangefuncs.out +++ b/src/test/regress/expected/rangefuncs.out @@ -2109,6 +2109,50 @@ select * from testrngfunc(); 7.136178 | 7.14 (1 row) +create or replace function testrngfunc() returns setof rngfunc_type as $$ + select 1, 2 union select 3, 4 order by 1; +$$ language sql immutable; +explain (verbose, costs off) +select testrngfunc(); + QUERY PLAN +------------------------- + ProjectSet + Output: testrngfunc() + -> Result +(3 rows) + +select testrngfunc(); + testrngfunc +----------------- + (1.000000,2.00) + (3.000000,4.00) +(2 rows) + +explain (verbose, costs off) +select * from testrngfunc(); + QUERY PLAN +---------------------------------------------------------- + Subquery Scan on "*SELECT*" + Output: "*SELECT*"."?column?", "*SELECT*"."?column?_1" + -> Unique + Output: (1), (2) + -> Sort + Output: (1), (2) + Sort Key: (1), (2) + -> Append + -> Result + Output: 1, 2 + -> Result + Output: 3, 4 +(12 rows) + +select * from testrngfunc(); + f1 | f2 +----------+------ + 1.000000 | 2.00 + 3.000000 | 4.00 +(2 rows) + -- Check a couple of error cases while we're here select * from testrngfunc() as t(f1 int8,f2 int8); -- fail, composite result ERROR: a column definition list is redundant for a function returning a named composite type diff --git a/src/test/regress/sql/rangefuncs.sql b/src/test/regress/sql/rangefuncs.sql index 5f41cb2d8d0b..3c436028daff 100644 --- a/src/test/regress/sql/rangefuncs.sql +++ b/src/test/regress/sql/rangefuncs.sql @@ -629,6 +629,17 @@ explain (verbose, costs off) select * from testrngfunc(); select * from testrngfunc(); +create or replace function testrngfunc() returns setof rngfunc_type as $$ + select 1, 2 union select 3, 4 order by 1; +$$ language sql immutable; + +explain (verbose, costs off) +select testrngfunc(); +select testrngfunc(); +explain (verbose, costs off) +select * from testrngfunc(); +select * from testrngfunc(); + -- Check a couple of error cases while we're here select * from testrngfunc() as t(f1 int8,f2 int8); -- fail, composite result select * from pg_get_keywords() as t(f1 int8,f2 int8); -- fail, OUT params From 8e5793ab60bba65ffaa0f2237b39c9580d8972c7 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 19 Oct 2020 19:03:46 -0400 Subject: [PATCH 341/589] Fix connection string handling in src/bin/scripts/ programs. When told to process all databases, clusterdb, reindexdb, and vacuumdb would reconnect by replacing their --maintenance-db parameter with the name of the target database. If that parameter is a connstring (which has been allowed for a long time, though we failed to document that before this patch), we'd lose any other options it might specify, for example SSL or GSS parameters, possibly resulting in failure to connect. Thus, this is the same bug as commit a45bc8a4f fixed in pg_dump and pg_restore. We can fix it in the same way, by using libpq's rules for handling multiple "dbname" parameters to add the target database name separately. I chose to apply the same refactoring approach as in that patch, with a struct to handle the command line parameters that need to be passed through to connectDatabase. (Maybe someday we can unify the very similar functions here and in pg_dump/pg_restore.) Per Peter Eisentraut's comments on bug #16604. Back-patch to all supported branches. Discussion: https://postgr.es/m/16604-933f4b8791227b15@postgresql.org --- doc/src/sgml/ref/clusterdb.sgml | 20 ++++--- doc/src/sgml/ref/createdb.sgml | 3 + doc/src/sgml/ref/dropdb.sgml | 3 + doc/src/sgml/ref/reindexdb.sgml | 20 ++++--- doc/src/sgml/ref/vacuumdb.sgml | 20 ++++--- src/bin/scripts/clusterdb.c | 67 ++++++++++------------ src/bin/scripts/common.c | 89 ++++++++++++++++++------------ src/bin/scripts/common.h | 26 ++++++--- src/bin/scripts/createdb.c | 11 +++- src/bin/scripts/createuser.c | 11 +++- src/bin/scripts/dropdb.c | 12 +++- src/bin/scripts/dropuser.c | 13 ++++- src/bin/scripts/reindexdb.c | 89 +++++++++++++++--------------- src/bin/scripts/scripts_parallel.c | 6 +- src/bin/scripts/scripts_parallel.h | 6 +- src/bin/scripts/vacuumdb.c | 70 ++++++++++------------- 16 files changed, 260 insertions(+), 206 deletions(-) diff --git a/doc/src/sgml/ref/clusterdb.sgml b/doc/src/sgml/ref/clusterdb.sgml index c53bacf8648c..c838b22c4405 100644 --- a/doc/src/sgml/ref/clusterdb.sgml +++ b/doc/src/sgml/ref/clusterdb.sgml @@ -90,9 +90,9 @@ PostgreSQL documentation - Specifies the name of the database to be clustered. - If this is not specified and (or - ) is not used, the database name is read + Specifies the name of the database to be clustered, + when / is not used. + If this is not specified, the database name is read from the environment variable PGDATABASE. If that is not set, the user name specified for the connection is used. The dbname can be a - Specifies the name of the database to connect to discover what other - databases should be clustered. If not specified, the - postgres database will be used, - and if that does not exist, template1 will be used. + Specifies the name of the database to connect to to discover which + databases should be clustered, + when / is used. + If not specified, the postgres database will be used, + or if that does not exist, template1 will be used. + This can be a connection + string. If so, connection string parameters will override any + conflicting command line options. Also, connection string parameters + other than the database name itself will be re-used when connecting + to other databases. diff --git a/doc/src/sgml/ref/createdb.sgml b/doc/src/sgml/ref/createdb.sgml index 95cc82dc88bd..86473455c9d0 100644 --- a/doc/src/sgml/ref/createdb.sgml +++ b/doc/src/sgml/ref/createdb.sgml @@ -284,6 +284,9 @@ PostgreSQL documentation database will be used; if that does not exist (or if it is the name of the new database being created), template1 will be used. + This can be a connection + string. If so, connection string parameters will override any + conflicting command line options. diff --git a/doc/src/sgml/ref/dropdb.sgml b/doc/src/sgml/ref/dropdb.sgml index fe523a2ee1d3..d36aed38c527 100644 --- a/doc/src/sgml/ref/dropdb.sgml +++ b/doc/src/sgml/ref/dropdb.sgml @@ -217,6 +217,9 @@ PostgreSQL documentation target database. If not specified, the postgres database will be used; if that does not exist (or is the database being dropped), template1 will be used. + This can be a connection + string. If so, connection string parameters will override any + conflicting command line options. diff --git a/doc/src/sgml/ref/reindexdb.sgml b/doc/src/sgml/ref/reindexdb.sgml index a6d93693c5d4..574144533378 100644 --- a/doc/src/sgml/ref/reindexdb.sgml +++ b/doc/src/sgml/ref/reindexdb.sgml @@ -134,9 +134,9 @@ PostgreSQL documentation - Specifies the name of the database to be reindexed. - If this is not specified and (or - ) is not used, the database name is read + Specifies the name of the database to be reindexed, + when / is not used. + If this is not specified, the database name is read from the environment variable PGDATABASE. If that is not set, the user name specified for the connection is used. The dbname can be a - Specifies the name of the database to connect to discover what other - databases should be reindexed. If not specified, the - postgres database will be used, - and if that does not exist, template1 will be used. + Specifies the name of the database to connect to to discover which + databases should be reindexed, + when / is used. + If not specified, the postgres database will be used, + or if that does not exist, template1 will be used. + This can be a connection + string. If so, connection string parameters will override any + conflicting command line options. Also, connection string parameters + other than the database name itself will be re-used when connecting + to other databases. diff --git a/doc/src/sgml/ref/vacuumdb.sgml b/doc/src/sgml/ref/vacuumdb.sgml index 6dcdab9cafd6..a90fc9322f99 100644 --- a/doc/src/sgml/ref/vacuumdb.sgml +++ b/doc/src/sgml/ref/vacuumdb.sgml @@ -92,9 +92,9 @@ PostgreSQL documentation - Specifies the name of the database to be cleaned or analyzed. - If this is not specified and (or - ) is not used, the database name is read + Specifies the name of the database to be cleaned or analyzed, + when / is not used. + If this is not specified, the database name is read from the environment variable PGDATABASE. If that is not set, the user name specified for the connection is used. The dbname can be a - Specifies the name of the database to connect to discover what other - databases should be vacuumed. If not specified, the - postgres database will be used, - and if that does not exist, template1 will be used. + Specifies the name of the database to connect to to discover which + databases should be vacuumed, + when / is used. + If not specified, the postgres database will be used, + or if that does not exist, template1 will be used. + This can be a connection + string. If so, connection string parameters will override any + conflicting command line options. Also, connection string parameters + other than the database name itself will be re-used when connecting + to other databases. diff --git a/src/bin/scripts/clusterdb.c b/src/bin/scripts/clusterdb.c index 12972de0e91e..2f786e61037b 100644 --- a/src/bin/scripts/clusterdb.c +++ b/src/bin/scripts/clusterdb.c @@ -17,15 +17,10 @@ #include "fe_utils/string_utils.h" -static void cluster_one_database(const char *dbname, bool verbose, const char *table, - const char *host, const char *port, - const char *username, enum trivalue prompt_password, - const char *progname, bool echo); -static void cluster_all_databases(bool verbose, const char *maintenance_db, - const char *host, const char *port, - const char *username, enum trivalue prompt_password, - const char *progname, bool echo, bool quiet); - +static void cluster_one_database(const ConnParams *cparams, const char *table, + const char *progname, bool verbose, bool echo); +static void cluster_all_databases(ConnParams *cparams, const char *progname, + bool verbose, bool echo, bool quiet); static void help(const char *progname); @@ -58,6 +53,7 @@ main(int argc, char *argv[]) char *port = NULL; char *username = NULL; enum trivalue prompt_password = TRI_DEFAULT; + ConnParams cparams; bool echo = false; bool quiet = false; bool alldb = false; @@ -134,6 +130,13 @@ main(int argc, char *argv[]) exit(1); } + /* fill cparams except for dbname, which is set below */ + cparams.pghost = host; + cparams.pgport = port; + cparams.pguser = username; + cparams.prompt_password = prompt_password; + cparams.override_dbname = NULL; + setup_cancel_handler(NULL); if (alldb) @@ -150,8 +153,9 @@ main(int argc, char *argv[]) exit(1); } - cluster_all_databases(verbose, maintenance_db, host, port, username, prompt_password, - progname, echo, quiet); + cparams.dbname = maintenance_db; + + cluster_all_databases(&cparams, progname, verbose, echo, quiet); } else { @@ -165,21 +169,21 @@ main(int argc, char *argv[]) dbname = get_user_name_or_exit(progname); } + cparams.dbname = dbname; + if (tables.head != NULL) { SimpleStringListCell *cell; for (cell = tables.head; cell; cell = cell->next) { - cluster_one_database(dbname, verbose, cell->val, - host, port, username, prompt_password, - progname, echo); + cluster_one_database(&cparams, cell->val, + progname, verbose, echo); } } else - cluster_one_database(dbname, verbose, NULL, - host, port, username, prompt_password, - progname, echo); + cluster_one_database(&cparams, NULL, + progname, verbose, echo); } exit(0); @@ -187,17 +191,14 @@ main(int argc, char *argv[]) static void -cluster_one_database(const char *dbname, bool verbose, const char *table, - const char *host, const char *port, - const char *username, enum trivalue prompt_password, - const char *progname, bool echo) +cluster_one_database(const ConnParams *cparams, const char *table, + const char *progname, bool verbose, bool echo) { PQExpBufferData sql; PGconn *conn; - conn = connectDatabase(dbname, host, port, username, prompt_password, - progname, echo, false, false); + conn = connectDatabase(cparams, progname, echo, false, false); initPQExpBuffer(&sql); @@ -228,22 +229,17 @@ cluster_one_database(const char *dbname, bool verbose, const char *table, static void -cluster_all_databases(bool verbose, const char *maintenance_db, - const char *host, const char *port, - const char *username, enum trivalue prompt_password, - const char *progname, bool echo, bool quiet) +cluster_all_databases(ConnParams *cparams, const char *progname, + bool verbose, bool echo, bool quiet) { PGconn *conn; PGresult *result; - PQExpBufferData connstr; int i; - conn = connectMaintenanceDatabase(maintenance_db, host, port, username, - prompt_password, progname, echo); + conn = connectMaintenanceDatabase(cparams, progname, echo); result = executeQuery(conn, "SELECT datname FROM pg_database WHERE datallowconn ORDER BY 1;", echo); PQfinish(conn); - initPQExpBuffer(&connstr); for (i = 0; i < PQntuples(result); i++) { char *dbname = PQgetvalue(result, i, 0); @@ -254,15 +250,10 @@ cluster_all_databases(bool verbose, const char *maintenance_db, fflush(stdout); } - resetPQExpBuffer(&connstr); - appendPQExpBufferStr(&connstr, "dbname="); - appendConnStrVal(&connstr, dbname); + cparams->override_dbname = dbname; - cluster_one_database(connstr.data, verbose, NULL, - host, port, username, prompt_password, - progname, echo); + cluster_one_database(cparams, NULL, progname, verbose, echo); } - termPQExpBuffer(&connstr); PQclear(result); } diff --git a/src/bin/scripts/common.c b/src/bin/scripts/common.c index e987eef23434..3362221a3114 100644 --- a/src/bin/scripts/common.c +++ b/src/bin/scripts/common.c @@ -54,7 +54,7 @@ handle_help_version_opts(int argc, char *argv[], * Make a database connection with the given parameters. * * An interactive password prompt is automatically issued if needed and - * allowed by prompt_password. + * allowed by cparams->prompt_password. * * If allow_password_reuse is true, we will try to re-use any password * given during previous calls to this routine. (Callers should not pass @@ -62,22 +62,23 @@ handle_help_version_opts(int argc, char *argv[], * as before, else we might create password exposure hazards.) */ PGconn * -connectDatabase(const char *dbname, const char *pghost, - const char *pgport, const char *pguser, - enum trivalue prompt_password, const char *progname, +connectDatabase(const ConnParams *cparams, const char *progname, bool echo, bool fail_ok, bool allow_password_reuse) { PGconn *conn; bool new_pass; static char *password = NULL; + /* Callers must supply at least dbname; other params can be NULL */ + Assert(cparams->dbname); + if (!allow_password_reuse && password) { free(password); password = NULL; } - if (!password && prompt_password == TRI_YES) + if (cparams->prompt_password == TRI_YES && password == NULL) password = simple_prompt("Password: ", false); /* @@ -86,23 +87,35 @@ connectDatabase(const char *dbname, const char *pghost, */ do { - const char *keywords[7]; - const char *values[7]; - - keywords[0] = "host"; - values[0] = pghost; - keywords[1] = "port"; - values[1] = pgport; - keywords[2] = "user"; - values[2] = pguser; - keywords[3] = "password"; - values[3] = password; - keywords[4] = "dbname"; - values[4] = dbname; - keywords[5] = "fallback_application_name"; - values[5] = progname; - keywords[6] = NULL; - values[6] = NULL; + const char *keywords[8]; + const char *values[8]; + int i = 0; + + /* + * If dbname is a connstring, its entries can override the other + * values obtained from cparams; but in turn, override_dbname can + * override the dbname component of it. + */ + keywords[i] = "host"; + values[i++] = cparams->pghost; + keywords[i] = "port"; + values[i++] = cparams->pgport; + keywords[i] = "user"; + values[i++] = cparams->pguser; + keywords[i] = "password"; + values[i++] = password; + keywords[i] = "dbname"; + values[i++] = cparams->dbname; + if (cparams->override_dbname) + { + keywords[i] = "dbname"; + values[i++] = cparams->override_dbname; + } + keywords[i] = "fallback_application_name"; + values[i++] = progname; + keywords[i] = NULL; + values[i++] = NULL; + Assert(i <= lengthof(keywords)); new_pass = false; conn = PQconnectdbParams(keywords, values, true); @@ -110,7 +123,7 @@ connectDatabase(const char *dbname, const char *pghost, if (!conn) { pg_log_error("could not connect to database %s: out of memory", - dbname); + cparams->dbname); exit(1); } @@ -119,7 +132,7 @@ connectDatabase(const char *dbname, const char *pghost, */ if (PQstatus(conn) == CONNECTION_BAD && PQconnectionNeedsPassword(conn) && - prompt_password != TRI_NO) + cparams->prompt_password != TRI_NO) { PQfinish(conn); if (password) @@ -138,10 +151,11 @@ connectDatabase(const char *dbname, const char *pghost, return NULL; } pg_log_error("could not connect to database %s: %s", - dbname, PQerrorMessage(conn)); + cparams->dbname, PQerrorMessage(conn)); exit(1); } + /* Start strict; callers may override this. */ PQclear(executeQuery(conn, ALWAYS_SECURE_SEARCH_PATH_SQL, echo)); return conn; @@ -149,27 +163,30 @@ connectDatabase(const char *dbname, const char *pghost, /* * Try to connect to the appropriate maintenance database. + * + * This differs from connectDatabase only in that it has a rule for + * inserting a default "dbname" if none was given (which is why cparams + * is not const). Note that cparams->dbname should typically come from + * a --maintenance-db command line parameter. */ PGconn * -connectMaintenanceDatabase(const char *maintenance_db, - const char *pghost, const char *pgport, - const char *pguser, enum trivalue prompt_password, +connectMaintenanceDatabase(ConnParams *cparams, const char *progname, bool echo) { PGconn *conn; /* If a maintenance database name was specified, just connect to it. */ - if (maintenance_db) - return connectDatabase(maintenance_db, pghost, pgport, pguser, - prompt_password, progname, echo, false, false); + if (cparams->dbname) + return connectDatabase(cparams, progname, echo, false, false); /* Otherwise, try postgres first and then template1. */ - conn = connectDatabase("postgres", pghost, pgport, pguser, prompt_password, - progname, echo, true, false); + cparams->dbname = "postgres"; + conn = connectDatabase(cparams, progname, echo, true, false); if (!conn) - conn = connectDatabase("template1", pghost, pgport, pguser, - prompt_password, progname, echo, false, false); - + { + cparams->dbname = "template1"; + conn = connectDatabase(cparams, progname, echo, false, false); + } return conn; } diff --git a/src/bin/scripts/common.h b/src/bin/scripts/common.h index ddf6320b47c8..9ec57cdd87c0 100644 --- a/src/bin/scripts/common.h +++ b/src/bin/scripts/common.h @@ -21,20 +21,32 @@ enum trivalue TRI_YES }; +/* Parameters needed by connectDatabase/connectMaintenanceDatabase */ +typedef struct _connParams +{ + /* These fields record the actual command line parameters */ + const char *dbname; /* this may be a connstring! */ + const char *pghost; + const char *pgport; + const char *pguser; + enum trivalue prompt_password; + /* If not NULL, this overrides the dbname obtained from command line */ + /* (but *only* the DB name, not anything else in the connstring) */ + const char *override_dbname; +} ConnParams; + typedef void (*help_handler) (const char *progname); extern void handle_help_version_opts(int argc, char *argv[], const char *fixed_progname, help_handler hlp); -extern PGconn *connectDatabase(const char *dbname, const char *pghost, - const char *pgport, const char *pguser, - enum trivalue prompt_password, const char *progname, - bool echo, bool fail_ok, bool allow_password_reuse); +extern PGconn *connectDatabase(const ConnParams *cparams, + const char *progname, + bool echo, bool fail_ok, + bool allow_password_reuse); -extern PGconn *connectMaintenanceDatabase(const char *maintenance_db, - const char *pghost, const char *pgport, - const char *pguser, enum trivalue prompt_password, +extern PGconn *connectMaintenanceDatabase(ConnParams *cparams, const char *progname, bool echo); extern void disconnectDatabase(PGconn *conn); diff --git a/src/bin/scripts/createdb.c b/src/bin/scripts/createdb.c index 1353af97c49e..91e6e2194bd7 100644 --- a/src/bin/scripts/createdb.c +++ b/src/bin/scripts/createdb.c @@ -51,6 +51,7 @@ main(int argc, char *argv[]) char *port = NULL; char *username = NULL; enum trivalue prompt_password = TRI_DEFAULT; + ConnParams cparams; bool echo = false; char *owner = NULL; char *tablespace = NULL; @@ -180,8 +181,14 @@ main(int argc, char *argv[]) if (maintenance_db == NULL && strcmp(dbname, "postgres") == 0) maintenance_db = "template1"; - conn = connectMaintenanceDatabase(maintenance_db, host, port, username, - prompt_password, progname, echo); + cparams.dbname = maintenance_db; + cparams.pghost = host; + cparams.pgport = port; + cparams.pguser = username; + cparams.prompt_password = prompt_password; + cparams.override_dbname = NULL; + + conn = connectMaintenanceDatabase(&cparams, progname, echo); initPQExpBuffer(&sql); diff --git a/src/bin/scripts/createuser.c b/src/bin/scripts/createuser.c index 6179199563c4..d6b56f15c3b6 100644 --- a/src/bin/scripts/createuser.c +++ b/src/bin/scripts/createuser.c @@ -59,6 +59,7 @@ main(int argc, char *argv[]) char *username = NULL; SimpleStringList roles = {NULL, NULL}; enum trivalue prompt_password = TRI_DEFAULT; + ConnParams cparams; bool echo = false; bool interactive = false; int conn_limit = -2; /* less than minimum valid value */ @@ -252,8 +253,14 @@ main(int argc, char *argv[]) if (login == 0) login = TRI_YES; - conn = connectDatabase("postgres", host, port, username, prompt_password, - progname, echo, false, false); + cparams.dbname = NULL; /* this program lacks any dbname option... */ + cparams.pghost = host; + cparams.pgport = port; + cparams.pguser = username; + cparams.prompt_password = prompt_password; + cparams.override_dbname = NULL; + + conn = connectMaintenanceDatabase(&cparams, progname, echo); initPQExpBuffer(&sql); diff --git a/src/bin/scripts/dropdb.c b/src/bin/scripts/dropdb.c index 581c7749c86a..ccbf78e91a86 100644 --- a/src/bin/scripts/dropdb.c +++ b/src/bin/scripts/dropdb.c @@ -48,6 +48,7 @@ main(int argc, char *argv[]) char *port = NULL; char *username = NULL; enum trivalue prompt_password = TRI_DEFAULT; + ConnParams cparams; bool echo = false; bool interactive = false; bool force = false; @@ -137,9 +138,14 @@ main(int argc, char *argv[]) if (maintenance_db == NULL && strcmp(dbname, "postgres") == 0) maintenance_db = "template1"; - conn = connectMaintenanceDatabase(maintenance_db, - host, port, username, prompt_password, - progname, echo); + cparams.dbname = maintenance_db; + cparams.pghost = host; + cparams.pgport = port; + cparams.pguser = username; + cparams.prompt_password = prompt_password; + cparams.override_dbname = NULL; + + conn = connectMaintenanceDatabase(&cparams, progname, echo); if (echo) printf("%s\n", sql.data); diff --git a/src/bin/scripts/dropuser.c b/src/bin/scripts/dropuser.c index f7ddd1402db2..73d7328a88d5 100644 --- a/src/bin/scripts/dropuser.c +++ b/src/bin/scripts/dropuser.c @@ -46,6 +46,7 @@ main(int argc, char *argv[]) char *port = NULL; char *username = NULL; enum trivalue prompt_password = TRI_DEFAULT; + ConnParams cparams; bool echo = false; bool interactive = false; @@ -129,13 +130,19 @@ main(int argc, char *argv[]) exit(0); } + cparams.dbname = NULL; /* this program lacks any dbname option... */ + cparams.pghost = host; + cparams.pgport = port; + cparams.pguser = username; + cparams.prompt_password = prompt_password; + cparams.override_dbname = NULL; + + conn = connectMaintenanceDatabase(&cparams, progname, echo); + initPQExpBuffer(&sql); appendPQExpBuffer(&sql, "DROP ROLE %s%s;", (if_exists ? "IF EXISTS " : ""), fmtId(dropuser)); - conn = connectDatabase("postgres", host, port, username, prompt_password, - progname, echo, false, false); - if (echo) printf("%s\n", sql.data); result = PQexec(conn, sql.data); diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c index 1efb53110efa..b32a7746baf4 100644 --- a/src/bin/scripts/reindexdb.c +++ b/src/bin/scripts/reindexdb.c @@ -34,15 +34,12 @@ static SimpleStringList *get_parallel_object_list(PGconn *conn, ReindexType type, SimpleStringList *user_list, bool echo); -static void reindex_one_database(const char *dbname, ReindexType type, - SimpleStringList *user_list, const char *host, - const char *port, const char *username, - enum trivalue prompt_password, const char *progname, +static void reindex_one_database(const ConnParams *cparams, ReindexType type, + SimpleStringList *user_list, + const char *progname, bool echo, bool verbose, bool concurrently, int concurrentCons); -static void reindex_all_databases(const char *maintenance_db, - const char *host, const char *port, - const char *username, enum trivalue prompt_password, +static void reindex_all_databases(ConnParams *cparams, const char *progname, bool echo, bool quiet, bool verbose, bool concurrently, int concurrentCons); @@ -86,6 +83,7 @@ main(int argc, char *argv[]) const char *port = NULL; const char *username = NULL; enum trivalue prompt_password = TRI_DEFAULT; + ConnParams cparams; bool syscatalog = false; bool alldb = false; bool echo = false; @@ -188,6 +186,13 @@ main(int argc, char *argv[]) exit(1); } + /* fill cparams except for dbname, which is set below */ + cparams.pghost = host; + cparams.pgport = port; + cparams.pguser = username; + cparams.prompt_password = prompt_password; + cparams.override_dbname = NULL; + setup_cancel_handler(NULL); if (alldb) @@ -218,8 +223,9 @@ main(int argc, char *argv[]) exit(1); } - reindex_all_databases(maintenance_db, host, port, username, - prompt_password, progname, echo, quiet, verbose, + cparams.dbname = maintenance_db; + + reindex_all_databases(&cparams, progname, echo, quiet, verbose, concurrently, concurrentCons); } else if (syscatalog) @@ -256,9 +262,11 @@ main(int argc, char *argv[]) dbname = get_user_name_or_exit(progname); } - reindex_one_database(dbname, REINDEX_SYSTEM, NULL, host, - port, username, prompt_password, progname, - echo, verbose, concurrently, 1); + cparams.dbname = dbname; + + reindex_one_database(&cparams, REINDEX_SYSTEM, NULL, + progname, echo, verbose, + concurrently, 1); } else { @@ -283,40 +291,40 @@ main(int argc, char *argv[]) dbname = get_user_name_or_exit(progname); } + cparams.dbname = dbname; + if (schemas.head != NULL) - reindex_one_database(dbname, REINDEX_SCHEMA, &schemas, host, - port, username, prompt_password, progname, - echo, verbose, concurrently, concurrentCons); + reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas, + progname, echo, verbose, + concurrently, concurrentCons); if (indexes.head != NULL) - reindex_one_database(dbname, REINDEX_INDEX, &indexes, host, - port, username, prompt_password, progname, - echo, verbose, concurrently, 1); + reindex_one_database(&cparams, REINDEX_INDEX, &indexes, + progname, echo, verbose, + concurrently, 1); if (tables.head != NULL) - reindex_one_database(dbname, REINDEX_TABLE, &tables, host, - port, username, prompt_password, progname, - echo, verbose, concurrently, - concurrentCons); + reindex_one_database(&cparams, REINDEX_TABLE, &tables, + progname, echo, verbose, + concurrently, concurrentCons); /* * reindex database only if neither index nor table nor schema is * specified */ if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL) - reindex_one_database(dbname, REINDEX_DATABASE, NULL, host, - port, username, prompt_password, progname, - echo, verbose, concurrently, concurrentCons); + reindex_one_database(&cparams, REINDEX_DATABASE, NULL, + progname, echo, verbose, + concurrently, concurrentCons); } exit(0); } static void -reindex_one_database(const char *dbname, ReindexType type, - SimpleStringList *user_list, const char *host, - const char *port, const char *username, - enum trivalue prompt_password, const char *progname, bool echo, +reindex_one_database(const ConnParams *cparams, ReindexType type, + SimpleStringList *user_list, + const char *progname, bool echo, bool verbose, bool concurrently, int concurrentCons) { PGconn *conn; @@ -328,8 +336,7 @@ reindex_one_database(const char *dbname, ReindexType type, bool failed = false; int items_count = 0; - conn = connectDatabase(dbname, host, port, username, prompt_password, - progname, echo, false, false); + conn = connectDatabase(cparams, progname, echo, false, false); if (concurrently && PQserverVersion(conn) < 120000) { @@ -436,8 +443,7 @@ reindex_one_database(const char *dbname, ReindexType type, Assert(process_list != NULL); - slots = ParallelSlotsSetup(dbname, host, port, username, prompt_password, - progname, echo, conn, concurrentCons); + slots = ParallelSlotsSetup(cparams, progname, echo, conn, concurrentCons); cell = process_list->head; do @@ -705,23 +711,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type, } static void -reindex_all_databases(const char *maintenance_db, - const char *host, const char *port, - const char *username, enum trivalue prompt_password, +reindex_all_databases(ConnParams *cparams, const char *progname, bool echo, bool quiet, bool verbose, bool concurrently, int concurrentCons) { PGconn *conn; PGresult *result; - PQExpBufferData connstr; int i; - conn = connectMaintenanceDatabase(maintenance_db, host, port, username, - prompt_password, progname, echo); + conn = connectMaintenanceDatabase(cparams, progname, echo); result = executeQuery(conn, "SELECT datname FROM pg_database WHERE datallowconn ORDER BY 1;", echo); PQfinish(conn); - initPQExpBuffer(&connstr); for (i = 0; i < PQntuples(result); i++) { char *dbname = PQgetvalue(result, i, 0); @@ -732,16 +733,12 @@ reindex_all_databases(const char *maintenance_db, fflush(stdout); } - resetPQExpBuffer(&connstr); - appendPQExpBufferStr(&connstr, "dbname="); - appendConnStrVal(&connstr, dbname); + cparams->override_dbname = dbname; - reindex_one_database(connstr.data, REINDEX_DATABASE, NULL, host, - port, username, prompt_password, + reindex_one_database(cparams, REINDEX_DATABASE, NULL, progname, echo, verbose, concurrently, concurrentCons); } - termPQExpBuffer(&connstr); PQclear(result); } diff --git a/src/bin/scripts/scripts_parallel.c b/src/bin/scripts/scripts_parallel.c index 01bc6dfeffc9..ec264a269a7d 100644 --- a/src/bin/scripts/scripts_parallel.c +++ b/src/bin/scripts/scripts_parallel.c @@ -205,8 +205,7 @@ ParallelSlotsGetIdle(ParallelSlot *slots, int numslots) * set. */ ParallelSlot * -ParallelSlotsSetup(const char *dbname, const char *host, const char *port, - const char *username, bool prompt_password, +ParallelSlotsSetup(const ConnParams *cparams, const char *progname, bool echo, PGconn *conn, int numslots) { @@ -221,8 +220,7 @@ ParallelSlotsSetup(const char *dbname, const char *host, const char *port, { for (i = 1; i < numslots; i++) { - conn = connectDatabase(dbname, host, port, username, prompt_password, - progname, echo, false, true); + conn = connectDatabase(cparams, progname, echo, false, true); /* * Fail and exit immediately if trying to use a socket in an diff --git a/src/bin/scripts/scripts_parallel.h b/src/bin/scripts/scripts_parallel.h index cf20449ce3e2..c9d9f0623e94 100644 --- a/src/bin/scripts/scripts_parallel.h +++ b/src/bin/scripts/scripts_parallel.h @@ -12,6 +12,7 @@ #ifndef SCRIPTS_PARALLEL_H #define SCRIPTS_PARALLEL_H +#include "common.h" #include "libpq-fe.h" @@ -23,10 +24,7 @@ typedef struct ParallelSlot extern ParallelSlot *ParallelSlotsGetIdle(ParallelSlot *slots, int numslots); -extern ParallelSlot *ParallelSlotsSetup(const char *dbname, const char *host, - const char *port, - const char *username, - bool prompt_password, +extern ParallelSlot *ParallelSlotsSetup(const ConnParams *cparams, const char *progname, bool echo, PGconn *conn, int numslots); diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c index 2a1247a1b042..8c2eade1d5d2 100644 --- a/src/bin/scripts/vacuumdb.c +++ b/src/bin/scripts/vacuumdb.c @@ -42,19 +42,16 @@ typedef struct vacuumingOptions } vacuumingOptions; -static void vacuum_one_database(const char *dbname, vacuumingOptions *vacopts, +static void vacuum_one_database(const ConnParams *cparams, + vacuumingOptions *vacopts, int stage, SimpleStringList *tables, - const char *host, const char *port, - const char *username, enum trivalue prompt_password, int concurrentCons, const char *progname, bool echo, bool quiet); -static void vacuum_all_databases(vacuumingOptions *vacopts, +static void vacuum_all_databases(ConnParams *cparams, + vacuumingOptions *vacopts, bool analyze_in_stages, - const char *maintenance_db, - const char *host, const char *port, - const char *username, enum trivalue prompt_password, int concurrentCons, const char *progname, bool echo, bool quiet); @@ -112,6 +109,7 @@ main(int argc, char *argv[]) char *port = NULL; char *username = NULL; enum trivalue prompt_password = TRI_DEFAULT; + ConnParams cparams; bool echo = false; bool quiet = false; vacuumingOptions vacopts; @@ -311,6 +309,13 @@ main(int argc, char *argv[]) } } + /* fill cparams except for dbname, which is set below */ + cparams.pghost = host; + cparams.pgport = port; + cparams.pguser = username; + cparams.prompt_password = prompt_password; + cparams.override_dbname = NULL; + setup_cancel_handler(NULL); /* Avoid opening extra connections. */ @@ -330,10 +335,10 @@ main(int argc, char *argv[]) exit(1); } - vacuum_all_databases(&vacopts, + cparams.dbname = maintenance_db; + + vacuum_all_databases(&cparams, &vacopts, analyze_in_stages, - maintenance_db, - host, port, username, prompt_password, concurrentCons, progname, echo, quiet); } @@ -349,25 +354,25 @@ main(int argc, char *argv[]) dbname = get_user_name_or_exit(progname); } + cparams.dbname = dbname; + if (analyze_in_stages) { int stage; for (stage = 0; stage < ANALYZE_NUM_STAGES; stage++) { - vacuum_one_database(dbname, &vacopts, + vacuum_one_database(&cparams, &vacopts, stage, &tables, - host, port, username, prompt_password, concurrentCons, progname, echo, quiet); } } else - vacuum_one_database(dbname, &vacopts, + vacuum_one_database(&cparams, &vacopts, ANALYZE_NO_STAGE, &tables, - host, port, username, prompt_password, concurrentCons, progname, echo, quiet); } @@ -389,11 +394,10 @@ main(int argc, char *argv[]) * a list of tables from the database. */ static void -vacuum_one_database(const char *dbname, vacuumingOptions *vacopts, +vacuum_one_database(const ConnParams *cparams, + vacuumingOptions *vacopts, int stage, SimpleStringList *tables, - const char *host, const char *port, - const char *username, enum trivalue prompt_password, int concurrentCons, const char *progname, bool echo, bool quiet) { @@ -424,8 +428,7 @@ vacuum_one_database(const char *dbname, vacuumingOptions *vacopts, Assert(stage == ANALYZE_NO_STAGE || (stage >= 0 && stage < ANALYZE_NUM_STAGES)); - conn = connectDatabase(dbname, host, port, username, prompt_password, - progname, echo, false, true); + conn = connectDatabase(cparams, progname, echo, false, true); if (vacopts->disable_page_skipping && PQserverVersion(conn) < 90600) { @@ -663,8 +666,7 @@ vacuum_one_database(const char *dbname, vacuumingOptions *vacopts, * for the first slot. If not in parallel mode, the first slot in the * array contains the connection. */ - slots = ParallelSlotsSetup(dbname, host, port, username, prompt_password, - progname, echo, conn, concurrentCons); + slots = ParallelSlotsSetup(cparams, progname, echo, conn, concurrentCons); /* * Prepare all the connections to run the appropriate analyze stage, if @@ -736,28 +738,23 @@ vacuum_one_database(const char *dbname, vacuumingOptions *vacopts, * quickly everywhere before generating more detailed ones. */ static void -vacuum_all_databases(vacuumingOptions *vacopts, +vacuum_all_databases(ConnParams *cparams, + vacuumingOptions *vacopts, bool analyze_in_stages, - const char *maintenance_db, const char *host, - const char *port, const char *username, - enum trivalue prompt_password, int concurrentCons, const char *progname, bool echo, bool quiet) { PGconn *conn; PGresult *result; - PQExpBufferData connstr; int stage; int i; - conn = connectMaintenanceDatabase(maintenance_db, host, port, username, - prompt_password, progname, echo); + conn = connectMaintenanceDatabase(cparams, progname, echo); result = executeQuery(conn, "SELECT datname FROM pg_database WHERE datallowconn ORDER BY 1;", echo); PQfinish(conn); - initPQExpBuffer(&connstr); if (analyze_in_stages) { /* @@ -772,14 +769,11 @@ vacuum_all_databases(vacuumingOptions *vacopts, { for (i = 0; i < PQntuples(result); i++) { - resetPQExpBuffer(&connstr); - appendPQExpBufferStr(&connstr, "dbname="); - appendConnStrVal(&connstr, PQgetvalue(result, i, 0)); + cparams->override_dbname = PQgetvalue(result, i, 0); - vacuum_one_database(connstr.data, vacopts, + vacuum_one_database(cparams, vacopts, stage, NULL, - host, port, username, prompt_password, concurrentCons, progname, echo, quiet); } @@ -789,19 +783,15 @@ vacuum_all_databases(vacuumingOptions *vacopts, { for (i = 0; i < PQntuples(result); i++) { - resetPQExpBuffer(&connstr); - appendPQExpBufferStr(&connstr, "dbname="); - appendConnStrVal(&connstr, PQgetvalue(result, i, 0)); + cparams->override_dbname = PQgetvalue(result, i, 0); - vacuum_one_database(connstr.data, vacopts, + vacuum_one_database(cparams, vacopts, ANALYZE_NO_STAGE, NULL, - host, port, username, prompt_password, concurrentCons, progname, echo, quiet); } } - termPQExpBuffer(&connstr); PQclear(result); } From 03d51b776d88badbeb4452c1ab452feb78dce36a Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Tue, 20 Oct 2020 10:24:36 +0530 Subject: [PATCH 342/589] Change the attribute name in pg_stat_replication_slots view. Change the attribute 'name' to 'slot_name' in pg_stat_replication_slots view to make it clear and that way we will be consistent with the other places like pg_stat_wal_receiver view where we display the same attribute. In the passing, fix the typo in one of the macros in the related code. Bump the catversion as we have modified the name in the catalog as well. Reported-by: Noriyoshi Shinoda Author: Noriyoshi Shinoda Reviewed-by: Sawada Masahiko and Amit Kapila Discussion: https://postgr.es/m/CA+fd4k5_pPAYRTDrO2PbtTOe0eHQpBvuqmCr8ic39uTNmR49Eg@mail.gmail.com --- contrib/test_decoding/expected/stats.out | 14 +++++++------- contrib/test_decoding/sql/stats.sql | 8 ++++---- doc/src/sgml/monitoring.sgml | 2 +- src/backend/catalog/system_views.sql | 2 +- src/backend/utils/adt/pgstatfuncs.c | 6 +++--- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.dat | 2 +- src/test/regress/expected/rules.out | 4 ++-- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/contrib/test_decoding/expected/stats.out b/contrib/test_decoding/expected/stats.out index bfffd1ac21df..dafca965201d 100644 --- a/contrib/test_decoding/expected/stats.out +++ b/contrib/test_decoding/expected/stats.out @@ -21,7 +21,7 @@ BEGIN ELSE (spill_txns > 0) END INTO updated - FROM pg_stat_replication_slots WHERE name='regression_slot'; + FROM pg_stat_replication_slots WHERE slot_name='regression_slot'; exit WHEN updated; @@ -57,8 +57,8 @@ SELECT wait_for_decode_stats(false); (1 row) -SELECT name, spill_txns > 0 AS spill_txns, spill_count > 0 AS spill_count FROM pg_stat_replication_slots; - name | spill_txns | spill_count +SELECT slot_name, spill_txns > 0 AS spill_txns, spill_count > 0 AS spill_count FROM pg_stat_replication_slots; + slot_name | spill_txns | spill_count -----------------+------------+------------- regression_slot | t | t (1 row) @@ -76,8 +76,8 @@ SELECT wait_for_decode_stats(true); (1 row) -SELECT name, spill_txns, spill_count FROM pg_stat_replication_slots; - name | spill_txns | spill_count +SELECT slot_name, spill_txns, spill_count FROM pg_stat_replication_slots; + slot_name | spill_txns | spill_count -----------------+------------+------------- regression_slot | 0 | 0 (1 row) @@ -95,8 +95,8 @@ SELECT wait_for_decode_stats(false); (1 row) -SELECT name, spill_txns > 0 AS spill_txns, spill_count > 0 AS spill_count FROM pg_stat_replication_slots; - name | spill_txns | spill_count +SELECT slot_name, spill_txns > 0 AS spill_txns, spill_count > 0 AS spill_count FROM pg_stat_replication_slots; + slot_name | spill_txns | spill_count -----------------+------------+------------- regression_slot | t | t (1 row) diff --git a/contrib/test_decoding/sql/stats.sql b/contrib/test_decoding/sql/stats.sql index b95adb16fa71..182df84030d0 100644 --- a/contrib/test_decoding/sql/stats.sql +++ b/contrib/test_decoding/sql/stats.sql @@ -19,7 +19,7 @@ BEGIN ELSE (spill_txns > 0) END INTO updated - FROM pg_stat_replication_slots WHERE name='regression_slot'; + FROM pg_stat_replication_slots WHERE slot_name='regression_slot'; exit WHEN updated; @@ -47,17 +47,17 @@ SELECT count(*) FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, -- exact stats count as that can vary if any background transaction (say by -- autovacuum) happens in parallel to the main transaction. SELECT wait_for_decode_stats(false); -SELECT name, spill_txns > 0 AS spill_txns, spill_count > 0 AS spill_count FROM pg_stat_replication_slots; +SELECT slot_name, spill_txns > 0 AS spill_txns, spill_count > 0 AS spill_count FROM pg_stat_replication_slots; -- reset the slot stats, and wait for stats collector to reset SELECT pg_stat_reset_replication_slot('regression_slot'); SELECT wait_for_decode_stats(true); -SELECT name, spill_txns, spill_count FROM pg_stat_replication_slots; +SELECT slot_name, spill_txns, spill_count FROM pg_stat_replication_slots; -- decode and check stats again. SELECT count(*) FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'skip-empty-xacts', '1'); SELECT wait_for_decode_stats(false); -SELECT name, spill_txns > 0 AS spill_txns, spill_count > 0 AS spill_count FROM pg_stat_replication_slots; +SELECT slot_name, spill_txns > 0 AS spill_txns, spill_count > 0 AS spill_count FROM pg_stat_replication_slots; DROP FUNCTION wait_for_decode_stats(bool); DROP TABLE stats_test; diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 66566765f0c9..f5cf163c8c68 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -2590,7 +2590,7 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i - name text + slot_name text A unique, cluster-wide identifier for the replication slot diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index c29390760fe4..85cd147e21bb 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -798,7 +798,7 @@ CREATE VIEW pg_stat_replication AS CREATE VIEW pg_stat_replication_slots AS SELECT - s.name, + s.slot_name, s.spill_txns, s.spill_count, s.spill_bytes, diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 0d0d2e6d2bb2..472fa596e1f8 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2153,7 +2153,7 @@ pg_stat_get_archiver(PG_FUNCTION_ARGS) Datum pg_stat_get_replication_slots(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_REPLICATION_SLOT_CLOS 5 +#define PG_STAT_GET_REPLICATION_SLOT_COLS 5 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; TupleDesc tupdesc; Tuplestorestate *tupstore; @@ -2190,8 +2190,8 @@ pg_stat_get_replication_slots(PG_FUNCTION_ARGS) slotstats = pgstat_fetch_replslot(&nstats); for (i = 0; i < nstats; i++) { - Datum values[PG_STAT_GET_REPLICATION_SLOT_CLOS]; - bool nulls[PG_STAT_GET_REPLICATION_SLOT_CLOS]; + Datum values[PG_STAT_GET_REPLICATION_SLOT_COLS]; + bool nulls[PG_STAT_GET_REPLICATION_SLOT_COLS]; PgStat_ReplSlotStats *s = &(slotstats[i]); MemSet(values, 0, sizeof(values)); diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 584c61728439..f44a09b0c251 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 202010081 +#define CATALOG_VERSION_NO 202010201 #endif diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 22340baf1c67..bbcac69d48f7 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5263,7 +5263,7 @@ prorettype => 'record', proargtypes => '', proallargtypes => '{text,int8,int8,int8,timestamptz}', proargmodes => '{o,o,o,o,o}', - proargnames => '{name,spill_txns,spill_count,spill_bytes,stats_reset}', + proargnames => '{slot_name,spill_txns,spill_count,spill_bytes,stats_reset}', prosrc => 'pg_stat_get_replication_slots' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', proisstrict => 'f', provolatile => 's', diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index cf2a9b44082e..492cdcf74c36 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2018,12 +2018,12 @@ pg_stat_replication| SELECT s.pid, FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, sslcompression, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, leader_pid) JOIN pg_stat_get_wal_senders() w(pid, state, sent_lsn, write_lsn, flush_lsn, replay_lsn, write_lag, flush_lag, replay_lag, sync_priority, sync_state, reply_time) ON ((s.pid = w.pid))) LEFT JOIN pg_authid u ON ((s.usesysid = u.oid))); -pg_stat_replication_slots| SELECT s.name, +pg_stat_replication_slots| SELECT s.slot_name, s.spill_txns, s.spill_count, s.spill_bytes, s.stats_reset - FROM pg_stat_get_replication_slots() s(name, spill_txns, spill_count, spill_bytes, stats_reset); + FROM pg_stat_get_replication_slots() s(slot_name, spill_txns, spill_count, spill_bytes, stats_reset); pg_stat_slru| SELECT s.name, s.blks_zeroed, s.blks_hit, From bbb927b4db9b3b449ccd0f76c1296de382a2f0c1 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Tue, 20 Oct 2020 19:22:09 -0300 Subject: [PATCH 343/589] Fix ALTER TABLE .. ENABLE/DISABLE TRIGGER recursion More precisely, correctly handle the ONLY flag indicating not to recurse. This was implemented in 86f575948c77 by recursing in trigger.c, but that's the wrong place; use ATSimpleRecursion instead, which behaves properly. However, because legacy inheritance has never recursed in that situation, make sure to do that only for new-style partitioning. I noticed this problem while testing a fix for another bug in the vicinity. This has been wrong all along, so backpatch to 11. Discussion: https://postgr.es/m/20201016235925.GA29829@alvherre.pgsql --- src/backend/commands/tablecmds.c | 2 + src/backend/commands/trigger.c | 21 ---------- src/test/regress/expected/triggers.out | 56 ++++++++++++++++++++++++++ src/test/regress/sql/triggers.sql | 35 ++++++++++++++++ 4 files changed, 93 insertions(+), 21 deletions(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 511f015a861c..a29c14bf1cf0 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -4321,6 +4321,8 @@ ATPrepCmd(List **wqueue, Relation rel, AlterTableCmd *cmd, case AT_DisableTrigAll: case AT_DisableTrigUser: ATSimplePermissions(rel, ATT_TABLE | ATT_FOREIGN_TABLE); + if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + ATSimpleRecursion(wqueue, rel, cmd, recurse, lockmode, context); pass = AT_PASS_MISC; break; case AT_EnableRule: /* ENABLE/DISABLE RULE variants */ diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c index 5719672f4ce1..28b98d10ae85 100644 --- a/src/backend/commands/trigger.c +++ b/src/backend/commands/trigger.c @@ -1531,27 +1531,6 @@ EnableDisableTrigger(Relation rel, const char *tgname, heap_freetuple(newtup); - /* - * When altering FOR EACH ROW triggers on a partitioned table, do - * the same on the partitions as well. - */ - if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE && - (TRIGGER_FOR_ROW(oldtrig->tgtype))) - { - PartitionDesc partdesc = RelationGetPartitionDesc(rel); - int i; - - for (i = 0; i < partdesc->nparts; i++) - { - Relation part; - - part = relation_open(partdesc->oids[i], lockmode); - EnableDisableTrigger(part, NameStr(oldtrig->tgname), - fires_when, skip_system, lockmode); - table_close(part, NoLock); /* keep lock till commit */ - } - } - changed = true; } diff --git a/src/test/regress/expected/triggers.out b/src/test/regress/expected/triggers.out index 5e76b3a47e7d..57efa290207a 100644 --- a/src/test/regress/expected/triggers.out +++ b/src/test/regress/expected/triggers.out @@ -2512,6 +2512,62 @@ select tgrelid::regclass, count(*) from pg_trigger (5 rows) drop table trg_clone; +-- Test the interaction between ALTER TABLE .. DISABLE TRIGGER and +-- both kinds of inheritance. Historically, legacy inheritance has +-- not recursed to children, so that behavior is preserved. +create table parent (a int); +create table child1 () inherits (parent); +create function trig_nothing() returns trigger language plpgsql + as $$ begin return null; end $$; +create trigger tg after insert on parent + for each row execute function trig_nothing(); +create trigger tg after insert on child1 + for each row execute function trig_nothing(); +alter table parent disable trigger tg; +select tgrelid::regclass, tgname, tgenabled from pg_trigger + where tgrelid in ('parent'::regclass, 'child1'::regclass) + order by tgrelid::regclass::text; + tgrelid | tgname | tgenabled +---------+--------+----------- + child1 | tg | O + parent | tg | D +(2 rows) + +alter table only parent enable always trigger tg; +select tgrelid::regclass, tgname, tgenabled from pg_trigger + where tgrelid in ('parent'::regclass, 'child1'::regclass) + order by tgrelid::regclass::text; + tgrelid | tgname | tgenabled +---------+--------+----------- + child1 | tg | O + parent | tg | A +(2 rows) + +drop table parent, child1; +create table parent (a int) partition by list (a); +create table child1 partition of parent for values in (1); +create trigger tg after insert on parent + for each row execute procedure trig_nothing(); +select tgrelid::regclass, tgname, tgenabled from pg_trigger + where tgrelid in ('parent'::regclass, 'child1'::regclass) + order by tgrelid::regclass::text; + tgrelid | tgname | tgenabled +---------+--------+----------- + child1 | tg | O + parent | tg | O +(2 rows) + +alter table only parent enable always trigger tg; +select tgrelid::regclass, tgname, tgenabled from pg_trigger + where tgrelid in ('parent'::regclass, 'child1'::regclass) + order by tgrelid::regclass::text; + tgrelid | tgname | tgenabled +---------+--------+----------- + child1 | tg | O + parent | tg | A +(2 rows) + +drop table parent, child1; -- -- Test the interaction between transition tables and both kinds of -- inheritance. We'll dump the contents of the transition tables in a diff --git a/src/test/regress/sql/triggers.sql b/src/test/regress/sql/triggers.sql index e228d0a8a5b6..8f66df9f3b41 100644 --- a/src/test/regress/sql/triggers.sql +++ b/src/test/regress/sql/triggers.sql @@ -1749,6 +1749,41 @@ select tgrelid::regclass, count(*) from pg_trigger group by tgrelid::regclass order by tgrelid::regclass; drop table trg_clone; +-- Test the interaction between ALTER TABLE .. DISABLE TRIGGER and +-- both kinds of inheritance. Historically, legacy inheritance has +-- not recursed to children, so that behavior is preserved. +create table parent (a int); +create table child1 () inherits (parent); +create function trig_nothing() returns trigger language plpgsql + as $$ begin return null; end $$; +create trigger tg after insert on parent + for each row execute function trig_nothing(); +create trigger tg after insert on child1 + for each row execute function trig_nothing(); +alter table parent disable trigger tg; +select tgrelid::regclass, tgname, tgenabled from pg_trigger + where tgrelid in ('parent'::regclass, 'child1'::regclass) + order by tgrelid::regclass::text; +alter table only parent enable always trigger tg; +select tgrelid::regclass, tgname, tgenabled from pg_trigger + where tgrelid in ('parent'::regclass, 'child1'::regclass) + order by tgrelid::regclass::text; +drop table parent, child1; + +create table parent (a int) partition by list (a); +create table child1 partition of parent for values in (1); +create trigger tg after insert on parent + for each row execute procedure trig_nothing(); +select tgrelid::regclass, tgname, tgenabled from pg_trigger + where tgrelid in ('parent'::regclass, 'child1'::regclass) + order by tgrelid::regclass::text; +alter table only parent enable always trigger tg; +select tgrelid::regclass, tgname, tgenabled from pg_trigger + where tgrelid in ('parent'::regclass, 'child1'::regclass) + order by tgrelid::regclass::text; +drop table parent, child1; + + -- -- Test the interaction between transition tables and both kinds of -- inheritance. We'll dump the contents of the transition tables in a From 19ae53c92d5f5bdfb971d560a562e84c5f65c8b0 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 21 Oct 2020 09:22:27 +0900 Subject: [PATCH 344/589] Review format of code generated by PerfectHash.pm 80f8eb7 has added to the normalization quick check headers some code generated by PerfectHash.pm that is incompatible with the settings of gitattributes for this repository, as whitespaces followed a set of tabs for the first element of a line in the table. Instead of adding a new exception to gitattributes, rework the format generated so as a right padding with spaces is used instead of a left padding. This keeps the table generated in a readable shape with its set of columns, making unnecessary an update of gitattributes. Reported-by: Peter Eisentraut Author: John Naylor Discussion: https://postgr.es/m/d601b3b5-a3c7-5457-2f84-3d6513d690fc@2ndquadrant.com --- src/include/common/unicode_normprops_table.h | 3076 +++++++++--------- src/tools/PerfectHash.pm | 13 +- 2 files changed, 1546 insertions(+), 1543 deletions(-) diff --git a/src/include/common/unicode_normprops_table.h b/src/include/common/unicode_normprops_table.h index 2ae13d847f2a..8c310f10d79c 100644 --- a/src/include/common/unicode_normprops_table.h +++ b/src/include/common/unicode_normprops_table.h @@ -1262,314 +1262,314 @@ static int NFC_QC_hash_func(const void *key) { static const int16 h[2463] = { - 0, -2717, 0, 221, 1293, 223, 1295, 225, - 226, 241, 0, 229, 230, 231, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - -386, 0, 0, 0, 0, 0, 0, 0, - -163, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - -246, -175, 1260, 0, 0, 0, -174, -173, - 0, -172, 0, 0, 0, 0, 0, 0, - 1049, 0, 300, 301, 1071, 0, 1071, 0, - 1071, 1071, 1057, 0, 0, 0, 0, 1061, - 0, -1053, 1664, 0, 2956, 0, 0, -13, - 0, 0, 0, 0, 2156, 0, 0, 0, - 0, 0, 0, 0, 71, 0, 1082, 0, - 1083, 1083, 0, 1084, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 359, 360, 361, - -1091, 363, -762, -130, -129, -128, -127, -126, - 137, -124, -708, -707, -706, -120, -185, -705, - -117, -184, -1307, -114, -113, -112, -111, 0, - 386, 387, 388, 389, -90, 391, 171, 172, - 394, -94, -183, 397, 398, 399, -98, -225, - 402, -1019, -636, -1019, -225, 407, 408, 409, - 410, 411, 674, 413, -171, -170, -169, 417, - 352, -168, 420, 353, -770, 423, 424, 425, - 426, 427, 428, 32767, 239, 239, 239, 239, - 239, 239, 239, 239, 239, 239, 239, 239, - 239, 239, 32767, 32767, 237, 32767, 236, 32767, - 32767, 234, 234, 234, 234, 617, 234, 234, - 234, -2483, 234, -1430, 1526, -1430, 1527, 47, - 48, 471, 230, 32767, 32767, 32767, 227, 227, - 227, 227, 227, 227, 227, 227, 227, 227, - 227, 227, 227, 227, 227, 227, 227, 227, - -159, 227, 227, 227, 227, 227, 227, 227, - 64, 227, 227, 227, 227, 227, 227, 227, - 227, 227, 227, 227, 227, 227, 227, 227, - 227, 227, 227, 227, 227, 227, 227, 227, - -19, 52, 1487, 227, 227, 227, 53, 54, - 227, 55, 227, 227, 227, 227, 227, 227, - 1276, 227, -989, 32767, 1296, 225, 1296, 225, - 1296, 1296, 1282, 225, 225, 225, 225, 1286, - 225, -828, 1889, 225, 3181, 225, 225, 212, - 225, 225, 225, 225, 2381, 225, 225, 225, - 225, 225, 225, 225, 296, 225, 1307, 225, - 1308, 1308, 225, 1309, 225, 225, 225, 225, - 225, 225, 225, 225, 225, 225, 225, 225, - 225, 225, 225, 225, 225, 584, 585, 586, - -866, 588, -537, 95, 96, 97, 98, 99, - 362, 101, -483, -482, -481, 105, 40, -480, - 108, 41, -1082, 111, 112, 113, 114, 225, - 611, 612, 613, 614, 135, 616, 396, 397, - 619, 131, 42, 622, 623, 624, 127, 0, - 627, -794, -411, -794, 0, 632, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - -272, 32767, 32767, 32767, 0, 32767, 32767, 32767, - 32767, 32767, -166, -165, 32767, 32767, 32767, 32767, - -164, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 397, 32767, 396, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 386, - 0, 386, 386, 386, 386, 386, 386, 386, - 223, 386, 386, 386, 32767, 385, 385, 385, - 385, 385, 32767, 384, 32767, 383, 383, 32767, - 382, 382, 32767, 381, 381, 381, 381, 381, - 135, 206, 1641, 381, 32767, 32767, 32767, 32767, - 32767, 32767, -160, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 1148, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 0, - 32767, 32767, 32767, 0, 0, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, -257, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, -910, -910, 32767, 32767, - 0, 32767, 0, 32767, 0, 32767, 0, 32767, - 147, 32767, 0, 32767, 0, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 0, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 143, 32767, 144, 32767, 145, - 32767, 146, 32767, 0, 32767, 148, 32767, 149, - 32767, 32767, 32767, -160, 32767, 32767, 32767, 32767, - 32767, 32767, 15, 32767, 32767, 0, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 145, 32767, 144, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 0, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 0, -148, 32767, 32767, 32767, 32767, - 32767, 32767, 2009, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 0, 32767, 32767, 135, -918, 32767, - 151, 32767, 32767, 0, 1, 2, 3, 4, - 133, 5, 6, 7, 8, 9, 10, 11, - 32767, 32767, -1248, 32767, 13, 154, 188, 188, - 32767, 32767, 32767, 32767, 32767, 155, 16, 32767, - 32767, 32767, 32767, 32767, 32767, -1853, -1054, 18, - -1052, -1051, -1036, 22, 32767, 157, 32767, 28, - 23, 1077, 673, 25, -2930, 0, 32767, 32767, - 32767, 32767, 32767, 27, 32767, 155, 32767, 154, - 32767, 32767, -62, 28, -42, 30, -1051, 32, - -1050, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 34, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 129, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 672, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 0, 32767, - 32767, 32767, 32767, 32767, -156, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, -155, 32767, 32767, - 32767, 0, 0, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 73, 32767, 32767, 32767, 32767, 74, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 675, - 32767, 32767, 32767, 32767, 32767, 75, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 165, 32767, 32767, 32767, 166, 167, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 170, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 689, 690, 691, 692, 693, 694, 695, - 696, 697, 698, 699, 700, 701, 702, 703, - 704, 705, 706, 707, 708, 709, 710, 711, - 712, 713, 714, 715, 716, 717, 718, 719, - 720, 721, 722, -304, -303, -302, -301, -300, - -299, -298, -297, 930, -295, -294, -293, -292, - -291, -290, -289, -288, -287, -286, -285, -284, - -283, -282, -281, -280, -279, -278, -277, -276, - -275, 753, 754, 755, 646, 757, -712, -1765, - 952, -712, 2244, -712, 2245, 765, 766, 767, - 768, 125, 770, 771, 772, 773, 774, 775, - 603, 777, 778, 779, 780, 781, 782, 783, - 784, 2011, 786, 787, 788, 789, 790, 791, - 792, 793, 794, 795, 796, 797, 798, 799, - 800, 801, 802, 803, 804, 805, 806, 603, - 603, 809, 603, 811, 603, 603, 814, 815, - 816, 817, 435, 819, 820, 821, 3539, 823, - 603, -468, 603, -468, 603, 603, 589, 831, - 603, 603, 603, 835, 836, 837, 838, 839, - 840, 841, 842, 843, 844, 845, 846, 847, - 848, 849, 850, 851, 852, 1239, 854, 855, - 856, 857, 858, 859, 860, 1024, 862, 863, - 864, 865, 866, 867, 868, 869, 870, 871, - 872, 873, 874, 875, 876, 877, 878, 879, - 880, 881, 882, 883, 884, 1131, 1061, -373, - 888, 889, 890, 1065, 1065, 893, 1066, 895, - 896, 897, 898, 899, 900, -148, 902, 603, - 603, -166, 906, -164, 908, -162, -161, -146, - 912, 913, 914, 915, -145, 917, 1971, -745, - 920, -2035, 922, 923, 937, 925, 926, 927, - 928, -1227, 930, 931, 932, 933, 934, 935, - 936, 866, 938, -143, 940, -142, -141, 943, - -140, 32767, 945, 946, 947, 948, 949, 950, - 951, 952, 953, 954, 955, 956, 957, 958, - 959, 960, 961, -65, -64, -63, -62, -61, - -60, -59, -58, 1169, -56, -55, -54, -53, - -52, -51, -50, -49, -48, -47, -46, -45, - -44, -43, -42, -41, -40, -39, -38, -37, - -36, 992, 993, 994, 885, 996, -473, -1526, - 1191, -473, 2483, -473, 2484, 1004, 1005, 1006, - 1007, 364, 1009, 1010, 1011, 1012, 1013, 1014, - 842, 1016, 1017, 1018, 1019, 1020, 1021, 1022, - 1023, 2250, 1025, 1026, 1027, 1028, 1029, 1030, - 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, - 1039, 1040, 1041, 1042, 1043, 1044, 1045, 842, - 842, 1048, 842, 1050, 842, 842, 1053, 1054, - 1055, 1056, 674, 1058, 1059, 1060, 3778, 1062, - 842, -229, 842, -229, 842, 842, 828, 1070, - 842, 842, 842, 1074, 1075, 1076, 1077, 1078, - 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, - 1087, 1088, 1089, 1090, 1091, 1478, 1093, 1094, - 1095, 1096, 1097, 1098, 1099, 1263, 1101, 1102, - 1103, 1104, 1105, 1106, 1107, 1108, 1109, 1110, - 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, - 1119, 1120, 1121, 1122, 1123, 1370, 1300, -134, - 1127, 1128, 1129, 1304, 1304, 1132, 1305, 1134, - 1135, 1136, 1137, 1138, 1139, 91, 1141, 842, - 842, 73, 1145, 75, 1147, 77, 78, 93, - 1151, 1152, 1153, 1154, 94, 1156, 2210, -506, - 1159, -1796, 1161, 1162, 1176, 1164, 1165, 1166, - 1167, -988, 1169, 1170, 1171, 1172, 1173, 1174, - 1175, 1105, 1177, 96, 1179, 97, 98, 1182, - 99, 1184, 1185, 1186, 1187, 1188, 1189, 1190, - 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, - 1199, 1200, 0, 174, 175, 176, 177, 178, - 179, 180, 181, 1408, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 194, - 195, 196, 197, 198, 199, 200, 201, 202, - 203, 0, 0, 206, 0, 208, 0, 0, - 211, 212, 213, 214, -168, 216, 217, 218, - 2936, 220, 0, -1071, 0, -1071, 0, 0, - -14, 228, 0, 0, 0, 232, 233, 234, - 235, 236, 237, 238, 239, 240, 241, 242, - 243, 244, 245, 246, 247, 248, 249, 636, - 251, 252, 253, 254, 255, 256, 257, 421, - 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, 273, 274, - 275, 276, 277, 278, 279, 280, 281, 528, - 458, -976, 285, 286, 287, 462, 462, 290, - 463, 292, 293, 294, 295, 296, 297, -751, - 299, 0, 0, -769, 303, -767, 305, -765, - -764, -749, 309, 310, 311, 312, -748, 314, - 1368, -1348, 317, -2638, 319, 320, 334, 322, - 323, 324, 325, -1830, 327, 328, 329, 330, - 331, 332, 333, 263, 335, -746, 337, -745, - -744, 340, -743, 342, 343, 344, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, - 355, 356, 357, 358, 0, 0, 0, 1453, - 0, 1126, 495, 495, 495, 495, 495, 233, - 495, 1080, 1080, 1080, 495, 561, 1082, 495, - 563, 1687, 495, 495, 495, 495, 385, 0, - 0, 0, 0, 480, 0, 221, 221, 0, - 489, 579, 0, 0, 0, 498, 626, 0, - 1422, 1040, 1424, 631, 0, 0, 0, 0, - 0, -262, 0, 585, 585, 585, 0, 66, - 587, 0, 68, 1192, 0, 0, 0, 0, - 0, 0, 32767, 32767, 32767, 32767, 669, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 670, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 142, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 115, 116, 117, 118, 119, 120, - 121, 122, 123, 124, 125, 126, 127, 128, - 129, 130, 131, 132, 133, 134, 135, 136, - 137, 138, 139, 140, 141, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1027, 1027, 1027, - 1027, 1027, 1027, 1027, 1027, -199, 1027, 1027, - 1027, 1027, 1027, 1027, 1027, 1027, 1027, 1027, - 1027, 1027, 1027, 1027, 1027, 1027, 1027, 1027, - 1027, 1027, 1027, 0, 0, 0, 110, 0, - 1470, 2524, -192, 1473, -1482, 1475, -1481, 0, - 0, 0, 0, 644, 0, 0, 0, 0, - 0, 0, 173, 0, 0, 0, 0, 0, - 0, 0, 0, -1226, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 204, 205, 0, 207, 0, 209, 210, - 0, 0, 0, 0, 383, 0, 0, + 0, -2717, 0, 221, 1293, 223, 1295, 225, + 226, 241, 0, 229, 230, 231, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + -386, 0, 0, 0, 0, 0, 0, 0, + -163, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + -246, -175, 1260, 0, 0, 0, -174, -173, + 0, -172, 0, 0, 0, 0, 0, 0, + 1049, 0, 300, 301, 1071, 0, 1071, 0, + 1071, 1071, 1057, 0, 0, 0, 0, 1061, + 0, -1053, 1664, 0, 2956, 0, 0, -13, + 0, 0, 0, 0, 2156, 0, 0, 0, + 0, 0, 0, 0, 71, 0, 1082, 0, + 1083, 1083, 0, 1084, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 359, 360, 361, + -1091, 363, -762, -130, -129, -128, -127, -126, + 137, -124, -708, -707, -706, -120, -185, -705, + -117, -184, -1307, -114, -113, -112, -111, 0, + 386, 387, 388, 389, -90, 391, 171, 172, + 394, -94, -183, 397, 398, 399, -98, -225, + 402, -1019, -636, -1019, -225, 407, 408, 409, + 410, 411, 674, 413, -171, -170, -169, 417, + 352, -168, 420, 353, -770, 423, 424, 425, + 426, 427, 428, 32767, 239, 239, 239, 239, + 239, 239, 239, 239, 239, 239, 239, 239, + 239, 239, 32767, 32767, 237, 32767, 236, 32767, + 32767, 234, 234, 234, 234, 617, 234, 234, + 234, -2483, 234, -1430, 1526, -1430, 1527, 47, + 48, 471, 230, 32767, 32767, 32767, 227, 227, + 227, 227, 227, 227, 227, 227, 227, 227, + 227, 227, 227, 227, 227, 227, 227, 227, + -159, 227, 227, 227, 227, 227, 227, 227, + 64, 227, 227, 227, 227, 227, 227, 227, + 227, 227, 227, 227, 227, 227, 227, 227, + 227, 227, 227, 227, 227, 227, 227, 227, + -19, 52, 1487, 227, 227, 227, 53, 54, + 227, 55, 227, 227, 227, 227, 227, 227, + 1276, 227, -989, 32767, 1296, 225, 1296, 225, + 1296, 1296, 1282, 225, 225, 225, 225, 1286, + 225, -828, 1889, 225, 3181, 225, 225, 212, + 225, 225, 225, 225, 2381, 225, 225, 225, + 225, 225, 225, 225, 296, 225, 1307, 225, + 1308, 1308, 225, 1309, 225, 225, 225, 225, + 225, 225, 225, 225, 225, 225, 225, 225, + 225, 225, 225, 225, 225, 584, 585, 586, + -866, 588, -537, 95, 96, 97, 98, 99, + 362, 101, -483, -482, -481, 105, 40, -480, + 108, 41, -1082, 111, 112, 113, 114, 225, + 611, 612, 613, 614, 135, 616, 396, 397, + 619, 131, 42, 622, 623, 624, 127, 0, + 627, -794, -411, -794, 0, 632, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + -272, 32767, 32767, 32767, 0, 32767, 32767, 32767, + 32767, 32767, -166, -165, 32767, 32767, 32767, 32767, + -164, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 397, 32767, 396, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 386, + 0, 386, 386, 386, 386, 386, 386, 386, + 223, 386, 386, 386, 32767, 385, 385, 385, + 385, 385, 32767, 384, 32767, 383, 383, 32767, + 382, 382, 32767, 381, 381, 381, 381, 381, + 135, 206, 1641, 381, 32767, 32767, 32767, 32767, + 32767, 32767, -160, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 1148, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 0, + 32767, 32767, 32767, 0, 0, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, -257, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, -910, -910, 32767, 32767, + 0, 32767, 0, 32767, 0, 32767, 0, 32767, + 147, 32767, 0, 32767, 0, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 0, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 143, 32767, 144, 32767, 145, + 32767, 146, 32767, 0, 32767, 148, 32767, 149, + 32767, 32767, 32767, -160, 32767, 32767, 32767, 32767, + 32767, 32767, 15, 32767, 32767, 0, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 145, 32767, 144, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 0, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 0, -148, 32767, 32767, 32767, 32767, + 32767, 32767, 2009, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 0, 32767, 32767, 135, -918, 32767, + 151, 32767, 32767, 0, 1, 2, 3, 4, + 133, 5, 6, 7, 8, 9, 10, 11, + 32767, 32767, -1248, 32767, 13, 154, 188, 188, + 32767, 32767, 32767, 32767, 32767, 155, 16, 32767, + 32767, 32767, 32767, 32767, 32767, -1853, -1054, 18, + -1052, -1051, -1036, 22, 32767, 157, 32767, 28, + 23, 1077, 673, 25, -2930, 0, 32767, 32767, + 32767, 32767, 32767, 27, 32767, 155, 32767, 154, + 32767, 32767, -62, 28, -42, 30, -1051, 32, + -1050, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 34, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 129, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 672, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 0, 32767, + 32767, 32767, 32767, 32767, -156, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, -155, 32767, 32767, + 32767, 0, 0, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 73, 32767, 32767, 32767, 32767, 74, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 675, + 32767, 32767, 32767, 32767, 32767, 75, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 165, 32767, 32767, 32767, 166, 167, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 170, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 689, 690, 691, 692, 693, 694, 695, + 696, 697, 698, 699, 700, 701, 702, 703, + 704, 705, 706, 707, 708, 709, 710, 711, + 712, 713, 714, 715, 716, 717, 718, 719, + 720, 721, 722, -304, -303, -302, -301, -300, + -299, -298, -297, 930, -295, -294, -293, -292, + -291, -290, -289, -288, -287, -286, -285, -284, + -283, -282, -281, -280, -279, -278, -277, -276, + -275, 753, 754, 755, 646, 757, -712, -1765, + 952, -712, 2244, -712, 2245, 765, 766, 767, + 768, 125, 770, 771, 772, 773, 774, 775, + 603, 777, 778, 779, 780, 781, 782, 783, + 784, 2011, 786, 787, 788, 789, 790, 791, + 792, 793, 794, 795, 796, 797, 798, 799, + 800, 801, 802, 803, 804, 805, 806, 603, + 603, 809, 603, 811, 603, 603, 814, 815, + 816, 817, 435, 819, 820, 821, 3539, 823, + 603, -468, 603, -468, 603, 603, 589, 831, + 603, 603, 603, 835, 836, 837, 838, 839, + 840, 841, 842, 843, 844, 845, 846, 847, + 848, 849, 850, 851, 852, 1239, 854, 855, + 856, 857, 858, 859, 860, 1024, 862, 863, + 864, 865, 866, 867, 868, 869, 870, 871, + 872, 873, 874, 875, 876, 877, 878, 879, + 880, 881, 882, 883, 884, 1131, 1061, -373, + 888, 889, 890, 1065, 1065, 893, 1066, 895, + 896, 897, 898, 899, 900, -148, 902, 603, + 603, -166, 906, -164, 908, -162, -161, -146, + 912, 913, 914, 915, -145, 917, 1971, -745, + 920, -2035, 922, 923, 937, 925, 926, 927, + 928, -1227, 930, 931, 932, 933, 934, 935, + 936, 866, 938, -143, 940, -142, -141, 943, + -140, 32767, 945, 946, 947, 948, 949, 950, + 951, 952, 953, 954, 955, 956, 957, 958, + 959, 960, 961, -65, -64, -63, -62, -61, + -60, -59, -58, 1169, -56, -55, -54, -53, + -52, -51, -50, -49, -48, -47, -46, -45, + -44, -43, -42, -41, -40, -39, -38, -37, + -36, 992, 993, 994, 885, 996, -473, -1526, + 1191, -473, 2483, -473, 2484, 1004, 1005, 1006, + 1007, 364, 1009, 1010, 1011, 1012, 1013, 1014, + 842, 1016, 1017, 1018, 1019, 1020, 1021, 1022, + 1023, 2250, 1025, 1026, 1027, 1028, 1029, 1030, + 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, + 1039, 1040, 1041, 1042, 1043, 1044, 1045, 842, + 842, 1048, 842, 1050, 842, 842, 1053, 1054, + 1055, 1056, 674, 1058, 1059, 1060, 3778, 1062, + 842, -229, 842, -229, 842, 842, 828, 1070, + 842, 842, 842, 1074, 1075, 1076, 1077, 1078, + 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, + 1087, 1088, 1089, 1090, 1091, 1478, 1093, 1094, + 1095, 1096, 1097, 1098, 1099, 1263, 1101, 1102, + 1103, 1104, 1105, 1106, 1107, 1108, 1109, 1110, + 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, + 1119, 1120, 1121, 1122, 1123, 1370, 1300, -134, + 1127, 1128, 1129, 1304, 1304, 1132, 1305, 1134, + 1135, 1136, 1137, 1138, 1139, 91, 1141, 842, + 842, 73, 1145, 75, 1147, 77, 78, 93, + 1151, 1152, 1153, 1154, 94, 1156, 2210, -506, + 1159, -1796, 1161, 1162, 1176, 1164, 1165, 1166, + 1167, -988, 1169, 1170, 1171, 1172, 1173, 1174, + 1175, 1105, 1177, 96, 1179, 97, 98, 1182, + 99, 1184, 1185, 1186, 1187, 1188, 1189, 1190, + 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, + 1199, 1200, 0, 174, 175, 176, 177, 178, + 179, 180, 181, 1408, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 198, 199, 200, 201, 202, + 203, 0, 0, 206, 0, 208, 0, 0, + 211, 212, 213, 214, -168, 216, 217, 218, + 2936, 220, 0, -1071, 0, -1071, 0, 0, + -14, 228, 0, 0, 0, 232, 233, 234, + 235, 236, 237, 238, 239, 240, 241, 242, + 243, 244, 245, 246, 247, 248, 249, 636, + 251, 252, 253, 254, 255, 256, 257, 421, + 259, 260, 261, 262, 263, 264, 265, 266, + 267, 268, 269, 270, 271, 272, 273, 274, + 275, 276, 277, 278, 279, 280, 281, 528, + 458, -976, 285, 286, 287, 462, 462, 290, + 463, 292, 293, 294, 295, 296, 297, -751, + 299, 0, 0, -769, 303, -767, 305, -765, + -764, -749, 309, 310, 311, 312, -748, 314, + 1368, -1348, 317, -2638, 319, 320, 334, 322, + 323, 324, 325, -1830, 327, 328, 329, 330, + 331, 332, 333, 263, 335, -746, 337, -745, + -744, 340, -743, 342, 343, 344, 345, 346, + 347, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 0, 0, 0, 1453, + 0, 1126, 495, 495, 495, 495, 495, 233, + 495, 1080, 1080, 1080, 495, 561, 1082, 495, + 563, 1687, 495, 495, 495, 495, 385, 0, + 0, 0, 0, 480, 0, 221, 221, 0, + 489, 579, 0, 0, 0, 498, 626, 0, + 1422, 1040, 1424, 631, 0, 0, 0, 0, + 0, -262, 0, 585, 585, 585, 0, 66, + 587, 0, 68, 1192, 0, 0, 0, 0, + 0, 0, 32767, 32767, 32767, 32767, 669, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 670, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 142, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 115, 116, 117, 118, 119, 120, + 121, 122, 123, 124, 125, 126, 127, 128, + 129, 130, 131, 132, 133, 134, 135, 136, + 137, 138, 139, 140, 141, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1027, 1027, 1027, + 1027, 1027, 1027, 1027, 1027, -199, 1027, 1027, + 1027, 1027, 1027, 1027, 1027, 1027, 1027, 1027, + 1027, 1027, 1027, 1027, 1027, 1027, 1027, 1027, + 1027, 1027, 1027, 0, 0, 0, 110, 0, + 1470, 2524, -192, 1473, -1482, 1475, -1481, 0, + 0, 0, 0, 644, 0, 0, 0, 0, + 0, 0, 173, 0, 0, 0, 0, 0, + 0, 0, 0, -1226, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 204, 205, 0, 207, 0, 209, 210, + 0, 0, 0, 0, 383, 0, 0 }; const unsigned char *k = (const unsigned char *) key; @@ -6520,1236 +6520,1236 @@ static int NFKC_QC_hash_func(const void *key) { static const int16 h[9837] = { - -2472, -2472, -2472, -2472, -2472, -2472, -2472, -2472, - -2472, -2472, -2472, -2472, -2472, -2472, -2472, -2472, - -2472, -2472, -2472, -2472, -2472, -2472, -2472, -2472, - -2472, -2472, -2472, -2472, -2472, 32767, 32767, 32767, - -2475, -2475, -2475, -2475, -2475, -2475, -2475, -2475, - -2475, -2475, -2475, -2475, -2475, -2475, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 865, 865, 865, 865, 865, 865, 865, - 865, 865, 865, 865, -2255, 32767, -5207, 32767, - -5207, 860, 860, 860, 860, 860, 860, 860, - 860, 860, 4250, 861, 861, 861, 3339, 3339, - 3339, 3339, 3339, 3339, 3339, 3339, 3339, 3339, - 3339, 3339, 3339, 3339, 3339, 3339, 3339, 3339, - 32767, 3338, 3338, 3338, 3338, 3338, 3338, 3338, - 3338, 3338, 3338, 3338, 3338, 3338, 3338, 3338, - 3338, 3338, 3338, 3338, 3338, 3338, 3338, 3338, - 3338, 3338, 3338, 3338, 3338, 3338, 3338, 3338, - 3338, 9, 10, 32767, 11, 12, 0, 32767, - 0, 2913, 2914, 2915, 2916, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 2917, 32767, 2918, -100, - 2919, 2920, 2921, 840, 840, 840, 2922, 0, - 0, 0, 0, 0, 2206, 0, 2923, 0, - 2924, 2925, 2926, 0, 0, 0, -2590, 0, - 0, 0, 0, 0, 0, 0, 2934, 0, - 2474, 2931, 2932, 0, 0, 0, 0, 0, - 14, 805, 0, 0, 2933, 0, 2934, 0, - 2935, 2936, 0, 0, 0, 16, 17, 0, - 0, 0, 0, 0, 0, 0, 0, 18, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, -790, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, -1675, 0, 0, 19, 0, -1679, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, -1694, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 20, 21, 22, 23, 24, 25, - 26, 27, 28, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 29, 30, 31, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 724, 2668, 724, 4350, -2633, -2633, - 2533, 2534, 2535, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 2518, 2519, 2520, 1431, 45, 46, - 32767, 32767, 47, 48, 49, 50, 51, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, -3011, 53, -1125, -3010, -3010, - 32767, -3334, -1123, -3011, 60, 61, 62, 63, - 32767, 32767, 64, 32767, 65, 32767, 66, 67, - 32767, 32767, 32767, 32767, 32767, 32767, 2268, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 69, 70, - 71, 72, 73, 74, 32767, 32767, 32767, 32767, - 75, 76, 32767, 77, 281, 32767, 32767, 32767, - 32767, 32767, 32767, 811, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 1341, 1342, 1343, 1344, 1345, - 1346, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 86, - 32767, 32767, 32767, 32767, 32767, 4550, 32767, 32767, - 32767, 1135, 32767, 32767, 32767, 32767, 32767, 1130, - 3016, 32767, 3017, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 677, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 2858, 2859, 651, 2861, -438, - 2863, 2864, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, -5305, -5305, -5305, 32767, -5306, - -5306, 32767, 32767, 32767, 2871, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 3022, 3023, 680, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, -272, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 4308, 4309, 4310, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 4311, 4312, 4313, - 4314, 4315, 4316, 4317, 4318, 4319, 4320, 4321, - 4322, 4323, 4324, 4325, 4326, 4307, 4307, 4307, - 4307, 4307, 4307, 4307, 4307, 4307, 4336, 4337, - 4338, 4339, 4340, 4341, 4342, 4343, 4344, 4345, - 4346, 4347, 4348, 4349, 4350, 4351, 4352, 4353, - 4354, 32767, 32767, 32767, 32767, 4355, 4356, 4357, - 4358, 4359, 4360, 4361, 4362, 4363, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 4364, 4365, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 2202, 0, 0, 0, 59, 0, - 0, 35, 0, 0, 0, 3549, 0, 0, - 0, 0, 0, 3394, 0, 0, 3399, 0, - 0, 0, 0, 0, 0, 0, 0, 2012, - 0, 0, 0, 0, 87, 2022, 0, 7490, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 2255, 0, 2256, 2256, 2256, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 32767, 0, 0, - 0, 0, 0, 0, -1759, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 4767, 0, 0, 4772, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 32767, 5977, 0, - 892, 32767, 0, 32767, 32767, 0, 0, 32767, - 32767, 2344, 4834, 4835, 4836, 32767, 0, 4840, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 32767, 0, 32767, 0, 0, 0, - 0, 0, 0, 0, 32767, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 32767, 32767, 0, 32767, 0, 0, 0, 32767, - 32767, 32767, 32767, 3261, 3262, 32767, 3007, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 106, 107, 108, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 109, 110, 111, 112, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 0, 0, -2344, - -2344, 0, 32767, 0, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, -1642, 1469, -1641, 1469, -1640, 1469, - 1469, 1457, 1469, 1469, 1469, -4254, -4254, -4254, - -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4254, - -4254, -4254, -4254, -4254, -4254, -4254, -3359, -4254, - -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4254, - -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4254, - -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4254, - -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4254, - -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4103, - -1478, 0, -4254, -4254, -4254, -4254, -4254, -4254, - -4254, -4254, -4254, -2433, -4254, -4254, -4254, -3658, - -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4254, - -4254, -4254, 0, -4253, -4253, -4253, -4253, -4253, - -4253, -4253, -4253, -4253, -678, -677, -676, -675, - -674, -673, -672, -4253, 314, -4253, -4253, -4253, - -4253, -4253, -4253, -4253, -4253, -4253, -4253, -4253, - -4253, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 1464, 1465, 1466, 1467, - 1468, 1469, 0, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 0, - 0, 0, 0, 0, 32767, 32767, 32767, 32767, - 32767, 0, 32767, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 827, 828, 829, -2469, -2469, -260, 0, - 0, 32767, 0, 32767, 0, 0, 32767, 0, - 0, 32767, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 3575, 3576, 3577, 3578, 3579, 3580, 3581, 0, - 4567, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 2201, 4411, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, -3338, 0, 0, 0, - 0, 0, 0, 0, -3337, 0, -3336, 0, - 0, 0, 0, -3335, 0, 0, -3334, -3333, - -3332, -3331, 0, 0, -3330, 0, 0, 32767, - 0, 0, 13, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 3073, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - -2556, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 3074, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 2355, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, -488, -488, -488, -302, -3067, -3067, - -3067, -3067, -488, -488, -488, -488, 2999, -488, - 2999, -488, -488, -488, -3067, -3067, -3067, -488, - -488, -3067, -3067, -3067, -488, -488, -488, 2463, - -488, -488, -488, -301, 2465, -488, 2466, 2467, - -3600, -493, -3599, -488, -3598, -488, -3597, -488, - -488, -500, -488, -488, -488, -488, -488, 2470, - 2471, 2472, -488, -488, -254, -488, -488, -488, - -488, -488, -104, -488, -488, -488, -102, -101, - -100, -99, -98, -97, -96, -95, -94, -93, - -92, -488, -488, -488, -488, -488, -488, -488, - -488, -488, -2194, -2194, -2194, -2194, -2194, -2194, - -2194, -2194, -2194, -2194, 5211, 3269, 5213, 3269, - 6895, -88, -88, 5078, 5079, 5080, 1773, -92, - -92, 1773, 1773, 1773, 1773, 1773, 1773, 5072, - 5073, 2865, 5075, 1776, 5077, 5078, 1778, 1778, - 6942, 6943, 1778, 1778, 1778, 5086, 6952, 6953, - 5089, 5090, 5091, 5092, 5093, 5094, 5095, 5096, - 4007, 5098, 2333, 2334, 2335, 2336, 2337, -3066, - -3066, -3066, 2341, -3066, -3066, 2344, 2345, 2346, - 5114, 317, 2349, 848, 849, 850, 2353, 852, - 853, 854, 855, 856, 857, 858, 859, 860, - 861, 692, 692, 692, 692, 692, 692, 692, - 692, 692, 692, 692, 692, 692, 692, 692, - 692, 692, 692, 692, 692, 692, 692, 692, - 692, 692, 692, 692, 692, 692, 692, 692, - 692, 692, 692, 692, 692, 692, 692, 692, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 3093, 3094, 3095, 3096, 3097, 3098, 3099, - 3100, 3101, 3102, 901, 3104, 3105, 3106, 3048, - 3108, 3109, 3075, 3111, 3112, 3113, -435, 3115, - 3116, 3117, 3118, 3119, -274, 3121, 3122, -276, - 3124, 3125, 3126, 3127, 3128, 3129, 3130, 3131, - 1120, 3133, 3134, 3135, 3136, 3050, 1116, 3139, - -4350, 3141, 3142, 3143, 3144, 3145, 3146, 3147, - 3148, 3149, 3150, 3151, 3152, 3153, 3154, 3155, - 3156, 902, 3158, 903, 904, 905, 3162, 3163, - 3164, 3165, 3166, 3167, 3168, 3169, 3170, 3171, - 3172, 3173, 3174, 3175, 3176, 3177, 32767, 3178, - 3179, 3180, 3181, 3182, 3183, 4943, 3185, 3186, - 3187, 3188, 3189, 3190, 3191, 3192, 3193, 3194, - 3195, 3196, 3197, 3198, 3199, 3200, 3201, 3202, - 3203, 3204, 3205, 3206, 3207, 3208, 3209, 3210, - 3211, 3212, 3213, 3214, 3215, 3216, 3217, 3218, - 3219, 3220, 3221, 3222, 3223, -1543, 3225, 3226, - -1545, 3228, 3229, 3230, 3231, 3232, 3233, 3234, - 3235, 3236, 3237, 3238, 3239, 3240, 3241, 3242, - 3243, 3244, 3245, 3246, 3247, 3248, -1251, -2728, - 3250, 32767, 32767, 3251, 906, 907, 3252, 3253, - 32767, 32767, 910, -1579, -1579, -1579, 32767, 3258, - -1581, 3260, 3261, 3262, 3263, 3264, 3265, 3266, - 3267, 3268, 3269, 32767, 3270, 32767, 3271, 3272, - 3273, 3274, 3275, 3276, 3277, 32767, 3278, 3279, - 3280, 3281, 3282, 3283, 3284, 3285, 3286, 3287, - 3288, 3289, 3290, 3291, 3292, 3293, 3294, 3295, - 3296, 3297, 3298, 3299, 3300, 3301, 3302, 3303, - 3304, 3305, 3306, 3307, 3308, 3309, 3310, 3311, - 3312, 3313, 3314, 3315, 3316, 3317, 3318, 3319, - 3320, 3321, 3322, 3323, 3324, 3325, 3326, 3327, - 3328, 3329, 3330, 3331, 3332, 3333, 3334, 3335, - 3336, 32767, 3337, 3338, 3339, 3340, 3341, 3342, - 0, 3343, 3344, 3345, 3346, 32767, 32767, 3347, - 3348, 3349, 3350, 3351, 3352, 3353, 3354, 32767, - 3355, 3356, 3357, 3358, 3359, 3360, 3361, 32767, - 3362, 3363, 3364, 3365, 3366, 3367, 3368, 3369, - 3370, 3371, 3372, 3373, 3374, 3375, 3376, 3377, - 3378, 3379, 3380, 3381, 3382, 3383, 3384, 3385, - 3386, 3387, 3388, 3389, 0, 3390, 3391, 3392, - 915, 916, 917, 918, 919, 920, 921, 922, - 923, 924, 925, 926, 927, 928, 929, 930, - 931, 932, 933, 934, 935, 936, 937, 938, - 939, 940, 941, 942, 943, 944, 945, 946, - 947, 948, 949, 950, 951, 952, 953, 954, - 955, 956, 957, 958, 959, 960, 961, 962, - 963, 964, 965, 966, 967, 968, 969, 970, - 971, 972, 973, 974, 975, 976, 3449, 3450, - 3451, 3452, 3453, 3454, 3455, 3456, 3457, 3458, - 3459, 3460, 3461, 3462, 3463, 3464, 3465, 3466, - 3467, 3468, 3469, 3470, 3471, 3472, 3473, 3474, - 3475, 3476, 3477, 3478, 3479, 3480, 3481, 3482, - 3483, 3484, 3485, 3486, 3487, 3488, 3489, 3490, - 3491, 3492, 3493, 3494, 3495, 3496, 3497, 3498, - 3499, 3500, 3501, 3502, 3503, 3504, 3505, 3506, - 3507, 3508, 3509, 3510, 3511, 3512, 3513, 3514, - 3515, 3516, 3517, 3518, 3519, 3520, 3521, 3522, - 3523, 3524, 3525, 3526, 3527, 3528, 3529, 3530, - 3531, 3532, 3533, 3534, 3535, 3536, 3537, 3538, - 3539, 3540, 3541, 3542, 3543, 3544, 3545, 3546, - 3547, 3548, 3549, 3550, 3551, 3552, 3553, 3554, - 3555, 3556, 3557, 3558, 3559, 3560, 3561, 3562, - 3563, 3564, 3565, 3566, 3567, 3568, 3569, 3570, - 3571, 3572, 3573, 3574, 3575, 3576, 3577, 6056, - 6057, 6058, 32767, 3581, 3582, 3583, 3584, 3585, - 4157, 4158, 4159, 3589, 4162, -4510, -1558, -1557, - -1556, -1742, -4507, -1553, -4506, -4506, 1562, -1544, - 1563, -1547, 1564, -1545, 1565, -1543, -1542, -1529, - -1540, -1539, -1538, -1537, -1536, -4493, -4493, -4493, - -1532, -1531, -1764, -1529, 3622, -1528, -1527, -1526, - -1909, -1524, -1523, -1522, -1907, -1907, -1907, -1907, - -1907, -1907, -1907, -1907, -1907, -1907, -1907, -1510, - -1509, 1071, 1072, 1073, 1074, 1075, 1076, 1077, - 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, - 1086, 1087, 1088, 1089, 1090, 3663, 3664, 3665, - 3666, 3667, 3668, 3669, 3670, 3671, 3672, 3673, - 3674, 1095, 1096, 1097, 1098, 1099, 1100, 1101, - 3682, 1103, 3684, 1105, 3686, 3687, 3688, 1109, - 1110, 1111, 3692, 1113, 1114, 1115, 1116, 1117, - 1118, 1119, 3700, 1121, 3702, 3703, 3704, 1125, - 1126, 1127, -1809, -1809, -1809, -1809, -1809, -1809, - 3720, 3721, 3722, 3717, 3718, 3719, 3720, 1140, - 1141, 1142, 1143, -1802, 1145, 1146, 1147, 1148, - 3730, -1797, 3732, 1152, 3734, 3735, 1155, 1156, - 3738, 3739, 3740, 3741, 3742, 3743, -1785, -1785, - -1785, -1779, -1324, 1168, 1169, 1170, 1171, 1172, - 3752, 3753, 1175, 1176, 1177, 992, 3758, 3759, - 3760, 3761, 1183, 1184, 1185, 1186, -2300, 1188, - -2298, 1190, 1191, 1192, 3772, 3773, 3774, 1196, - 1197, 3777, 3778, 3779, 1201, 1202, 1203, -1747, - 1205, 1206, 1207, 1021, -1744, 1210, -1743, -1743, - 4325, 1219, 4326, 1216, 4327, 1218, 4328, 1220, - 1221, 1234, 1223, 1224, 1225, 1226, 1227, -1730, - -1730, -1730, 1231, 1232, 999, 1234, 1235, 1236, - 1237, 1238, 855, 1240, 1241, 1242, 857, 857, - 857, 857, 857, 857, 857, 857, 857, 857, - 857, 1254, 1255, 1256, 1257, 1258, 1259, 1260, - 1261, 1262, 2969, 2970, 2971, 2972, 2973, 2974, - 2975, 2976, 2977, 2978, -4426, -2483, -4426, -2481, - -6106, 878, 879, -4286, -4286, -4286, -978, 888, - 889, -975, -974, -973, -972, -971, -970, -4268, - -4268, -2059, -4268, -968, -4268, -4268, -967, -966, - -6129, -6129, -963, -962, -961, -4268, -6133, -6133, - -4268, -4268, -4268, -4268, -4268, -4268, -4268, -4268, - -3178, -4268, -1502, -1502, -1502, -1502, -1502, 3902, - 3903, 3904, -1502, 3906, 3907, -1502, -1502, -1502, - -4269, 529, -1502, 0, 0, 0, -1502, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 170, 171, 172, 173, 174, 175, 176, - 177, 178, 179, 180, 181, 182, 183, 184, - 185, 186, 187, 188, 189, 190, 191, 192, - 193, 194, 195, 196, 197, 198, 199, 200, - 201, 202, 203, 204, 205, 206, 207, 208, - 209, 210, 211, 212, 213, 214, 215, 216, - 217, 218, 219, -3194, 221, 222, 223, 224, - -1657, 226, 227, -1657, 229, 230, -1655, 555, - -1655, 234, 235, 236, 732, 238, 239, 240, - 241, 242, 243, -1655, 245, 246, 247, 248, - -1655, 250, -1655, 252, -1655, -1655, -1655, -1655, - -1655, -1655, 259, -1655, -1655, -1655, -1655, 264, - -1655, 266, -1655, 268, -1655, -3620, 271, 272, - -1655, 274, 275, -1655, 277, -1655, -1655, 280, - -1655, 282, 5746, 5747, 5748, 5749, -1655, 288, - -1655, 290, -3335, 3649, 3650, -1515, -1515, -1515, - 1793, 3659, 3660, 1796, 1797, 1798, 1799, 1800, - 1801, -1497, -1497, 712, -1497, 1803, -1497, -1497, - 1804, 1805, -3358, -3358, 1808, 1809, 1810, -1497, - -3362, -3362, -1497, -1497, -1497, -1497, -1497, -1497, - -1497, -1497, -407, -1497, -1497, -1497, -1497, -1497, - -1497, 3667, 3668, -1497, -1497, -1497, 1811, 3677, - 3678, 32767, 1814, 32767, 1815, 32767, 32767, 1816, - 1817, 32767, 32767, 32767, 1818, 1819, 1820, 1821, - -3342, -3342, 1824, 1825, 1826, 1827, 1828, 1829, - 1830, 1831, 1832, 1833, 1834, 1835, 1836, 1837, - 1838, 1839, 1840, 1841, 1842, 1843, 1844, 1845, - 1846, 1847, 1848, 1849, 1850, 1851, 1852, 1853, - 1854, 1855, 1856, 1857, 1858, 1859, 1860, 1861, - 1862, 1863, 1864, 1865, 1866, 1867, 1868, 1869, - 1870, 1871, 1872, 1873, 1874, 1875, 1876, -1537, - 1878, 1879, 1880, 1881, 0, 1883, 1884, 0, - 529, 0, 0, 2210, 0, 1889, 1890, 1891, - 2387, 1893, 1894, 1895, 1896, 1897, 1898, 0, - 1900, 1901, 1902, 1903, 0, 1905, 0, 1907, - 0, 0, 0, 0, 0, 0, 1914, 0, - 0, 0, 0, 1919, 0, 1921, 0, 1923, - 0, -1965, 1926, 1927, 0, 1929, 1930, 0, - 1932, 0, 0, 1935, 0, 1937, 7401, 7402, - 7403, 7404, 0, 1943, 0, 1945, 1946, 0, - 1948, 0, 0, 1951, 1952, 1953, 1954, 0, - 1956, 1957, 1958, 1959, 1960, 1961, 1962, 0, - 1964, 1965, 1966, 1967, 0, 1969, 1970, 1971, - 1972, 0, 1974, 0, 1976, 1977, 1978, 1979, - 1980, 1981, 1982, 1983, 1984, 1985, 0, 1987, - 1988, 1989, 1990, 1991, 566, 566, 566, 5141, - 5142, 566, 566, 566, 566, 566, 566, 566, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 8673, 5722, 5722, 5722, 0, 8676, - 5723, 8677, 8678, 2611, 5718, 2612, 5723, 2613, - 5723, 2614, 5723, 5723, 5711, 5723, 5723, 5723, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 895, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 151, 2776, 4254, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1821, 0, - 0, 0, 596, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, -2856, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, -2901, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, -1025, 32767, 32767, 32767, - 32767, -2910, 32767, 32767, 32767, 32767, 157, 32767, - 32767, 32767, 32767, 158, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 2359, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 160, 32767, 161, 162, 163, 164, - 165, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 898, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 1428, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 1254, 32767, 32767, 32767, - 32767, 1250, 32767, 32767, 32767, 32767, 1246, 32767, - 32767, 32767, 32767, 1243, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 1231, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 1842, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 3177, 1235, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, -4323, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 0, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 0, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 174, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 1830, -112, 1832, -112, 3514, -3469, - -3469, 1697, 1698, 1699, -1608, -3473, -3473, -1608, - -1608, -1608, -1608, -1608, -1608, 1691, 1692, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, -1623, -1623, -1623, 3541, 3542, -1623, -1623, - -1623, -1623, -1623, -1623, -1623, -1623, -1623, -1623, - -1623, -1623, -1623, -1623, -1623, -1623, -1623, -1623, - -1623, -1623, -1623, -1623, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, -766, 2253, 2254, 2255, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 1531, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 0, 0, 32767, 0, 0, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, -173, -173, -173, -173, -173, - -173, -173, -173, -173, -173, -173, -173, 3241, - -173, -173, -173, -173, 1709, -173, -173, 1712, - -173, -173, 1713, -496, 1715, -173, -173, -173, - -668, -173, -173, -173, -173, -173, -173, 1726, - -173, -173, -173, -173, 1731, -173, 1733, -173, - 1735, 1736, 1737, 1738, 1739, 1740, -173, 1742, - 1743, 1744, 1745, -173, 1747, -173, 1749, -173, - 1751, 3717, -173, -173, 1755, -173, -173, 1758, - -173, 1760, 1761, -173, 1763, -173, -5636, -5636, - -5636, -5636, 1769, -173, 1771, -173, 3453, -3530, - -3530, 1636, 1637, 1638, -1669, -3534, -3534, -1669, - -1669, -1669, -1669, -1669, -1669, 1630, 1631, -577, - 1633, -1666, 1635, 1636, -1664, -1664, 3500, 3501, - -1664, -1664, -1664, 1644, 3510, 3511, 1647, 1648, - 1649, 1650, 1651, 1652, 1653, 1654, 565, 1656, - 1657, 1658, 1659, 1660, 1661, -3502, -3502, 1664, - 1665, 1666, 1667, 1668, 1669, 1670, 1671, 1672, - 1673, 1674, 1675, 1676, 1677, 1678, 1679, 1680, - 1681, 1682, 1683, 1684, 1685, 1686, 1687, 1688, - 1689, 1690, 1691, 1692, 1693, 1694, 1695, 1696, - 1697, 1698, 1699, 1700, 1701, 1702, 1703, 1704, - 1705, 1706, 1707, 1708, 1709, 1710, 1711, 1712, - 1713, 1714, 1715, 1716, -1697, 1718, 1719, 1720, - 1721, -160, 1723, 1724, -160, 1726, 1727, -158, - 2052, -158, 1731, 1732, 1733, 2229, 1735, 1736, - 1737, 1738, 1739, 1740, -158, 1742, 1743, 1744, - 1745, -158, 1747, -158, 1749, -158, -158, -158, - -158, -158, -158, 1756, -158, -158, -158, -158, - 1761, -158, 1763, -158, 1765, -158, -2123, 1768, - 1769, -158, 1771, 1772, -158, 1774, -158, -158, - 1777, -158, 1779, 7243, 7244, 7245, 7246, -158, - 1785, -158, 1787, -1838, 5146, 5147, -18, -18, - -18, 3290, 5156, 5157, 3293, 3294, 3295, 3296, - 3297, 3298, 0, 0, 2209, 0, 3300, 0, - 0, 3301, 3302, -1861, -1861, 3305, 3306, 3307, - 0, -1865, -1865, 0, 0, 0, 0, 0, - 0, 0, 0, 1090, 0, 0, 0, 0, - 0, 0, 5164, 5165, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 3414, 0, 0, 0, 0, 1882, 0, - 0, 1885, 0, 0, 1886, -323, 1888, 0, - 0, 0, -495, 0, 0, 0, 0, 0, - 0, 1899, 0, 0, 0, 0, 1904, 0, - 1906, 0, 1908, 1909, 1910, 1911, 1912, 1913, - 0, 1915, 1916, 1917, 1918, 0, 1920, 0, - 1922, 0, 1924, 3890, 0, 0, 1928, 0, - 0, 1931, 0, 1933, 1934, 0, 1936, 0, - -5463, -5463, -5463, -5463, 1942, 0, 1944, 0, - 0, 1947, 0, 1949, 1950, 0, 0, 0, - 0, 1955, 0, 0, 0, 0, 0, 0, - 0, 1963, 0, 0, 0, 0, 1968, 0, - 0, 0, 0, 1973, 0, 1975, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 1986, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 527, 527, 527, 527, 0, - 528, 528, 528, 528, 528, 528, 528, 528, - 528, 528, 528, 1998, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 1999, 2000, 2001, 2002, 2003, 32767, 32767, 32767, - 32767, 32767, 2004, 32767, 2005, 2006, 2007, 2008, - 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, - 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, - 2025, 2026, 1200, 1200, 32767, 4498, 4499, 2291, - 2032, 2033, 32767, 2034, 32767, 2035, 2036, 32767, - 2037, 2038, 32767, 2039, 2040, 2041, 2042, 2043, - 2044, 2045, 2046, 2047, 2048, 2049, 2050, 2051, - 2052, 2053, 2054, 2055, 2056, 2057, 2058, 2059, - 2060, 2061, 2062, 2063, 2064, 2065, 2066, 2067, - 2068, -1506, -1506, -1506, -1506, -1506, -1506, -1506, - 2076, -2490, 2078, 2079, 2080, 2081, 2082, 2083, - 2084, 2085, 2086, 2087, 2088, 2089, 2090, 2091, - 2092, 2093, 2094, 2095, -105, -2314, 2098, 2099, - 2100, 2101, 2102, 2103, 2104, 2105, 2106, 2107, - 2108, 2109, 2110, 2111, 2112, 2113, 2114, 2115, - 2116, 2117, 2118, 2119, 2120, 5459, 2122, 2123, - 2124, 2125, 2126, 2127, 2128, 5466, 2130, 5467, - 2132, 2133, 2134, 2135, 5471, 2137, 2138, 5473, - 5473, 5473, 5473, 2143, 2144, 5475, 2146, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 2147, 2148, 2149, 2150, 2151, 2152, 2153, 2154, - 2155, 2156, 2157, 2158, 2159, 2160, 2161, 2162, - 2163, 2164, 2165, 2166, 2167, 2168, 2169, 2170, - 2171, 2172, 2173, 2174, 2175, 2176, 2177, 2178, - 2179, 2180, 2181, 2182, 2183, 2184, 2185, 2186, - 2187, 2188, 2189, 2190, 2191, 32767, -726, 2293, - -725, -725, -725, 1357, 1358, 1359, -722, 2201, - 2202, 2203, 2204, 2205, 0, 2207, -715, 2209, - -714, -714, -714, 2213, 2214, 2215, 4806, 2217, - 2218, 2219, 2220, 2221, 2222, 2223, -710, 2225, - -248, -704, -704, 2229, 2230, 2231, 2232, 2233, - 2220, 1430, 2236, 2237, -695, 2239, -694, 2241, - -693, -693, 2244, 2245, 2246, 2231, 2231, 2249, - 2250, 2251, 2252, 2253, 2254, 2255, 2256, 2239, - 2258, 2259, 2260, 2261, 2262, 2263, 2264, 2265, - 2266, 2267, 2268, 2269, 2270, 2271, 2272, 2273, - 2274, 2275, 2276, 2277, 2278, 2279, 2280, 2281, - 2282, 2283, 2284, 2285, 2286, 2287, 2288, 2289, - 2290, 2291, 2292, 2293, 3084, 2295, 2296, 2297, - 2298, 2299, 2300, 2301, 2302, 2303, 2304, 2305, - 2306, 2307, 3983, 2309, 2310, 2292, 2312, 3992, - 2314, 2315, 2316, 2317, 2318, 2319, 2320, 2321, - 2322, 2323, 2324, 2325, 2326, 2327, 2328, 4023, - 2330, 2331, 2332, 2333, 2334, 2335, 2336, 2337, - 2338, 2339, 2340, 2341, 2342, 2343, 2344, 2345, - 2346, 2347, 2348, 2349, 2350, 2351, 2352, 2353, - 2354, 2355, 2356, 2357, 2358, 2359, 2360, 2361, - 2362, 2363, 2364, 2365, 2366, 2367, 2368, 2369, - 2370, 2371, 2372, 2373, 2374, 2375, 2376, 2377, - 2378, 2379, 2360, 2360, 2360, 2360, 2360, 2360, - 2360, 2360, 2360, 2389, 2390, 2391, 2392, 2393, - 2394, 2395, 2396, 2397, 2398, 2399, 2400, 2401, - 2402, 2403, 2404, 2405, 2406, 2407, 2408, 2409, - 2410, 2411, 2412, 2413, 2414, 2415, 2416, 2417, - 2418, 2419, 2420, 2421, 2422, 2423, 2424, 2425, - 2426, 2427, 2428, 2429, 2430, 2431, 2432, 2433, - 2434, 2435, 2436, 2437, 2438, 2439, 2440, 2441, - 2442, 2443, 2444, 2445, 2446, 2447, 32767, 2448, - 2449, 2450, 2451, 2452, 2453, 2454, 2455, 2456, - 2457, 2458, 2459, 2460, 2461, 2462, 2463, 2464, - 2465, 2466, 2467, 2468, 2469, 2470, 2471, 2472, - 2473, 2474, 2475, 2476, 2477, 2478, 2479, 2480, - 2481, 2482, 2483, 2484, 2485, 2486, 2487, 2488, - 2489, 2490, 2491, 2492, 2493, 2494, 2495, 2496, - 2497, 2498, 2499, 2500, 2501, 2502, 2503, 2504, - 2505, 2506, 2507, 2508, 2509, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 2510, - 2511, 2512, 2513, 3266, 3266, 3266, 3266, 2518, - 3267, 3267, 3267, 2522, 3268, 3268, 3268, 3268, - 3268, 3268, 3268, 6682, 3268, 3268, 3268, 2534, - 5151, 3269, 2537, 2538, 3271, 3271, 5157, 2948, - 5159, 2544, 2545, 3273, 2778, 3273, 2549, 3274, - 2551, 3275, 2553, 5175, 2555, 3277, 3277, 3277, - 5181, 2560, 5184, 3278, 5186, 2564, 5189, 5190, - 5191, 5192, 3279, 5194, 5195, 2572, 5198, 32767, - 32767, 3278, 5200, 3278, 2577, 2578, 2579, 2580, - 5210, 3282, 3282, 5213, 3282, 2586, 2587, 2588, - 2589, 2590, 2591, -2175, -2175, -2175, 5230, 3288, - 5232, 3288, 6914, -69, -69, 5097, 5098, 5099, - 1792, -73, -73, 1792, 1792, 1792, 1792, 1792, - 1792, 5091, 5092, 2884, 5094, 1795, 5096, 5097, - 1797, 1797, 6961, 6962, 1797, 1797, 1797, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 2578, 2578, 2578, 2578, 2578, - 2578, 872, 872, 872, 872, 872, 872, 872, - 872, 872, 872, 8277, 6335, 8279, 6335, 9961, - 2978, 2978, 8144, 8145, 8146, 4839, 2974, 2974, - 4839, 4839, 4839, 4839, 4839, 4839, 8138, 8139, - 5931, 8141, 4842, 8143, 8144, 4844, 4844, 10008, - 10009, 4844, 4844, 4844, 8152, 10018, 10019, 8155, - 8156, 8157, 8158, 8159, 8160, 8161, 8162, 7073, - 8164, 5399, 5400, 5401, 5402, 5403, 0, 0, - 0, 5407, 0, 0, 5410, 5411, 5412, 8180, - 3383, 5415, 3914, 3915, 3916, 5419, 3918, 3919, - 3920, 3921, 3922, 3923, 3924, 3925, 3926, 3927, - 3758, 3758, 3758, 3758, 3758, 3758, 3758, 3758, - 3758, 3758, 3758, 3758, 3758, 3758, 3758, 3758, - 3758, 3758, 3758, 3758, 3758, 3758, 3758, 3758, - 3758, 3758, 3758, 3758, 3758, 3758, 3758, 3758, - 3758, 3758, 3758, 3758, 3758, 3758, 3758, 3758, - 3758, 3758, 3758, 3758, 3758, 3758, 3758, 3758, - 3758, 3758, 7172, 3758, 3758, 3758, 3758, 5640, - 3758, 3758, 5643, 3758, 3758, 5644, 3435, 5646, - 3758, 3758, 3758, 3263, 3758, 3758, 3758, 3758, - 3758, 3758, 5657, 3758, 3758, 3758, 3758, 5662, - 3758, 5664, 3758, 5666, 5667, 5668, 5669, 5670, - 5671, 3758, 5673, 5674, 5675, 5676, 3758, 5678, - 3758, 5680, 3758, 5682, 7648, 3758, 3758, 5686, - 3758, 3758, 5689, 3758, 5691, 5692, 3758, -1707, - -1707, -1707, -1707, -1707, -1707, 5698, 3756, 5700, - 3756, 7382, 399, 399, 5565, 5566, 5567, 2260, - 395, 395, 2260, 2260, 2260, 2260, 2260, 2260, - 5559, 5560, 3352, 5562, 2263, 5564, 5565, 2265, - 2265, 7429, 7430, 2265, 2265, 2265, 5573, 7439, - 7440, 5576, 5577, 5578, 5579, 5580, 5581, 5582, - 5583, 4494, 5585, 2820, 2821, 2822, 2823, 2824, - -2579, -2579, -2579, 2828, -2579, -2579, 2831, 2832, - 2833, 5601, 804, 2836, 1335, 1336, 1337, 2840, - 1339, 1340, 1341, 1342, 1343, 1344, 1345, 1346, - 1347, 1348, 1179, 1179, 1179, 1179, 1179, 1179, - 1179, 1179, 1179, 1179, 1179, 1179, 1179, 1179, - 1179, 1179, 1179, 1179, 1179, 1179, 1179, 1179, - 1179, 1179, 1179, 1179, 1179, 1179, 1179, 1179, - 1179, 1179, 1179, 1179, 1179, 1179, 1179, 1179, - 1179, 1179, 1179, 1179, 1179, 1179, 1179, 1179, - 1179, 1179, 1179, 1179, 4593, 1179, 1179, 1179, - 1179, 3061, 1179, 1179, 3064, 1179, 1179, 3065, - 856, 3067, 1179, 1179, 1179, 684, 1179, 1179, - 1179, 1179, 1179, 1179, 3078, 1179, 1179, 1179, - 1179, 3083, 1179, 3085, 1179, 3087, 3088, 3089, - 3090, 3091, 3092, 1179, 3094, 3095, 3096, 3097, - 1179, 3099, 1179, 3101, 1179, 3103, 5069, 1179, - 1179, 3107, 1179, 1179, 3110, 1179, 3112, 3113, - 1179, 3115, 1179, -4284, -4284, -4284, -4284, 3121, - 1179, 3123, 1179, 4805, -2178, -2178, 2988, 2989, - 2990, -317, -2182, -2182, -317, -317, -317, -317, - -317, -317, 2982, 2983, 775, 2985, -314, 2987, - 2988, -312, -312, 4852, 4853, -312, -312, -312, - 2996, 4862, 4863, 2999, 3000, 3001, 3002, 3003, - 3004, 3005, 3006, 1917, 3008, 3009, 3010, 3011, - 3012, 3013, -2150, -2150, 3016, 3017, 3018, 3019, - 3020, 3021, 3022, 3023, 3024, 3025, 3026, 3027, - 3028, 3029, 3030, 3031, 3032, 3033, 3034, 3035, - 32767, 32767, 32767, 3036, 3037, 3038, 3039, 3040, - 3041, 32767, 32767, 3042, 3043, 3044, 3045, 3046, - 3047, 32767, 32767, 3048, 3049, 3050, 3051, 3052, - 3053, 32767, 32767, 3054, 3055, 3056, 32767, 32767, - 32767, -357, 3058, 3059, 3060, 3061, 1180, 3063, - 0, 1179, 3065, 3066, 1181, 3391, 1181, 3070, - 0, 0, 0, 0, 32767, 0, 0, 32767, - 0, 32767, 0, 0, -4973, 32767, 32767, -7368, - -2202, -2201, -2200, -5507, -7372, -7372, -5507, -5507, - -5507, 32767, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 32767, 0, 0, 32767, 0, - -203, -2234, -732, -732, -732, -2234, -732, -732, - -2763, -1261, -1261, -1261, -2763, -1261, -1261, -1261, - -1261, -1261, -1261, -1261, -1261, -1261, -1261, -1091, - -1090, -1089, -1088, -1087, 32767, 32767, -1086, -1085, - -1084, -1083, -1082, -1081, -1080, -1079, -1078, -1077, - -1076, -1075, 32767, -1074, -1073, -1072, -1071, -1070, - -1069, -1068, -1067, -1066, -1065, -1064, -1063, -1062, - -1061, -1060, -1059, -1058, -1057, -1056, 32767, -1055, - -1054, -1053, -1052, 0, 32767, 32767, 32767, -1051, - -1050, -4463, 32767, -1048, 32767, -1047, -2928, -1045, - -1044, -2928, -1042, -1041, -2926, -716, -2926, -1037, - -1036, -1035, -539, -1033, -1032, -1031, -1030, -1029, - -1028, -2926, -1026, -1025, -1024, -1023, -2926, -1021, - -2926, -1019, -2926, -2926, -2926, -2926, -2926, -2926, - -1012, -2926, -2926, -2926, -2926, -1007, -2926, -1005, - -2926, -1003, -2926, -4891, -1000, -999, -2926, -997, - -996, -2926, -994, -2926, -2926, -991, 4475, 4476, - 4477, 4478, 4479, 4480, -2924, -981, -2924, -979, - -4604, 2380, 2381, -2784, -2784, -2784, 524, 2390, - 2391, 527, 528, 529, 530, 531, 532, -2766, - -2766, -557, -2766, 534, -2766, -2766, 535, 536, - -4627, -4627, 539, 540, 541, -2766, -4631, -4631, - -2766, -2766, -2766, -2766, -2766, -2766, -2766, -2766, - -1676, -2766, 0, 0, 0, 0, 0, 5404, - 5405, 5406, 0, 5408, 5409, 0, 0, 0, - -2767, 2031, 0, 1502, 1502, 1502, 0, 1502, - 1502, 1502, 1502, 1502, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 224, 225, 226, 32767, 227, 228, 229, - 230, 231, 232, 233, 234, 235, 236, 67, - 32767, 66, 66, 66, 66, 66, 66, 66, - 66, 66, 66, 66, 66, 66, 66, 66, - 66, 66, 66, 32767, 65, 65, 65, 65, - 65, 65, 65, 65, 65, 65, 65, 65, - 65, 65, 65, 65, 65, 65, 65, 65, - 65, 65, 65, 65, 65, 65, 65, 65, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, -271, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 1940, 18, 1942, 3908, 18, 18, 1946, 18, - 18, 1949, 18, 1951, 1952, 18, 1954, 18, - -5445, -5445, -5445, -5445, 1960, 18, 1962, 18, - 3644, -3339, -3339, 1827, 1828, 1829, -1478, -3343, - -3343, -1478, -1478, -1478, -1478, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 0, 0, 0, - 0, 32767, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1340, 1341, - 1342, 1343, 1344, 1345, 1346, 1347, 1348, 1349, - -2064, 1351, 1352, 1353, 1354, 32767, 1355, 1356, - 32767, 0, 32767, 32767, 1679, 32767, 1357, 1358, - 1359, 1855, 1361, 1362, 1363, 1364, 1365, 1366, - 32767, 1367, 1368, 1369, 1370, 32767, 1371, 32767, - 1372, 32767, 32767, 32767, 32767, 32767, 32767, 1373, - 32767, 32767, 32767, 32767, 1374, 32767, 1375, 32767, - 1376, 32767, -2513, 1378, 1379, 32767, 1380, 1381, - 32767, 1382, 32767, 32767, 1383, 32767, 1384, 32767, - 6848, 32767, 6849, 32767, 1387, 32767, 1388, 1389, - 32767, 1390, 32767, 32767, 1391, 1392, 1393, 1394, - 32767, 1395, 1396, 1397, 1398, 1399, 1400, 1401, - 32767, 1402, 1403, 1404, 1405, 32767, 1406, 1407, - 1408, 1409, 32767, 1410, 32767, 1411, 1412, 1413, - 1414, 1415, 1416, 1417, 1418, 1419, 1420, 32767, - 1421, 1422, 1423, 1424, 1425, 0, 0, 0, - 4575, 4576, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, -571, -571, - -571, 0, -572, 8101, 5150, 5150, 5150, 5337, - 8103, 5150, 8104, 8105, 2038, 5145, 2039, 5150, - 2040, 5150, 2041, 5150, 5150, 5138, 5150, 5150, - 5150, 5150, 5150, 8108, 8109, 8110, 5150, 5150, - 5384, 5150, 0, 5151, 5151, 5151, 5535, 5151, - 5151, 5151, 5537, 5538, 5539, 5540, 5541, 5542, - 5543, 5544, 5545, 5546, 5547, 5151, 5151, 2572, - 2572, 2572, 2572, 2572, 2572, 2572, 2572, 2572, - 2572, 2572, 2572, 2572, 2572, 2572, 2572, 2572, - 2572, 2572, 2572, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 2580, - 2580, 2580, 2580, 2580, 2580, 2580, 0, 2580, - 0, 2580, 0, 0, 0, 2580, 2580, 2580, - 0, 2580, 2580, 2580, 2580, 2580, 2580, 2580, - 0, 2580, 0, 0, 0, 2580, 2580, 2580, - 5517, 5518, 5519, 5520, 5521, 5522, -6, -6, - -6, 0, 0, 0, 0, 2581, 2581, 2581, - 2581, 5527, 2581, 2581, 2581, 2581, 0, 5528, - 0, 2581, 0, 0, 2581, 2581, 0, 0, - 0, 0, 0, 0, 5529, 5530, 5531, 32767, - 32767, 2579, 2579, 2579, 2579, 2579, 0, 0, - 2579, 2579, 2579, 2765, 0, 0, 0, 0, - 2579, 2579, 2579, 2579, 6066, 2579, 6066, 2579, - 2579, 2579, 0, 0, 0, 2579, 2579, 0, - 0, 0, 2579, 2579, 2579, 5530, 2579, 2579, - 2579, 2766, 5532, 2579, 5533, 5534, -533, 2574, - -532, 2579, -531, 2579, -530, 2579, 2579, 2567, - 2579, 2579, 2579, 2579, 2579, 5537, 5538, 5539, - 2579, 2579, 2813, 2579, 2579, 2579, 2579, 2579, - 2963, 2579, 2579, 2579, 2965, 2966, 2967, 2968, - 2969, 2970, 2971, 2972, 2973, 2974, 2975, 2579, - 2579, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 32767, 32767, 32767, - 32767, 32767, 331, 32767, 332, -2580, -2580, -2580, - -2580, 0, 0, 0, 0, 0, 0, 0, - -2580, 0, -2580, 0, -2580, -2580, -2580, 0, - 0, 0, -2580, 0, 0, 0, 0, 0, - 0, 0, -2580, 0, -2580, -2580, -2580, 0, - 0, 0, 2937, 2938, 2939, 2940, 2941, 2942, - -2586, -2586, -2586, -2580, -2125, -2581, -2581, 0, - 0, 0, 0, 2946, 0, 0, 0, 0, - -2581, 2947, -2581, 0, -2581, -2581, 0, 0, - -2581, -2581, -2581, -2581, -2581, -2581, 2948, 2949, - 2950, 2945, 2491, 0, 0, 0, 0, 0, - -2579, -2579, 0, 0, 0, 186, -2579, -2579, - -2579, -2579, 0, 0, 0, 0, 3487, 0, - 3487, 0, 0, 0, -2579, -2579, -2579, 0, - 0, -2579, -2579, -2579, 0, 0, 0, 2951, - 0, 0, 0, 187, 2953, 0, 2954, 2955, - -3112, -5, -3111, 0, -3110, 0, -3109, 0, - 0, -12, 0, 0, 0, 0, 0, 2958, - 2959, 2960, 0, 0, 234, 0, 0, 0, - 0, 0, 384, 0, 0, 0, 386, 387, - 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 0, 0, 0, 0, 0, 0, - 0, 0, -1706, -1706, -1706, 0, 0, 0, - 0, 385, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 397, - 398, 399, 400, 401, 402, 403, 404, 405, - 2112, 2113, 2114, 409, 410, 411, 412, 32767, - 413, 414, 415, 416, 417, 418, 419, 420, - 421, 422, 423, 424, 425, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - -1688, 32767, 32767, 32767, 32767, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 0, 0, 0, - 0, -752, -751, -750, -749, 0, -748, -747, - -746, 0, -745, -744, -743, -742, -741, -740, - -739, -4152, -737, -736, -735, 0, -2616, -733, - 0, 0, -732, -731, -2616, -406, -2616, 0, - 0, -727, -231, -725, 0, -724, 0, -723, - 0, -2621, 0, -721, -720, -719, -2622, 0, - -2623, -716, -2623, 0, -2624, -2624, -2624, -2624, - -710, -2624, -2624, 0, -2625, -706, -2625, -704, - -2625, -702, 0, 0, 0, 0, -2629, -700, - -699, -2629, -697, 0, 0, 0, 0, 0, - 0, 4767, 4768, 4769, -2635, -692, -2635, -690, - -4315, 2669, 2670, -2495, -2495, -2495, 813, 2679, - 2680, 816, 817, 818, 819, 820, 821, -2477, - -2477, -268, -2477, 823, -2477, -2477, 824, 825, - -4338, -4338, 828, 829, 830, -2477, -4342, -4342, - -2477, -2477, -2477, -2477, -2477, -2477, -2477, -2477, - -1387, 0, 0, 32767, 32767, 0, 0, 0, - 0, 0, -2486, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 1756, 1757, 1758, - 1759, -5645, -3702, -5645, -3700, -7325, -341, -340, - -5505, -5505, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 532, 533, - 32767, 534, 535, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, -781, 1084, 1084, 1084, 1084, - 1084, 1084, 4383, 4384, 2176, 4386, 1087, 4388, - 4389, 1089, 1089, 6253, 6254, 1089, 1089, 1089, - 4397, 6263, 6264, 4400, 4401, 4402, 4403, 4404, - 4405, 4406, 4407, 3318, 4409, 4410, 4411, 4412, - 4413, 4414, -749, -749, 4417, 4418, 4419, 4420, - 4421, 4422, 4423, 4424, 4425, 4426, 4427, 4428, - 4429, 4430, 4431, 4432, 4433, 4434, 4435, 4436, - 4437, 4438, 4439, 4440, 4441, 4442, 4443, 4444, - 4445, 4446, 4447, 4448, 4449, 4450, 4451, 4452, - 4453, 4454, 4455, 4456, 4457, 4458, 4459, 4460, - 4461, 4462, 4463, 4464, 4465, 4466, 4467, 4468, - 4469, 1056, 4471, 4472, 4473, 4474, 2593, 4476, - 4477, 2593, 4479, 4480, 2595, 4805, 2595, 4484, - 4485, 4486, 4982, 4488, 4489, 4490, 4491, 4492, - 4493, 2595, 4495, 4496, 4497, 4498, 2595, 4500, - 2595, 4502, 2595, 2595, 2595, 2595, 2595, 2595, - 4509, 2595, 2595, 2595, 2595, 4514, 2595, 4516, - 2595, 4518, 2595, 630, 4521, 4522, 2595, 4524, - 4525, 2595, 4527, 2595, 2595, 4530, 2595, 4532, - 9996, 9997, 9998, 9999, 2595, 4538, 2595, 4540, - 4541, 2595, 4543, 2595, 2595, 4546, 4547, 4548, - 4549, 2595, 4551, 4552, 4553, 4554, 4555, 4556, - 4557, 2595, 4559, 4560, 4561, 4562, 2595, 4564, - 4565, 4566, 4567, 2595, 4569, 2595, 4571, 4572, - 4573, 4574, 4575, 4576, 4577, 4578, 4579, 4580, - 2595, 4582, 4583, 4584, 4585, 4586, 4587, 4588, - 4589, 4590, 4591, 4592, 4593, 4594, 4595, 4596, - 4597, 4598, 4599, 4600, 4601, 4602, 4603, 4604, - 4605, 4606, 4607, 4608, 4609, 4610, 4611, 4612, - 4613, 4614, 4615, 4089, 4090, 4091, 4092, 4620, - 4093, 4094, 4095, 4096, 4097, 4098, 4099, 4100, - 4101, 4102, 4103, 4104, 2765, 2765, 2765, 2765, - 2765, 2765, 2765, 2765, 2765, 2765, 6179, 2765, - 2765, 2765, 2765, 4647, 2765, 2765, 4650, 4122, - 4652, 4653, 2444, 4655, 2767, 2767, 2767, 2272, - 2767, 2767, 2767, 2767, 2767, 2767, 4666, 2767, - 2767, 2767, 2767, 4671, 2767, 4673, 2767, 4675, - 4676, 4677, 4678, 4679, 4680, 2767, 4682, 4683, - 4684, 4685, 2767, 4687, 2767, 4689, 2767, 4691, - 6657, 2767, 2767, 4695, 2767, 2767, 4698, 2767, - 4700, 4701, 2767, 4703, 2767, -2696, -2696, -2696, - -2696, 4709, 2767, 4711, 2767, 2767, 4714, 2767, - 4716, 4717, 2767, 2767, 2767, 2767, 4722, 2767, - 2767, 2767, 2767, 2767, 2767, 2767, 4730, 2767, - 2767, 2767, 2767, 4735, 2767, 2767, 2767, 2767, - 4740, 2767, 4742, 2767, 2767, 2767, 2767, 2767, - 2767, 2767, 2767, 2767, 2767, 4753, 2767, 2767, - 2767, 2767, 2767, 4193, 4194, 4195, -379, -379, - 4198, 4199, 4200, 4201, 4202, 4203, 4204, 4771, - 4772, 4773, 4774, 4775, 4776, 4777, 4778, 4779, - 4780, -3892, -940, -939, -938, 4785, -3890, -936, - -3889, -3889, 2179, -927, 2180, -930, 2181, -928, - 2182, -926, -925, -912, -923, -922, -921, 4803, - 4804, 4805, 4806, 4807, 4808, 4809, 4810, 4811, - 4812, 4813, 4814, 4815, 4816, 4817, 4818, 4819, - 3925, 4821, 4822, 4823, 4824, 4825, 4826, 4827, - 4828, 4829, 4830, 4831, 4832, 4833, 4834, 4835, - 4836, 4837, 4838, 4839, 4840, 4841, 4842, 4843, - 4844, 4845, 4846, 4847, 4848, 4849, 4850, 4851, - 4852, 4853, 4854, 4855, 4856, 4857, 4858, 4859, - 4860, 4710, 2086, 609, 4864, 4865, 4866, 4867, - 4868, 4869, 4870, 4871, 4872, 3052, 4874, 4875, - 4876, 4281, 4878, 4879, 4880, 4881, 4882, 4883, - 4884, 4885, 4886, 4887, 634, 4888, 4889, 4890, - 4891, 4892, 4893, 4894, 4895, 4896, 1322, 1322, - 1322, 1322, 1322, 1322, 1322, 4904, 338, 4906, - 4907, 4908, 4909, 4910, 4911, 4912, 4913, 4914, - 4915, 4916, 4917, 665, 666, 667, 668, 669, - 670, 671, 672, 673, 674, 675, 676, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, - 32767, 32767, 32767, 32767, 32767, 32767, 32767, 0, - 0, 0, 0, 0, 0, 32767, 0, 0, - 0, 0, 32767, 32767, 0, 0, 0, 0, - 0, 0, 0, 0, 32767, 0, 0, 0, - 0, 0, 0, 0, 32767, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 32767, 0, 0, 0, 2478, 32767, 2477, - 2477, 2477, 2477, 2477, 32767, 2476, 32767, 32767, - 32767, 2473, 2473, 2473, 2473, 2473, 2473, 2473, - 32767, 2472, 2472, 2472, 2472, 2472, 2472, 2472, - 2472, 2472, 2472, 2472, 2472, 2472, 2472, 2472, - 2472, 2472, 2472, 2472, 2472, 2472, 2472, 2472, - 2472, 2472, 2472, 2472, 2472, 2472, 2472, 2472, - 2472, 2472, 2472, 2472, 2472, 2472, 2472, 2472, - 2472, 2472, 2472, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, -2478, -2478, -2478, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, + -2472, -2472, -2472, -2472, -2472, -2472, -2472, -2472, + -2472, -2472, -2472, -2472, -2472, -2472, -2472, -2472, + -2472, -2472, -2472, -2472, -2472, -2472, -2472, -2472, + -2472, -2472, -2472, -2472, -2472, 32767, 32767, 32767, + -2475, -2475, -2475, -2475, -2475, -2475, -2475, -2475, + -2475, -2475, -2475, -2475, -2475, -2475, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 865, 865, 865, 865, 865, 865, 865, + 865, 865, 865, 865, -2255, 32767, -5207, 32767, + -5207, 860, 860, 860, 860, 860, 860, 860, + 860, 860, 4250, 861, 861, 861, 3339, 3339, + 3339, 3339, 3339, 3339, 3339, 3339, 3339, 3339, + 3339, 3339, 3339, 3339, 3339, 3339, 3339, 3339, + 32767, 3338, 3338, 3338, 3338, 3338, 3338, 3338, + 3338, 3338, 3338, 3338, 3338, 3338, 3338, 3338, + 3338, 3338, 3338, 3338, 3338, 3338, 3338, 3338, + 3338, 3338, 3338, 3338, 3338, 3338, 3338, 3338, + 3338, 9, 10, 32767, 11, 12, 0, 32767, + 0, 2913, 2914, 2915, 2916, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 2917, 32767, 2918, -100, + 2919, 2920, 2921, 840, 840, 840, 2922, 0, + 0, 0, 0, 0, 2206, 0, 2923, 0, + 2924, 2925, 2926, 0, 0, 0, -2590, 0, + 0, 0, 0, 0, 0, 0, 2934, 0, + 2474, 2931, 2932, 0, 0, 0, 0, 0, + 14, 805, 0, 0, 2933, 0, 2934, 0, + 2935, 2936, 0, 0, 0, 16, 17, 0, + 0, 0, 0, 0, 0, 0, 0, 18, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, -790, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, -1675, 0, 0, 19, 0, -1679, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, -1694, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 29, 30, 31, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 724, 2668, 724, 4350, -2633, -2633, + 2533, 2534, 2535, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 2518, 2519, 2520, 1431, 45, 46, + 32767, 32767, 47, 48, 49, 50, 51, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, -3011, 53, -1125, -3010, -3010, + 32767, -3334, -1123, -3011, 60, 61, 62, 63, + 32767, 32767, 64, 32767, 65, 32767, 66, 67, + 32767, 32767, 32767, 32767, 32767, 32767, 2268, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 69, 70, + 71, 72, 73, 74, 32767, 32767, 32767, 32767, + 75, 76, 32767, 77, 281, 32767, 32767, 32767, + 32767, 32767, 32767, 811, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 1341, 1342, 1343, 1344, 1345, + 1346, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 86, + 32767, 32767, 32767, 32767, 32767, 4550, 32767, 32767, + 32767, 1135, 32767, 32767, 32767, 32767, 32767, 1130, + 3016, 32767, 3017, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 677, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 2858, 2859, 651, 2861, -438, + 2863, 2864, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, -5305, -5305, -5305, 32767, -5306, + -5306, 32767, 32767, 32767, 2871, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 3022, 3023, 680, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, -272, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 4308, 4309, 4310, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 4311, 4312, 4313, + 4314, 4315, 4316, 4317, 4318, 4319, 4320, 4321, + 4322, 4323, 4324, 4325, 4326, 4307, 4307, 4307, + 4307, 4307, 4307, 4307, 4307, 4307, 4336, 4337, + 4338, 4339, 4340, 4341, 4342, 4343, 4344, 4345, + 4346, 4347, 4348, 4349, 4350, 4351, 4352, 4353, + 4354, 32767, 32767, 32767, 32767, 4355, 4356, 4357, + 4358, 4359, 4360, 4361, 4362, 4363, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 4364, 4365, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 2202, 0, 0, 0, 59, 0, + 0, 35, 0, 0, 0, 3549, 0, 0, + 0, 0, 0, 3394, 0, 0, 3399, 0, + 0, 0, 0, 0, 0, 0, 0, 2012, + 0, 0, 0, 0, 87, 2022, 0, 7490, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 2255, 0, 2256, 2256, 2256, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 32767, 0, 0, + 0, 0, 0, 0, -1759, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 4767, 0, 0, 4772, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 32767, 5977, 0, + 892, 32767, 0, 32767, 32767, 0, 0, 32767, + 32767, 2344, 4834, 4835, 4836, 32767, 0, 4840, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 32767, 0, 32767, 0, 0, 0, + 0, 0, 0, 0, 32767, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 32767, 32767, 0, 32767, 0, 0, 0, 32767, + 32767, 32767, 32767, 3261, 3262, 32767, 3007, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 106, 107, 108, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 109, 110, 111, 112, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 0, 0, -2344, + -2344, 0, 32767, 0, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, -1642, 1469, -1641, 1469, -1640, 1469, + 1469, 1457, 1469, 1469, 1469, -4254, -4254, -4254, + -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4254, + -4254, -4254, -4254, -4254, -4254, -4254, -3359, -4254, + -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4254, + -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4254, + -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4254, + -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4254, + -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4103, + -1478, 0, -4254, -4254, -4254, -4254, -4254, -4254, + -4254, -4254, -4254, -2433, -4254, -4254, -4254, -3658, + -4254, -4254, -4254, -4254, -4254, -4254, -4254, -4254, + -4254, -4254, 0, -4253, -4253, -4253, -4253, -4253, + -4253, -4253, -4253, -4253, -678, -677, -676, -675, + -674, -673, -672, -4253, 314, -4253, -4253, -4253, + -4253, -4253, -4253, -4253, -4253, -4253, -4253, -4253, + -4253, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 1464, 1465, 1466, 1467, + 1468, 1469, 0, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 0, + 0, 0, 0, 0, 32767, 32767, 32767, 32767, + 32767, 0, 32767, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 827, 828, 829, -2469, -2469, -260, 0, + 0, 32767, 0, 32767, 0, 0, 32767, 0, + 0, 32767, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 3575, 3576, 3577, 3578, 3579, 3580, 3581, 0, + 4567, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 2201, 4411, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, -3338, 0, 0, 0, + 0, 0, 0, 0, -3337, 0, -3336, 0, + 0, 0, 0, -3335, 0, 0, -3334, -3333, + -3332, -3331, 0, 0, -3330, 0, 0, 32767, + 0, 0, 13, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 3073, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + -2556, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 3074, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 2355, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, -488, -488, -488, -302, -3067, -3067, + -3067, -3067, -488, -488, -488, -488, 2999, -488, + 2999, -488, -488, -488, -3067, -3067, -3067, -488, + -488, -3067, -3067, -3067, -488, -488, -488, 2463, + -488, -488, -488, -301, 2465, -488, 2466, 2467, + -3600, -493, -3599, -488, -3598, -488, -3597, -488, + -488, -500, -488, -488, -488, -488, -488, 2470, + 2471, 2472, -488, -488, -254, -488, -488, -488, + -488, -488, -104, -488, -488, -488, -102, -101, + -100, -99, -98, -97, -96, -95, -94, -93, + -92, -488, -488, -488, -488, -488, -488, -488, + -488, -488, -2194, -2194, -2194, -2194, -2194, -2194, + -2194, -2194, -2194, -2194, 5211, 3269, 5213, 3269, + 6895, -88, -88, 5078, 5079, 5080, 1773, -92, + -92, 1773, 1773, 1773, 1773, 1773, 1773, 5072, + 5073, 2865, 5075, 1776, 5077, 5078, 1778, 1778, + 6942, 6943, 1778, 1778, 1778, 5086, 6952, 6953, + 5089, 5090, 5091, 5092, 5093, 5094, 5095, 5096, + 4007, 5098, 2333, 2334, 2335, 2336, 2337, -3066, + -3066, -3066, 2341, -3066, -3066, 2344, 2345, 2346, + 5114, 317, 2349, 848, 849, 850, 2353, 852, + 853, 854, 855, 856, 857, 858, 859, 860, + 861, 692, 692, 692, 692, 692, 692, 692, + 692, 692, 692, 692, 692, 692, 692, 692, + 692, 692, 692, 692, 692, 692, 692, 692, + 692, 692, 692, 692, 692, 692, 692, 692, + 692, 692, 692, 692, 692, 692, 692, 692, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 3093, 3094, 3095, 3096, 3097, 3098, 3099, + 3100, 3101, 3102, 901, 3104, 3105, 3106, 3048, + 3108, 3109, 3075, 3111, 3112, 3113, -435, 3115, + 3116, 3117, 3118, 3119, -274, 3121, 3122, -276, + 3124, 3125, 3126, 3127, 3128, 3129, 3130, 3131, + 1120, 3133, 3134, 3135, 3136, 3050, 1116, 3139, + -4350, 3141, 3142, 3143, 3144, 3145, 3146, 3147, + 3148, 3149, 3150, 3151, 3152, 3153, 3154, 3155, + 3156, 902, 3158, 903, 904, 905, 3162, 3163, + 3164, 3165, 3166, 3167, 3168, 3169, 3170, 3171, + 3172, 3173, 3174, 3175, 3176, 3177, 32767, 3178, + 3179, 3180, 3181, 3182, 3183, 4943, 3185, 3186, + 3187, 3188, 3189, 3190, 3191, 3192, 3193, 3194, + 3195, 3196, 3197, 3198, 3199, 3200, 3201, 3202, + 3203, 3204, 3205, 3206, 3207, 3208, 3209, 3210, + 3211, 3212, 3213, 3214, 3215, 3216, 3217, 3218, + 3219, 3220, 3221, 3222, 3223, -1543, 3225, 3226, + -1545, 3228, 3229, 3230, 3231, 3232, 3233, 3234, + 3235, 3236, 3237, 3238, 3239, 3240, 3241, 3242, + 3243, 3244, 3245, 3246, 3247, 3248, -1251, -2728, + 3250, 32767, 32767, 3251, 906, 907, 3252, 3253, + 32767, 32767, 910, -1579, -1579, -1579, 32767, 3258, + -1581, 3260, 3261, 3262, 3263, 3264, 3265, 3266, + 3267, 3268, 3269, 32767, 3270, 32767, 3271, 3272, + 3273, 3274, 3275, 3276, 3277, 32767, 3278, 3279, + 3280, 3281, 3282, 3283, 3284, 3285, 3286, 3287, + 3288, 3289, 3290, 3291, 3292, 3293, 3294, 3295, + 3296, 3297, 3298, 3299, 3300, 3301, 3302, 3303, + 3304, 3305, 3306, 3307, 3308, 3309, 3310, 3311, + 3312, 3313, 3314, 3315, 3316, 3317, 3318, 3319, + 3320, 3321, 3322, 3323, 3324, 3325, 3326, 3327, + 3328, 3329, 3330, 3331, 3332, 3333, 3334, 3335, + 3336, 32767, 3337, 3338, 3339, 3340, 3341, 3342, + 0, 3343, 3344, 3345, 3346, 32767, 32767, 3347, + 3348, 3349, 3350, 3351, 3352, 3353, 3354, 32767, + 3355, 3356, 3357, 3358, 3359, 3360, 3361, 32767, + 3362, 3363, 3364, 3365, 3366, 3367, 3368, 3369, + 3370, 3371, 3372, 3373, 3374, 3375, 3376, 3377, + 3378, 3379, 3380, 3381, 3382, 3383, 3384, 3385, + 3386, 3387, 3388, 3389, 0, 3390, 3391, 3392, + 915, 916, 917, 918, 919, 920, 921, 922, + 923, 924, 925, 926, 927, 928, 929, 930, + 931, 932, 933, 934, 935, 936, 937, 938, + 939, 940, 941, 942, 943, 944, 945, 946, + 947, 948, 949, 950, 951, 952, 953, 954, + 955, 956, 957, 958, 959, 960, 961, 962, + 963, 964, 965, 966, 967, 968, 969, 970, + 971, 972, 973, 974, 975, 976, 3449, 3450, + 3451, 3452, 3453, 3454, 3455, 3456, 3457, 3458, + 3459, 3460, 3461, 3462, 3463, 3464, 3465, 3466, + 3467, 3468, 3469, 3470, 3471, 3472, 3473, 3474, + 3475, 3476, 3477, 3478, 3479, 3480, 3481, 3482, + 3483, 3484, 3485, 3486, 3487, 3488, 3489, 3490, + 3491, 3492, 3493, 3494, 3495, 3496, 3497, 3498, + 3499, 3500, 3501, 3502, 3503, 3504, 3505, 3506, + 3507, 3508, 3509, 3510, 3511, 3512, 3513, 3514, + 3515, 3516, 3517, 3518, 3519, 3520, 3521, 3522, + 3523, 3524, 3525, 3526, 3527, 3528, 3529, 3530, + 3531, 3532, 3533, 3534, 3535, 3536, 3537, 3538, + 3539, 3540, 3541, 3542, 3543, 3544, 3545, 3546, + 3547, 3548, 3549, 3550, 3551, 3552, 3553, 3554, + 3555, 3556, 3557, 3558, 3559, 3560, 3561, 3562, + 3563, 3564, 3565, 3566, 3567, 3568, 3569, 3570, + 3571, 3572, 3573, 3574, 3575, 3576, 3577, 6056, + 6057, 6058, 32767, 3581, 3582, 3583, 3584, 3585, + 4157, 4158, 4159, 3589, 4162, -4510, -1558, -1557, + -1556, -1742, -4507, -1553, -4506, -4506, 1562, -1544, + 1563, -1547, 1564, -1545, 1565, -1543, -1542, -1529, + -1540, -1539, -1538, -1537, -1536, -4493, -4493, -4493, + -1532, -1531, -1764, -1529, 3622, -1528, -1527, -1526, + -1909, -1524, -1523, -1522, -1907, -1907, -1907, -1907, + -1907, -1907, -1907, -1907, -1907, -1907, -1907, -1510, + -1509, 1071, 1072, 1073, 1074, 1075, 1076, 1077, + 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, + 1086, 1087, 1088, 1089, 1090, 3663, 3664, 3665, + 3666, 3667, 3668, 3669, 3670, 3671, 3672, 3673, + 3674, 1095, 1096, 1097, 1098, 1099, 1100, 1101, + 3682, 1103, 3684, 1105, 3686, 3687, 3688, 1109, + 1110, 1111, 3692, 1113, 1114, 1115, 1116, 1117, + 1118, 1119, 3700, 1121, 3702, 3703, 3704, 1125, + 1126, 1127, -1809, -1809, -1809, -1809, -1809, -1809, + 3720, 3721, 3722, 3717, 3718, 3719, 3720, 1140, + 1141, 1142, 1143, -1802, 1145, 1146, 1147, 1148, + 3730, -1797, 3732, 1152, 3734, 3735, 1155, 1156, + 3738, 3739, 3740, 3741, 3742, 3743, -1785, -1785, + -1785, -1779, -1324, 1168, 1169, 1170, 1171, 1172, + 3752, 3753, 1175, 1176, 1177, 992, 3758, 3759, + 3760, 3761, 1183, 1184, 1185, 1186, -2300, 1188, + -2298, 1190, 1191, 1192, 3772, 3773, 3774, 1196, + 1197, 3777, 3778, 3779, 1201, 1202, 1203, -1747, + 1205, 1206, 1207, 1021, -1744, 1210, -1743, -1743, + 4325, 1219, 4326, 1216, 4327, 1218, 4328, 1220, + 1221, 1234, 1223, 1224, 1225, 1226, 1227, -1730, + -1730, -1730, 1231, 1232, 999, 1234, 1235, 1236, + 1237, 1238, 855, 1240, 1241, 1242, 857, 857, + 857, 857, 857, 857, 857, 857, 857, 857, + 857, 1254, 1255, 1256, 1257, 1258, 1259, 1260, + 1261, 1262, 2969, 2970, 2971, 2972, 2973, 2974, + 2975, 2976, 2977, 2978, -4426, -2483, -4426, -2481, + -6106, 878, 879, -4286, -4286, -4286, -978, 888, + 889, -975, -974, -973, -972, -971, -970, -4268, + -4268, -2059, -4268, -968, -4268, -4268, -967, -966, + -6129, -6129, -963, -962, -961, -4268, -6133, -6133, + -4268, -4268, -4268, -4268, -4268, -4268, -4268, -4268, + -3178, -4268, -1502, -1502, -1502, -1502, -1502, 3902, + 3903, 3904, -1502, 3906, 3907, -1502, -1502, -1502, + -4269, 529, -1502, 0, 0, 0, -1502, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 170, 171, 172, 173, 174, 175, 176, + 177, 178, 179, 180, 181, 182, 183, 184, + 185, 186, 187, 188, 189, 190, 191, 192, + 193, 194, 195, 196, 197, 198, 199, 200, + 201, 202, 203, 204, 205, 206, 207, 208, + 209, 210, 211, 212, 213, 214, 215, 216, + 217, 218, 219, -3194, 221, 222, 223, 224, + -1657, 226, 227, -1657, 229, 230, -1655, 555, + -1655, 234, 235, 236, 732, 238, 239, 240, + 241, 242, 243, -1655, 245, 246, 247, 248, + -1655, 250, -1655, 252, -1655, -1655, -1655, -1655, + -1655, -1655, 259, -1655, -1655, -1655, -1655, 264, + -1655, 266, -1655, 268, -1655, -3620, 271, 272, + -1655, 274, 275, -1655, 277, -1655, -1655, 280, + -1655, 282, 5746, 5747, 5748, 5749, -1655, 288, + -1655, 290, -3335, 3649, 3650, -1515, -1515, -1515, + 1793, 3659, 3660, 1796, 1797, 1798, 1799, 1800, + 1801, -1497, -1497, 712, -1497, 1803, -1497, -1497, + 1804, 1805, -3358, -3358, 1808, 1809, 1810, -1497, + -3362, -3362, -1497, -1497, -1497, -1497, -1497, -1497, + -1497, -1497, -407, -1497, -1497, -1497, -1497, -1497, + -1497, 3667, 3668, -1497, -1497, -1497, 1811, 3677, + 3678, 32767, 1814, 32767, 1815, 32767, 32767, 1816, + 1817, 32767, 32767, 32767, 1818, 1819, 1820, 1821, + -3342, -3342, 1824, 1825, 1826, 1827, 1828, 1829, + 1830, 1831, 1832, 1833, 1834, 1835, 1836, 1837, + 1838, 1839, 1840, 1841, 1842, 1843, 1844, 1845, + 1846, 1847, 1848, 1849, 1850, 1851, 1852, 1853, + 1854, 1855, 1856, 1857, 1858, 1859, 1860, 1861, + 1862, 1863, 1864, 1865, 1866, 1867, 1868, 1869, + 1870, 1871, 1872, 1873, 1874, 1875, 1876, -1537, + 1878, 1879, 1880, 1881, 0, 1883, 1884, 0, + 529, 0, 0, 2210, 0, 1889, 1890, 1891, + 2387, 1893, 1894, 1895, 1896, 1897, 1898, 0, + 1900, 1901, 1902, 1903, 0, 1905, 0, 1907, + 0, 0, 0, 0, 0, 0, 1914, 0, + 0, 0, 0, 1919, 0, 1921, 0, 1923, + 0, -1965, 1926, 1927, 0, 1929, 1930, 0, + 1932, 0, 0, 1935, 0, 1937, 7401, 7402, + 7403, 7404, 0, 1943, 0, 1945, 1946, 0, + 1948, 0, 0, 1951, 1952, 1953, 1954, 0, + 1956, 1957, 1958, 1959, 1960, 1961, 1962, 0, + 1964, 1965, 1966, 1967, 0, 1969, 1970, 1971, + 1972, 0, 1974, 0, 1976, 1977, 1978, 1979, + 1980, 1981, 1982, 1983, 1984, 1985, 0, 1987, + 1988, 1989, 1990, 1991, 566, 566, 566, 5141, + 5142, 566, 566, 566, 566, 566, 566, 566, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 8673, 5722, 5722, 5722, 0, 8676, + 5723, 8677, 8678, 2611, 5718, 2612, 5723, 2613, + 5723, 2614, 5723, 5723, 5711, 5723, 5723, 5723, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 895, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 151, 2776, 4254, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1821, 0, + 0, 0, 596, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, -2856, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, -2901, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, -1025, 32767, 32767, 32767, + 32767, -2910, 32767, 32767, 32767, 32767, 157, 32767, + 32767, 32767, 32767, 158, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 2359, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 160, 32767, 161, 162, 163, 164, + 165, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 898, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 1428, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 1254, 32767, 32767, 32767, + 32767, 1250, 32767, 32767, 32767, 32767, 1246, 32767, + 32767, 32767, 32767, 1243, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 1231, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 1842, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 3177, 1235, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, -4323, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 0, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 0, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 174, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 1830, -112, 1832, -112, 3514, -3469, + -3469, 1697, 1698, 1699, -1608, -3473, -3473, -1608, + -1608, -1608, -1608, -1608, -1608, 1691, 1692, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, -1623, -1623, -1623, 3541, 3542, -1623, -1623, + -1623, -1623, -1623, -1623, -1623, -1623, -1623, -1623, + -1623, -1623, -1623, -1623, -1623, -1623, -1623, -1623, + -1623, -1623, -1623, -1623, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, -766, 2253, 2254, 2255, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 1531, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 0, 0, 32767, 0, 0, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, -173, -173, -173, -173, -173, + -173, -173, -173, -173, -173, -173, -173, 3241, + -173, -173, -173, -173, 1709, -173, -173, 1712, + -173, -173, 1713, -496, 1715, -173, -173, -173, + -668, -173, -173, -173, -173, -173, -173, 1726, + -173, -173, -173, -173, 1731, -173, 1733, -173, + 1735, 1736, 1737, 1738, 1739, 1740, -173, 1742, + 1743, 1744, 1745, -173, 1747, -173, 1749, -173, + 1751, 3717, -173, -173, 1755, -173, -173, 1758, + -173, 1760, 1761, -173, 1763, -173, -5636, -5636, + -5636, -5636, 1769, -173, 1771, -173, 3453, -3530, + -3530, 1636, 1637, 1638, -1669, -3534, -3534, -1669, + -1669, -1669, -1669, -1669, -1669, 1630, 1631, -577, + 1633, -1666, 1635, 1636, -1664, -1664, 3500, 3501, + -1664, -1664, -1664, 1644, 3510, 3511, 1647, 1648, + 1649, 1650, 1651, 1652, 1653, 1654, 565, 1656, + 1657, 1658, 1659, 1660, 1661, -3502, -3502, 1664, + 1665, 1666, 1667, 1668, 1669, 1670, 1671, 1672, + 1673, 1674, 1675, 1676, 1677, 1678, 1679, 1680, + 1681, 1682, 1683, 1684, 1685, 1686, 1687, 1688, + 1689, 1690, 1691, 1692, 1693, 1694, 1695, 1696, + 1697, 1698, 1699, 1700, 1701, 1702, 1703, 1704, + 1705, 1706, 1707, 1708, 1709, 1710, 1711, 1712, + 1713, 1714, 1715, 1716, -1697, 1718, 1719, 1720, + 1721, -160, 1723, 1724, -160, 1726, 1727, -158, + 2052, -158, 1731, 1732, 1733, 2229, 1735, 1736, + 1737, 1738, 1739, 1740, -158, 1742, 1743, 1744, + 1745, -158, 1747, -158, 1749, -158, -158, -158, + -158, -158, -158, 1756, -158, -158, -158, -158, + 1761, -158, 1763, -158, 1765, -158, -2123, 1768, + 1769, -158, 1771, 1772, -158, 1774, -158, -158, + 1777, -158, 1779, 7243, 7244, 7245, 7246, -158, + 1785, -158, 1787, -1838, 5146, 5147, -18, -18, + -18, 3290, 5156, 5157, 3293, 3294, 3295, 3296, + 3297, 3298, 0, 0, 2209, 0, 3300, 0, + 0, 3301, 3302, -1861, -1861, 3305, 3306, 3307, + 0, -1865, -1865, 0, 0, 0, 0, 0, + 0, 0, 0, 1090, 0, 0, 0, 0, + 0, 0, 5164, 5165, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 3414, 0, 0, 0, 0, 1882, 0, + 0, 1885, 0, 0, 1886, -323, 1888, 0, + 0, 0, -495, 0, 0, 0, 0, 0, + 0, 1899, 0, 0, 0, 0, 1904, 0, + 1906, 0, 1908, 1909, 1910, 1911, 1912, 1913, + 0, 1915, 1916, 1917, 1918, 0, 1920, 0, + 1922, 0, 1924, 3890, 0, 0, 1928, 0, + 0, 1931, 0, 1933, 1934, 0, 1936, 0, + -5463, -5463, -5463, -5463, 1942, 0, 1944, 0, + 0, 1947, 0, 1949, 1950, 0, 0, 0, + 0, 1955, 0, 0, 0, 0, 0, 0, + 0, 1963, 0, 0, 0, 0, 1968, 0, + 0, 0, 0, 1973, 0, 1975, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1986, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 527, 527, 527, 527, 0, + 528, 528, 528, 528, 528, 528, 528, 528, + 528, 528, 528, 1998, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 1999, 2000, 2001, 2002, 2003, 32767, 32767, 32767, + 32767, 32767, 2004, 32767, 2005, 2006, 2007, 2008, + 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, + 2025, 2026, 1200, 1200, 32767, 4498, 4499, 2291, + 2032, 2033, 32767, 2034, 32767, 2035, 2036, 32767, + 2037, 2038, 32767, 2039, 2040, 2041, 2042, 2043, + 2044, 2045, 2046, 2047, 2048, 2049, 2050, 2051, + 2052, 2053, 2054, 2055, 2056, 2057, 2058, 2059, + 2060, 2061, 2062, 2063, 2064, 2065, 2066, 2067, + 2068, -1506, -1506, -1506, -1506, -1506, -1506, -1506, + 2076, -2490, 2078, 2079, 2080, 2081, 2082, 2083, + 2084, 2085, 2086, 2087, 2088, 2089, 2090, 2091, + 2092, 2093, 2094, 2095, -105, -2314, 2098, 2099, + 2100, 2101, 2102, 2103, 2104, 2105, 2106, 2107, + 2108, 2109, 2110, 2111, 2112, 2113, 2114, 2115, + 2116, 2117, 2118, 2119, 2120, 5459, 2122, 2123, + 2124, 2125, 2126, 2127, 2128, 5466, 2130, 5467, + 2132, 2133, 2134, 2135, 5471, 2137, 2138, 5473, + 5473, 5473, 5473, 2143, 2144, 5475, 2146, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 2147, 2148, 2149, 2150, 2151, 2152, 2153, 2154, + 2155, 2156, 2157, 2158, 2159, 2160, 2161, 2162, + 2163, 2164, 2165, 2166, 2167, 2168, 2169, 2170, + 2171, 2172, 2173, 2174, 2175, 2176, 2177, 2178, + 2179, 2180, 2181, 2182, 2183, 2184, 2185, 2186, + 2187, 2188, 2189, 2190, 2191, 32767, -726, 2293, + -725, -725, -725, 1357, 1358, 1359, -722, 2201, + 2202, 2203, 2204, 2205, 0, 2207, -715, 2209, + -714, -714, -714, 2213, 2214, 2215, 4806, 2217, + 2218, 2219, 2220, 2221, 2222, 2223, -710, 2225, + -248, -704, -704, 2229, 2230, 2231, 2232, 2233, + 2220, 1430, 2236, 2237, -695, 2239, -694, 2241, + -693, -693, 2244, 2245, 2246, 2231, 2231, 2249, + 2250, 2251, 2252, 2253, 2254, 2255, 2256, 2239, + 2258, 2259, 2260, 2261, 2262, 2263, 2264, 2265, + 2266, 2267, 2268, 2269, 2270, 2271, 2272, 2273, + 2274, 2275, 2276, 2277, 2278, 2279, 2280, 2281, + 2282, 2283, 2284, 2285, 2286, 2287, 2288, 2289, + 2290, 2291, 2292, 2293, 3084, 2295, 2296, 2297, + 2298, 2299, 2300, 2301, 2302, 2303, 2304, 2305, + 2306, 2307, 3983, 2309, 2310, 2292, 2312, 3992, + 2314, 2315, 2316, 2317, 2318, 2319, 2320, 2321, + 2322, 2323, 2324, 2325, 2326, 2327, 2328, 4023, + 2330, 2331, 2332, 2333, 2334, 2335, 2336, 2337, + 2338, 2339, 2340, 2341, 2342, 2343, 2344, 2345, + 2346, 2347, 2348, 2349, 2350, 2351, 2352, 2353, + 2354, 2355, 2356, 2357, 2358, 2359, 2360, 2361, + 2362, 2363, 2364, 2365, 2366, 2367, 2368, 2369, + 2370, 2371, 2372, 2373, 2374, 2375, 2376, 2377, + 2378, 2379, 2360, 2360, 2360, 2360, 2360, 2360, + 2360, 2360, 2360, 2389, 2390, 2391, 2392, 2393, + 2394, 2395, 2396, 2397, 2398, 2399, 2400, 2401, + 2402, 2403, 2404, 2405, 2406, 2407, 2408, 2409, + 2410, 2411, 2412, 2413, 2414, 2415, 2416, 2417, + 2418, 2419, 2420, 2421, 2422, 2423, 2424, 2425, + 2426, 2427, 2428, 2429, 2430, 2431, 2432, 2433, + 2434, 2435, 2436, 2437, 2438, 2439, 2440, 2441, + 2442, 2443, 2444, 2445, 2446, 2447, 32767, 2448, + 2449, 2450, 2451, 2452, 2453, 2454, 2455, 2456, + 2457, 2458, 2459, 2460, 2461, 2462, 2463, 2464, + 2465, 2466, 2467, 2468, 2469, 2470, 2471, 2472, + 2473, 2474, 2475, 2476, 2477, 2478, 2479, 2480, + 2481, 2482, 2483, 2484, 2485, 2486, 2487, 2488, + 2489, 2490, 2491, 2492, 2493, 2494, 2495, 2496, + 2497, 2498, 2499, 2500, 2501, 2502, 2503, 2504, + 2505, 2506, 2507, 2508, 2509, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 2510, + 2511, 2512, 2513, 3266, 3266, 3266, 3266, 2518, + 3267, 3267, 3267, 2522, 3268, 3268, 3268, 3268, + 3268, 3268, 3268, 6682, 3268, 3268, 3268, 2534, + 5151, 3269, 2537, 2538, 3271, 3271, 5157, 2948, + 5159, 2544, 2545, 3273, 2778, 3273, 2549, 3274, + 2551, 3275, 2553, 5175, 2555, 3277, 3277, 3277, + 5181, 2560, 5184, 3278, 5186, 2564, 5189, 5190, + 5191, 5192, 3279, 5194, 5195, 2572, 5198, 32767, + 32767, 3278, 5200, 3278, 2577, 2578, 2579, 2580, + 5210, 3282, 3282, 5213, 3282, 2586, 2587, 2588, + 2589, 2590, 2591, -2175, -2175, -2175, 5230, 3288, + 5232, 3288, 6914, -69, -69, 5097, 5098, 5099, + 1792, -73, -73, 1792, 1792, 1792, 1792, 1792, + 1792, 5091, 5092, 2884, 5094, 1795, 5096, 5097, + 1797, 1797, 6961, 6962, 1797, 1797, 1797, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 2578, 2578, 2578, 2578, 2578, + 2578, 872, 872, 872, 872, 872, 872, 872, + 872, 872, 872, 8277, 6335, 8279, 6335, 9961, + 2978, 2978, 8144, 8145, 8146, 4839, 2974, 2974, + 4839, 4839, 4839, 4839, 4839, 4839, 8138, 8139, + 5931, 8141, 4842, 8143, 8144, 4844, 4844, 10008, + 10009, 4844, 4844, 4844, 8152, 10018, 10019, 8155, + 8156, 8157, 8158, 8159, 8160, 8161, 8162, 7073, + 8164, 5399, 5400, 5401, 5402, 5403, 0, 0, + 0, 5407, 0, 0, 5410, 5411, 5412, 8180, + 3383, 5415, 3914, 3915, 3916, 5419, 3918, 3919, + 3920, 3921, 3922, 3923, 3924, 3925, 3926, 3927, + 3758, 3758, 3758, 3758, 3758, 3758, 3758, 3758, + 3758, 3758, 3758, 3758, 3758, 3758, 3758, 3758, + 3758, 3758, 3758, 3758, 3758, 3758, 3758, 3758, + 3758, 3758, 3758, 3758, 3758, 3758, 3758, 3758, + 3758, 3758, 3758, 3758, 3758, 3758, 3758, 3758, + 3758, 3758, 3758, 3758, 3758, 3758, 3758, 3758, + 3758, 3758, 7172, 3758, 3758, 3758, 3758, 5640, + 3758, 3758, 5643, 3758, 3758, 5644, 3435, 5646, + 3758, 3758, 3758, 3263, 3758, 3758, 3758, 3758, + 3758, 3758, 5657, 3758, 3758, 3758, 3758, 5662, + 3758, 5664, 3758, 5666, 5667, 5668, 5669, 5670, + 5671, 3758, 5673, 5674, 5675, 5676, 3758, 5678, + 3758, 5680, 3758, 5682, 7648, 3758, 3758, 5686, + 3758, 3758, 5689, 3758, 5691, 5692, 3758, -1707, + -1707, -1707, -1707, -1707, -1707, 5698, 3756, 5700, + 3756, 7382, 399, 399, 5565, 5566, 5567, 2260, + 395, 395, 2260, 2260, 2260, 2260, 2260, 2260, + 5559, 5560, 3352, 5562, 2263, 5564, 5565, 2265, + 2265, 7429, 7430, 2265, 2265, 2265, 5573, 7439, + 7440, 5576, 5577, 5578, 5579, 5580, 5581, 5582, + 5583, 4494, 5585, 2820, 2821, 2822, 2823, 2824, + -2579, -2579, -2579, 2828, -2579, -2579, 2831, 2832, + 2833, 5601, 804, 2836, 1335, 1336, 1337, 2840, + 1339, 1340, 1341, 1342, 1343, 1344, 1345, 1346, + 1347, 1348, 1179, 1179, 1179, 1179, 1179, 1179, + 1179, 1179, 1179, 1179, 1179, 1179, 1179, 1179, + 1179, 1179, 1179, 1179, 1179, 1179, 1179, 1179, + 1179, 1179, 1179, 1179, 1179, 1179, 1179, 1179, + 1179, 1179, 1179, 1179, 1179, 1179, 1179, 1179, + 1179, 1179, 1179, 1179, 1179, 1179, 1179, 1179, + 1179, 1179, 1179, 1179, 4593, 1179, 1179, 1179, + 1179, 3061, 1179, 1179, 3064, 1179, 1179, 3065, + 856, 3067, 1179, 1179, 1179, 684, 1179, 1179, + 1179, 1179, 1179, 1179, 3078, 1179, 1179, 1179, + 1179, 3083, 1179, 3085, 1179, 3087, 3088, 3089, + 3090, 3091, 3092, 1179, 3094, 3095, 3096, 3097, + 1179, 3099, 1179, 3101, 1179, 3103, 5069, 1179, + 1179, 3107, 1179, 1179, 3110, 1179, 3112, 3113, + 1179, 3115, 1179, -4284, -4284, -4284, -4284, 3121, + 1179, 3123, 1179, 4805, -2178, -2178, 2988, 2989, + 2990, -317, -2182, -2182, -317, -317, -317, -317, + -317, -317, 2982, 2983, 775, 2985, -314, 2987, + 2988, -312, -312, 4852, 4853, -312, -312, -312, + 2996, 4862, 4863, 2999, 3000, 3001, 3002, 3003, + 3004, 3005, 3006, 1917, 3008, 3009, 3010, 3011, + 3012, 3013, -2150, -2150, 3016, 3017, 3018, 3019, + 3020, 3021, 3022, 3023, 3024, 3025, 3026, 3027, + 3028, 3029, 3030, 3031, 3032, 3033, 3034, 3035, + 32767, 32767, 32767, 3036, 3037, 3038, 3039, 3040, + 3041, 32767, 32767, 3042, 3043, 3044, 3045, 3046, + 3047, 32767, 32767, 3048, 3049, 3050, 3051, 3052, + 3053, 32767, 32767, 3054, 3055, 3056, 32767, 32767, + 32767, -357, 3058, 3059, 3060, 3061, 1180, 3063, + 0, 1179, 3065, 3066, 1181, 3391, 1181, 3070, + 0, 0, 0, 0, 32767, 0, 0, 32767, + 0, 32767, 0, 0, -4973, 32767, 32767, -7368, + -2202, -2201, -2200, -5507, -7372, -7372, -5507, -5507, + -5507, 32767, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 32767, 0, 0, 32767, 0, + -203, -2234, -732, -732, -732, -2234, -732, -732, + -2763, -1261, -1261, -1261, -2763, -1261, -1261, -1261, + -1261, -1261, -1261, -1261, -1261, -1261, -1261, -1091, + -1090, -1089, -1088, -1087, 32767, 32767, -1086, -1085, + -1084, -1083, -1082, -1081, -1080, -1079, -1078, -1077, + -1076, -1075, 32767, -1074, -1073, -1072, -1071, -1070, + -1069, -1068, -1067, -1066, -1065, -1064, -1063, -1062, + -1061, -1060, -1059, -1058, -1057, -1056, 32767, -1055, + -1054, -1053, -1052, 0, 32767, 32767, 32767, -1051, + -1050, -4463, 32767, -1048, 32767, -1047, -2928, -1045, + -1044, -2928, -1042, -1041, -2926, -716, -2926, -1037, + -1036, -1035, -539, -1033, -1032, -1031, -1030, -1029, + -1028, -2926, -1026, -1025, -1024, -1023, -2926, -1021, + -2926, -1019, -2926, -2926, -2926, -2926, -2926, -2926, + -1012, -2926, -2926, -2926, -2926, -1007, -2926, -1005, + -2926, -1003, -2926, -4891, -1000, -999, -2926, -997, + -996, -2926, -994, -2926, -2926, -991, 4475, 4476, + 4477, 4478, 4479, 4480, -2924, -981, -2924, -979, + -4604, 2380, 2381, -2784, -2784, -2784, 524, 2390, + 2391, 527, 528, 529, 530, 531, 532, -2766, + -2766, -557, -2766, 534, -2766, -2766, 535, 536, + -4627, -4627, 539, 540, 541, -2766, -4631, -4631, + -2766, -2766, -2766, -2766, -2766, -2766, -2766, -2766, + -1676, -2766, 0, 0, 0, 0, 0, 5404, + 5405, 5406, 0, 5408, 5409, 0, 0, 0, + -2767, 2031, 0, 1502, 1502, 1502, 0, 1502, + 1502, 1502, 1502, 1502, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 224, 225, 226, 32767, 227, 228, 229, + 230, 231, 232, 233, 234, 235, 236, 67, + 32767, 66, 66, 66, 66, 66, 66, 66, + 66, 66, 66, 66, 66, 66, 66, 66, + 66, 66, 66, 32767, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 65, 65, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, -271, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 1940, 18, 1942, 3908, 18, 18, 1946, 18, + 18, 1949, 18, 1951, 1952, 18, 1954, 18, + -5445, -5445, -5445, -5445, 1960, 18, 1962, 18, + 3644, -3339, -3339, 1827, 1828, 1829, -1478, -3343, + -3343, -1478, -1478, -1478, -1478, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 0, 0, 0, + 0, 32767, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1340, 1341, + 1342, 1343, 1344, 1345, 1346, 1347, 1348, 1349, + -2064, 1351, 1352, 1353, 1354, 32767, 1355, 1356, + 32767, 0, 32767, 32767, 1679, 32767, 1357, 1358, + 1359, 1855, 1361, 1362, 1363, 1364, 1365, 1366, + 32767, 1367, 1368, 1369, 1370, 32767, 1371, 32767, + 1372, 32767, 32767, 32767, 32767, 32767, 32767, 1373, + 32767, 32767, 32767, 32767, 1374, 32767, 1375, 32767, + 1376, 32767, -2513, 1378, 1379, 32767, 1380, 1381, + 32767, 1382, 32767, 32767, 1383, 32767, 1384, 32767, + 6848, 32767, 6849, 32767, 1387, 32767, 1388, 1389, + 32767, 1390, 32767, 32767, 1391, 1392, 1393, 1394, + 32767, 1395, 1396, 1397, 1398, 1399, 1400, 1401, + 32767, 1402, 1403, 1404, 1405, 32767, 1406, 1407, + 1408, 1409, 32767, 1410, 32767, 1411, 1412, 1413, + 1414, 1415, 1416, 1417, 1418, 1419, 1420, 32767, + 1421, 1422, 1423, 1424, 1425, 0, 0, 0, + 4575, 4576, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, -571, -571, + -571, 0, -572, 8101, 5150, 5150, 5150, 5337, + 8103, 5150, 8104, 8105, 2038, 5145, 2039, 5150, + 2040, 5150, 2041, 5150, 5150, 5138, 5150, 5150, + 5150, 5150, 5150, 8108, 8109, 8110, 5150, 5150, + 5384, 5150, 0, 5151, 5151, 5151, 5535, 5151, + 5151, 5151, 5537, 5538, 5539, 5540, 5541, 5542, + 5543, 5544, 5545, 5546, 5547, 5151, 5151, 2572, + 2572, 2572, 2572, 2572, 2572, 2572, 2572, 2572, + 2572, 2572, 2572, 2572, 2572, 2572, 2572, 2572, + 2572, 2572, 2572, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 2580, + 2580, 2580, 2580, 2580, 2580, 2580, 0, 2580, + 0, 2580, 0, 0, 0, 2580, 2580, 2580, + 0, 2580, 2580, 2580, 2580, 2580, 2580, 2580, + 0, 2580, 0, 0, 0, 2580, 2580, 2580, + 5517, 5518, 5519, 5520, 5521, 5522, -6, -6, + -6, 0, 0, 0, 0, 2581, 2581, 2581, + 2581, 5527, 2581, 2581, 2581, 2581, 0, 5528, + 0, 2581, 0, 0, 2581, 2581, 0, 0, + 0, 0, 0, 0, 5529, 5530, 5531, 32767, + 32767, 2579, 2579, 2579, 2579, 2579, 0, 0, + 2579, 2579, 2579, 2765, 0, 0, 0, 0, + 2579, 2579, 2579, 2579, 6066, 2579, 6066, 2579, + 2579, 2579, 0, 0, 0, 2579, 2579, 0, + 0, 0, 2579, 2579, 2579, 5530, 2579, 2579, + 2579, 2766, 5532, 2579, 5533, 5534, -533, 2574, + -532, 2579, -531, 2579, -530, 2579, 2579, 2567, + 2579, 2579, 2579, 2579, 2579, 5537, 5538, 5539, + 2579, 2579, 2813, 2579, 2579, 2579, 2579, 2579, + 2963, 2579, 2579, 2579, 2965, 2966, 2967, 2968, + 2969, 2970, 2971, 2972, 2973, 2974, 2975, 2579, + 2579, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 32767, 32767, 32767, + 32767, 32767, 331, 32767, 332, -2580, -2580, -2580, + -2580, 0, 0, 0, 0, 0, 0, 0, + -2580, 0, -2580, 0, -2580, -2580, -2580, 0, + 0, 0, -2580, 0, 0, 0, 0, 0, + 0, 0, -2580, 0, -2580, -2580, -2580, 0, + 0, 0, 2937, 2938, 2939, 2940, 2941, 2942, + -2586, -2586, -2586, -2580, -2125, -2581, -2581, 0, + 0, 0, 0, 2946, 0, 0, 0, 0, + -2581, 2947, -2581, 0, -2581, -2581, 0, 0, + -2581, -2581, -2581, -2581, -2581, -2581, 2948, 2949, + 2950, 2945, 2491, 0, 0, 0, 0, 0, + -2579, -2579, 0, 0, 0, 186, -2579, -2579, + -2579, -2579, 0, 0, 0, 0, 3487, 0, + 3487, 0, 0, 0, -2579, -2579, -2579, 0, + 0, -2579, -2579, -2579, 0, 0, 0, 2951, + 0, 0, 0, 187, 2953, 0, 2954, 2955, + -3112, -5, -3111, 0, -3110, 0, -3109, 0, + 0, -12, 0, 0, 0, 0, 0, 2958, + 2959, 2960, 0, 0, 234, 0, 0, 0, + 0, 0, 384, 0, 0, 0, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, + 396, 0, 0, 0, 0, 0, 0, 0, + 0, 0, -1706, -1706, -1706, 0, 0, 0, + 0, 385, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 397, + 398, 399, 400, 401, 402, 403, 404, 405, + 2112, 2113, 2114, 409, 410, 411, 412, 32767, + 413, 414, 415, 416, 417, 418, 419, 420, + 421, 422, 423, 424, 425, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + -1688, 32767, 32767, 32767, 32767, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 0, 0, 0, + 0, -752, -751, -750, -749, 0, -748, -747, + -746, 0, -745, -744, -743, -742, -741, -740, + -739, -4152, -737, -736, -735, 0, -2616, -733, + 0, 0, -732, -731, -2616, -406, -2616, 0, + 0, -727, -231, -725, 0, -724, 0, -723, + 0, -2621, 0, -721, -720, -719, -2622, 0, + -2623, -716, -2623, 0, -2624, -2624, -2624, -2624, + -710, -2624, -2624, 0, -2625, -706, -2625, -704, + -2625, -702, 0, 0, 0, 0, -2629, -700, + -699, -2629, -697, 0, 0, 0, 0, 0, + 0, 4767, 4768, 4769, -2635, -692, -2635, -690, + -4315, 2669, 2670, -2495, -2495, -2495, 813, 2679, + 2680, 816, 817, 818, 819, 820, 821, -2477, + -2477, -268, -2477, 823, -2477, -2477, 824, 825, + -4338, -4338, 828, 829, 830, -2477, -4342, -4342, + -2477, -2477, -2477, -2477, -2477, -2477, -2477, -2477, + -1387, 0, 0, 32767, 32767, 0, 0, 0, + 0, 0, -2486, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 1756, 1757, 1758, + 1759, -5645, -3702, -5645, -3700, -7325, -341, -340, + -5505, -5505, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 532, 533, + 32767, 534, 535, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, -781, 1084, 1084, 1084, 1084, + 1084, 1084, 4383, 4384, 2176, 4386, 1087, 4388, + 4389, 1089, 1089, 6253, 6254, 1089, 1089, 1089, + 4397, 6263, 6264, 4400, 4401, 4402, 4403, 4404, + 4405, 4406, 4407, 3318, 4409, 4410, 4411, 4412, + 4413, 4414, -749, -749, 4417, 4418, 4419, 4420, + 4421, 4422, 4423, 4424, 4425, 4426, 4427, 4428, + 4429, 4430, 4431, 4432, 4433, 4434, 4435, 4436, + 4437, 4438, 4439, 4440, 4441, 4442, 4443, 4444, + 4445, 4446, 4447, 4448, 4449, 4450, 4451, 4452, + 4453, 4454, 4455, 4456, 4457, 4458, 4459, 4460, + 4461, 4462, 4463, 4464, 4465, 4466, 4467, 4468, + 4469, 1056, 4471, 4472, 4473, 4474, 2593, 4476, + 4477, 2593, 4479, 4480, 2595, 4805, 2595, 4484, + 4485, 4486, 4982, 4488, 4489, 4490, 4491, 4492, + 4493, 2595, 4495, 4496, 4497, 4498, 2595, 4500, + 2595, 4502, 2595, 2595, 2595, 2595, 2595, 2595, + 4509, 2595, 2595, 2595, 2595, 4514, 2595, 4516, + 2595, 4518, 2595, 630, 4521, 4522, 2595, 4524, + 4525, 2595, 4527, 2595, 2595, 4530, 2595, 4532, + 9996, 9997, 9998, 9999, 2595, 4538, 2595, 4540, + 4541, 2595, 4543, 2595, 2595, 4546, 4547, 4548, + 4549, 2595, 4551, 4552, 4553, 4554, 4555, 4556, + 4557, 2595, 4559, 4560, 4561, 4562, 2595, 4564, + 4565, 4566, 4567, 2595, 4569, 2595, 4571, 4572, + 4573, 4574, 4575, 4576, 4577, 4578, 4579, 4580, + 2595, 4582, 4583, 4584, 4585, 4586, 4587, 4588, + 4589, 4590, 4591, 4592, 4593, 4594, 4595, 4596, + 4597, 4598, 4599, 4600, 4601, 4602, 4603, 4604, + 4605, 4606, 4607, 4608, 4609, 4610, 4611, 4612, + 4613, 4614, 4615, 4089, 4090, 4091, 4092, 4620, + 4093, 4094, 4095, 4096, 4097, 4098, 4099, 4100, + 4101, 4102, 4103, 4104, 2765, 2765, 2765, 2765, + 2765, 2765, 2765, 2765, 2765, 2765, 6179, 2765, + 2765, 2765, 2765, 4647, 2765, 2765, 4650, 4122, + 4652, 4653, 2444, 4655, 2767, 2767, 2767, 2272, + 2767, 2767, 2767, 2767, 2767, 2767, 4666, 2767, + 2767, 2767, 2767, 4671, 2767, 4673, 2767, 4675, + 4676, 4677, 4678, 4679, 4680, 2767, 4682, 4683, + 4684, 4685, 2767, 4687, 2767, 4689, 2767, 4691, + 6657, 2767, 2767, 4695, 2767, 2767, 4698, 2767, + 4700, 4701, 2767, 4703, 2767, -2696, -2696, -2696, + -2696, 4709, 2767, 4711, 2767, 2767, 4714, 2767, + 4716, 4717, 2767, 2767, 2767, 2767, 4722, 2767, + 2767, 2767, 2767, 2767, 2767, 2767, 4730, 2767, + 2767, 2767, 2767, 4735, 2767, 2767, 2767, 2767, + 4740, 2767, 4742, 2767, 2767, 2767, 2767, 2767, + 2767, 2767, 2767, 2767, 2767, 4753, 2767, 2767, + 2767, 2767, 2767, 4193, 4194, 4195, -379, -379, + 4198, 4199, 4200, 4201, 4202, 4203, 4204, 4771, + 4772, 4773, 4774, 4775, 4776, 4777, 4778, 4779, + 4780, -3892, -940, -939, -938, 4785, -3890, -936, + -3889, -3889, 2179, -927, 2180, -930, 2181, -928, + 2182, -926, -925, -912, -923, -922, -921, 4803, + 4804, 4805, 4806, 4807, 4808, 4809, 4810, 4811, + 4812, 4813, 4814, 4815, 4816, 4817, 4818, 4819, + 3925, 4821, 4822, 4823, 4824, 4825, 4826, 4827, + 4828, 4829, 4830, 4831, 4832, 4833, 4834, 4835, + 4836, 4837, 4838, 4839, 4840, 4841, 4842, 4843, + 4844, 4845, 4846, 4847, 4848, 4849, 4850, 4851, + 4852, 4853, 4854, 4855, 4856, 4857, 4858, 4859, + 4860, 4710, 2086, 609, 4864, 4865, 4866, 4867, + 4868, 4869, 4870, 4871, 4872, 3052, 4874, 4875, + 4876, 4281, 4878, 4879, 4880, 4881, 4882, 4883, + 4884, 4885, 4886, 4887, 634, 4888, 4889, 4890, + 4891, 4892, 4893, 4894, 4895, 4896, 1322, 1322, + 1322, 1322, 1322, 1322, 1322, 4904, 338, 4906, + 4907, 4908, 4909, 4910, 4911, 4912, 4913, 4914, + 4915, 4916, 4917, 665, 666, 667, 668, 669, + 670, 671, 672, 673, 674, 675, 676, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 0, + 0, 0, 0, 0, 0, 32767, 0, 0, + 0, 0, 32767, 32767, 0, 0, 0, 0, + 0, 0, 0, 0, 32767, 0, 0, 0, + 0, 0, 0, 0, 32767, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 32767, 0, 0, 0, 2478, 32767, 2477, + 2477, 2477, 2477, 2477, 32767, 2476, 32767, 32767, + 32767, 2473, 2473, 2473, 2473, 2473, 2473, 2473, + 32767, 2472, 2472, 2472, 2472, 2472, 2472, 2472, + 2472, 2472, 2472, 2472, 2472, 2472, 2472, 2472, + 2472, 2472, 2472, 2472, 2472, 2472, 2472, 2472, + 2472, 2472, 2472, 2472, 2472, 2472, 2472, 2472, + 2472, 2472, 2472, 2472, 2472, 2472, 2472, 2472, + 2472, 2472, 2472, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, -2478, -2478, -2478, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0 }; const unsigned char *k = (const unsigned char *) key; diff --git a/src/tools/PerfectHash.pm b/src/tools/PerfectHash.pm index d6841589a39b..964f79b71a27 100644 --- a/src/tools/PerfectHash.pm +++ b/src/tools/PerfectHash.pm @@ -121,13 +121,16 @@ sub generate_hash_function { $f .= sprintf "%s(const void *key, size_t keylen)\n{\n", $funcname; } - $f .= sprintf "\tstatic const %s h[%d] = {\n", $elemtype, $nhash; + $f .= sprintf "\tstatic const %s h[%d] = {\n\t\t", $elemtype, $nhash; for (my $i = 0; $i < $nhash; $i++) { - $f .= sprintf "%s%6d,%s", - ($i % 8 == 0 ? "\t\t" : " "), - $hashtab[$i], - ($i % 8 == 7 ? "\n" : ""); + # Hash element. + $f .= sprintf "%d", $hashtab[$i]; + next if ($i == $nhash - 1); + + # Optional indentation and newline, with eight items per line. + $f .= sprintf ",%s", + ($i % 8 == 7 ? "\n\t\t" : ' ' x (6 - length($hashtab[$i]))); } $f .= sprintf "\n" if ($nhash % 8 != 0); $f .= sprintf "\t};\n\n"; From 8a58347a3c10126ce9eb38872a46c55e91faa174 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 21 Oct 2020 08:17:51 +0200 Subject: [PATCH 345/589] Fix -Wcast-function-type warnings on Windows/MinGW After de8feb1f3a23465b5737e8a8c160e8ca62f61339, some warnings remained that were only visible when using GCC on Windows. Fix those as well. Note that the ecpg test source files don't use the full pg_config.h, so we can't use pg_funcptr_t there but have to do it the long way. --- src/backend/libpq/auth.c | 4 ++-- src/backend/port/win32/crashdump.c | 2 +- src/bin/pg_ctl/pg_ctl.c | 12 ++++++------ src/common/restricted_token.c | 2 +- src/interfaces/ecpg/test/expected/thread-thread.c | 2 +- .../ecpg/test/expected/thread-thread_implicit.c | 2 +- src/interfaces/ecpg/test/thread/thread.pgc | 2 +- src/interfaces/ecpg/test/thread/thread_implicit.pgc | 2 +- src/port/getaddrinfo.c | 12 ++++++------ src/port/win32env.c | 2 +- src/port/win32stat.c | 2 +- 11 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c index 36565df4fc1e..d132c5cb48bd 100644 --- a/src/backend/libpq/auth.c +++ b/src/backend/libpq/auth.c @@ -1521,7 +1521,7 @@ pg_SSPI_recvauth(Port *port) (errmsg("could not load library \"%s\": error code %lu", "SECUR32.DLL", GetLastError()))); - _QuerySecurityContextToken = (QUERY_SECURITY_CONTEXT_TOKEN_FN) + _QuerySecurityContextToken = (QUERY_SECURITY_CONTEXT_TOKEN_FN) (pg_funcptr_t) GetProcAddress(secur32, "QuerySecurityContextToken"); if (_QuerySecurityContextToken == NULL) { @@ -2522,7 +2522,7 @@ InitializeLDAPConnection(Port *port, LDAP **ldap) ldap_unbind(*ldap); return STATUS_ERROR; } - _ldap_start_tls_sA = (__ldap_start_tls_sA) GetProcAddress(ldaphandle, "ldap_start_tls_sA"); + _ldap_start_tls_sA = (__ldap_start_tls_sA) (pg_funcptr_t) GetProcAddress(ldaphandle, "ldap_start_tls_sA"); if (_ldap_start_tls_sA == NULL) { ereport(LOG, diff --git a/src/backend/port/win32/crashdump.c b/src/backend/port/win32/crashdump.c index e6c68379b20e..47114d916cc1 100644 --- a/src/backend/port/win32/crashdump.c +++ b/src/backend/port/win32/crashdump.c @@ -122,7 +122,7 @@ crashDumpHandler(struct _EXCEPTION_POINTERS *pExceptionInfo) return EXCEPTION_CONTINUE_SEARCH; } - pDump = (MINIDUMPWRITEDUMP) GetProcAddress(hDll, "MiniDumpWriteDump"); + pDump = (MINIDUMPWRITEDUMP) (pg_funcptr_t) GetProcAddress(hDll, "MiniDumpWriteDump"); if (pDump == NULL) { diff --git a/src/bin/pg_ctl/pg_ctl.c b/src/bin/pg_ctl/pg_ctl.c index 1cdc3ebaa338..fc07f1aba6e7 100644 --- a/src/bin/pg_ctl/pg_ctl.c +++ b/src/bin/pg_ctl/pg_ctl.c @@ -1778,7 +1778,7 @@ CreateRestrictedProcess(char *cmd, PROCESS_INFORMATION *processInfo, bool as_ser Advapi32Handle = LoadLibrary("ADVAPI32.DLL"); if (Advapi32Handle != NULL) { - _CreateRestrictedToken = (__CreateRestrictedToken) GetProcAddress(Advapi32Handle, "CreateRestrictedToken"); + _CreateRestrictedToken = (__CreateRestrictedToken) (pg_funcptr_t) GetProcAddress(Advapi32Handle, "CreateRestrictedToken"); } if (_CreateRestrictedToken == NULL) @@ -1852,11 +1852,11 @@ CreateRestrictedProcess(char *cmd, PROCESS_INFORMATION *processInfo, bool as_ser Kernel32Handle = LoadLibrary("KERNEL32.DLL"); if (Kernel32Handle != NULL) { - _IsProcessInJob = (__IsProcessInJob) GetProcAddress(Kernel32Handle, "IsProcessInJob"); - _CreateJobObject = (__CreateJobObject) GetProcAddress(Kernel32Handle, "CreateJobObjectA"); - _SetInformationJobObject = (__SetInformationJobObject) GetProcAddress(Kernel32Handle, "SetInformationJobObject"); - _AssignProcessToJobObject = (__AssignProcessToJobObject) GetProcAddress(Kernel32Handle, "AssignProcessToJobObject"); - _QueryInformationJobObject = (__QueryInformationJobObject) GetProcAddress(Kernel32Handle, "QueryInformationJobObject"); + _IsProcessInJob = (__IsProcessInJob) (pg_funcptr_t) GetProcAddress(Kernel32Handle, "IsProcessInJob"); + _CreateJobObject = (__CreateJobObject) (pg_funcptr_t) GetProcAddress(Kernel32Handle, "CreateJobObjectA"); + _SetInformationJobObject = (__SetInformationJobObject) (pg_funcptr_t) GetProcAddress(Kernel32Handle, "SetInformationJobObject"); + _AssignProcessToJobObject = (__AssignProcessToJobObject) (pg_funcptr_t) GetProcAddress(Kernel32Handle, "AssignProcessToJobObject"); + _QueryInformationJobObject = (__QueryInformationJobObject) (pg_funcptr_t) GetProcAddress(Kernel32Handle, "QueryInformationJobObject"); } /* Verify that we found all functions */ diff --git a/src/common/restricted_token.c b/src/common/restricted_token.c index d8d3aeffcdc2..dcc88a75c59d 100644 --- a/src/common/restricted_token.c +++ b/src/common/restricted_token.c @@ -66,7 +66,7 @@ CreateRestrictedProcess(char *cmd, PROCESS_INFORMATION *processInfo) return 0; } - _CreateRestrictedToken = (__CreateRestrictedToken) GetProcAddress(Advapi32Handle, "CreateRestrictedToken"); + _CreateRestrictedToken = (__CreateRestrictedToken) (pg_funcptr_t) GetProcAddress(Advapi32Handle, "CreateRestrictedToken"); if (_CreateRestrictedToken == NULL) { diff --git a/src/interfaces/ecpg/test/expected/thread-thread.c b/src/interfaces/ecpg/test/expected/thread-thread.c index a7e401570a40..0e75c47fab26 100644 --- a/src/interfaces/ecpg/test/expected/thread-thread.c +++ b/src/interfaces/ecpg/test/expected/thread-thread.c @@ -99,7 +99,7 @@ int main() #ifndef WIN32 pthread_create(&threads[n], NULL, test_thread, (void *) (n + 1)); #else - threads[n] = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE)test_thread, (void *) (n + 1), 0, NULL); + threads[n] = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE) (void (*) (void)) test_thread, (void *) (n + 1), 0, NULL); #endif } diff --git a/src/interfaces/ecpg/test/expected/thread-thread_implicit.c b/src/interfaces/ecpg/test/expected/thread-thread_implicit.c index 6c7adb062c80..0df2794530c9 100644 --- a/src/interfaces/ecpg/test/expected/thread-thread_implicit.c +++ b/src/interfaces/ecpg/test/expected/thread-thread_implicit.c @@ -99,7 +99,7 @@ int main() #ifndef WIN32 pthread_create(&threads[n], NULL, test_thread, (void *) (n + 1)); #else - threads[n] = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE) test_thread, (void *) (n+1), 0, NULL); + threads[n] = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE) (void (*) (void)) test_thread, (void *) (n+1), 0, NULL); #endif } diff --git a/src/interfaces/ecpg/test/thread/thread.pgc b/src/interfaces/ecpg/test/thread/thread.pgc index e149b91d976b..e7d8c00af6f4 100644 --- a/src/interfaces/ecpg/test/thread/thread.pgc +++ b/src/interfaces/ecpg/test/thread/thread.pgc @@ -68,7 +68,7 @@ int main() #ifndef WIN32 pthread_create(&threads[n], NULL, test_thread, (void *) (n + 1)); #else - threads[n] = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE)test_thread, (void *) (n + 1), 0, NULL); + threads[n] = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE) (void (*) (void)) test_thread, (void *) (n + 1), 0, NULL); #endif } diff --git a/src/interfaces/ecpg/test/thread/thread_implicit.pgc b/src/interfaces/ecpg/test/thread/thread_implicit.pgc index 3209da22bc54..b4cae7e1aecf 100644 --- a/src/interfaces/ecpg/test/thread/thread_implicit.pgc +++ b/src/interfaces/ecpg/test/thread/thread_implicit.pgc @@ -68,7 +68,7 @@ int main() #ifndef WIN32 pthread_create(&threads[n], NULL, test_thread, (void *) (n + 1)); #else - threads[n] = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE) test_thread, (void *) (n+1), 0, NULL); + threads[n] = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE) (void (*) (void)) test_thread, (void *) (n+1), 0, NULL); #endif } diff --git a/src/port/getaddrinfo.c b/src/port/getaddrinfo.c index 3b51eea4815e..495ad343f392 100644 --- a/src/port/getaddrinfo.c +++ b/src/port/getaddrinfo.c @@ -79,12 +79,12 @@ haveNativeWindowsIPv6routines(void) { /* We found a dll, so now get the addresses of the routines */ - getaddrinfo_ptr = (getaddrinfo_ptr_t) GetProcAddress(hLibrary, - "getaddrinfo"); - freeaddrinfo_ptr = (freeaddrinfo_ptr_t) GetProcAddress(hLibrary, - "freeaddrinfo"); - getnameinfo_ptr = (getnameinfo_ptr_t) GetProcAddress(hLibrary, - "getnameinfo"); + getaddrinfo_ptr = (getaddrinfo_ptr_t) (pg_funcptr_t) GetProcAddress(hLibrary, + "getaddrinfo"); + freeaddrinfo_ptr = (freeaddrinfo_ptr_t) (pg_funcptr_t) GetProcAddress(hLibrary, + "freeaddrinfo"); + getnameinfo_ptr = (getnameinfo_ptr_t) (pg_funcptr_t) GetProcAddress(hLibrary, + "getnameinfo"); /* * If any one of the routines is missing, let's play it safe and diff --git a/src/port/win32env.c b/src/port/win32env.c index 2021f3d5aa6e..177488cc67ea 100644 --- a/src/port/win32env.c +++ b/src/port/win32env.c @@ -95,7 +95,7 @@ pgwin32_putenv(const char *envval) { PUTENVPROC putenvFunc; - putenvFunc = (PUTENVPROC) GetProcAddress(hmodule, "_putenv"); + putenvFunc = (PUTENVPROC) (pg_funcptr_t) GetProcAddress(hmodule, "_putenv"); if (putenvFunc) putenvFunc(envval); FreeLibrary(hmodule); diff --git a/src/port/win32stat.c b/src/port/win32stat.c index 9051c713e7bd..4351aa4d08f2 100644 --- a/src/port/win32stat.c +++ b/src/port/win32stat.c @@ -227,7 +227,7 @@ _pgstat64(const char *name, struct stat *buf) return -1; } - _NtQueryInformationFile = (PFN_NTQUERYINFORMATIONFILE) + _NtQueryInformationFile = (PFN_NTQUERYINFORMATIONFILE) (pg_funcptr_t) GetProcAddress(ntdll, "NtQueryInformationFile"); if (_NtQueryInformationFile == NULL) { From 555eb1a4f0114e1c4017093fe840045dde1ed9ea Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 21 Oct 2020 14:20:50 +0200 Subject: [PATCH 346/589] Remove obsolete ifdefs Commit 8dace66e0735ca39b779922d02c24ea2686e6521 added #ifdefs for a number of errno symbols because they were not present on Windows. Later, commit 125ad539a275db5ab8f4647828b80a16d02eabd2 added replacement #defines for some of those symbols. So some of the changes from the first commit are made dead code by the second commit and can now be removed. Discussion: https://www.postgresql.org/message-id/flat/6dee8574-b0ad-fc49-9c8c-2edc796f0033@2ndquadrant.com --- src/port/strerror.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/port/strerror.c b/src/port/strerror.c index 43a9761d90bd..127bc5ef1fee 100644 --- a/src/port/strerror.c +++ b/src/port/strerror.c @@ -118,14 +118,10 @@ get_errno_symbol(int errnum) return "E2BIG"; case EACCES: return "EACCES"; -#ifdef EADDRINUSE case EADDRINUSE: return "EADDRINUSE"; -#endif -#ifdef EADDRNOTAVAIL case EADDRNOTAVAIL: return "EADDRNOTAVAIL"; -#endif case EAFNOSUPPORT: return "EAFNOSUPPORT"; #ifdef EAGAIN @@ -176,10 +172,8 @@ get_errno_symbol(int errnum) return "EINVAL"; case EIO: return "EIO"; -#ifdef EISCONN case EISCONN: return "EISCONN"; -#endif case EISDIR: return "EISDIR"; #ifdef ELOOP @@ -216,20 +210,16 @@ get_errno_symbol(int errnum) return "ENOSPC"; case ENOSYS: return "ENOSYS"; -#ifdef ENOTCONN case ENOTCONN: return "ENOTCONN"; -#endif case ENOTDIR: return "ENOTDIR"; #if defined(ENOTEMPTY) && (ENOTEMPTY != EEXIST) /* same code on AIX */ case ENOTEMPTY: return "ENOTEMPTY"; #endif -#ifdef ENOTSOCK case ENOTSOCK: return "ENOTSOCK"; -#endif #ifdef ENOTSUP case ENOTSUP: return "ENOTSUP"; From 8a2121185b26d7eb439860c7a96a08c0aa3c8508 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 21 Oct 2020 12:08:48 -0400 Subject: [PATCH 347/589] Remove the option to build thread_test.c outside configure. Theoretically one could go into src/test/thread and build/run this program there. In practice, that hasn't worked since 96bf88d52, and probably much longer on some platforms (likely including just the sort of hoary leftovers where this test might be of interest). While it wouldn't be too hard to repair the breakage, the fact that nobody has noticed for two years shows that there is zero usefulness in maintaining this build pathway. Let's get rid of it and decree that thread_test.c is *only* meant to be built/used in configure. Given that decision, it makes sense to put thread_test.c under config/ and get rid of src/test/thread altogether, so that's what I did. In passing, update src/test/README, which had been ignored by some not-so-recent additions of subdirectories. Discussion: https://postgr.es/m/227659.1603041612@sss.pgh.pa.us --- {src/test/thread => config}/thread_test.c | 30 ++----------- configure | 11 ++--- configure.ac | 10 ++--- src/Makefile | 2 - src/port/thread.c | 3 -- src/test/Makefile | 2 +- src/test/README | 11 +++-- src/test/thread/.gitignore | 1 - src/test/thread/Makefile | 24 ---------- src/test/thread/README | 54 ----------------------- 10 files changed, 20 insertions(+), 128 deletions(-) rename {src/test/thread => config}/thread_test.c (93%) delete mode 100644 src/test/thread/.gitignore delete mode 100644 src/test/thread/Makefile delete mode 100644 src/test/thread/README diff --git a/src/test/thread/thread_test.c b/config/thread_test.c similarity index 93% rename from src/test/thread/thread_test.c rename to config/thread_test.c index 09603c95dd68..ff2eace87d84 100644 --- a/src/test/thread/thread_test.c +++ b/config/thread_test.c @@ -1,12 +1,12 @@ /*------------------------------------------------------------------------- * * thread_test.c - * libc thread test program + * libc threading test program * * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * src/test/thread/thread_test.c + * config/thread_test.c * * This program tests to see if your standard libc functions use * pthread_setspecific()/pthread_getspecific() to be thread-safe. @@ -20,12 +20,7 @@ *------------------------------------------------------------------------- */ -#if !defined(IN_CONFIGURE) && !defined(WIN32) -#include "postgres.h" - -/* we want to know what the native strerror does, not pg_strerror */ -#undef strerror -#endif +/* We cannot use c.h, as port.h will not exist yet */ #include #include @@ -36,6 +31,7 @@ #include #include #include +#include /* CYGWIN requires this for MAXHOSTNAMELEN */ #ifdef __CYGWIN__ @@ -47,25 +43,11 @@ #include #endif - /* Test for POSIX.1c 2-arg sigwait() and fail on single-arg version */ #include int sigwait(const sigset_t *set, int *sig); -#if !defined(ENABLE_THREAD_SAFETY) && !defined(IN_CONFIGURE) && !defined(WIN32) -int -main(int argc, char *argv[]) -{ - fprintf(stderr, "This PostgreSQL build does not support threads.\n"); - fprintf(stderr, "Perhaps rerun 'configure' using '--enable-thread-safety'.\n"); - return 1; -} -#else - -/* This must be down here because this is the code that uses threads. */ -#include - #define TEMP_FILENAME_1 "thread_test.1" #define TEMP_FILENAME_2 "thread_test.2" @@ -119,11 +101,9 @@ main(int argc, char *argv[]) return 1; } -#ifdef IN_CONFIGURE /* Send stdout to 'config.log' */ close(1); dup(5); -#endif #ifdef WIN32 err = WSAStartup(MAKEWORD(2, 2), &wsaData); @@ -455,5 +435,3 @@ func_call_2(void) pthread_mutex_lock(&init_mutex); /* wait for parent to test */ pthread_mutex_unlock(&init_mutex); } - -#endif /* !ENABLE_THREAD_SAFETY && !IN_CONFIGURE */ diff --git a/configure b/configure index 071d050ef009..ace4ed5decf1 100755 --- a/configure +++ b/configure @@ -18992,23 +18992,21 @@ $as_echo_n "checking thread safety of required library functions... " >&6; } _CFLAGS="$CFLAGS" _LIBS="$LIBS" -CFLAGS="$CFLAGS $PTHREAD_CFLAGS -DIN_CONFIGURE" +CFLAGS="$CFLAGS $PTHREAD_CFLAGS" LIBS="$LIBS $PTHREAD_LIBS" if test "$cross_compiling" = yes; then : { $as_echo "$as_me:${as_lineno-$LINENO}: result: maybe" >&5 $as_echo "maybe" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: *** Skipping thread test program because of cross-compile build. -*** Run the program in src/test/thread on the target machine. " >&5 $as_echo "$as_me: WARNING: *** Skipping thread test program because of cross-compile build. -*** Run the program in src/test/thread on the target machine. " >&2;} else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include "$srcdir/src/test/thread/thread_test.c" +#include "$srcdir/config/thread_test.c" _ACEOF if ac_fn_c_try_run "$LINENO"; then : { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 @@ -19017,9 +19015,8 @@ else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } as_fn_error $? "thread test program failed -This platform is not thread-safe. Check the file 'config.log' or compile -and run src/test/thread/thread_test for the exact reason. -Use --disable-thread-safety to disable thread safety." "$LINENO" 5 +This platform is not thread-safe. Check the file 'config.log' for the +exact reason, or use --disable-thread-safety to disable thread safety." "$LINENO" 5 fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ conftest.$ac_objext conftest.beam conftest.$ac_ext diff --git a/configure.ac b/configure.ac index e9ce611d23cb..5b91c83fd07c 100644 --- a/configure.ac +++ b/configure.ac @@ -2295,20 +2295,18 @@ AC_MSG_CHECKING([thread safety of required library functions]) _CFLAGS="$CFLAGS" _LIBS="$LIBS" -CFLAGS="$CFLAGS $PTHREAD_CFLAGS -DIN_CONFIGURE" +CFLAGS="$CFLAGS $PTHREAD_CFLAGS" LIBS="$LIBS $PTHREAD_LIBS" AC_RUN_IFELSE( - [AC_LANG_SOURCE([[#include "$srcdir/src/test/thread/thread_test.c"]])], + [AC_LANG_SOURCE([[#include "$srcdir/config/thread_test.c"]])], [AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no) AC_MSG_ERROR([thread test program failed -This platform is not thread-safe. Check the file 'config.log' or compile -and run src/test/thread/thread_test for the exact reason. -Use --disable-thread-safety to disable thread safety.])], +This platform is not thread-safe. Check the file 'config.log' for the +exact reason, or use --disable-thread-safety to disable thread safety.])], [AC_MSG_RESULT(maybe) AC_MSG_WARN([ *** Skipping thread test program because of cross-compile build. -*** Run the program in src/test/thread on the target machine. ])]) CFLAGS="$_CFLAGS" LIBS="$_LIBS" diff --git a/src/Makefile b/src/Makefile index bcdbd9588aa4..79e274a4769b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -66,13 +66,11 @@ clean: $(MAKE) -C test $@ $(MAKE) -C tutorial NO_PGXS=1 $@ $(MAKE) -C test/isolation $@ - $(MAKE) -C test/thread $@ distclean maintainer-clean: $(MAKE) -C test $@ $(MAKE) -C tutorial NO_PGXS=1 $@ $(MAKE) -C test/isolation $@ - $(MAKE) -C test/thread $@ rm -f Makefile.port Makefile.global diff --git a/src/port/thread.c b/src/port/thread.c index 171df0f9e381..0941cb6a88f4 100644 --- a/src/port/thread.c +++ b/src/port/thread.c @@ -47,9 +47,6 @@ * use non-*_r functions if they are thread-safe * * One thread-safe solution for gethostbyname() might be to use getaddrinfo(). - * - * Run src/test/thread to test if your operating system has thread-safe - * non-*_r functions. */ diff --git a/src/test/Makefile b/src/test/Makefile index efb206aa7501..9774f534d93f 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -37,7 +37,7 @@ endif # clean" etc to recurse into them. (We must filter out those that we # have conditionally included into SUBDIRS above, else there will be # make confusion.) -ALWAYS_SUBDIRS = $(filter-out $(SUBDIRS),examples kerberos ldap locale thread ssl) +ALWAYS_SUBDIRS = $(filter-out $(SUBDIRS),examples kerberos ldap locale ssl) # We want to recurse to all subdirs for all standard targets, except that # installcheck and install should not recurse into the subdirectory "modules". diff --git a/src/test/README b/src/test/README index b5ccfc0cf673..afdc76765190 100644 --- a/src/test/README +++ b/src/test/README @@ -9,7 +9,7 @@ Not all these tests get run by "make check". Check src/test/Makefile to see which tests get run automatically. authentication/ - Tests for authentication + Tests for authentication (but see also below) examples/ Demonstration programs for libpq that double as regression tests via @@ -18,6 +18,12 @@ examples/ isolation/ Tests for concurrent behavior at the SQL level +kerberos/ + Tests for Kerberos/GSSAPI authentication and encryption + +ldap/ + Tests for LDAP-based authentication + locale/ Sanity checks for locale data, encodings, etc @@ -42,6 +48,3 @@ ssl/ subscription/ Tests for logical replication - -thread/ - A thread-safety-testing utility used by configure diff --git a/src/test/thread/.gitignore b/src/test/thread/.gitignore deleted file mode 100644 index 1d54d546a8ce..000000000000 --- a/src/test/thread/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/thread_test diff --git a/src/test/thread/Makefile b/src/test/thread/Makefile deleted file mode 100644 index a13c0c6cf530..000000000000 --- a/src/test/thread/Makefile +++ /dev/null @@ -1,24 +0,0 @@ -#------------------------------------------------------------------------- -# -# Makefile for tools/thread -# -# Copyright (c) 2003-2020, PostgreSQL Global Development Group -# -# src/test/thread/Makefile -# -#------------------------------------------------------------------------- - -subdir = src/tools/thread -top_builddir = ../../.. -include $(top_builddir)/src/Makefile.global - -override CFLAGS += $(PTHREAD_CFLAGS) - -all: thread_test - -thread_test: thread_test.o -# no need for $LIBS, might not be compiled yet - $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(PTHREAD_LIBS) -o $@$(X) - -clean distclean maintainer-clean: - rm -f thread_test$(X) thread_test.o diff --git a/src/test/thread/README b/src/test/thread/README deleted file mode 100644 index 4da23440f6b5..000000000000 --- a/src/test/thread/README +++ /dev/null @@ -1,54 +0,0 @@ -src/test/thread/README - -Threading -========= - -This program is run by configure to determine if threading is -properly supported on the platform. - -You can run the program manually to see details, which shows if your -native libc functions are thread-safe, or if we use *_r functions or -thread locking. - -To use this program manually, you must: - - o run "configure" - o compile the main source tree - o compile and run this program - -If your platform requires special thread flags that are not tested by -/config/acx_pthread.m4, add PTHREAD_CFLAGS and PTHREAD_LIBS defines to -your template/${port} file. - -Windows Systems -=============== - -Windows systems do not vary in their thread-safeness in the same way that -other systems might, nor do they generally have pthreads installed, hence -on Windows this test is skipped by the configure program (pthreads is -required by the test program, but not PostgreSQL itself). If you do wish -to test your system however, you can do so as follows: - -1) Install pthreads in you Mingw/Msys environment. You can download pthreads - from ftp://sources.redhat.com/pub/pthreads-win32/. - -2) Build the test program: - - gcc -o thread_test.exe \ - -D_REENTRANT \ - -D_THREAD_SAFE \ - -D_POSIX_PTHREAD_SEMANTICS \ - -I../../../src/include/port/win32 \ - thread_test.c \ - -lws2_32 \ - -lpthreadgc2 - -3) Run thread_test.exe. You should see output like: - - dpage@PC30:/cvs/pgsql/src/tools/thread$ ./thread_test - Your GetLastError() is thread-safe. - Your system uses strerror() which is thread-safe. - getpwuid_r()/getpwuid() are not applicable to Win32 platforms. - Your system uses gethostbyname which is thread-safe. - - Your platform is thread-safe. From 831611b11c34ed0066633864f42ff67a10286aee Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Wed, 21 Oct 2020 14:37:26 -0300 Subject: [PATCH 348/589] Use fast checkpoint in PostgresNode::backup() Should cause tests to be a bit faster --- src/test/perl/PostgresNode.pm | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index 97e9d932ce70..ebcaeb44fec5 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -553,8 +553,10 @@ sub backup my $name = $self->name; print "# Taking pg_basebackup $backup_name from node \"$name\"\n"; - TestLib::system_or_bail('pg_basebackup', '-D', $backup_path, '-h', - $self->host, '-p', $self->port, '--no-sync'); + TestLib::system_or_bail( + 'pg_basebackup', '-D', $backup_path, '-h', + $self->host, '-p', $self->port, '--checkpoint', + 'fast', '--no-sync'); print "# Backup finished\n"; return; } From 85c54287af56fe351b53913ea2b81e9d6145f964 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 21 Oct 2020 16:18:40 -0400 Subject: [PATCH 349/589] Fix connection string handling in psql's \connect command. psql's \connect claims to be able to re-use previous connection parameters, but in fact it only re-uses the database name, user name, host name (and possibly hostaddr, depending on version), and port. This is problematic for assorted use cases. Notably, pg_dump[all] emits "\connect databasename" commands which we would like to have re-use all other parameters. If such a script is loaded in a psql run that initially had "-d connstring" with some non-default parameters, those other parameters would be lost, potentially causing connection failure. (Thus, this is the same kind of bug addressed in commits a45bc8a4f and 8e5793ab6, although the details are much different.) To fix, redesign do_connect() so that it pulls out all properties of the old PGconn using PQconninfo(), and then replaces individual properties in that array. In the case where we don't wish to re-use anything, get libpq's default settings using PQconndefaults() and replace entries in that, so that we don't need different code paths for the two cases. This does result in an additional behavioral change for cases where the original connection parameters allowed multiple hosts, say "psql -h host1,host2", and the \connect request allows re-use of the host setting. Because the previous coding relied on PQhost(), it would only permit reconnection to the same host originally selected. Although one can think of scenarios where that's a good thing, there are others where it is not. Moreover, that behavior doesn't seem to meet the principle of least surprise, nor was it documented; nor is it even clear it was intended, since that coding long pre-dates the addition of multi-host support to libpq. Hence, this patch is content to drop it and re-use the host list as given. Per Peter Eisentraut's comments on bug #16604. Back-patch to all supported branches. Discussion: https://postgr.es/m/16604-933f4b8791227b15@postgresql.org --- doc/src/sgml/ref/psql-ref.sgml | 45 +++-- src/bin/psql/command.c | 298 +++++++++++++++++++++------------ 2 files changed, 217 insertions(+), 126 deletions(-) diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml index ee3fc0957798..7d0d3616573f 100644 --- a/doc/src/sgml/ref/psql-ref.sgml +++ b/doc/src/sgml/ref/psql-ref.sgml @@ -889,35 +889,47 @@ testdb=> Establishes a new connection to a PostgreSQL server. The connection parameters to use can be specified either - using a positional syntax, or using conninfo connection - strings as detailed in . + using a positional syntax (one or more of database name, user, + host, and port), or using a conninfo + connection string as detailed in + . If no arguments are given, a + new connection is made using the same parameters as before. - Where the command omits database name, user, host, or port, the new - connection can reuse values from the previous connection. By default, - values from the previous connection are reused except when processing - a conninfo string. Passing a first argument - of -reuse-previous=on - or -reuse-previous=off overrides that default. - When the command neither specifies nor reuses a particular parameter, - the libpq default is used. Specifying any + Specifying any of dbname, username, host or port as - is equivalent to omitting that parameter. - If hostaddr was specified in the original - connection's conninfo, that address is reused - for the new connection (disregarding any other host specification). + + + + The new connection can re-use connection parameters from the previous + connection; not only database name, user, host, and port, but other + settings such as sslmode. By default, + parameters are re-used in the positional syntax, but not when + a conninfo string is given. Passing a + first argument of -reuse-previous=on + or -reuse-previous=off overrides that default. If + parameters are re-used, then any parameter not explicitly specified as + a positional parameter or in the conninfo + string is taken from the existing connection's parameters. An + exception is that if the host setting + is changed from its previous value using the positional syntax, + any hostaddr setting present in the + existing connection's parameters is dropped. + When the command neither specifies nor reuses a particular parameter, + the libpq default is used. If the new connection is successfully made, the previous connection is closed. - If the connection attempt failed (wrong user name, access - denied, etc.), the previous connection will only be kept if - psql is in interactive mode. When + If the connection attempt fails (wrong user name, access + denied, etc.), the previous connection will be kept if + psql is in interactive mode. But when executing a non-interactive script, processing will immediately stop with an error. This distinction was chosen as a user convenience against typos on the one hand, and a safety @@ -932,6 +944,7 @@ testdb=> => \c mydb myuser host.dom 6432 => \c service=foo => \c "host=localhost port=5432 dbname=mydb connect_timeout=10 sslmode=disable" +=> \c -reuse-previous=on sslmode=require -- changes only sslmode => \c postgresql://tom@localhost/mydb?application_name=myapp diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c index d4aa0976b5bf..ba3ea39aaa4d 100644 --- a/src/bin/psql/command.c +++ b/src/bin/psql/command.c @@ -3011,26 +3011,6 @@ param_is_newly_set(const char *old_val, const char *new_val) return false; } -/* return whether the connection has 'hostaddr' in its conninfo */ -static bool -has_hostaddr(PGconn *conn) -{ - bool used = false; - PQconninfoOption *ciopt = PQconninfo(conn); - - for (PQconninfoOption *p = ciopt; p->keyword != NULL; p++) - { - if (strcmp(p->keyword, "hostaddr") == 0 && p->val != NULL) - { - used = true; - break; - } - } - - PQconninfoFree(ciopt); - return used; -} - /* * do_connect -- handler for \connect * @@ -3048,13 +3028,15 @@ do_connect(enum trivalue reuse_previous_specification, char *dbname, char *user, char *host, char *port) { PGconn *o_conn = pset.db, - *n_conn; + *n_conn = NULL; + PQconninfoOption *cinfo; + int nconnopts = 0; + bool same_host = false; char *password = NULL; - char *hostaddr = NULL; - bool keep_password; + bool success = true; + bool keep_password = true; bool has_connection_string; bool reuse_previous; - PQExpBufferData connstr; if (!o_conn && (!dbname || !user || !host || !port)) { @@ -3096,55 +3078,125 @@ do_connect(enum trivalue reuse_previous_specification, } /* - * Grab missing values from the old connection. If we grab host (or host - * is the same as before) and hostaddr was set, grab that too. + * If we intend to re-use connection parameters, collect them out of the + * old connection, then replace individual values as necessary. Otherwise, + * obtain a PQconninfoOption array containing libpq's defaults, and modify + * that. Note this function assumes that PQconninfo, PQconndefaults, and + * PQconninfoParse will all produce arrays containing the same options in + * the same order. */ if (reuse_previous) + cinfo = PQconninfo(o_conn); + else + cinfo = PQconndefaults(); + + if (cinfo) { - if (!user) - user = PQuser(o_conn); - if (host && strcmp(host, PQhost(o_conn)) == 0 && - has_hostaddr(o_conn)) + if (has_connection_string) { - hostaddr = PQhostaddr(o_conn); + /* Parse the connstring and insert values into cinfo */ + PQconninfoOption *replcinfo; + char *errmsg; + + replcinfo = PQconninfoParse(dbname, &errmsg); + if (replcinfo) + { + PQconninfoOption *ci; + PQconninfoOption *replci; + + for (ci = cinfo, replci = replcinfo; + ci->keyword && replci->keyword; + ci++, replci++) + { + Assert(strcmp(ci->keyword, replci->keyword) == 0); + /* Insert value from connstring if one was provided */ + if (replci->val) + { + /* + * We know that both val strings were allocated by + * libpq, so the least messy way to avoid memory leaks + * is to swap them. + */ + char *swap = replci->val; + + replci->val = ci->val; + ci->val = swap; + } + } + Assert(ci->keyword == NULL && replci->keyword == NULL); + + /* While here, determine how many option slots there are */ + nconnopts = ci - cinfo; + + PQconninfoFree(replcinfo); + + /* We never re-use a password with a conninfo string. */ + keep_password = false; + + /* Don't let code below try to inject dbname into params. */ + dbname = NULL; + } + else + { + /* PQconninfoParse failed */ + if (errmsg) + { + pg_log_error("%s", errmsg); + PQfreemem(errmsg); + } + else + pg_log_error("out of memory"); + success = false; + } } - if (!host) + else { - host = PQhost(o_conn); - if (has_hostaddr(o_conn)) - hostaddr = PQhostaddr(o_conn); + /* + * If dbname isn't a connection string, then we'll inject it and + * the other parameters into the keyword array below. (We can't + * easily insert them into the cinfo array because of memory + * management issues: PQconninfoFree would misbehave on Windows.) + * However, to avoid dependencies on the order in which parameters + * appear in the array, make a preliminary scan to set + * keep_password and same_host correctly. + * + * While any change in user, host, or port causes us to ignore the + * old connection's password, we don't force that for dbname, + * since passwords aren't database-specific. + */ + PQconninfoOption *ci; + + for (ci = cinfo; ci->keyword; ci++) + { + if (user && strcmp(ci->keyword, "user") == 0) + { + if (!(ci->val && strcmp(user, ci->val) == 0)) + keep_password = false; + } + else if (host && strcmp(ci->keyword, "host") == 0) + { + if (ci->val && strcmp(host, ci->val) == 0) + same_host = true; + else + keep_password = false; + } + else if (port && strcmp(ci->keyword, "port") == 0) + { + if (!(ci->val && strcmp(port, ci->val) == 0)) + keep_password = false; + } + } + + /* While here, determine how many option slots there are */ + nconnopts = ci - cinfo; } - if (!port) - port = PQport(o_conn); } - - /* - * Any change in the parameters read above makes us discard the password. - * We also discard it if we're to use a conninfo rather than the - * positional syntax. - */ - if (has_connection_string) - keep_password = false; else - keep_password = - (user && PQuser(o_conn) && strcmp(user, PQuser(o_conn)) == 0) && - (host && PQhost(o_conn) && strcmp(host, PQhost(o_conn)) == 0) && - (port && PQport(o_conn) && strcmp(port, PQport(o_conn)) == 0); - - /* - * Grab missing dbname from old connection. No password discard if this - * changes: passwords aren't (usually) database-specific. - */ - if (!dbname && reuse_previous) { - initPQExpBuffer(&connstr); - appendPQExpBufferStr(&connstr, "dbname="); - appendConnStrVal(&connstr, PQdb(o_conn)); - dbname = connstr.data; - /* has_connection_string=true would be a dead store */ + /* We failed to create the cinfo structure */ + pg_log_error("out of memory"); + success = false; } - else - connstr.data = NULL; /* * If the user asked to be prompted for a password, ask for one now. If @@ -3156,13 +3208,13 @@ do_connect(enum trivalue reuse_previous_specification, * the postmaster's log. But libpq offers no API that would let us obtain * a password and then continue with the first connection attempt. */ - if (pset.getPassword == TRI_YES) + if (pset.getPassword == TRI_YES && success) { /* - * If a connstring or URI is provided, we can't be sure we know which - * username will be used, since we haven't parsed that argument yet. + * If a connstring or URI is provided, we don't know which username + * will be used, since we haven't dug that out of the connstring. * Don't risk issuing a misleading prompt. As in startup.c, it does - * not seem worth working harder, since this getPassword option is + * not seem worth working harder, since this getPassword setting is * normally only used in noninteractive cases. */ password = prompt_for_password(has_connection_string ? NULL : user); @@ -3176,57 +3228,60 @@ do_connect(enum trivalue reuse_previous_specification, password = NULL; } - while (true) + /* Loop till we have a connection or fail, which we might've already */ + while (success) { -#define PARAMS_ARRAY_SIZE 9 - const char **keywords = pg_malloc(PARAMS_ARRAY_SIZE * sizeof(*keywords)); - const char **values = pg_malloc(PARAMS_ARRAY_SIZE * sizeof(*values)); - int paramnum = -1; - - keywords[++paramnum] = "host"; - values[paramnum] = host; - if (hostaddr && *hostaddr) - { - keywords[++paramnum] = "hostaddr"; - values[paramnum] = hostaddr; - } - keywords[++paramnum] = "port"; - values[paramnum] = port; - keywords[++paramnum] = "user"; - values[paramnum] = user; + const char **keywords = pg_malloc((nconnopts + 1) * sizeof(*keywords)); + const char **values = pg_malloc((nconnopts + 1) * sizeof(*values)); + int paramnum = 0; + PQconninfoOption *ci; /* - * Position in the array matters when the dbname is a connection - * string, because settings in a connection string override earlier - * array entries only. Thus, user= in the connection string always - * takes effect, but client_encoding= often will not. + * Copy non-default settings into the PQconnectdbParams parameter + * arrays; but override any values specified old-style, as well as the + * password and a couple of fields we want to set forcibly. * - * If you change this code, also change the initial-connection code in + * If you change this code, see also the initial-connection code in * main(). For no good reason, a connection string password= takes * precedence in main() but not here. */ - keywords[++paramnum] = "dbname"; - values[paramnum] = dbname; - keywords[++paramnum] = "password"; - values[paramnum] = password; - keywords[++paramnum] = "fallback_application_name"; - values[paramnum] = pset.progname; - keywords[++paramnum] = "client_encoding"; - values[paramnum] = (pset.notty || getenv("PGCLIENTENCODING")) ? NULL : "auto"; - + for (ci = cinfo; ci->keyword; ci++) + { + keywords[paramnum] = ci->keyword; + + if (dbname && strcmp(ci->keyword, "dbname") == 0) + values[paramnum++] = dbname; + else if (user && strcmp(ci->keyword, "user") == 0) + values[paramnum++] = user; + else if (host && strcmp(ci->keyword, "host") == 0) + values[paramnum++] = host; + else if (host && !same_host && strcmp(ci->keyword, "hostaddr") == 0) + { + /* If we're changing the host value, drop any old hostaddr */ + values[paramnum++] = NULL; + } + else if (port && strcmp(ci->keyword, "port") == 0) + values[paramnum++] = port; + else if (strcmp(ci->keyword, "password") == 0) + values[paramnum++] = password; + else if (strcmp(ci->keyword, "fallback_application_name") == 0) + values[paramnum++] = pset.progname; + else if (strcmp(ci->keyword, "client_encoding") == 0) + values[paramnum++] = (pset.notty || getenv("PGCLIENTENCODING")) ? NULL : "auto"; + else if (ci->val) + values[paramnum++] = ci->val; + /* else, don't bother making libpq parse this keyword */ + } /* add array terminator */ - keywords[++paramnum] = NULL; + keywords[paramnum] = NULL; values[paramnum] = NULL; - n_conn = PQconnectdbParams(keywords, values, true); + /* Note we do not want libpq to re-expand the dbname parameter */ + n_conn = PQconnectdbParams(keywords, values, false); pg_free(keywords); pg_free(values); - /* We can immediately discard the password -- no longer needed */ - if (password) - pg_free(password); - if (PQstatus(n_conn) == CONNECTION_OK) break; @@ -3242,9 +3297,28 @@ do_connect(enum trivalue reuse_previous_specification, */ password = prompt_for_password(PQuser(n_conn)); PQfinish(n_conn); + n_conn = NULL; continue; } + /* + * We'll report the error below ... unless n_conn is NULL, indicating + * that libpq didn't have enough memory to make a PGconn. + */ + if (n_conn == NULL) + pg_log_error("out of memory"); + + success = false; + } /* end retry loop */ + + /* Release locally allocated data, whether we succeeded or not */ + if (password) + pg_free(password); + if (cinfo) + PQconninfoFree(cinfo); + + if (!success) + { /* * Failed to connect to the database. In interactive mode, keep the * previous connection to the DB; in scripting mode, close our @@ -3252,7 +3326,11 @@ do_connect(enum trivalue reuse_previous_specification, */ if (pset.cur_cmd_interactive) { - pg_log_info("%s", PQerrorMessage(n_conn)); + if (n_conn) + { + pg_log_info("%s", PQerrorMessage(n_conn)); + PQfinish(n_conn); + } /* pset.db is left unmodified */ if (o_conn) @@ -3260,7 +3338,12 @@ do_connect(enum trivalue reuse_previous_specification, } else { - pg_log_error("\\connect: %s", PQerrorMessage(n_conn)); + if (n_conn) + { + pg_log_error("\\connect: %s", PQerrorMessage(n_conn)); + PQfinish(n_conn); + } + if (o_conn) { /* @@ -3274,13 +3357,8 @@ do_connect(enum trivalue reuse_previous_specification, } } - PQfinish(n_conn); - if (connstr.data) - termPQExpBuffer(&connstr); return false; } - if (connstr.data) - termPQExpBuffer(&connstr); /* * Replace the old connection with the new one, and update From e7c2b95d37a2b9c01367f7ccc55703555b39c81c Mon Sep 17 00:00:00 2001 From: David Rowley Date: Thu, 22 Oct 2020 14:36:32 +1300 Subject: [PATCH 350/589] Optimize a few list_delete_ptr calls There is a handful of places where we called list_delete_ptr() to remove some element from a List. In many of these places we know, or with very little additional effort know the index of the ListCell that we need to remove. Here we change all of those places to instead either use one of; list_delete_nth_cell(), foreach_delete_current() or list_delete_last(). Each of these saves from having to iterate over the list to search for the element to remove by its pointer value. There are some small performance gains to be had by doing this, but in the general case, none of these lists are likely to be very large, so the lookup was probably never that expensive anyway. However, some of the calls are in fairly hot code paths, e.g process_equivalence(). So any small gains there are useful. Author: Zhijie Hou and David Rowley Discussion: https://postgr.es/m/b3517353ec7c4f87aa560678fbb1034b@G08CNEXMBPEKD05.g08.fujitsu.local --- src/backend/optimizer/path/equivclass.c | 9 +++++++-- src/backend/optimizer/path/joinpath.c | 4 ++-- src/backend/parser/parse_expr.c | 5 +++-- src/backend/parser/parse_utilcmd.c | 4 +++- src/backend/rewrite/rewriteHandler.c | 6 +----- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 823422edad02..690b753369e8 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -137,6 +137,7 @@ process_equivalence(PlannerInfo *root, EquivalenceMember *em1, *em2; ListCell *lc1; + int ec2_idx; /* Should not already be marked as having generated an eclass */ Assert(restrictinfo->left_ec == NULL); @@ -258,6 +259,7 @@ process_equivalence(PlannerInfo *root, */ ec1 = ec2 = NULL; em1 = em2 = NULL; + ec2_idx = -1; foreach(lc1, root->eq_classes) { EquivalenceClass *cur_ec = (EquivalenceClass *) lfirst(lc1); @@ -311,6 +313,7 @@ process_equivalence(PlannerInfo *root, equal(item2, cur_em->em_expr)) { ec2 = cur_ec; + ec2_idx = foreach_current_index(lc1); em2 = cur_em; if (ec1) break; @@ -371,7 +374,7 @@ process_equivalence(PlannerInfo *root, ec1->ec_max_security = Max(ec1->ec_max_security, ec2->ec_max_security); ec2->ec_merged = ec1; - root->eq_classes = list_delete_ptr(root->eq_classes, ec2); + root->eq_classes = list_delete_nth_cell(root->eq_classes, ec2_idx); /* just to avoid debugging confusion w/ dangling pointers: */ ec2->ec_members = NIL; ec2->ec_sources = NIL; @@ -1964,6 +1967,7 @@ reconsider_full_join_clause(PlannerInfo *root, RestrictInfo *rinfo) bool matchleft; bool matchright; ListCell *lc2; + int coal_idx = -1; /* Ignore EC unless it contains pseudoconstants */ if (!cur_ec->ec_has_const) @@ -2008,6 +2012,7 @@ reconsider_full_join_clause(PlannerInfo *root, RestrictInfo *rinfo) if (equal(leftvar, cfirst) && equal(rightvar, csecond)) { + coal_idx = foreach_current_index(lc2); match = true; break; } @@ -2072,7 +2077,7 @@ reconsider_full_join_clause(PlannerInfo *root, RestrictInfo *rinfo) */ if (matchleft && matchright) { - cur_ec->ec_members = list_delete_ptr(cur_ec->ec_members, coal_em); + cur_ec->ec_members = list_delete_nth_cell(cur_ec->ec_members, coal_idx); return true; } diff --git a/src/backend/optimizer/path/joinpath.c b/src/backend/optimizer/path/joinpath.c index db54a6ba2ea9..4a35903b29f7 100644 --- a/src/backend/optimizer/path/joinpath.c +++ b/src/backend/optimizer/path/joinpath.c @@ -1005,8 +1005,8 @@ sort_inner_and_outer(PlannerInfo *root, /* Make a pathkey list with this guy first */ if (l != list_head(all_pathkeys)) outerkeys = lcons(front_pathkey, - list_delete_ptr(list_copy(all_pathkeys), - front_pathkey)); + list_delete_nth_cell(list_copy(all_pathkeys), + foreach_current_index(l))); else outerkeys = all_pathkeys; /* no work at first one... */ diff --git a/src/backend/parser/parse_expr.c b/src/backend/parser/parse_expr.c index d24420c58319..f5165863d779 100644 --- a/src/backend/parser/parse_expr.c +++ b/src/backend/parser/parse_expr.c @@ -1698,11 +1698,12 @@ transformMultiAssignRef(ParseState *pstate, MultiAssignRef *maref) /* * If we're at the last column, delete the RowExpr from * p_multiassign_exprs; we don't need it anymore, and don't want it in - * the finished UPDATE tlist. + * the finished UPDATE tlist. We assume this is still the last entry + * in p_multiassign_exprs. */ if (maref->colno == maref->ncolumns) pstate->p_multiassign_exprs = - list_delete_ptr(pstate->p_multiassign_exprs, tle); + list_delete_last(pstate->p_multiassign_exprs); return result; } diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index 0dc03dd98408..015b0538e339 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -360,6 +360,7 @@ generateSerialExtraStmts(CreateStmtContext *cxt, ColumnDef *column, CreateSeqStmt *seqstmt; AlterSeqStmt *altseqstmt; List *attnamelist; + int nameEl_idx = -1; /* * Determine namespace and name to use for the sequence. @@ -386,6 +387,7 @@ generateSerialExtraStmts(CreateStmtContext *cxt, ColumnDef *column, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("conflicting or redundant options"))); nameEl = defel; + nameEl_idx = foreach_current_index(option); } } @@ -405,7 +407,7 @@ generateSerialExtraStmts(CreateStmtContext *cxt, ColumnDef *column, } sname = rv->relname; /* Remove the SEQUENCE NAME item from seqoptions */ - seqoptions = list_delete_ptr(seqoptions, nameEl); + seqoptions = list_delete_nth_cell(seqoptions, nameEl_idx); } else { diff --git a/src/backend/rewrite/rewriteHandler.c b/src/backend/rewrite/rewriteHandler.c index fe777c3103df..1faaafab08a6 100644 --- a/src/backend/rewrite/rewriteHandler.c +++ b/src/backend/rewrite/rewriteHandler.c @@ -650,11 +650,7 @@ adjustJoinTreeList(Query *parsetree, bool removert, int rt_index) if (IsA(rtr, RangeTblRef) && rtr->rtindex == rt_index) { - newjointree = list_delete_ptr(newjointree, rtr); - - /* - * foreach is safe because we exit loop after list_delete... - */ + newjointree = foreach_delete_current(newjointree, l); break; } } From f8721bd752790859df747905bc44fb5ad8dbf07d Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 22 Oct 2020 13:29:39 +0200 Subject: [PATCH 351/589] Use croak instead of die in Perl code when appropriate --- src/test/perl/TestLib.pm | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/test/perl/TestLib.pm b/src/test/perl/TestLib.pm index cbe87f868431..1baf6bd00173 100644 --- a/src/test/perl/TestLib.pm +++ b/src/test/perl/TestLib.pm @@ -43,6 +43,7 @@ package TestLib; use strict; use warnings; +use Carp; use Config; use Cwd; use Exporter 'import'; @@ -421,7 +422,7 @@ sub slurp_dir { my ($dir) = @_; opendir(my $dh, $dir) - or die "could not opendir \"$dir\": $!"; + or croak "could not opendir \"$dir\": $!"; my @direntries = readdir $dh; closedir $dh; return @direntries; @@ -443,19 +444,19 @@ sub slurp_file if ($Config{osname} ne 'MSWin32') { open(my $in, '<', $filename) - or die "could not read \"$filename\": $!"; + or croak "could not read \"$filename\": $!"; $contents = <$in>; close $in; } else { my $fHandle = createFile($filename, "r", "rwd") - or die "could not open \"$filename\": $^E"; + or croak "could not open \"$filename\": $^E"; OsFHandleOpen(my $fh = IO::Handle->new(), $fHandle, 'r') - or die "could not read \"$filename\": $^E\n"; + or croak "could not read \"$filename\": $^E\n"; $contents = <$fh>; CloseHandle($fHandle) - or die "could not close \"$filename\": $^E\n"; + or croak "could not close \"$filename\": $^E\n"; } $contents =~ s/\r\n/\n/g if $Config{osname} eq 'msys'; return $contents; @@ -474,7 +475,7 @@ sub append_to_file { my ($filename, $str) = @_; open my $fh, ">>", $filename - or die "could not write \"$filename\": $!"; + or croak "could not write \"$filename\": $!"; print $fh $str; close $fh; return; From 866e24d47db1743dfcff5bd595b57e3a143f2cb1 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 22 Oct 2020 08:44:18 -0400 Subject: [PATCH 352/589] Extend amcheck to check heap pages. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mark Dilger, reviewed by Peter Geoghegan, Andres Freund, Álvaro Herrera, Michael Paquier, Amul Sul, and by me. Some last-minute cosmetic revisions by me. Discussion: http://postgr.es/m/12ED3DA8-25F0-4B68-937D-D907CFBF08E7@enterprisedb.com --- contrib/amcheck/Makefile | 7 +- contrib/amcheck/amcheck--1.2--1.3.sql | 30 + contrib/amcheck/amcheck.control | 2 +- contrib/amcheck/expected/check_heap.out | 194 +++ contrib/amcheck/sql/check_heap.sql | 116 ++ contrib/amcheck/t/001_verify_heapam.pl | 242 ++++ contrib/amcheck/verify_heapam.c | 1447 +++++++++++++++++++++++ doc/src/sgml/amcheck.sgml | 235 +++- src/backend/access/heap/hio.c | 11 + src/backend/access/transam/multixact.c | 19 + src/include/access/multixact.h | 1 + src/tools/pgindent/typedefs.list | 4 + 12 files changed, 2299 insertions(+), 9 deletions(-) create mode 100644 contrib/amcheck/amcheck--1.2--1.3.sql create mode 100644 contrib/amcheck/expected/check_heap.out create mode 100644 contrib/amcheck/sql/check_heap.sql create mode 100644 contrib/amcheck/t/001_verify_heapam.pl create mode 100644 contrib/amcheck/verify_heapam.c diff --git a/contrib/amcheck/Makefile b/contrib/amcheck/Makefile index a2b1b1036b3e..b82f221e50bb 100644 --- a/contrib/amcheck/Makefile +++ b/contrib/amcheck/Makefile @@ -3,13 +3,16 @@ MODULE_big = amcheck OBJS = \ $(WIN32RES) \ + verify_heapam.o \ verify_nbtree.o EXTENSION = amcheck -DATA = amcheck--1.1--1.2.sql amcheck--1.0--1.1.sql amcheck--1.0.sql +DATA = amcheck--1.2--1.3.sql amcheck--1.1--1.2.sql amcheck--1.0--1.1.sql amcheck--1.0.sql PGFILEDESC = "amcheck - function for verifying relation integrity" -REGRESS = check check_btree +REGRESS = check check_btree check_heap + +TAP_TESTS = 1 ifdef USE_PGXS PG_CONFIG = pg_config diff --git a/contrib/amcheck/amcheck--1.2--1.3.sql b/contrib/amcheck/amcheck--1.2--1.3.sql new file mode 100644 index 000000000000..7237ab738ce7 --- /dev/null +++ b/contrib/amcheck/amcheck--1.2--1.3.sql @@ -0,0 +1,30 @@ +/* contrib/amcheck/amcheck--1.2--1.3.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION amcheck UPDATE TO '1.3'" to load this file. \quit + +-- +-- verify_heapam() +-- +CREATE FUNCTION verify_heapam(relation regclass, + on_error_stop boolean default false, + check_toast boolean default false, + skip text default 'none', + startblock bigint default null, + endblock bigint default null, + blkno OUT bigint, + offnum OUT integer, + attnum OUT integer, + msg OUT text) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'verify_heapam' +LANGUAGE C; + +-- Don't want this to be available to public +REVOKE ALL ON FUNCTION verify_heapam(regclass, + boolean, + boolean, + text, + bigint, + bigint) +FROM PUBLIC; diff --git a/contrib/amcheck/amcheck.control b/contrib/amcheck/amcheck.control index c6e310046d4e..ab50931f754a 100644 --- a/contrib/amcheck/amcheck.control +++ b/contrib/amcheck/amcheck.control @@ -1,5 +1,5 @@ # amcheck extension comment = 'functions for verifying relation integrity' -default_version = '1.2' +default_version = '1.3' module_pathname = '$libdir/amcheck' relocatable = true diff --git a/contrib/amcheck/expected/check_heap.out b/contrib/amcheck/expected/check_heap.out new file mode 100644 index 000000000000..882f853d56ac --- /dev/null +++ b/contrib/amcheck/expected/check_heap.out @@ -0,0 +1,194 @@ +CREATE TABLE heaptest (a integer, b text); +REVOKE ALL ON heaptest FROM PUBLIC; +-- Check that invalid skip option is rejected +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'rope'); +ERROR: invalid skip option +HINT: Valid skip options are "all-visible", "all-frozen", and "none". +-- Check specifying invalid block ranges when verifying an empty table +SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 0, endblock := 0); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 5, endblock := 8); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +-- Check that valid options are not rejected nor corruption reported +-- for an empty table, and that skip enum-like parameter is case-insensitive +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'none'); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-frozen'); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-visible'); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'None'); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'All-Frozen'); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'All-Visible'); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'NONE'); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'ALL-FROZEN'); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'ALL-VISIBLE'); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +-- Add some data so subsequent tests are not entirely trivial +INSERT INTO heaptest (a, b) + (SELECT gs, repeat('x', gs) + FROM generate_series(1,50) gs); +-- Check that valid options are not rejected nor corruption reported +-- for a non-empty table +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'none'); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-frozen'); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-visible'); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 0, endblock := 0); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +CREATE ROLE regress_heaptest_role; +-- verify permissions are checked (error due to function not callable) +SET ROLE regress_heaptest_role; +SELECT * FROM verify_heapam(relation := 'heaptest'); +ERROR: permission denied for function verify_heapam +RESET ROLE; +GRANT EXECUTE ON FUNCTION verify_heapam(regclass, boolean, boolean, text, bigint, bigint) TO regress_heaptest_role; +-- verify permissions are now sufficient +SET ROLE regress_heaptest_role; +SELECT * FROM verify_heapam(relation := 'heaptest'); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +RESET ROLE; +-- Check specifying invalid block ranges when verifying a non-empty table. +SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 0, endblock := 10000); +ERROR: ending block number must be between 0 and 0 +SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 10000, endblock := 11000); +ERROR: starting block number must be between 0 and 0 +-- Vacuum freeze to change the xids encountered in subsequent tests +VACUUM FREEZE heaptest; +-- Check that valid options are not rejected nor corruption reported +-- for a non-empty frozen table +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'none'); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-frozen'); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-visible'); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 0, endblock := 0); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +-- Check that partitioned tables (the parent ones) which don't have visibility +-- maps are rejected +CREATE TABLE test_partitioned (a int, b text default repeat('x', 5000)) + PARTITION BY list (a); +SELECT * FROM verify_heapam('test_partitioned', + startblock := NULL, + endblock := NULL); +ERROR: "test_partitioned" is not a table, materialized view, or TOAST table +-- Check that valid options are not rejected nor corruption reported +-- for an empty partition table (the child one) +CREATE TABLE test_partition partition OF test_partitioned FOR VALUES IN (1); +SELECT * FROM verify_heapam('test_partition', + startblock := NULL, + endblock := NULL); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +-- Check that valid options are not rejected nor corruption reported +-- for a non-empty partition table (the child one) +INSERT INTO test_partitioned (a) (SELECT 1 FROM generate_series(1,1000) gs); +SELECT * FROM verify_heapam('test_partition', + startblock := NULL, + endblock := NULL); + blkno | offnum | attnum | msg +-------+--------+--------+----- +(0 rows) + +-- Check that indexes are rejected +CREATE INDEX test_index ON test_partition (a); +SELECT * FROM verify_heapam('test_index', + startblock := NULL, + endblock := NULL); +ERROR: "test_index" is not a table, materialized view, or TOAST table +-- Check that views are rejected +CREATE VIEW test_view AS SELECT 1; +SELECT * FROM verify_heapam('test_view', + startblock := NULL, + endblock := NULL); +ERROR: "test_view" is not a table, materialized view, or TOAST table +-- Check that sequences are rejected +CREATE SEQUENCE test_sequence; +SELECT * FROM verify_heapam('test_sequence', + startblock := NULL, + endblock := NULL); +ERROR: "test_sequence" is not a table, materialized view, or TOAST table +-- Check that foreign tables are rejected +CREATE FOREIGN DATA WRAPPER dummy; +CREATE SERVER dummy_server FOREIGN DATA WRAPPER dummy; +CREATE FOREIGN TABLE test_foreign_table () SERVER dummy_server; +SELECT * FROM verify_heapam('test_foreign_table', + startblock := NULL, + endblock := NULL); +ERROR: "test_foreign_table" is not a table, materialized view, or TOAST table +-- cleanup +DROP TABLE heaptest; +DROP TABLE test_partition; +DROP TABLE test_partitioned; +DROP OWNED BY regress_heaptest_role; -- permissions +DROP ROLE regress_heaptest_role; diff --git a/contrib/amcheck/sql/check_heap.sql b/contrib/amcheck/sql/check_heap.sql new file mode 100644 index 000000000000..c10a25f21cb8 --- /dev/null +++ b/contrib/amcheck/sql/check_heap.sql @@ -0,0 +1,116 @@ +CREATE TABLE heaptest (a integer, b text); +REVOKE ALL ON heaptest FROM PUBLIC; + +-- Check that invalid skip option is rejected +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'rope'); + +-- Check specifying invalid block ranges when verifying an empty table +SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 0, endblock := 0); +SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 5, endblock := 8); + +-- Check that valid options are not rejected nor corruption reported +-- for an empty table, and that skip enum-like parameter is case-insensitive +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'none'); +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-frozen'); +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-visible'); +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'None'); +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'All-Frozen'); +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'All-Visible'); +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'NONE'); +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'ALL-FROZEN'); +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'ALL-VISIBLE'); + +-- Add some data so subsequent tests are not entirely trivial +INSERT INTO heaptest (a, b) + (SELECT gs, repeat('x', gs) + FROM generate_series(1,50) gs); + +-- Check that valid options are not rejected nor corruption reported +-- for a non-empty table +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'none'); +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-frozen'); +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-visible'); +SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 0, endblock := 0); + +CREATE ROLE regress_heaptest_role; + +-- verify permissions are checked (error due to function not callable) +SET ROLE regress_heaptest_role; +SELECT * FROM verify_heapam(relation := 'heaptest'); +RESET ROLE; + +GRANT EXECUTE ON FUNCTION verify_heapam(regclass, boolean, boolean, text, bigint, bigint) TO regress_heaptest_role; + +-- verify permissions are now sufficient +SET ROLE regress_heaptest_role; +SELECT * FROM verify_heapam(relation := 'heaptest'); +RESET ROLE; + +-- Check specifying invalid block ranges when verifying a non-empty table. +SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 0, endblock := 10000); +SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 10000, endblock := 11000); + +-- Vacuum freeze to change the xids encountered in subsequent tests +VACUUM FREEZE heaptest; + +-- Check that valid options are not rejected nor corruption reported +-- for a non-empty frozen table +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'none'); +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-frozen'); +SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-visible'); +SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 0, endblock := 0); + +-- Check that partitioned tables (the parent ones) which don't have visibility +-- maps are rejected +CREATE TABLE test_partitioned (a int, b text default repeat('x', 5000)) + PARTITION BY list (a); +SELECT * FROM verify_heapam('test_partitioned', + startblock := NULL, + endblock := NULL); + +-- Check that valid options are not rejected nor corruption reported +-- for an empty partition table (the child one) +CREATE TABLE test_partition partition OF test_partitioned FOR VALUES IN (1); +SELECT * FROM verify_heapam('test_partition', + startblock := NULL, + endblock := NULL); + +-- Check that valid options are not rejected nor corruption reported +-- for a non-empty partition table (the child one) +INSERT INTO test_partitioned (a) (SELECT 1 FROM generate_series(1,1000) gs); +SELECT * FROM verify_heapam('test_partition', + startblock := NULL, + endblock := NULL); + +-- Check that indexes are rejected +CREATE INDEX test_index ON test_partition (a); +SELECT * FROM verify_heapam('test_index', + startblock := NULL, + endblock := NULL); + +-- Check that views are rejected +CREATE VIEW test_view AS SELECT 1; +SELECT * FROM verify_heapam('test_view', + startblock := NULL, + endblock := NULL); + +-- Check that sequences are rejected +CREATE SEQUENCE test_sequence; +SELECT * FROM verify_heapam('test_sequence', + startblock := NULL, + endblock := NULL); + +-- Check that foreign tables are rejected +CREATE FOREIGN DATA WRAPPER dummy; +CREATE SERVER dummy_server FOREIGN DATA WRAPPER dummy; +CREATE FOREIGN TABLE test_foreign_table () SERVER dummy_server; +SELECT * FROM verify_heapam('test_foreign_table', + startblock := NULL, + endblock := NULL); + +-- cleanup +DROP TABLE heaptest; +DROP TABLE test_partition; +DROP TABLE test_partitioned; +DROP OWNED BY regress_heaptest_role; -- permissions +DROP ROLE regress_heaptest_role; diff --git a/contrib/amcheck/t/001_verify_heapam.pl b/contrib/amcheck/t/001_verify_heapam.pl new file mode 100644 index 000000000000..e7526c17b80e --- /dev/null +++ b/contrib/amcheck/t/001_verify_heapam.pl @@ -0,0 +1,242 @@ +use strict; +use warnings; + +use PostgresNode; +use TestLib; + +use Test::More tests => 65; + +my ($node, $result); + +# +# Test set-up +# +$node = get_new_node('test'); +$node->init; +$node->append_conf('postgresql.conf', 'autovacuum=off'); +$node->start; +$node->safe_psql('postgres', q(CREATE EXTENSION amcheck)); + +# +# Check a table with data loaded but no corruption, freezing, etc. +# +fresh_test_table('test'); +check_all_options_uncorrupted('test', 'plain'); + +# +# Check a corrupt table +# +fresh_test_table('test'); +corrupt_first_page('test'); +detects_corruption( + "verify_heapam('test')", + "plain corrupted table"); +detects_corruption( + "verify_heapam('test', skip := 'all-visible')", + "plain corrupted table skipping all-visible"); +detects_corruption( + "verify_heapam('test', skip := 'all-frozen')", + "plain corrupted table skipping all-frozen"); +detects_corruption( + "verify_heapam('test', check_toast := false)", + "plain corrupted table skipping toast"); +detects_corruption( + "verify_heapam('test', startblock := 0, endblock := 0)", + "plain corrupted table checking only block zero"); + +# +# Check a corrupt table with all-frozen data +# +fresh_test_table('test'); +$node->safe_psql('postgres', q(VACUUM FREEZE test)); +corrupt_first_page('test'); +detects_corruption( + "verify_heapam('test')", + "all-frozen corrupted table"); +detects_no_corruption( + "verify_heapam('test', skip := 'all-frozen')", + "all-frozen corrupted table skipping all-frozen"); + +# +# Check a corrupt table with corrupt page header +# +fresh_test_table('test'); +corrupt_first_page_and_header('test'); +detects_corruption( + "verify_heapam('test')", + "corrupted test table with bad page header"); + +# +# Check an uncorrupted table with corrupt toast page header +# +fresh_test_table('test'); +my $toast = get_toast_for('test'); +corrupt_first_page_and_header($toast); +detects_corruption( + "verify_heapam('test', check_toast := true)", + "table with corrupted toast page header checking toast"); +detects_no_corruption( + "verify_heapam('test', check_toast := false)", + "table with corrupted toast page header skipping toast"); +detects_corruption( + "verify_heapam('$toast')", + "corrupted toast page header"); + +# +# Check an uncorrupted table with corrupt toast +# +fresh_test_table('test'); +$toast = get_toast_for('test'); +corrupt_first_page($toast); +detects_corruption( + "verify_heapam('test', check_toast := true)", + "table with corrupted toast checking toast"); +detects_no_corruption( + "verify_heapam('test', check_toast := false)", + "table with corrupted toast skipping toast"); +detects_corruption( + "verify_heapam('$toast')", + "corrupted toast table"); + +# +# Check an uncorrupted all-frozen table with corrupt toast +# +fresh_test_table('test'); +$node->safe_psql('postgres', q(VACUUM FREEZE test)); +$toast = get_toast_for('test'); +corrupt_first_page($toast); +detects_corruption( + "verify_heapam('test', check_toast := true)", + "all-frozen table with corrupted toast checking toast"); +detects_no_corruption( + "verify_heapam('test', check_toast := false)", + "all-frozen table with corrupted toast skipping toast"); +detects_corruption( + "verify_heapam('$toast')", + "corrupted toast table of all-frozen table"); + +# Returns the filesystem path for the named relation. +sub relation_filepath +{ + my ($relname) = @_; + + my $pgdata = $node->data_dir; + my $rel = $node->safe_psql('postgres', + qq(SELECT pg_relation_filepath('$relname'))); + die "path not found for relation $relname" unless defined $rel; + return "$pgdata/$rel"; +} + +# Returns the fully qualified name of the toast table for the named relation +sub get_toast_for +{ + my ($relname) = @_; + $node->safe_psql('postgres', qq( + SELECT 'pg_toast.' || t.relname + FROM pg_catalog.pg_class c, pg_catalog.pg_class t + WHERE c.relname = '$relname' + AND c.reltoastrelid = t.oid)); +} + +# (Re)create and populate a test table of the given name. +sub fresh_test_table +{ + my ($relname) = @_; + $node->safe_psql('postgres', qq( + DROP TABLE IF EXISTS $relname CASCADE; + CREATE TABLE $relname (a integer, b text); + ALTER TABLE $relname SET (autovacuum_enabled=false); + ALTER TABLE $relname ALTER b SET STORAGE external; + INSERT INTO $relname (a, b) + (SELECT gs, repeat('b',gs*10) FROM generate_series(1,1000) gs); + )); +} + +# Stops the test node, corrupts the first page of the named relation, and +# restarts the node. +sub corrupt_first_page_internal +{ + my ($relname, $corrupt_header) = @_; + my $relpath = relation_filepath($relname); + + $node->stop; + my $fh; + open($fh, '+<', $relpath); + binmode $fh; + + # If we corrupt the header, postgres won't allow the page into the buffer. + syswrite($fh, '\xFF\xFF\xFF\xFF', 8) if ($corrupt_header); + + # Corrupt at least the line pointers. Exactly what this corrupts will + # depend on the page, as it may run past the line pointers into the user + # data. We stop short of writing 2048 bytes (2k), the smallest supported + # page size, as we don't want to corrupt the next page. + seek($fh, 32, 0); + syswrite($fh, '\x77\x77\x77\x77', 500); + close($fh); + $node->start; +} + +sub corrupt_first_page +{ + corrupt_first_page_internal($_[0], undef); +} + +sub corrupt_first_page_and_header +{ + corrupt_first_page_internal($_[0], 1); +} + +sub detects_corruption +{ + my ($function, $testname) = @_; + + my $result = $node->safe_psql('postgres', + qq(SELECT COUNT(*) > 0 FROM $function)); + is($result, 't', $testname); +} + +sub detects_no_corruption +{ + my ($function, $testname) = @_; + + my $result = $node->safe_psql('postgres', + qq(SELECT COUNT(*) = 0 FROM $function)); + is($result, 't', $testname); +} + +# Check various options are stable (don't abort) and do not report corruption +# when running verify_heapam on an uncorrupted test table. +# +# The relname *must* be an uncorrupted table, or this will fail. +# +# The prefix is used to identify the test, along with the options, +# and should be unique. +sub check_all_options_uncorrupted +{ + my ($relname, $prefix) = @_; + for my $stop (qw(true false)) + { + for my $check_toast (qw(true false)) + { + for my $skip ("'none'", "'all-frozen'", "'all-visible'") + { + for my $startblock (qw(NULL 0)) + { + for my $endblock (qw(NULL 0)) + { + my $opts = "on_error_stop := $stop, " . + "check_toast := $check_toast, " . + "skip := $skip, " . + "startblock := $startblock, " . + "endblock := $endblock"; + + detects_no_corruption( + "verify_heapam('$relname', $opts)", + "$prefix: $opts"); + } + } + } + } + } +} diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c new file mode 100644 index 000000000000..0156c1e74aa1 --- /dev/null +++ b/contrib/amcheck/verify_heapam.c @@ -0,0 +1,1447 @@ +/*------------------------------------------------------------------------- + * + * verify_heapam.c + * Functions to check postgresql heap relations for corruption + * + * Copyright (c) 2016-2020, PostgreSQL Global Development Group + * + * contrib/amcheck/verify_heapam.c + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/detoast.h" +#include "access/genam.h" +#include "access/heapam.h" +#include "access/heaptoast.h" +#include "access/multixact.h" +#include "access/toast_internals.h" +#include "access/visibilitymap.h" +#include "catalog/pg_am.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "storage/bufmgr.h" +#include "storage/procarray.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" + +PG_FUNCTION_INFO_V1(verify_heapam); + +/* The number of columns in tuples returned by verify_heapam */ +#define HEAPCHECK_RELATION_COLS 4 + +/* + * Despite the name, we use this for reporting problems with both XIDs and + * MXIDs. + */ +typedef enum XidBoundsViolation +{ + XID_INVALID, + XID_IN_FUTURE, + XID_PRECEDES_CLUSTERMIN, + XID_PRECEDES_RELMIN, + XID_BOUNDS_OK +} XidBoundsViolation; + +typedef enum XidCommitStatus +{ + XID_COMMITTED, + XID_IN_PROGRESS, + XID_ABORTED +} XidCommitStatus; + +typedef enum SkipPages +{ + SKIP_PAGES_ALL_FROZEN, + SKIP_PAGES_ALL_VISIBLE, + SKIP_PAGES_NONE +} SkipPages; + +/* + * Struct holding the running context information during + * a lifetime of a verify_heapam execution. + */ +typedef struct HeapCheckContext +{ + /* + * Cached copies of values from ShmemVariableCache and computed values + * from them. + */ + FullTransactionId next_fxid; /* ShmemVariableCache->nextXid */ + TransactionId next_xid; /* 32-bit version of next_fxid */ + TransactionId oldest_xid; /* ShmemVariableCache->oldestXid */ + FullTransactionId oldest_fxid; /* 64-bit version of oldest_xid, computed + * relative to next_fxid */ + + /* + * Cached copy of value from MultiXactState + */ + MultiXactId next_mxact; /* MultiXactState->nextMXact */ + MultiXactId oldest_mxact; /* MultiXactState->oldestMultiXactId */ + + /* + * Cached copies of the most recently checked xid and its status. + */ + TransactionId cached_xid; + XidCommitStatus cached_status; + + /* Values concerning the heap relation being checked */ + Relation rel; + TransactionId relfrozenxid; + FullTransactionId relfrozenfxid; + TransactionId relminmxid; + Relation toast_rel; + Relation *toast_indexes; + Relation valid_toast_index; + int num_toast_indexes; + + /* Values for iterating over pages in the relation */ + BlockNumber blkno; + BufferAccessStrategy bstrategy; + Buffer buffer; + Page page; + + /* Values for iterating over tuples within a page */ + OffsetNumber offnum; + ItemId itemid; + uint16 lp_len; + HeapTupleHeader tuphdr; + int natts; + + /* Values for iterating over attributes within the tuple */ + uint32 offset; /* offset in tuple data */ + AttrNumber attnum; + + /* Values for iterating over toast for the attribute */ + int32 chunkno; + int32 attrsize; + int32 endchunk; + int32 totalchunks; + + /* Whether verify_heapam has yet encountered any corrupt tuples */ + bool is_corrupt; + + /* The descriptor and tuplestore for verify_heapam's result tuples */ + TupleDesc tupdesc; + Tuplestorestate *tupstore; +} HeapCheckContext; + +/* Internal implementation */ +static void sanity_check_relation(Relation rel); +static void check_tuple(HeapCheckContext *ctx); +static void check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx); + +static bool check_tuple_attribute(HeapCheckContext *ctx); +static bool check_tuple_header_and_visibilty(HeapTupleHeader tuphdr, + HeapCheckContext *ctx); + +static void report_corruption(HeapCheckContext *ctx, char *msg); +static TupleDesc verify_heapam_tupdesc(void); +static FullTransactionId FullTransactionIdFromXidAndCtx(TransactionId xid, + const HeapCheckContext *ctx); +static void update_cached_xid_range(HeapCheckContext *ctx); +static void update_cached_mxid_range(HeapCheckContext *ctx); +static XidBoundsViolation check_mxid_in_range(MultiXactId mxid, + HeapCheckContext *ctx); +static XidBoundsViolation check_mxid_valid_in_rel(MultiXactId mxid, + HeapCheckContext *ctx); +static XidBoundsViolation get_xid_status(TransactionId xid, + HeapCheckContext *ctx, + XidCommitStatus *status); + +/* + * Scan and report corruption in heap pages, optionally reconciling toasted + * attributes with entries in the associated toast table. Intended to be + * called from SQL with the following parameters: + * + * relation: + * The Oid of the heap relation to be checked. + * + * on_error_stop: + * Whether to stop at the end of the first page for which errors are + * detected. Note that multiple rows may be returned. + * + * check_toast: + * Whether to check each toasted attribute against the toast table to + * verify that it can be found there. + * + * skip: + * What kinds of pages in the heap relation should be skipped. Valid + * options are "all-visible", "all-frozen", and "none". + * + * Returns to the SQL caller a set of tuples, each containing the location + * and a description of a corruption found in the heap. + * + * This code goes to some trouble to avoid crashing the server even if the + * table pages are badly corrupted, but it's probably not perfect. If + * check_toast is true, we'll use regular index lookups to try to fetch TOAST + * tuples, which can certainly cause crashes if the right kind of corruption + * exists in the toast table or index. No matter what parameters you pass, + * we can't protect against crashes that might occur trying to look up the + * commit status of transaction IDs (though we avoid trying to do such lookups + * for transaction IDs that can't legally appear in the table). + */ +Datum +verify_heapam(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + MemoryContext old_context; + bool random_access; + HeapCheckContext ctx; + Buffer vmbuffer = InvalidBuffer; + Oid relid; + bool on_error_stop; + bool check_toast; + SkipPages skip_option = SKIP_PAGES_NONE; + BlockNumber first_block; + BlockNumber last_block; + BlockNumber nblocks; + const char *skip; + + /* Check to see if caller supports us returning a tuplestore */ + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that cannot accept a set"))); + if (!(rsinfo->allowedModes & SFRM_Materialize)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("materialize mode required, but it is not allowed in this context"))); + + /* Check supplied arguments */ + if (PG_ARGISNULL(0)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("relation cannot be null"))); + relid = PG_GETARG_OID(0); + + if (PG_ARGISNULL(1)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("on_error_stop cannot be null"))); + on_error_stop = PG_GETARG_BOOL(1); + + if (PG_ARGISNULL(2)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("check_toast cannot be null"))); + check_toast = PG_GETARG_BOOL(2); + + if (PG_ARGISNULL(3)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("skip cannot be null"))); + skip = text_to_cstring(PG_GETARG_TEXT_PP(3)); + if (pg_strcasecmp(skip, "all-visible") == 0) + skip_option = SKIP_PAGES_ALL_VISIBLE; + else if (pg_strcasecmp(skip, "all-frozen") == 0) + skip_option = SKIP_PAGES_ALL_FROZEN; + else if (pg_strcasecmp(skip, "none") == 0) + skip_option = SKIP_PAGES_NONE; + else + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid skip option"), + errhint("Valid skip options are \"all-visible\", \"all-frozen\", and \"none\"."))); + + memset(&ctx, 0, sizeof(HeapCheckContext)); + ctx.cached_xid = InvalidTransactionId; + + /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ + old_context = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); + random_access = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; + ctx.tupdesc = verify_heapam_tupdesc(); + ctx.tupstore = tuplestore_begin_heap(random_access, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = ctx.tupstore; + rsinfo->setDesc = ctx.tupdesc; + MemoryContextSwitchTo(old_context); + + /* Open relation, check relkind and access method, and check privileges */ + ctx.rel = relation_open(relid, AccessShareLock); + sanity_check_relation(ctx.rel); + + /* Early exit if the relation is empty */ + nblocks = RelationGetNumberOfBlocks(ctx.rel); + if (!nblocks) + { + relation_close(ctx.rel, AccessShareLock); + PG_RETURN_NULL(); + } + + ctx.bstrategy = GetAccessStrategy(BAS_BULKREAD); + ctx.buffer = InvalidBuffer; + ctx.page = NULL; + + /* Validate block numbers, or handle nulls. */ + if (PG_ARGISNULL(4)) + first_block = 0; + else + { + int64 fb = PG_GETARG_INT64(4); + + if (fb < 0 || fb >= nblocks) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("starting block number must be between 0 and %u", + nblocks - 1))); + first_block = (BlockNumber) fb; + } + if (PG_ARGISNULL(5)) + last_block = nblocks - 1; + else + { + int64 lb = PG_GETARG_INT64(5); + + if (lb < 0 || lb >= nblocks) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("ending block number must be between 0 and %u", + nblocks - 1))); + last_block = (BlockNumber) lb; + } + + /* Optionally open the toast relation, if any. */ + if (ctx.rel->rd_rel->reltoastrelid && check_toast) + { + int offset; + + /* Main relation has associated toast relation */ + ctx.toast_rel = table_open(ctx.rel->rd_rel->reltoastrelid, + AccessShareLock); + offset = toast_open_indexes(ctx.toast_rel, + AccessShareLock, + &(ctx.toast_indexes), + &(ctx.num_toast_indexes)); + ctx.valid_toast_index = ctx.toast_indexes[offset]; + } + else + { + /* + * Main relation has no associated toast relation, or we're + * intentionally skipping it. + */ + ctx.toast_rel = NULL; + ctx.toast_indexes = NULL; + ctx.num_toast_indexes = 0; + } + + update_cached_xid_range(&ctx); + update_cached_mxid_range(&ctx); + ctx.relfrozenxid = ctx.rel->rd_rel->relfrozenxid; + ctx.relfrozenfxid = FullTransactionIdFromXidAndCtx(ctx.relfrozenxid, &ctx); + ctx.relminmxid = ctx.rel->rd_rel->relminmxid; + + if (TransactionIdIsNormal(ctx.relfrozenxid)) + ctx.oldest_xid = ctx.relfrozenxid; + + for (ctx.blkno = first_block; ctx.blkno <= last_block; ctx.blkno++) + { + OffsetNumber maxoff; + + /* Optionally skip over all-frozen or all-visible blocks */ + if (skip_option != SKIP_PAGES_NONE) + { + int32 mapbits; + + mapbits = (int32) visibilitymap_get_status(ctx.rel, ctx.blkno, + &vmbuffer); + if (skip_option == SKIP_PAGES_ALL_FROZEN) + { + if ((mapbits & VISIBILITYMAP_ALL_FROZEN) != 0) + continue; + } + + if (skip_option == SKIP_PAGES_ALL_VISIBLE) + { + if ((mapbits & VISIBILITYMAP_ALL_VISIBLE) != 0) + continue; + } + } + + /* Read and lock the next page. */ + ctx.buffer = ReadBufferExtended(ctx.rel, MAIN_FORKNUM, ctx.blkno, + RBM_NORMAL, ctx.bstrategy); + LockBuffer(ctx.buffer, BUFFER_LOCK_SHARE); + ctx.page = BufferGetPage(ctx.buffer); + + /* Perform tuple checks */ + maxoff = PageGetMaxOffsetNumber(ctx.page); + for (ctx.offnum = FirstOffsetNumber; ctx.offnum <= maxoff; + ctx.offnum = OffsetNumberNext(ctx.offnum)) + { + ctx.itemid = PageGetItemId(ctx.page, ctx.offnum); + + /* Skip over unused/dead line pointers */ + if (!ItemIdIsUsed(ctx.itemid) || ItemIdIsDead(ctx.itemid)) + continue; + + /* + * If this line pointer has been redirected, check that it + * redirects to a valid offset within the line pointer array. + */ + if (ItemIdIsRedirected(ctx.itemid)) + { + OffsetNumber rdoffnum = ItemIdGetRedirect(ctx.itemid); + ItemId rditem; + + if (rdoffnum < FirstOffsetNumber || rdoffnum > maxoff) + { + report_corruption(&ctx, + psprintf("line pointer redirection to item at offset %u exceeds maximum offset %u", + (unsigned) rdoffnum, + (unsigned) maxoff)); + continue; + } + rditem = PageGetItemId(ctx.page, rdoffnum); + if (!ItemIdIsUsed(rditem)) + report_corruption(&ctx, + psprintf("line pointer redirection to unused item at offset %u", + (unsigned) rdoffnum)); + continue; + } + + /* Set up context information about this next tuple */ + ctx.lp_len = ItemIdGetLength(ctx.itemid); + ctx.tuphdr = (HeapTupleHeader) PageGetItem(ctx.page, ctx.itemid); + ctx.natts = HeapTupleHeaderGetNatts(ctx.tuphdr); + + /* Ok, ready to check this next tuple */ + check_tuple(&ctx); + } + + /* clean up */ + UnlockReleaseBuffer(ctx.buffer); + + if (on_error_stop && ctx.is_corrupt) + break; + } + + if (vmbuffer != InvalidBuffer) + ReleaseBuffer(vmbuffer); + + /* Close the associated toast table and indexes, if any. */ + if (ctx.toast_indexes) + toast_close_indexes(ctx.toast_indexes, ctx.num_toast_indexes, + AccessShareLock); + if (ctx.toast_rel) + table_close(ctx.toast_rel, AccessShareLock); + + /* Close the main relation */ + relation_close(ctx.rel, AccessShareLock); + + PG_RETURN_NULL(); +} + +/* + * Check that a relation's relkind and access method are both supported, + * and that the caller has select privilege on the relation. + */ +static void +sanity_check_relation(Relation rel) +{ + if (rel->rd_rel->relkind != RELKIND_RELATION && + rel->rd_rel->relkind != RELKIND_MATVIEW && + rel->rd_rel->relkind != RELKIND_TOASTVALUE) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("\"%s\" is not a table, materialized view, or TOAST table", + RelationGetRelationName(rel)))); + if (rel->rd_rel->relam != HEAP_TABLE_AM_OID) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("only heap AM is supported"))); +} + +/* + * Record a single corruption found in the table. The values in ctx should + * reflect the location of the corruption, and the msg argument should contain + * a human readable description of the corruption. + * + * The msg argument is pfree'd by this function. + */ +static void +report_corruption(HeapCheckContext *ctx, char *msg) +{ + Datum values[HEAPCHECK_RELATION_COLS]; + bool nulls[HEAPCHECK_RELATION_COLS]; + HeapTuple tuple; + + MemSet(values, 0, sizeof(values)); + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int64GetDatum(ctx->blkno); + values[1] = Int32GetDatum(ctx->offnum); + values[2] = Int32GetDatum(ctx->attnum); + nulls[2] = (ctx->attnum < 0); + values[3] = CStringGetTextDatum(msg); + + /* + * In principle, there is nothing to prevent a scan over a large, highly + * corrupted table from using work_mem worth of memory building up the + * tuplestore. That's ok, but if we also leak the msg argument memory + * until the end of the query, we could exceed work_mem by more than a + * trivial amount. Therefore, free the msg argument each time we are + * called rather than waiting for our current memory context to be freed. + */ + pfree(msg); + + tuple = heap_form_tuple(ctx->tupdesc, values, nulls); + tuplestore_puttuple(ctx->tupstore, tuple); + ctx->is_corrupt = true; +} + +/* + * Construct the TupleDesc used to report messages about corruptions found + * while scanning the heap. + */ +static TupleDesc +verify_heapam_tupdesc(void) +{ + TupleDesc tupdesc; + AttrNumber a = 0; + + tupdesc = CreateTemplateTupleDesc(HEAPCHECK_RELATION_COLS); + TupleDescInitEntry(tupdesc, ++a, "blkno", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, ++a, "offnum", INT4OID, -1, 0); + TupleDescInitEntry(tupdesc, ++a, "attnum", INT4OID, -1, 0); + TupleDescInitEntry(tupdesc, ++a, "msg", TEXTOID, -1, 0); + Assert(a == HEAPCHECK_RELATION_COLS); + + return BlessTupleDesc(tupdesc); +} + +/* + * Check for tuple header corruption and tuple visibility. + * + * Since we do not hold a snapshot, tuple visibility is not a question of + * whether we should be able to see the tuple relative to any particular + * snapshot, but rather a question of whether it is safe and reasonable to + * to check the tuple attributes. + * + * Some kinds of corruption make it unsafe to check the tuple attributes, for + * example when the line pointer refers to a range of bytes outside the page. + * In such cases, we return false (not visible) after recording appropriate + * corruption messages. + * + * Some other kinds of tuple header corruption confuse the question of where + * the tuple attributes begin, or how long the nulls bitmap is, etc., making it + * unreasonable to attempt to check attributes, even if all candidate answers + * to those questions would not result in reading past the end of the line + * pointer or page. In such cases, like above, we record corruption messages + * about the header and then return false. + * + * Other kinds of tuple header corruption do not bear on the question of + * whether the tuple attributes can be checked, so we record corruption + * messages for them but do not base our visibility determination on them. (In + * other words, we do not return false merely because we detected them.) + * + * For visibility determination not specifically related to corruption, what we + * want to know is if a tuple is potentially visible to any running + * transaction. If you are tempted to replace this function's visibility logic + * with a call to another visibility checking function, keep in mind that this + * function does not update hint bits, as it seems imprudent to write hint bits + * (or anything at all) to a table during a corruption check. Nor does this + * function bother classifying tuple visibility beyond a boolean visible vs. + * not visible. + * + * The caller should already have checked that xmin and xmax are not out of + * bounds for the relation. + * + * Returns whether the tuple is both visible and sufficiently sensible to + * undergo attribute checks. + */ +static bool +check_tuple_header_and_visibilty(HeapTupleHeader tuphdr, HeapCheckContext *ctx) +{ + uint16 infomask = tuphdr->t_infomask; + bool header_garbled = false; + unsigned expected_hoff;; + + if (ctx->tuphdr->t_hoff > ctx->lp_len) + { + report_corruption(ctx, + psprintf("data begins at offset %u beyond the tuple length %u", + ctx->tuphdr->t_hoff, ctx->lp_len)); + header_garbled = true; + } + if ((ctx->tuphdr->t_infomask & HEAP_XMAX_LOCK_ONLY) && + (ctx->tuphdr->t_infomask2 & HEAP_KEYS_UPDATED)) + { + report_corruption(ctx, + pstrdup("tuple is marked as only locked, but also claims key columns were updated")); + header_garbled = true; + } + + if ((ctx->tuphdr->t_infomask & HEAP_XMAX_COMMITTED) && + (ctx->tuphdr->t_infomask & HEAP_XMAX_IS_MULTI)) + { + report_corruption(ctx, + pstrdup("multixact should not be marked committed")); + + /* + * This condition is clearly wrong, but we do not consider the header + * garbled, because we don't rely on this property for determining if + * the tuple is visible or for interpreting other relevant header + * fields. + */ + } + + if (infomask & HEAP_HASNULL) + expected_hoff = MAXALIGN(SizeofHeapTupleHeader + BITMAPLEN(ctx->natts)); + else + expected_hoff = MAXALIGN(SizeofHeapTupleHeader); + if (ctx->tuphdr->t_hoff != expected_hoff) + { + if ((infomask & HEAP_HASNULL) && ctx->natts == 1) + report_corruption(ctx, + psprintf("tuple data should begin at byte %u, but actually begins at byte %u (1 attribute, has nulls)", + expected_hoff, ctx->tuphdr->t_hoff)); + else if ((infomask & HEAP_HASNULL)) + report_corruption(ctx, + psprintf("tuple data should begin at byte %u, but actually begins at byte %u (%u attributes, has nulls)", + expected_hoff, ctx->tuphdr->t_hoff, ctx->natts)); + else if (ctx->natts == 1) + report_corruption(ctx, + psprintf("tuple data should begin at byte %u, but actually begins at byte %u (1 attribute, no nulls)", + expected_hoff, ctx->tuphdr->t_hoff)); + else + report_corruption(ctx, + psprintf("tuple data should begin at byte %u, but actually begins at byte %u (%u attributes, no nulls)", + expected_hoff, ctx->tuphdr->t_hoff, ctx->natts)); + header_garbled = true; + } + + if (header_garbled) + return false; /* checking of this tuple should not continue */ + + /* + * Ok, we can examine the header for tuple visibility purposes, though we + * still need to be careful about a few remaining types of header + * corruption. This logic roughly follows that of + * HeapTupleSatisfiesVacuum. Where possible the comments indicate which + * HTSV_Result we think that function might return for this tuple. + */ + if (!HeapTupleHeaderXminCommitted(tuphdr)) + { + TransactionId raw_xmin = HeapTupleHeaderGetRawXmin(tuphdr); + + if (HeapTupleHeaderXminInvalid(tuphdr)) + return false; /* HEAPTUPLE_DEAD */ + /* Used by pre-9.0 binary upgrades */ + else if (infomask & HEAP_MOVED_OFF || + infomask & HEAP_MOVED_IN) + { + XidCommitStatus status; + TransactionId xvac = HeapTupleHeaderGetXvac(tuphdr); + + switch (get_xid_status(xvac, ctx, &status)) + { + case XID_INVALID: + report_corruption(ctx, + pstrdup("old-style VACUUM FULL transaction ID is invalid")); + return false; /* corrupt */ + case XID_IN_FUTURE: + report_corruption(ctx, + psprintf("old-style VACUUM FULL transaction ID %u equals or exceeds next valid transaction ID %u:%u", + xvac, + EpochFromFullTransactionId(ctx->next_fxid), + XidFromFullTransactionId(ctx->next_fxid))); + return false; /* corrupt */ + case XID_PRECEDES_RELMIN: + report_corruption(ctx, + psprintf("old-style VACUUM FULL transaction ID %u precedes relation freeze threshold %u:%u", + xvac, + EpochFromFullTransactionId(ctx->relfrozenfxid), + XidFromFullTransactionId(ctx->relfrozenfxid))); + return false; /* corrupt */ + break; + case XID_PRECEDES_CLUSTERMIN: + report_corruption(ctx, + psprintf("old-style VACUUM FULL transaction ID %u precedes oldest valid transaction ID %u:%u", + xvac, + EpochFromFullTransactionId(ctx->oldest_fxid), + XidFromFullTransactionId(ctx->oldest_fxid))); + return false; /* corrupt */ + break; + case XID_BOUNDS_OK: + switch (status) + { + case XID_IN_PROGRESS: + return true; /* HEAPTUPLE_DELETE_IN_PROGRESS */ + case XID_COMMITTED: + case XID_ABORTED: + return false; /* HEAPTUPLE_DEAD */ + } + } + } + else + { + XidCommitStatus status; + + switch (get_xid_status(raw_xmin, ctx, &status)) + { + case XID_INVALID: + report_corruption(ctx, + pstrdup("raw xmin is invalid")); + return false; + case XID_IN_FUTURE: + report_corruption(ctx, + psprintf("raw xmin %u equals or exceeds next valid transaction ID %u:%u", + raw_xmin, + EpochFromFullTransactionId(ctx->next_fxid), + XidFromFullTransactionId(ctx->next_fxid))); + return false; /* corrupt */ + case XID_PRECEDES_RELMIN: + report_corruption(ctx, + psprintf("raw xmin %u precedes relation freeze threshold %u:%u", + raw_xmin, + EpochFromFullTransactionId(ctx->relfrozenfxid), + XidFromFullTransactionId(ctx->relfrozenfxid))); + return false; /* corrupt */ + case XID_PRECEDES_CLUSTERMIN: + report_corruption(ctx, + psprintf("raw xmin %u precedes oldest valid transaction ID %u:%u", + raw_xmin, + EpochFromFullTransactionId(ctx->oldest_fxid), + XidFromFullTransactionId(ctx->oldest_fxid))); + return false; /* corrupt */ + case XID_BOUNDS_OK: + switch (status) + { + case XID_COMMITTED: + break; + case XID_IN_PROGRESS: + return true; /* insert or delete in progress */ + case XID_ABORTED: + return false; /* HEAPTUPLE_DEAD */ + } + } + } + } + + if (!(infomask & HEAP_XMAX_INVALID) && !HEAP_XMAX_IS_LOCKED_ONLY(infomask)) + { + if (infomask & HEAP_XMAX_IS_MULTI) + { + XidCommitStatus status; + TransactionId xmax = HeapTupleGetUpdateXid(tuphdr); + + switch (get_xid_status(xmax, ctx, &status)) + { + /* not LOCKED_ONLY, so it has to have an xmax */ + case XID_INVALID: + report_corruption(ctx, + pstrdup("xmax is invalid")); + return false; /* corrupt */ + case XID_IN_FUTURE: + report_corruption(ctx, + psprintf("xmax %u equals or exceeds next valid transaction ID %u:%u", + xmax, + EpochFromFullTransactionId(ctx->next_fxid), + XidFromFullTransactionId(ctx->next_fxid))); + return false; /* corrupt */ + case XID_PRECEDES_RELMIN: + report_corruption(ctx, + psprintf("xmax %u precedes relation freeze threshold %u:%u", + xmax, + EpochFromFullTransactionId(ctx->relfrozenfxid), + XidFromFullTransactionId(ctx->relfrozenfxid))); + return false; /* corrupt */ + case XID_PRECEDES_CLUSTERMIN: + report_corruption(ctx, + psprintf("xmax %u precedes oldest valid transaction ID %u:%u", + xmax, + EpochFromFullTransactionId(ctx->oldest_fxid), + XidFromFullTransactionId(ctx->oldest_fxid))); + return false; /* corrupt */ + case XID_BOUNDS_OK: + switch (status) + { + case XID_IN_PROGRESS: + return true; /* HEAPTUPLE_DELETE_IN_PROGRESS */ + case XID_COMMITTED: + case XID_ABORTED: + return false; /* HEAPTUPLE_RECENTLY_DEAD or + * HEAPTUPLE_DEAD */ + } + } + + /* Ok, the tuple is live */ + } + else if (!(infomask & HEAP_XMAX_COMMITTED)) + return true; /* HEAPTUPLE_DELETE_IN_PROGRESS or + * HEAPTUPLE_LIVE */ + else + return false; /* HEAPTUPLE_RECENTLY_DEAD or HEAPTUPLE_DEAD */ + } + return true; /* not dead */ +} + +/* + * Check the current toast tuple against the state tracked in ctx, recording + * any corruption found in ctx->tupstore. + * + * This is not equivalent to running verify_heapam on the toast table itself, + * and is not hardened against corruption of the toast table. Rather, when + * validating a toasted attribute in the main table, the sequence of toast + * tuples that store the toasted value are retrieved and checked in order, with + * each toast tuple being checked against where we are in the sequence, as well + * as each toast tuple having its varlena structure sanity checked. + */ +static void +check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx) +{ + int32 curchunk; + Pointer chunk; + bool isnull; + int32 chunksize; + int32 expected_size; + + /* + * Have a chunk, extract the sequence number and the data + */ + curchunk = DatumGetInt32(fastgetattr(toasttup, 2, + ctx->toast_rel->rd_att, &isnull)); + if (isnull) + { + report_corruption(ctx, + pstrdup("toast chunk sequence number is null")); + return; + } + chunk = DatumGetPointer(fastgetattr(toasttup, 3, + ctx->toast_rel->rd_att, &isnull)); + if (isnull) + { + report_corruption(ctx, + pstrdup("toast chunk data is null")); + return; + } + if (!VARATT_IS_EXTENDED(chunk)) + chunksize = VARSIZE(chunk) - VARHDRSZ; + else if (VARATT_IS_SHORT(chunk)) + { + /* + * could happen due to heap_form_tuple doing its thing + */ + chunksize = VARSIZE_SHORT(chunk) - VARHDRSZ_SHORT; + } + else + { + /* should never happen */ + uint32 header = ((varattrib_4b *) chunk)->va_4byte.va_header; + + report_corruption(ctx, + psprintf("corrupt extended toast chunk has invalid varlena header: %0x (sequence number %d)", + header, curchunk)); + return; + } + + /* + * Some checks on the data we've found + */ + if (curchunk != ctx->chunkno) + { + report_corruption(ctx, + psprintf("toast chunk sequence number %u does not match the expected sequence number %u", + curchunk, ctx->chunkno)); + return; + } + if (curchunk > ctx->endchunk) + { + report_corruption(ctx, + psprintf("toast chunk sequence number %u exceeds the end chunk sequence number %u", + curchunk, ctx->endchunk)); + return; + } + + expected_size = curchunk < ctx->totalchunks - 1 ? TOAST_MAX_CHUNK_SIZE + : ctx->attrsize - ((ctx->totalchunks - 1) * TOAST_MAX_CHUNK_SIZE); + if (chunksize != expected_size) + { + report_corruption(ctx, + psprintf("toast chunk size %u differs from the expected size %u", + chunksize, expected_size)); + return; + } +} + +/* + * Check the current attribute as tracked in ctx, recording any corruption + * found in ctx->tupstore. + * + * This function follows the logic performed by heap_deform_tuple(), and in the + * case of a toasted value, optionally continues along the logic of + * detoast_external_attr(), checking for any conditions that would result in + * either of those functions Asserting or crashing the backend. The checks + * performed by Asserts present in those two functions are also performed here. + * In cases where those two functions are a bit cavalier in their assumptions + * about data being correct, we perform additional checks not present in either + * of those two functions. Where some condition is checked in both of those + * functions, we perform it here twice, as we parallel the logical flow of + * those two functions. The presence of duplicate checks seems a reasonable + * price to pay for keeping this code tightly coupled with the code it + * protects. + * + * Returns true if the tuple attribute is sane enough for processing to + * continue on to the next attribute, false otherwise. + */ +static bool +check_tuple_attribute(HeapCheckContext *ctx) +{ + struct varatt_external toast_pointer; + ScanKeyData toastkey; + SysScanDesc toastscan; + SnapshotData SnapshotToast; + HeapTuple toasttup; + bool found_toasttup; + Datum attdatum; + struct varlena *attr; + char *tp; /* pointer to the tuple data */ + uint16 infomask; + Form_pg_attribute thisatt; + + infomask = ctx->tuphdr->t_infomask; + thisatt = TupleDescAttr(RelationGetDescr(ctx->rel), ctx->attnum); + + tp = (char *) ctx->tuphdr + ctx->tuphdr->t_hoff; + + if (ctx->tuphdr->t_hoff + ctx->offset > ctx->lp_len) + { + report_corruption(ctx, + psprintf("attribute %u with length %u starts at offset %u beyond total tuple length %u", + ctx->attnum, + thisatt->attlen, + ctx->tuphdr->t_hoff + ctx->offset, + ctx->lp_len)); + return false; + } + + /* Skip null values */ + if (infomask & HEAP_HASNULL && att_isnull(ctx->attnum, ctx->tuphdr->t_bits)) + return true; + + /* Skip non-varlena values, but update offset first */ + if (thisatt->attlen != -1) + { + ctx->offset = att_align_nominal(ctx->offset, thisatt->attalign); + ctx->offset = att_addlength_pointer(ctx->offset, thisatt->attlen, + tp + ctx->offset); + if (ctx->tuphdr->t_hoff + ctx->offset > ctx->lp_len) + { + report_corruption(ctx, + psprintf("attribute %u with length %u ends at offset %u beyond total tuple length %u", + ctx->attnum, + thisatt->attlen, + ctx->tuphdr->t_hoff + ctx->offset, + ctx->lp_len)); + return false; + } + return true; + } + + /* Ok, we're looking at a varlena attribute. */ + ctx->offset = att_align_pointer(ctx->offset, thisatt->attalign, -1, + tp + ctx->offset); + + /* Get the (possibly corrupt) varlena datum */ + attdatum = fetchatt(thisatt, tp + ctx->offset); + + /* + * We have the datum, but we cannot decode it carelessly, as it may still + * be corrupt. + */ + + /* + * Check that VARTAG_SIZE won't hit a TrapMacro on a corrupt va_tag before + * risking a call into att_addlength_pointer + */ + if (VARATT_IS_EXTERNAL(tp + ctx->offset)) + { + uint8 va_tag = VARTAG_EXTERNAL(tp + ctx->offset); + + if (va_tag != VARTAG_ONDISK) + { + report_corruption(ctx, + psprintf("toasted attribute %u has unexpected TOAST tag %u", + ctx->attnum, + va_tag)); + /* We can't know where the next attribute begins */ + return false; + } + } + + /* Ok, should be safe now */ + ctx->offset = att_addlength_pointer(ctx->offset, thisatt->attlen, + tp + ctx->offset); + + if (ctx->tuphdr->t_hoff + ctx->offset > ctx->lp_len) + { + report_corruption(ctx, + psprintf("attribute %u with length %u ends at offset %u beyond total tuple length %u", + ctx->attnum, + thisatt->attlen, + ctx->tuphdr->t_hoff + ctx->offset, + ctx->lp_len)); + + return false; + } + + /* + * heap_deform_tuple would be done with this attribute at this point, + * having stored it in values[], and would continue to the next attribute. + * We go further, because we need to check if the toast datum is corrupt. + */ + + attr = (struct varlena *) DatumGetPointer(attdatum); + + /* + * Now we follow the logic of detoast_external_attr(), with the same + * caveats about being paranoid about corruption. + */ + + /* Skip values that are not external */ + if (!VARATT_IS_EXTERNAL(attr)) + return true; + + /* It is external, and we're looking at a page on disk */ + + /* The tuple header better claim to contain toasted values */ + if (!(infomask & HEAP_HASEXTERNAL)) + { + report_corruption(ctx, + psprintf("attribute %u is external but tuple header flag HEAP_HASEXTERNAL not set", + ctx->attnum)); + return true; + } + + /* The relation better have a toast table */ + if (!ctx->rel->rd_rel->reltoastrelid) + { + report_corruption(ctx, + psprintf("attribute %u is external but relation has no toast relation", + ctx->attnum)); + return true; + } + + /* If we were told to skip toast checking, then we're done. */ + if (ctx->toast_rel == NULL) + return true; + + /* + * Must copy attr into toast_pointer for alignment considerations + */ + VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); + + ctx->attrsize = toast_pointer.va_extsize; + ctx->endchunk = (ctx->attrsize - 1) / TOAST_MAX_CHUNK_SIZE; + ctx->totalchunks = ctx->endchunk + 1; + + /* + * Setup a scan key to find chunks in toast table with matching va_valueid + */ + ScanKeyInit(&toastkey, + (AttrNumber) 1, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(toast_pointer.va_valueid)); + + /* + * Check if any chunks for this toasted object exist in the toast table, + * accessible via the index. + */ + init_toast_snapshot(&SnapshotToast); + toastscan = systable_beginscan_ordered(ctx->toast_rel, + ctx->valid_toast_index, + &SnapshotToast, 1, + &toastkey); + ctx->chunkno = 0; + found_toasttup = false; + while ((toasttup = + systable_getnext_ordered(toastscan, + ForwardScanDirection)) != NULL) + { + found_toasttup = true; + check_toast_tuple(toasttup, ctx); + ctx->chunkno++; + } + if (ctx->chunkno != (ctx->endchunk + 1)) + report_corruption(ctx, + psprintf("final toast chunk number %u differs from expected value %u", + ctx->chunkno, (ctx->endchunk + 1))); + if (!found_toasttup) + report_corruption(ctx, + psprintf("toasted value for attribute %u missing from toast table", + ctx->attnum)); + systable_endscan_ordered(toastscan); + + return true; +} + +/* + * Check the current tuple as tracked in ctx, recording any corruption found in + * ctx->tupstore. + */ +static void +check_tuple(HeapCheckContext *ctx) +{ + TransactionId xmin; + TransactionId xmax; + bool fatal = false; + uint16 infomask = ctx->tuphdr->t_infomask; + + /* + * If we report corruption before iterating over individual attributes, we + * need attnum to be reported as NULL. Set that up before any corruption + * reporting might happen. + */ + ctx->attnum = -1; + + /* + * If the line pointer for this tuple does not reserve enough space for a + * complete tuple header, we dare not read the tuple header. + */ + if (ctx->lp_len < MAXALIGN(SizeofHeapTupleHeader)) + { + report_corruption(ctx, + psprintf("line pointer length %u is less than the minimum tuple header size %u", + ctx->lp_len, (uint32) MAXALIGN(SizeofHeapTupleHeader))); + return; + } + + /* If xmin is normal, it should be within valid range */ + xmin = HeapTupleHeaderGetXmin(ctx->tuphdr); + switch (get_xid_status(xmin, ctx, NULL)) + { + case XID_INVALID: + case XID_BOUNDS_OK: + break; + case XID_IN_FUTURE: + report_corruption(ctx, + psprintf("xmin %u equals or exceeds next valid transaction ID %u:%u", + xmin, + EpochFromFullTransactionId(ctx->next_fxid), + XidFromFullTransactionId(ctx->next_fxid))); + fatal = true; + break; + case XID_PRECEDES_CLUSTERMIN: + report_corruption(ctx, + psprintf("xmin %u precedes oldest valid transaction ID %u:%u", + xmin, + EpochFromFullTransactionId(ctx->oldest_fxid), + XidFromFullTransactionId(ctx->oldest_fxid))); + fatal = true; + break; + case XID_PRECEDES_RELMIN: + report_corruption(ctx, + psprintf("xmin %u precedes relation freeze threshold %u:%u", + xmin, + EpochFromFullTransactionId(ctx->relfrozenfxid), + XidFromFullTransactionId(ctx->relfrozenfxid))); + fatal = true; + break; + } + + xmax = HeapTupleHeaderGetRawXmax(ctx->tuphdr); + + if (infomask & HEAP_XMAX_IS_MULTI) + { + /* xmax is a multixact, so it should be within valid MXID range */ + switch (check_mxid_valid_in_rel(xmax, ctx)) + { + case XID_INVALID: + report_corruption(ctx, + pstrdup("multitransaction ID is invalid")); + fatal = true; + break; + case XID_PRECEDES_RELMIN: + report_corruption(ctx, + psprintf("multitransaction ID %u precedes relation minimum multitransaction ID threshold %u", + xmax, ctx->relminmxid)); + fatal = true; + break; + case XID_PRECEDES_CLUSTERMIN: + report_corruption(ctx, + psprintf("multitransaction ID %u precedes oldest valid multitransaction ID threshold %u", + xmax, ctx->oldest_mxact)); + fatal = true; + break; + case XID_IN_FUTURE: + report_corruption(ctx, + psprintf("multitransaction ID %u equals or exceeds next valid multitransaction ID %u", + xmax, + ctx->next_mxact)); + fatal = true; + break; + case XID_BOUNDS_OK: + break; + } + } + else + { + /* + * xmax is not a multixact and is normal, so it should be within the + * valid XID range. + */ + switch (get_xid_status(xmax, ctx, NULL)) + { + case XID_INVALID: + case XID_BOUNDS_OK: + break; + case XID_IN_FUTURE: + report_corruption(ctx, + psprintf("xmax %u equals or exceeds next valid transaction ID %u:%u", + xmax, + EpochFromFullTransactionId(ctx->next_fxid), + XidFromFullTransactionId(ctx->next_fxid))); + fatal = true; + break; + case XID_PRECEDES_CLUSTERMIN: + report_corruption(ctx, + psprintf("xmax %u precedes oldest valid transaction ID %u:%u", + xmax, + EpochFromFullTransactionId(ctx->oldest_fxid), + XidFromFullTransactionId(ctx->oldest_fxid))); + fatal = true; + break; + case XID_PRECEDES_RELMIN: + report_corruption(ctx, + psprintf("xmax %u precedes relation freeze threshold %u:%u", + xmax, + EpochFromFullTransactionId(ctx->relfrozenfxid), + XidFromFullTransactionId(ctx->relfrozenfxid))); + fatal = true; + } + } + + /* + * Cannot process tuple data if tuple header was corrupt, as the offsets + * within the page cannot be trusted, leaving too much risk of reading + * garbage if we continue. + * + * We also cannot process the tuple if the xmin or xmax were invalid + * relative to relfrozenxid or relminmxid, as clog entries for the xids + * may already be gone. + */ + if (fatal) + return; + + /* + * Check various forms of tuple header corruption. If the header is too + * corrupt to continue checking, or if the tuple is not visible to anyone, + * we cannot continue with other checks. + */ + if (!check_tuple_header_and_visibilty(ctx->tuphdr, ctx)) + return; + + /* + * The tuple is visible, so it must be compatible with the current version + * of the relation descriptor. It might have fewer columns than are + * present in the relation descriptor, but it cannot have more. + */ + if (RelationGetDescr(ctx->rel)->natts < ctx->natts) + { + report_corruption(ctx, + psprintf("number of attributes %u exceeds maximum expected for table %u", + ctx->natts, + RelationGetDescr(ctx->rel)->natts)); + return; + } + + /* + * Check each attribute unless we hit corruption that confuses what to do + * next, at which point we abort further attribute checks for this tuple. + * Note that we don't abort for all types of corruption, only for those + * types where we don't know how to continue. + */ + ctx->offset = 0; + for (ctx->attnum = 0; ctx->attnum < ctx->natts; ctx->attnum++) + if (!check_tuple_attribute(ctx)) + break; /* cannot continue */ +} + +/* + * Convert a TransactionId into a FullTransactionId using our cached values of + * the valid transaction ID range. It is the caller's responsibility to have + * already updated the cached values, if necessary. + */ +static FullTransactionId +FullTransactionIdFromXidAndCtx(TransactionId xid, const HeapCheckContext *ctx) +{ + uint32 epoch; + + if (!TransactionIdIsNormal(xid)) + return FullTransactionIdFromEpochAndXid(0, xid); + epoch = EpochFromFullTransactionId(ctx->next_fxid); + if (xid > ctx->next_xid) + epoch--; + return FullTransactionIdFromEpochAndXid(epoch, xid); +} + +/* + * Update our cached range of valid transaction IDs. + */ +static void +update_cached_xid_range(HeapCheckContext *ctx) +{ + /* Make cached copies */ + LWLockAcquire(XidGenLock, LW_SHARED); + ctx->next_fxid = ShmemVariableCache->nextXid; + ctx->oldest_xid = ShmemVariableCache->oldestXid; + LWLockRelease(XidGenLock); + + /* And compute alternate versions of the same */ + ctx->oldest_fxid = FullTransactionIdFromXidAndCtx(ctx->oldest_xid, ctx); + ctx->next_xid = XidFromFullTransactionId(ctx->next_fxid); +} + +/* + * Update our cached range of valid multitransaction IDs. + */ +static void +update_cached_mxid_range(HeapCheckContext *ctx) +{ + ReadMultiXactIdRange(&ctx->oldest_mxact, &ctx->next_mxact); +} + +/* + * Return whether the given FullTransactionId is within our cached valid + * transaction ID range. + */ +static inline bool +fxid_in_cached_range(FullTransactionId fxid, const HeapCheckContext *ctx) +{ + return (FullTransactionIdPrecedesOrEquals(ctx->oldest_fxid, fxid) && + FullTransactionIdPrecedes(fxid, ctx->next_fxid)); +} + +/* + * Checks wheter a multitransaction ID is in the cached valid range, returning + * the nature of the range violation, if any. + */ +static XidBoundsViolation +check_mxid_in_range(MultiXactId mxid, HeapCheckContext *ctx) +{ + if (!TransactionIdIsValid(mxid)) + return XID_INVALID; + if (MultiXactIdPrecedes(mxid, ctx->relminmxid)) + return XID_PRECEDES_RELMIN; + if (MultiXactIdPrecedes(mxid, ctx->oldest_mxact)) + return XID_PRECEDES_CLUSTERMIN; + if (MultiXactIdPrecedesOrEquals(ctx->next_mxact, mxid)) + return XID_IN_FUTURE; + return XID_BOUNDS_OK; +} + +/* + * Checks whether the given mxid is valid to appear in the heap being checked, + * returning the nature of the range violation, if any. + * + * This function attempts to return quickly by caching the known valid mxid + * range in ctx. Callers should already have performed the initial setup of + * the cache prior to the first call to this function. + */ +static XidBoundsViolation +check_mxid_valid_in_rel(MultiXactId mxid, HeapCheckContext *ctx) +{ + XidBoundsViolation result; + + result = check_mxid_in_range(mxid, ctx); + if (result == XID_BOUNDS_OK) + return XID_BOUNDS_OK; + + /* The range may have advanced. Recheck. */ + update_cached_mxid_range(ctx); + return check_mxid_in_range(mxid, ctx); +} + +/* + * Checks whether the given transaction ID is (or was recently) valid to appear + * in the heap being checked, or whether it is too old or too new to appear in + * the relation, returning information about the nature of the bounds violation. + * + * We cache the range of valid transaction IDs. If xid is in that range, we + * conclude that it is valid, even though concurrent changes to the table might + * invalidate it under certain corrupt conditions. (For example, if the table + * contains corrupt all-frozen bits, a concurrent vacuum might skip the page(s) + * containing the xid and then truncate clog and advance the relfrozenxid + * beyond xid.) Reporting the xid as valid under such conditions seems + * acceptable, since if we had checked it earlier in our scan it would have + * truly been valid at that time. + * + * If the status argument is not NULL, and if and only if the transaction ID + * appears to be valid in this relation, clog will be consulted and the commit + * status argument will be set with the status of the transaction ID. + */ +static XidBoundsViolation +get_xid_status(TransactionId xid, HeapCheckContext *ctx, + XidCommitStatus *status) +{ + XidBoundsViolation result; + FullTransactionId fxid; + FullTransactionId clog_horizon; + + /* Quick check for special xids */ + if (!TransactionIdIsValid(xid)) + result = XID_INVALID; + else if (xid == BootstrapTransactionId || xid == FrozenTransactionId) + result = XID_BOUNDS_OK; + else + { + /* Check if the xid is within bounds */ + fxid = FullTransactionIdFromXidAndCtx(xid, ctx); + if (!fxid_in_cached_range(fxid, ctx)) + { + /* + * We may have been checking against stale values. Update the + * cached range to be sure, and since we relied on the cached + * range when we performed the full xid conversion, reconvert. + */ + update_cached_xid_range(ctx); + fxid = FullTransactionIdFromXidAndCtx(xid, ctx); + } + + if (FullTransactionIdPrecedesOrEquals(ctx->next_fxid, fxid)) + result = XID_IN_FUTURE; + else if (FullTransactionIdPrecedes(fxid, ctx->oldest_fxid)) + result = XID_PRECEDES_CLUSTERMIN; + else if (FullTransactionIdPrecedes(fxid, ctx->relfrozenfxid)) + result = XID_PRECEDES_RELMIN; + else + result = XID_BOUNDS_OK; + } + + /* + * Early return if the caller does not request clog checking, or if the + * xid is already known to be out of bounds. We dare not check clog for + * out of bounds transaction IDs. + */ + if (status == NULL || result != XID_BOUNDS_OK) + return result; + + /* Early return if we just checked this xid in a prior call */ + if (xid == ctx->cached_xid) + { + *status = ctx->cached_status; + return result; + } + + *status = XID_COMMITTED; + LWLockAcquire(XactTruncationLock, LW_SHARED); + clog_horizon = + FullTransactionIdFromXidAndCtx(ShmemVariableCache->oldestClogXid, + ctx); + if (FullTransactionIdPrecedesOrEquals(clog_horizon, fxid)) + { + if (TransactionIdIsCurrentTransactionId(xid)) + *status = XID_IN_PROGRESS; + else if (TransactionIdDidCommit(xid)) + *status = XID_COMMITTED; + else if (TransactionIdDidAbort(xid)) + *status = XID_ABORTED; + else + *status = XID_IN_PROGRESS; + } + LWLockRelease(XactTruncationLock); + ctx->cached_xid = xid; + ctx->cached_status = *status; + return result; +} diff --git a/doc/src/sgml/amcheck.sgml b/doc/src/sgml/amcheck.sgml index a9df2c1a9d22..25e4bb2bfec2 100644 --- a/doc/src/sgml/amcheck.sgml +++ b/doc/src/sgml/amcheck.sgml @@ -9,12 +9,11 @@ The amcheck module provides functions that allow you to - verify the logical consistency of the structure of relations. If the - structure appears to be valid, no error is raised. + verify the logical consistency of the structure of relations. - The functions verify various invariants in the + The B-Tree checking functions verify various invariants in the structure of the representation of particular relations. The correctness of the access method functions behind index scans and other important operations relies on these invariants always @@ -24,7 +23,7 @@ collated lexical order). If that particular invariant somehow fails to hold, we can expect binary searches on the affected page to incorrectly guide index scans, resulting in wrong answers to SQL - queries. + queries. If the structure appears to be valid, no error is raised. Verification is performed using the same procedures as those used by @@ -35,7 +34,22 @@ functions. - amcheck functions may only be used by superusers. + Unlike the B-Tree checking functions which report corruption by raising + errors, the heap checking function verify_heapam checks + a table and attempts to return a set of rows, one row per corruption + detected. Despite this, if facilities that + verify_heapam relies upon are themselves corrupted, the + function may be unable to continue and may instead raise an error. + + + Permission to execute amcheck functions may be granted + to non-superusers, but before granting such permissions careful consideration + should be given to data security and privacy concerns. Although the + corruption reports generated by these functions do not focus on the contents + of the corrupted data so much as on the structure of that data and the nature + of the corruptions found, an attacker who gains permission to execute these + functions, particularly if the attacker can also induce corruption, might be + able to infer something of the data itself from such messages. @@ -187,12 +201,221 @@ SET client_min_messages = DEBUG1; + + + + + verify_heapam(relation regclass, + on_error_stop boolean, + check_toast boolean, + skip cstring, + startblock bigint, + endblock bigint, + blkno OUT bigint, + offnum OUT integer, + attnum OUT integer, + msg OUT text) + returns record + + + + + Checks a table for structural corruption, where pages in the relation + contain data that is invalidly formatted, and for logical corruption, + where pages are structurally valid but inconsistent with the rest of the + database cluster. Example usage: + +test=# select * from verify_heapam('mytable', check_toast := true); + blkno | offnum | attnum | msg +-------+--------+--------+-------------------------------------------------------------------------------------------------- + 17 | 12 | | xmin 4294967295 precedes relation freeze threshold 17:1134217582 + 960 | 4 | | data begins at offset 152 beyond the tuple length 58 + 960 | 4 | | tuple data should begin at byte 24, but actually begins at byte 152 (3 attributes, no nulls) + 960 | 5 | | tuple data should begin at byte 24, but actually begins at byte 27 (3 attributes, no nulls) + 960 | 6 | | tuple data should begin at byte 24, but actually begins at byte 16 (3 attributes, no nulls) + 960 | 7 | | tuple data should begin at byte 24, but actually begins at byte 21 (3 attributes, no nulls) + 1147 | 2 | | number of attributes 2047 exceeds maximum expected for table 3 + 1147 | 10 | | tuple data should begin at byte 280, but actually begins at byte 24 (2047 attributes, has nulls) + 1147 | 15 | | number of attributes 67 exceeds maximum expected for table 3 + 1147 | 16 | 1 | attribute 1 with length 4294967295 ends at offset 416848000 beyond total tuple length 58 + 1147 | 18 | 2 | final toast chunk number 0 differs from expected value 6 + 1147 | 19 | 2 | toasted value for attribute 2 missing from toast table + 1147 | 21 | | tuple is marked as only locked, but also claims key columns were updated + 1147 | 22 | | multitransaction ID 1775655 is from before relation cutoff 2355572 +(14 rows) + + As this example shows, the Tuple ID (TID) of the corrupt tuple is given + in the (blkno, offnum) columns, and + for corruptions specific to a particular attribute in the tuple, the + attnum field shows which one. + + + Structural corruption can happen due to faulty storage hardware, or + relation files being overwritten or modified by unrelated software. + This kind of corruption can also be detected with + data page + checksums. + + + Relation pages which are correctly formatted, internally consistent, and + correct relative to their own internal checksums may still contain + logical corruption. As such, this kind of corruption cannot be detected + with checksums. Examples include toasted + values in the main table which lack a corresponding entry in the toast + table, and tuples in the main table with a Transaction ID that is older + than the oldest valid Transaction ID in the database or cluster. + + + Multiple causes of logical corruption have been observed in production + systems, including bugs in the PostgreSQL + server software, faulty and ill-conceived backup and restore tools, and + user error. + + + Corrupt relations are most concerning in live production environments, + precisely the same environments where high risk activities are least + welcome. For this reason, verify_heapam has been + designed to diagnose corruption without undue risk. It cannot guard + against all causes of backend crashes, as even executing the calling + query could be unsafe on a badly corrupted system. Access to catalog tables are performed and could + be problematic if the catalogs themselves are corrupted. + + + The design principle adhered to in verify_heapam is + that, if the rest of the system and server hardware are correct, under + default options, verify_heapam will not crash the + server due merely to structural or logical corruption in the target + table. + + + The check_toast attempts to reconcile the target + table against entries in its corresponding toast table. This option is + disabled by default and is known to be slow. + If the target relation's corresponding toast table or toast index is + corrupt, reconciling the target table against toast values could + conceivably crash the server, although in many cases this would + just produce an error. + + + The following optional arguments are recognized: + + + + on_error_stop + + + If true, corruption checking stops at the end of the first block on + which any corruptions are found. + + + Defaults to false. + + + + + check_toast + + + If true, toasted values are checked gainst the corresponding + TOAST table. + + + Defaults to false. + + + + + skip + + + If not none, corruption checking skips blocks that + are marked as all-visible or all-frozen, as given. + Valid options are all-visible, + all-frozen and none. + + + Defaults to none. + + + + + startblock + + + If specified, corruption checking begins at the specified block, + skipping all previous blocks. It is an error to specify a + startblock outside the range of blocks in the + target table. + + + By default, does not skip any blocks. + + + + + endblock + + + If specified, corruption checking ends at the specified block, + skipping all remaining blocks. It is an error to specify an + endblock outside the range of blocks in the target + table. + + + By default, does not skip any blocks. + + + + + + For each corruption detected, verify_heapam returns + a row with the following columns: + + + + blkno + + + The number of the block containing the corrupt page. + + + + + offnum + + + The OffsetNumber of the corrupt tuple. + + + + + attnum + + + The attribute number of the corrupt column in the tuple, if the + corruption is specific to a column and not the tuple as a whole. + + + + + msg + + + A human readable message describing the corruption in the page. + + + + + + + Optional <parameter>heapallindexed</parameter> Verification - When the heapallindexed argument to + When the heapallindexed argument to B-Tree verification functions is true, an additional phase of verification is performed against the table associated with the target index relation. This consists of a dummy diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c index aa3f14c019c4..ca357410a293 100644 --- a/src/backend/access/heap/hio.c +++ b/src/backend/access/heap/hio.c @@ -47,6 +47,17 @@ RelationPutHeapTuple(Relation relation, */ Assert(!token || HeapTupleHeaderIsSpeculative(tuple->t_data)); + /* + * Do not allow tuples with invalid combinations of hint bits to be placed + * on a page. These combinations are detected as corruption by the + * contrib/amcheck logic, so if you disable one or both of these + * assertions, make corresponding changes there. + */ + Assert(!((tuple->t_data->t_infomask & HEAP_XMAX_LOCK_ONLY) && + (tuple->t_data->t_infomask2 & HEAP_KEYS_UPDATED))); + Assert(!((tuple->t_data->t_infomask & HEAP_XMAX_COMMITTED) && + (tuple->t_data->t_infomask & HEAP_XMAX_IS_MULTI))); + /* Add the tuple to the page */ pageHeader = BufferGetPage(buffer); diff --git a/src/backend/access/transam/multixact.c b/src/backend/access/transam/multixact.c index 6ccdc5b58cba..43653fe57212 100644 --- a/src/backend/access/transam/multixact.c +++ b/src/backend/access/transam/multixact.c @@ -735,6 +735,25 @@ ReadNextMultiXactId(void) return mxid; } +/* + * ReadMultiXactIdRange + * Get the range of IDs that may still be referenced by a relation. + */ +void +ReadMultiXactIdRange(MultiXactId *oldest, MultiXactId *next) +{ + LWLockAcquire(MultiXactGenLock, LW_SHARED); + *oldest = MultiXactState->oldestMultiXactId; + *next = MultiXactState->nextMXact; + LWLockRelease(MultiXactGenLock); + + if (*oldest < FirstMultiXactId) + *oldest = FirstMultiXactId; + if (*next < FirstMultiXactId) + *next = FirstMultiXactId; +} + + /* * MultiXactIdCreateFromMembers * Make a new MultiXactId from the specified set of members diff --git a/src/include/access/multixact.h b/src/include/access/multixact.h index 58c42ffe1fe6..9a303809019a 100644 --- a/src/include/access/multixact.h +++ b/src/include/access/multixact.h @@ -109,6 +109,7 @@ extern MultiXactId MultiXactIdCreateFromMembers(int nmembers, MultiXactMember *members); extern MultiXactId ReadNextMultiXactId(void); +extern void ReadMultiXactIdRange(MultiXactId *oldest, MultiXactId *next); extern bool MultiXactIdIsRunning(MultiXactId multi, bool isLockOnly); extern void MultiXactIdSetOldestMember(void); extern int GetMultiXactIdMembers(MultiXactId multi, MultiXactMember **xids, diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index c52f20d4ba4d..ff853634bc52 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1020,6 +1020,7 @@ HbaToken HeadlineJsonState HeadlineParsedText HeadlineWordEntry +HeapCheckContext HeapScanDesc HeapTuple HeapTupleData @@ -2290,6 +2291,7 @@ SimpleStringList SimpleStringListCell SingleBoundSortItem Size +SkipPages SlabBlock SlabChunk SlabContext @@ -2791,6 +2793,8 @@ XactCallback XactCallbackItem XactEvent XactLockTableWaitInfo +XidBoundsViolation +XidCommitStatus XidHorizonPrefetchState XidStatus XmlExpr From 94929f1cf6cb3ea070d0919d1303379b525a72a3 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 22 Oct 2020 14:04:21 -0400 Subject: [PATCH 353/589] Clean up some unpleasant behaviors in psql's \connect command. The check for whether to complain about not having an old connection to get parameters from was seriously out of date: it had not been rethought when we invented connstrings, nor when we invented the -reuse-previous option. Replace it with a check that throws an error if reuse-previous is active and we lack an old connection to reuse. While that doesn't move the goalposts very far in terms of easing reconnection after a server crash, at least it's consistent. If the user specifies a connstring plus additional parameters (which is invalid per the documentation), the extra parameters were silently ignored. That seems like it could be really confusing, so let's throw a syntax error instead. Teach the connstring code path to re-use the old connection's password in the same cases as the old-style-syntax code path would, ie if we are reusing parameters and the values of username, host/hostaddr, and port are not being changed. Document this behavior, too, since it was unmentioned before. Also simplify the implementation a bit, giving rise to two new and useful properties: if there's a "password=xxx" in the connstring, we'll use it not ignore it, and by default (i.e., except with --no-password) we will prompt for a password if the re-used password or connstring password doesn't work. The previous code just failed if the re-used password didn't work. Given the paucity of field complaints about these issues, I don't think that they rise to the level of back-patchable bug fixes, and in any case they might represent undesirable behavior changes in minor releases. So no back-patch. Discussion: https://postgr.es/m/235210.1603321144@sss.pgh.pa.us --- doc/src/sgml/ref/psql-ref.sgml | 2 + src/bin/psql/command.c | 91 +++++++++++++++++++--------------- 2 files changed, 54 insertions(+), 39 deletions(-) diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml index 7d0d3616573f..f6f77dbac3ac 100644 --- a/doc/src/sgml/ref/psql-ref.sgml +++ b/doc/src/sgml/ref/psql-ref.sgml @@ -920,6 +920,8 @@ testdb=> is changed from its previous value using the positional syntax, any hostaddr setting present in the existing connection's parameters is dropped. + Also, any password used for the existing connection will be re-used + only if the user, host, and port settings are not changed. When the command neither specifies nor reuses a particular parameter, the libpq default is used. diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c index ba3ea39aaa4d..39a460d85ce9 100644 --- a/src/bin/psql/command.c +++ b/src/bin/psql/command.c @@ -3014,11 +3014,10 @@ param_is_newly_set(const char *old_val, const char *new_val) /* * do_connect -- handler for \connect * - * Connects to a database with given parameters. Absent an established - * connection, all parameters are required. Given -reuse-previous=off or a - * connection string without -reuse-previous=on, NULL values will pass through - * to PQconnectdbParams(), so the libpq defaults will be used. Otherwise, NULL - * values will be replaced with the ones in the current connection. + * Connects to a database with given parameters. If we are told to re-use + * parameters, parameters from the previous connection are used where the + * command's own options do not supply a value. Otherwise, libpq defaults + * are used. * * In interactive mode, if connection fails with the given parameters, * the old connection will be kept. @@ -3038,20 +3037,16 @@ do_connect(enum trivalue reuse_previous_specification, bool has_connection_string; bool reuse_previous; - if (!o_conn && (!dbname || !user || !host || !port)) + has_connection_string = dbname ? + recognized_connection_string(dbname) : false; + + /* Complain if we have additional arguments after a connection string. */ + if (has_connection_string && (user || host || port)) { - /* - * We don't know the supplied connection parameters and don't want to - * connect to the wrong database by using defaults, so require all - * parameters to be specified. - */ - pg_log_error("All connection parameters must be supplied because no " - "database connection exists"); + pg_log_error("Do not give user, host, or port separately when using a connection string"); return false; } - has_connection_string = dbname ? - recognized_connection_string(dbname) : false; switch (reuse_previous_specification) { case TRI_YES: @@ -3065,16 +3060,14 @@ do_connect(enum trivalue reuse_previous_specification, break; } - /* If the old connection does not exist, there is nothing to reuse. */ - if (!o_conn) - reuse_previous = false; - - /* Silently ignore arguments subsequent to a connection string. */ - if (has_connection_string) + /* + * If we are to re-use parameters, there'd better be an old connection to + * get them from. + */ + if (reuse_previous && !o_conn) { - user = NULL; - host = NULL; - port = NULL; + pg_log_error("No database connection exists to re-use parameters from"); + return false; } /* @@ -3103,6 +3096,7 @@ do_connect(enum trivalue reuse_previous_specification, { PQconninfoOption *ci; PQconninfoOption *replci; + bool have_password = false; for (ci = cinfo, replci = replcinfo; ci->keyword && replci->keyword; @@ -3121,6 +3115,26 @@ do_connect(enum trivalue reuse_previous_specification, replci->val = ci->val; ci->val = swap; + + /* + * Check whether connstring provides options affecting + * password re-use. While any change in user, host, + * hostaddr, or port causes us to ignore the old + * connection's password, we don't force that for + * dbname, since passwords aren't database-specific. + */ + if (replci->val == NULL || + strcmp(ci->val, replci->val) != 0) + { + if (strcmp(replci->keyword, "user") == 0 || + strcmp(replci->keyword, "host") == 0 || + strcmp(replci->keyword, "hostaddr") == 0 || + strcmp(replci->keyword, "port") == 0) + keep_password = false; + } + /* Also note whether connstring contains a password. */ + if (strcmp(replci->keyword, "password") == 0) + have_password = true; } } Assert(ci->keyword == NULL && replci->keyword == NULL); @@ -3130,8 +3144,13 @@ do_connect(enum trivalue reuse_previous_specification, PQconninfoFree(replcinfo); - /* We never re-use a password with a conninfo string. */ - keep_password = false; + /* + * If the connstring contains a password, tell the loop below + * that we may use it, regardless of other settings (i.e., + * cinfo's password is no longer an "old" password). + */ + if (have_password) + keep_password = true; /* Don't let code below try to inject dbname into params. */ dbname = NULL; @@ -3219,14 +3238,6 @@ do_connect(enum trivalue reuse_previous_specification, */ password = prompt_for_password(has_connection_string ? NULL : user); } - else if (o_conn && keep_password) - { - password = PQpass(o_conn); - if (password && *password) - password = pg_strdup(password); - else - password = NULL; - } /* Loop till we have a connection or fail, which we might've already */ while (success) @@ -3238,12 +3249,12 @@ do_connect(enum trivalue reuse_previous_specification, /* * Copy non-default settings into the PQconnectdbParams parameter - * arrays; but override any values specified old-style, as well as the - * password and a couple of fields we want to set forcibly. + * arrays; but inject any values specified old-style, as well as any + * interactively-obtained password, and a couple of fields we want to + * set forcibly. * * If you change this code, see also the initial-connection code in - * main(). For no good reason, a connection string password= takes - * precedence in main() but not here. + * main(). */ for (ci = cinfo; ci->keyword; ci++) { @@ -3262,7 +3273,9 @@ do_connect(enum trivalue reuse_previous_specification, } else if (port && strcmp(ci->keyword, "port") == 0) values[paramnum++] = port; - else if (strcmp(ci->keyword, "password") == 0) + /* If !keep_password, we unconditionally drop old password */ + else if ((password || !keep_password) && + strcmp(ci->keyword, "password") == 0) values[paramnum++] = password; else if (strcmp(ci->keyword, "fallback_application_name") == 0) values[paramnum++] = pset.progname; From 8bb0c9770e80fa79f15b27af8f3c3f2833225aa3 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 22 Oct 2020 16:13:58 -0400 Subject: [PATCH 354/589] Try to avoid a compiler warning about using fxid uninitialized. Mark Dilger, with a couple of stray semicolons removed by me. Discussion: http://postgr.es/m/2A7DA1A8-C4AA-43DF-A985-3CA52F4DC775@enterprisedb.com --- contrib/amcheck/verify_heapam.c | 67 +++++++++++++++------------------ 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index 0156c1e74aa1..37b40a0404d5 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -555,7 +555,7 @@ check_tuple_header_and_visibilty(HeapTupleHeader tuphdr, HeapCheckContext *ctx) { uint16 infomask = tuphdr->t_infomask; bool header_garbled = false; - unsigned expected_hoff;; + unsigned expected_hoff; if (ctx->tuphdr->t_hoff > ctx->lp_len) { @@ -1368,60 +1368,55 @@ check_mxid_valid_in_rel(MultiXactId mxid, HeapCheckContext *ctx) * truly been valid at that time. * * If the status argument is not NULL, and if and only if the transaction ID - * appears to be valid in this relation, clog will be consulted and the commit - * status argument will be set with the status of the transaction ID. + * appears to be valid in this relation, the status argument will be set with + * the commit status of the transaction ID. */ static XidBoundsViolation get_xid_status(TransactionId xid, HeapCheckContext *ctx, XidCommitStatus *status) { - XidBoundsViolation result; FullTransactionId fxid; FullTransactionId clog_horizon; /* Quick check for special xids */ if (!TransactionIdIsValid(xid)) - result = XID_INVALID; + return XID_INVALID; else if (xid == BootstrapTransactionId || xid == FrozenTransactionId) - result = XID_BOUNDS_OK; - else { - /* Check if the xid is within bounds */ - fxid = FullTransactionIdFromXidAndCtx(xid, ctx); - if (!fxid_in_cached_range(fxid, ctx)) - { - /* - * We may have been checking against stale values. Update the - * cached range to be sure, and since we relied on the cached - * range when we performed the full xid conversion, reconvert. - */ - update_cached_xid_range(ctx); - fxid = FullTransactionIdFromXidAndCtx(xid, ctx); - } + if (status != NULL) + *status = XID_COMMITTED; + return XID_BOUNDS_OK; + } - if (FullTransactionIdPrecedesOrEquals(ctx->next_fxid, fxid)) - result = XID_IN_FUTURE; - else if (FullTransactionIdPrecedes(fxid, ctx->oldest_fxid)) - result = XID_PRECEDES_CLUSTERMIN; - else if (FullTransactionIdPrecedes(fxid, ctx->relfrozenfxid)) - result = XID_PRECEDES_RELMIN; - else - result = XID_BOUNDS_OK; + /* Check if the xid is within bounds */ + fxid = FullTransactionIdFromXidAndCtx(xid, ctx); + if (!fxid_in_cached_range(fxid, ctx)) + { + /* + * We may have been checking against stale values. Update the + * cached range to be sure, and since we relied on the cached + * range when we performed the full xid conversion, reconvert. + */ + update_cached_xid_range(ctx); + fxid = FullTransactionIdFromXidAndCtx(xid, ctx); } - /* - * Early return if the caller does not request clog checking, or if the - * xid is already known to be out of bounds. We dare not check clog for - * out of bounds transaction IDs. - */ - if (status == NULL || result != XID_BOUNDS_OK) - return result; + if (FullTransactionIdPrecedesOrEquals(ctx->next_fxid, fxid)) + return XID_IN_FUTURE; + if (FullTransactionIdPrecedes(fxid, ctx->oldest_fxid)) + return XID_PRECEDES_CLUSTERMIN; + if (FullTransactionIdPrecedes(fxid, ctx->relfrozenfxid)) + return XID_PRECEDES_RELMIN; + + /* Early return if the caller does not request clog checking */ + if (status == NULL) + return XID_BOUNDS_OK; /* Early return if we just checked this xid in a prior call */ if (xid == ctx->cached_xid) { *status = ctx->cached_status; - return result; + return XID_BOUNDS_OK; } *status = XID_COMMITTED; @@ -1443,5 +1438,5 @@ get_xid_status(TransactionId xid, HeapCheckContext *ctx, LWLockRelease(XactTruncationLock); ctx->cached_xid = xid; ctx->cached_status = *status; - return result; + return XID_BOUNDS_OK; } From 3dfb1942d9b8748b93094a430289e2f7f3b3ae0d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 22 Oct 2020 17:34:32 -0400 Subject: [PATCH 355/589] Avoid premature de-doubling of quote marks in ECPG strings. If you write the literal 'abc''def' in an EXEC SQL command, that will come out the other end as 'abc'def', triggering a syntax error in the backend. Likewise, "abc""def" is reduced to "abc"def" which is wrong syntax for a quoted identifier. The cause is that the lexer thinks it should emit just one quote mark, whereas what it really should do is keep the string as-is. Add some docs and test cases, too. Although this seems clearly a bug, I fear users wouldn't appreciate changing it in minor releases. Some may well be working around it by applying an extra doubling of affected quotes, as for example sql/dyntest.pgc has been doing. Per investigation of a report from 1250kv, although this isn't exactly what he/she was on about. Discussion: https://postgr.es/m/673825.1603223178@sss.pgh.pa.us --- doc/src/sgml/ecpg.sgml | 17 ++++++++++++++--- src/interfaces/ecpg/preproc/pgc.l | 9 +++------ .../ecpg/test/expected/preproc-strings.c | 2 +- .../ecpg/test/expected/preproc-strings.stderr | 8 ++++---- .../ecpg/test/expected/preproc-strings.stdout | 2 +- src/interfaces/ecpg/test/preproc/strings.pgc | 8 ++++---- src/interfaces/ecpg/test/sql/dyntest.pgc | 2 +- 7 files changed, 28 insertions(+), 20 deletions(-) diff --git a/doc/src/sgml/ecpg.sgml b/doc/src/sgml/ecpg.sgml index 6e3ca788f6ec..419574e9ea68 100644 --- a/doc/src/sgml/ecpg.sgml +++ b/doc/src/sgml/ecpg.sgml @@ -31,7 +31,7 @@ specially marked sections. To build the program, the source code (*.pgc) is first passed through the embedded SQL preprocessor, which converts it to an ordinary C program (*.c), and afterwards it can be processed by a C - compiler. (For details about the compiling and linking see ). + compiler. (For details about the compiling and linking see .) Converted ECPG applications call functions in the libpq library through the embedded SQL library (ecpglib), and communicate with the PostgreSQL server using the normal frontend-backend protocol. @@ -63,11 +63,22 @@ EXEC SQL ...; These statements syntactically take the place of a C statement. Depending on the particular statement, they can appear at the - global level or within a function. Embedded + global level or within a function. + + + + Embedded SQL statements follow the case-sensitivity rules of normal SQL code, and not those of C. Also they allow nested - C-style comments that are part of the SQL standard. The C part of the + C-style comments as per the SQL standard. The C part of the program, however, follows the C standard of not accepting nested comments. + Embedded SQL statements likewise use SQL rules, not + C rules, for parsing quoted strings and identifiers. + (See and + respectively. Note that + ECPG assumes that standard_conforming_strings + is on.) + Of course, the C part of the program follows C quoting rules. diff --git a/src/interfaces/ecpg/preproc/pgc.l b/src/interfaces/ecpg/preproc/pgc.l index 466bbac6a7b0..e98aa6c48628 100644 --- a/src/interfaces/ecpg/preproc/pgc.l +++ b/src/interfaces/ecpg/preproc/pgc.l @@ -623,11 +623,8 @@ cppline {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+ } } -{xqdouble} { addlitchar('\''); } -{xqcquote} { - addlitchar('\\'); - addlitchar('\''); - } +{xqdouble} { addlit(yytext, yyleng); } +{xqcquote} { addlit(yytext, yyleng); } {xqinside} { addlit(yytext, yyleng); } {xeinside} { addlit(yytext, yyleng); @@ -736,7 +733,7 @@ cppline {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+ return UIDENT; } {xddouble} { - addlitchar('"'); + addlit(yytext, yyleng); } {xdinside} { addlit(yytext, yyleng); diff --git a/src/interfaces/ecpg/test/expected/preproc-strings.c b/src/interfaces/ecpg/test/expected/preproc-strings.c index e695007b1336..1e50cd36c382 100644 --- a/src/interfaces/ecpg/test/expected/preproc-strings.c +++ b/src/interfaces/ecpg/test/expected/preproc-strings.c @@ -45,7 +45,7 @@ int main(void) #line 13 "strings.pgc" - { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select 'abcdef' , N'abcdef' as foo , E'abc\\bdef' as \"foo\" , U&'d\\0061t\\0061' as U&\"foo\" , U&'d!+000061t!+000061' UESCAPE '!' , $foo$abc$def$foo$", ECPGt_EOIT, + { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select 'abc''d\\ef' , N'abc''d\\ef' as foo , E'abc''d\\\\ef' as \"foo\"\"bar\" , U&'d\\0061t\\0061' as U&\"foo\"\"bar\" , U&'d!+000061t!+000061' UESCAPE '!' , $foo$abc$def$foo$", ECPGt_EOIT, ECPGt_char,&(s1),(long)0,(long)1,(1)*sizeof(char), ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_char,&(s2),(long)0,(long)1,(1)*sizeof(char), diff --git a/src/interfaces/ecpg/test/expected/preproc-strings.stderr b/src/interfaces/ecpg/test/expected/preproc-strings.stderr index dbc9e5c0b8db..4c3a8eee5aad 100644 --- a/src/interfaces/ecpg/test/expected/preproc-strings.stderr +++ b/src/interfaces/ecpg/test/expected/preproc-strings.stderr @@ -8,7 +8,7 @@ [NO_PID]: sqlca: code: 0, state: 00000 [NO_PID]: ecpg_process_output on line 13: OK: SET [NO_PID]: sqlca: code: 0, state: 00000 -[NO_PID]: ecpg_execute on line 15: query: select 'abcdef' , N'abcdef' as foo , E'abc\bdef' as "foo" , U&'d\0061t\0061' as U&"foo" , U&'d!+000061t!+000061' UESCAPE '!' , $foo$abc$def$foo$; with 0 parameter(s) on connection ecpg1_regression +[NO_PID]: ecpg_execute on line 15: query: select 'abc''d\ef' , N'abc''d\ef' as foo , E'abc''d\\ef' as "foo""bar" , U&'d\0061t\0061' as U&"foo""bar" , U&'d!+000061t!+000061' UESCAPE '!' , $foo$abc$def$foo$; with 0 parameter(s) on connection ecpg1_regression [NO_PID]: sqlca: code: 0, state: 00000 [NO_PID]: ecpg_execute on line 15: using PQexec [NO_PID]: sqlca: code: 0, state: 00000 @@ -16,15 +16,15 @@ [NO_PID]: sqlca: code: 0, state: 00000 [NO_PID]: ecpg_store_result on line 15: allocating memory for 1 tuples [NO_PID]: sqlca: code: 0, state: 00000 -[NO_PID]: ecpg_get_data on line 15: RESULT: abcdef offset: -1; array: no +[NO_PID]: ecpg_get_data on line 15: RESULT: abc'd\ef offset: -1; array: no [NO_PID]: sqlca: code: 0, state: 00000 [NO_PID]: ecpg_store_result on line 15: allocating memory for 1 tuples [NO_PID]: sqlca: code: 0, state: 00000 -[NO_PID]: ecpg_get_data on line 15: RESULT: abcdef offset: -1; array: no +[NO_PID]: ecpg_get_data on line 15: RESULT: abc'd\ef offset: -1; array: no [NO_PID]: sqlca: code: 0, state: 00000 [NO_PID]: ecpg_store_result on line 15: allocating memory for 1 tuples [NO_PID]: sqlca: code: 0, state: 00000 -[NO_PID]: ecpg_get_data on line 15: RESULT: abcdef offset: -1; array: no +[NO_PID]: ecpg_get_data on line 15: RESULT: abc'd\ef offset: -1; array: no [NO_PID]: sqlca: code: 0, state: 00000 [NO_PID]: ecpg_store_result on line 15: allocating memory for 1 tuples [NO_PID]: sqlca: code: 0, state: 00000 diff --git a/src/interfaces/ecpg/test/expected/preproc-strings.stdout b/src/interfaces/ecpg/test/expected/preproc-strings.stdout index 730d72dd64e1..1456b152d78e 100644 --- a/src/interfaces/ecpg/test/expected/preproc-strings.stdout +++ b/src/interfaces/ecpg/test/expected/preproc-strings.stdout @@ -1 +1 @@ -abcdef abcdef abcdef data data abc$def +abc'd\ef abc'd\ef abc'd\ef data data abc$def diff --git a/src/interfaces/ecpg/test/preproc/strings.pgc b/src/interfaces/ecpg/test/preproc/strings.pgc index f004ddf6dc1e..25157f136c2a 100644 --- a/src/interfaces/ecpg/test/preproc/strings.pgc +++ b/src/interfaces/ecpg/test/preproc/strings.pgc @@ -12,10 +12,10 @@ int main(void) exec sql set standard_conforming_strings to on; - exec sql select 'abcdef', - N'abcdef' AS foo, - E'abc\bdef' AS "foo", - U&'d\0061t\0061' AS U&"foo", + exec sql select 'abc''d\ef', + N'abc''d\ef' AS foo, + E'abc''d\\ef' AS "foo""bar", + U&'d\0061t\0061' AS U&"foo""bar", U&'d!+000061t!+000061' uescape '!', $foo$abc$def$foo$ into :s1, :s2, :s3, :s4, :s5, :s6; diff --git a/src/interfaces/ecpg/test/sql/dyntest.pgc b/src/interfaces/ecpg/test/sql/dyntest.pgc index 5f02fd5dd695..0222c8985154 100644 --- a/src/interfaces/ecpg/test/sql/dyntest.pgc +++ b/src/interfaces/ecpg/test/sql/dyntest.pgc @@ -51,7 +51,7 @@ main () exec sql create table dyntest (name char (14), d float8, i int, bignumber int8, b boolean, comment text, day date); - exec sql insert into dyntest values ('first entry', 14.7, 14, 123045607890, true, 'The world''''s most advanced open source database.', '1987-07-14'); + exec sql insert into dyntest values ('first entry', 14.7, 14, 123045607890, true, 'The world''s most advanced open source database.', '1987-07-14'); exec sql insert into dyntest values ('second entry', 1407.87, 1407, 987065403210, false, 'The elephant never forgets.', '1999-11-5'); exec sql prepare MYQUERY from :QUERY; From c16a1bbcf498f0aa053a3e55008f57d7f67357dd Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 22 Oct 2020 18:29:40 -0400 Subject: [PATCH 356/589] Add documentation and tests for quote marks in ECPG literal queries. ECPG's PREPARE ... FROM and EXECUTE IMMEDIATE can optionally take the target query as a simple literal, rather than the more usual string-variable reference. This was previously documented as being a C string literal, but that's a lie in one critical respect: you can't write a data double quote as \" in such literals. That's because the lexer is in SQL mode at this point, so it'll parse double-quoted strings as SQL identifiers, within which backslash is not special, so \" ends the literal. I looked into making this work as documented, but getting the lexer to switch behaviors at just the right point is somewhere between very difficult and impossible. It's not really worth the trouble, because these cases are next to useless: if you have a fixed SQL statement to execute or prepare, you might as well write it as a direct EXEC SQL, saving the messiness of converting it into a string literal and gaining the opportunity for compile-time SQL syntax checking. Instead, let's just document (and test) the workaround of writing a double quote as an octal escape (\042) in such cases. There's no code behavioral change here, so in principle this could be back-patched, but it's such a niche case I doubt it's worth the trouble. Per report from 1250kv. Discussion: https://postgr.es/m/673825.1603223178@sss.pgh.pa.us --- doc/src/sgml/ecpg.sgml | 59 +++++++++++++++++-- src/interfaces/ecpg/preproc/pgc.l | 9 ++- .../ecpg/test/expected/sql-execute.c | 4 +- .../ecpg/test/expected/sql-execute.stderr | 2 +- src/interfaces/ecpg/test/sql/execute.pgc | 4 +- 5 files changed, 67 insertions(+), 11 deletions(-) diff --git a/doc/src/sgml/ecpg.sgml b/doc/src/sgml/ecpg.sgml index 419574e9ea68..14dcbdb4e32a 100644 --- a/doc/src/sgml/ecpg.sgml +++ b/doc/src/sgml/ecpg.sgml @@ -7066,7 +7066,7 @@ EXECUTE IMMEDIATE string string - A literal C string or a host variable containing the SQL + A literal string or a host variable containing the SQL statement to be executed. @@ -7074,6 +7074,30 @@ EXECUTE IMMEDIATE string + + Notes + + + In typical usage, the string is a host + variable reference to a string containing a dynamically-constructed + SQL statement. The case of a literal string is not very useful; + you might as well just write the SQL statement directly, without + the extra typing of EXECUTE IMMEDIATE. + + + + If you do use a literal string, keep in mind that any double quotes + you might wish to include in the SQL statement must be written as + octal escapes (\042) not the usual C + idiom \". This is because the string is inside + an EXEC SQL section, so the ECPG lexer parses it + according to SQL rules not C rules. Any embedded backslashes will + later be handled according to C rules; but \" + causes an immediate syntax error because it is seen as ending the + literal. + + + Examples @@ -7388,7 +7412,7 @@ EXEC SQL OPEN :curname1; -PREPARE name FROM string +PREPARE prepared_name FROM string @@ -7421,15 +7445,40 @@ PREPARE name FROM string - A literal C string or a host variable containing a preparable - statement, one of the SELECT, INSERT, UPDATE, or - DELETE. + A literal string or a host variable containing a preparable + SQL statement, one of SELECT, INSERT, UPDATE, or DELETE. + Use question marks (?) for parameter values + to be supplied at execution. + + Notes + + + In typical usage, the string is a host + variable reference to a string containing a dynamically-constructed + SQL statement. The case of a literal string is not very useful; + you might as well just write a direct SQL PREPARE + statement. + + + + If you do use a literal string, keep in mind that any double quotes + you might wish to include in the SQL statement must be written as + octal escapes (\042) not the usual C + idiom \". This is because the string is inside + an EXEC SQL section, so the ECPG lexer parses it + according to SQL rules not C rules. Any embedded backslashes will + later be handled according to C rules; but \" + causes an immediate syntax error because it is seen as ending the + literal. + + + Examples diff --git a/src/interfaces/ecpg/preproc/pgc.l b/src/interfaces/ecpg/preproc/pgc.l index e98aa6c48628..91d8b635787b 100644 --- a/src/interfaces/ecpg/preproc/pgc.l +++ b/src/interfaces/ecpg/preproc/pgc.l @@ -715,7 +715,14 @@ cppline {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+ BEGIN(state_before_str_start); if (literallen == 0) mmerror(PARSE_ERROR, ET_ERROR, "zero-length delimited identifier"); - /* The backend will truncate the identifier here. We do not as it does not change the result. */ + /* + * The server will truncate the identifier here. We do + * not, as (1) it does not change the result; (2) we don't + * know what NAMEDATALEN the server might use; (3) this + * code path is also taken for literal query strings in + * PREPARE and EXECUTE IMMEDIATE, which can certainly be + * longer than NAMEDATALEN. + */ base_yylval.str = mm_strdup(literalbuf); return CSTRING; } diff --git a/src/interfaces/ecpg/test/expected/sql-execute.c b/src/interfaces/ecpg/test/expected/sql-execute.c index cac91dc5999d..10e9ad56b51b 100644 --- a/src/interfaces/ecpg/test/expected/sql-execute.c +++ b/src/interfaces/ecpg/test/expected/sql-execute.c @@ -77,8 +77,8 @@ if (sqlca.sqlcode < 0) sqlprint();} #line 26 "execute.pgc" - sprintf(command, "insert into test (name, amount, letter) values ('db: ''r1''', 1, 'f')"); - { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_exec_immediate, command, ECPGt_EOIT, ECPGt_EORT); + /* test handling of embedded quotes in EXECUTE IMMEDIATE "literal" */ + { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_exec_immediate, "insert into test (name, \042amount\042, letter) values ('db: ''r1''', 1, 'f')", ECPGt_EOIT, ECPGt_EORT); #line 29 "execute.pgc" if (sqlca.sqlcode < 0) sqlprint();} diff --git a/src/interfaces/ecpg/test/expected/sql-execute.stderr b/src/interfaces/ecpg/test/expected/sql-execute.stderr index 96b46bd15847..d8bc3c6524ab 100644 --- a/src/interfaces/ecpg/test/expected/sql-execute.stderr +++ b/src/interfaces/ecpg/test/expected/sql-execute.stderr @@ -10,7 +10,7 @@ [NO_PID]: sqlca: code: 0, state: 00000 [NO_PID]: ECPGtrans on line 26: action "commit"; connection "main" [NO_PID]: sqlca: code: 0, state: 00000 -[NO_PID]: ecpg_execute on line 29: query: insert into test (name, amount, letter) values ('db: ''r1''', 1, 'f'); with 0 parameter(s) on connection main +[NO_PID]: ecpg_execute on line 29: query: insert into test (name, "amount", letter) values ('db: ''r1''', 1, 'f'); with 0 parameter(s) on connection main [NO_PID]: sqlca: code: 0, state: 00000 [NO_PID]: ecpg_execute on line 29: using PQexec [NO_PID]: sqlca: code: 0, state: 00000 diff --git a/src/interfaces/ecpg/test/sql/execute.pgc b/src/interfaces/ecpg/test/sql/execute.pgc index cc9814e9bea7..43171bb77c99 100644 --- a/src/interfaces/ecpg/test/sql/execute.pgc +++ b/src/interfaces/ecpg/test/sql/execute.pgc @@ -25,8 +25,8 @@ exec sql end declare section; exec sql create table test (name char(8), amount int, letter char(1)); exec sql commit; - sprintf(command, "insert into test (name, amount, letter) values ('db: ''r1''', 1, 'f')"); - exec sql execute immediate :command; + /* test handling of embedded quotes in EXECUTE IMMEDIATE "literal" */ + exec sql execute immediate "insert into test (name, \042amount\042, letter) values ('db: ''r1''', 1, 'f')"; sprintf(command, "insert into test (name, amount, letter) values ('db: ''r1''', 2, 't')"); exec sql execute immediate :command; From c5054da0d7ee59a93ca6d7f7fc669ef0210dbad3 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 22 Oct 2020 21:15:22 -0400 Subject: [PATCH 357/589] Sync our copy of the timezone library with IANA release tzcode2020d. There's no functional change at all here, but I'm curious to see whether this change successfully shuts up Coverity's warning about a useless strcmp(), which appeared with the previous update. Discussion: http://mm.icann.org/pipermail/tz/2020-October/029370.html --- src/timezone/README | 2 +- src/timezone/zic.c | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/timezone/README b/src/timezone/README index 10aeedce92a0..f588d1f5add2 100644 --- a/src/timezone/README +++ b/src/timezone/README @@ -55,7 +55,7 @@ match properly on the old version. Time Zone code ============== -The code in this directory is currently synced with tzcode release 2020c. +The code in this directory is currently synced with tzcode release 2020d. There are many cosmetic (and not so cosmetic) differences from the original tzcode library, but diffs in the upstream version should usually be propagated to our version. Here are some notes about that. diff --git a/src/timezone/zic.c b/src/timezone/zic.c index a1c64051f788..0ea6ead2db3a 100644 --- a/src/timezone/zic.c +++ b/src/timezone/zic.c @@ -634,7 +634,8 @@ static const char *leapsec; static const char *tzdefault; /* -1 if the TZif output file should be slim, 0 if default, 1 if the - output should be fat for backward compatibility. The default is slim. */ + output should be fat for backward compatibility. ZIC_BLOAT_DEFAULT + determines the default. */ static int bloat; static bool @@ -787,7 +788,16 @@ main(int argc, char **argv) if (optind == argc - 1 && strcmp(argv[optind], "=") == 0) usage(stderr, EXIT_FAILURE); /* usage message by request */ if (bloat == 0) - bloat = strcmp(ZIC_BLOAT_DEFAULT, "slim") == 0 ? -1 : 1; + { + static char const bloat_default[] = ZIC_BLOAT_DEFAULT; + + if (strcmp(bloat_default, "slim") == 0) + bloat = -1; + else if (strcmp(bloat_default, "fat") == 0) + bloat = 1; + else + abort(); /* Configuration error. */ + } if (directory == NULL) directory = "data"; if (tzdefault == NULL) From 7d6d6bce43c60bb7b77237e2cc6ab845646b911f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 22 Oct 2020 21:23:47 -0400 Subject: [PATCH 358/589] Update time zone data files to tzdata release 2020d. DST law changes in Palestine, with a whopping 120 hours' notice. Also some historical corrections for Palestine. --- src/timezone/data/tzdata.zi | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/timezone/data/tzdata.zi b/src/timezone/data/tzdata.zi index df175e61cd8d..09afb428717f 100644 --- a/src/timezone/data/tzdata.zi +++ b/src/timezone/data/tzdata.zi @@ -1,4 +1,4 @@ -# version 2020c +# version 2020d # This zic input file is in the public domain. R d 1916 o - Jun 14 23s 1 S R d 1916 1919 - O Su>=1 23s 0 - @@ -1096,10 +1096,10 @@ R P 2004 o - O 1 1 0 - R P 2005 o - O 4 2 0 - R P 2006 2007 - Ap 1 0 1 S R P 2006 o - S 22 0 0 - -R P 2007 o - S Th>=8 2 0 - +R P 2007 o - S 13 2 0 - R P 2008 2009 - Mar lastF 0 1 S R P 2008 o - S 1 0 0 - -R P 2009 o - S F>=1 1 0 - +R P 2009 o - S 4 1 0 - R P 2010 o - Mar 26 0 1 S R P 2010 o - Au 11 0 0 - R P 2011 o - Ap 1 0:1 1 S @@ -1108,12 +1108,16 @@ R P 2011 o - Au 30 0 1 S R P 2011 o - S 30 0 0 - R P 2012 2014 - Mar lastTh 24 1 S R P 2012 o - S 21 1 0 - -R P 2013 o - S F>=21 0 0 - -R P 2014 2015 - O F>=21 0 0 - -R P 2015 o - Mar lastF 24 1 S +R P 2013 o - S 27 0 0 - +R P 2014 o - O 24 0 0 - +R P 2015 o - Mar 28 0 1 S +R P 2015 o - O 23 1 0 - R P 2016 2018 - Mar Sa>=24 1 1 S -R P 2016 ma - O lastSa 1 0 - -R P 2019 ma - Mar lastF 0 1 S +R P 2016 2018 - O Sa>=24 1 0 - +R P 2019 o - Mar 29 0 1 S +R P 2019 o - O Sa>=24 0 0 - +R P 2020 ma - Mar Sa>=24 0 1 S +R P 2020 ma - O Sa>=24 1 0 - Z Asia/Gaza 2:17:52 - LMT 1900 O 2 Z EET/EEST 1948 May 15 2 K EE%sT 1967 Jun 5 From 783f0cc64dcc05e3d112a06b1cd181e5a1ca9099 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Fri, 23 Oct 2020 11:05:46 +0900 Subject: [PATCH 359/589] Improve performance of Unicode {de,re}composition in the backend This replaces the existing binary search with two perfect hash functions for the composition and the decomposition in the backend code, at the cost of slightly-larger binaries there (35kB in libpgcommon_srv.a). Per the measurements done, this improves the speed of the recomposition and decomposition by up to 30~40 times for the NFC and NFKC conversions, while all other operations get at least 40% faster. This is not as "good" as what libicu has, but it closes the gap a lot as per the feedback from Daniel Verite. The decomposition table remains the same, getting used for the binary search in the frontend code, where we care more about the size of the libraries like libpq over performance as this gets involved only in code paths related to the SCRAM authentication. In consequence, note that the perfect hash function for the recomposition needs to use a new inverse lookup array back to to the existing decomposition table. The size of all frontend deliverables remains unchanged, even with --enable-debug, including libpq. Author: John Naylor Reviewed-by: Michael Paquier, Tom Lane Discussion: https://postgr.es/m/CAFBsxsHUuMFCt6-pU+oG-F1==CmEp8wR+O+bRouXWu6i8kXuqA@mail.gmail.com --- src/common/unicode/Makefile | 4 +- .../unicode/generate-unicode_norm_table.pl | 226 +- src/common/unicode_norm.c | 106 +- src/include/common/unicode_norm_hashfunc.h | 2932 +++++++++++++++++ src/tools/pgindent/exclude_file_patterns | 3 +- 5 files changed, 3227 insertions(+), 44 deletions(-) create mode 100644 src/include/common/unicode_norm_hashfunc.h diff --git a/src/common/unicode/Makefile b/src/common/unicode/Makefile index 93a9d1615f1c..eb14add28ad6 100644 --- a/src/common/unicode/Makefile +++ b/src/common/unicode/Makefile @@ -18,7 +18,7 @@ LIBS += $(PTHREAD_LIBS) # By default, do nothing. all: -update-unicode: unicode_norm_table.h unicode_combining_table.h unicode_normprops_table.h +update-unicode: unicode_norm_table.h unicode_combining_table.h unicode_normprops_table.h unicode_norm_hashfunc.h mv $^ ../../../src/include/common/ $(MAKE) normalization-check @@ -30,6 +30,8 @@ UnicodeData.txt DerivedNormalizationProps.txt CompositionExclusions.txt Normaliz # Generation of conversion tables used for string normalization with # UTF-8 strings. +unicode_norm_hashfunc.h: unicode_norm_table.h + unicode_norm_table.h: generate-unicode_norm_table.pl UnicodeData.txt CompositionExclusions.txt $(PERL) generate-unicode_norm_table.pl diff --git a/src/common/unicode/generate-unicode_norm_table.pl b/src/common/unicode/generate-unicode_norm_table.pl index 7ce15e1a0395..e4d3ccc2346a 100644 --- a/src/common/unicode/generate-unicode_norm_table.pl +++ b/src/common/unicode/generate-unicode_norm_table.pl @@ -1,16 +1,22 @@ #!/usr/bin/perl # -# Generate a composition table, using Unicode data files as input +# Generate a composition table and its lookup utilities, using Unicode data +# files as input. # # Input: UnicodeData.txt and CompositionExclusions.txt -# Output: unicode_norm_table.h +# Output: unicode_norm_table.h and unicode_norm_hashfunc.h # # Copyright (c) 2000-2020, PostgreSQL Global Development Group use strict; use warnings; -my $output_file = "unicode_norm_table.h"; +use FindBin; +use lib "$FindBin::RealBin/../../tools/"; +use PerfectHash; + +my $output_table_file = "unicode_norm_table.h"; +my $output_func_file = "unicode_norm_hashfunc.h"; my $FH; @@ -64,11 +70,13 @@ my $num_characters = scalar @characters; -# Start writing out the output file -open my $OUTPUT, '>', $output_file - or die "Could not open output file $output_file: $!\n"; +# Start writing out the output files +open my $OT, '>', $output_table_file + or die "Could not open output file $output_table_file: $!\n"; +open my $OF, '>', $output_func_file + or die "Could not open output file $output_func_file: $!\n"; -print $OUTPUT <{code}; foreach my $char (@characters) @@ -121,6 +174,9 @@ my $class = $char->{class}; my $decomp = $char->{decomp}; + # Save the code point bytes as a string in network order. + push @dec_cp_packed, pack('N', hex($char->{code})); + # The character decomposition mapping field in UnicodeData.txt is a list # of unicode codepoints, separated by space. But it can be prefixed with # so-called compatibility formatting tag, like "", or "". @@ -163,7 +219,7 @@ { foreach my $lcode (@composition_exclusion_codes) { - if ($lcode eq $char->{code}) + if ($lcode eq $code) { $flags .= " | DECOMP_NO_COMPOSE"; $comment = "in exclusion list"; @@ -171,11 +227,26 @@ } } } + + # Save info for recomposeable codepoints. + # Note that this MUST match the macro DECOMPOSITION_NO_COMPOSE in C + # above! See also the inverse lookup in recompose_code() found in + # src/common/unicode_norm.c. + if (!($flags =~ /DECOMP_COMPAT/ || $flags =~ /DECOMP_NO_COMPOSE/)) + { + push @rec_info, + { + code => $code, + main_index => $main_index, + first => $first_decomp, + second => $decomp_elts[0] + }; + } } if ($decomp_size == 0) { - print $OUTPUT "\t{0x$code, $class, 0$flags, 0}"; + print $OT "\t{0x$code, $class, 0$flags, 0}"; } elsif ($decomp_size == 1 && length($first_decomp) <= 4) { @@ -183,12 +254,11 @@ # The decomposition consists of a single codepoint, and it fits # in a uint16, so we can store it "inline" in the main table. $flags .= " | DECOMP_INLINE"; - print $OUTPUT "\t{0x$code, $class, 1$flags, 0x$first_decomp}"; + print $OT "\t{0x$code, $class, 1$flags, 0x$first_decomp}"; } else { - print $OUTPUT - "\t{0x$code, $class, $decomp_size$flags, $decomp_index}"; + print $OT "\t{0x$code, $class, $decomp_size$flags, $decomp_index}"; # Now save the decompositions into a dedicated area that will # be written afterwards. First build the entry dedicated to @@ -205,25 +275,17 @@ } # Print a comma after all items except the last one. - print $OUTPUT "," unless ($code eq $last_code); - if ($comment ne "") - { + print $OT "," unless ($code eq $last_code); - # If the line is wide already, indent the comment with one tab, - # otherwise with two. This is to make the output match the way - # pgindent would mangle it. (This is quite hacky. To do this - # properly, we should actually track how long the line is so far, - # but this works for now.) - print $OUTPUT "\t" if ($decomp_index < 10); + print $OT "\t/* $comment */" if ($comment ne ""); + print $OT "\n"; - print $OUTPUT "\t/* $comment */" if ($comment ne ""); - } - print $OUTPUT "\n"; + $main_index++; } -print $OUTPUT "\n};\n\n"; +print $OT "\n};\n\n"; # Print the array of decomposed codes. -print $OUTPUT < 4); +print $OF "/* Perfect hash function for decomposition */\n"; +print $OF "static $dec_func\n"; + +# Emit the structure that wraps the hash lookup information into +# one variable. +print $OF <{first}) << 32) | hex($rec->{second}); + + # We are only interested in the lowest code point that decomposes + # to the given code pair. + next if $seenit{$hashkey}; + + # Save the hash key bytes in network order + push @rec_cp_packed, pack('Q>', $hashkey); + + # Append inverse lookup element + $recomp_string .= ",\n" if !$firstentry; + $recomp_string .= sprintf "\t/* U+%s+%s -> U+%s */ %s", + $rec->{first}, + $rec->{second}, + $rec->{code}, + $rec->{main_index}; + + $seenit{$hashkey} = 1; + $firstentry = 0; +} + +# Emit the inverse lookup array containing indexes into UnicodeDecompMain. +my $num_recomps = scalar @rec_cp_packed; +print $OF < 8); +print $OF "/* Perfect hash function for recomposition */\n"; +print $OF "static $rec_func\n"; + +# Emit the structure that wraps the hash lookup information into +# one variable. +print $OF <{first}); + my $b1 = hex($b->{first}); + + my $a2 = hex($a->{second}); + my $b2 = hex($b->{second}); + + # First sort by the first code point + return -1 if $a1 < $b1; + return 1 if $a1 > $b1; + + # Then sort by the second code point + return -1 if $a2 < $b2; + return 1 if $a2 > $b2; + + # Finally sort by the code point that decomposes into first and + # second ones. + my $acode = hex($a->{code}); + my $bcode = hex($b->{code}); + + return -1 if $acode < $bcode; + return -1 if $acode > $bcode; + + die "found duplicate entries of recomposeable code pairs"; +} diff --git a/src/common/unicode_norm.c b/src/common/unicode_norm.c index 4bb6a0f58738..4ffce0e61930 100644 --- a/src/common/unicode_norm.c +++ b/src/common/unicode_norm.c @@ -19,9 +19,11 @@ #endif #include "common/unicode_norm.h" -#include "common/unicode_norm_table.h" #ifndef FRONTEND +#include "common/unicode_norm_hashfunc.h" #include "common/unicode_normprops_table.h" +#else +#include "common/unicode_norm_table.h" #endif #include "port/pg_bswap.h" @@ -44,6 +46,46 @@ #define NCOUNT VCOUNT * TCOUNT #define SCOUNT LCOUNT * NCOUNT +/* + * get_code_entry + * + * Get the entry corresponding to code in the decomposition lookup table. + * The backend version of this code uses a perfect hash function for the + * lookup, while the frontend version uses a binary search. + */ +#ifndef FRONTEND + +static const pg_unicode_decomposition * +get_code_entry(pg_wchar code) +{ + int h; + uint32 hashkey; + pg_unicode_decompinfo decompinfo = UnicodeDecompInfo; + + /* + * Compute the hash function. The hash key is the codepoint with the bytes + * in network order. + */ + hashkey = pg_hton32(code); + h = decompinfo.hash(&hashkey); + + /* An out-of-range result implies no match */ + if (h < 0 || h >= decompinfo.num_decomps) + return NULL; + + /* + * Since it's a perfect hash, we need only match to the specific codepoint + * it identifies. + */ + if (code != decompinfo.decomps[h].codepoint) + return NULL; + + /* Success! */ + return &decompinfo.decomps[h]; +} + +#else + /* comparison routine for bsearch() of decomposition lookup table. */ static int conv_compare(const void *p1, const void *p2) @@ -56,10 +98,7 @@ conv_compare(const void *p1, const void *p2) return (v1 > v2) ? 1 : ((v1 == v2) ? 0 : -1); } -/* - * Get the entry corresponding to code in the decomposition lookup table. - */ -static pg_unicode_decomposition * +static const pg_unicode_decomposition * get_code_entry(pg_wchar code) { return bsearch(&(code), @@ -69,6 +108,8 @@ get_code_entry(pg_wchar code) conv_compare); } +#endif /* !FRONTEND */ + /* * Given a decomposition entry looked up earlier, get the decomposed * characters. @@ -77,7 +118,7 @@ get_code_entry(pg_wchar code) * is only valid until next call to this function! */ static const pg_wchar * -get_code_decomposition(pg_unicode_decomposition *entry, int *dec_size) +get_code_decomposition(const pg_unicode_decomposition *entry, int *dec_size) { static pg_wchar x; @@ -104,7 +145,7 @@ get_code_decomposition(pg_unicode_decomposition *entry, int *dec_size) static int get_decomposed_size(pg_wchar code, bool compat) { - pg_unicode_decomposition *entry; + const pg_unicode_decomposition *entry; int size = 0; int i; const uint32 *decomp; @@ -191,17 +232,51 @@ recompose_code(uint32 start, uint32 code, uint32 *result) } else { - int i; + const pg_unicode_decomposition *entry; /* * Do an inverse lookup of the decomposition tables to see if anything * matches. The comparison just needs to be a perfect match on the * sub-table of size two, because the start character has already been - * recomposed partially. + * recomposed partially. This lookup uses a perfect hash function for + * the backend code. */ +#ifndef FRONTEND + + int h, + inv_lookup_index; + uint64 hashkey; + pg_unicode_recompinfo recompinfo = UnicodeRecompInfo; + + /* + * Compute the hash function. The hash key is formed by concatenating + * bytes of the two codepoints in network order. See also + * src/common/unicode/generate-unicode_norm_table.pl. + */ + hashkey = pg_hton64(((uint64) start << 32) | (uint64) code); + h = recompinfo.hash(&hashkey); + + /* An out-of-range result implies no match */ + if (h < 0 || h >= recompinfo.num_recomps) + return false; + + inv_lookup_index = recompinfo.inverse_lookup[h]; + entry = &UnicodeDecompMain[inv_lookup_index]; + + if (start == UnicodeDecomp_codepoints[entry->dec_index] && + code == UnicodeDecomp_codepoints[entry->dec_index + 1]) + { + *result = entry->codepoint; + return true; + } + +#else + + int i; + for (i = 0; i < lengthof(UnicodeDecompMain); i++) { - const pg_unicode_decomposition *entry = &UnicodeDecompMain[i]; + entry = &UnicodeDecompMain[i]; if (DECOMPOSITION_SIZE(entry) != 2) continue; @@ -216,6 +291,7 @@ recompose_code(uint32 start, uint32 code, uint32 *result) return true; } } +#endif /* !FRONTEND */ } return false; @@ -231,7 +307,7 @@ recompose_code(uint32 start, uint32 code, uint32 *result) static void decompose_code(pg_wchar code, bool compat, pg_wchar **result, int *current) { - pg_unicode_decomposition *entry; + const pg_unicode_decomposition *entry; int i; const uint32 *decomp; int dec_size; @@ -358,8 +434,8 @@ unicode_normalize(UnicodeNormalizationForm form, const pg_wchar *input) pg_wchar prev = decomp_chars[count - 1]; pg_wchar next = decomp_chars[count]; pg_wchar tmp; - pg_unicode_decomposition *prevEntry = get_code_entry(prev); - pg_unicode_decomposition *nextEntry = get_code_entry(next); + const pg_unicode_decomposition *prevEntry = get_code_entry(prev); + const pg_unicode_decomposition *nextEntry = get_code_entry(next); /* * If no entries are found, the character used is either an Hangul @@ -417,7 +493,7 @@ unicode_normalize(UnicodeNormalizationForm form, const pg_wchar *input) for (count = 1; count < decomp_size; count++) { pg_wchar ch = decomp_chars[count]; - pg_unicode_decomposition *ch_entry = get_code_entry(ch); + const pg_unicode_decomposition *ch_entry = get_code_entry(ch); int ch_class = (ch_entry == NULL) ? 0 : ch_entry->comb_class; pg_wchar composite; @@ -458,7 +534,7 @@ unicode_normalize(UnicodeNormalizationForm form, const pg_wchar *input) static uint8 get_canonical_class(pg_wchar ch) { - pg_unicode_decomposition *entry = get_code_entry(ch); + const pg_unicode_decomposition *entry = get_code_entry(ch); if (!entry) return 0; diff --git a/src/include/common/unicode_norm_hashfunc.h b/src/include/common/unicode_norm_hashfunc.h new file mode 100644 index 000000000000..e6acb2a8d0f7 --- /dev/null +++ b/src/include/common/unicode_norm_hashfunc.h @@ -0,0 +1,2932 @@ +/*------------------------------------------------------------------------- + * + * unicode_norm_hashfunc.h + * Perfect hash functions used for Unicode normalization + * + * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/common/unicode_norm_hashfunc.h + * + *------------------------------------------------------------------------- + */ + +/* + * File auto-generated by src/common/unicode/generate-unicode_norm_table.pl, + * do not edit. There is deliberately not an #ifndef PG_UNICODE_NORM_HASHFUNC_H + * here. + */ + +#include "common/unicode_norm_table.h" + +/* Typedef for perfect hash functions */ +typedef int (*cp_hash_func) (const void *key); + +/* Information for lookups with perfect hash functions */ +typedef struct +{ + const pg_unicode_decomposition *decomps; + cp_hash_func hash; + int num_decomps; +} pg_unicode_decompinfo; + +typedef struct +{ + const uint16 *inverse_lookup; + cp_hash_func hash; + int num_recomps; +} pg_unicode_recompinfo; + +/* Perfect hash function for decomposition */ +static int +Decomp_hash_func(const void *key) +{ + static const int16 h[13209] = { + 0, 1515, 4744, 4745, 0, 0, 0, 0, + 0, 0, 0, 0, 3890, 3890, 0, 0, + 3891, 3891, -2046, 2800, 3890, 3890, 3890, -4396, + 4361, 4362, -4441, -4441, -4396, 1773, 1773, 1773, + 4372, 4373, -4438, -4438, -4393, -4393, 2619, 17, + -4347, -4393, -4393, -4393, -4393, -4393, 2619, 2619, + 1560, 4346, 4347, 4348, 1917, 1873, 1874, 1875, + -7856, 4358, 17619, 2622, 2622, 2622, 6357, 6358, + 6359, 6360, 6361, 6362, 6363, 2622, -4390, -4390, + 4414, -5356, -5356, 4374, 4375, -5356, -5356, -6335, + -3020, 2511, -5356, -5356, -3583, -3583, -3583, -3583, + -995, 0, 0, -9799, -9754, 2874, 2875, 2876, + 2877, 2878, -9830, -3591, -9756, -9756, -2744, -5346, + -9710, -9756, 342, -5346, -9756, -5346, -2743, -449, + 348, 2894, 2895, -2853, 2897, 2898, 2899, 2900, + 2901, 2902, 2903, 2904, 2905, 2906, 2907, 2908, + 2909, 2910, 2911, 2912, 2913, 2914, 2915, 2916, + 2917, 2918, 2919, 2920, 2921, 2922, 2923, 2924, + 2925, 2926, 2927, 2928, 2929, 2930, 2931, 2932, + 2933, 2934, 32767, 32767, 32767, 32767, 32767, 32767, + -8721, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 1, 32767, 48, 32767, 32767, 32767, 32767, 49, + 32767, 32767, -8687, -8687, -6255, -6210, 32767, 32767, + -8689, -8689, -21949,32767, -18635,-15320,-15320,32767, + -12006,-8691, -8691, -8691, -8691, -8691, 32767, 66, + -8737, -8737, -8692, -8692, -8692, -8692, 73, 74, + 32767, -8738, -8693, -8693, -8693, -8693, -8693, 32767, + 32767, -8695, -8695, -8695, -8695, -8695, 32767, 32767, + 40, 41, -2390, -2434, 44, 45, 32767, 46, + 13307, 9993, 9994, 6680, 6681, 3367, 3368, 54, + 0, 55, 56, 57, -8699, -8699, 105, 32767, + 32767, 61, 62, 63, -8701, -8701, 32767, 111, + 32767, 67, 68, 69, 70, 1890, 3687, -1272, + 3690, 75, 76, 77, 78, 79, 80, 81, + 82, 32767, 32767, 83, 84, 85, 86, 87, + 88, 89, 90, 91, 92, 93, 94, 95, + 96, 97, 98, 99, 100, 101, 102, 32767, + 32767, 103, 104, 105, 106, 107, 108, 109, + -8660, -8660, 32767, -8661, -8661, -8661, -8661, -8661, + -8661, 32767, 73, 74, 75, 76, -2355, -2399, + 79, 80, 32767, 32767, 13341, 10027, 10028, 6714, + 6715, 3401, 3402, 32767, 32767, 88, 89, 90, + -8666, -8666, 138, 32767, 32767, 94, 95, 96, + -8668, -8668, 144, 145, 101, -2553, -2553, -2553, + -2553, -4983, -2553, -2553, 154, -2553, 156, 32767, + 32767, 6114, 158, -3153, -3152, -3151, -12891,-6888, + -931, -3149, 166, -3148, -4728, 169, -3147, -3146, + -3145, -3144, -3143, -3142, -3141, -2543, -3139, -3138, + 180, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 3314, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 0, 3660, 3661, 2131, 2132, 2133, 2134, 2135, + 2136, 2137, 2138, 2139, 2140, 2141, 2142, 2143, + 2144, 2145, -5472, -5472, -3612, -3612, -3612, -3612, + -3612, 2652, -3612, -3612, -3612, -3612, -3612, -3612, + -3612, -3612, 3693, -3613, -7015, -7015, 1742, 1743, + -7060, -7060, -7015, -846, -846, -846, 1753, 1754, + -7057, -7057, -7012, -7012, 0, -2602, -6966, -7012, + -7012, -7012, -7012, -7012, 0, 0, 1725, 1726, + 1727, 1728, -703, -747, -746, 0, 1735, 1736, + 14997, 0, 0, 0, 3735, 3736, 3737, 3738, + 3739, 3740, 3741, 0, -7012, -7012, 1792, 1793, + 1749, 1750, 1751, -7980, -7980, -8959, -5644, -113, + -7980, -113, -2382, -6116, -6116, -6116, -6116, -6116, + -6116, -6116, -2374, 4639, 4640, -4163, 5608, 5609, + -4120, -4120, 5612, 5613, 6593, 3279, -2251, 5617, + 5618, 3846, 3847, 3848, 3849, 1262, 1262, 10066, + 10067, 10023, 3855, 3856, 3857, 1259, 1259, 10071, + 3861, 10027, 10028, 3017, 5620, 9985, 10032, -65, + 5624, 10035, 5626, 3024, 731, -65, 1298, 12530, + 3727, 3727, 3772, 3772, 3772, 13504, 13505, 14485, + 11171, 5641, 13509, 5643, 7913, 11648, 11649, 11650, + 11651, 11652, 11653, 11654, 7913, 901, 901, 9705, + -65, -65, 9665, 9666, -65, -65, -1044, 2271, + 7802, -65, -65, 1708, 1708, 1708, 1708, 4296, + 4297, -4506, -4506, -4461, 1708, 1708, 1708, 4307, + 4308, -4503, 1708, -4457, -4457, 2555, -47, -4411, + -4457, 5641, -47, -4457, -47, 2556, 4850, 5647, + 4285, -6946, 1858, 1859, 1815, 1816, 1817, -7914, + -7914, -8893, -5578, -47, -7914, -47, -2316, -6050, + -6050, -6050, -6050, -6050, -6050, -6050, -2308, 4705, + 4706, -4097, 5674, 5675, -4054, -4054, 5678, 5679, + 6659, 3345, -2185, 5683, 5684, 3912, 3913, 3914, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, -3083, -3083, 232, 287, 233, 233, + 233, 8990, 8991, 32767, 32767, 3668, 32767, 3667, + 3667, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 208, 208, 208, 208, 208, 208, + 32767, 32767, 206, 206, 206, 206, 206, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 304, 305, -1274, 307, 308, + 309, 6753, -1374, 10488, 4486, -1470, 4488, 316, + 4489, -5607, 4490, 4491, 4492, 322, 760, 324, + 325, 326, 166, 763, 329, -2553, 765, 332, + 333, 334, 335, 772, 337, 6310, 339, 340, + 341, 342, 343, 344, 345, 346, -2542, -2542, + -2542, 350, 351, 352, 353, 354, 355, 356, + 357, 358, 359, 360, 361, 362, -6008, 364, + 365, 366, 367, 368, 369, 370, 254, 372, + 373, 374, 375, 376, 377, 378, 379, 380, + 381, 382, 32767, 383, 384, -3606, -3605, -3604, + -3603, 389, -3600, -3599, -3598, 2340, -1238, -3595, + -3594, -3593, 4694, -4062, -4062, 4742, 4743, 4699, + -1469, -1468, -1467, -4065, -4065, 4747, -1463, 4703, + 4704, -2307, 296, 32767, 0, 32767, 32767, 4708, + -1376, -1376, -1376, 32767, 32767, -1246, 506, 506, + 0, -1559, 32767, 32767, 32767, 32767, 32767, 305, + 419, 308, 2578, 6313, 6314, 424, 32767, -6030, + 32767, 426, 427, 428, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 0, 32767, 0, + 32767, 0, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 0, 32767, 429, -5407, 431, + -5406, 433, -3601, 435, 32767, -3751, 32767, 32767, + 32767, 32767, -3755, 32767, 32767, 32767, 32767, 0, + 32767, 32767, 32767, 32767, 0, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 436, -11425,-5422, + 535, -5422, 535, -5422, 4675, -5421, -5421, -5421, + -5421, -5421, 4681, 0, 0, 0, 4682, 4683, + 4684, 4685, 4686, 4687, 0, 0, 32767, 32767, + 0, 0, -5684, 0, 4688, 4689, 4690, 4691, + 4692, 4693, 4694, 4695, -1257, -1257, 4696, -5441, + -5441, 4699, 4700, 4701, -5443, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 454, 0, 32767, 456, + 32767, 32767, 0, 457, 32767, 32767, 32767, 0, + 458, 459, 460, 32767, 0, 32767, 32767, 32767, + 32767, 32767, 32767, 4703, 4704, 4705, 4706, 32767, + 32767, 0, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 4655, 4656, 4657, 4658, + 4659, 4712, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 462, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 463, 464, 32767, 465, + 32767, 32767, 32767, 466, 32767, 32767, 32767, 32767, + 467, 468, 469, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 3011, 3011, 3011, + 3011, 3011, 3011, 3011, 32767, 32767, 32767, 32767, + 32767, 32767, 470, 471, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 472, + 473, 474, 475, 476, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 4713, 4714, 4715, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 477, 478, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 479, 480, 481, 482, + 32767, 32767, 483, 484, 32767, 32767, 485, 486, + 487, 488, 489, 490, 32767, 32767, 491, 492, + 493, 494, 495, 496, 32767, 32767, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 665, -255, 667, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 693, 694, 695, 696, + 697, 698, 699, 700, 701, 702, 703, 704, + 705, 706, 707, 708, 709, 710, 711, 712, + 7183, 714, -1580, 716, 2547, 718, 7194, 720, + 2553, 722, 723, 7204, 725, 726, 727, 728, + 729, 730, 731, 732, 733, 734, 735, 736, + 0, 0, 8114, 8159, 745, -1535, 747, 748, + 8161, -5019, -5019, -5019, -5019, 1938, 0, 0, + 0, 0, 0, 0, 767, 768, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 32767, 32767, 32767, 32767, 32767, 0, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, -2875, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, -2884, -2884, + -2884, -2884, -2884, -2884, -2884, -2884, -2884, -2884, + -2884, -2884, -4271, -2884, -2884, -2884, -2884, -2884, + -2884, -2884, -2884, -2884, -2884, -2884, -2884, -2884, + -2884, -2884, -2884, -2884, -2884, -2884, -2884, -2884, + -2884, -2884, -2884, -2884, -2884, -2884, -2884, -2884, + -2884, -2884, -2884, 32767, -2885, 32767, -2886, -2886, + 32767, -2887, -2887, 32767, -2888, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 563, 564, + 565, 566, 567, 568, 569, 570, 571, 572, + 573, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 574, 575, 576, 577, 578, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, -294, -294, -294, -3047, 583, 584, 585, + -4462, -4418, -4418, -4418, -4418, -4418, -4462, -4462, + -4462, 595, 596, 597, 598, 599, 32767, 32767, + 32767, 32767, -4471, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 4716, 4717, 4718, 4719, + 4720, 4721, 4722, 4723, 4724, 4725, 4726, 4727, + 4728, 4729, 4730, 4731, 4732, 4733, 4734, 4735, + 3826, 4737, 4738, 4739, 4740, 4741, 4742, 3832, + 4744, 3833, 3120, 3121, 3835, 3835, 3124, 3836, + 3836, 4753, 4754, 4755, 4756, 4757, 4758, 4759, + 4760, 4761, 4762, 4763, 4764, 4765, 4766, 4767, + 4768, 4769, 4770, 4771, 4772, 4773, 4774, 4775, + 4776, 4777, 4778, 4779, 4780, 4781, 6619, 6620, + 6621, 11272, 6623, 6624, 4788, 4789, 4790, 3874, + 4761, 3874, 4794, 3874, 4796, 4797, 4798, 3874, + 4800, 32767, 0, 4802, 4803, 4804, 4805, 4806, + 4807, 4808, 4809, 4810, 4811, 4812, 4813, 4814, + 4815, 4816, 4817, 4818, 4819, 4820, 4821, 4822, + 4823, 4824, 4825, 4826, 4827, 4828, 11299, 4830, + 2536, 4832, 6663, 4834, 11310, 4836, 6669, 4838, + 4839, 11320, 4841, 4842, 4843, 4844, 4845, 4846, + 4847, 4848, 4849, 4850, 4851, 4852, 1188, 4854, + 4855, 4856, 4857, 2577, 4859, 4860, 12273, -907, + -907, -907, -907, -907, -907, 4868, 4869, 4870, + 4871, 32767, 4872, 4873, 32767, 32767, 4874, 32767, + 627, 4875, 4876, 32767, 32767, 4877, 4878, 4879, + 6722, 32767, 4881, 4882, 4883, 6730, 6731, 7446, + 6733, 4888, 7449, 7449, 4891, 4892, 32767, 4893, + 32767, 4894, 4895, 4896, 4897, 4898, 4899, 3512, + 3513, 3514, 3515, 3516, 4904, 3518, 3519, 3520, + 3521, 3522, 3523, 3524, 3525, 3526, 3527, 3528, + 3529, 3530, 3531, 3532, 3533, 3534, 3535, 3536, + 3537, 3538, 4926, 6797, 4928, 6800, 4930, 4931, + 4932, 4933, 4934, 4935, 6813, 4937, 4938, 6816, + 6817, 4941, 4942, 4943, 0, 4945, 6821, 0, + 0, 4949, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 32767, -127, -127, -127, + 7285, -127, -127, 0, -128, -128, -128, -128, + 0, 32767, -130, 4971, -129, 5613, 5614, 5615, + 4976, 5618, 32767, 5619, 5620, 5621, 4981, 5624, + 4983, 4984, 32767, 5630, 5631, -1986, -1986, -126, + -126, 5078, 4992, 5037, 5038, 5039, 5040, 5041, + 5086, 5087, 5088, 5089, -2322, 5091, 5092, 5093, + 5094, 5095, 5096, 5097, 5098, 5099, 5100, 0, + 5101, -640, -640, -640, 0, -641, -641, -641, + -641, -641, 0, -642, 0, 0, 32767, -645, + -645, 6973, 6974, 5115, 5116, -87, 0, -44, + -44, -44, -44, -44, -88, -88, -88, -88, + 7324, -88, -88, -88, -88, -88, -88, -88, + -88, -88, -88, -88, -88, 5654, 5655, 5656, + 5657, 5658, 5659, 5660, 5661, 5662, 5663, 5664, + 5665, 5666, 5667, 5668, 5669, -1948, -1948, -88, + -88, 5116, 5117, 5074, 5075, 5076, 5077, 5078, + 5123, 5124, 5125, 5126, -2285, 5128, 5129, 5130, + 5131, 5132, 5133, 5134, 5135, 5136, 5137, 5138, + 5139, -602, -602, -602, -602, -602, -602, -602, + -602, -602, -602, -602, -602, -602, -602, -602, + -602, 7016, 7017, 5158, 5159, -44, -44, 0, + 0, 0, 0, 0, -44, -44, -44, -44, + 7368, -44, -44, -44, -44, -44, -44, -44, + -44, -44, -44, -44, -44, 5698, 5699, 5700, + 5701, 5702, 5703, 5704, 5705, 5706, 5707, 5708, + 5709, 5710, 5711, 5712, 5713, -1904, -1904, -44, + -44, 5160, 5161, 5118, 5119, 5120, 5121, 5122, + 5167, 5168, 5169, 5170, -2241, 5172, 5173, 5174, + 5175, 5176, 5177, 5178, 5179, 5180, 5181, 5182, + 5183, -558, -558, -558, -558, -558, -558, -558, + -558, -558, -558, -558, -558, -558, -558, -558, + -558, 7060, 7061, 5202, 5203, 0, 0, 44, + 44, 44, 44, 44, 0, 0, 0, 0, + 7412, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 5742, 5743, 5744, + 5745, 5746, 5747, 5748, 5749, 5750, 5751, 5752, + 5753, 5754, 5755, 5756, 5757, -1860, -1860, 0, + 0, 0, 0, 0, 6264, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, -3402, + -3402, 5355, 5356, -3447, -3447, -3402, -3402, -3402, + -3402, 5363, 5364, -3447, -3447, -3402, -3402, -3402, + -3358, -3358, -3404, -3404, -3404, -3404, -3404, -3404, + -3404, 5331, 5332, 5333, 5334, 2903, 2859, 5337, + 5338, 5339, 5340, 18601, 15287, 15288, 11974, 11975, + 8661, 8662, 5348, 5349, 5350, 5351, 5352, -3404, + -3404, 5400, 5401, 5357, 5358, 5359, 5360, -3404, + -3404, 5408, 5409, 5365, 5366, 5367, 5324, 5325, + 5372, 5373, 5374, 5375, 5376, 5377, 5378, -3356, + -3356, -3356, -3356, -924, -879, -3356, -3356, -3356, + -3356, -16616,-13301,-13301,-9986, -9986, -6671, -6671, + -3356, -3356, -3356, -3356, -3356, 5401, 5402, -3401, + -3401, -3356, -3356, -3356, -3356, 5409, 5410, -3401, + -3401, -3356, -3356, -3356, -3312, -3312, -3358, -3358, + -3358, -3358, -3358, -3358, -3358, 5377, 5378, 5379, + 5380, 2949, 2905, 5383, 5384, 5385, 5386, 18647, + 15333, 15334, 12020, 12021, 8707, 8708, 5394, 5395, + 5396, 5397, 5398, -3358, -3358, 5446, 5447, 5403, + 5404, 5405, 5406, -3358, -3358, 5454, 5455, 5411, + 5412, 5413, 5414, 5415, 5416, 5417, 5418, 5419, + 5420, 5421, 5422, -3312, -3312, -3312, -3312, -880, + -835, -3312, -3312, -3312, -3312, -16572,-13257,-13257, + -9942, -9942, -6627, -6627, -3312, -3312, -3312, -3312, + -3312, 5445, 5446, -3357, -3357, -3312, -3312, -3312, + -3312, 5453, 5454, -3357, -3357, -3312, -3312, -3312, + -3312, -3312, -3312, -3312, -3312, -3312, -3312, -3312, + -3312, 5423, 5424, 5425, 5426, 2995, 2951, 5429, + 5430, 5431, 5432, 18693, 15379, 15380, 12066, 12067, + 8753, 8754, 5440, 5441, 5442, 5443, 5444, -3312, + -3312, 5492, 5493, 5449, 5450, 5451, 5452, -3312, + -3312, 5500, 5501, 5457, 2803, 2803, 2803, 2803, + 373, 2803, 2803, 5510, 2803, 5512, 11470, 5514, + 11472, 5516, 2205, 2206, 2207, -7533, -1530, 4427, + 2209, 5524, 2210, 630, 5527, 2211, 2212, 2213, + 2214, 2215, 2216, 2217, 2815, 2219, 2220, 5538, + 2221, 5540, 2222, 5542, 5543, 2223, -3312, -3312, + -3312, 5548, 5549, -3312, -3312, 2803, 2803, 2803, + 5555, 5556, 5557, 2803, 2803, 2803, 2803, 2803, + 2803, 2803, 2803, 2803, 2803, 2803, 2803, 2803, + 9050, 9051, 2803, 2803, 2803, 2803, 2803, 2803, + 2803, 2803, 2803, 2803, 2803, 2803, 4318, 7547, + 7548, 2803, 2803, 2803, 2803, 2803, 2803, 2803, + 2803, 6693, 6693, 2803, 2803, 6694, 6694, 757, + 5603, 6693, 6693, 6693, -1593, 7164, 7165, -1638, + -1638, -1593, 4576, 4576, 4576, 7175, 7176, -1635, + -1635, -1590, -1590, 5422, 2820, -1544, -1590, -1590, + -1590, -1590, -1590, 5422, 5422, 4363, 7149, 7150, + 7151, 4720, 4676, 4677, 4678, -5053, 7161, 20422, + 5425, 5425, 5425, 9160, 9161, 9162, 9163, 9164, + 9165, 9166, 5425, -1587, -1587, 7217, -2553, -2553, + 7177, 7178, -2553, 32767, 32767, -219, 5312, -2555, + -2555, -782, -782, -782, -782, 1806, 2801, 2801, + -6998, -6953, 5675, 5676, 5677, 5678, 5679, -7029, + -790, -6955, -6955, 57, -2545, -6909, -6955, 3143, + -2545, -6955, -2545, 58, 2352, 3149, 5695, 5696, + -52, 5698, 5699, 5700, 5701, 5702, 5703, 5704, + 5705, 5706, 5707, 5708, 5709, 5710, 5711, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, -1838, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 6927, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, -973, 32767, 32767, + 32767, 32767, 0, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 0, 4567, 4568, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, -437, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, -448, 32767, 32767, -450, -450, + -450, 0, 32767, 32767, 32767, -2166, 32767, 32767, + 32767, 32767, 32767, 32767, 0, 0, 32767, -464, + -464, 32767, 0, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, -514, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 5757, 5758, 5759, 0, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, -4186, -4186, -12097,-4186, 32767, + -4187, -4187, -8787, 32767, 0, 0, 5952, 0, + 0, -4183, -4183, -4183, 0, -2386, -4182, 778, + -4183, -5935, 32767, 32767, -4690, -6249, -4184, -4184, + -4184, 32767, 32767, -4186, -4186, -77, 32767, -77, + 32767, -4188, 0, -4189, 32767, 0, 0, 0, + 0, 32767, 0, 0, 0, 32767, 0, 0, + 0, 0, 0, 0, 0, 32767, 0, 0, + 0, 0, 0, 0, 32767, 32767, 32767, 32767, + 0, 0, 0, 0, 0, 32767, 32767, 32767, + 32767, 32767, 32767, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, -5937, -2358, 0, 0, 0, + -8286, 471, 472, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 1747, 32767, -2126, 32767, 32767, 1748, + 1749, 1750, 1751, 1752, 1753, 8224, 1755, -539, + 1757, 781, 32767, 32767, 32767, -1991, -2035, 32767, + 32767, 782, -3784, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 837, 32767, 32767, 32767, 32767, 32767, -4008, + -4008, -4008, 2949, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 0, -797, 1806, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 4605, 4606, + 32767, 32767, 0, 455, 32767, 0, 32767, 32767, + 32767, 0, 32767, 32767, 32767, 32767, 0, 0, + 0, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, -4244, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 784, 32767, 32767, 2950, 2951, 32767, 32767, 32767, + 32767, 32767, 32767, 786, 787, 32767, 1252, 1253, + 32767, 790, 32767, 0, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 0, 0, 32767, 0, 32767, 32767, + 32767, 0, 32767, 32767, 32767, 32767, 0, 0, + 0, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 0, 0, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 0, 0, 0, + 0, 0, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, -200, -200, -200, + -200, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + -5932, -5932, 32767, 32767, 2952, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, -5387, + -5387, -5387, -5387, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 0, 0, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 0, 0, 0, 0, 32767, 32767, + 0, 0, 32767, 32767, 0, 0, 0, 0, + 0, 0, 32767, 32767, 0, 0, 0, 0, + 0, 0, 32767, 32767, 497, 498, 499, 500, + 501, 502, 503, 504, 505, 506, 507, 508, + 32767, 32767, -156, 765, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, -861, + 32767, 6106, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 2953, 2954, 32767, 797, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 2955, 32767, 32767, 32767, -8929, + 32767, -8885, -8885, -8885, 32767, 32767, 32767, 32767, + 32767, 32767, -749, 7119, 7120, 32767, 32767, 32767, + 32767, 2760, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 0, 0, 0, 32767, 32767, 32767, 32767, + 32767, -1181, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, -5587, 0, 7596, + 7597, 0, 0, 0, 0, 0, 0, 32767, + 32767, 32767, 32767, 32767, 32767, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, -714, 0, + 0, -713, -712, 0, -711, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1859, + 0, 3247, 32767, 32767, 0, 3247, 0, 3248, + 0, 3249, 0, 3250, 0, 3251, 0, 3252, + 808, 3252, 0, 3253, 0, 3254, 0, 0, + 3256, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 32767, 0, 0, 0, + 0, 32767, 32767, 32767, 32767, 0, 0, 6824, + 32767, 0, 32767, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 4207, 4208, 0, 0, 0, 0, 0, 1896, + 0, 0, 1898, 1898, 1898, 1898, 0, 0, + 0, 1901, 1901, 0, 0, 0, 0, 0, + 0, -1319, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 7618, 7619, 7620, + 3, 3, 1863, 1863, 7067, 7068, 7025, 7026, + 7027, 7028, 7029, 7074, 7075, 7076, 7077, -334, + 7079, 7080, 7081, 7082, 7083, 7084, 7085, 7086, + 7087, 7088, 7089, 7090, 1349, 1349, 1349, 1349, + 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, + 1349, 1349, 1349, 1349, 8967, 8968, 7109, 7110, + 1907, 1907, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 2976, 2977, 2978, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 0, 0, 0, 820, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 821, + 2381, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 2005, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 823, 32767, 824, 32767, + 825, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 826, 32767, 32767, 32767, 32767, 32767, + 32767, 4575, 4576, 4577, 4578, 4579, 4580, 4581, + 4582, 4583, 4584, 4585, 32767, 32767, 829, 32767, + 32767, 32767, 32767, 830, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 6253, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 6253, -3848, 834, 835, 836, -3845, -3845, -3845, + -3845, -3845, -3845, 843, 844, -4280, 32767, 845, + 846, 6531, 848, -3839, 32767, -3840, -3840, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 1946, 32767, + 32767, 32767, -3849, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 853, 32767, 32767, 32767, + 32767, 854, 32767, 32767, 32767, 32767, 855, 32767, + 32767, 32767, 32767, 856, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 857, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, -3799, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 8266, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 859, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 860, + 32767, 861, -5065, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 10746, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 4526, + 32767, 4573, 4574, 4575, 32767, 32767, -2436, -1376, + 32767, 32767, 32767, 32767, 32767, -1689, -1689, 4349, + -4171, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 4588, 32767, + 4589, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 4590, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 4591, 4592, 32767, + 32767, 32767, 32767, 32767, 32767, 2933, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 864, 32767, 32767, 32767, + 0, 32767, 0, 32767, 32767, -2977, 335, 335, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 2992, 2993, 2994, 2995, + 32767, 32767, 32767, 4596, 2550, 32767, 32767, 32767, + -1188, 4769, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 4600, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 0, 0, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 2997, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 4601, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 2013, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, -11287,32767, 32767, 32767, 32767, + 32767, 32767, 32767, -4664, 32767, 32767, -4711, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, -4718, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 4049, + 32767, 32767, 32767, 4050, 4051, 4052, 17313, 32767, + 32767, 32767, 10684, 7370, 7371, 4057, 4058, 4059, + 4060, 4061, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 4603, 8793, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 1283, 4897, 4898, 4899, 12175, 4901, 4902, 32767, + 4903, 4904, 4905, 4906, 4907, 10276, -1469, 1282, + 1282, 1282, 1282, 1282, 1282, 1282, 1282, 1282, + 1282, 32767, 32767, 4920, 4921, 4063, -2051, -2050, + 4925, 4926, 32767, 7332, 7333, 32767, 7334, 7335, + 7336, 7337, 5045, 32767, 32767, 32767, -2049, -2048, + 32767, -8294, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 0, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1132, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 20166, 16852, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 6908, 6909, 6910, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + -4510, -4510, -4510, -4510, -4510, -4510, -4510, 0, + 0, 0, 0, 0, 0, -1831, -1831, -1831, + -15091,-11776,-11776,-8461, 0, 0, 0, -1834, + -1834, -1834, -1834, -1834, 0, 0, 0, 0, + 0, 0, 0, 0, 32767, 32767, 32767, 32767, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, -1819, -3615, 1345, -3616, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 32767, 32767, 0, + 0, 0, 0, 0, 0, 0, 8770, 8771, + 8772, 8773, 8774, 8775, 8776, 8777, 8778, 8779, + 45, 45, 45, 45, 2477, 2522, 45, 45, + 45, 45, -13215,-9900, -9900, -6585, -6585, -3270, + -3270, 45, 45, 45, 45, 45, 8802, 8803, + 0, 0, 45, 45, 45, 45, 8810, 8811, + 0, 0, 45, 2700, 2701, 2702, 2703, 5134, + 2705, 2706, 0, 2708, 0, -5957, 0, -5957, + 0, 3312, 3312, 3312, 13053, 7051, 1095, 3314, + 0, 3315, 4896, 0, 3317, 3317, 3317, 3317, + 3317, 3317, 3317, 2720, 3317, 3317, 0, 3318, + 0, 3319, 0, 0, 3321, 8857, 8858, 8859, + 0, 0, 8862, 8863, 2749, 2750, 2751, 0, + 0, 0, 2755, 2756, 2757, 2758, 2759, 2760, + 2761, 2762, 2763, 2764, 2765, 2766, 2767, -3479, + -3479, 2770, 2771, 2772, 2773, 2774, 2775, 2776, + 2777, 2778, 2779, 2780, 2781, 1267, -1961, -1961, + 2785, 2786, 2787, 2788, 2789, 2790, 2791, 2792, + -1097, -1096, 2795, 2796, -1094, -1093, 4845, 0, + -1089, -1088, -1087, 7200, -1556, -1556, 7248, 7249, + 7205, 1037, 1038, 1039, -1559, -1559, 7253, 7254, + 7210, 7211, 200, 2803, 7168, 7215, 7216, 7217, + 7218, 7219, 208, 209, 1269, -1516, -1516, -1516, + 916, 961, 961, 961, 10693, -1520, -14780,218, + 219, 220, -3514, -3514, -3514, -3514, -3514, -3514, + -3514, 228, 7241, 7242, -1561, 8210, 8211, -1518, + -1518, 8214, 8215, 9195, 5881, 351, 8219, 8220, + 6448, 6449, 6450, 6451, 3864, 2870, 2871, 12671, + 12627, 0, 0, 0, 0, 0, 12709, 6471, + 12637, 12638, 5627, 8230, 12595, 12642, 2545, 8234, + 12645, 8236, 5634, 3341, 2545, 0, 0, 5749, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 0, 0, 0, 0, 0, 11602, + 0, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 0, 0, 1466, + 0, 0, 32767, 32767, 32767, 32767, 32767, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 5760, 0, 0, 0, 0, 0, 32767, + 0, 32767, 0, 0, 32767, 0, 0, 32767, + 0, 3507, 3508, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1644, 1645, 1646, 1647, -5764, 1649, 1650, 1651, + 1652, 1653, 1654, 1655, 1656, 1657, 1658, 1659, + 1660, -4081, -4081, -4081, -4081, -4081, -4081, -4081, + -4081, -4081, -4081, -4081, -4081, -4081, -4081, -4081, + -4081, 3537, 3538, 1679, 3582, 3583, 3584, -3482, + -3482, -3482, -3482, -3482, -3526, -3526, -3526, -3526, + 3886, -3526, -3526, -3526, -3526, 3599, 3600, 3601, + 3602, 3603, 3604, 3605, 3606, 3607, 3608, 3609, + 3610, 3611, 3612, 3613, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 0, 0, 0, + -7275, 0, 0, -7234, 0, 0, 0, 0, + 0, -5368, 6378, 3628, 3629, 3630, 3631, 3632, + 3633, 3634, 3635, 3636, 3637, 3638, 3639, 0, + 0, 859, 6974, 6974, 0, 0, 3647, -2405, + -2405, 3650, -2405, -2405, -2405, -2405, -112, -2405, + -3201, 3658, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 0, 32767, 32767, 32767, + 32767, 5280, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 4637, 4638, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 4014, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 802, 32767, 32767, + 32767, 32767, 803, -1055, 805, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 4639, 32767, + 32767, 32767, 806, -2445, 0, -2443, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 810, 32767, 32767, + 32767, 32767, 811, 812, 813, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, -6211, -6211, -6211, -6211, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, -6271, -6271, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 935, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, -10300,32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 0, 0, 32767, 32767, 4640, 4641, 32767, + 32767, 32767, 32767, 32767, 4624, 32767, 32767, 32767, + -4233, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 1859, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 872, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, -4568, -1253, 32767, + -3590, 32767, 32767, 32767, -1820, -1820, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 0, 0, 0, 0, 0, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 873, 874, 875, 3629, 0, 0, + 0, 5048, 5005, 5006, 5007, 5008, 5009, 5054, + 5055, 5056, 0, 0, 0, 0, 0, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, -4118, + 32767, 32767, 32767, 32767, -4122, -4122, -4122, -4122, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, -4193, + 32767, -4194, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, -4209, 32767, 32767, -4211, -4211, -4211, + -4211, -4211, -4211, -4211, 32767, 32767, -4213, -10683, + -4213, -1918, -4213, -6043, 32767, 32767, -4215, -6047, + 32767, -4216, -10696,-4216, -4216, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 4646, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 876, 877, 0, 32767, 0, 32767, 0, + 32767, 0, 32767, 0, 32767, 32767, 32767, 0, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 1844, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 0, 0, 0, 0, + 0, 0, 0, 0, 0, -2899, 0, 32767, + 0, 32767, 0, 32767, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 836, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 32767, 0, 0, 0, 879, + 880, 881, 882, 883, 884, 885, 886, 0, + 0, 887, 0, 920, 0, 922, 923, 924, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 5431, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 0, 0, + 0, 32767, 3639, 889, 890, 891, 892, 893, + 894, 895, 896, 897, 898, 899, 900, -2739, + 927, -1881, 4234, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 0, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, -459, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, -458, + -457, 904, 32767, 905, 32767, 906, 32767, 907, + 32767, 908, 32767, 32767, 32767, 909, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 910, + 0, 0, 0, 0, 0, 0, 911, 0, + 912, 1626, 1626, 913, 914, 1626, 915, 916, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, -1837, -1837, -1837, + -6487, -1837, -1837, 0, 0, 0, 917, 31, + 919, 0, 921, 0, 0, 0, 925, 0, + 32767, 4801, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, -6470, 0, 2295, + 0, -1830, 0, -6475, 0, -1832, 0, 0, + -6480, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 3665, 0, 0, + 0, 0, 2281, 0, 0, -7412, 5769, 5770, + 5771, 5772, 5773, 5774, 0, 0, 0, 0, + 32767, 0, 0, 32767, 32767, 0, 32767, 32767, + 0, 0, 32767, 32767, 0, 0, 0, -1842, + 32767, 0, 0, 0, -1846, -1846, -2560, -1846, + 0, -2560, -2559, 0, 0, 32767, 0, 32767, + 0, 0, 0, 0, 0, 0, 1388, 0, + 1387, 1387, 1387, 0, 1387, 1387, 1387, 1387, + 1387, 1387, 1387, 1387, 1387, 1387, 1387, 1387, + 1387, 1387, 1387, 1387, 1387, 1387, 1387, 1387, + 1387, 0, -1870, 0, -1871, 0, 0, 0, + 0, 0, 0, -1877, 0, 0, -1877, -1877, + 0, 0, 0, 4944, 0, -1875, 4947, 4948, + 0, 4950, 4951, 4952, 4953, 4954, 4955, 4956, + 4957, 4958, 4959, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 0, + 32767, 32767, 0, 0, 0, 0, 32767, 32767, + 32767, 0, 0, 931, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 4650, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 5375, + 5376, 5377, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 13180, 0, 0, + 0, 0, 0, 0, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, -4011, 933, -4011, 32767, + 935, 936, -4012, 938, 939, 940, 941, 942, + 943, 944, 945, 946, 947, 32767, 1075, 1076, + 1077, -6334, 1079, 1080, 954, 32767, 32767, 32767, + 32767, 955, 32767, 32767, 32767, 32767, 32767, 32767, + -4659, 32767, 32767, 32767, -4662, -4662, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 0, 0, 0, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 959, 960, 961, 32767, 962, 963, 964, + 965, 966, 967, 968, 969, 970, 971, 972, + 32767, 973, 974, 975, 976, 977, 978, 979, + 980, 981, 982, 983, 984, 985, 986, 987, + 988, 989, 990, 32767, 991, 992, 993, 994, + 995, 996, 997, 998, 999, 1000, 1001, 1002, + 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, + 1011, 1012, 1013, 1014, 1015, 1016, 1017, -362, + -362, 32767, 32767, 32767, 32767, -410, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 1019, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 164, 1021, -3551, -3551, 1024, 1025, 1026, 1027, + 1028, 1029, 1030, 1031, 1032, 1033, 1034, 1035, + 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043, + 1044, 1045, 1046, 1047, 1048, 1049, 1050, 1051, + 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059, + 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1067, + 1068, 1069, 1070, 1071, 1072, 1073, 1074, 1075, + 1076, 1077, 1078, 1079, 1080, 1081, 1082, 1083, + 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, + 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, + 1100, 1101, 1102, 1103, 1104, 1105, 1106, 1107, + 1108, 1109, 1110, 1111, 1112, 1113, 1114, 32767, + 1115, 1116, 1117, 1118, 1119, 32767, 1120, 1121, + 1122, 1123, 1124, 1125, 1126, 1127, 1128, 1129, + 1130, 1131, 0, 1133, 1134, 1135, 1136, 1137, + 1138, 1139, 1140, 1141, 1142, 1143, 1144, 1145, + 1146, 1147, 1148, 1149, 1150, 1151, 1152, 1153, + 1154, 1155, 1156, 1157, 1158, 1159, 1160, 1161, + 1162, 1163, 1164, 1165, 1166, 1167, 1168, 1169, + 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1177, + 1178, 1179, 1180, 1181, 1182, 1183, 1184, 1185, + 1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, + 1194, 1195, 1196, 1197, 1198, 1199, 1200, 1201, + 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, + -18956,-15641,1212, 1213, 1214, 1215, 1216, 1217, + 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, + -5682, -5682, -5682, 1229, 1230, 1231, 1232, 1233, + 1234, 1235, 1236, 1237, 1238, 1239, 5750, 5751, + 5752, 5753, 5754, 5755, 5756, 1247, 1248, 1249, + 1250, 1251, 1252, 3084, 3085, 3086, 16347, 13033, + 13034, 9720, 1260, 1261, 1262, 3097, 3098, 3099, + 3100, 3101, 1268, 1269, 1270, 1271, 1272, 1273, + 1274, 1275, 32767, 32767, 32767, 32767, 1276, 1277, + 1278, 1279, 1280, 1281, 1282, 1283, 1284, 1285, + 1286, 1287, 1288, 1289, 1290, 1291, 1292, 1293, + 1294, 1295, 1296, 1297, 1298, 1299, 1300, 1301, + 1302, 1303, 1304, 1305, 1306, 1307, 1308, 1309, + 1310, 1311, 1312, 1313, 1314, 1315, 1316, 1317, + 1318, 1319, 1320, 1321, 1322, 1323, 1324, 1325, + 1326, 1327, 1328, 1329, 1330, 1331, 1332, 1333, + 1334, 1335, 1336, 1337, 1338, 1339, 1340, 1341, + 1342, 3162, 4959, 0, 4962, 1347, 1348, 1349, + 1350, 1351, 1352, 1353, 1354, 1355, 1356, 1357, + 1358, 1359, 1360, 1361, 1362, 1363, 1364, 1365, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 7481, + 7482, 7483, 7484, 5053, 5009, 7487, 7488, 7489, + 7490, 20751, 17437, 17438, 14124, 14125, 10811, 10812, + 7498, 7499, 7500, 7501, 7502, 32767, 32767, 7548, + 7549, 7505, 7506, 7507, 7508, 32767, 32767, 7554, + 7555, 7511, 4857, 4857, 4857, 4857, 2427, 4857, + 4857, 7564, 4857, 7566, 13524, 7568, 13526, 7570, + 4259, 4260, 4261, -5479, 524, 6481, 4263, 7578, + 4264, 2684, 1421, -7842, -4527, -4527, -1212, -1212, + -1212, -1212, -1212, 7545, 7546, 0, 0, -1214, + -1214, -1214, -1214, 7551, 7552, 32767, 1610, -1216, + 1439, 1440, 1441, 1442, 3873, 1444, 1445, 32767, + 1446, 32767, -7220, 32767, -7221, 0, 2047, 2047, + 2047, 11788, 5786, -170, 2049, -1265, 2050, 3631, + -1265, 2052, 2052, 2052, 2052, 2052, 2052, 2052, + 1455, 2052, 2052, -1265, 2053, -1265, 2054, -1265, + -1265, 2056, 7592, 7593, 7594, 32767, 32767, 7595, + 7596, 1482, 1483, 1484, -1267, -1267, -1267, 1488, + 1489, 1490, 1491, 1492, 1493, 1494, 1495, 1496, + 1497, 1498, 1499, 1500, -4746, -4746, 1503, 1504, + 1505, 1506, 1507, 1508, 1509, 1510, 1511, 1512, + 1513, 1514, 0, -3228, -3228, 1518, 1519, 1520, + 1521, 1522, 1523, 1524, 1525, -2364, -2363, 1528, + 1529, -2361, -2360, 3578, 0, -2357, -2356, -2355, + 5932, -2824, -2824, 5980, 5981, 5937, -231, -230, + -229, -2827, -2827, 5985, -225, 5941, 5942, -1069, + 1534, 5899, 5946, 5947, 5948, 5949, 5950, -1061, + -1060, 0, -2785, 0, -355, -355, -310, -310, + -310, 9422, -2791, 32767, -1054, -1053, -1052, -4786, + -4786, -4786, -4786, -4786, -4786, -4786, -1044, 5969, + 5970, -2833, 6938, 6939, -2790, -2790, 6942, 0, + 32767, 4607, -923, 6945, 32767, 5173, 5174, 5175, + 5176, 2589, 1595, 1596, 11396, 11352, 32767, 32767, + 6126, 2812, 2813, 2814, 2815, 2816, -5940, -5940, + 1607, 1608, 2823, 32767, 32767, 1516, 0, -8581, + 0, 0, 728, 1525, 163, -11068,0, -2262, + -2306, -2305, 32767, 32767, 0, 0, 1580, 0, + 0, 0, -6443, 1685, -10176,-4173, 1784, -4173, + 0, -4172, 5925, -4171, -4171, -4171, 0, -437, + 0, 0, 0, 161, -435, 0, 2883, -434, + 0, 0, 0, 0, -436, 0, -5972, 0, + 0, 0, 0, 0, 0, 0, 0, 2889, + 2890, 2891, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 6371, + 0, 0, 0, 0, 0, 0, 0, 117, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 32767, 0, 0, 3991, 3991, + 3991, 3991, 0, 3990, 3990, 3990, -1947, 1632, + 3990, 3990, 3990, -4296, 4461, 4462, -4341, -4341, + -4296, 1873, 1873, 1873, 4472, 4473, -4338, 1873, + -4292, -4292, 2720, 118, -4246, -4292, -4292, 117, + -4293, -4293, 2719, 2719, 1660, 4446, 1662, 2018, + 2019, 1975, 1976, 1977, -7754, -7754, -8733, -5418, + 113, 0, 112, -2157, -5891, -5891, 0, -5892, + 6455, -5893, 0, 0, 0, 32767, 32767, 32767, + 5826, 32767, 32767, 32767, 32767, 6806, 32767, -2039, + 32767, 5829, 32767, 5830, 5831, 5832, 32767, 5833, + 5834, 32767, 5835, 32767, 32767, -3520, 0, 5837, + 0, 5838, 0, 4035, 0, 5840, 32767, 10251, + 154, 1671, 10253, 1673, 1674, 947, 151, 1514, + 12746, 1679, 3942, 3987, 3987, 3987, 13719, 13720, + 14700, 103, 5855, 13723, 5857, 8127, 0, 11862, + 5860, -96, 5862, 1690, 5863, -4233, 5864, 5865, + 5866, 5867, 5868, 5869, 5870, 5871, 5872, 5873, + 32767, 5874, 5875, 5876, 5877, 5878, 5879, 5880, + 5881, 5882, 5883, 13795, 5885, 5886, 5887, 5888, + 10489, 5890, 1703, 1704, -4247, 1706, 1707, 5891, + 5892, 5893, 1711, 4098, 5895, 5896, 5897, 7650, + 32767, 5899, 6406, 7966, 5902, 5903, 5904, 5905, + 5906, 5907, 5908, 1800, 5910, 1801, 5912, 5913, + 5914, 5915, 32767, 1727, 1728, 1729, 1730, 32767, + 1731, 1732, 1733, 32767, 1734, 1735, 1736, 1737, + 1738, 1739, 1740, 32767, 1741, 1742, 1743, 1744, + 1745, 1746, 32767, 32767, 32767, 32767, 1747, 1748, + 1749, 1750, 1751, 32767, 32767, 32767, 32767, 32767, + 32767, 1752, 1753, 1754, 1755, 1756, 1757, 1758, + 1759, 1760, 1761, 1762, 1763, 1764, 1765, 1766, + 1767, 1768, 1769, 1770, 1771, 1772, 1773, 1774, + 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1782, + 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790, + 1791, 7729, 4151, 1794, 1795, 1796, 10083, 1327, + 1327, 10131, 10132, 10088, 3920, 3921, 3922, 1324, + 1324, 10136, 3926, 10092, 10093, 3082, 5685, 10050, + 10097, 0, 5689, 10100, 5691, 3089, 796, 0, + 1363, 12595, 3792, 3792, 3837, 3837, 3837, 13569, + 13570, 14550, 11236, 5706, 13574, 5708, 7978, 11713, + 11714, 11715, 11716, 11717, 11718, 11719, 7978, 966, + 966, 9770, 0, 0, 9730, 9731, 0, 0, + -979, 2336, 7867, 0, 0, 32767, 0, 0, + 0, 32767, 0, 0, 32767, 0, 32767, 32767, + 9356, 32767, 0, 32767, 0, 32767, 1804, 2602, + 0, -4364, -4410, 5688, 0, -4410, 0, 2603, + 4897, 5694, 4332, -6899, 1905, 1906, 1862, 1863, + 1864, -7867, -7867, -8846, -5531, 0, -7867, 0, + -2269, -6003, -6003, 0, 5957, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, -7911, 0, + 0, 0, 0, -4600, 0, 0, 4156, 32767, + 32767, 0, 0, 0, 0, 0, 1796, 0, + 0, 0, -1752, 0, 0, -506, -2065, 0, + 0, 0, 0, 0, 0, 0, 4109, 0, + 4110, 0, 0, 0, 0, 0, 4111, 17372, + 0, 14058, 10744, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, -4650, 0, 0, 4161, 32767, + 32767, 4117, 32767, 4118, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, -7946, 32767, -4632, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, -4642, + -4642, 4123, 4124, -4687, 0, 0, -4644, -4644, + 0, 0, -4646, -4646, 32767, 32767, 32767, 32767, + 32767, 32767, 4084, 4085, 32767, 32767, 1609, 4087, + 32767, 32767, 4088, 17349, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 10092, 4136, + 10094, 4138, 10096, 0, 10097, 10098, 10099, 10100, + 10101, 0, 32767, 32767, 32767, 0, 0, 0, + 0, 0, 0, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 0, 0, 0, 0, 0, + 0, 0, 0, 32767, 32767, 0, 10138, 10139, + 0, 0, 0, 10145, 32767, 32767, 32767, 32767, + 32767, 32767, -1425, 8316, 2314, -3642, 32767, 0, + 32767, 32767, 32767, 32767, -1426, -1426, -1426, -1426, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 0, 0, 0, 0, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 52, 52, 52, 52, 52, + 0, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 1849, 1850, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 100, 101, 102, 103, 104, 105, 106, 107, + 108, -5633, -5633, -5633, -5633, -5633, -5633, -5633, + -5633, -5633, -5633, -5633, -5633, -5633, -5633, -5633, + -5633, 1985, 1986, 127, 2030, 2031, 2032, -5034, + 32767, 32767, 32767, 32767, 32767, 0, 32767, 32767, + 32767, 5916, 5917, 5918, 5919, 5920, 5921, 5922, + 5923, 5924, 8824, 5926, 32767, 32767, 0, 32767, + 0, 5927, 5928, 5929, 5930, 5931, 5932, 5933, + 5934, 5935, 5936, 5937, 5938, 5939, 5940, 5105, + 5942, 5943, 5944, 5945, 5946, 5947, 5948, 5949, + 5950, 5951, 5952, 5953, 5954, 5955, 5956, 5957, + 32767, 5958, 5959, 5960, 5082, 5082, 5082, 5082, + 5082, 5082, 5082, 5082, 5969, 5970, 5084, 5972, + 5053, 5974, 5053, 5053, 5053, 5978, 5979, 5980, + 5981, 5982, 5983, 5984, 5985, 5986, 5987, 5988, + 5989, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 2552, 32767, 32767, 32767, + 32767, 32767, 32767, 5990, 5991, 5992, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 5993, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 6936, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 0, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 1851, 1852, 1853, 1854, + 1855, 1856, 1857, 1858, 1859, 1860, 1861, 1862, + 1863, 1864, 1200, 2121, 1200, 1868, 1869, 1870, + 1871, 1872, 1873, 1874, 1875, 1876, 1877, 1878, + 1879, 1880, 1188, 1188, 1188, 1188, 1188, 1188, + 1188, 1188, 1188, 1188, 1188, 1188, 1188, 1188, + 1188, 1188, 1188, 1188, 1188, 1188, -5282, 1188, + 3483, 1188, -642, 1188, -5287, 1188, -644, 1188, + 1188, -5292, 1188, 1188, 1188, 1188, 1188, 1188, + 1188, 1188, 1188, 1188, 1188, 1188, 1925, 1926, + -6187, -6231, 1184, 3465, 1184, 1184, -6228, 6953, + 6954, 6955, 6956, 0, 1939, 1940, 1941, 1942, + 1943, 1944, 1178, 1178, 1947, 1948, 1949, 1950, + 1951, 1952, 1953, 1954, 1955, 1956, 1957, 1958, + 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, + 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1974, + 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, + 1983, 1984, 1985, 1986, 1987, 1988, 1989, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 0, 0, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 0, 0, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 709, 666, 667, 668, 32767, 669, + 714, 715, 716, 717, -6694, 719, 720, 721, + 32767, 722, 723, 724, 32767, 725, 726, 727, + 728, -5013, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 6052, 0, 0, 6055, + 0, 0, 0, 0, 2293, 0, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 1244, 1245, 1246, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, -4660, + -4660, -4660, -4660, 4097, 4098, -4705, -4705, -4660, + -4660, -4660, -4660, 4105, 4106, -4705, 32767, -4661, + -4661, -4661, -4617, -4617, -4663, -4663, -4663, -4663, + -4663, -4663, -4663, 4072, 4073, 4074, 4075, 1644, + 1600, 4078, 4079, 4080, 4081, 17342, 14028, 14029, + 10715, 10716, 7402, 7403, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 0, 0, + 0, 32767, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 32767, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 32767, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1380, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 0, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 856, 0, 4573, + 4574, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 32767, 0, 0, 0, + 0, 0, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 5204, 5161, 5162, 5163, 5164, 5165, 5210, 5211, + 5212, 5213, -2198, 5215, 5216, 5217, 5218, 5219, + 5220, 5221, 5222, 5223, 5224, 5225, 5226, -515, + -515, -515, -515, -515, -515, -515, -515, -515, + -515, -515, -515, -515, -515, -515, -515, 7103, + 7104, 5245, 5246, 5247, 5248, 5249, -1014, 5251, + 5252, 5253, 5254, 5255, 5256, 5257, 5258, 5259, + 5260, 8663, 8664, -92, -92, 8712, 8713, 8669, + 8670, 8671, 8672, -92, -92, 8720, 8721, 8677, + 8678, 8679, 8636, 8637, 8684, 8685, 8686, 8687, + 8688, 8689, 8690, -44, -44, -44, -44, 2388, + 2433, -44, -44, -44, -44, -13304,-9989, -9989, + -6674, -6674, -3359, -3359, -44, -44, -44, -44, + -44, 8713, 8714, -89, -89, -44, -44, -44, + -44, 8721, 8722, -89, -89, -44, -44, -44, + 0, 0, -46, -46, -46, -46, -46, -46, + -46, 8689, 8690, 8691, 8692, 6261, 6217, 8695, + 8696, 8697, 8698, 21959, 18645, 18646, 15332, 15333, + 12019, 12020, 8706, 8707, 8708, 8709, 8710, -46, + -46, 8758, 8759, 8715, 8716, 8717, 8718, -46, + -46, 8766, 8767, 8723, 8724, 8725, 8726, 8727, + 8728, 8729, 8730, 8731, 8732, 8733, 8734, 0, + 0, 0, 0, 2432, 2477, 0, 0, 0, + 0, -13260,-9945, -9945, -6630, -6630, -3315, -3315, + 0, 0, 0, 0, 0, 8757, 8758, -45, + -45, 0, 0, 0, 0, 8765, 8766, -45, + -45, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 8735, 8736, 8737, + 8738, 6307, 6263, 8741, 8742, 8743, 8744, 22005, + 18691, 18692, 15378, 15379, 12065, 12066, 8752, 8753, + 8754, 8755, 8756, 0, 0, 8804, 8805, 8761, + 8762, 8763, 8764, 0, 0, 8812, 8813, 8769, + 6115, 6115, 6115, 6115, 3685, 6115, 6115, 8822, + 6115, 8824, 14782, 8826, 14784, 8828, 5517, 5518, + 5519, -4221, 1782, 7739, 5521, 8836, 5522, 3942, + 8839, 5523, 5524, 5525, 5526, 5527, 5528, 5529, + 6127, 5531, 5532, 8850, 5533, 8852, 5534, 8854, + 8855, 5535, 0, 0, 0, 8860, 8861, 0, + 0, 0, 13252, 9939, 9939, 6626, 6626, 3313, + 3313, 0, 0, 0, -9269, -3312, 0, 0, + 0, 9741, 32767, 32767, 0, 32767, 0, 32767, + 32767, 0, 0, 0, 0, 0, 0, 0, + -597, 0, 0, 32767, 0, 32767, 0, 32767, + 32767, 0, 0, 32767, 32767, 32767, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 32767, 32767, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, -1387, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 0, 0, 0, 0, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, -1773, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 0, + 0, 0, 0, 0, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, -4161, 1581, 1582, 32767, 32767, 1990, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 0, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 1539, 32767, 32767, 6150, 6151, 6152, 411, + 411, 411, 411, 411, 411, 411, 411, 411, + 411, 411, 411, 411, 411, 411, 411, 8029, + 8030, 6171, 6172, 969, 969, 1013, 1013, 1013, + 1013, 1013, 969, 969, 969, 969, 8381, 969, + 969, 969, 969, 969, 969, 969, 969, 969, + 969, 969, 969, 6711, 6712, 6713, 6714, 6715, + 6716, 6717, 6718, 6719, 6720, 6721, 6722, 6723, + 6724, 6725, 6726, -891, -891, 969, 969, 6173, + 6174, 6131, 6132, 6133, 6134, 6135, 6180, 6181, + 6182, 6183, -1228, 6185, 6186, 6187, 6188, 6189, + 6190, 6191, 6192, 6193, 6194, 6195, 6196, 455, + 455, 455, 455, 455, 455, 455, 455, 455, + 455, 455, 455, 455, 455, 455, 455, 8073, + 8074, 6215, 6216, 1013, 1013, 1057, 1057, 1057, + 1057, 1057, 1013, 1013, 1013, 1013, 8425, 1013, + 1013, 1013, 1013, 1013, 1013, 1013, 1013, 1013, + 1013, 1013, 1013, 6755, 6756, 6757, 6758, 6759, + 6760, 6761, 6762, 6763, 6764, 6765, 6766, 6767, + 6768, 6769, 6770, -847, -847, 1013, 1013, 6217, + 6218, 6175, 6176, 6177, 6178, 6179, 6224, 6225, + 6226, 6227, -1184, 6229, 6230, 6231, 6232, 6233, + 6234, 6235, 6236, 6237, 6238, 6239, 6240, 499, + 499, 499, 499, 499, 499, 499, 499, 499, + 499, 499, 499, 499, 499, 499, 499, 8117, + 8118, 6259, 6260, 6261, 6262, 6263, 0, 6265, + 6266, 6267, 6268, 6269, 6270, 6271, 6272, 6273, + 6274, 9677, 9678, 922, 922, 9726, 9727, 9683, + 9684, 9685, 9686, 922, 922, 9734, 9735, 9691, + 9692, 9693, 9650, 9651, 9698, 9699, 9700, 9701, + 9702, 9703, 9704, 970, 970, 970, 970, 3402, + 3447, 970, 970, 970, 970, -12290,-8975, -8975, + -5660, -5660, -2345, -2345, -2345, -2345, -2345, 6412, + 6413, -2390, -2390, -2345, -2345, -2345, -2345, 6420, + 6421, -2390, -2390, -2345, -2345, -2345, -2301, -2301, + -2347, -2347, -2347, -2347, -2347, -2347, -2347, 6388, + 6389, 6390, 6391, 3960, 3916, 6394, 6395, 6396, + 6397, 19658, 16344, 16345, 13031, 13032, 9718, 9719, + 6405, 6406, 6407, 6408, 6409, -2347, -2347, 6457, + 6458, 6414, 6415, 6416, 6417, -2347, -2347, 6465, + 6466, 6422, 6423, 6424, 6381, 6382, 6429, 6430, + 6431, 6432, 6433, 6434, 6435, -2299, -2299, -2299, + -2299, 133, 178, -2299, -2299, -2299, -2299, -15559, + -12244,-12244,-8929, -8929, -5614, -5614, -2299, -2299, + -2299, -2299, -2299, 6458, 6459, -2344, -2344, -2299, + -2299, -2299, -2299, 6466, 6467, -2344, -2344, -2299, + -2299, -2299, -2299, -2299, -2299, -2299, -2299, -2299, + -2299, -2299, -2299, 6436, 6437, 6438, 6439, 4008, + 3964, 6442, 6443, 6444, 6445, 19706, 16392, 16393, + 13079, 13080, 9766, 9767, 6453, 6454, 6455, 6456, + 6457, -2299, -2299, 6505, 6506, 6462, 6463, 6464, + 6465, -2299, -2299, 6513, 6514, 6470, 6471, 6472, + 6473, 6474, 6475, 6476, 6477, 6478, 6479, 6480, + 6481, -2253, -2253, -2253, -2253, 179, 224, -2253, + -2253, -2253, -2253, -15513,-12198,-12198,-8883, -8883, + -5568, -5568, -2253, -2253, -2253, -2253, -2253, 6504, + 6505, -2298, -2298, -2253, -2253, -2253, -2253, 6512, + 6513, -2298, -2298, -2253, 402, 403, 404, 405, + 2836, 407, 408, -2298, 410, -2298, -8255, -2298, + -8255, -2298, 1014, 1014, 1014, 10755, 4753, -1203, + 1016, -2298, 1017, 2598, -2298, 1019, 1019, 1019, + 1019, 1019, 1019, 1019, 422, 1019, 1019, -2298, + 1020, -2298, 1021, -2298, -2298, 1023, 6559, 6560, + 6561, -2298, -2298, 6564, 6565, 6566, -6685, -3371, + -3370, -56, -55, 3259, 3260, 3261, 12531, 6575, + 3264, 3265, 3266, -6474, -471, 5486, 3268, 6583, + 3269, 1689, 6586, 3270, 3271, 3272, 3273, 3274, + 3275, 3276, 3874, 3278, 3279, 6597, 3280, 6599, + 3281, 6601, 6602, 3282, 3283, 32767, 32767, 32767, + 3284, 3285, 3286, 3287, 3288, 3289, 3290, 3291, + 3292, 3293, 3294, 3295, 3296, 3297, 3298, 3299, + 3300, 3301, 3302, 3303, 3304, 3305, 3306, 3307, + 3308, 3309, 3310, 3311, 3312, 3313, 3314, 3315, + 3316, 3317, 3318, 3319, 3320, 3321, 3322, 3323, + 3324, 3325, 3326, 3327, 3328, 3329, 3330, 3331, + 3332, 3333, 3334, 3335, 3336, 3337, 3338, 3339, + 3340, 3341, 3342, 3343, 3344, 3345, 3346, 3347, + 3348, 3349, 3350, 3351, 32767, 32767, 3352, 3353, + 3354, 3355, 3356, 3357, 3358, 3359, 3360, 3361, + 3362, 3363, 3364, 3365, 3366, 3367, 3368, 3369, + 3370, 3371, 3372, 3373, 3374, 3375, 3376, 3377, + 3378, 3379, 3380, 3381, 3382, 3383, 3384, 3385, + 3386, 3387, 3388, 3389, 3390, 3391, 3392, 3393, + 3394, 3395, 3396, 3397, 3398, 3399, 3400, 3401, + 3402, 3403, 3404, 3405, 3406, 3407, 4795, 3409, + 3410, 3411, 3412, 3413, 3414, 3415, 3416, 3417, + 3418, 3419, 3420, 3421, 3422, 3423, 3424, 3425, + 3426, 3427, 3428, 3429, 3430, 3431, 3432, 3433, + 3434, 3435, 3436, 3437, 3438, 3439, 3440, 3441, + 3442, 3443, 3444, 3445, 3446, 3447, 3448, 3449, + 3450, 3451, 3452, 3453, 3454, 3455, 3456, 3457, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 3458, + 3459, 3460, 3461, 3462, -8139, 3464, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 3465, 3466, 2001, 3468, 3469, 32767, + 32767, 32767, 32767, 32767, 3470, 3471, 3472, 3473, + 3474, 3475, 3476, 3477, 3478, 3479, 3480, 3481, + 3482, 3483, 3484, 3485, 3486, 3487, 3488, 3489, + 3490, 3491, 3492, 3493, 3494, 3495, 32767, 3496, + 3497, 3498, 3499, 3500, 32767, 3501, 32767, 3502, + 3503, 32767, 3504, 3505, 32767, 3506, 0, 0, + 3509, 3510, 3511, 3512, 3513, 3514, 3515, 3516, + 3517, 3518, 3519, 3520, 3521, 3522, 3523, 3524, + 3525, 3526, 3527, 3528, 3529, 3530, 3531, 3532, + 3533, 3534, 3535, 3536, 3537, 3538, 3539, 3540, + 3541, 3542, 3543, 3544, 3545, 1902, 1902, 1902, + 1902, 9314, 1902, 1902, 1902, 1902, 1902, 1902, + 1902, 1902, 1902, 1902, 1902, 1902, 7644, 7645, + 7646, 7647, 7648, 7649, 7650, 7651, 7652, 7653, + 7654, 7655, 7656, 7657, 7658, 7659, 42, 42, + 1902, 0, 0, 0, 7067, 7068, 7069, 7070, + 7071, 7116, 7117, 7118, 7119, -292, 7121, 7122, + 7123, 7124, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 3614, 3615, 3616, 10892, 3618, 3619, + 10854, 3621, 3622, 3623, 3624, 3625, 8994, -2751, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3640, 3641, 2783, -3331, + -3330, 3645, 3646, 0, 6053, 6054, 0, 6056, + 6057, 6058, 6059, 3767, 6061, 6858, 0, 0, + 3659, 0, 0, 1531, 1531, 1531, 1531, 1531, + 1531, 1531, 1531, 1531, 1531, 1531, 1531, 1531, + 1531, 1531, 9149, 9150, 7291, 7292, 7293, 7294, + 7295, 1032, 7297, 7298, 7299, 7300, 7301, 7302, + 7303, 7304, 0, 7307, 10710, 10711, 1955, 1955, + 10759, 10760, 10716, 4548, 4549, 4550, 1952, 1952, + 10764, 10765, 10721, 10722, 3711, 6314, 10679, 10726, + 10727, 10728, 10729, 10730, 3719, 3720, 1996, 1996, + 1996, 1996, 4428, 4473, 4473, 3728, 1994, 1994, + -11266,3732, 3733, 3734, 0, 0, 0, 0, + 0, 0, 0, 3742, 10755, 10756, 1953, 1953, + 1998, 1998, 1998, 11730, 11731, 12711, 9397, 3867, + 11735, 3869, 6139, 9874, 9875, 9876, 9877, 9878, + 9879, 9880, 6139, -873, -873, 7931, -1839, -1839, + 7891, 7892, -1839, -1839, -2818, 497, 6028, -1839, + -1839, -66, -66, -66, -66, 2522, 2523, -6280, + -6280, -6235, -66, -66, -66, 2533, 2534, -6277, + -66, -6231, -6231, 781, -1821, -6185, -6231, 3867, + -1821, -6231, -1821, 782, 3076, 3873, 2511, -8720, + 84, 85, 41, 42, 43, -9688, -9688, -10667, + -7352, -1821, -9688, -1821, -4090, -7824, -7824, -7824, + -7824, -7824, -7824, -7824, -4082, 2931, 2932, -5871, + 3900, 3901, -5828, -5828, 3904, 3905, 4885, 1571, + -3959, 3909, 3910, 2138, 2139, 2140, 2141, -446, + -446, 8358, 8359, 8315, 2147, 2148, 2149, -449, + -449, 8363, 2153, 8319, 8320, 1309, 3912, 8277, + 8324, -1773, 3916, 8327, 3918, 1316, -977, -1773, + -410, 10822, 2019, 2019, 2064, 2064, 2064, 11796, + 11797, 12777, 9463, 3933, 11801, 3935, 6205, 9940, + 9941, 9942, 9943, 9944, 9945, 9946, 6205, -807, + -807, 7997, -1773, -1773, 7957, 7958, -1773, -1773, + -2752, 563, 6094, -1773, -1773, 0, 0, 0, + 0, 2588, 2589, -6214, -6214, -6169, 0, 0, + 0, 2599, 2600, -6211, 0, -6165, -6165, 847, + -1755, -6119, -6165, 3933, -1755, -6165, -1755, 848, + 3142, 3939, 2577, -8654, 150, 151, 107, 108, + 109, -9622, -9622, -10601,-7286, -1755, -9622, -1755, + -4024, -7758, -7758, -7758, -7758, -7758, -7758, -7758, + -4016, 2997, 2998, -5805, 3966, 3967, -5762, -5762, + 3970, 3971, 4951, 1637, -3893, 3975, 3976, 2204, + 2205, 2206, 2207, -380, -380, 8424, 8425, 8381, + 2213, 2214, 2215, -383, -383, 8429, 2219, 8385, + 8386, 1375, 3978, 8343, 8390, -1707, 3982, 8393, + 3984, 1382, -911, -1707, -344, 10888, 2085, 2085, + 2130, 2130, 2130, 11862, 11863, 12843, 9529, 3999, + 11867, 4001, 6271, 10006, 10007, 4005, -1951, 4007, + 4008, 4009, 4010, 4011, 4012, 4013, 4014, 4015, + 4016, 4017, 4018, 4019, 4020, 4021, 4022, 4023, + 4024, 4025, 4026, 4027, 4028, 4029, 4030, 4031, + 11943, 4033, 4034, 4035, 4036, 8637, 4038, 4039, + -116, 32767, 32767, 4041, 4042, 4043, 4044, 4045, + 2250, 4047, 4048, 4049, 5802, 4051, 4052, 4559, + 6119, 4055, 4056, 4057, 4058, 4059, 4060, 4061, + -47, 4063, -46, 4065, 4066, 4067, 4068, 4069, + -41, -13301,4072, -9985, -6670, 4075, 4076, 4077, + 4078, 4079, 4080, 4081, 4082, 4083, 4084, 4085, + 4086, 4087, 4088, 4089, 4090, 8741, 4092, 4093, + -67, 32767, 32767, 32767, 32767, 32767, 2257, 32767, + 2258, 2259, 2260, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 2261, 32767, 2262, 32767, + 2263, 32767, 2264, 32767, 2265, 32767, 2266, 32767, + 2267, 8737, 8738, -26, -26, 8786, 4100, 4101, + 8746, 8747, 4104, 4105, 8752, 8753, 32767, 2274, + 32767, 2275, 32767, 32767, 32767, 32767, 32767, 32767, + 2276, 2277, 32767, 2278, 2279, 32767, 2280, 0, + 32767, 2282, 9695, 4109, -3486, -3486, 4112, 4113, + 4114, 4115, 4116, 4117, 32767, 32767, 32767, 32767, + 32767, 32767, 4118, 4119, 4120, 4121, 4122, 4123, + 4124, 4125, 4126, 4127, 4128, 4129, 4130, 4131, + 4132, 4133, 4134, 4849, 4136, 4137, 4851, 4851, + 4140, 4852, 4142, 4143, 4144, 4145, 4146, 4147, + 4148, 4149, 4150, 4151, 2293, 4153, 907, 32767, + 2295, 4155, 909, 4157, 910, 4159, 911, 4161, + 912, 4163, 913, 4165, 914, 32767, 915, 4168, + 916, 4170, 917, 4172, 4173, 918, 4175, 4176, + 4177, 4178, 4179, 4180, 4181, 4182, 4183, 4184, + 4185, 2309, 4186, 4187, 4188, 4189, 2312, 2313, + 32767, 2314, 4190, 4191, -2632, 2317, 4193, 32767, + 4194, 4195, 4196, 4197, 4198, 4199, 4200, 4201, + 4202, 4203, 4204, 4205, 4206, 0, 0, 4209, + 4210, 4211, 4212, 4213, 2318, 4215, 4216, 2319, + 2320, 2321, 2322, 4221, 4222, 4223, 2323, 2324, + 4226, 4227, 4228, 4229, 4230, 4231, 5551, 4233, + 4234, 4235, 4236, 4237, 4238, 4239, 4240, 4241, + 4242, 4243, 4244, 4245, 4246, 4247, 4248, 4249, + 4250, 4251, 4252, 4253, 4254, 4255, 4256, 4257, + 4258, 4259, 4260, 4261, 4262, 4263, 4264, 4265, + 4266, 4267, 4268, 4269, 4270, 4271, 4272, 4273, + 4274, 4275, -3342, -3342, -3342, 4276, 4277, 2418, + 2419, -2784, -2784, -2740, -2740, -2740, -2740, -2740, + -2784, -2784, -2784, -2784, 4628, -2784, -2784, -2784, + -2784, -2784, -2784, -2784, -2784, -2784, -2784, -2784, + -2784, 2958, 2959, 2960, 2961, 2962, 2963, 2964, + 2965, 2966, 2967, 2968, 2969, 2970, 2971, 2972, + 2973, -4644, -4644, -2784, -2784, 2420, 2421, 2378, + 2379, 2380, 2381, 2382, 2427, 2428, 2429, 2430, + -4981, 2432, 2433, 2434, 2435, 2436, 2437, 2438, + 2439, 2440, 2441, 2442, 2443, -3298, -3298, -3298, + -3298, -3298, -3298, -3298, -3298, -3298, -3298, -3298, + -3298, -3298, -3298, -3298, -3298, 4320, 4321, 2462, + 4365, 4366, 4367, -2699, -2699, -2699, -2699, -2699, + -2743, -2743, -2743, -2743, 4669, -2743, -2743, -2743, + -2743, 4382, 4383, 4384, 4385, 4386, 4387, 4388, + 4389, 4390, 4391, 4392, 4393, 4394, 4395, 4396, + 4397, 4398, 4399, 4400, 4401, 4402, 4403, 4404, + 4405, 4406, 4407, 4408, 4409, 4410, 4411, 4412, + 4413, 4414, 4415, 4416, 4417, 4418, 4419, 4420, + 4421, 4422, 4423, 4424, 4425, 4426, 4427, 4428, + 4429, 816, 816, 816, -6459, 816, 816, -6418, + 816, 816, 816, 816, 816, -4552, 7194, 4444, + 4445, 4446, 4447, 4448, 4449, 4450, 4451, 4452, + 4453, 4454, 4455, 816, 816, 1675, 7790, 7790, + 816, 816, 4463, -1589, -1589, 4466, -1589, -1589, + -1589, -1589, 704, -1589, -2385, 4474, 4475, 817, + 4477, 4478, 2948, 2949, 2950, 2951, 2952, 2953, + 2954, 2955, 2956, 2957, 2958, 2959, 2960, 2961, + 2962, -4655, -4655, -2795, -2795, -2795, -2795, -2795, + 3469, -2795, -2795, -2795, -2795, -2795, -2795, -2795, + -2795, 4510, -2796, -6198, -6198, 2559, 2560, -6243, + -6243, -6198, -6198, -6198, -6198, 2567, 2568, -6243, + -6243, -6198, -6198, -6198, -6154, -6154, -6200, -6200, + -6200, -6200, -6200, -6200, -6200, 2535, 2536, 2537, + 2538, 107, 63, 2541, 2542, 2543, 2544, 15805, + 12491, 12492, 32767, 4540, 4541, 4542, 4543, 4544, + 4545, 4546, 2548, -6208, -6208, 2596, 2597, 2553, + 2554, 2555, 2556, -6208, -6208, 2604, 2605, 2561, + 2562, 2563, 2520, 2521, 2568, 2569, 2570, 2571, + 2572, 2573, 2574, -6160, -6160, -6160, -6160, -3728, + -3683, -6160, -6160, -6160, -6160, -19420,-16105,-16105, + -12790,-12790,-9475, -9475, -6160, -6160, -6160, -6160, + -6160, 32767, 2597, -6206, -6206, -6161, -6161, -6161, + -6161, 2604, 2605, -6206, -6206, -6161, -6161, -6161, + -6161, -6161, -6161, -6161, -6161, -6161, -6161, -6161, + -6161, 2574, 2575, 2576, 2577, 146, 102, 2580, + 2581, 2582, 2583, 15844, 12530, 12531, 9217, 9218, + 5904, 5905, 2591, 2592, 2593, 2594, 2595, -6161, + -6161, 2643, 2644, 2600, 2601, 2602, 2603, -6161, + -6161, 2651, 2652, 2608, 2609, 2610, 2611, 2612, + 2613, 2614, 2615, 2616, 2617, 2618, 2619, -6115, + -6115, -6115, -6115, -3683, -3638, -6115, -6115, -6115, + -6115, -19375,-16060,-16060,-12745,-12745,-9430, -9430, + -6115, -6115, -6115, -6115, -6115, 2642, 2643, -6160, + -6160, -6115, -6115, -6115, -6115, 2650, 2651, -6160, + -6160, -6115, -3460, -3459, -3458, -3457, -1026, -3455, + -3454, -6160, -3452, -6160, -12117,-6160, -12117,-6160, + -2848, -2848, -2848, 6893, 891, -5065, -2846, -6160, + -2845, -1264, 0, 9264, 5950, 5951, 2637, 2638, + 2639, 2640, 2641, -6115, -6115, 2689, 2690, 2646, + 2647, 2648, 2649, -6115, -6115, 2697, 2698, 2654, + 0, 0, 0, 0, -2430, 0, 0, 2707, + 0, 2709, 8667, 2711, 8669, 2713, -598, -597, + -596, -10336,-4333, 1624, -594, 2721, -593, -2173, + 2724, -592, -591, -590, -589, -588, -587, -586, + 12, -584, -583, 2735, -582, 2737, -581, 2739, + 2740, -580, -6115, -6115, -6115, 2745, 2746, -6115, + -6115, 0, 0, 0, 2752, 2753, 2754, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 6247, 6248, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0 + }; + + const unsigned char *k = (const unsigned char *) key; + size_t keylen = 4; + uint32 a = 0; + uint32 b = 1; + + while (keylen--) + { + unsigned char c = *k++; + + a = a * 257 + c; + b = b * 8191 + c; + } + return h[a % 13209] + h[b % 13209]; +} + +/* Hash lookup information for decomposition */ +static const pg_unicode_decompinfo UnicodeDecompInfo = +{ + UnicodeDecompMain, + Decomp_hash_func, + 6604 +}; + +/* Inverse lookup array -- contains indexes into UnicodeDecompMain[] */ +static const uint16 RecompInverseLookup[941] = +{ + /* U+003C+0338 -> U+226E */ 1823, + /* U+003D+0338 -> U+2260 */ 1820, + /* U+003E+0338 -> U+226F */ 1824, + /* U+0041+0300 -> U+00C0 */ 14, + /* U+0041+0301 -> U+00C1 */ 15, + /* U+0041+0302 -> U+00C2 */ 16, + /* U+0041+0303 -> U+00C3 */ 17, + /* U+0041+0304 -> U+0100 */ 67, + /* U+0041+0306 -> U+0102 */ 69, + /* U+0041+0307 -> U+0226 */ 270, + /* U+0041+0308 -> U+00C4 */ 18, + /* U+0041+0309 -> U+1EA2 */ 1278, + /* U+0041+030A -> U+00C5 */ 19, + /* U+0041+030C -> U+01CD */ 194, + /* U+0041+030F -> U+0200 */ 240, + /* U+0041+0311 -> U+0202 */ 242, + /* U+0041+0323 -> U+1EA0 */ 1276, + /* U+0041+0325 -> U+1E00 */ 1120, + /* U+0041+0328 -> U+0104 */ 71, + /* U+0042+0307 -> U+1E02 */ 1122, + /* U+0042+0323 -> U+1E04 */ 1124, + /* U+0042+0331 -> U+1E06 */ 1126, + /* U+0043+0301 -> U+0106 */ 73, + /* U+0043+0302 -> U+0108 */ 75, + /* U+0043+0307 -> U+010A */ 77, + /* U+0043+030C -> U+010C */ 79, + /* U+0043+0327 -> U+00C7 */ 20, + /* U+0044+0307 -> U+1E0A */ 1130, + /* U+0044+030C -> U+010E */ 81, + /* U+0044+0323 -> U+1E0C */ 1132, + /* U+0044+0327 -> U+1E10 */ 1136, + /* U+0044+032D -> U+1E12 */ 1138, + /* U+0044+0331 -> U+1E0E */ 1134, + /* U+0045+0300 -> U+00C8 */ 21, + /* U+0045+0301 -> U+00C9 */ 22, + /* U+0045+0302 -> U+00CA */ 23, + /* U+0045+0303 -> U+1EBC */ 1304, + /* U+0045+0304 -> U+0112 */ 83, + /* U+0045+0306 -> U+0114 */ 85, + /* U+0045+0307 -> U+0116 */ 87, + /* U+0045+0308 -> U+00CB */ 24, + /* U+0045+0309 -> U+1EBA */ 1302, + /* U+0045+030C -> U+011A */ 91, + /* U+0045+030F -> U+0204 */ 244, + /* U+0045+0311 -> U+0206 */ 246, + /* U+0045+0323 -> U+1EB8 */ 1300, + /* U+0045+0327 -> U+0228 */ 272, + /* U+0045+0328 -> U+0118 */ 89, + /* U+0045+032D -> U+1E18 */ 1144, + /* U+0045+0330 -> U+1E1A */ 1146, + /* U+0046+0307 -> U+1E1E */ 1150, + /* U+0047+0301 -> U+01F4 */ 230, + /* U+0047+0302 -> U+011C */ 93, + /* U+0047+0304 -> U+1E20 */ 1152, + /* U+0047+0306 -> U+011E */ 95, + /* U+0047+0307 -> U+0120 */ 97, + /* U+0047+030C -> U+01E6 */ 216, + /* U+0047+0327 -> U+0122 */ 99, + /* U+0048+0302 -> U+0124 */ 101, + /* U+0048+0307 -> U+1E22 */ 1154, + /* U+0048+0308 -> U+1E26 */ 1158, + /* U+0048+030C -> U+021E */ 268, + /* U+0048+0323 -> U+1E24 */ 1156, + /* U+0048+0327 -> U+1E28 */ 1160, + /* U+0048+032E -> U+1E2A */ 1162, + /* U+0049+0300 -> U+00CC */ 25, + /* U+0049+0301 -> U+00CD */ 26, + /* U+0049+0302 -> U+00CE */ 27, + /* U+0049+0303 -> U+0128 */ 103, + /* U+0049+0304 -> U+012A */ 105, + /* U+0049+0306 -> U+012C */ 107, + /* U+0049+0307 -> U+0130 */ 111, + /* U+0049+0308 -> U+00CF */ 28, + /* U+0049+0309 -> U+1EC8 */ 1316, + /* U+0049+030C -> U+01CF */ 196, + /* U+0049+030F -> U+0208 */ 248, + /* U+0049+0311 -> U+020A */ 250, + /* U+0049+0323 -> U+1ECA */ 1318, + /* U+0049+0328 -> U+012E */ 109, + /* U+0049+0330 -> U+1E2C */ 1164, + /* U+004A+0302 -> U+0134 */ 114, + /* U+004B+0301 -> U+1E30 */ 1168, + /* U+004B+030C -> U+01E8 */ 218, + /* U+004B+0323 -> U+1E32 */ 1170, + /* U+004B+0327 -> U+0136 */ 116, + /* U+004B+0331 -> U+1E34 */ 1172, + /* U+004C+0301 -> U+0139 */ 118, + /* U+004C+030C -> U+013D */ 122, + /* U+004C+0323 -> U+1E36 */ 1174, + /* U+004C+0327 -> U+013B */ 120, + /* U+004C+032D -> U+1E3C */ 1180, + /* U+004C+0331 -> U+1E3A */ 1178, + /* U+004D+0301 -> U+1E3E */ 1182, + /* U+004D+0307 -> U+1E40 */ 1184, + /* U+004D+0323 -> U+1E42 */ 1186, + /* U+004E+0300 -> U+01F8 */ 232, + /* U+004E+0301 -> U+0143 */ 126, + /* U+004E+0303 -> U+00D1 */ 29, + /* U+004E+0307 -> U+1E44 */ 1188, + /* U+004E+030C -> U+0147 */ 130, + /* U+004E+0323 -> U+1E46 */ 1190, + /* U+004E+0327 -> U+0145 */ 128, + /* U+004E+032D -> U+1E4A */ 1194, + /* U+004E+0331 -> U+1E48 */ 1192, + /* U+004F+0300 -> U+00D2 */ 30, + /* U+004F+0301 -> U+00D3 */ 31, + /* U+004F+0302 -> U+00D4 */ 32, + /* U+004F+0303 -> U+00D5 */ 33, + /* U+004F+0304 -> U+014C */ 133, + /* U+004F+0306 -> U+014E */ 135, + /* U+004F+0307 -> U+022E */ 278, + /* U+004F+0308 -> U+00D6 */ 34, + /* U+004F+0309 -> U+1ECE */ 1322, + /* U+004F+030B -> U+0150 */ 137, + /* U+004F+030C -> U+01D1 */ 198, + /* U+004F+030F -> U+020C */ 252, + /* U+004F+0311 -> U+020E */ 254, + /* U+004F+031B -> U+01A0 */ 181, + /* U+004F+0323 -> U+1ECC */ 1320, + /* U+004F+0328 -> U+01EA */ 220, + /* U+0050+0301 -> U+1E54 */ 1204, + /* U+0050+0307 -> U+1E56 */ 1206, + /* U+0052+0301 -> U+0154 */ 139, + /* U+0052+0307 -> U+1E58 */ 1208, + /* U+0052+030C -> U+0158 */ 143, + /* U+0052+030F -> U+0210 */ 256, + /* U+0052+0311 -> U+0212 */ 258, + /* U+0052+0323 -> U+1E5A */ 1210, + /* U+0052+0327 -> U+0156 */ 141, + /* U+0052+0331 -> U+1E5E */ 1214, + /* U+0053+0301 -> U+015A */ 145, + /* U+0053+0302 -> U+015C */ 147, + /* U+0053+0307 -> U+1E60 */ 1216, + /* U+0053+030C -> U+0160 */ 151, + /* U+0053+0323 -> U+1E62 */ 1218, + /* U+0053+0326 -> U+0218 */ 264, + /* U+0053+0327 -> U+015E */ 149, + /* U+0054+0307 -> U+1E6A */ 1226, + /* U+0054+030C -> U+0164 */ 155, + /* U+0054+0323 -> U+1E6C */ 1228, + /* U+0054+0326 -> U+021A */ 266, + /* U+0054+0327 -> U+0162 */ 153, + /* U+0054+032D -> U+1E70 */ 1232, + /* U+0054+0331 -> U+1E6E */ 1230, + /* U+0055+0300 -> U+00D9 */ 35, + /* U+0055+0301 -> U+00DA */ 36, + /* U+0055+0302 -> U+00DB */ 37, + /* U+0055+0303 -> U+0168 */ 157, + /* U+0055+0304 -> U+016A */ 159, + /* U+0055+0306 -> U+016C */ 161, + /* U+0055+0308 -> U+00DC */ 38, + /* U+0055+0309 -> U+1EE6 */ 1346, + /* U+0055+030A -> U+016E */ 163, + /* U+0055+030B -> U+0170 */ 165, + /* U+0055+030C -> U+01D3 */ 200, + /* U+0055+030F -> U+0214 */ 260, + /* U+0055+0311 -> U+0216 */ 262, + /* U+0055+031B -> U+01AF */ 183, + /* U+0055+0323 -> U+1EE4 */ 1344, + /* U+0055+0324 -> U+1E72 */ 1234, + /* U+0055+0328 -> U+0172 */ 167, + /* U+0055+032D -> U+1E76 */ 1238, + /* U+0055+0330 -> U+1E74 */ 1236, + /* U+0056+0303 -> U+1E7C */ 1244, + /* U+0056+0323 -> U+1E7E */ 1246, + /* U+0057+0300 -> U+1E80 */ 1248, + /* U+0057+0301 -> U+1E82 */ 1250, + /* U+0057+0302 -> U+0174 */ 169, + /* U+0057+0307 -> U+1E86 */ 1254, + /* U+0057+0308 -> U+1E84 */ 1252, + /* U+0057+0323 -> U+1E88 */ 1256, + /* U+0058+0307 -> U+1E8A */ 1258, + /* U+0058+0308 -> U+1E8C */ 1260, + /* U+0059+0300 -> U+1EF2 */ 1358, + /* U+0059+0301 -> U+00DD */ 39, + /* U+0059+0302 -> U+0176 */ 171, + /* U+0059+0303 -> U+1EF8 */ 1364, + /* U+0059+0304 -> U+0232 */ 282, + /* U+0059+0307 -> U+1E8E */ 1262, + /* U+0059+0308 -> U+0178 */ 173, + /* U+0059+0309 -> U+1EF6 */ 1362, + /* U+0059+0323 -> U+1EF4 */ 1360, + /* U+005A+0301 -> U+0179 */ 174, + /* U+005A+0302 -> U+1E90 */ 1264, + /* U+005A+0307 -> U+017B */ 176, + /* U+005A+030C -> U+017D */ 178, + /* U+005A+0323 -> U+1E92 */ 1266, + /* U+005A+0331 -> U+1E94 */ 1268, + /* U+0061+0300 -> U+00E0 */ 40, + /* U+0061+0301 -> U+00E1 */ 41, + /* U+0061+0302 -> U+00E2 */ 42, + /* U+0061+0303 -> U+00E3 */ 43, + /* U+0061+0304 -> U+0101 */ 68, + /* U+0061+0306 -> U+0103 */ 70, + /* U+0061+0307 -> U+0227 */ 271, + /* U+0061+0308 -> U+00E4 */ 44, + /* U+0061+0309 -> U+1EA3 */ 1279, + /* U+0061+030A -> U+00E5 */ 45, + /* U+0061+030C -> U+01CE */ 195, + /* U+0061+030F -> U+0201 */ 241, + /* U+0061+0311 -> U+0203 */ 243, + /* U+0061+0323 -> U+1EA1 */ 1277, + /* U+0061+0325 -> U+1E01 */ 1121, + /* U+0061+0328 -> U+0105 */ 72, + /* U+0062+0307 -> U+1E03 */ 1123, + /* U+0062+0323 -> U+1E05 */ 1125, + /* U+0062+0331 -> U+1E07 */ 1127, + /* U+0063+0301 -> U+0107 */ 74, + /* U+0063+0302 -> U+0109 */ 76, + /* U+0063+0307 -> U+010B */ 78, + /* U+0063+030C -> U+010D */ 80, + /* U+0063+0327 -> U+00E7 */ 46, + /* U+0064+0307 -> U+1E0B */ 1131, + /* U+0064+030C -> U+010F */ 82, + /* U+0064+0323 -> U+1E0D */ 1133, + /* U+0064+0327 -> U+1E11 */ 1137, + /* U+0064+032D -> U+1E13 */ 1139, + /* U+0064+0331 -> U+1E0F */ 1135, + /* U+0065+0300 -> U+00E8 */ 47, + /* U+0065+0301 -> U+00E9 */ 48, + /* U+0065+0302 -> U+00EA */ 49, + /* U+0065+0303 -> U+1EBD */ 1305, + /* U+0065+0304 -> U+0113 */ 84, + /* U+0065+0306 -> U+0115 */ 86, + /* U+0065+0307 -> U+0117 */ 88, + /* U+0065+0308 -> U+00EB */ 50, + /* U+0065+0309 -> U+1EBB */ 1303, + /* U+0065+030C -> U+011B */ 92, + /* U+0065+030F -> U+0205 */ 245, + /* U+0065+0311 -> U+0207 */ 247, + /* U+0065+0323 -> U+1EB9 */ 1301, + /* U+0065+0327 -> U+0229 */ 273, + /* U+0065+0328 -> U+0119 */ 90, + /* U+0065+032D -> U+1E19 */ 1145, + /* U+0065+0330 -> U+1E1B */ 1147, + /* U+0066+0307 -> U+1E1F */ 1151, + /* U+0067+0301 -> U+01F5 */ 231, + /* U+0067+0302 -> U+011D */ 94, + /* U+0067+0304 -> U+1E21 */ 1153, + /* U+0067+0306 -> U+011F */ 96, + /* U+0067+0307 -> U+0121 */ 98, + /* U+0067+030C -> U+01E7 */ 217, + /* U+0067+0327 -> U+0123 */ 100, + /* U+0068+0302 -> U+0125 */ 102, + /* U+0068+0307 -> U+1E23 */ 1155, + /* U+0068+0308 -> U+1E27 */ 1159, + /* U+0068+030C -> U+021F */ 269, + /* U+0068+0323 -> U+1E25 */ 1157, + /* U+0068+0327 -> U+1E29 */ 1161, + /* U+0068+032E -> U+1E2B */ 1163, + /* U+0068+0331 -> U+1E96 */ 1270, + /* U+0069+0300 -> U+00EC */ 51, + /* U+0069+0301 -> U+00ED */ 52, + /* U+0069+0302 -> U+00EE */ 53, + /* U+0069+0303 -> U+0129 */ 104, + /* U+0069+0304 -> U+012B */ 106, + /* U+0069+0306 -> U+012D */ 108, + /* U+0069+0308 -> U+00EF */ 54, + /* U+0069+0309 -> U+1EC9 */ 1317, + /* U+0069+030C -> U+01D0 */ 197, + /* U+0069+030F -> U+0209 */ 249, + /* U+0069+0311 -> U+020B */ 251, + /* U+0069+0323 -> U+1ECB */ 1319, + /* U+0069+0328 -> U+012F */ 110, + /* U+0069+0330 -> U+1E2D */ 1165, + /* U+006A+0302 -> U+0135 */ 115, + /* U+006A+030C -> U+01F0 */ 226, + /* U+006B+0301 -> U+1E31 */ 1169, + /* U+006B+030C -> U+01E9 */ 219, + /* U+006B+0323 -> U+1E33 */ 1171, + /* U+006B+0327 -> U+0137 */ 117, + /* U+006B+0331 -> U+1E35 */ 1173, + /* U+006C+0301 -> U+013A */ 119, + /* U+006C+030C -> U+013E */ 123, + /* U+006C+0323 -> U+1E37 */ 1175, + /* U+006C+0327 -> U+013C */ 121, + /* U+006C+032D -> U+1E3D */ 1181, + /* U+006C+0331 -> U+1E3B */ 1179, + /* U+006D+0301 -> U+1E3F */ 1183, + /* U+006D+0307 -> U+1E41 */ 1185, + /* U+006D+0323 -> U+1E43 */ 1187, + /* U+006E+0300 -> U+01F9 */ 233, + /* U+006E+0301 -> U+0144 */ 127, + /* U+006E+0303 -> U+00F1 */ 55, + /* U+006E+0307 -> U+1E45 */ 1189, + /* U+006E+030C -> U+0148 */ 131, + /* U+006E+0323 -> U+1E47 */ 1191, + /* U+006E+0327 -> U+0146 */ 129, + /* U+006E+032D -> U+1E4B */ 1195, + /* U+006E+0331 -> U+1E49 */ 1193, + /* U+006F+0300 -> U+00F2 */ 56, + /* U+006F+0301 -> U+00F3 */ 57, + /* U+006F+0302 -> U+00F4 */ 58, + /* U+006F+0303 -> U+00F5 */ 59, + /* U+006F+0304 -> U+014D */ 134, + /* U+006F+0306 -> U+014F */ 136, + /* U+006F+0307 -> U+022F */ 279, + /* U+006F+0308 -> U+00F6 */ 60, + /* U+006F+0309 -> U+1ECF */ 1323, + /* U+006F+030B -> U+0151 */ 138, + /* U+006F+030C -> U+01D2 */ 199, + /* U+006F+030F -> U+020D */ 253, + /* U+006F+0311 -> U+020F */ 255, + /* U+006F+031B -> U+01A1 */ 182, + /* U+006F+0323 -> U+1ECD */ 1321, + /* U+006F+0328 -> U+01EB */ 221, + /* U+0070+0301 -> U+1E55 */ 1205, + /* U+0070+0307 -> U+1E57 */ 1207, + /* U+0072+0301 -> U+0155 */ 140, + /* U+0072+0307 -> U+1E59 */ 1209, + /* U+0072+030C -> U+0159 */ 144, + /* U+0072+030F -> U+0211 */ 257, + /* U+0072+0311 -> U+0213 */ 259, + /* U+0072+0323 -> U+1E5B */ 1211, + /* U+0072+0327 -> U+0157 */ 142, + /* U+0072+0331 -> U+1E5F */ 1215, + /* U+0073+0301 -> U+015B */ 146, + /* U+0073+0302 -> U+015D */ 148, + /* U+0073+0307 -> U+1E61 */ 1217, + /* U+0073+030C -> U+0161 */ 152, + /* U+0073+0323 -> U+1E63 */ 1219, + /* U+0073+0326 -> U+0219 */ 265, + /* U+0073+0327 -> U+015F */ 150, + /* U+0074+0307 -> U+1E6B */ 1227, + /* U+0074+0308 -> U+1E97 */ 1271, + /* U+0074+030C -> U+0165 */ 156, + /* U+0074+0323 -> U+1E6D */ 1229, + /* U+0074+0326 -> U+021B */ 267, + /* U+0074+0327 -> U+0163 */ 154, + /* U+0074+032D -> U+1E71 */ 1233, + /* U+0074+0331 -> U+1E6F */ 1231, + /* U+0075+0300 -> U+00F9 */ 61, + /* U+0075+0301 -> U+00FA */ 62, + /* U+0075+0302 -> U+00FB */ 63, + /* U+0075+0303 -> U+0169 */ 158, + /* U+0075+0304 -> U+016B */ 160, + /* U+0075+0306 -> U+016D */ 162, + /* U+0075+0308 -> U+00FC */ 64, + /* U+0075+0309 -> U+1EE7 */ 1347, + /* U+0075+030A -> U+016F */ 164, + /* U+0075+030B -> U+0171 */ 166, + /* U+0075+030C -> U+01D4 */ 201, + /* U+0075+030F -> U+0215 */ 261, + /* U+0075+0311 -> U+0217 */ 263, + /* U+0075+031B -> U+01B0 */ 184, + /* U+0075+0323 -> U+1EE5 */ 1345, + /* U+0075+0324 -> U+1E73 */ 1235, + /* U+0075+0328 -> U+0173 */ 168, + /* U+0075+032D -> U+1E77 */ 1239, + /* U+0075+0330 -> U+1E75 */ 1237, + /* U+0076+0303 -> U+1E7D */ 1245, + /* U+0076+0323 -> U+1E7F */ 1247, + /* U+0077+0300 -> U+1E81 */ 1249, + /* U+0077+0301 -> U+1E83 */ 1251, + /* U+0077+0302 -> U+0175 */ 170, + /* U+0077+0307 -> U+1E87 */ 1255, + /* U+0077+0308 -> U+1E85 */ 1253, + /* U+0077+030A -> U+1E98 */ 1272, + /* U+0077+0323 -> U+1E89 */ 1257, + /* U+0078+0307 -> U+1E8B */ 1259, + /* U+0078+0308 -> U+1E8D */ 1261, + /* U+0079+0300 -> U+1EF3 */ 1359, + /* U+0079+0301 -> U+00FD */ 65, + /* U+0079+0302 -> U+0177 */ 172, + /* U+0079+0303 -> U+1EF9 */ 1365, + /* U+0079+0304 -> U+0233 */ 283, + /* U+0079+0307 -> U+1E8F */ 1263, + /* U+0079+0308 -> U+00FF */ 66, + /* U+0079+0309 -> U+1EF7 */ 1363, + /* U+0079+030A -> U+1E99 */ 1273, + /* U+0079+0323 -> U+1EF5 */ 1361, + /* U+007A+0301 -> U+017A */ 175, + /* U+007A+0302 -> U+1E91 */ 1265, + /* U+007A+0307 -> U+017C */ 177, + /* U+007A+030C -> U+017E */ 179, + /* U+007A+0323 -> U+1E93 */ 1267, + /* U+007A+0331 -> U+1E95 */ 1269, + /* U+00A8+0300 -> U+1FED */ 1584, + /* U+00A8+0301 -> U+0385 */ 419, + /* U+00A8+0342 -> U+1FC1 */ 1544, + /* U+00C2+0300 -> U+1EA6 */ 1282, + /* U+00C2+0301 -> U+1EA4 */ 1280, + /* U+00C2+0303 -> U+1EAA */ 1286, + /* U+00C2+0309 -> U+1EA8 */ 1284, + /* U+00C4+0304 -> U+01DE */ 210, + /* U+00C5+0301 -> U+01FA */ 234, + /* U+00C6+0301 -> U+01FC */ 236, + /* U+00C6+0304 -> U+01E2 */ 214, + /* U+00C7+0301 -> U+1E08 */ 1128, + /* U+00CA+0300 -> U+1EC0 */ 1308, + /* U+00CA+0301 -> U+1EBE */ 1306, + /* U+00CA+0303 -> U+1EC4 */ 1312, + /* U+00CA+0309 -> U+1EC2 */ 1310, + /* U+00CF+0301 -> U+1E2E */ 1166, + /* U+00D4+0300 -> U+1ED2 */ 1326, + /* U+00D4+0301 -> U+1ED0 */ 1324, + /* U+00D4+0303 -> U+1ED6 */ 1330, + /* U+00D4+0309 -> U+1ED4 */ 1328, + /* U+00D5+0301 -> U+1E4C */ 1196, + /* U+00D5+0304 -> U+022C */ 276, + /* U+00D5+0308 -> U+1E4E */ 1198, + /* U+00D6+0304 -> U+022A */ 274, + /* U+00D8+0301 -> U+01FE */ 238, + /* U+00DC+0300 -> U+01DB */ 208, + /* U+00DC+0301 -> U+01D7 */ 204, + /* U+00DC+0304 -> U+01D5 */ 202, + /* U+00DC+030C -> U+01D9 */ 206, + /* U+00E2+0300 -> U+1EA7 */ 1283, + /* U+00E2+0301 -> U+1EA5 */ 1281, + /* U+00E2+0303 -> U+1EAB */ 1287, + /* U+00E2+0309 -> U+1EA9 */ 1285, + /* U+00E4+0304 -> U+01DF */ 211, + /* U+00E5+0301 -> U+01FB */ 235, + /* U+00E6+0301 -> U+01FD */ 237, + /* U+00E6+0304 -> U+01E3 */ 215, + /* U+00E7+0301 -> U+1E09 */ 1129, + /* U+00EA+0300 -> U+1EC1 */ 1309, + /* U+00EA+0301 -> U+1EBF */ 1307, + /* U+00EA+0303 -> U+1EC5 */ 1313, + /* U+00EA+0309 -> U+1EC3 */ 1311, + /* U+00EF+0301 -> U+1E2F */ 1167, + /* U+00F4+0300 -> U+1ED3 */ 1327, + /* U+00F4+0301 -> U+1ED1 */ 1325, + /* U+00F4+0303 -> U+1ED7 */ 1331, + /* U+00F4+0309 -> U+1ED5 */ 1329, + /* U+00F5+0301 -> U+1E4D */ 1197, + /* U+00F5+0304 -> U+022D */ 277, + /* U+00F5+0308 -> U+1E4F */ 1199, + /* U+00F6+0304 -> U+022B */ 275, + /* U+00F8+0301 -> U+01FF */ 239, + /* U+00FC+0300 -> U+01DC */ 209, + /* U+00FC+0301 -> U+01D8 */ 205, + /* U+00FC+0304 -> U+01D6 */ 203, + /* U+00FC+030C -> U+01DA */ 207, + /* U+0102+0300 -> U+1EB0 */ 1292, + /* U+0102+0301 -> U+1EAE */ 1290, + /* U+0102+0303 -> U+1EB4 */ 1296, + /* U+0102+0309 -> U+1EB2 */ 1294, + /* U+0103+0300 -> U+1EB1 */ 1293, + /* U+0103+0301 -> U+1EAF */ 1291, + /* U+0103+0303 -> U+1EB5 */ 1297, + /* U+0103+0309 -> U+1EB3 */ 1295, + /* U+0112+0300 -> U+1E14 */ 1140, + /* U+0112+0301 -> U+1E16 */ 1142, + /* U+0113+0300 -> U+1E15 */ 1141, + /* U+0113+0301 -> U+1E17 */ 1143, + /* U+014C+0300 -> U+1E50 */ 1200, + /* U+014C+0301 -> U+1E52 */ 1202, + /* U+014D+0300 -> U+1E51 */ 1201, + /* U+014D+0301 -> U+1E53 */ 1203, + /* U+015A+0307 -> U+1E64 */ 1220, + /* U+015B+0307 -> U+1E65 */ 1221, + /* U+0160+0307 -> U+1E66 */ 1222, + /* U+0161+0307 -> U+1E67 */ 1223, + /* U+0168+0301 -> U+1E78 */ 1240, + /* U+0169+0301 -> U+1E79 */ 1241, + /* U+016A+0308 -> U+1E7A */ 1242, + /* U+016B+0308 -> U+1E7B */ 1243, + /* U+017F+0307 -> U+1E9B */ 1275, + /* U+01A0+0300 -> U+1EDC */ 1336, + /* U+01A0+0301 -> U+1EDA */ 1334, + /* U+01A0+0303 -> U+1EE0 */ 1340, + /* U+01A0+0309 -> U+1EDE */ 1338, + /* U+01A0+0323 -> U+1EE2 */ 1342, + /* U+01A1+0300 -> U+1EDD */ 1337, + /* U+01A1+0301 -> U+1EDB */ 1335, + /* U+01A1+0303 -> U+1EE1 */ 1341, + /* U+01A1+0309 -> U+1EDF */ 1339, + /* U+01A1+0323 -> U+1EE3 */ 1343, + /* U+01AF+0300 -> U+1EEA */ 1350, + /* U+01AF+0301 -> U+1EE8 */ 1348, + /* U+01AF+0303 -> U+1EEE */ 1354, + /* U+01AF+0309 -> U+1EEC */ 1352, + /* U+01AF+0323 -> U+1EF0 */ 1356, + /* U+01B0+0300 -> U+1EEB */ 1351, + /* U+01B0+0301 -> U+1EE9 */ 1349, + /* U+01B0+0303 -> U+1EEF */ 1355, + /* U+01B0+0309 -> U+1EED */ 1353, + /* U+01B0+0323 -> U+1EF1 */ 1357, + /* U+01B7+030C -> U+01EE */ 224, + /* U+01EA+0304 -> U+01EC */ 222, + /* U+01EB+0304 -> U+01ED */ 223, + /* U+0226+0304 -> U+01E0 */ 212, + /* U+0227+0304 -> U+01E1 */ 213, + /* U+0228+0306 -> U+1E1C */ 1148, + /* U+0229+0306 -> U+1E1D */ 1149, + /* U+022E+0304 -> U+0230 */ 280, + /* U+022F+0304 -> U+0231 */ 281, + /* U+0292+030C -> U+01EF */ 225, + /* U+0391+0300 -> U+1FBA */ 1537, + /* U+0391+0301 -> U+0386 */ 420, + /* U+0391+0304 -> U+1FB9 */ 1536, + /* U+0391+0306 -> U+1FB8 */ 1535, + /* U+0391+0313 -> U+1F08 */ 1374, + /* U+0391+0314 -> U+1F09 */ 1375, + /* U+0391+0345 -> U+1FBC */ 1539, + /* U+0395+0300 -> U+1FC8 */ 1550, + /* U+0395+0301 -> U+0388 */ 422, + /* U+0395+0313 -> U+1F18 */ 1388, + /* U+0395+0314 -> U+1F19 */ 1389, + /* U+0397+0300 -> U+1FCA */ 1552, + /* U+0397+0301 -> U+0389 */ 423, + /* U+0397+0313 -> U+1F28 */ 1402, + /* U+0397+0314 -> U+1F29 */ 1403, + /* U+0397+0345 -> U+1FCC */ 1554, + /* U+0399+0300 -> U+1FDA */ 1566, + /* U+0399+0301 -> U+038A */ 424, + /* U+0399+0304 -> U+1FD9 */ 1565, + /* U+0399+0306 -> U+1FD8 */ 1564, + /* U+0399+0308 -> U+03AA */ 429, + /* U+0399+0313 -> U+1F38 */ 1418, + /* U+0399+0314 -> U+1F39 */ 1419, + /* U+039F+0300 -> U+1FF8 */ 1592, + /* U+039F+0301 -> U+038C */ 425, + /* U+039F+0313 -> U+1F48 */ 1432, + /* U+039F+0314 -> U+1F49 */ 1433, + /* U+03A1+0314 -> U+1FEC */ 1583, + /* U+03A5+0300 -> U+1FEA */ 1581, + /* U+03A5+0301 -> U+038E */ 426, + /* U+03A5+0304 -> U+1FE9 */ 1580, + /* U+03A5+0306 -> U+1FE8 */ 1579, + /* U+03A5+0308 -> U+03AB */ 430, + /* U+03A5+0314 -> U+1F59 */ 1446, + /* U+03A9+0300 -> U+1FFA */ 1594, + /* U+03A9+0301 -> U+038F */ 427, + /* U+03A9+0313 -> U+1F68 */ 1458, + /* U+03A9+0314 -> U+1F69 */ 1459, + /* U+03A9+0345 -> U+1FFC */ 1596, + /* U+03AC+0345 -> U+1FB4 */ 1532, + /* U+03AE+0345 -> U+1FC4 */ 1547, + /* U+03B1+0300 -> U+1F70 */ 1466, + /* U+03B1+0301 -> U+03AC */ 431, + /* U+03B1+0304 -> U+1FB1 */ 1529, + /* U+03B1+0306 -> U+1FB0 */ 1528, + /* U+03B1+0313 -> U+1F00 */ 1366, + /* U+03B1+0314 -> U+1F01 */ 1367, + /* U+03B1+0342 -> U+1FB6 */ 1533, + /* U+03B1+0345 -> U+1FB3 */ 1531, + /* U+03B5+0300 -> U+1F72 */ 1468, + /* U+03B5+0301 -> U+03AD */ 432, + /* U+03B5+0313 -> U+1F10 */ 1382, + /* U+03B5+0314 -> U+1F11 */ 1383, + /* U+03B7+0300 -> U+1F74 */ 1470, + /* U+03B7+0301 -> U+03AE */ 433, + /* U+03B7+0313 -> U+1F20 */ 1394, + /* U+03B7+0314 -> U+1F21 */ 1395, + /* U+03B7+0342 -> U+1FC6 */ 1548, + /* U+03B7+0345 -> U+1FC3 */ 1546, + /* U+03B9+0300 -> U+1F76 */ 1472, + /* U+03B9+0301 -> U+03AF */ 434, + /* U+03B9+0304 -> U+1FD1 */ 1559, + /* U+03B9+0306 -> U+1FD0 */ 1558, + /* U+03B9+0308 -> U+03CA */ 436, + /* U+03B9+0313 -> U+1F30 */ 1410, + /* U+03B9+0314 -> U+1F31 */ 1411, + /* U+03B9+0342 -> U+1FD6 */ 1562, + /* U+03BF+0300 -> U+1F78 */ 1474, + /* U+03BF+0301 -> U+03CC */ 438, + /* U+03BF+0313 -> U+1F40 */ 1426, + /* U+03BF+0314 -> U+1F41 */ 1427, + /* U+03C1+0313 -> U+1FE4 */ 1575, + /* U+03C1+0314 -> U+1FE5 */ 1576, + /* U+03C5+0300 -> U+1F7A */ 1476, + /* U+03C5+0301 -> U+03CD */ 439, + /* U+03C5+0304 -> U+1FE1 */ 1572, + /* U+03C5+0306 -> U+1FE0 */ 1571, + /* U+03C5+0308 -> U+03CB */ 437, + /* U+03C5+0313 -> U+1F50 */ 1438, + /* U+03C5+0314 -> U+1F51 */ 1439, + /* U+03C5+0342 -> U+1FE6 */ 1577, + /* U+03C9+0300 -> U+1F7C */ 1478, + /* U+03C9+0301 -> U+03CE */ 440, + /* U+03C9+0313 -> U+1F60 */ 1450, + /* U+03C9+0314 -> U+1F61 */ 1451, + /* U+03C9+0342 -> U+1FF6 */ 1590, + /* U+03C9+0345 -> U+1FF3 */ 1588, + /* U+03CA+0300 -> U+1FD2 */ 1560, + /* U+03CA+0301 -> U+0390 */ 428, + /* U+03CA+0342 -> U+1FD7 */ 1563, + /* U+03CB+0300 -> U+1FE2 */ 1573, + /* U+03CB+0301 -> U+03B0 */ 435, + /* U+03CB+0342 -> U+1FE7 */ 1578, + /* U+03CE+0345 -> U+1FF4 */ 1589, + /* U+03D2+0301 -> U+03D3 */ 444, + /* U+03D2+0308 -> U+03D4 */ 445, + /* U+0406+0308 -> U+0407 */ 457, + /* U+0410+0306 -> U+04D0 */ 479, + /* U+0410+0308 -> U+04D2 */ 481, + /* U+0413+0301 -> U+0403 */ 456, + /* U+0415+0300 -> U+0400 */ 454, + /* U+0415+0306 -> U+04D6 */ 483, + /* U+0415+0308 -> U+0401 */ 455, + /* U+0416+0306 -> U+04C1 */ 477, + /* U+0416+0308 -> U+04DC */ 487, + /* U+0417+0308 -> U+04DE */ 489, + /* U+0418+0300 -> U+040D */ 459, + /* U+0418+0304 -> U+04E2 */ 491, + /* U+0418+0306 -> U+0419 */ 461, + /* U+0418+0308 -> U+04E4 */ 493, + /* U+041A+0301 -> U+040C */ 458, + /* U+041E+0308 -> U+04E6 */ 495, + /* U+0423+0304 -> U+04EE */ 501, + /* U+0423+0306 -> U+040E */ 460, + /* U+0423+0308 -> U+04F0 */ 503, + /* U+0423+030B -> U+04F2 */ 505, + /* U+0427+0308 -> U+04F4 */ 507, + /* U+042B+0308 -> U+04F8 */ 509, + /* U+042D+0308 -> U+04EC */ 499, + /* U+0430+0306 -> U+04D1 */ 480, + /* U+0430+0308 -> U+04D3 */ 482, + /* U+0433+0301 -> U+0453 */ 465, + /* U+0435+0300 -> U+0450 */ 463, + /* U+0435+0306 -> U+04D7 */ 484, + /* U+0435+0308 -> U+0451 */ 464, + /* U+0436+0306 -> U+04C2 */ 478, + /* U+0436+0308 -> U+04DD */ 488, + /* U+0437+0308 -> U+04DF */ 490, + /* U+0438+0300 -> U+045D */ 468, + /* U+0438+0304 -> U+04E3 */ 492, + /* U+0438+0306 -> U+0439 */ 462, + /* U+0438+0308 -> U+04E5 */ 494, + /* U+043A+0301 -> U+045C */ 467, + /* U+043E+0308 -> U+04E7 */ 496, + /* U+0443+0304 -> U+04EF */ 502, + /* U+0443+0306 -> U+045E */ 469, + /* U+0443+0308 -> U+04F1 */ 504, + /* U+0443+030B -> U+04F3 */ 506, + /* U+0447+0308 -> U+04F5 */ 508, + /* U+044B+0308 -> U+04F9 */ 510, + /* U+044D+0308 -> U+04ED */ 500, + /* U+0456+0308 -> U+0457 */ 466, + /* U+0474+030F -> U+0476 */ 470, + /* U+0475+030F -> U+0477 */ 471, + /* U+04D8+0308 -> U+04DA */ 485, + /* U+04D9+0308 -> U+04DB */ 486, + /* U+04E8+0308 -> U+04EA */ 497, + /* U+04E9+0308 -> U+04EB */ 498, + /* U+0627+0653 -> U+0622 */ 574, + /* U+0627+0654 -> U+0623 */ 575, + /* U+0627+0655 -> U+0625 */ 577, + /* U+0648+0654 -> U+0624 */ 576, + /* U+064A+0654 -> U+0626 */ 578, + /* U+06C1+0654 -> U+06C2 */ 606, + /* U+06D2+0654 -> U+06D3 */ 607, + /* U+06D5+0654 -> U+06C0 */ 605, + /* U+0928+093C -> U+0929 */ 733, + /* U+0930+093C -> U+0931 */ 734, + /* U+0933+093C -> U+0934 */ 735, + /* U+09C7+09BE -> U+09CB */ 751, + /* U+09C7+09D7 -> U+09CC */ 752, + /* U+0B47+0B3E -> U+0B4B */ 770, + /* U+0B47+0B56 -> U+0B48 */ 769, + /* U+0B47+0B57 -> U+0B4C */ 771, + /* U+0B92+0BD7 -> U+0B94 */ 775, + /* U+0BC6+0BBE -> U+0BCA */ 776, + /* U+0BC6+0BD7 -> U+0BCC */ 778, + /* U+0BC7+0BBE -> U+0BCB */ 777, + /* U+0C46+0C56 -> U+0C48 */ 780, + /* U+0CBF+0CD5 -> U+0CC0 */ 785, + /* U+0CC6+0CC2 -> U+0CCA */ 788, + /* U+0CC6+0CD5 -> U+0CC7 */ 786, + /* U+0CC6+0CD6 -> U+0CC8 */ 787, + /* U+0CCA+0CD5 -> U+0CCB */ 789, + /* U+0D46+0D3E -> U+0D4A */ 793, + /* U+0D46+0D57 -> U+0D4C */ 795, + /* U+0D47+0D3E -> U+0D4B */ 794, + /* U+0DD9+0DCA -> U+0DDA */ 798, + /* U+0DD9+0DCF -> U+0DDC */ 799, + /* U+0DD9+0DDF -> U+0DDE */ 801, + /* U+0DDC+0DCA -> U+0DDD */ 800, + /* U+1025+102E -> U+1026 */ 859, + /* U+1B05+1B35 -> U+1B06 */ 904, + /* U+1B07+1B35 -> U+1B08 */ 905, + /* U+1B09+1B35 -> U+1B0A */ 906, + /* U+1B0B+1B35 -> U+1B0C */ 907, + /* U+1B0D+1B35 -> U+1B0E */ 908, + /* U+1B11+1B35 -> U+1B12 */ 909, + /* U+1B3A+1B35 -> U+1B3B */ 911, + /* U+1B3C+1B35 -> U+1B3D */ 912, + /* U+1B3E+1B35 -> U+1B40 */ 913, + /* U+1B3F+1B35 -> U+1B41 */ 914, + /* U+1B42+1B35 -> U+1B43 */ 915, + /* U+1E36+0304 -> U+1E38 */ 1176, + /* U+1E37+0304 -> U+1E39 */ 1177, + /* U+1E5A+0304 -> U+1E5C */ 1212, + /* U+1E5B+0304 -> U+1E5D */ 1213, + /* U+1E62+0307 -> U+1E68 */ 1224, + /* U+1E63+0307 -> U+1E69 */ 1225, + /* U+1EA0+0302 -> U+1EAC */ 1288, + /* U+1EA0+0306 -> U+1EB6 */ 1298, + /* U+1EA1+0302 -> U+1EAD */ 1289, + /* U+1EA1+0306 -> U+1EB7 */ 1299, + /* U+1EB8+0302 -> U+1EC6 */ 1314, + /* U+1EB9+0302 -> U+1EC7 */ 1315, + /* U+1ECC+0302 -> U+1ED8 */ 1332, + /* U+1ECD+0302 -> U+1ED9 */ 1333, + /* U+1F00+0300 -> U+1F02 */ 1368, + /* U+1F00+0301 -> U+1F04 */ 1370, + /* U+1F00+0342 -> U+1F06 */ 1372, + /* U+1F00+0345 -> U+1F80 */ 1480, + /* U+1F01+0300 -> U+1F03 */ 1369, + /* U+1F01+0301 -> U+1F05 */ 1371, + /* U+1F01+0342 -> U+1F07 */ 1373, + /* U+1F01+0345 -> U+1F81 */ 1481, + /* U+1F02+0345 -> U+1F82 */ 1482, + /* U+1F03+0345 -> U+1F83 */ 1483, + /* U+1F04+0345 -> U+1F84 */ 1484, + /* U+1F05+0345 -> U+1F85 */ 1485, + /* U+1F06+0345 -> U+1F86 */ 1486, + /* U+1F07+0345 -> U+1F87 */ 1487, + /* U+1F08+0300 -> U+1F0A */ 1376, + /* U+1F08+0301 -> U+1F0C */ 1378, + /* U+1F08+0342 -> U+1F0E */ 1380, + /* U+1F08+0345 -> U+1F88 */ 1488, + /* U+1F09+0300 -> U+1F0B */ 1377, + /* U+1F09+0301 -> U+1F0D */ 1379, + /* U+1F09+0342 -> U+1F0F */ 1381, + /* U+1F09+0345 -> U+1F89 */ 1489, + /* U+1F0A+0345 -> U+1F8A */ 1490, + /* U+1F0B+0345 -> U+1F8B */ 1491, + /* U+1F0C+0345 -> U+1F8C */ 1492, + /* U+1F0D+0345 -> U+1F8D */ 1493, + /* U+1F0E+0345 -> U+1F8E */ 1494, + /* U+1F0F+0345 -> U+1F8F */ 1495, + /* U+1F10+0300 -> U+1F12 */ 1384, + /* U+1F10+0301 -> U+1F14 */ 1386, + /* U+1F11+0300 -> U+1F13 */ 1385, + /* U+1F11+0301 -> U+1F15 */ 1387, + /* U+1F18+0300 -> U+1F1A */ 1390, + /* U+1F18+0301 -> U+1F1C */ 1392, + /* U+1F19+0300 -> U+1F1B */ 1391, + /* U+1F19+0301 -> U+1F1D */ 1393, + /* U+1F20+0300 -> U+1F22 */ 1396, + /* U+1F20+0301 -> U+1F24 */ 1398, + /* U+1F20+0342 -> U+1F26 */ 1400, + /* U+1F20+0345 -> U+1F90 */ 1496, + /* U+1F21+0300 -> U+1F23 */ 1397, + /* U+1F21+0301 -> U+1F25 */ 1399, + /* U+1F21+0342 -> U+1F27 */ 1401, + /* U+1F21+0345 -> U+1F91 */ 1497, + /* U+1F22+0345 -> U+1F92 */ 1498, + /* U+1F23+0345 -> U+1F93 */ 1499, + /* U+1F24+0345 -> U+1F94 */ 1500, + /* U+1F25+0345 -> U+1F95 */ 1501, + /* U+1F26+0345 -> U+1F96 */ 1502, + /* U+1F27+0345 -> U+1F97 */ 1503, + /* U+1F28+0300 -> U+1F2A */ 1404, + /* U+1F28+0301 -> U+1F2C */ 1406, + /* U+1F28+0342 -> U+1F2E */ 1408, + /* U+1F28+0345 -> U+1F98 */ 1504, + /* U+1F29+0300 -> U+1F2B */ 1405, + /* U+1F29+0301 -> U+1F2D */ 1407, + /* U+1F29+0342 -> U+1F2F */ 1409, + /* U+1F29+0345 -> U+1F99 */ 1505, + /* U+1F2A+0345 -> U+1F9A */ 1506, + /* U+1F2B+0345 -> U+1F9B */ 1507, + /* U+1F2C+0345 -> U+1F9C */ 1508, + /* U+1F2D+0345 -> U+1F9D */ 1509, + /* U+1F2E+0345 -> U+1F9E */ 1510, + /* U+1F2F+0345 -> U+1F9F */ 1511, + /* U+1F30+0300 -> U+1F32 */ 1412, + /* U+1F30+0301 -> U+1F34 */ 1414, + /* U+1F30+0342 -> U+1F36 */ 1416, + /* U+1F31+0300 -> U+1F33 */ 1413, + /* U+1F31+0301 -> U+1F35 */ 1415, + /* U+1F31+0342 -> U+1F37 */ 1417, + /* U+1F38+0300 -> U+1F3A */ 1420, + /* U+1F38+0301 -> U+1F3C */ 1422, + /* U+1F38+0342 -> U+1F3E */ 1424, + /* U+1F39+0300 -> U+1F3B */ 1421, + /* U+1F39+0301 -> U+1F3D */ 1423, + /* U+1F39+0342 -> U+1F3F */ 1425, + /* U+1F40+0300 -> U+1F42 */ 1428, + /* U+1F40+0301 -> U+1F44 */ 1430, + /* U+1F41+0300 -> U+1F43 */ 1429, + /* U+1F41+0301 -> U+1F45 */ 1431, + /* U+1F48+0300 -> U+1F4A */ 1434, + /* U+1F48+0301 -> U+1F4C */ 1436, + /* U+1F49+0300 -> U+1F4B */ 1435, + /* U+1F49+0301 -> U+1F4D */ 1437, + /* U+1F50+0300 -> U+1F52 */ 1440, + /* U+1F50+0301 -> U+1F54 */ 1442, + /* U+1F50+0342 -> U+1F56 */ 1444, + /* U+1F51+0300 -> U+1F53 */ 1441, + /* U+1F51+0301 -> U+1F55 */ 1443, + /* U+1F51+0342 -> U+1F57 */ 1445, + /* U+1F59+0300 -> U+1F5B */ 1447, + /* U+1F59+0301 -> U+1F5D */ 1448, + /* U+1F59+0342 -> U+1F5F */ 1449, + /* U+1F60+0300 -> U+1F62 */ 1452, + /* U+1F60+0301 -> U+1F64 */ 1454, + /* U+1F60+0342 -> U+1F66 */ 1456, + /* U+1F60+0345 -> U+1FA0 */ 1512, + /* U+1F61+0300 -> U+1F63 */ 1453, + /* U+1F61+0301 -> U+1F65 */ 1455, + /* U+1F61+0342 -> U+1F67 */ 1457, + /* U+1F61+0345 -> U+1FA1 */ 1513, + /* U+1F62+0345 -> U+1FA2 */ 1514, + /* U+1F63+0345 -> U+1FA3 */ 1515, + /* U+1F64+0345 -> U+1FA4 */ 1516, + /* U+1F65+0345 -> U+1FA5 */ 1517, + /* U+1F66+0345 -> U+1FA6 */ 1518, + /* U+1F67+0345 -> U+1FA7 */ 1519, + /* U+1F68+0300 -> U+1F6A */ 1460, + /* U+1F68+0301 -> U+1F6C */ 1462, + /* U+1F68+0342 -> U+1F6E */ 1464, + /* U+1F68+0345 -> U+1FA8 */ 1520, + /* U+1F69+0300 -> U+1F6B */ 1461, + /* U+1F69+0301 -> U+1F6D */ 1463, + /* U+1F69+0342 -> U+1F6F */ 1465, + /* U+1F69+0345 -> U+1FA9 */ 1521, + /* U+1F6A+0345 -> U+1FAA */ 1522, + /* U+1F6B+0345 -> U+1FAB */ 1523, + /* U+1F6C+0345 -> U+1FAC */ 1524, + /* U+1F6D+0345 -> U+1FAD */ 1525, + /* U+1F6E+0345 -> U+1FAE */ 1526, + /* U+1F6F+0345 -> U+1FAF */ 1527, + /* U+1F70+0345 -> U+1FB2 */ 1530, + /* U+1F74+0345 -> U+1FC2 */ 1545, + /* U+1F7C+0345 -> U+1FF2 */ 1587, + /* U+1FB6+0345 -> U+1FB7 */ 1534, + /* U+1FBF+0300 -> U+1FCD */ 1555, + /* U+1FBF+0301 -> U+1FCE */ 1556, + /* U+1FBF+0342 -> U+1FCF */ 1557, + /* U+1FC6+0345 -> U+1FC7 */ 1549, + /* U+1FF6+0345 -> U+1FF7 */ 1591, + /* U+1FFE+0300 -> U+1FDD */ 1568, + /* U+1FFE+0301 -> U+1FDE */ 1569, + /* U+1FFE+0342 -> U+1FDF */ 1570, + /* U+2190+0338 -> U+219A */ 1801, + /* U+2192+0338 -> U+219B */ 1802, + /* U+2194+0338 -> U+21AE */ 1803, + /* U+21D0+0338 -> U+21CD */ 1804, + /* U+21D2+0338 -> U+21CF */ 1806, + /* U+21D4+0338 -> U+21CE */ 1805, + /* U+2203+0338 -> U+2204 */ 1807, + /* U+2208+0338 -> U+2209 */ 1808, + /* U+220B+0338 -> U+220C */ 1809, + /* U+2223+0338 -> U+2224 */ 1810, + /* U+2225+0338 -> U+2226 */ 1811, + /* U+223C+0338 -> U+2241 */ 1816, + /* U+2243+0338 -> U+2244 */ 1817, + /* U+2245+0338 -> U+2247 */ 1818, + /* U+2248+0338 -> U+2249 */ 1819, + /* U+224D+0338 -> U+226D */ 1822, + /* U+2261+0338 -> U+2262 */ 1821, + /* U+2264+0338 -> U+2270 */ 1825, + /* U+2265+0338 -> U+2271 */ 1826, + /* U+2272+0338 -> U+2274 */ 1827, + /* U+2273+0338 -> U+2275 */ 1828, + /* U+2276+0338 -> U+2278 */ 1829, + /* U+2277+0338 -> U+2279 */ 1830, + /* U+227A+0338 -> U+2280 */ 1831, + /* U+227B+0338 -> U+2281 */ 1832, + /* U+227C+0338 -> U+22E0 */ 1841, + /* U+227D+0338 -> U+22E1 */ 1842, + /* U+2282+0338 -> U+2284 */ 1833, + /* U+2283+0338 -> U+2285 */ 1834, + /* U+2286+0338 -> U+2288 */ 1835, + /* U+2287+0338 -> U+2289 */ 1836, + /* U+2291+0338 -> U+22E2 */ 1843, + /* U+2292+0338 -> U+22E3 */ 1844, + /* U+22A2+0338 -> U+22AC */ 1837, + /* U+22A8+0338 -> U+22AD */ 1838, + /* U+22A9+0338 -> U+22AE */ 1839, + /* U+22AB+0338 -> U+22AF */ 1840, + /* U+22B2+0338 -> U+22EA */ 1845, + /* U+22B3+0338 -> U+22EB */ 1846, + /* U+22B4+0338 -> U+22EC */ 1847, + /* U+22B5+0338 -> U+22ED */ 1848, + /* U+3046+3099 -> U+3094 */ 2286, + /* U+304B+3099 -> U+304C */ 2261, + /* U+304D+3099 -> U+304E */ 2262, + /* U+304F+3099 -> U+3050 */ 2263, + /* U+3051+3099 -> U+3052 */ 2264, + /* U+3053+3099 -> U+3054 */ 2265, + /* U+3055+3099 -> U+3056 */ 2266, + /* U+3057+3099 -> U+3058 */ 2267, + /* U+3059+3099 -> U+305A */ 2268, + /* U+305B+3099 -> U+305C */ 2269, + /* U+305D+3099 -> U+305E */ 2270, + /* U+305F+3099 -> U+3060 */ 2271, + /* U+3061+3099 -> U+3062 */ 2272, + /* U+3064+3099 -> U+3065 */ 2273, + /* U+3066+3099 -> U+3067 */ 2274, + /* U+3068+3099 -> U+3069 */ 2275, + /* U+306F+3099 -> U+3070 */ 2276, + /* U+306F+309A -> U+3071 */ 2277, + /* U+3072+3099 -> U+3073 */ 2278, + /* U+3072+309A -> U+3074 */ 2279, + /* U+3075+3099 -> U+3076 */ 2280, + /* U+3075+309A -> U+3077 */ 2281, + /* U+3078+3099 -> U+3079 */ 2282, + /* U+3078+309A -> U+307A */ 2283, + /* U+307B+3099 -> U+307C */ 2284, + /* U+307B+309A -> U+307D */ 2285, + /* U+309D+3099 -> U+309E */ 2291, + /* U+30A6+3099 -> U+30F4 */ 2318, + /* U+30AB+3099 -> U+30AC */ 2293, + /* U+30AD+3099 -> U+30AE */ 2294, + /* U+30AF+3099 -> U+30B0 */ 2295, + /* U+30B1+3099 -> U+30B2 */ 2296, + /* U+30B3+3099 -> U+30B4 */ 2297, + /* U+30B5+3099 -> U+30B6 */ 2298, + /* U+30B7+3099 -> U+30B8 */ 2299, + /* U+30B9+3099 -> U+30BA */ 2300, + /* U+30BB+3099 -> U+30BC */ 2301, + /* U+30BD+3099 -> U+30BE */ 2302, + /* U+30BF+3099 -> U+30C0 */ 2303, + /* U+30C1+3099 -> U+30C2 */ 2304, + /* U+30C4+3099 -> U+30C5 */ 2305, + /* U+30C6+3099 -> U+30C7 */ 2306, + /* U+30C8+3099 -> U+30C9 */ 2307, + /* U+30CF+3099 -> U+30D0 */ 2308, + /* U+30CF+309A -> U+30D1 */ 2309, + /* U+30D2+3099 -> U+30D3 */ 2310, + /* U+30D2+309A -> U+30D4 */ 2311, + /* U+30D5+3099 -> U+30D6 */ 2312, + /* U+30D5+309A -> U+30D7 */ 2313, + /* U+30D8+3099 -> U+30D9 */ 2314, + /* U+30D8+309A -> U+30DA */ 2315, + /* U+30DB+3099 -> U+30DC */ 2316, + /* U+30DB+309A -> U+30DD */ 2317, + /* U+30EF+3099 -> U+30F7 */ 2319, + /* U+30F0+3099 -> U+30F8 */ 2320, + /* U+30F1+3099 -> U+30F9 */ 2321, + /* U+30F2+3099 -> U+30FA */ 2322, + /* U+30FD+3099 -> U+30FE */ 2323, + /* U+11099+110BA -> U+1109A */ 4588, + /* U+1109B+110BA -> U+1109C */ 4589, + /* U+110A5+110BA -> U+110AB */ 4590, + /* U+11131+11127 -> U+1112E */ 4596, + /* U+11132+11127 -> U+1112F */ 4597, + /* U+11347+1133E -> U+1134B */ 4609, + /* U+11347+11357 -> U+1134C */ 4610, + /* U+114B9+114B0 -> U+114BC */ 4628, + /* U+114B9+114BA -> U+114BB */ 4627, + /* U+114B9+114BD -> U+114BE */ 4629, + /* U+115B8+115AF -> U+115BA */ 4632, + /* U+115B9+115AF -> U+115BB */ 4633, + /* U+11935+11930 -> U+11938 */ 4642 +}; + +/* Perfect hash function for recomposition */ +static int +Recomp_hash_func(const void *key) +{ + static const int16 h[1883] = { + 772, 773, 621, 32767, 32767, 387, 653, 196, + 32767, 32767, 855, 463, -19, 651, 32767, 32767, + 32767, 364, 32767, 32767, -108, 32767, 32767, 32767, + 32767, 0, -568, 32767, 32767, 32767, 0, 0, + 0, -103, 364, 0, 210, 732, 0, 0, + -506, 0, 0, 0, 32767, 32767, 0, 32767, + 407, -140, 32767, 409, 32767, 772, 0, 86, + 842, 934, 32767, 32767, -499, -355, 32767, 32767, + 532, 138, 174, -243, 860, 1870, 742, 32767, + 32767, 339, 32767, 1290, 0, 32767, 32767, 0, + -449, -1386, 1633, 560, 561, 32767, 1219, 1004, + 139, -804, 32767, -179, 141, 579, 1586, 32767, + 32767, 32767, 142, 199, 32767, 32767, 143, 0, + 32767, 32767, 314, 896, 32767, 32767, 428, 129, + 286, -58, 0, 68, 32767, 0, 244, -566, + 32767, 32767, 32767, 246, 32767, 32767, 0, 32767, + 32767, 271, -108, 928, 32767, 715, 32767, 32767, + -211, -497, 32767, 0, 1055, 1339, 32767, 0, + 32767, 32767, -968, -144, 32767, 32767, 248, 32767, + -161, 32767, 32767, 282, 32767, -372, 0, 2, + -137, 1116, 32767, 687, 32767, 459, 913, 0, + 461, 879, -816, 443, 32767, 32767, 462, 1089, + 32767, 1054, 0, 314, 447, -26, 480, 32767, + 64, 0, 0, 112, 32767, 66, 0, 646, + 603, 22, -292, 0, 710, 475, 32767, 24, + -781, 32767, 32767, 32767, 281, 307, 32767, 1289, + 32767, 0, 1064, -149, 454, 118, 32767, 32767, + 0, 32767, -126, 0, 32767, 32767, 858, 32767, + 32767, 32767, 1029, 886, 665, 209, 0, 26, + 359, 0, 0, -108, -508, -603, 894, 906, + 32767, 32767, 14, 0, 0, 534, 984, 876, + 32767, -93, 110, -367, 167, 843, 32767, 32767, + -947, -290, 169, 0, 0, 32767, -42, 564, + 0, -927, 32767, 817, 32767, 32767, 32767, 110, + 0, 32767, 32767, -38, 32767, 32767, -101, 694, + -142, 190, 191, 1288, 32767, -687, 194, -579, + 534, -452, 0, -72, 536, 765, 823, 266, + -259, 684, 767, 32767, 654, 32767, 32767, 64, + 920, 32767, 32767, 32767, 0, 1653, 0, 0, + 32767, 32767, -452, -222, 855, 0, 32767, -1153, + 127, 490, 449, 863, 32767, -144, 32767, -379, + 545, 32767, 32767, 32767, 530, 32767, 32767, 1331, + 611, -612, 332, 545, -73, 0, 604, 201, + 32767, -279, 338, 836, 340, 408, 32767, -60, + -358, 32767, 343, 69, 707, 0, -129, 582, + 32767, 0, 32767, 96, 392, 490, 639, 157, + -4, 406, 32767, 32767, -571, 1077, 546, 32767, + 551, 0, 0, 0, 32767, 32767, 348, 32767, + 498, -181, 0, -433, 1057, 260, 0, 32767, + 32767, 397, 32767, 816, -130, 32767, 624, 0, + 0, 32767, 32767, 32767, 485, 0, 32767, 32767, + 32767, 32767, 32767, 0, 32767, 32767, 32767, 1222, + -230, 32767, 797, -538, 32767, 974, 32767, 32767, + 831, 70, -658, 145, 0, 147, 0, 32767, + 1295, 32767, 0, 0, 895, 0, 0, -385, + 491, -287, 32767, -587, 32767, 32767, 32767, 813, + -471, -13, 32767, 32767, 32767, 0, 203, 411, + 470, 0, -546, -179, 146, 0, 0, 32767, + -468, 32767, 0, 0, 32767, 32767, 32767, 211, + 32767, 32767, 0, 32767, 0, 52, 32767, 0, + 32767, 0, 692, 990, 32767, 32767, 32767, 56, + -507, 784, 951, 0, 32767, 0, 697, 32767, + 187, 0, 32767, 32767, 430, 1209, 682, 32767, + 130, 0, -25, 0, -1006, 0, 32767, 214, + 433, 22, 0, -1119, 32767, 285, 32767, 32767, + 32767, 216, 32767, 32767, 32767, 217, 527, 32767, + 32767, 32767, 829, 485, 419, 717, 620, 731, + 32767, 470, 0, -145, -620, 1162, -644, 848, + 287, -632, 32767, 32767, 32767, 32767, 381, 32767, + 510, 511, -554, -2, 32767, 0, 0, 698, + 32767, 32767, 436, 1154, 32767, 463, 32767, 32767, + 627, 517, 32767, 32767, 854, 579, 723, 396, + 110, -42, 354, 32767, 664, 32767, 32767, 0, + 0, 32767, 65, -163, 67, 140, 69, 341, + 70, 71, 402, 73, 623, 544, 624, 417, + -1375, 648, 32767, -26, 904, 0, 548, 0, + 0, 32767, 32767, 855, 32767, 488, -524, 599, + 130, 131, 32767, 32767, 542, -1110, -324, -462, + 32767, -405, -440, 0, 0, 629, 850, 0, + 741, 257, 258, 32767, 32767, 0, 32767, 923, + 0, 32767, 0, 32767, 1559, 32767, 32767, 32767, + 671, 32767, 134, 32767, 32767, -336, -104, 576, + 577, 829, 32767, 32767, 762, 902, 32767, 0, + 32767, 0, 1506, 887, 32767, 636, 601, 2465, + 426, 0, 236, 317, 427, 968, 32767, -975, + -559, -343, 341, 32767, 937, 241, 0, 32767, + 32767, 547, 32767, 32767, 32767, 32767, 32767, 789, + 0, 32767, 32767, 32767, 0, 0, 0, 32767, + -192, 859, 1185, 1153, 69, 32767, 32767, 32767, + -539, 32767, 32767, 0, 32767, 32767, 32767, 32767, + 640, 578, 32767, 32767, -766, 32767, 32767, 32767, + 32767, 1050, -572, 32767, 32767, 32767, 32767, 1268, + 32767, 32767, 32767, 754, 32767, 32767, 1640, 179, + 804, 32767, 32767, 32767, 32767, 0, 684, 943, + 1006, 32767, 32767, 652, 0, 32767, 1041, 32767, + 718, 791, 32767, 274, 697, 32767, 32767, 0, + 32767, 32767, 32767, 0, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 735, + 0, 32767, 32767, 32767, 275, 358, 688, 32767, + 32767, 32767, 548, -87, 770, 32767, -42, 0, + 551, 32767, 691, 222, 32767, 32767, 32767, 32767, + 0, 1273, 403, -121, 806, 553, 554, 163, + 32767, 32767, 892, 825, 32767, 32767, -490, 32767, + 32767, 32767, 32767, 32767, -109, 744, 910, 32767, + 91, 32767, 32767, 0, 0, 32767, 32767, 32767, + 1521, 50, 701, 32767, 32767, 32767, 32767, 164, + 658, 32767, 288, 0, 32767, 0, 51, 0, + 32767, 32767, 32767, 32767, 555, 1547, 32767, 32767, + 595, 585, 429, 32767, -80, 32767, 1258, 0, + 540, 486, -434, 865, 0, 192, 0, 884, + 0, 0, 0, 175, 555, 0, 32767, 32767, + 0, 32767, -566, 866, 591, 32767, 32767, 32767, + 32767, 32767, 496, 495, -215, 32767, 849, -772, + 32767, 32767, 502, 178, 483, 32767, 912, 793, + 794, 0, 32767, 32767, 32767, -556, 499, 838, + 32767, 32767, -506, 331, 0, 0, -1096, 512, + 880, 0, 774, -338, 649, 32767, 270, 32767, + 32767, -624, 328, 459, 32767, 32767, 32767, 32767, + 329, -201, -835, 813, -879, 560, 0, -212, + -114, 35, -494, 37, 523, 653, 751, -653, + -743, 32767, 1356, 818, 32767, 32767, 856, 0, + 44, 902, 0, 0, 0, 0, 32767, -26, + 526, 795, 456, 32767, 104, -209, -341, 133, + -372, 0, 45, 110, 111, 0, 511, 47, + 114, 32767, 32767, 93, 48, 116, -1031, -279, + 32767, 192, 0, 32767, 453, 415, 0, -190, + 32767, 471, 240, 175, 29, 665, 684, 0, + -11, -95, -344, 32767, 245, 148, 0, 530, + 0, 1185, -615, -712, 693, 784, 32767, 0, + -776, 32767, 32767, -813, 0, 0, 0, 207, + 208, 32767, 674, 32767, 742, -289, 249, 32767, + 520, 929, -50, 781, 0, -778, 32767, 0, + 302, 32767, 720, -465, 0, 32767, 32767, 32767, + 0, 0, 32767, 833, 328, 806, 32767, -403, + 0, 32767, -77, 32767, 0, 441, 930, 32767, + 643, 0, 32767, 1938, 0, 1334, 381, 32767, + 216, 32767, 32767, 0, 32767, 484, 383, 0, + 242, 395, 0, 32767, 32767, 32767, -781, 355, + 356, 32767, 292, 706, 32767, 32767, 32767, 32767, + 32767, -410, 32767, 32767, 782, 32767, 189, 32767, + 32767, 943, 0, -212, 407, 335, 0, 135, + 32767, 616, 0, -497, 0, -67, 853, 32767, + 700, 32767, 0, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 0, 459, -48, 32767, 58, 0, + -856, 1017, 32767, 59, 916, -731, 32767, 940, + -855, 347, 650, 0, 678, 32767, 0, 32767, + 32767, 530, 32767, 0, -80, 32767, -730, 32767, + 1214, 799, 58, 651, 841, 0, 0, -589, + -1530, -478, 651, 652, 93, 576, -1215, 32767, + 125, 32767, 1279, 32767, 32767, 0, 32767, 0, + -367, 416, -1236, 32767, 418, 32767, 815, 558, + 559, 781, 419, 32767, 739, 32767, 0, 32767, + 128, 570, 1349, -298, -66, 0, 147, -488, + 32767, 590, 189, 274, 524, 32767, 1082, -209, + 32767, 423, 32767, 32767, 975, 573, 32767, 424, + 32767, 32767, 1241, 32767, 32767, 32767, 32767, 32767, + 612, 391, 32767, 0, -803, 1004, -561, 32767, + 32767, 735, 870, 32767, 0, 32767, 32767, -123, + 99, 210, 600, 1294, 109, 1053, 32767, 307, + 834, 32767, 0, 1651, 32767, 644, 32767, 32767, + 0, 32767, -801, 385, 379, 32767, -368, 32767, + 32767, 830, 0, 32767, 32767, 739, 371, 372, + -275, 32767, 32767, 331, -780, 32767, 0, 1229, + -1462, 913, 266, 827, 125, 32767, 32767, 32767, + 393, 32767, 631, -33, -883, -661, -204, 6, + -19, 257, 8, 9, 118, 519, 615, -541, + -893, 0, 32767, 0, 1156, 15, 900, 32767, + 32767, 32767, 32767, 32767, 32767, 1022, 376, 0, + 32767, 32767, -972, 676, 840, -661, 631, 58, + 0, 17, 32767, 0, -799, 82, 0, 32767, + 32767, 680, 32767, 905, 0, 0, 32767, 32767, + 0, 0, 32767, 0, 828, 386, 802, 0, + 146, 0, 148, 32767, -1146, 0, 150, 151, + -743, 153, 154, 32767, 32767, 442, 32767, 743, + 0, 0, 746, 0, 32767, 32767, 32767, 98, + 32767, 157, 0, 696, 0, 32767, 32767, -294, + 32767, 158, 159, 32767, 0, 32767, 160, 32767, + 933, 32767, 32767, -50, 759, 824, 162, 672, + 32767, 356, 0, 356, 32767, 32767, 0, 0, + 656, 692, 253, 254, -374, 102, 256, 32767, + 0, 0, 32767, 32767, 259, 32767, 63, 260, + 510, 261, 32767, 0, 32767, 1061, 32767, 521, + 32767, 32767, 32767, 32767, 32767, 32767, 316, 317, + 846, 0, 32767, -500, 318, 0, 32767, 32767, + 263, 0, 790, 872, 32767, 32767, 32767, 2171, + 264, 32767, 32767, 32767, 32767, 486, 334, 465, + 32767, 466, 32767, 444, 606, 32767, 0, 445, + 320, -317, 0, 520, 322, 718, 32767, 32767, + 32767, 0, 1013, 32767, 32767, 32767, 32767, 32767, + 32767, 611, 32767, 0, 0, 32767, 32767, -120, + 156, 613, 0, 0, 32767, -68, 32767, 622, + 32767, 32767, 32767, 32767, 32767, 455, 32767, 32767, + 32767, 403, 533, 0, -161, 405, 95, 96, + 32767, 97, 32767, 0, 29, 0, 32767, 32767, + 30, 32767, 99, 32767, 32767, 0, 161, 32767, + 97, 0, 32, 32767, 32767, 0, 0, 315, + 32767, 32767, 414, 966, 0, 585, 32767, 32767, + -616, -256, 171, 172, 666, 101, 562, 563, + 32767, 95, 0, 0, 1492, 390, -251, 103, + 32767, 0, 32767, 188, 1487, 32767, 0, 0, + 586, 668, -126, 0, 0, 32767, 32767, 204, + 32767, 32767, 32767, 32767, 32767, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 32767, 656, 32767, 32767, + 599, 0, 222, 32767, 0, 1368, -412, 435, + 32767, 936, 32767, -17, 32767, 832, 32767, 437, + 0, -518, 787, 32767, 864, -449, 0, 636, + 713, 206, 592, 572, 0, 483, -139, 32767, + 32767, 180, 818, 32767, 32767, 1304, 0, 32767, + 274, 0, 0, 0, 0, 705, 32767, 32767, + 32767, 0, -272, 0, 502, 503, 319, 0, + 32767, 0, 13, 32767, 32767, 0, 32767, 270, + 737, 0, 32767, 32767, 32767, 901, 32767, 616, + 180, 32767, 721, 353, 32767, 0, 32767, 32767, + -199, 0, 280, 788, 32767, 940, 32767, 51, + 0, 400, 53, 0, 54, -637, 0, -453, + 0, 0, 0, 380, 0, 32767, 504, 0, + 2049, 0, -964, 32767, 0, 32767, 32767, 32767, + 32767, 32767, 32767, 798, 32767, 32767, 32767, 0, + 538, 488, 0, 32767, -528, 57, 819, 32767, + 32767, 1244, 0, 488, 739, 908, 32767, 32767, + 0, 32767, 32767, 0, 55, 533, 0, 32767, + 814, 0, 32767, 458, 0, 32767, 32767, 32767, + 32767, 32767, 32767, 32767, 776, 777, 920, 0, + 0, 755, 32767, 0, 32767, 32767, 0, 32767, + 55, -954, 0, 372, 166, 218, 165, 857, + 221, 675, 0, 223, 224, -155, 226, 32767, + 1851, 227, 32767, 32767, 1192, 0, 229, 0, + -72, 0, 865, 0, 0, -330, 0, 683, + 32767, -550, -196, 725, -573, 293, 102, 32767, + -589, 296, 297, 298, 231, -256, 300, 32767, + 32767, 301, 233, 868, 32767, 234, 0, 811, + 1187, 32767, 32767, 0, 32767, 518, 0, 361, + 362, 466, 0, 365, 32767, -179, 366, 367, + 874, 369, 305, 0, 32767, 0, 32767, 0, + 32767, 2000, 1215, 451, 652, 0, 0, 799, + 32767, 32767, 32767 + }; + + const unsigned char *k = (const unsigned char *) key; + size_t keylen = 8; + uint32 a = 0; + uint32 b = 0; + + while (keylen--) + { + unsigned char c = *k++; + + a = a * 257 + c; + b = b * 17 + c; + } + return h[a % 1883] + h[b % 1883]; +} + +/* Hash lookup information for recomposition */ +static const pg_unicode_recompinfo UnicodeRecompInfo = +{ + RecompInverseLookup, + Recomp_hash_func, + 941 +}; diff --git a/src/tools/pgindent/exclude_file_patterns b/src/tools/pgindent/exclude_file_patterns index 86bdd9d6dcbf..f08180b0d089 100644 --- a/src/tools/pgindent/exclude_file_patterns +++ b/src/tools/pgindent/exclude_file_patterns @@ -18,9 +18,10 @@ src/backend/utils/fmgrprotos\.h$ # they match pgindent style, they'd look worse not better, so exclude them. kwlist_d\.h$ # -# This is generated by the scripts from src/common/unicode/. It uses +# These are generated by the scripts from src/common/unicode/. They use # hash functions generated by PerfectHash.pm whose format looks worse with # pgindent. +src/include/common/unicode_norm_hashfunc\.h$ src/include/common/unicode_normprops_table\.h$ # # Exclude ecpg test files to avoid breaking the ecpg regression tests From 22b73d3cb0b5bb4c141421f98dd67f091dda3c20 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 23 Oct 2020 09:30:08 +0300 Subject: [PATCH 360/589] Fix initialization of es_result_relations in EvalPlanQualStart(). Thinko in commit 1375422c782. EvalPlanQualStart() was mistakenly resetting the parent EState's es_result_relations, when it should initialize the field in the child EPQ EState it just created. That was clearly wrong, but it didn't cause any ill effects, because es_result_relations is currently not used after the ExecInit* phase. Author: Amit Langote Discussion: https://www.postgresql.org/message-id/CA%2BHiwqFEuq8AAAmxXsTDVZ1r38cHbfYuiPQx_%3DYyKe2DC-6q4A%40mail.gmail.com --- src/backend/executor/execMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index aea04794487f..7179f589f949 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -2693,7 +2693,7 @@ EvalPlanQualStart(EPQState *epqstate, Plan *planTree) * ResultRelInfos needed by subplans are initialized from scratch when the * subplans themselves are initialized. */ - parentestate->es_result_relations = NULL; + rcestate->es_result_relations = NULL; /* es_trig_target_relations must NOT be copied */ rcestate->es_top_eflags = parentestate->es_top_eflags; rcestate->es_instrument = parentestate->es_instrument; From 902b57c9bf27b3ea2963ef832cddb38b1203130e Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 23 Oct 2020 11:49:59 +0300 Subject: [PATCH 361/589] doc: Remove reference to pre-8.2 pg_dump behaviour The behavioural change in the -t/--table option happened around 15 years ago and there seems little point in keeping it around. Author: Ian Barwick Discussion: https://www.postgresql.org/message-id/CAB8KJ%3Dh-XALik4M7gv-pX48%3D%2BSPWexfaYwa%2ByTnPwD3DxceXrg%40mail.gmail.com --- doc/src/sgml/ref/pg_dump.sgml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index a9d389455497..0aa35cf0c3b4 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -554,18 +554,6 @@ PostgreSQL documentation - - - The behavior of the switch is not entirely upward - compatible with pre-8.2 PostgreSQL - versions. Formerly, writing -t tab would dump all - tables named tab, but now it just dumps whichever one - is visible in your default search path. To get the old behavior - you can write -t '*.tab'. Also, you must write something - like -t sch.tab to select a table in a particular schema, - rather than the old locution of -n sch -t tab. - - From 83d727e5b23c43f3fb7221963ddec24277c1126f Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 23 Oct 2020 13:01:39 +0200 Subject: [PATCH 362/589] doc: Fix order of protocol messages in listing The order of AuthenticationGSSContinue and AuthenticationSSPI was swapped, based on the other Authentication* protocol messages being listed in subcode order. --- doc/src/sgml/protocol.sgml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index a46cb728b79d..3a64db6f5509 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -3238,7 +3238,7 @@ AuthenticationGSS (B) -AuthenticationSSPI (B) +AuthenticationGSSContinue (B) @@ -3256,7 +3256,7 @@ AuthenticationSSPI (B) - Int32(8) + Int32 @@ -3266,11 +3266,21 @@ AuthenticationSSPI (B) - Int32(9) + Int32(8) - Specifies that SSPI authentication is required. + Specifies that this message contains GSSAPI or SSPI data. + + + + + + Byten + + + + GSSAPI or SSPI authentication data. @@ -3283,7 +3293,7 @@ AuthenticationSSPI (B) -AuthenticationGSSContinue (B) +AuthenticationSSPI (B) @@ -3300,32 +3310,22 @@ AuthenticationGSSContinue (B) - - Int32 - - - - Length of message contents in bytes, including self. - - - - Int32(8) - Specifies that this message contains GSSAPI or SSPI data. + Length of message contents in bytes, including self. - Byten + Int32(9) - GSSAPI or SSPI authentication data. + Specifies that SSPI authentication is required. From 87a174c0e77eed0bec7d53ef6d470d60335f6444 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 23 Oct 2020 11:32:33 -0400 Subject: [PATCH 363/589] Fix broken XML formatting in EXPLAIN output for incremental sorts. The ExplainCloseGroup arguments for incremental sort usage data didn't match the corresponding ExplainOpenGroup. This only matters for XML-format output, which is probably why we'd not noticed. Daniel Gustafsson, per bug #16683 from Frits Jalvingh Discussion: https://postgr.es/m/16683-8005033324ad34e9@postgresql.org --- src/backend/commands/explain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 41317f183743..43f9b01e833b 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -2827,7 +2827,7 @@ show_incremental_sort_group_info(IncrementalSortGroupInfo *groupInfo, ExplainPropertyInteger("Peak Sort Space Used", "kB", groupInfo->maxMemorySpaceUsed, es); - ExplainCloseGroup("Sort Spaces", memoryName.data, true, es); + ExplainCloseGroup("Sort Space", memoryName.data, true, es); } if (groupInfo->maxDiskSpaceUsed > 0) { @@ -2844,7 +2844,7 @@ show_incremental_sort_group_info(IncrementalSortGroupInfo *groupInfo, ExplainPropertyInteger("Peak Sort Space Used", "kB", groupInfo->maxDiskSpaceUsed, es); - ExplainCloseGroup("Sort Spaces", diskName.data, true, es); + ExplainCloseGroup("Sort Space", diskName.data, true, es); } ExplainCloseGroup("Incremental Sort Groups", groupName.data, true, es); From 860593ec3bd15e8969effdfcb5cbd98c561dd722 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 23 Oct 2020 14:02:00 -0400 Subject: [PATCH 364/589] Fix portability issues in new amcheck test. The tests added by commit 866e24d47 failed on big-endian machines due to lack of attention to endianness considerations. Fix that. While here, improve a few small cosmetic things, such as running it through perltidy. Mark Dilger Discussion: https://postgr.es/m/30B8E99A-2D9C-48D4-A55C-741C9D5F1563@enterprisedb.com --- contrib/amcheck/t/001_verify_heapam.pl | 148 ++++++++----------------- 1 file changed, 46 insertions(+), 102 deletions(-) diff --git a/contrib/amcheck/t/001_verify_heapam.pl b/contrib/amcheck/t/001_verify_heapam.pl index e7526c17b80e..f8ee5384f328 100644 --- a/contrib/amcheck/t/001_verify_heapam.pl +++ b/contrib/amcheck/t/001_verify_heapam.pl @@ -4,7 +4,7 @@ use PostgresNode; use TestLib; -use Test::More tests => 65; +use Test::More tests => 55; my ($node, $result); @@ -28,19 +28,17 @@ # fresh_test_table('test'); corrupt_first_page('test'); -detects_corruption( - "verify_heapam('test')", - "plain corrupted table"); -detects_corruption( +detects_heap_corruption("verify_heapam('test')", "plain corrupted table"); +detects_heap_corruption( "verify_heapam('test', skip := 'all-visible')", "plain corrupted table skipping all-visible"); -detects_corruption( +detects_heap_corruption( "verify_heapam('test', skip := 'all-frozen')", "plain corrupted table skipping all-frozen"); -detects_corruption( +detects_heap_corruption( "verify_heapam('test', check_toast := false)", "plain corrupted table skipping toast"); -detects_corruption( +detects_heap_corruption( "verify_heapam('test', startblock := 0, endblock := 0)", "plain corrupted table checking only block zero"); @@ -50,79 +48,20 @@ fresh_test_table('test'); $node->safe_psql('postgres', q(VACUUM FREEZE test)); corrupt_first_page('test'); -detects_corruption( - "verify_heapam('test')", +detects_heap_corruption("verify_heapam('test')", "all-frozen corrupted table"); detects_no_corruption( "verify_heapam('test', skip := 'all-frozen')", "all-frozen corrupted table skipping all-frozen"); -# -# Check a corrupt table with corrupt page header -# -fresh_test_table('test'); -corrupt_first_page_and_header('test'); -detects_corruption( - "verify_heapam('test')", - "corrupted test table with bad page header"); - -# -# Check an uncorrupted table with corrupt toast page header -# -fresh_test_table('test'); -my $toast = get_toast_for('test'); -corrupt_first_page_and_header($toast); -detects_corruption( - "verify_heapam('test', check_toast := true)", - "table with corrupted toast page header checking toast"); -detects_no_corruption( - "verify_heapam('test', check_toast := false)", - "table with corrupted toast page header skipping toast"); -detects_corruption( - "verify_heapam('$toast')", - "corrupted toast page header"); - -# -# Check an uncorrupted table with corrupt toast -# -fresh_test_table('test'); -$toast = get_toast_for('test'); -corrupt_first_page($toast); -detects_corruption( - "verify_heapam('test', check_toast := true)", - "table with corrupted toast checking toast"); -detects_no_corruption( - "verify_heapam('test', check_toast := false)", - "table with corrupted toast skipping toast"); -detects_corruption( - "verify_heapam('$toast')", - "corrupted toast table"); - -# -# Check an uncorrupted all-frozen table with corrupt toast -# -fresh_test_table('test'); -$node->safe_psql('postgres', q(VACUUM FREEZE test)); -$toast = get_toast_for('test'); -corrupt_first_page($toast); -detects_corruption( - "verify_heapam('test', check_toast := true)", - "all-frozen table with corrupted toast checking toast"); -detects_no_corruption( - "verify_heapam('test', check_toast := false)", - "all-frozen table with corrupted toast skipping toast"); -detects_corruption( - "verify_heapam('$toast')", - "corrupted toast table of all-frozen table"); - # Returns the filesystem path for the named relation. sub relation_filepath { my ($relname) = @_; my $pgdata = $node->data_dir; - my $rel = $node->safe_psql('postgres', - qq(SELECT pg_relation_filepath('$relname'))); + my $rel = $node->safe_psql('postgres', + qq(SELECT pg_relation_filepath('$relname'))); die "path not found for relation $relname" unless defined $rel; return "$pgdata/$rel"; } @@ -131,7 +70,9 @@ sub relation_filepath sub get_toast_for { my ($relname) = @_; - $node->safe_psql('postgres', qq( + + return $node->safe_psql( + 'postgres', qq( SELECT 'pg_toast.' || t.relname FROM pg_catalog.pg_class c, pg_catalog.pg_class t WHERE c.relname = '$relname' @@ -142,7 +83,9 @@ sub get_toast_for sub fresh_test_table { my ($relname) = @_; - $node->safe_psql('postgres', qq( + + return $node->safe_psql( + 'postgres', qq( DROP TABLE IF EXISTS $relname CASCADE; CREATE TABLE $relname (a integer, b text); ALTER TABLE $relname SET (autovacuum_enabled=false); @@ -154,55 +97,54 @@ sub fresh_test_table # Stops the test node, corrupts the first page of the named relation, and # restarts the node. -sub corrupt_first_page_internal +sub corrupt_first_page { - my ($relname, $corrupt_header) = @_; + my ($relname) = @_; my $relpath = relation_filepath($relname); $node->stop; + my $fh; - open($fh, '+<', $relpath); + open($fh, '+<', $relpath) + or BAIL_OUT("open failed: $!"); binmode $fh; - # If we corrupt the header, postgres won't allow the page into the buffer. - syswrite($fh, '\xFF\xFF\xFF\xFF', 8) if ($corrupt_header); + # Corrupt two line pointers. To be stable across platforms, we use + # 0x55555555 and 0xAAAAAAAA for the two, which are bitwise reverses of each + # other. + seek($fh, 32, 0) + or BAIL_OUT("seek failed: $!"); + syswrite($fh, pack("L*", 0x55555555, 0xAAAAAAAA)) + or BAIL_OUT("syswrite failed: $!"); + close($fh) + or BAIL_OUT("close failed: $!"); - # Corrupt at least the line pointers. Exactly what this corrupts will - # depend on the page, as it may run past the line pointers into the user - # data. We stop short of writing 2048 bytes (2k), the smallest supported - # page size, as we don't want to corrupt the next page. - seek($fh, 32, 0); - syswrite($fh, '\x77\x77\x77\x77', 500); - close($fh); $node->start; } -sub corrupt_first_page +sub detects_heap_corruption { - corrupt_first_page_internal($_[0], undef); -} + my ($function, $testname) = @_; -sub corrupt_first_page_and_header -{ - corrupt_first_page_internal($_[0], 1); + detects_corruption($function, $testname, + qr/line pointer redirection to item at offset \d+ exceeds maximum offset \d+/ + ); } sub detects_corruption { - my ($function, $testname) = @_; + my ($function, $testname, $re) = @_; - my $result = $node->safe_psql('postgres', - qq(SELECT COUNT(*) > 0 FROM $function)); - is($result, 't', $testname); + my $result = $node->safe_psql('postgres', qq(SELECT * FROM $function)); + like($result, $re, $testname); } sub detects_no_corruption { my ($function, $testname) = @_; - my $result = $node->safe_psql('postgres', - qq(SELECT COUNT(*) = 0 FROM $function)); - is($result, 't', $testname); + my $result = $node->safe_psql('postgres', qq(SELECT * FROM $function)); + is($result, '', $testname); } # Check various options are stable (don't abort) and do not report corruption @@ -215,6 +157,7 @@ sub detects_no_corruption sub check_all_options_uncorrupted { my ($relname, $prefix) = @_; + for my $stop (qw(true false)) { for my $check_toast (qw(true false)) @@ -225,11 +168,12 @@ sub check_all_options_uncorrupted { for my $endblock (qw(NULL 0)) { - my $opts = "on_error_stop := $stop, " . - "check_toast := $check_toast, " . - "skip := $skip, " . - "startblock := $startblock, " . - "endblock := $endblock"; + my $opts = + "on_error_stop := $stop, " + . "check_toast := $check_toast, " + . "skip := $skip, " + . "startblock := $startblock, " + . "endblock := $endblock"; detects_no_corruption( "verify_heapam('$relname', $opts)", From 1b62d0fb3e50ede570d0d4e4a2be69d5645b48a7 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 23 Oct 2020 17:07:15 -0400 Subject: [PATCH 365/589] Allow psql to re-use connection parameters after a connection loss. Instead of immediately PQfinish'ing a dead connection, save it aside so that we can still extract its parameters for \connect attempts. (This works because PQconninfo doesn't care whether the PGconn is in CONNECTION_BAD state.) This allows developers to reconnect with just \c after a database crash and restart. It's tempting to use the same approach instead of closing the old connection after a failed non-interactive \connect command. However, that would not be very safe: consider a script containing \c db1 user1 live_server \c db2 user2 dead_server \c db3 The script would be expecting to connect to db3 at dead_server, but if we re-use parameters from the first connection then it might successfully connect to db3 at live_server. This'd defeat the goal of not letting a script accidentally execute commands against the wrong database. Discussion: https://postgr.es/m/38464.1603394584@sss.pgh.pa.us --- doc/src/sgml/ref/psql-ref.sgml | 17 ++++++++-- src/bin/psql/command.c | 57 +++++++++++++++++++++++----------- src/bin/psql/common.c | 10 ++++-- src/bin/psql/describe.c | 2 +- src/bin/psql/settings.h | 7 +++++ src/bin/psql/startup.c | 6 +++- 6 files changed, 73 insertions(+), 26 deletions(-) diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml index f6f77dbac3ac..221a967bfe66 100644 --- a/doc/src/sgml/ref/psql-ref.sgml +++ b/doc/src/sgml/ref/psql-ref.sgml @@ -931,12 +931,23 @@ testdb=> connection is closed. If the connection attempt fails (wrong user name, access denied, etc.), the previous connection will be kept if - psql is in interactive mode. But when - executing a non-interactive script, processing will - immediately stop with an error. This distinction was chosen as + psql is in interactive mode. But when + executing a non-interactive script, the old connection is closed + and an error is reported. That may or may not terminate the + script; if it does not, all database-accessing commands will fail + until another \connect command is successfully + executed. This distinction was chosen as a user convenience against typos on the one hand, and a safety mechanism that scripts are not accidentally acting on the wrong database on the other hand. + Note that whenever a \connect command attempts + to re-use parameters, the values re-used are those of the last + successful connection, not of any failed attempts made subsequently. + However, in the case of a + non-interactive \connect failure, no parameters + are allowed to be re-used later, since the script would likely be + expecting the values from the failed \connect + to be re-used. diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c index 39a460d85ce9..c7a83d5dfc59 100644 --- a/src/bin/psql/command.c +++ b/src/bin/psql/command.c @@ -3060,26 +3060,28 @@ do_connect(enum trivalue reuse_previous_specification, break; } - /* - * If we are to re-use parameters, there'd better be an old connection to - * get them from. - */ - if (reuse_previous && !o_conn) - { - pg_log_error("No database connection exists to re-use parameters from"); - return false; - } - /* * If we intend to re-use connection parameters, collect them out of the - * old connection, then replace individual values as necessary. Otherwise, - * obtain a PQconninfoOption array containing libpq's defaults, and modify - * that. Note this function assumes that PQconninfo, PQconndefaults, and - * PQconninfoParse will all produce arrays containing the same options in - * the same order. + * old connection, then replace individual values as necessary. (We may + * need to resort to looking at pset.dead_conn, if the connection died + * previously.) Otherwise, obtain a PQconninfoOption array containing + * libpq's defaults, and modify that. Note this function assumes that + * PQconninfo, PQconndefaults, and PQconninfoParse will all produce arrays + * containing the same options in the same order. */ if (reuse_previous) - cinfo = PQconninfo(o_conn); + { + if (o_conn) + cinfo = PQconninfo(o_conn); + else if (pset.dead_conn) + cinfo = PQconninfo(pset.dead_conn); + else + { + /* This is reachable after a non-interactive \connect failure */ + pg_log_error("No database connection exists to re-use parameters from"); + return false; + } + } else cinfo = PQconndefaults(); @@ -3360,14 +3362,26 @@ do_connect(enum trivalue reuse_previous_specification, if (o_conn) { /* - * Transition to having no connection. Keep this bit in sync - * with CheckConnection(). + * Transition to having no connection. + * + * Unlike CheckConnection(), we close the old connection + * immediately to prevent its parameters from being re-used. + * This is so that a script cannot accidentally reuse + * parameters it did not expect to. Otherwise, the state + * cleanup should be the same as in CheckConnection(). */ PQfinish(o_conn); pset.db = NULL; ResetCancelConn(); UnsyncVariables(); } + + /* On the same reasoning, release any dead_conn to prevent reuse */ + if (pset.dead_conn) + { + PQfinish(pset.dead_conn); + pset.dead_conn = NULL; + } } return false; @@ -3421,8 +3435,15 @@ do_connect(enum trivalue reuse_previous_specification, PQdb(pset.db), PQuser(pset.db)); } + /* Drop no-longer-needed connection(s) */ if (o_conn) PQfinish(o_conn); + if (pset.dead_conn) + { + PQfinish(pset.dead_conn); + pset.dead_conn = NULL; + } + return true; } diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c index 6323a35c91ca..ff673665d869 100644 --- a/src/bin/psql/common.c +++ b/src/bin/psql/common.c @@ -313,10 +313,14 @@ CheckConnection(void) fprintf(stderr, _("Failed.\n")); /* - * Transition to having no connection. Keep this bit in sync with - * do_connect(). + * Transition to having no connection; but stash away the failed + * connection so that we can still refer to its parameters in a + * later \connect attempt. Keep the state cleanup here in sync + * with do_connect(). */ - PQfinish(pset.db); + if (pset.dead_conn) + PQfinish(pset.dead_conn); + pset.dead_conn = pset.db; pset.db = NULL; ResetCancelConn(); UnsyncVariables(); diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 6bb0316bd98b..07d640021c27 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -2744,7 +2744,7 @@ describeOneTableDetails(const char *schemaname, /* Show the stats target if it's not default */ if (strcmp(PQgetvalue(result, i, 8), "-1") != 0) appendPQExpBuffer(&buf, "; STATISTICS %s", - PQgetvalue(result, i, 8)); + PQgetvalue(result, i, 8)); printTableAddFooter(&cont, buf.data); } diff --git a/src/bin/psql/settings.h b/src/bin/psql/settings.h index 97941aa10c67..9601f6e90ce8 100644 --- a/src/bin/psql/settings.h +++ b/src/bin/psql/settings.h @@ -117,6 +117,13 @@ typedef struct _psqlSettings VariableSpace vars; /* "shell variable" repository */ + /* + * If we get a connection failure, the now-unusable PGconn is stashed here + * until we can successfully reconnect. Never attempt to do anything with + * this PGconn except extract parameters for a \connect attempt. + */ + PGconn *dead_conn; /* previous connection to backend */ + /* * The remaining fields are set by assign hooks associated with entries in * "vars". They should not be set directly except by those hook diff --git a/src/bin/psql/startup.c b/src/bin/psql/startup.c index 8232a0143bc9..e8d35a108f36 100644 --- a/src/bin/psql/startup.c +++ b/src/bin/psql/startup.c @@ -145,6 +145,7 @@ main(int argc, char *argv[]) pset.progname = get_progname(argv[0]); pset.db = NULL; + pset.dead_conn = NULL; setDecimalLocale(); pset.encoding = PQenv2encoding(); pset.queryFout = stdout; @@ -442,7 +443,10 @@ main(int argc, char *argv[]) /* clean up */ if (pset.logfile) fclose(pset.logfile); - PQfinish(pset.db); + if (pset.db) + PQfinish(pset.db); + if (pset.dead_conn) + PQfinish(pset.dead_conn); setQFout(NULL); return successResult; From 321633e17b07968e68ca5341429e2c8bbf15c331 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 23 Oct 2020 19:08:01 -0400 Subject: [PATCH 366/589] Fix more portability issues in new amcheck code. verify_heapam() wasn't being careful to sanity-check tuple line pointers before using them, resulting in SIGBUS on alignment-picky architectures. Fix that, add some more test coverage. Mark Dilger, some tweaking by me Discussion: https://postgr.es/m/30B8E99A-2D9C-48D4-A55C-741C9D5F1563@enterprisedb.com --- contrib/amcheck/t/001_verify_heapam.pl | 30 ++++++---- contrib/amcheck/verify_heapam.c | 78 +++++++++++++++++--------- 2 files changed, 73 insertions(+), 35 deletions(-) diff --git a/contrib/amcheck/t/001_verify_heapam.pl b/contrib/amcheck/t/001_verify_heapam.pl index f8ee5384f328..1581e51f3ca7 100644 --- a/contrib/amcheck/t/001_verify_heapam.pl +++ b/contrib/amcheck/t/001_verify_heapam.pl @@ -4,7 +4,7 @@ use PostgresNode; use TestLib; -use Test::More tests => 55; +use Test::More tests => 79; my ($node, $result); @@ -109,13 +109,17 @@ sub corrupt_first_page or BAIL_OUT("open failed: $!"); binmode $fh; - # Corrupt two line pointers. To be stable across platforms, we use - # 0x55555555 and 0xAAAAAAAA for the two, which are bitwise reverses of each - # other. + # Corrupt some line pointers. The values are chosen to hit the + # various line-pointer-corruption checks in verify_heapam.c + # on both little-endian and big-endian architectures. seek($fh, 32, 0) or BAIL_OUT("seek failed: $!"); - syswrite($fh, pack("L*", 0x55555555, 0xAAAAAAAA)) - or BAIL_OUT("syswrite failed: $!"); + syswrite( + $fh, + pack("L*", + 0xAAA15550, 0xAAA0D550, 0x00010000, + 0x00008000, 0x0000800F, 0x001e8000) + ) or BAIL_OUT("syswrite failed: $!"); close($fh) or BAIL_OUT("close failed: $!"); @@ -126,17 +130,23 @@ sub detects_heap_corruption { my ($function, $testname) = @_; - detects_corruption($function, $testname, - qr/line pointer redirection to item at offset \d+ exceeds maximum offset \d+/ + detects_corruption( + $function, + $testname, + qr/line pointer redirection to item at offset \d+ precedes minimum offset \d+/, + qr/line pointer redirection to item at offset \d+ exceeds maximum offset \d+/, + qr/line pointer to page offset \d+ is not maximally aligned/, + qr/line pointer length \d+ is less than the minimum tuple header size \d+/, + qr/line pointer to page offset \d+ with length \d+ ends beyond maximum page offset \d+/, ); } sub detects_corruption { - my ($function, $testname, $re) = @_; + my ($function, $testname, @re) = @_; my $result = $node->safe_psql('postgres', qq(SELECT * FROM $function)); - like($result, $re, $testname); + like($result, $_, $testname) for (@re); } sub detects_no_corruption diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index 37b40a0404d5..8bb890438aa9 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -105,6 +105,7 @@ typedef struct HeapCheckContext OffsetNumber offnum; ItemId itemid; uint16 lp_len; + uint16 lp_off; HeapTupleHeader tuphdr; int natts; @@ -247,6 +248,13 @@ verify_heapam(PG_FUNCTION_ARGS) memset(&ctx, 0, sizeof(HeapCheckContext)); ctx.cached_xid = InvalidTransactionId; + /* + * If we report corruption when not examining some individual attribute, + * we need attnum to be reported as NULL. Set that up before any + * corruption reporting might happen. + */ + ctx.attnum = -1; + /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ old_context = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); random_access = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; @@ -378,14 +386,22 @@ verify_heapam(PG_FUNCTION_ARGS) /* * If this line pointer has been redirected, check that it - * redirects to a valid offset within the line pointer array. + * redirects to a valid offset within the line pointer array */ if (ItemIdIsRedirected(ctx.itemid)) { OffsetNumber rdoffnum = ItemIdGetRedirect(ctx.itemid); ItemId rditem; - if (rdoffnum < FirstOffsetNumber || rdoffnum > maxoff) + if (rdoffnum < FirstOffsetNumber) + { + report_corruption(&ctx, + psprintf("line pointer redirection to item at offset %u precedes minimum offset %u", + (unsigned) rdoffnum, + (unsigned) FirstOffsetNumber)); + continue; + } + if (rdoffnum > maxoff) { report_corruption(&ctx, psprintf("line pointer redirection to item at offset %u exceeds maximum offset %u", @@ -401,8 +417,36 @@ verify_heapam(PG_FUNCTION_ARGS) continue; } - /* Set up context information about this next tuple */ + /* Sanity-check the line pointer's offset and length values */ ctx.lp_len = ItemIdGetLength(ctx.itemid); + ctx.lp_off = ItemIdGetOffset(ctx.itemid); + + if (ctx.lp_off != MAXALIGN(ctx.lp_off)) + { + report_corruption(&ctx, + psprintf("line pointer to page offset %u is not maximally aligned", + ctx.lp_off)); + continue; + } + if (ctx.lp_len < MAXALIGN(SizeofHeapTupleHeader)) + { + report_corruption(&ctx, + psprintf("line pointer length %u is less than the minimum tuple header size %u", + ctx.lp_len, + (unsigned) MAXALIGN(SizeofHeapTupleHeader))); + continue; + } + if (ctx.lp_off + ctx.lp_len > BLCKSZ) + { + report_corruption(&ctx, + psprintf("line pointer to page offset %u with length %u ends beyond maximum page offset %u", + ctx.lp_off, + ctx.lp_len, + (unsigned) BLCKSZ)); + continue; + } + + /* It should be safe to examine the tuple's header, at least */ ctx.tuphdr = (HeapTupleHeader) PageGetItem(ctx.page, ctx.itemid); ctx.natts = HeapTupleHeaderGetNatts(ctx.tuphdr); @@ -1088,25 +1132,6 @@ check_tuple(HeapCheckContext *ctx) bool fatal = false; uint16 infomask = ctx->tuphdr->t_infomask; - /* - * If we report corruption before iterating over individual attributes, we - * need attnum to be reported as NULL. Set that up before any corruption - * reporting might happen. - */ - ctx->attnum = -1; - - /* - * If the line pointer for this tuple does not reserve enough space for a - * complete tuple header, we dare not read the tuple header. - */ - if (ctx->lp_len < MAXALIGN(SizeofHeapTupleHeader)) - { - report_corruption(ctx, - psprintf("line pointer length %u is less than the minimum tuple header size %u", - ctx->lp_len, (uint32) MAXALIGN(SizeofHeapTupleHeader))); - return; - } - /* If xmin is normal, it should be within valid range */ xmin = HeapTupleHeaderGetXmin(ctx->tuphdr); switch (get_xid_status(xmin, ctx, NULL)) @@ -1256,6 +1281,9 @@ check_tuple(HeapCheckContext *ctx) for (ctx->attnum = 0; ctx->attnum < ctx->natts; ctx->attnum++) if (!check_tuple_attribute(ctx)) break; /* cannot continue */ + + /* revert attnum to -1 until we again examine individual attributes */ + ctx->attnum = -1; } /* @@ -1393,9 +1421,9 @@ get_xid_status(TransactionId xid, HeapCheckContext *ctx, if (!fxid_in_cached_range(fxid, ctx)) { /* - * We may have been checking against stale values. Update the - * cached range to be sure, and since we relied on the cached - * range when we performed the full xid conversion, reconvert. + * We may have been checking against stale values. Update the cached + * range to be sure, and since we relied on the cached range when we + * performed the full xid conversion, reconvert. */ update_cached_xid_range(ctx); fxid = FullTransactionIdFromXidAndCtx(xid, ctx); From 0b46e82c06b0c4b0dc6a94a890d23945ebf720fd Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Sat, 24 Oct 2020 10:29:55 +0900 Subject: [PATCH 367/589] Add tab completion for ALTER TABLE .. FORCE ROW LEVEL SECURITY in psql This completes both the FORCE and NO FORCE options, NO INHERIT needing a small adjustment. Author: Li Japin Discussion: https://postgr.es/m/15B10F9F-5847-4F5E-BD66-8E25AA473C95@hotmail.com --- src/bin/psql/tab-complete.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 561fe1dff93b..b2b4f1fd4d13 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -1974,10 +1974,10 @@ psql_completion(const char *text, int start, int end) */ else if (Matches("ALTER", "TABLE", MatchAny)) COMPLETE_WITH("ADD", "ALTER", "CLUSTER ON", "DISABLE", "DROP", - "ENABLE", "INHERIT", "NO INHERIT", "RENAME", "RESET", + "ENABLE", "INHERIT", "NO", "RENAME", "RESET", "OWNER TO", "SET", "VALIDATE CONSTRAINT", "REPLICA IDENTITY", "ATTACH PARTITION", - "DETACH PARTITION"); + "DETACH PARTITION", "FORCE ROW LEVEL SECURITY"); /* ALTER TABLE xxx ENABLE */ else if (Matches("ALTER", "TABLE", MatchAny, "ENABLE")) COMPLETE_WITH("ALWAYS", "REPLICA", "ROW LEVEL SECURITY", "RULE", @@ -2007,6 +2007,9 @@ psql_completion(const char *text, int start, int end) /* ALTER TABLE xxx INHERIT */ else if (Matches("ALTER", "TABLE", MatchAny, "INHERIT")) COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_tables, ""); + /* ALTER TABLE xxx NO */ + else if (Matches("ALTER", "TABLE", MatchAny, "NO")) + COMPLETE_WITH("FORCE ROW LEVEL SECURITY", "INHERIT"); /* ALTER TABLE xxx NO INHERIT */ else if (Matches("ALTER", "TABLE", MatchAny, "NO", "INHERIT")) COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_tables, ""); From 2771fcee18be0b86d2e008add20f73d175e06e90 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Sat, 24 Oct 2020 14:20:38 +0900 Subject: [PATCH 368/589] Fix issue with --enable-coverage and the new unicode {de,re}composition code genhtml has been generating the following warning with this new code: WARNING: function data mismatch at /path/src/common/unicode_norm.c:102 HTML coverage reports care about the uniqueness of functions defined in source files, ignoring any assumptions around CFLAGS. 783f0cc introduced a duplicated definition of get_code_entry(), leading to a warning and potentially some incorrect data generated in the reports. This refactors the code so as the code has only one function declaration, fixing the warning. Oversight in 783f0cc. Reported-by: Tom Lane Author: Michael Paquier Reviewed-by: Tom Lane Discussion: https://postgr.es/m/207789.1603469272@sss.pgh.pa.us --- src/common/unicode_norm.c | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/src/common/unicode_norm.c b/src/common/unicode_norm.c index 4ffce0e61930..abb83cbf985e 100644 --- a/src/common/unicode_norm.c +++ b/src/common/unicode_norm.c @@ -46,6 +46,21 @@ #define NCOUNT VCOUNT * TCOUNT #define SCOUNT LCOUNT * NCOUNT +#ifdef FRONTEND +/* comparison routine for bsearch() of decomposition lookup table. */ +static int +conv_compare(const void *p1, const void *p2) +{ + uint32 v1, + v2; + + v1 = *(const uint32 *) p1; + v2 = ((const pg_unicode_decomposition *) p2)->codepoint; + return (v1 > v2) ? 1 : ((v1 == v2) ? 0 : -1); +} + +#endif + /* * get_code_entry * @@ -53,11 +68,10 @@ * The backend version of this code uses a perfect hash function for the * lookup, while the frontend version uses a binary search. */ -#ifndef FRONTEND - static const pg_unicode_decomposition * get_code_entry(pg_wchar code) { +#ifndef FRONTEND int h; uint32 hashkey; pg_unicode_decompinfo decompinfo = UnicodeDecompInfo; @@ -82,33 +96,15 @@ get_code_entry(pg_wchar code) /* Success! */ return &decompinfo.decomps[h]; -} - #else - -/* comparison routine for bsearch() of decomposition lookup table. */ -static int -conv_compare(const void *p1, const void *p2) -{ - uint32 v1, - v2; - - v1 = *(const uint32 *) p1; - v2 = ((const pg_unicode_decomposition *) p2)->codepoint; - return (v1 > v2) ? 1 : ((v1 == v2) ? 0 : -1); -} - -static const pg_unicode_decomposition * -get_code_entry(pg_wchar code) -{ return bsearch(&(code), UnicodeDecompMain, lengthof(UnicodeDecompMain), sizeof(pg_unicode_decomposition), conv_compare); +#endif } -#endif /* !FRONTEND */ /* * Given a decomposition entry looked up earlier, get the decomposed From 21d36747d4fafe16539a0c55ebb91a01e4053e3c Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 24 Oct 2020 13:12:08 -0400 Subject: [PATCH 369/589] Fix ancient bug in ecpg's pthread_once() emulation for Windows. We must not set the "done" flag until after we've executed the initialization function. Otherwise, other threads can fall through the initial unlocked test before initialization is really complete. This has been seen to cause rare failures of ecpg's thread/descriptor test, and it could presumably cause other sorts of misbehavior in threaded ECPG-using applications, since ecpglib relies on pthread_once() in several places. Diagnosis and patch by me, based on investigation by Alexander Lakhin. Back-patch to all supported branches (the bug dates to 2007). Discussion: https://postgr.es/m/16685-d6cd241872c101d3@postgresql.org --- src/interfaces/ecpg/ecpglib/misc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/interfaces/ecpg/ecpglib/misc.c b/src/interfaces/ecpg/ecpglib/misc.c index a07d0dfb9fc2..1feb5c03e174 100644 --- a/src/interfaces/ecpg/ecpglib/misc.c +++ b/src/interfaces/ecpg/ecpglib/misc.c @@ -477,8 +477,8 @@ win32_pthread_once(volatile pthread_once_t *once, void (*fn) (void)) pthread_mutex_lock(&win32_pthread_once_lock); if (!*once) { - *once = true; fn(); + *once = true; } pthread_mutex_unlock(&win32_pthread_once_lock); } From e83c9f913c6197586af8ac53c1d3652db15a3c91 Mon Sep 17 00:00:00 2001 From: David Rowley Date: Sun, 25 Oct 2020 22:39:00 +1300 Subject: [PATCH 370/589] Fix incorrect parameter name in a function header comment Author: Zhijie Hou Discussion: https://postgr.es/m/14cd74ea00204cc8a7ea5d738ac82cd1@G08CNEXMBPEKD05.g08.fujitsu.local Backpatch-through: 12, where the mistake was introduced --- src/backend/utils/cache/lsyscache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/utils/cache/lsyscache.c b/src/backend/utils/cache/lsyscache.c index f3bf413829fc..140339073b63 100644 --- a/src/backend/utils/cache/lsyscache.c +++ b/src/backend/utils/cache/lsyscache.c @@ -874,7 +874,7 @@ get_attnum(Oid relid, const char *attname) /* * get_attgenerated * - * Given the relation id and the attribute name, + * Given the relation id and the attribute number, * return the "attgenerated" field from the attribute relation. * * Errors if not found. From ba9f18abd3650e385e9a35df7145a7c38af17e92 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 25 Oct 2020 13:57:46 -0400 Subject: [PATCH 371/589] Fix corner case for a BEFORE ROW UPDATE trigger returning OLD. If the old row has any "missing" attributes that are supposed to be retrieved from an associated tuple descriptor, the wrong things happened because the trigger result is shoved directly into an executor slot that lacks the missing-attribute data. Notably, CHECK-constraint verification would incorrectly see those columns as NULL, and so would RETURNING-list evaluation. Band-aid around this by forcibly expanding the tuple before passing it to the trigger function. (IMO it was a fundamental misdesign to put the missing-attribute data into tuple constraints, which so much of the system considers to be optional. But we're probably stuck with that now, and will have to continue to apply band-aids as we find other places with similar issues.) Back-patch to v12. v11 would also have the issue, except that commit 920311ab1 already applied a similar band-aid. That forced expansion in more cases than seem really necessary, though, so this isn't a directly equivalent fix. Amit Langote, with some cosmetic changes by me Discussion: https://postgr.es/m/16644-5da7ef98a7ac4545@postgresql.org --- src/backend/commands/trigger.c | 41 +++++++++++++++++++++++++- src/test/regress/expected/triggers.out | 32 ++++++++++++++++++++ src/test/regress/sql/triggers.sql | 18 +++++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c index 28b98d10ae85..59289f8d4d37 100644 --- a/src/backend/commands/trigger.c +++ b/src/backend/commands/trigger.c @@ -89,6 +89,8 @@ static bool GetTupleForTrigger(EState *estate, LockTupleMode lockmode, TupleTableSlot *oldslot, TupleTableSlot **newSlot); +static HeapTuple MaterializeTupleForTrigger(TupleTableSlot *slot, + bool *shouldFree); static bool TriggerEnabled(EState *estate, ResultRelInfo *relinfo, Trigger *trigger, TriggerEvent event, Bitmapset *modifiedCols, @@ -2672,7 +2674,7 @@ ExecBRUpdateTriggers(EState *estate, EPQState *epqstate, ExecCopySlot(newslot, epqslot_clean); } - trigtuple = ExecFetchSlotHeapTuple(oldslot, true, &should_free_trig); + trigtuple = MaterializeTupleForTrigger(oldslot, &should_free_trig); } else { @@ -2925,6 +2927,9 @@ ExecASTruncateTriggers(EState *estate, ResultRelInfo *relinfo) } +/* + * Fetch tuple into "oldslot", dealing with locking and EPQ if necessary + */ static bool GetTupleForTrigger(EState *estate, EPQState *epqstate, @@ -3038,6 +3043,40 @@ GetTupleForTrigger(EState *estate, return true; } +/* + * Extract a HeapTuple that we can pass off to trigger functions. + * + * We must materialize the tuple and make sure it is not dependent on any + * attrmissing data. This is needed for the old row in BEFORE UPDATE + * triggers, since they can choose to pass back this exact tuple as the update + * result, causing the tuple to be inserted into an executor slot that lacks + * the attrmissing data. + * + * Currently we don't seem to need to remove the attrmissing dependency in any + * other cases, but keep this as a separate function to simplify fixing things + * if that changes. + */ +static HeapTuple +MaterializeTupleForTrigger(TupleTableSlot *slot, bool *shouldFree) +{ + HeapTuple tup; + TupleDesc tupdesc = slot->tts_tupleDescriptor; + + tup = ExecFetchSlotHeapTuple(slot, true, shouldFree); + if (HeapTupleHeaderGetNatts(tup->t_data) < tupdesc->natts && + tupdesc->constr && tupdesc->constr->missing) + { + HeapTuple newtup; + + newtup = heap_expand_tuple(tup, tupdesc); + if (*shouldFree) + heap_freetuple(tup); + *shouldFree = true; + tup = newtup; + } + return tup; +} + /* * Is trigger enabled to fire? */ diff --git a/src/test/regress/expected/triggers.out b/src/test/regress/expected/triggers.out index 57efa290207a..c19aac967428 100644 --- a/src/test/regress/expected/triggers.out +++ b/src/test/regress/expected/triggers.out @@ -214,6 +214,38 @@ select * from trigtest; ----+---- (0 rows) +drop table trigtest; +-- Check behavior with an implicit column default, too (bug #16644) +create table trigtest (a integer); +create trigger trigger_return_old + before insert or delete or update on trigtest + for each row execute procedure trigger_return_old(); +insert into trigtest values(1); +select * from trigtest; + a +--- + 1 +(1 row) + +alter table trigtest add column b integer default 42 not null; +select * from trigtest; + a | b +---+---- + 1 | 42 +(1 row) + +update trigtest set a = 2 where a = 1 returning *; + a | b +---+---- + 1 | 42 +(1 row) + +select * from trigtest; + a | b +---+---- + 1 | 42 +(1 row) + drop table trigtest; create sequence ttdummy_seq increment 10 start 0 minvalue 0; create table tttest ( diff --git a/src/test/regress/sql/triggers.sql b/src/test/regress/sql/triggers.sql index 8f66df9f3b41..bf2e73abf606 100644 --- a/src/test/regress/sql/triggers.sql +++ b/src/test/regress/sql/triggers.sql @@ -154,6 +154,24 @@ select * from trigtest; drop table trigtest; +-- Check behavior with an implicit column default, too (bug #16644) +create table trigtest (a integer); + +create trigger trigger_return_old + before insert or delete or update on trigtest + for each row execute procedure trigger_return_old(); + +insert into trigtest values(1); +select * from trigtest; + +alter table trigtest add column b integer default 42 not null; + +select * from trigtest; +update trigtest set a = 2 where a = 1 returning *; +select * from trigtest; + +drop table trigtest; + create sequence ttdummy_seq increment 10 start 0 minvalue 0; create table tttest ( From d401c5769ef6aeef0a28c147f3fb5afedcd59984 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 26 Oct 2020 09:55:28 +0900 Subject: [PATCH 372/589] Extend PageIsVerified() to handle more custom options This is useful for checks of relation pages without having to load the pages into the shared buffers, and two cases can make use of that: page verification in base backups and the online, lock-safe, flavor. Compatibility is kept with past versions using a macro that calls the new extended routine with the set of options compatible with the original version. Extracted from a larger patch by the same author. Author: Anastasia Lubennikova Reviewed-by: Michael Paquier, Julien Rouhaud Discussion: https://postgr.es/m/608f3476-0598-2514-2c03-e05c7d2b0cbd@postgrespro.ru --- src/backend/catalog/storage.c | 3 ++- src/backend/storage/buffer/bufmgr.c | 6 ++++-- src/backend/storage/page/bufpage.c | 22 +++++++++++++++------- src/include/storage/bufpage.h | 22 ++++++++++++++++------ 4 files changed, 37 insertions(+), 16 deletions(-) diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c index dbbd3aa31fe1..d538f25726f4 100644 --- a/src/backend/catalog/storage.c +++ b/src/backend/catalog/storage.c @@ -443,7 +443,8 @@ RelationCopyStorage(SMgrRelation src, SMgrRelation dst, smgrread(src, forkNum, blkno, buf.data); - if (!PageIsVerified(page, blkno)) + if (!PageIsVerifiedExtended(page, blkno, + PIV_LOG_WARNING | PIV_REPORT_STAT)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("invalid page in block %u of relation %s", diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index e549fa1d309f..3eee86afe5c4 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -625,7 +625,8 @@ ReadBuffer(Relation reln, BlockNumber blockNum) * * In RBM_NORMAL mode, the page is read from disk, and the page header is * validated. An error is thrown if the page header is not valid. (But - * note that an all-zero page is considered "valid"; see PageIsVerified().) + * note that an all-zero page is considered "valid"; see + * PageIsVerifiedExtended().) * * RBM_ZERO_ON_ERROR is like the normal mode, but if the page header is not * valid, the page is zeroed instead of throwing an error. This is intended @@ -917,7 +918,8 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, } /* check for garbage data */ - if (!PageIsVerified((Page) bufBlock, blockNum)) + if (!PageIsVerifiedExtended((Page) bufBlock, blockNum, + PIV_LOG_WARNING | PIV_REPORT_STAT)) { if (mode == RBM_ZERO_ON_ERROR || zero_damaged_pages) { diff --git a/src/backend/storage/page/bufpage.c b/src/backend/storage/page/bufpage.c index 4bc2bf955dfd..ddf18079e2fb 100644 --- a/src/backend/storage/page/bufpage.c +++ b/src/backend/storage/page/bufpage.c @@ -61,7 +61,7 @@ PageInit(Page page, Size pageSize, Size specialSize) /* - * PageIsVerified + * PageIsVerifiedExtended * Check that the page header and checksum (if any) appear valid. * * This is called when a page has just been read in from disk. The idea is @@ -77,9 +77,15 @@ PageInit(Page page, Size pageSize, Size specialSize) * allow zeroed pages here, and are careful that the page access macros * treat such a page as empty and without free space. Eventually, VACUUM * will clean up such a page and make it usable. + * + * If flag PIV_LOG_WARNING is set, a WARNING is logged in the event of + * a checksum failure. + * + * If flag PIV_REPORT_STAT is set, a checksum failure is reported directly + * to pgstat. */ bool -PageIsVerified(Page page, BlockNumber blkno) +PageIsVerifiedExtended(Page page, BlockNumber blkno, int flags) { PageHeader p = (PageHeader) page; size_t *pagebytes; @@ -140,12 +146,14 @@ PageIsVerified(Page page, BlockNumber blkno) */ if (checksum_failure) { - ereport(WARNING, - (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("page verification failed, calculated checksum %u but expected %u", - checksum, p->pd_checksum))); + if ((flags & PIV_LOG_WARNING) != 0) + ereport(WARNING, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("page verification failed, calculated checksum %u but expected %u", + checksum, p->pd_checksum))); - pgstat_report_checksum_failure(); + if ((flags & PIV_REPORT_STAT) != 0) + pgstat_report_checksum_failure(); if (header_sane && ignore_checksum_failure) return true; diff --git a/src/include/storage/bufpage.h b/src/include/storage/bufpage.h index 51b8f994ac0a..d0a52f8e08fc 100644 --- a/src/include/storage/bufpage.h +++ b/src/include/storage/bufpage.h @@ -404,26 +404,36 @@ do { \ * extern declarations * ---------------------------------------------------------------- */ + +/* flags for PageAddItemExtended() */ #define PAI_OVERWRITE (1 << 0) #define PAI_IS_HEAP (1 << 1) +/* flags for PageIsVerifiedExtended() */ +#define PIV_LOG_WARNING (1 << 0) +#define PIV_REPORT_STAT (1 << 1) + #define PageAddItem(page, item, size, offsetNumber, overwrite, is_heap) \ PageAddItemExtended(page, item, size, offsetNumber, \ ((overwrite) ? PAI_OVERWRITE : 0) | \ ((is_heap) ? PAI_IS_HEAP : 0)) +#define PageIsVerified(page, blkno) \ + PageIsVerifiedExtended(page, blkno, \ + PIV_LOG_WARNING | PIV_REPORT_STAT) + /* - * Check that BLCKSZ is a multiple of sizeof(size_t). In PageIsVerified(), - * it is much faster to check if a page is full of zeroes using the native - * word size. Note that this assertion is kept within a header to make - * sure that StaticAssertDecl() works across various combinations of - * platforms and compilers. + * Check that BLCKSZ is a multiple of sizeof(size_t). In + * PageIsVerifiedExtended(), it is much faster to check if a page is + * full of zeroes using the native word size. Note that this assertion + * is kept within a header to make sure that StaticAssertDecl() works + * across various combinations of platforms and compilers. */ StaticAssertDecl(BLCKSZ == ((BLCKSZ / sizeof(size_t)) * sizeof(size_t)), "BLCKSZ has to be a multiple of sizeof(size_t)"); extern void PageInit(Page page, Size pageSize, Size specialSize); -extern bool PageIsVerified(Page page, BlockNumber blkno); +extern bool PageIsVerifiedExtended(Page page, BlockNumber blkno, int flags); extern OffsetNumber PageAddItemExtended(Page page, Item item, Size size, OffsetNumber offsetNumber, int flags); extern Page PageGetTempPage(Page page); From fa42c2ecb0f6e89f74bc1cc37b56a1d43e45d513 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 26 Oct 2020 09:07:14 +0200 Subject: [PATCH 373/589] docs: Remove notes about incompatibilies with very old versions. These are old enough that they'll cause more confusion and distraction to new readers, than they could help anyone upgrade from very old servers. Discussion: https://www.postgresql.org/message-id/fd93f1c5-7818-a02c-01e5-1075ac0d4def%40iki.fi --- doc/src/sgml/func.sgml | 37 ----------------------------- doc/src/sgml/high-availability.sgml | 36 ---------------------------- doc/src/sgml/indexam.sgml | 1 - doc/src/sgml/installation.sgml | 3 --- doc/src/sgml/pgfreespacemap.sgml | 7 ------ doc/src/sgml/ref/pg_dumpall.sgml | 5 +--- 6 files changed, 1 insertion(+), 88 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c99499e52bdb..f7f401b534c7 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -6848,30 +6848,6 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); constraints, and the longest/shortest-match (rather than first-match) matching semantics. - - - Two significant incompatibilities exist between AREs and the ERE syntax - recognized by pre-7.4 releases of PostgreSQL: - - - - - In AREs, \ followed by an alphanumeric character is either - an escape or an error, while in previous releases, it was just another - way of writing the alphanumeric. - This should not be much of a problem because there was no reason to - write such a sequence in earlier releases. - - - - - In AREs, \ remains a special character within - [], so a literal \ within a bracket - expression must be written \\. - - - - @@ -17106,16 +17082,6 @@ nextval('foo') searches search path for fo - - Before PostgreSQL 8.1, the arguments of the - sequence functions were of type text, not regclass, and - the above-described conversion from a text string to an OID value would - happen at run time during each call. For backward compatibility, this - facility still exists, but internally it is now handled as an implicit - coercion from text to regclass before the function is - invoked. - - When you write the argument of a sequence function as an unadorned literal string, it becomes a constant of type regclass. @@ -17129,9 +17095,6 @@ nextval('foo') searches search path for fo nextval('foo'::text) foo is looked up at runtime - Note that late binding was the only behavior supported in - PostgreSQL releases before 8.1, so you - might need to do this to preserve the semantics of old applications. diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index 339ed38d42c8..19d7bd2b28fa 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -1632,42 +1632,6 @@ if (!triggered) improvement in server robustness, nor would it be described as HA. - - - Record-Based Log Shipping - - - It is also possible to implement record-based log shipping using this - alternative method, though this requires custom development, and changes - will still only become visible to hot standby queries after a full WAL - file has been shipped. - - - - An external program can call the pg_walfile_name_offset() - function (see ) - to find out the file name and the exact byte offset within it of - the current end of WAL. It can then access the WAL file directly - and copy the data from the last known end of WAL through the current end - over to the standby servers. With this approach, the window for data - loss is the polling cycle time of the copying program, which can be very - small, and there is no wasted bandwidth from forcing partially-used - segment files to be archived. Note that the standby servers' - restore_command scripts can only deal with whole WAL files, - so the incrementally copied data is not ordinarily made available to - the standby servers. It is of use only when the primary dies — - then the last partial WAL file is fed to the standby before allowing - it to come up. The correct implementation of this process requires - cooperation of the restore_command script with the data - copying program. - - - - Starting with PostgreSQL version 9.0, you can use - streaming replication (see ) to - achieve the same benefits with less effort. - - diff --git a/doc/src/sgml/indexam.sgml b/doc/src/sgml/indexam.sgml index 649020b7daa2..80473e0f1a2b 100644 --- a/doc/src/sgml/indexam.sgml +++ b/doc/src/sgml/indexam.sgml @@ -368,7 +368,6 @@ amvacuumcleanup (IndexVacuumInfo *info, - As of PostgreSQL 8.4, amvacuumcleanup will also be called at completion of an ANALYZE operation. In this case stats is always NULL and any return value will be ignored. This case can be distinguished diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 3ac588dfb5c9..0ac1cb999992 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -560,9 +560,6 @@ build-postgresql: The standard installation provides all the header files needed for client application development as well as for server-side program development, such as custom functions or data types written in C. - (Prior to PostgreSQL 8.0, a separate make - install-all-headers command was needed for the latter, but this - step has been folded into the standard install.) diff --git a/doc/src/sgml/pgfreespacemap.sgml b/doc/src/sgml/pgfreespacemap.sgml index 0122d278e39e..5025498249d8 100644 --- a/doc/src/sgml/pgfreespacemap.sgml +++ b/doc/src/sgml/pgfreespacemap.sgml @@ -68,13 +68,6 @@ space within pages. Therefore, the values are not meaningful, just whether a page is full or empty. - - - - The interface was changed in version 8.4, to reflect the new FSM - implementation introduced in the same version. - - diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml index 4360b2cf5773..874964309218 100644 --- a/doc/src/sgml/ref/pg_dumpall.sgml +++ b/doc/src/sgml/ref/pg_dumpall.sgml @@ -375,10 +375,7 @@ PostgreSQL documentation the dump. Instead, fail if unable to lock a table within the specified timeout. The timeout may be specified in any of the formats accepted by SET - statement_timeout. Allowed values vary depending on the server - version you are dumping from, but an integer number of milliseconds - is accepted by all versions since 7.3. This option is ignored when - dumping from a pre-7.3 server. + statement_timeout. From 20d3fe9009ddbbbb3da3a2da298f922054b43f8c Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 26 Oct 2020 11:36:53 -0400 Subject: [PATCH 374/589] In INSERT/UPDATE, use the table's real tuple descriptor as target. Previously, ExecInitModifyTable relied on ExecInitJunkFilter, and thence ExecCleanTypeFromTL, to build the target descriptor from the query tlist. While we just checked (in ExecCheckPlanOutput) that the tlist produces compatible output, this is not a great substitute for the relation's actual tuple descriptor that's available from the relcache. For one thing, dropped columns will not be correctly marked attisdropped; it's a bit surprising that we've gotten away with that this long. But the real reason for being concerned with this is that using the table's descriptor means that the slot will have correct attrmissing data, allowing us to revert the klugy fix of commit ba9f18abd. (This commit undoes that one's changes in trigger.c, but keeps the new test case.) Thus we can solve the bogus-trigger-tuple problem with fewer cycles rather than more. No back-patch, since this doesn't fix any additional bug, and it seems somewhat more likely to have unforeseen side effects than ba9f18abd's narrow fix. Discussion: https://postgr.es/m/16644-5da7ef98a7ac4545@postgresql.org --- src/backend/commands/trigger.c | 38 +---------------------- src/backend/executor/execJunk.c | 42 +++++++++++++++++++++----- src/backend/executor/nodeModifyTable.c | 22 +++++++++++--- src/include/executor/executor.h | 3 ++ 4 files changed, 55 insertions(+), 50 deletions(-) diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c index 59289f8d4d37..092ac1646de4 100644 --- a/src/backend/commands/trigger.c +++ b/src/backend/commands/trigger.c @@ -89,8 +89,6 @@ static bool GetTupleForTrigger(EState *estate, LockTupleMode lockmode, TupleTableSlot *oldslot, TupleTableSlot **newSlot); -static HeapTuple MaterializeTupleForTrigger(TupleTableSlot *slot, - bool *shouldFree); static bool TriggerEnabled(EState *estate, ResultRelInfo *relinfo, Trigger *trigger, TriggerEvent event, Bitmapset *modifiedCols, @@ -2674,7 +2672,7 @@ ExecBRUpdateTriggers(EState *estate, EPQState *epqstate, ExecCopySlot(newslot, epqslot_clean); } - trigtuple = MaterializeTupleForTrigger(oldslot, &should_free_trig); + trigtuple = ExecFetchSlotHeapTuple(oldslot, true, &should_free_trig); } else { @@ -3043,40 +3041,6 @@ GetTupleForTrigger(EState *estate, return true; } -/* - * Extract a HeapTuple that we can pass off to trigger functions. - * - * We must materialize the tuple and make sure it is not dependent on any - * attrmissing data. This is needed for the old row in BEFORE UPDATE - * triggers, since they can choose to pass back this exact tuple as the update - * result, causing the tuple to be inserted into an executor slot that lacks - * the attrmissing data. - * - * Currently we don't seem to need to remove the attrmissing dependency in any - * other cases, but keep this as a separate function to simplify fixing things - * if that changes. - */ -static HeapTuple -MaterializeTupleForTrigger(TupleTableSlot *slot, bool *shouldFree) -{ - HeapTuple tup; - TupleDesc tupdesc = slot->tts_tupleDescriptor; - - tup = ExecFetchSlotHeapTuple(slot, true, shouldFree); - if (HeapTupleHeaderGetNatts(tup->t_data) < tupdesc->natts && - tupdesc->constr && tupdesc->constr->missing) - { - HeapTuple newtup; - - newtup = heap_expand_tuple(tup, tupdesc); - if (*shouldFree) - heap_freetuple(tup); - *shouldFree = true; - tup = newtup; - } - return tup; -} - /* * Is trigger enabled to fire? */ diff --git a/src/backend/executor/execJunk.c b/src/backend/executor/execJunk.c index 40d700dd9e23..1a822ff24b38 100644 --- a/src/backend/executor/execJunk.c +++ b/src/backend/executor/execJunk.c @@ -54,23 +54,48 @@ * * The source targetlist is passed in. The output tuple descriptor is * built from the non-junk tlist entries. - * An optional resultSlot can be passed as well. + * An optional resultSlot can be passed as well; otherwise, we create one. */ JunkFilter * ExecInitJunkFilter(List *targetList, TupleTableSlot *slot) { - JunkFilter *junkfilter; TupleDesc cleanTupType; - int cleanLength; - AttrNumber *cleanMap; - ListCell *t; - AttrNumber cleanResno; /* * Compute the tuple descriptor for the cleaned tuple. */ cleanTupType = ExecCleanTypeFromTL(targetList); + /* + * The rest is the same as ExecInitJunkFilterInsertion, ie, we want to map + * every non-junk targetlist column into the output tuple. + */ + return ExecInitJunkFilterInsertion(targetList, cleanTupType, slot); +} + +/* + * ExecInitJunkFilterInsertion + * + * Initialize a JunkFilter for insertions into a table. + * + * Here, we are given the target "clean" tuple descriptor rather than + * inferring it from the targetlist. Although the target descriptor can + * contain deleted columns, that is not of concern here, since the targetlist + * should contain corresponding NULL constants (cf. ExecCheckPlanOutput). + * It is assumed that the caller has checked that the table's columns match up + * with the non-junk columns of the targetlist. + */ +JunkFilter * +ExecInitJunkFilterInsertion(List *targetList, + TupleDesc cleanTupType, + TupleTableSlot *slot) +{ + JunkFilter *junkfilter; + int cleanLength; + AttrNumber *cleanMap; + ListCell *t; + AttrNumber cleanResno; + /* * Use the given slot, or make a new slot if we weren't given one. */ @@ -93,17 +118,18 @@ ExecInitJunkFilter(List *targetList, TupleTableSlot *slot) if (cleanLength > 0) { cleanMap = (AttrNumber *) palloc(cleanLength * sizeof(AttrNumber)); - cleanResno = 1; + cleanResno = 0; foreach(t, targetList) { TargetEntry *tle = lfirst(t); if (!tle->resjunk) { - cleanMap[cleanResno - 1] = tle->resno; + cleanMap[cleanResno] = tle->resno; cleanResno++; } } + Assert(cleanResno == cleanLength); } else cleanMap = NULL; diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index a33423c896e3..29e07b722875 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -2591,15 +2591,27 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) TupleTableSlot *junkresslot; subplan = mtstate->mt_plans[i]->plan; - if (operation == CMD_INSERT || operation == CMD_UPDATE) - ExecCheckPlanOutput(resultRelInfo->ri_RelationDesc, - subplan->targetlist); junkresslot = ExecInitExtraTupleSlot(estate, NULL, table_slot_callbacks(resultRelInfo->ri_RelationDesc)); - j = ExecInitJunkFilter(subplan->targetlist, - junkresslot); + + /* + * For an INSERT or UPDATE, the result tuple must always match + * the target table's descriptor. For a DELETE, it won't + * (indeed, there's probably no non-junk output columns). + */ + if (operation == CMD_INSERT || operation == CMD_UPDATE) + { + ExecCheckPlanOutput(resultRelInfo->ri_RelationDesc, + subplan->targetlist); + j = ExecInitJunkFilterInsertion(subplan->targetlist, + RelationGetDescr(resultRelInfo->ri_RelationDesc), + junkresslot); + } + else + j = ExecInitJunkFilter(subplan->targetlist, + junkresslot); if (operation == CMD_UPDATE || operation == CMD_DELETE) { diff --git a/src/include/executor/executor.h b/src/include/executor/executor.h index b7978cd22ebc..0c48d2a519e8 100644 --- a/src/include/executor/executor.h +++ b/src/include/executor/executor.h @@ -156,6 +156,9 @@ extern void ResetTupleHashTable(TupleHashTable hashtable); */ extern JunkFilter *ExecInitJunkFilter(List *targetList, TupleTableSlot *slot); +extern JunkFilter *ExecInitJunkFilterInsertion(List *targetList, + TupleDesc cleanTupType, + TupleTableSlot *slot); extern JunkFilter *ExecInitJunkFilterConversion(List *targetList, TupleDesc cleanTupType, TupleTableSlot *slot); From e9661f2b0f3310d8bacc3f0802b309f599300cfb Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 26 Oct 2020 19:17:05 -0400 Subject: [PATCH 375/589] doc: make blooms docs match reality Parallel execution changed the way bloom queries are executed, so update the EXPLAIN output, and restructure the docs to be clearer and more accurate. Reported-by: Daniel Westermann Discussion: https://postgr.es/m/ZR0P278MB0122119FAE78721A694C30C8D2340@ZR0P278MB0122.CHEP278.PROD.OUTLOOK.COM Author: Daniel Westermann and me Backpatch-through: 9.6 --- doc/src/sgml/bloom.sgml | 117 +++++++++++++++++++++------------------- 1 file changed, 62 insertions(+), 55 deletions(-) diff --git a/doc/src/sgml/bloom.sgml b/doc/src/sgml/bloom.sgml index 285b67b3f10d..d1cf9ac24a75 100644 --- a/doc/src/sgml/bloom.sgml +++ b/doc/src/sgml/bloom.sgml @@ -110,75 +110,70 @@ CREATE INDEX bloomidx ON tbloom USING bloom (i1,i2,i3) FROM generate_series(1,10000000); SELECT 10000000 -=# CREATE INDEX bloomidx ON tbloom USING bloom (i1, i2, i3, i4, i5, i6); -CREATE INDEX -=# SELECT pg_size_pretty(pg_relation_size('bloomidx')); - pg_size_pretty ----------------- - 153 MB -(1 row) -=# CREATE index btreeidx ON tbloom (i1, i2, i3, i4, i5, i6); -CREATE INDEX -=# SELECT pg_size_pretty(pg_relation_size('btreeidx')); - pg_size_pretty ----------------- - 387 MB -(1 row) A sequential scan over this large table takes a long time: =# EXPLAIN ANALYZE SELECT * FROM tbloom WHERE i2 = 898732 AND i5 = 123451; - QUERY PLAN --------------------------------------------------------------------&zwsp;----------------------------------------- - Seq Scan on tbloom (cost=0.00..213694.08 rows=1 width=24) (actual time=1445.438..1445.438 rows=0 loops=1) + QUERY PLAN +-------------------------------------------------------------------&zwsp;----------------------------------- + Seq Scan on tbloom (cost=0.00..2137.14 rows=3 width=24) (actual time=16.971..16.971 rows=0 loops=1) Filter: ((i2 = 898732) AND (i5 = 123451)) - Rows Removed by Filter: 10000000 - Planning time: 0.177 ms - Execution time: 1445.473 ms + Rows Removed by Filter: 100000 + Planning Time: 0.346 ms + Execution Time: 16.988 ms (5 rows) - So the planner will usually select an index scan if possible. - With a btree index, we get results like this: + Even with the btree index defined the result will still be a + sequential scan: +=# CREATE INDEX btreeidx ON tbloom (i1, i2, i3, i4, i5, i6); +CREATE INDEX +=# SELECT pg_size_pretty(pg_relation_size('btreeidx')); + pg_size_pretty +---------------- + 3976 kB +(1 row) =# EXPLAIN ANALYZE SELECT * FROM tbloom WHERE i2 = 898732 AND i5 = 123451; - QUERY PLAN --------------------------------------------------------------------&zwsp;------------------------------------------------------------- - Index Only Scan using btreeidx on tbloom (cost=0.56..298311.96 rows=1 width=24) (actual time=445.709..445.709 rows=0 loops=1) - Index Cond: ((i2 = 898732) AND (i5 = 123451)) - Heap Fetches: 0 - Planning time: 0.193 ms - Execution time: 445.770 ms + QUERY PLAN +-------------------------------------------------------------------&zwsp;----------------------------------- + Seq Scan on tbloom (cost=0.00..2137.00 rows=2 width=24) (actual time=12.805..12.805 rows=0 loops=1) + Filter: ((i2 = 898732) AND (i5 = 123451)) + Rows Removed by Filter: 100000 + Planning Time: 0.138 ms + Execution Time: 12.817 ms (5 rows) - Bloom is better than btree in handling this type of search: + Having the bloom index defined on the table is better than btree in + handling this type of search: +=# CREATE INDEX bloomidx ON tbloom USING bloom (i1, i2, i3, i4, i5, i6); +CREATE INDEX +=# SELECT pg_size_pretty(pg_relation_size('bloomidx')); + pg_size_pretty +---------------- + 1584 kB +(1 row) =# EXPLAIN ANALYZE SELECT * FROM tbloom WHERE i2 = 898732 AND i5 = 123451; - QUERY PLAN --------------------------------------------------------------------&zwsp;-------------------------------------------------------- - Bitmap Heap Scan on tbloom (cost=178435.39..178439.41 rows=1 width=24) (actual time=76.698..76.698 rows=0 loops=1) + QUERY PLAN +-------------------------------------------------------------------&zwsp;-------------------------------------------------- + Bitmap Heap Scan on tbloom (cost=1792.00..1799.69 rows=2 width=24) (actual time=0.388..0.388 rows=0 loops=1) Recheck Cond: ((i2 = 898732) AND (i5 = 123451)) - Rows Removed by Index Recheck: 2439 - Heap Blocks: exact=2408 - -> Bitmap Index Scan on bloomidx (cost=0.00..178435.39 rows=1 width=0) (actual time=72.455..72.455 rows=2439 loops=1) + Rows Removed by Index Recheck: 29 + Heap Blocks: exact=28 + -> Bitmap Index Scan on bloomidx (cost=0.00..1792.00 rows=2 width=0) (actual time=0.356..0.356 rows=29 loops=1) Index Cond: ((i2 = 898732) AND (i5 = 123451)) - Planning time: 0.475 ms - Execution time: 76.778 ms + Planning Time: 0.099 ms + Execution Time: 0.408 ms (8 rows) - Note the relatively large number of false positives: 2439 rows were - selected to be visited in the heap, but none actually matched the - query. We could reduce that by specifying a larger signature length. - In this example, creating the index with length=200 - reduced the number of false positives to 55; but it doubled the index size - (to 306 MB) and ended up being slower for this query (125 ms overall). @@ -187,24 +182,36 @@ CREATE INDEX A better strategy for btree is to create a separate index on each column. Then the planner will choose something like this: +=# CREATE INDEX btreeidx1 ON tbloom (i1); +CREATE INDEX +=# CREATE INDEX btreeidx2 ON tbloom (i2); +CREATE INDEX +=# CREATE INDEX btreeidx3 ON tbloom (i3); +CREATE INDEX +=# CREATE INDEX btreeidx4 ON tbloom (i4); +CREATE INDEX +=# CREATE INDEX btreeidx5 ON tbloom (i5); +CREATE INDEX +=# CREATE INDEX btreeidx6 ON tbloom (i6); +CREATE INDEX =# EXPLAIN ANALYZE SELECT * FROM tbloom WHERE i2 = 898732 AND i5 = 123451; - QUERY PLAN --------------------------------------------------------------------&zwsp;----------------------------------------------------------- - Bitmap Heap Scan on tbloom (cost=9.29..13.30 rows=1 width=24) (actual time=0.148..0.148 rows=0 loops=1) + QUERY PLAN +-------------------------------------------------------------------&zwsp;-------------------------------------------------------- + Bitmap Heap Scan on tbloom (cost=24.34..32.03 rows=2 width=24) (actual time=0.028..0.029 rows=0 loops=1) Recheck Cond: ((i5 = 123451) AND (i2 = 898732)) - -> BitmapAnd (cost=9.29..9.29 rows=1 width=0) (actual time=0.145..0.145 rows=0 loops=1) - -> Bitmap Index Scan on tbloom_i5_idx (cost=0.00..4.52 rows=11 width=0) (actual time=0.089..0.089 rows=10 loops=1) + -> BitmapAnd (cost=24.34..24.34 rows=2 width=0) (actual time=0.027..0.027 rows=0 loops=1) + -> Bitmap Index Scan on btreeidx5 (cost=0.00..12.04 rows=500 width=0) (actual time=0.026..0.026 rows=0 loops=1) Index Cond: (i5 = 123451) - -> Bitmap Index Scan on tbloom_i2_idx (cost=0.00..4.52 rows=11 width=0) (actual time=0.048..0.048 rows=8 loops=1) + -> Bitmap Index Scan on btreeidx2 (cost=0.00..12.04 rows=500 width=0) (never executed) Index Cond: (i2 = 898732) - Planning time: 2.049 ms - Execution time: 0.280 ms + Planning Time: 0.491 ms + Execution Time: 0.055 ms (9 rows) Although this query runs much faster than with either of the single - indexes, we pay a large penalty in index size. Each of the single-column - btree indexes occupies 214 MB, so the total space needed is over 1.2GB, - more than 8 times the space used by the bloom index. + indexes, we pay a penalty in index size. Each of the single-column + btree indexes occupies 2 MB, so the total space needed is 12 MB, + eight times the space used by the bloom index. From 2f17fe431860fc434084d99d8b789e6ec7e6977b Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 26 Oct 2020 22:38:11 -0400 Subject: [PATCH 376/589] doc: simplify wording of function error affects Reported-by: bob.henkel@gmail.com Discussion: https://postgr.es/m/160324449781.693.8298142858847611071@wrigleys.postgresql.org Backpatch-through: 9.5 --- doc/src/sgml/plpgsql.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/plpgsql.sgml b/doc/src/sgml/plpgsql.sgml index 94a3b36458df..9ec168b0c4ce 100644 --- a/doc/src/sgml/plpgsql.sgml +++ b/doc/src/sgml/plpgsql.sgml @@ -2670,8 +2670,8 @@ NOTICE: row = {10,11,12} By default, any error occurring in a PL/pgSQL - function aborts execution of the function, and indeed of the - surrounding transaction as well. You can trap errors and recover + function aborts execution of the function and the + surrounding transaction. You can trap errors and recover from them by using a BEGIN block with an EXCEPTION clause. The syntax is an extension of the normal syntax for a BEGIN block: From 2f0760c9ff827bb3d23ee327e6b46038000c7ef9 Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Tue, 27 Oct 2020 08:22:39 +0530 Subject: [PATCH 377/589] Update description of spilled counters in pg_stat_replication_slots view. This is to make the description of spilled counters clear. Author: Amit Kapila Reviewed-by: Sawada Masahiko Discussion: https://postgr.es/m/CA+fd4k5_pPAYRTDrO2PbtTOe0eHQpBvuqmCr8ic39uTNmR49Eg@mail.gmail.com --- doc/src/sgml/monitoring.sgml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index f5cf163c8c68..aeb694d4a86d 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -2603,9 +2603,9 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i Number of transactions spilled to disk after the memory used by - logical decoding exceeds logical_decoding_work_mem. The - counter gets incremented both for toplevel transactions and - subtransactions. + logical decoding of changes from WAL for this slot exceeds + logical_decoding_work_mem. The counter gets + incremented both for toplevel transactions and subtransactions. @@ -2614,7 +2614,8 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i spill_count bigint - Number of times transactions were spilled to disk. Transactions + Number of times transactions were spilled to disk while performing + decoding of changes from WAL for this slot. Transactions may get spilled repeatedly, and this counter gets incremented on every such invocation. @@ -2625,7 +2626,10 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i spill_bytes bigint - Amount of decoded transaction data spilled to disk. + Amount of decoded transaction data spilled to disk while performing + decoding of changes from WAL for this slot. This and other spill + counters can be used to gauge the I/O occurred during logical decoding + and accordingly can tune logical_decoding_work_mem. From 8fed2eadb8558185d162cc7fd09192c2e10b915f Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 27 Oct 2020 08:43:35 +0100 Subject: [PATCH 378/589] doc: Fix order of protocol messages in listing Move GSSENCRequest to the correct alphabetical position. --- doc/src/sgml/protocol.sgml | 75 +++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 3a64db6f5509..9a95d7b734da 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -4858,6 +4858,44 @@ FunctionCallResponse (B) + + +GSSENCRequest (F) + + + + + + + + Int32(8) + + + + Length of message contents in bytes, including self. + + + + + + Int32(80877104) + + + + The GSSAPI Encryption request code. The value is chosen to contain + 1234 in the most significant 16 bits, and 5680 in the + least significant 16 bits. (To avoid confusion, this code + must not be the same as any protocol version number.) + + + + + + + + + + GSSResponse (F) @@ -5823,43 +5861,6 @@ SSLRequest (F) - - -GSSENCRequest (F) - - - - - - - - Int32(8) - - - - Length of message contents in bytes, including self. - - - - - - Int32(80877104) - - - - The GSSAPI Encryption request code. The value is chosen to contain - 1234 in the most significant 16 bits, and 5680 in the - least significant 16 bits. (To avoid confusion, this code - must not be the same as any protocol version number.) - - - - - - - - - From 9213462c539e6412fe0498a7f8e20b662e15c4ec Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 27 Oct 2020 08:58:48 +0100 Subject: [PATCH 379/589] Make procedure OUT parameters work with JDBC The JDBC driver sends OUT parameters with type void. This makes sense when calling a function, so that the parameters are ignored in ParseFuncOrColumn(). For a procedure call we want to treat them as unknown. Reviewed-by: Andrew Dunstan Discussion: https://www.postgresql.org/message-id/flat/d7e49540-ea92-b4e2-5fff-42036102f968%402ndquadrant.com --- src/backend/parser/parse_param.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/backend/parser/parse_param.c b/src/backend/parser/parse_param.c index 17a96abfa8c3..93c9d82d017d 100644 --- a/src/backend/parser/parse_param.c +++ b/src/backend/parser/parse_param.c @@ -163,6 +163,15 @@ variable_paramref_hook(ParseState *pstate, ParamRef *pref) if (*pptype == InvalidOid) *pptype = UNKNOWNOID; + /* + * If the argument is of type void and it's procedure call, interpret it + * as unknown. This allows the JDBC driver to not have to distinguish + * function and procedure calls. See also another component of this hack + * in ParseFuncOrColumn(). + */ + if (*pptype == VOIDOID && pstate->p_expr_kind == EXPR_KIND_CALL_ARGUMENT) + *pptype = UNKNOWNOID; + param = makeNode(Param); param->paramkind = PARAM_EXTERN; param->paramid = paramno; From 0525572860335d050a1bea194a5278c8833304d1 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 27 Oct 2020 11:50:18 +0100 Subject: [PATCH 380/589] Fix enum errdetail to mention bytes, not chars The enum label length is in terms of bytes, not charactes. Author: Ian Lawrence Barwick Reviewed-by: Julien Rouhaud Discussion: https://www.postgresql.org/message-id/flat/CAB8KJ=itZEJ7C9BacTHSYgeUysH4xx8wDiOnyppnSLyn6-g+Bw@mail.gmail.com --- src/backend/catalog/pg_enum.c | 6 +++--- src/test/regress/expected/enum.out | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/catalog/pg_enum.c b/src/backend/catalog/pg_enum.c index 27e4100a6f4d..6a2c6685a0a5 100644 --- a/src/backend/catalog/pg_enum.c +++ b/src/backend/catalog/pg_enum.c @@ -125,7 +125,7 @@ EnumValuesCreate(Oid enumTypeOid, List *vals) ereport(ERROR, (errcode(ERRCODE_INVALID_NAME), errmsg("invalid enum label \"%s\"", lab), - errdetail("Labels must be %d characters or less.", + errdetail("Labels must be %d bytes or less.", NAMEDATALEN - 1))); values[Anum_pg_enum_oid - 1] = ObjectIdGetDatum(oids[elemno]); @@ -228,7 +228,7 @@ AddEnumLabel(Oid enumTypeOid, ereport(ERROR, (errcode(ERRCODE_INVALID_NAME), errmsg("invalid enum label \"%s\"", newVal), - errdetail("Labels must be %d characters or less.", + errdetail("Labels must be %d bytes or less.", NAMEDATALEN - 1))); /* @@ -523,7 +523,7 @@ RenameEnumLabel(Oid enumTypeOid, ereport(ERROR, (errcode(ERRCODE_INVALID_NAME), errmsg("invalid enum label \"%s\"", newVal), - errdetail("Labels must be %d characters or less.", + errdetail("Labels must be %d bytes or less.", NAMEDATALEN - 1))); /* diff --git a/src/test/regress/expected/enum.out b/src/test/regress/expected/enum.out index dffff88928e8..908f67efffb8 100644 --- a/src/test/regress/expected/enum.out +++ b/src/test/regress/expected/enum.out @@ -92,7 +92,7 @@ ORDER BY enumlabel::planets; ALTER TYPE planets ADD VALUE 'plutoplutoplutoplutoplutoplutoplutoplutoplutoplutoplutoplutoplutopluto'; ERROR: invalid enum label "plutoplutoplutoplutoplutoplutoplutoplutoplutoplutoplutoplutoplutopluto" -DETAIL: Labels must be 63 characters or less. +DETAIL: Labels must be 63 bytes or less. ALTER TYPE planets ADD VALUE 'pluto' AFTER 'zeus'; ERROR: "zeus" is not an existing enum label -- if not exists tests From 4066b642909176aede03b6c64d29734ce2a34716 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 27 Oct 2020 12:43:11 -0400 Subject: [PATCH 381/589] docs: remove reference to src/tools/thread This directory and the ability to build the thread test independently were removed in commit 8a2121185b. Reported-by: e.indrupskaya@postgrespro.ru Discussion: https://postgr.es/m/160379609706.24746.7506163279454026608@wrigleys.postgresql.org Backpatch-through: 9.5 --- doc/src/sgml/libpq.sgml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index de60281fcb49..9ce32fb39b54 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -8052,14 +8052,6 @@ int PQisthreadsafe(); libpq source code for a way to do cooperative locking between libpq and your application. - - - If you experience problems with threaded applications, run the program - in src/tools/thread to see if your platform has - thread-unsafe functions. This program is run by - configure, but for binary distributions your - library might not match the library used to build the binaries. - From 59ab4ac32460a6a93b665f4e487d7ff64979ba4d Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Tue, 27 Oct 2020 13:49:19 -0300 Subject: [PATCH 382/589] Accept relations of any kind in LOCK TABLE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The restriction that only tables and views can be locked by LOCK TABLE is quite arbitrary, since the underlying mechanism can lock any relation type. Drop the restriction so that programs such as pg_dump can lock all relations they're interested in, preventing schema changes that could cause a dump to fail after expending much effort. Backpatch to 9.5. Author: Álvaro Herrera Reviewed-by: Tom Lane Reported-by: Wells Oliver Discussion: https://postgr.es/m/20201021200659.GA32358@alvherre.pgsql --- doc/src/sgml/ref/lock.sgml | 15 +++++++++------ src/backend/commands/lockcmds.c | 28 +++++++++------------------- src/test/regress/expected/lock.out | 11 +++++++++++ src/test/regress/sql/lock.sql | 13 +++++++++++++ 4 files changed, 42 insertions(+), 25 deletions(-) diff --git a/doc/src/sgml/ref/lock.sgml b/doc/src/sgml/ref/lock.sgml index 4cdfae2279e3..37881f25ac57 100644 --- a/doc/src/sgml/ref/lock.sgml +++ b/doc/src/sgml/ref/lock.sgml @@ -16,7 +16,7 @@ PostgreSQL documentation LOCK - lock a table + lock a named relation (table, etc) @@ -34,7 +34,9 @@ LOCK [ TABLE ] [ ONLY ] name [ * ] Description - LOCK TABLE obtains a table-level lock, waiting + LOCK TABLE obtains a table-level lock on a + relation (table, partitioned table, foreign table, view, + materialized view, index, composite type, sequence), waiting if necessary for any conflicting locks to be released. If NOWAIT is specified, LOCK TABLE does not wait to acquire the desired lock: if it @@ -115,17 +117,18 @@ LOCK [ TABLE ] [ ONLY ] name [ * ] name - The name (optionally schema-qualified) of an existing table to - lock. If ONLY is specified before the table name, only that + The name (optionally schema-qualified) of an existing relation to + lock. If ONLY is specified before a table name, only that table is locked. If ONLY is not specified, the table and all its descendant tables (if any) are locked. Optionally, * can be specified after the table name to explicitly indicate that - descendant tables are included. + descendant tables are included. When locking a view, all relations appearing + in the view definition are locked, regardless of ONLY. The command LOCK TABLE a, b; is equivalent to - LOCK TABLE a; LOCK TABLE b;. The tables are locked + LOCK TABLE a; LOCK TABLE b;. The relations are locked one-by-one in the order specified in the LOCK TABLE command. diff --git a/src/backend/commands/lockcmds.c b/src/backend/commands/lockcmds.c index d8cafc42bb54..f64246dba391 100644 --- a/src/backend/commands/lockcmds.c +++ b/src/backend/commands/lockcmds.c @@ -83,14 +83,6 @@ RangeVarCallbackForLockTable(const RangeVar *rv, Oid relid, Oid oldrelid, return; /* woops, concurrently dropped; no permissions * check */ - /* Currently, we only allow plain tables or views to be locked */ - if (relkind != RELKIND_RELATION && relkind != RELKIND_PARTITIONED_TABLE && - relkind != RELKIND_VIEW) - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("\"%s\" is not a table or view", - rv->relname))); - /* * Make note if a temporary relation has been accessed in this * transaction. @@ -186,11 +178,13 @@ LockViewRecurse_walker(Node *node, LockViewRecurse_context *context) foreach(rtable, query->rtable) { RangeTblEntry *rte = lfirst(rtable); + Oid relid; AclResult aclresult; - Oid relid = rte->relid; - char relkind = rte->relkind; - char *relname = get_rel_name(relid); + /* ignore all non-relation RTEs */ + if (rte->rtekind != RTE_RELATION) + continue; + relid = rte->relid; /* * The OLD and NEW placeholder entries in the view's rtable are @@ -201,11 +195,6 @@ LockViewRecurse_walker(Node *node, LockViewRecurse_context *context) strcmp(rte->eref->aliasname, "new") == 0)) continue; - /* Currently, we only allow plain tables or views to be locked. */ - if (relkind != RELKIND_RELATION && relkind != RELKIND_PARTITIONED_TABLE && - relkind != RELKIND_VIEW) - continue; - /* Check infinite recursion in the view definition. */ if (list_member_oid(context->ancestor_views, relid)) ereport(ERROR, @@ -216,7 +205,8 @@ LockViewRecurse_walker(Node *node, LockViewRecurse_context *context) /* Check permissions with the view owner's privilege. */ aclresult = LockTableAclCheck(relid, context->lockmode, context->viewowner); if (aclresult != ACLCHECK_OK) - aclcheck_error(aclresult, get_relkind_objtype(relkind), relname); + aclcheck_error(aclresult, get_relkind_objtype(rte->relkind), + get_rel_name(relid)); /* We have enough rights to lock the relation; do so. */ if (!context->nowait) @@ -225,9 +215,9 @@ LockViewRecurse_walker(Node *node, LockViewRecurse_context *context) ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), errmsg("could not obtain lock on relation \"%s\"", - relname))); + get_rel_name(relid)))); - if (relkind == RELKIND_VIEW) + if (rte->relkind == RELKIND_VIEW) LockViewRecurse(relid, context->lockmode, context->nowait, context->ancestor_views); else if (rte->inh) LockTableRecurse(relid, context->lockmode, context->nowait); diff --git a/src/test/regress/expected/lock.out b/src/test/regress/expected/lock.out index 2e54688dffe3..1d6a4e9f1e00 100644 --- a/src/test/regress/expected/lock.out +++ b/src/test/regress/expected/lock.out @@ -12,6 +12,9 @@ CREATE VIEW lock_view3 AS SELECT * from lock_view2; CREATE VIEW lock_view4 AS SELECT (select a from lock_tbl1a limit 1) from lock_tbl1; CREATE VIEW lock_view5 AS SELECT * from lock_tbl1 where a in (select * from lock_tbl1a); CREATE VIEW lock_view6 AS SELECT * from (select * from lock_tbl1) sub; +CREATE MATERIALIZED VIEW lock_mv1 AS SELECT * FROM lock_view6; +CREATE INDEX lock_mvi1 ON lock_mv1 (a); +CREATE SEQUENCE lock_seq; CREATE ROLE regress_rol_lock1; ALTER ROLE regress_rol_lock1 SET search_path = lock_schema1; GRANT USAGE ON SCHEMA lock_schema1 TO regress_rol_lock1; @@ -154,9 +157,16 @@ BEGIN; LOCK TABLE ONLY lock_tbl1; ROLLBACK; RESET ROLE; +-- Lock other relations +BEGIN TRANSACTION; +LOCK TABLE lock_mv1; +LOCK TABLE lock_mvi1; +LOCK TABLE lock_seq; +ROLLBACK; -- -- Clean up -- +DROP MATERIALIZED VIEW lock_mv1; DROP VIEW lock_view7; DROP VIEW lock_view6; DROP VIEW lock_view5; @@ -168,6 +178,7 @@ DROP TABLE lock_tbl3; DROP TABLE lock_tbl2; DROP TABLE lock_tbl1; DROP TABLE lock_tbl1a; +DROP SEQUENCE lock_seq; DROP SCHEMA lock_schema1 CASCADE; DROP ROLE regress_rol_lock1; -- atomic ops tests diff --git a/src/test/regress/sql/lock.sql b/src/test/regress/sql/lock.sql index e50cb6f06428..98d13fc8e1e8 100644 --- a/src/test/regress/sql/lock.sql +++ b/src/test/regress/sql/lock.sql @@ -13,6 +13,9 @@ CREATE VIEW lock_view3 AS SELECT * from lock_view2; CREATE VIEW lock_view4 AS SELECT (select a from lock_tbl1a limit 1) from lock_tbl1; CREATE VIEW lock_view5 AS SELECT * from lock_tbl1 where a in (select * from lock_tbl1a); CREATE VIEW lock_view6 AS SELECT * from (select * from lock_tbl1) sub; +CREATE MATERIALIZED VIEW lock_mv1 AS SELECT * FROM lock_view6; +CREATE INDEX lock_mvi1 ON lock_mv1 (a); +CREATE SEQUENCE lock_seq; CREATE ROLE regress_rol_lock1; ALTER ROLE regress_rol_lock1 SET search_path = lock_schema1; GRANT USAGE ON SCHEMA lock_schema1 TO regress_rol_lock1; @@ -117,9 +120,18 @@ LOCK TABLE ONLY lock_tbl1; ROLLBACK; RESET ROLE; +-- Lock other relations +BEGIN TRANSACTION; +LOCK TABLE lock_mv1; +LOCK TABLE lock_mvi1; +LOCK TABLE lock_seq; +ROLLBACK; + + -- -- Clean up -- +DROP MATERIALIZED VIEW lock_mv1; DROP VIEW lock_view7; DROP VIEW lock_view6; DROP VIEW lock_view5; @@ -130,6 +142,7 @@ DROP TABLE lock_tbl3; DROP TABLE lock_tbl2; DROP TABLE lock_tbl1; DROP TABLE lock_tbl1a; +DROP SEQUENCE lock_seq; DROP SCHEMA lock_schema1 CASCADE; DROP ROLE regress_rol_lock1; From f893e68d761adbee7f888109b1adf76151e3e17a Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 27 Oct 2020 17:39:23 +0100 Subject: [PATCH 383/589] Add select_common_typmod() This accompanies select_common_type() and select_common_collation(). Typmods were previously combined using hand-coded logic in several places. The logic in select_common_typmod() isn't very exciting, but it makes the code more compact and readable in a few locations, and in the future we can perhaps do more complicated things if desired. As a small enhancement, the type unification of the direct and aggregate arguments of hypothetical-set aggregates now unifies the typmod as well using this new function, instead of just dropping it. Reviewed-by: Heikki Linnakangas Discussion: https://www.postgresql.org/message-id/flat/97df3af9-8b5e-fb7f-a029-3eb7e80d7af9@2ndquadrant.com --- src/backend/parser/analyze.c | 26 +++++----------------- src/backend/parser/parse_clause.c | 23 +++++-------------- src/backend/parser/parse_coerce.c | 37 +++++++++++++++++++++++++++++++ src/backend/parser/parse_func.c | 8 +++++-- src/include/parser/parse_coerce.h | 2 ++ 5 files changed, 57 insertions(+), 39 deletions(-) diff --git a/src/backend/parser/analyze.c b/src/backend/parser/analyze.c index c159fb2957ba..98a83db8b5cf 100644 --- a/src/backend/parser/analyze.c +++ b/src/backend/parser/analyze.c @@ -1434,9 +1434,8 @@ transformValuesClause(ParseState *pstate, SelectStmt *stmt) for (i = 0; i < sublist_length; i++) { Oid coltype; - int32 coltypmod = -1; + int32 coltypmod; Oid colcoll; - bool first = true; coltype = select_common_type(pstate, colexprs[i], "VALUES", NULL); @@ -1446,19 +1445,9 @@ transformValuesClause(ParseState *pstate, SelectStmt *stmt) col = coerce_to_common_type(pstate, col, coltype, "VALUES"); lfirst(lc) = (void *) col; - if (first) - { - coltypmod = exprTypmod(col); - first = false; - } - else - { - /* As soon as we see a non-matching typmod, fall back to -1 */ - if (coltypmod >= 0 && coltypmod != exprTypmod(col)) - coltypmod = -1; - } } + coltypmod = select_common_typmod(pstate, colexprs[i], coltype); colcoll = select_common_collation(pstate, colexprs[i], true); coltypes = lappend_oid(coltypes, coltype); @@ -2020,8 +2009,6 @@ transformSetOperationTree(ParseState *pstate, SelectStmt *stmt, Node *rcolnode = (Node *) rtle->expr; Oid lcoltype = exprType(lcolnode); Oid rcoltype = exprType(rcolnode); - int32 lcoltypmod = exprTypmod(lcolnode); - int32 rcoltypmod = exprTypmod(rcolnode); Node *bestexpr; int bestlocation; Oid rescoltype; @@ -2034,11 +2021,6 @@ transformSetOperationTree(ParseState *pstate, SelectStmt *stmt, context, &bestexpr); bestlocation = exprLocation(bestexpr); - /* if same type and same typmod, use typmod; else default */ - if (lcoltype == rcoltype && lcoltypmod == rcoltypmod) - rescoltypmod = lcoltypmod; - else - rescoltypmod = -1; /* * Verify the coercions are actually possible. If not, we'd fail @@ -2089,6 +2071,10 @@ transformSetOperationTree(ParseState *pstate, SelectStmt *stmt, rtle->expr = (Expr *) rcolnode; } + rescoltypmod = select_common_typmod(pstate, + list_make2(lcolnode, rcolnode), + rescoltype); + /* * Select common collation. A common collation is required for * all set operators except UNION ALL; see SQL:2008 7.13 vartype; - outcoltypmod = l_colvar->vartypmod; - if (outcoltype != r_colvar->vartype) - { - outcoltype = select_common_type(pstate, + outcoltype = select_common_type(pstate, + list_make2(l_colvar, r_colvar), + "JOIN/USING", + NULL); + outcoltypmod = select_common_typmod(pstate, list_make2(l_colvar, r_colvar), - "JOIN/USING", - NULL); - outcoltypmod = -1; /* ie, unknown */ - } - else if (outcoltypmod != r_colvar->vartypmod) - { - /* same type, but not same typmod */ - outcoltypmod = -1; /* ie, unknown */ - } + outcoltype); /* * Insert coercion functions if needed. Note that a difference in typmod diff --git a/src/backend/parser/parse_coerce.c b/src/backend/parser/parse_coerce.c index 2ffe47026b9b..a2924e3d1ce0 100644 --- a/src/backend/parser/parse_coerce.c +++ b/src/backend/parser/parse_coerce.c @@ -1522,6 +1522,43 @@ coerce_to_common_type(ParseState *pstate, Node *node, return node; } +/* + * select_common_typmod() + * Determine the common typmod of a list of input expressions. + * + * common_type is the selected common type of the expressions, typically + * computed using select_common_type(). + */ +int32 +select_common_typmod(ParseState *pstate, List *exprs, Oid common_type) +{ + ListCell *lc; + bool first = true; + int32 result = -1; + + foreach(lc, exprs) + { + Node *expr = (Node *) lfirst(lc); + + /* Types must match */ + if (exprType(expr) != common_type) + return -1; + else if (first) + { + result = exprTypmod(expr); + first = false; + } + else + { + /* As soon as we see a non-matching typmod, fall back to -1 */ + if (result != exprTypmod(expr)) + return -1; + } + } + + return result; +} + /* * check_generic_type_consistency() * Are the actual arguments potentially compatible with a diff --git a/src/backend/parser/parse_func.c b/src/backend/parser/parse_func.c index 9c3b6ad91662..a7a31704fb4f 100644 --- a/src/backend/parser/parse_func.c +++ b/src/backend/parser/parse_func.c @@ -1750,6 +1750,7 @@ unify_hypothetical_args(ParseState *pstate, ListCell *harg = list_nth_cell(fargs, hargpos); ListCell *aarg = list_nth_cell(fargs, aargpos); Oid commontype; + int32 commontypmod; /* A mismatch means AggregateCreate didn't check properly ... */ if (declared_arg_types[hargpos] != declared_arg_types[aargpos]) @@ -1768,6 +1769,9 @@ unify_hypothetical_args(ParseState *pstate, list_make2(lfirst(aarg), lfirst(harg)), "WITHIN GROUP", NULL); + commontypmod = select_common_typmod(pstate, + list_make2(lfirst(aarg), lfirst(harg)), + commontype); /* * Perform the coercions. We don't need to worry about NamedArgExprs @@ -1776,7 +1780,7 @@ unify_hypothetical_args(ParseState *pstate, lfirst(harg) = coerce_type(pstate, (Node *) lfirst(harg), actual_arg_types[hargpos], - commontype, -1, + commontype, commontypmod, COERCION_IMPLICIT, COERCE_IMPLICIT_CAST, -1); @@ -1784,7 +1788,7 @@ unify_hypothetical_args(ParseState *pstate, lfirst(aarg) = coerce_type(pstate, (Node *) lfirst(aarg), actual_arg_types[aargpos], - commontype, -1, + commontype, commontypmod, COERCION_IMPLICIT, COERCE_IMPLICIT_CAST, -1); diff --git a/src/include/parser/parse_coerce.h b/src/include/parser/parse_coerce.h index 8686eaacbc9b..33d7cfc8b646 100644 --- a/src/include/parser/parse_coerce.h +++ b/src/include/parser/parse_coerce.h @@ -71,6 +71,8 @@ extern Node *coerce_to_common_type(ParseState *pstate, Node *node, Oid targetTypeId, const char *context); +extern int32 select_common_typmod(ParseState *pstate, List *exprs, Oid common_type); + extern bool check_generic_type_consistency(const Oid *actual_arg_types, const Oid *declared_arg_types, int nargs); From 403a3d91c841beabf3efd7bffddb47a2bce4481f Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Tue, 27 Oct 2020 14:31:37 -0300 Subject: [PATCH 384/589] pg_dump: Lock all relations, not just plain tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that LOCK TABLE can take any relation type, acquire lock on all relations that are to be dumped. This prevents schema changes or deadlock errors that could cause a dump to fail after expending much effort. The server is tested to have the capability and the feature disabled if it doesn't, so that a patched pg_dump doesn't fail when connecting to an unpatched server. Backpatch to 9.5. Author: Álvaro Herrera Reviewed-by: Tom Lane Reported-by: Wells Oliver Discussion: https://postgr.es/m/20201021200659.GA32358@alvherre.pgsql --- src/bin/pg_dump/pg_backup.h | 2 ++ src/bin/pg_dump/pg_backup_db.c | 64 ++++++++++++++++++++++++++++++++++ src/bin/pg_dump/pg_backup_db.h | 2 ++ src/bin/pg_dump/pg_dump.c | 17 +++++---- 4 files changed, 78 insertions(+), 7 deletions(-) diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index a6a8e6f2fd86..6c7931916414 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -198,6 +198,8 @@ typedef struct Archive int minRemoteVersion; /* allowable range */ int maxRemoteVersion; + bool hasGenericLockTable; /* can LOCK TABLE do non-table rels */ + int numWorkers; /* number of parallel processes */ char *sync_snapshot_id; /* sync snapshot id for parallel operation */ diff --git a/src/bin/pg_dump/pg_backup_db.c b/src/bin/pg_dump/pg_backup_db.c index 5ba43441f50a..28b2892538ae 100644 --- a/src/bin/pg_dump/pg_backup_db.c +++ b/src/bin/pg_dump/pg_backup_db.c @@ -531,6 +531,70 @@ EndDBCopyMode(Archive *AHX, const char *tocEntryTag) } } +/* + * Does LOCK TABLE work on non-table relations on this server? + * + * Note: assumes it is called out of any transaction + */ +bool +IsLockTableGeneric(Archive *AHX) +{ + ArchiveHandle *AH = (ArchiveHandle *) AHX; + PGresult *res; + char *sqlstate; + bool retval; + + if (AHX->remoteVersion >= 140000) + return true; + else if (AHX->remoteVersion < 90500) + return false; + + StartTransaction(AHX); + + /* + * Try a LOCK TABLE on a well-known non-table catalog; WRONG_OBJECT_TYPE + * tells us that this server doesn't support locking non-table rels, while + * LOCK_NOT_AVAILABLE and INSUFFICIENT_PRIVILEGE tell us that it does. + * Report anything else as a fatal problem. + */ +#define ERRCODE_INSUFFICIENT_PRIVILEGE "42501" +#define ERRCODE_WRONG_OBJECT_TYPE "42809" +#define ERRCODE_LOCK_NOT_AVAILABLE "55P03" + res = PQexec(AH->connection, + "LOCK TABLE pg_catalog.pg_class_tblspc_relfilenode_index IN ACCESS SHARE MODE NOWAIT"); + switch (PQresultStatus(res)) + { + case PGRES_COMMAND_OK: + retval = true; + break; + case PGRES_FATAL_ERROR: + sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE); + if (strcmp(sqlstate, ERRCODE_WRONG_OBJECT_TYPE) == 0) + { + retval = false; + break; + } + else if (strcmp(sqlstate, ERRCODE_LOCK_NOT_AVAILABLE) == 0 || + strcmp(sqlstate, ERRCODE_INSUFFICIENT_PRIVILEGE) == 0) + { + retval = true; + break; + } + /* else, falls through */ + default: + warn_or_exit_horribly(AH, "LOCK TABLE failed for \"%s\": %s", + "pg_catalog.pg_class_tblspc_relfilenode_index", + PQerrorMessage(AH->connection)); + retval = false; /* not reached */ + break; + } + PQclear(res); + + CommitTransaction(AHX); + + return retval; +} + void StartTransaction(Archive *AHX) { diff --git a/src/bin/pg_dump/pg_backup_db.h b/src/bin/pg_dump/pg_backup_db.h index 8888dd34b9b0..355beec1f7c7 100644 --- a/src/bin/pg_dump/pg_backup_db.h +++ b/src/bin/pg_dump/pg_backup_db.h @@ -20,6 +20,8 @@ extern PGresult *ExecuteSqlQueryForSingleRow(Archive *fout, const char *query); extern void EndDBCopyMode(Archive *AHX, const char *tocEntryTag); +extern bool IsLockTableGeneric(Archive *AHX); + extern void StartTransaction(Archive *AHX); extern void CommitTransaction(Archive *AHX); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index ff45e3fb8c37..03f9d4d9e84f 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -1176,6 +1176,9 @@ setup_connection(Archive *AH, const char *dumpencoding, ExecuteSqlStatement(AH, "SET row_security = off"); } + /* Detect whether LOCK TABLE can handle non-table relations */ + AH->hasGenericLockTable = IsLockTableGeneric(AH); + /* * Start transaction-snapshot mode transaction to dump consistent data. */ @@ -6843,16 +6846,16 @@ getTables(Archive *fout, int *numTables) * assume our lock on the child is enough to prevent schema * alterations to parent tables. * - * NOTE: it'd be kinda nice to lock other relations too, not only - * plain or partitioned tables, but the backend doesn't presently - * allow that. - * - * We only need to lock the table for certain components; see + * We only need to lock the relation for certain components; see * pg_dump.h + * + * On server versions that support it, we lock all relations not just + * plain tables. */ if (tblinfo[i].dobj.dump && - (tblinfo[i].relkind == RELKIND_RELATION || - tblinfo->relkind == RELKIND_PARTITIONED_TABLE) && + (fout->hasGenericLockTable || + tblinfo[i].relkind == RELKIND_PARTITIONED_TABLE || + tblinfo[i].relkind == RELKIND_RELATION) && (tblinfo[i].dobj.dump & DUMP_COMPONENTS_REQUIRING_LOCK)) { resetPQExpBuffer(query); From 8e5d1aef8eed26977739b662cddcea2de5efe834 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 27 Oct 2020 14:00:38 -0400 Subject: [PATCH 385/589] Makefile comment: remove reference to tools/thread/thread_test You can't compile thread_test alone anymore, and the location moved too. Reported-by: Tom Lane Discussion: https://postgr.es/m/1062278.1603819969@sss.pgh.pa.us Backpatch-through: 9.5 --- src/template/netbsd | 1 - 1 file changed, 1 deletion(-) diff --git a/src/template/netbsd b/src/template/netbsd index d97f995c748d..aaa560cd92c9 100644 --- a/src/template/netbsd +++ b/src/template/netbsd @@ -1,5 +1,4 @@ # src/template/netbsd -# tools/thread/thread_test must be run # Extra CFLAGS for code that will go into a shared library CFLAGS_SL="-fPIC -DPIC" From 8d132b2850d4c95ed5666a09b295dea645bbc884 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 27 Oct 2020 14:31:12 -0400 Subject: [PATCH 386/589] Doc: improve explanation of how to use our code coverage infrastructure. The reference to running "make coverage" in a subdirectory was a bit obscure, so clarify what happens when you do that. Do a little desultory copy-editing, too. Per a question from Peter Smith. Discussion: https://postgr.es/m/CAHut+Pu0r3AjRSyu5E0v2-zRj8r24OSrkWs3fEBxOuaw1i8DKA@mail.gmail.com --- doc/src/sgml/regress.sgml | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/doc/src/sgml/regress.sgml b/doc/src/sgml/regress.sgml index d98187c970e9..083d0bf46b89 100644 --- a/doc/src/sgml/regress.sgml +++ b/doc/src/sgml/regress.sgml @@ -806,12 +806,12 @@ make check PROVE_TESTS='t/001_test1.pl t/003_test3.pl' instrumentation, so that it becomes possible to examine which parts of the code are covered by the regression tests or any other test suite that is run with the code. This is currently supported - when compiling with GCC and requires the gcov + when compiling with GCC, and it requires the gcov and lcov programs. - A typical workflow would look like this: + A typical workflow looks like this: ./configure --enable-coverage ... OTHER OPTIONS ... make @@ -820,12 +820,11 @@ make coverage-html Then point your HTML browser to coverage/index.html. - The make commands also work in subdirectories. If you don't have lcov or prefer text output over an - HTML report, you can also run + HTML report, you can run make coverage @@ -837,11 +836,23 @@ make coverage - To reset the execution counts between test runs, run: + You can run several different tests before making the coverage report; + the execution counts will accumulate. If you want + to reset the execution counts between test runs, run: make coverage-clean + + + You can run the make coverage-html or make + coverage command in a subdirectory if you want a coverage + report for only a portion of the code tree. + + + + Use make distclean to clean up when done. + From 9e0f87a4955ede0769d2f9a93171145b6ddab901 Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Wed, 28 Oct 2020 07:28:51 +0530 Subject: [PATCH 387/589] Minor improvements in description of spilled counters in pg_stat_replication_slots view. Per a suggestion by Justin Pryzby. Discussion: https://postgr.es/m/CA+fd4k5_pPAYRTDrO2PbtTOe0eHQpBvuqmCr8ic39uTNmR49Eg@mail.gmail.com --- doc/src/sgml/monitoring.sgml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index aeb694d4a86d..313e44ed5498 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -2614,10 +2614,9 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i spill_count bigint - Number of times transactions were spilled to disk while performing - decoding of changes from WAL for this slot. Transactions - may get spilled repeatedly, and this counter gets incremented on every - such invocation. + Number of times transactions were spilled to disk while decoding changes + from WAL for this slot. Transactions may get spilled repeatedly, and + this counter gets incremented on every such invocation. @@ -2628,8 +2627,8 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i Amount of decoded transaction data spilled to disk while performing decoding of changes from WAL for this slot. This and other spill - counters can be used to gauge the I/O occurred during logical decoding - and accordingly can tune logical_decoding_work_mem. + counters can be used to gauge the I/O which occurred during logical + decoding and allow tuning logical_decoding_work_mem. From c780a7a90a8e7b074405ea2007e34f94e227e695 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 28 Oct 2020 11:12:46 +0900 Subject: [PATCH 388/589] Add CheckBuffer() to check on-disk pages without shared buffer loading CheckBuffer() is designed to be a concurrent-safe function able to run sanity checks on a relation page without loading it into the shared buffers. The operation is done using a lock on the partition involved in the shared buffer mapping hashtable and an I/O lock for the buffer itself, preventing the risk of false positives due to any concurrent activity. The primary use of this function is the detection of on-disk corruptions for relation pages. If a page is found in shared buffers, the on-disk page is checked if not dirty (a follow-up checkpoint would flush a valid version of the page if dirty anyway), as it could be possible that a page was present for a long time in shared buffers with its on-disk version corrupted. Such a scenario could lead to a corrupted cluster if a host is plugged off for example. If the page is not found in shared buffers, its on-disk state is checked. PageIsVerifiedExtended() is used to apply the same sanity checks as when a page gets loaded into shared buffers. This function will be used by an upcoming patch able to check the state of on-disk relation pages using a SQL function. Author: Julien Rouhaud, Michael Paquier Reviewed-by: Masahiko Sawada Discussion: https://postgr.es/m/CAOBaU_aVvMjQn=ge5qPiJOPMmOj5=ii3st5Q0Y+WuLML5sR17w@mail.gmail.com --- src/backend/storage/buffer/bufmgr.c | 92 +++++++++++++++++++++++++++++ src/include/storage/bufmgr.h | 3 + 2 files changed, 95 insertions(+) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 3eee86afe5c4..2fa0b065a287 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -4585,3 +4585,95 @@ TestForOldSnapshot_impl(Snapshot snapshot, Relation relation) (errcode(ERRCODE_SNAPSHOT_TOO_OLD), errmsg("snapshot too old"))); } + + +/* + * CheckBuffer + * + * Check the state of a buffer without loading it into the shared buffers. To + * avoid torn pages and possible false positives when reading data, a shared + * LWLock is taken on the target buffer pool partition mapping, and we check + * if the page is in shared buffers or not. An I/O lock is taken on the block + * to prevent any concurrent activity from happening. + * + * If the page is found as dirty in the shared buffers, it is ignored as + * it will be flushed to disk either before the end of the next checkpoint + * or during recovery in the event of an unsafe shutdown. + * + * If the page is found in the shared buffers but is not dirty, we still + * check the state of its data on disk, as it could be possible that the + * page stayed in shared buffers for a rather long time while the on-disk + * data got corrupted. + * + * If the page is not found in shared buffers, the block is read from disk + * while holding the buffer pool partition mapping LWLock. + * + * The page data is stored in a private memory area local to this function + * while running the checks. + */ +bool +CheckBuffer(SMgrRelation smgr, ForkNumber forknum, BlockNumber blkno) +{ + char buffer[BLCKSZ]; + BufferTag buf_tag; /* identity of requested block */ + uint32 buf_hash; /* hash value for buf_tag */ + LWLock *partLock; /* buffer partition lock for the buffer */ + BufferDesc *bufdesc; + int buf_id; + + Assert(smgrexists(smgr, forknum)); + + /* create a tag so we can look after the buffer */ + INIT_BUFFERTAG(buf_tag, smgr->smgr_rnode.node, forknum, blkno); + + /* determine its hash code and partition lock ID */ + buf_hash = BufTableHashCode(&buf_tag); + partLock = BufMappingPartitionLock(buf_hash); + + /* see if the block is in the buffer pool or not */ + LWLockAcquire(partLock, LW_SHARED); + buf_id = BufTableLookup(&buf_tag, buf_hash); + if (buf_id >= 0) + { + uint32 buf_state; + + /* + * Found it. Now, retrieve its state to know what to do with it, and + * release the pin immediately. We do so to limit overhead as much as + * possible. We keep the shared LWLock on the target buffer mapping + * partition for now, so this buffer cannot be evicted, and we acquire + * an I/O Lock on the buffer as we may need to read its contents from + * disk. + */ + bufdesc = GetBufferDescriptor(buf_id); + + LWLockAcquire(BufferDescriptorGetIOLock(bufdesc), LW_SHARED); + buf_state = LockBufHdr(bufdesc); + UnlockBufHdr(bufdesc, buf_state); + + /* If the page is dirty or invalid, skip it */ + if ((buf_state & BM_DIRTY) != 0 || (buf_state & BM_TAG_VALID) == 0) + { + LWLockRelease(BufferDescriptorGetIOLock(bufdesc)); + LWLockRelease(partLock); + return true; + } + + /* Read the buffer from disk, with the I/O lock still held */ + smgrread(smgr, forknum, blkno, buffer); + LWLockRelease(BufferDescriptorGetIOLock(bufdesc)); + } + else + { + /* + * Simply read the buffer. There's no risk of modification on it as + * we are holding the buffer pool partition mapping lock. + */ + smgrread(smgr, forknum, blkno, buffer); + } + + /* buffer lookup done, so now do its check */ + LWLockRelease(partLock); + + return PageIsVerifiedExtended(buffer, blkno, PIV_REPORT_STAT); +} diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index ee91b8fa26c1..a21cab2eaf8c 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -240,6 +240,9 @@ extern void AtProcExit_LocalBuffers(void); extern void TestForOldSnapshot_impl(Snapshot snapshot, Relation relation); +extern bool CheckBuffer(struct SMgrRelationData *smgr, ForkNumber forknum, + BlockNumber blkno); + /* in freelist.c */ extern BufferAccessStrategy GetAccessStrategy(BufferAccessStrategyType btype); extern void FreeAccessStrategy(BufferAccessStrategy strategy); From f2b883969557f4572cdfa87e1a40083d2b1272e7 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 28 Oct 2020 12:15:00 +0900 Subject: [PATCH 389/589] Add pg_relation_check_pages() to check on-disk pages of a relation This makes use of CheckBuffer() introduced in c780a7a, adding a SQL wrapper able to do checks for all the pages of a relation. By default, all the fork types of a relation are checked, and it is possible to check only a given relation fork. Note that if the relation given in input has no physical storage or is temporary, then no errors are generated, allowing full-database checks when coupled with a simple scan of pg_class for example. This is not limited to clusters with data checksums enabled, as clusters without data checksums can still apply checks on pages using the page headers or for the case of a page full of zeros. This function returns a set of tuples consisting of: - The physical file where a broken page has been detected (without the segment number as that can be AM-dependent, which can be guessed from the block number for heap). A relative path from PGPATH is used. - The block number of the broken page. By default, only superusers have an access to this function but execution rights can be granted to other users. The feature introduced here is still minimal, and more improvements could be done, like: - Addition of a start and end block number to run checks on a range of blocks, which would apply only if one fork type is checked. - Addition of some progress reporting. - Throttling, with configuration parameters in function input or potentially some cost-based GUCs. Regression tests are added for positive cases in the main regression test suite, and TAP tests are added for cases involving the emulation of page corruptions. Bump catalog version. Author: Julien Rouhaud, Michael Paquier Reviewed-by: Masahiko Sawada, Justin Pryzby Discussion: https://postgr.es/m/CAOBaU_aVvMjQn=ge5qPiJOPMmOj5=ii3st5Q0Y+WuLML5sR17w@mail.gmail.com --- doc/src/sgml/func.sgml | 50 +++++ src/backend/catalog/system_views.sql | 9 + src/backend/utils/adt/Makefile | 1 + src/backend/utils/adt/pagefuncs.c | 229 +++++++++++++++++++++++ src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.dat | 7 + src/test/recovery/t/022_page_check.pl | 231 ++++++++++++++++++++++++ src/test/regress/expected/pagefuncs.out | 72 ++++++++ src/test/regress/parallel_schedule | 2 +- src/test/regress/serial_schedule | 1 + src/test/regress/sql/pagefuncs.sql | 41 +++++ src/tools/pgindent/typedefs.list | 1 + 12 files changed, 644 insertions(+), 2 deletions(-) create mode 100644 src/backend/utils/adt/pagefuncs.c create mode 100644 src/test/recovery/t/022_page_check.pl create mode 100644 src/test/regress/expected/pagefuncs.out create mode 100644 src/test/regress/sql/pagefuncs.sql diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index f7f401b534c7..7ef2ec997259 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -26182,6 +26182,56 @@ SELECT convert_from(pg_read_binary_file('file_in_utf8.txt'), 'UTF8'); + + Data Sanity Functions + + + The functions shown in + provide ways to check the sanity of data files in the cluster. + + + + Data Sanity Functions + + + Name Return Type Description + + + + + + + pg_relation_check_pages(relation regclass [, fork text DEFAULT NULL ]) + + setof record + Check the pages of a relation. + + + + +
+ + + pg_relation_check_pages + + + pg_relation_check_pages iterates over all blocks of a + given relation and verifies if they are in a state where they can safely + be loaded into the shared buffers. If defined, + fork specifies that only the pages of the given + fork are to be verified. Fork can be 'main' for the + main data fork, 'fsm' for the free space map, + 'vm' for the visibility map, or + 'init' for the initialization fork. The default of + NULL means that all the forks of the relation are + checked. The function returns a list of blocks that are considered as + corrupted with the path of the related file. Use of this function is + restricted to superusers by default but access may be granted to others + using GRANT. + + + + diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 85cd147e21bb..c6dd084fbccb 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1300,6 +1300,14 @@ LANGUAGE INTERNAL STRICT VOLATILE AS 'pg_create_logical_replication_slot'; +CREATE OR REPLACE FUNCTION pg_relation_check_pages( + IN relation regclass, IN fork text DEFAULT NULL, + OUT path text, OUT failed_block_num bigint) +RETURNS SETOF record +LANGUAGE internal +VOLATILE PARALLEL RESTRICTED +AS 'pg_relation_check_pages'; + CREATE OR REPLACE FUNCTION make_interval(years int4 DEFAULT 0, months int4 DEFAULT 0, weeks int4 DEFAULT 0, days int4 DEFAULT 0, hours int4 DEFAULT 0, mins int4 DEFAULT 0, @@ -1444,6 +1452,7 @@ AS 'unicode_is_normalized'; -- can later change who can access these functions, or leave them as only -- available to superuser / cluster owner, if they choose. -- +REVOKE EXECUTE ON FUNCTION pg_relation_check_pages(regclass, text) FROM public; REVOKE EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_stop_backup() FROM public; REVOKE EXECUTE ON FUNCTION pg_stop_backup(boolean, boolean) FROM public; diff --git a/src/backend/utils/adt/Makefile b/src/backend/utils/adt/Makefile index b4d55e849b3d..e2279af1e52b 100644 --- a/src/backend/utils/adt/Makefile +++ b/src/backend/utils/adt/Makefile @@ -69,6 +69,7 @@ OBJS = \ oid.o \ oracle_compat.o \ orderedsetaggs.o \ + pagefuncs.o \ partitionfuncs.o \ pg_locale.o \ pg_lsn.o \ diff --git a/src/backend/utils/adt/pagefuncs.c b/src/backend/utils/adt/pagefuncs.c new file mode 100644 index 000000000000..2ef133ba45bd --- /dev/null +++ b/src/backend/utils/adt/pagefuncs.c @@ -0,0 +1,229 @@ +/*------------------------------------------------------------------------- + * + * pagefuncs.c + * Functions for features related to relation pages. + * + * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/utils/adt/pagefuncs.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/relation.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "storage/bufmgr.h" +#include "storage/lmgr.h" +#include "storage/smgr.h" +#include "utils/builtins.h" +#include "utils/syscache.h" + +static void check_one_relation(TupleDesc tupdesc, Tuplestorestate *tupstore, + Oid relid, ForkNumber single_forknum); +static void check_relation_fork(TupleDesc tupdesc, Tuplestorestate *tupstore, + Relation relation, ForkNumber forknum); + +/* + * callback arguments for check_pages_error_callback() + */ +typedef struct CheckPagesErrorInfo +{ + char *path; + BlockNumber blkno; +} CheckPagesErrorInfo; + +/* + * Error callback specific to check_relation_fork(). + */ +static void +check_pages_error_callback(void *arg) +{ + CheckPagesErrorInfo *errinfo = (CheckPagesErrorInfo *) arg; + + errcontext("while checking page %u of path %s", + errinfo->blkno, errinfo->path); +} + +/* + * pg_relation_check_pages + * + * Check the state of all the pages for one or more fork types in the given + * relation. + */ +Datum +pg_relation_check_pages(PG_FUNCTION_ARGS) +{ + Oid relid; + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + TupleDesc tupdesc; + Tuplestorestate *tupstore; + MemoryContext per_query_ctx; + MemoryContext oldcontext; + ForkNumber forknum; + + /* Switch into long-lived context to construct returned data structures */ + per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; + oldcontext = MemoryContextSwitchTo(per_query_ctx); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + tupstore = tuplestore_begin_heap(true, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; + + MemoryContextSwitchTo(oldcontext); + + /* handle arguments */ + if (PG_ARGISNULL(0)) + { + /* Just leave if nothing is defined */ + PG_RETURN_VOID(); + } + + /* By default all the forks of a relation are checked */ + if (PG_ARGISNULL(1)) + forknum = InvalidForkNumber; + else + { + const char *forkname = TextDatumGetCString(PG_GETARG_TEXT_PP(1)); + + forknum = forkname_to_number(forkname); + } + + relid = PG_GETARG_OID(0); + + check_one_relation(tupdesc, tupstore, relid, forknum); + tuplestore_donestoring(tupstore); + + return (Datum) 0; +} + +/* + * Perform the check on a single relation, possibly filtered with a single + * fork. This function will check if the given relation exists or not, as + * a relation could be dropped after checking for the list of relations and + * before getting here, and we don't want to error out in this case. + */ +static void +check_one_relation(TupleDesc tupdesc, Tuplestorestate *tupstore, + Oid relid, ForkNumber single_forknum) +{ + Relation relation; + ForkNumber forknum; + + /* Check if relation exists. leaving if there is no such relation */ + if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(relid))) + return; + + relation = relation_open(relid, AccessShareLock); + + /* + * Sanity checks, returning no results if not supported. Temporary + * relations and relations without storage are out of scope. + */ + if (!RELKIND_HAS_STORAGE(relation->rd_rel->relkind) || + relation->rd_rel->relpersistence == RELPERSISTENCE_TEMP) + { + relation_close(relation, AccessShareLock); + return; + } + + RelationOpenSmgr(relation); + + for (forknum = 0; forknum <= MAX_FORKNUM; forknum++) + { + if (single_forknum != InvalidForkNumber && single_forknum != forknum) + continue; + + if (smgrexists(relation->rd_smgr, forknum)) + check_relation_fork(tupdesc, tupstore, relation, forknum); + } + + relation_close(relation, AccessShareLock); +} + +/* + * For a given relation and fork, do the real work of iterating over all pages + * and doing the check. Caller must hold an AccessShareLock lock on the given + * relation. + */ +static void +check_relation_fork(TupleDesc tupdesc, Tuplestorestate *tupstore, + Relation relation, ForkNumber forknum) +{ + BlockNumber blkno, + nblocks; + SMgrRelation smgr = relation->rd_smgr; + char *path; + CheckPagesErrorInfo errinfo; + ErrorContextCallback errcallback; + + /* Number of output arguments in the SRF */ +#define PG_CHECK_RELATION_COLS 2 + + Assert(CheckRelationLockedByMe(relation, AccessShareLock, true)); + + /* + * We remember the number of blocks here. Since caller must hold a lock + * on the relation, we know that it won't be truncated while we are + * iterating over the blocks. Any block added after this function started + * will not be checked. + */ + nblocks = RelationGetNumberOfBlocksInFork(relation, forknum); + + path = relpathbackend(smgr->smgr_rnode.node, + smgr->smgr_rnode.backend, + forknum); + + /* + * Error context to print some information about blocks and relations + * impacted by corruptions. + */ + errinfo.path = pstrdup(path); + errinfo.blkno = 0; + errcallback.callback = check_pages_error_callback; + errcallback.arg = (void *) &errinfo; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + for (blkno = 0; blkno < nblocks; blkno++) + { + Datum values[PG_CHECK_RELATION_COLS]; + bool nulls[PG_CHECK_RELATION_COLS]; + int i = 0; + + /* Update block number for the error context */ + errinfo.blkno = blkno; + + CHECK_FOR_INTERRUPTS(); + + /* Check the given buffer */ + if (CheckBuffer(smgr, forknum, blkno)) + continue; + + memset(values, 0, sizeof(values)); + memset(nulls, 0, sizeof(nulls)); + + values[i++] = CStringGetTextDatum(path); + values[i++] = UInt32GetDatum(blkno); + + Assert(i == PG_CHECK_RELATION_COLS); + + /* Save the corrupted blocks in the tuplestore. */ + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + + pfree(path); + } + + /* Pop the error context stack */ + error_context_stack = errcallback.previous; +} diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index f44a09b0c251..8cf02fc0d852 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 202010201 +#define CATALOG_VERSION_NO 202010281 #endif diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index bbcac69d48f7..a66870bcc080 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10958,6 +10958,13 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '9147', descr => 'check pages of a relation', + proname => 'pg_relation_check_pages', procost => '10000', prorows => '20', + proisstrict => 'f', proretset => 't', provolatile => 'v', proparallel => 'r', + prorettype => 'record', proargtypes => 'regclass text', + proallargtypes => '{regclass,text,text,int8}', proargmodes => '{i,i,o,o}', + proargnames => '{relation,fork,path,failed_block_num}', + prosrc => 'pg_relation_check_pages' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/recovery/t/022_page_check.pl b/src/test/recovery/t/022_page_check.pl new file mode 100644 index 000000000000..9dd4225c5a0d --- /dev/null +++ b/src/test/recovery/t/022_page_check.pl @@ -0,0 +1,231 @@ +# Emulate on-disk corruptions of relation pages and find such corruptions +# using pg_relation_check_pages(). + +use strict; +use warnings; + +use PostgresNode; +use TestLib; +use Test::More tests => 20; + +our $CHECKSUM_UINT16_OFFSET = 4; +our $PD_UPPER_UINT16_OFFSET = 7; +our $BLOCKSIZE; +our $TOTAL_NB_ERR = 0; + +# Grab a relation page worth a size of BLOCKSIZE from given $filename. +# $blkno is the same block number as for a relation file. +sub read_page +{ + my ($filename, $blkno) = @_; + my $block; + + open(my $infile, '<', $filename) or die; + binmode($infile); + + my $success = read($infile, $block, $BLOCKSIZE, ($blkno * $BLOCKSIZE)); + die($!) if !defined($success); + + close($infile); + + return ($block); +} + +# Update an existing page of size BLOCKSIZE with new contents in given +# $filename. $blkno is the block number assigned in the relation file. +sub write_page +{ + my ($filename, $block, $blkno) = @_; + + open(my $outfile, '>', $filename) or die; + binmode($outfile); + + my $nb = syswrite($outfile, $block, $BLOCKSIZE, ($blkno * $BLOCKSIZE)); + + die($!) if not defined $nb; + die("Write error") if ($nb != $BLOCKSIZE); + + $outfile->flush(); + + close($outfile); + return; +} + +# Read 2 bytes from relation page at a given offset. +sub get_uint16_from_page +{ + my ($block, $offset) = @_; + + return (unpack("S*", $block))[$offset]; +} + +# Write 2 bytes to relation page at a given offset. +sub set_uint16_to_page +{ + my ($block, $data, $offset) = @_; + + my $pack = pack("S", $data); + + # vec with 16B or more won't preserve endianness. + vec($block, 2 * $offset, 8) = (unpack('C*', $pack))[0]; + vec($block, (2 * $offset) + 1, 8) = (unpack('C*', $pack))[1]; + + return $block; +} + +# Sanity check on pg_stat_database looking after the number of checksum +# failures. +sub check_pg_stat_database +{ + my ($node, $test_prefix) = @_; + + my $stdout = $node->safe_psql('postgres', + "SELECT " + . " sum(checksum_failures)" + . " FROM pg_catalog.pg_stat_database"); + is($stdout, $TOTAL_NB_ERR, + "$test_prefix: pg_stat_database should have $TOTAL_NB_ERR error"); + + return; +} + +# Run a round of page checks for any relation present in this test run. +# $expected_broken is the psql output marking all the pages found as +# corrupted using relname|blkno as format for each tuple returned. $nb +# is the new number of checksum errors added to the global counter +# matched with the contents of pg_stat_database. +# +# Note that this has no need to check system relations as these would have +# no corruptions: this test does not manipulate them and should by no mean +# break the cluster. +sub run_page_checks +{ + my ($node, $num_checksum, $expected_broken, $test_prefix) = @_; + + my $stdout = $node->safe_psql('postgres', + "SELECT relname, failed_block_num" + . " FROM (SELECT relname, (pg_catalog.pg_relation_check_pages(oid)).*" + . " FROM pg_class " + . " WHERE relkind in ('r','i', 'm') AND oid >= 16384) AS s"); + + # Check command result + is($stdout, $expected_broken, + "$test_prefix: output mismatch with pg_relation_check_pages()"); + + $TOTAL_NB_ERR += $num_checksum; + return; +} + +# Perform various tests that modify a specified block at the given +# offset, checking that a page corruption is correctly detected. The +# original contents of the page are restored back once done. +# $broken_pages is the set of pages that are expected to be broken +# as of the returned result of pg_relation_check_pages(). $num_checksum +# is the number of checksum failures expected to be added to the contents +# of pg_stat_database after this function is done. +sub corrupt_and_test_block +{ + my ($node, $filename, $blkno, $offset, $broken_pages, $num_checksum, + $test_prefix) + = @_; + my $fake_data = hex '0x0000'; + + # Stop the server cleanly to flush any pages, and to prevent any + # concurrent updates on what is going to be updated. + $node->stop; + my $original_block = read_page($filename, 0); + my $original_data = get_uint16_from_page($original_block, $offset); + + isnt($original_data, $fake_data, + "$test_prefix: fake data at offset $offset should be different from the existing one" + ); + + my $new_block = set_uint16_to_page($original_block, $fake_data, $offset); + isnt( + $original_data, + get_uint16_from_page($new_block, $offset), + "$test_prefix: The data at offset $offset should have been changed in memory" + ); + + write_page($filename, $new_block, 0); + + my $written_data = get_uint16_from_page(read_page($filename, 0), $offset); + + # Some offline checks to validate that the corrupted data is in place. + isnt($original_data, $written_data, + "$test_prefix: data written at offset $offset should be different from the original one" + ); + is( get_uint16_from_page($new_block, $offset), + $written_data, + "$test_prefix: data written at offset $offset should be the same as the one in memory" + ); + is($written_data, $fake_data, + "$test_prefix: The data written at offset $offset should be the one we wanted to write" + ); + + # The corruption is in place, start the server to run the checks. + $node->start; + run_page_checks($node, $num_checksum, $broken_pages, $test_prefix); + + # Stop the server, put the original page back in place. + $node->stop; + + $new_block = set_uint16_to_page($original_block, $original_data, $offset); + is( $original_data, + get_uint16_from_page($new_block, $offset), + "$test_prefix: data at offset $offset should have been restored in memory" + ); + + write_page($filename, $new_block, 0); + is( $original_data, + get_uint16_from_page(read_page($filename, $blkno), $offset), + "$test_prefix: data at offset $offset should have been restored on disk" + ); + + # There should be no errors now that the contents are back in place. + $node->start; + run_page_checks($node, 0, '', $test_prefix); +} + +# Data checksums are necessary for this test. +my $node = get_new_node('main'); +$node->init(extra => ['--data-checksums']); +$node->start; + +my $stdout = + $node->safe_psql('postgres', "SELECT" . " current_setting('block_size')"); + +$BLOCKSIZE = $stdout; + +# Basic schema to corrupt and check +$node->safe_psql( + 'postgres', q| + CREATE TABLE public.t1(id integer); + INSERT INTO public.t1 SELECT generate_series(1, 100); + CHECKPOINT; +|); + +# Get the path to the relation file that will get manipulated by the +# follow-up tests with some on-disk corruptions. +$stdout = $node->safe_psql('postgres', + "SELECT" + . " current_setting('data_directory') || '/' || pg_relation_filepath('t1')" +); + +my $filename = $stdout; + +# Normal case without corruptions, this passes, with pg_stat_database +# reporting no errors. +check_pg_stat_database($node, 'start'); + +# Test with a modified checksum. +corrupt_and_test_block($node, $filename, 0, $CHECKSUM_UINT16_OFFSET, 't1|0', + 1, 'broken checksum'); + +# Test corruption making the block looking like it validates PageIsNew(). +corrupt_and_test_block($node, $filename, 0, $PD_UPPER_UINT16_OFFSET, 't1|0', + 0, 'new page'); + +# Check that the number of errors in pg_stat_database match what we +# expect with the corruptions previously introduced. +check_pg_stat_database($node, 'end'); diff --git a/src/test/regress/expected/pagefuncs.out b/src/test/regress/expected/pagefuncs.out new file mode 100644 index 000000000000..38a72b01b3fd --- /dev/null +++ b/src/test/regress/expected/pagefuncs.out @@ -0,0 +1,72 @@ +-- +-- Tests for functions related to relation pages +-- +-- Restricted to superusers by default +CREATE ROLE regress_pgfunc_user; +SET ROLE regress_pgfunc_user; +SELECT pg_relation_check_pages('pg_class'); -- error +ERROR: permission denied for function pg_relation_check_pages +SELECT pg_relation_check_pages('pg_class', 'main'); -- error +ERROR: permission denied for function pg_relation_check_pages +RESET ROLE; +DROP ROLE regress_pgfunc_user; +-- NULL and simple sanity checks +SELECT pg_relation_check_pages(NULL); -- empty result + pg_relation_check_pages +------------------------- +(0 rows) + +SELECT pg_relation_check_pages(NULL, NULL); -- empty result + pg_relation_check_pages +------------------------- +(0 rows) + +SELECT pg_relation_check_pages('pg_class', 'invalid_fork'); -- error +ERROR: invalid fork name +HINT: Valid fork names are "main", "fsm", "vm", and "init". +-- Relation types that are supported +CREATE TABLE pgfunc_test_tab (id int); +CREATE INDEX pgfunc_test_ind ON pgfunc_test_tab(id); +INSERT INTO pgfunc_test_tab VALUES (generate_series(1,1000)); +SELECT pg_relation_check_pages('pgfunc_test_tab'); + pg_relation_check_pages +------------------------- +(0 rows) + +SELECT pg_relation_check_pages('pgfunc_test_ind'); + pg_relation_check_pages +------------------------- +(0 rows) + +DROP TABLE pgfunc_test_tab; +CREATE MATERIALIZED VIEW pgfunc_test_matview AS SELECT 1; +SELECT pg_relation_check_pages('pgfunc_test_matview'); + pg_relation_check_pages +------------------------- +(0 rows) + +DROP MATERIALIZED VIEW pgfunc_test_matview; +CREATE SEQUENCE pgfunc_test_seq; +SELECT pg_relation_check_pages('pgfunc_test_seq'); + pg_relation_check_pages +------------------------- +(0 rows) + +DROP SEQUENCE pgfunc_test_seq; +-- pg_relation_check_pages() returns no results if passed relations that +-- do not support the operation, like relations without storage or temporary +-- relations. +CREATE TEMPORARY TABLE pgfunc_test_temp AS SELECT generate_series(1,10) AS a; +SELECT pg_relation_check_pages('pgfunc_test_temp'); + pg_relation_check_pages +------------------------- +(0 rows) + +DROP TABLE pgfunc_test_temp; +CREATE VIEW pgfunc_test_view AS SELECT 1; +SELECT pg_relation_check_pages('pgfunc_test_view'); + pg_relation_check_pages +------------------------- +(0 rows) + +DROP VIEW pgfunc_test_view; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index ae89ed7f0b40..7a46a132525c 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -112,7 +112,7 @@ test: plancache limit plpgsql copy2 temp domain rangefuncs prepare conversion tr # ---------- # Another group of parallel tests # ---------- -test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain +test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain pagefuncs # event triggers cannot run concurrently with any test that runs DDL test: event_trigger diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule index 525bdc804f61..9a80b80f73ad 100644 --- a/src/test/regress/serial_schedule +++ b/src/test/regress/serial_schedule @@ -197,6 +197,7 @@ test: hash_part test: indexing test: partition_aggregate test: partition_info +test: pagefuncs test: tuplesort test: explain test: event_trigger diff --git a/src/test/regress/sql/pagefuncs.sql b/src/test/regress/sql/pagefuncs.sql new file mode 100644 index 000000000000..12d32eeae473 --- /dev/null +++ b/src/test/regress/sql/pagefuncs.sql @@ -0,0 +1,41 @@ +-- +-- Tests for functions related to relation pages +-- + +-- Restricted to superusers by default +CREATE ROLE regress_pgfunc_user; +SET ROLE regress_pgfunc_user; +SELECT pg_relation_check_pages('pg_class'); -- error +SELECT pg_relation_check_pages('pg_class', 'main'); -- error +RESET ROLE; +DROP ROLE regress_pgfunc_user; + +-- NULL and simple sanity checks +SELECT pg_relation_check_pages(NULL); -- empty result +SELECT pg_relation_check_pages(NULL, NULL); -- empty result +SELECT pg_relation_check_pages('pg_class', 'invalid_fork'); -- error + +-- Relation types that are supported +CREATE TABLE pgfunc_test_tab (id int); +CREATE INDEX pgfunc_test_ind ON pgfunc_test_tab(id); +INSERT INTO pgfunc_test_tab VALUES (generate_series(1,1000)); +SELECT pg_relation_check_pages('pgfunc_test_tab'); +SELECT pg_relation_check_pages('pgfunc_test_ind'); +DROP TABLE pgfunc_test_tab; + +CREATE MATERIALIZED VIEW pgfunc_test_matview AS SELECT 1; +SELECT pg_relation_check_pages('pgfunc_test_matview'); +DROP MATERIALIZED VIEW pgfunc_test_matview; +CREATE SEQUENCE pgfunc_test_seq; +SELECT pg_relation_check_pages('pgfunc_test_seq'); +DROP SEQUENCE pgfunc_test_seq; + +-- pg_relation_check_pages() returns no results if passed relations that +-- do not support the operation, like relations without storage or temporary +-- relations. +CREATE TEMPORARY TABLE pgfunc_test_temp AS SELECT generate_series(1,10) AS a; +SELECT pg_relation_check_pages('pgfunc_test_temp'); +DROP TABLE pgfunc_test_temp; +CREATE VIEW pgfunc_test_view AS SELECT 1; +SELECT pg_relation_check_pages('pgfunc_test_view'); +DROP VIEW pgfunc_test_view; diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index ff853634bc52..b6acade6c678 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -332,6 +332,7 @@ CatCacheHeader CatalogId CatalogIndexState ChangeVarNodes_context +CheckPagesErrorInfo CheckPoint CheckPointStmt CheckpointStatsData From ce7f772c5e6066e0bbafea5759e652c9757c8e6b Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 28 Oct 2020 13:59:18 +0900 Subject: [PATCH 390/589] Use correct GetDatum() in pg_relation_check_pages() UInt32GetDatum() was getting used, while the result needs Int64GetDatum(). Oversight in f2b8839. Per buildfarm member florican. Discussion: https://postgr.es/m/1226629.1603859189@sss.pgh.pa.us --- src/backend/utils/adt/pagefuncs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/utils/adt/pagefuncs.c b/src/backend/utils/adt/pagefuncs.c index 2ef133ba45bd..b6a23a2a4fa8 100644 --- a/src/backend/utils/adt/pagefuncs.c +++ b/src/backend/utils/adt/pagefuncs.c @@ -214,7 +214,7 @@ check_relation_fork(TupleDesc tupdesc, Tuplestorestate *tupstore, memset(nulls, 0, sizeof(nulls)); values[i++] = CStringGetTextDatum(path); - values[i++] = UInt32GetDatum(blkno); + values[i++] = Int64GetDatum((int64) blkno); Assert(i == PG_CHECK_RELATION_COLS); From ad1c36b0709e47cdb3cc4abd6c939fe64279b63f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 28 Oct 2020 11:15:47 -0400 Subject: [PATCH 391/589] Fix foreign-key selectivity estimation in the presence of constants. get_foreign_key_join_selectivity() looks for join clauses that equate the two sides of the FK constraint. However, if we have a query like "WHERE fktab.a = pktab.a and fktab.a = 1", it won't find any such join clause, because equivclass.c replaces the given clauses with "fktab.a = 1 and pktab.a = 1", which can be enforced at the scan level, leaving nothing to be done for column "a" at the join level. We can fix that expectation without much trouble, but then a new problem arises: applying the foreign-key-based selectivity rule produces a rowcount underestimate, because we're effectively double-counting the selectivity of the "fktab.a = 1" clause. So we have to cancel that selectivity out of the estimate. To fix, refactor process_implied_equality() so that it can pass back the new RestrictInfo to its callers in equivclass.c, allowing the generated "fktab.a = 1" clause to be saved in the EquivalenceClass's ec_derives list. Then it's not much trouble to dig out the relevant RestrictInfo when we need to adjust an FK selectivity estimate. (While at it, we can also remove the expensive use of initialize_mergeclause_eclasses() to set up the new RestrictInfo's left_ec and right_ec pointers. The equivclass.c code can set those basically for free.) This seems like clearly a bug fix, but I'm hesitant to back-patch it, first because there's some API/ABI risk for extensions and second because we're usually loath to destabilize plan choices in stable branches. Per report from Sigrid Ehrenreich. Discussion: https://postgr.es/m/1019549.1603770457@sss.pgh.pa.us Discussion: https://postgr.es/m/AM6PR02MB5287A0ADD936C1FA80973E72AB190@AM6PR02MB5287.eurprd02.prod.outlook.com --- src/backend/nodes/outfuncs.c | 1 + src/backend/optimizer/path/costsize.c | 57 +++++++- src/backend/optimizer/path/equivclass.c | 121 ++++++++++++---- src/backend/optimizer/plan/initsplan.c | 184 ++++++++++++++++-------- src/backend/optimizer/util/plancat.c | 2 + src/include/nodes/pathnodes.h | 3 + src/include/optimizer/paths.h | 2 + src/include/optimizer/planmain.h | 20 +-- src/test/regress/expected/join.out | 50 +++++++ src/test/regress/sql/join.sql | 29 ++++ 10 files changed, 366 insertions(+), 103 deletions(-) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 08a049232e0a..530328af4335 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2352,6 +2352,7 @@ _outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node) WRITE_ATTRNUMBER_ARRAY(confkey, node->nkeys); WRITE_OID_ARRAY(conpfeqop, node->nkeys); WRITE_INT_FIELD(nmatched_ec); + WRITE_INT_FIELD(nconst_ec); WRITE_INT_FIELD(nmatched_rcols); WRITE_INT_FIELD(nmatched_ri); /* for compactness, just print the number of matches per column: */ diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 733f7ea54328..f1dfdc1a4a1d 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -5066,9 +5066,16 @@ get_foreign_key_join_selectivity(PlannerInfo *root, * remove back into the worklist. * * Since the matching clauses are known not outerjoin-delayed, they - * should certainly have appeared in the initial joinclause list. If - * we didn't find them, they must have been matched to, and removed - * by, some other FK in a previous iteration of this loop. (A likely + * would normally have appeared in the initial joinclause list. If we + * didn't find them, there are two possibilities: + * + * 1. If the FK match is based on an EC that is ec_has_const, it won't + * have generated any join clauses at all. We discount such ECs while + * checking to see if we have "all" the clauses. (Below, we'll adjust + * the selectivity estimate for this case.) + * + * 2. The clauses were matched to some other FK in a previous + * iteration of this loop, and thus removed from worklist. (A likely * case is that two FKs are matched to the same EC; there will be only * one EC-derived clause in the initial list, so the first FK will * consume it.) Applying both FKs' selectivity independently risks @@ -5078,8 +5085,9 @@ get_foreign_key_join_selectivity(PlannerInfo *root, * Later we might think of a reasonable way to combine the estimates, * but for now, just punt, since this is a fairly uncommon situation. */ - if (list_length(removedlist) != - (fkinfo->nmatched_ec + fkinfo->nmatched_ri)) + if (removedlist == NIL || + list_length(removedlist) != + (fkinfo->nmatched_ec - fkinfo->nconst_ec + fkinfo->nmatched_ri)) { worklist = list_concat(worklist, removedlist); continue; @@ -5138,9 +5146,48 @@ get_foreign_key_join_selectivity(PlannerInfo *root, fkselec *= 1.0 / ref_tuples; } + + /* + * If any of the FK columns participated in ec_has_const ECs, then + * equivclass.c will have generated "var = const" restrictions for + * each side of the join, thus reducing the sizes of both input + * relations. Taking the fkselec at face value would amount to + * double-counting the selectivity of the constant restriction for the + * referencing Var. Hence, look for the restriction clause(s) that + * were applied to the referencing Var(s), and divide out their + * selectivity to correct for this. + */ + if (fkinfo->nconst_ec > 0) + { + for (int i = 0; i < fkinfo->nkeys; i++) + { + EquivalenceClass *ec = fkinfo->eclass[i]; + + if (ec && ec->ec_has_const) + { + EquivalenceMember *em = fkinfo->fk_eclass_member[i]; + RestrictInfo *rinfo = find_derived_clause_for_ec_member(ec, + em); + + if (rinfo) + { + Selectivity s0; + + s0 = clause_selectivity(root, + (Node *) rinfo, + 0, + jointype, + sjinfo); + if (s0 > 0) + fkselec /= s0; + } + } + } + } } *restrictlist = worklist; + CLAMP_PROBABILITY(fkselec); return fkselec; } diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 690b753369e8..a21b3b4756cd 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -840,10 +840,8 @@ find_em_expr_for_rel(EquivalenceClass *ec, RelOptInfo *rel) * scanning of the quals and before Path construction begins. * * We make no attempt to avoid generating duplicate RestrictInfos here: we - * don't search ec_sources for matches, nor put the created RestrictInfos - * into ec_derives. Doing so would require some slightly ugly changes in - * initsplan.c's API, and there's no real advantage, because the clauses - * generated here can't duplicate anything we will generate for joins anyway. + * don't search ec_sources or ec_derives for matches. It doesn't really + * seem worth the trouble to do so. */ void generate_base_implied_equalities(PlannerInfo *root) @@ -969,6 +967,7 @@ generate_base_implied_equalities_const(PlannerInfo *root, { EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc); Oid eq_op; + RestrictInfo *rinfo; Assert(!cur_em->em_is_child); /* no children yet */ if (cur_em == const_em) @@ -982,14 +981,31 @@ generate_base_implied_equalities_const(PlannerInfo *root, ec->ec_broken = true; break; } - process_implied_equality(root, eq_op, ec->ec_collation, - cur_em->em_expr, const_em->em_expr, - bms_copy(ec->ec_relids), - bms_union(cur_em->em_nullable_relids, - const_em->em_nullable_relids), - ec->ec_min_security, - ec->ec_below_outer_join, - cur_em->em_is_const); + rinfo = process_implied_equality(root, eq_op, ec->ec_collation, + cur_em->em_expr, const_em->em_expr, + bms_copy(ec->ec_relids), + bms_union(cur_em->em_nullable_relids, + const_em->em_nullable_relids), + ec->ec_min_security, + ec->ec_below_outer_join, + cur_em->em_is_const); + + /* + * If the clause didn't degenerate to a constant, fill in the correct + * markings for a mergejoinable clause, and save it in ec_derives. (We + * will not re-use such clauses directly, but selectivity estimation + * may consult the list later. Note that this use of ec_derives does + * not overlap with its use for join clauses, since we never generate + * join clauses from an ec_has_const eclass.) + */ + if (rinfo && rinfo->mergeopfamilies) + { + /* it's not redundant, so don't set parent_ec */ + rinfo->left_ec = rinfo->right_ec = ec; + rinfo->left_em = cur_em; + rinfo->right_em = const_em; + ec->ec_derives = lappend(ec->ec_derives, rinfo); + } } } @@ -1028,6 +1044,7 @@ generate_base_implied_equalities_no_const(PlannerInfo *root, { EquivalenceMember *prev_em = prev_ems[relid]; Oid eq_op; + RestrictInfo *rinfo; eq_op = select_equality_operator(ec, prev_em->em_datatype, @@ -1038,14 +1055,29 @@ generate_base_implied_equalities_no_const(PlannerInfo *root, ec->ec_broken = true; break; } - process_implied_equality(root, eq_op, ec->ec_collation, - prev_em->em_expr, cur_em->em_expr, - bms_copy(ec->ec_relids), - bms_union(prev_em->em_nullable_relids, - cur_em->em_nullable_relids), - ec->ec_min_security, - ec->ec_below_outer_join, - false); + rinfo = process_implied_equality(root, eq_op, ec->ec_collation, + prev_em->em_expr, cur_em->em_expr, + bms_copy(ec->ec_relids), + bms_union(prev_em->em_nullable_relids, + cur_em->em_nullable_relids), + ec->ec_min_security, + ec->ec_below_outer_join, + false); + + /* + * If the clause didn't degenerate to a constant, fill in the + * correct markings for a mergejoinable clause. We don't put it + * in ec_derives however; we don't currently need to re-find such + * clauses, and we don't want to clutter that list with non-join + * clauses. + */ + if (rinfo && rinfo->mergeopfamilies) + { + /* it's not redundant, so don't set parent_ec */ + rinfo->left_ec = rinfo->right_ec = ec; + rinfo->left_em = prev_em; + rinfo->right_em = cur_em; + } } prev_ems[relid] = cur_em; } @@ -2151,6 +2183,10 @@ exprs_known_equal(PlannerInfo *root, Node *item1, Node *item2) * we ignore that fine point here.) This is much like exprs_known_equal, * except that we insist on the comparison operator matching the eclass, so * that the result is definite not approximate. + * + * On success, we also set fkinfo->eclass[colno] to the matching eclass, + * and set fkinfo->fk_eclass_member[colno] to the eclass member for the + * referencing Var. */ EquivalenceClass * match_eclasses_to_foreign_key_col(PlannerInfo *root, @@ -2180,8 +2216,8 @@ match_eclasses_to_foreign_key_col(PlannerInfo *root, { EquivalenceClass *ec = (EquivalenceClass *) list_nth(root->eq_classes, i); - bool item1member = false; - bool item2member = false; + EquivalenceMember *item1_em = NULL; + EquivalenceMember *item2_em = NULL; ListCell *lc2; /* Never match to a volatile EC */ @@ -2206,12 +2242,12 @@ match_eclasses_to_foreign_key_col(PlannerInfo *root, /* Match? */ if (var->varno == var1varno && var->varattno == var1attno) - item1member = true; + item1_em = em; else if (var->varno == var2varno && var->varattno == var2attno) - item2member = true; + item2_em = em; /* Have we found both PK and FK column in this EC? */ - if (item1member && item2member) + if (item1_em && item2_em) { /* * Succeed if eqop matches EC's opfamilies. We could test @@ -2221,7 +2257,11 @@ match_eclasses_to_foreign_key_col(PlannerInfo *root, if (opfamilies == NIL) /* compute if we didn't already */ opfamilies = get_mergejoin_opfamilies(eqop); if (equal(opfamilies, ec->ec_opfamilies)) + { + fkinfo->eclass[colno] = ec; + fkinfo->fk_eclass_member[colno] = item2_em; return ec; + } /* Otherwise, done with this EC, move on to the next */ break; } @@ -2230,6 +2270,37 @@ match_eclasses_to_foreign_key_col(PlannerInfo *root, return NULL; } +/* + * find_derived_clause_for_ec_member + * Search for a previously-derived clause mentioning the given EM. + * + * The eclass should be an ec_has_const EC, of which the EM is a non-const + * member. This should ensure there is just one derived clause mentioning + * the EM (and equating it to a constant). + * Returns NULL if no such clause can be found. + */ +RestrictInfo * +find_derived_clause_for_ec_member(EquivalenceClass *ec, + EquivalenceMember *em) +{ + ListCell *lc; + + Assert(ec->ec_has_const); + Assert(!em->em_is_const); + foreach(lc, ec->ec_derives) + { + RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + + /* + * generate_base_implied_equalities_const will have put non-const + * members on the left side of derived clauses. + */ + if (rinfo->left_em == em) + return rinfo; + } + return NULL; +} + /* * add_child_rel_equivalences diff --git a/src/backend/optimizer/plan/initsplan.c b/src/backend/optimizer/plan/initsplan.c index e978b491f6c2..aae5df09f9b7 100644 --- a/src/backend/optimizer/plan/initsplan.c +++ b/src/backend/optimizer/plan/initsplan.c @@ -62,14 +62,12 @@ static SpecialJoinInfo *make_outerjoininfo(PlannerInfo *root, JoinType jointype, List *clause); static void compute_semijoin_info(SpecialJoinInfo *sjinfo, List *clause); static void distribute_qual_to_rels(PlannerInfo *root, Node *clause, - bool is_deduced, bool below_outer_join, JoinType jointype, Index security_level, Relids qualscope, Relids ojscope, Relids outerjoin_nonnullable, - Relids deduced_nullable_relids, List **postponed_qual_list); static bool check_outerjoin_delay(PlannerInfo *root, Relids *relids_p, Relids *nullable_relids_p, bool is_pushed_down); @@ -815,9 +813,9 @@ deconstruct_recurse(PlannerInfo *root, Node *jtnode, bool below_outer_join, if (bms_is_subset(pq->relids, *qualscope)) distribute_qual_to_rels(root, pq->qual, - false, below_outer_join, JOIN_INNER, + below_outer_join, JOIN_INNER, root->qual_security_level, - *qualscope, NULL, NULL, NULL, + *qualscope, NULL, NULL, NULL); else *postponed_qual_list = lappend(*postponed_qual_list, pq); @@ -831,9 +829,9 @@ deconstruct_recurse(PlannerInfo *root, Node *jtnode, bool below_outer_join, Node *qual = (Node *) lfirst(l); distribute_qual_to_rels(root, qual, - false, below_outer_join, JOIN_INNER, + below_outer_join, JOIN_INNER, root->qual_security_level, - *qualscope, NULL, NULL, NULL, + *qualscope, NULL, NULL, postponed_qual_list); } } @@ -1008,10 +1006,10 @@ deconstruct_recurse(PlannerInfo *root, Node *jtnode, bool below_outer_join, Node *qual = (Node *) lfirst(l); distribute_qual_to_rels(root, qual, - false, below_outer_join, j->jointype, + below_outer_join, j->jointype, root->qual_security_level, *qualscope, - ojscope, nonnullable_rels, NULL, + ojscope, nonnullable_rels, postponed_qual_list); } @@ -1110,14 +1108,12 @@ process_security_barrier_quals(PlannerInfo *root, * than being pushed up to top of tree, which we don't want. */ distribute_qual_to_rels(root, qual, - false, below_outer_join, JOIN_INNER, security_level, qualscope, qualscope, NULL, - NULL, NULL); } security_level++; @@ -1581,7 +1577,6 @@ compute_semijoin_info(SpecialJoinInfo *sjinfo, List *clause) * as belonging to a higher join level, just add it to postponed_qual_list. * * 'clause': the qual clause to be distributed - * 'is_deduced': true if the qual came from implied-equality deduction * 'below_outer_join': true if the qual is from a JOIN/ON that is below the * nullable side of a higher-level outer join * 'jointype': type of join the qual is from (JOIN_INNER for a WHERE clause) @@ -1593,8 +1588,6 @@ compute_semijoin_info(SpecialJoinInfo *sjinfo, List *clause) * baserels appearing on the outer (nonnullable) side of the join * (for FULL JOIN this includes both sides of the join, and must in fact * equal qualscope) - * 'deduced_nullable_relids': if is_deduced is true, the nullable relids to - * impute to the clause; otherwise NULL * 'postponed_qual_list': list of PostponedQual structs, which we can add * this qual to if it turns out to belong to a higher join level. * Can be NULL if caller knows postponement is impossible. @@ -1603,23 +1596,17 @@ compute_semijoin_info(SpecialJoinInfo *sjinfo, List *clause) * 'ojscope' is needed if we decide to force the qual up to the outer-join * level, which will be ojscope not necessarily qualscope. * - * In normal use (when is_deduced is false), at the time this is called, - * root->join_info_list must contain entries for all and only those special - * joins that are syntactically below this qual. But when is_deduced is true, - * we are adding new deduced clauses after completion of deconstruct_jointree, - * so it cannot be assumed that root->join_info_list has anything to do with - * qual placement. + * At the time this is called, root->join_info_list must contain entries for + * all and only those special joins that are syntactically below this qual. */ static void distribute_qual_to_rels(PlannerInfo *root, Node *clause, - bool is_deduced, bool below_outer_join, JoinType jointype, Index security_level, Relids qualscope, Relids ojscope, Relids outerjoin_nonnullable, - Relids deduced_nullable_relids, List **postponed_qual_list) { Relids relids; @@ -1653,7 +1640,6 @@ distribute_qual_to_rels(PlannerInfo *root, Node *clause, Assert(root->hasLateralRTEs); /* shouldn't happen otherwise */ Assert(jointype == JOIN_INNER); /* mustn't postpone past outer join */ - Assert(!is_deduced); /* shouldn't be deduced, either */ pq->qual = clause; pq->relids = relids; *postponed_qual_list = lappend(*postponed_qual_list, pq); @@ -1754,24 +1740,7 @@ distribute_qual_to_rels(PlannerInfo *root, Node *clause, * This seems like another reason why it should perhaps be rethought. *---------- */ - if (is_deduced) - { - /* - * If the qual came from implied-equality deduction, it should not be - * outerjoin-delayed, else deducer blew it. But we can't check this - * because the join_info_list may now contain OJs above where the qual - * belongs. For the same reason, we must rely on caller to supply the - * correct nullable_relids set. - */ - Assert(!ojscope); - is_pushed_down = true; - outerjoin_delayed = false; - nullable_relids = deduced_nullable_relids; - /* Don't feed it back for more deductions */ - maybe_equivalence = false; - maybe_outer_join = false; - } - else if (bms_overlap(relids, outerjoin_nonnullable)) + if (bms_overlap(relids, outerjoin_nonnullable)) { /* * The qual is attached to an outer join and mentions (some of the) @@ -2277,14 +2246,18 @@ distribute_restrictinfo_to_rels(PlannerInfo *root, * can produce constant TRUE or constant FALSE. (Otherwise it's not, * because the expressions went through eval_const_expressions already.) * + * Returns the generated RestrictInfo, if any. The result will be NULL + * if both_const is true and we successfully reduced the clause to + * constant TRUE. + * * Note: this function will copy item1 and item2, but it is caller's * responsibility to make sure that the Relids parameters are fresh copies * not shared with other uses. * - * This is currently used only when an EquivalenceClass is found to - * contain pseudoconstants. See path/pathkeys.c for more details. + * Note: we do not do initialize_mergeclause_eclasses() here. It is + * caller's responsibility that left_ec/right_ec be set as necessary. */ -void +RestrictInfo * process_implied_equality(PlannerInfo *root, Oid opno, Oid collation, @@ -2296,24 +2269,27 @@ process_implied_equality(PlannerInfo *root, bool below_outer_join, bool both_const) { - Expr *clause; + RestrictInfo *restrictinfo; + Node *clause; + Relids relids; + bool pseudoconstant = false; /* * Build the new clause. Copy to ensure it shares no substructure with * original (this is necessary in case there are subselects in there...) */ - clause = make_opclause(opno, - BOOLOID, /* opresulttype */ - false, /* opretset */ - copyObject(item1), - copyObject(item2), - InvalidOid, - collation); + clause = (Node *) make_opclause(opno, + BOOLOID, /* opresulttype */ + false, /* opretset */ + copyObject(item1), + copyObject(item2), + InvalidOid, + collation); /* If both constant, try to reduce to a boolean constant. */ if (both_const) { - clause = (Expr *) eval_const_expressions(root, (Node *) clause); + clause = eval_const_expressions(root, clause); /* If we produced const TRUE, just drop the clause */ if (clause && IsA(clause, Const)) @@ -2322,25 +2298,106 @@ process_implied_equality(PlannerInfo *root, Assert(cclause->consttype == BOOLOID); if (!cclause->constisnull && DatumGetBool(cclause->constvalue)) - return; + return NULL; + } + } + + /* + * The rest of this is a very cut-down version of distribute_qual_to_rels. + * We can skip most of the work therein, but there are a couple of special + * cases we still have to handle. + * + * Retrieve all relids mentioned within the possibly-simplified clause. + */ + relids = pull_varnos(clause); + Assert(bms_is_subset(relids, qualscope)); + + /* + * If the clause is variable-free, our normal heuristic for pushing it + * down to just the mentioned rels doesn't work, because there are none. + * Apply at the given qualscope, or at the top of tree if it's nonvolatile + * (which it very likely is, but we'll check, just to be sure). + */ + if (bms_is_empty(relids)) + { + /* eval at original syntactic level */ + relids = bms_copy(qualscope); + if (!contain_volatile_functions(clause)) + { + /* mark as gating qual */ + pseudoconstant = true; + /* tell createplan.c to check for gating quals */ + root->hasPseudoConstantQuals = true; + /* if not below outer join, push it to top of tree */ + if (!below_outer_join) + { + relids = + get_relids_in_jointree((Node *) root->parse->jointree, + false); + } } } + /* + * Build the RestrictInfo node itself. + */ + restrictinfo = make_restrictinfo((Expr *) clause, + true, /* is_pushed_down */ + false, /* outerjoin_delayed */ + pseudoconstant, + security_level, + relids, + NULL, /* outer_relids */ + nullable_relids); + + /* + * If it's a join clause, add vars used in the clause to targetlists of + * their relations, so that they will be emitted by the plan nodes that + * scan those relations (else they won't be available at the join node!). + * + * Typically, we'd have already done this when the component expressions + * were first seen by distribute_qual_to_rels; but it is possible that + * some of the Vars could have missed having that done because they only + * appeared in single-relation clauses originally. So do it here for + * safety. + */ + if (bms_membership(relids) == BMS_MULTIPLE) + { + List *vars = pull_var_clause(clause, + PVC_RECURSE_AGGREGATES | + PVC_RECURSE_WINDOWFUNCS | + PVC_INCLUDE_PLACEHOLDERS); + + add_vars_to_targetlist(root, vars, relids, false); + list_free(vars); + } + + /* + * Check mergejoinability. This will usually succeed, since the op came + * from an EquivalenceClass; but we could have reduced the original clause + * to a constant. + */ + check_mergejoinable(restrictinfo); + + /* + * Note we don't do initialize_mergeclause_eclasses(); the caller can + * handle that much more cheaply than we can. It's okay to call + * distribute_restrictinfo_to_rels() before that happens. + */ + /* * Push the new clause into all the appropriate restrictinfo lists. */ - distribute_qual_to_rels(root, (Node *) clause, - true, below_outer_join, JOIN_INNER, - security_level, - qualscope, NULL, NULL, nullable_relids, - NULL); + distribute_restrictinfo_to_rels(root, restrictinfo); + + return restrictinfo; } /* * build_implied_join_equality --- build a RestrictInfo for a derived equality * * This overlaps the functionality of process_implied_equality(), but we - * must return the RestrictInfo, not push it into the joininfo tree. + * must not push the RestrictInfo into the joininfo tree. * * Note: this function will copy item1 and item2, but it is caller's * responsibility to make sure that the Relids parameters are fresh copies @@ -2455,18 +2512,19 @@ match_foreign_keys_to_quals(PlannerInfo *root) */ for (colno = 0; colno < fkinfo->nkeys; colno++) { + EquivalenceClass *ec; AttrNumber con_attno, ref_attno; Oid fpeqop; ListCell *lc2; - fkinfo->eclass[colno] = match_eclasses_to_foreign_key_col(root, - fkinfo, - colno); + ec = match_eclasses_to_foreign_key_col(root, fkinfo, colno); /* Don't bother looking for loose quals if we got an EC match */ - if (fkinfo->eclass[colno] != NULL) + if (ec != NULL) { fkinfo->nmatched_ec++; + if (ec->ec_has_const) + fkinfo->nconst_ec++; continue; } diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index f9d0d67aa75a..4f0da51c2697 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -567,9 +567,11 @@ get_relation_foreign_keys(PlannerInfo *root, RelOptInfo *rel, memcpy(info->conpfeqop, cachedfk->conpfeqop, sizeof(info->conpfeqop)); /* zero out fields to be filled by match_foreign_keys_to_quals */ info->nmatched_ec = 0; + info->nconst_ec = 0; info->nmatched_rcols = 0; info->nmatched_ri = 0; memset(info->eclass, 0, sizeof(info->eclass)); + memset(info->fk_eclass_member, 0, sizeof(info->fk_eclass_member)); memset(info->rinfos, 0, sizeof(info->rinfos)); root->fkey_list = lappend(root->fkey_list, info); diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 3dd16b9ad534..45cbf6045a38 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -889,10 +889,13 @@ typedef struct ForeignKeyOptInfo /* Derived info about whether FK's equality conditions match the query: */ int nmatched_ec; /* # of FK cols matched by ECs */ + int nconst_ec; /* # of these ECs that are ec_has_const */ int nmatched_rcols; /* # of FK cols matched by non-EC rinfos */ int nmatched_ri; /* total # of non-EC rinfos matched to FK */ /* Pointer to eclass matching each column's condition, if there is one */ struct EquivalenceClass *eclass[INDEX_MAX_KEYS]; + /* Pointer to eclass member for the referencing Var, if there is one */ + struct EquivalenceMember *fk_eclass_member[INDEX_MAX_KEYS]; /* List of non-EC RestrictInfos matching each column's condition */ List *rinfos[INDEX_MAX_KEYS]; } ForeignKeyOptInfo; diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 10b6e8107963..2134227ebcbb 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -149,6 +149,8 @@ extern bool exprs_known_equal(PlannerInfo *root, Node *item1, Node *item2); extern EquivalenceClass *match_eclasses_to_foreign_key_col(PlannerInfo *root, ForeignKeyOptInfo *fkinfo, int colno); +extern RestrictInfo *find_derived_clause_for_ec_member(EquivalenceClass *ec, + EquivalenceMember *em); extern void add_child_rel_equivalences(PlannerInfo *root, AppendRelInfo *appinfo, RelOptInfo *parent_rel, diff --git a/src/include/optimizer/planmain.h b/src/include/optimizer/planmain.h index f3cefe67b8d3..81c4a7e56078 100644 --- a/src/include/optimizer/planmain.h +++ b/src/include/optimizer/planmain.h @@ -77,16 +77,16 @@ extern void create_lateral_join_info(PlannerInfo *root); extern List *deconstruct_jointree(PlannerInfo *root); extern void distribute_restrictinfo_to_rels(PlannerInfo *root, RestrictInfo *restrictinfo); -extern void process_implied_equality(PlannerInfo *root, - Oid opno, - Oid collation, - Expr *item1, - Expr *item2, - Relids qualscope, - Relids nullable_relids, - Index security_level, - bool below_outer_join, - bool both_const); +extern RestrictInfo *process_implied_equality(PlannerInfo *root, + Oid opno, + Oid collation, + Expr *item1, + Expr *item2, + Relids qualscope, + Relids nullable_relids, + Index security_level, + bool below_outer_join, + bool both_const); extern RestrictInfo *build_implied_join_equality(Oid opno, Oid collation, Expr *item1, diff --git a/src/test/regress/expected/join.out b/src/test/regress/expected/join.out index a46b1573bd4e..6c9a5e26ddeb 100644 --- a/src/test/regress/expected/join.out +++ b/src/test/regress/expected/join.out @@ -5843,6 +5843,56 @@ select t1.b, ss.phv from join_ut1 t1 left join lateral drop table join_pt1; drop table join_ut1; -- +-- test estimation behavior with multi-column foreign key and constant qual +-- +begin; +create table fkest (x integer, x10 integer, x10b integer, x100 integer); +insert into fkest select x, x/10, x/10, x/100 from generate_series(1,1000) x; +create unique index on fkest(x, x10, x100); +analyze fkest; +explain (costs off) +select * from fkest f1 + join fkest f2 on (f1.x = f2.x and f1.x10 = f2.x10b and f1.x100 = f2.x100) + join fkest f3 on f1.x = f3.x + where f1.x100 = 2; + QUERY PLAN +----------------------------------------------------------- + Nested Loop + -> Hash Join + Hash Cond: ((f2.x = f1.x) AND (f2.x10b = f1.x10)) + -> Seq Scan on fkest f2 + Filter: (x100 = 2) + -> Hash + -> Seq Scan on fkest f1 + Filter: (x100 = 2) + -> Index Scan using fkest_x_x10_x100_idx on fkest f3 + Index Cond: (x = f1.x) +(10 rows) + +alter table fkest add constraint fk + foreign key (x, x10b, x100) references fkest (x, x10, x100); +explain (costs off) +select * from fkest f1 + join fkest f2 on (f1.x = f2.x and f1.x10 = f2.x10b and f1.x100 = f2.x100) + join fkest f3 on f1.x = f3.x + where f1.x100 = 2; + QUERY PLAN +----------------------------------------------------- + Hash Join + Hash Cond: ((f2.x = f1.x) AND (f2.x10b = f1.x10)) + -> Hash Join + Hash Cond: (f3.x = f2.x) + -> Seq Scan on fkest f3 + -> Hash + -> Seq Scan on fkest f2 + Filter: (x100 = 2) + -> Hash + -> Seq Scan on fkest f1 + Filter: (x100 = 2) +(11 rows) + +rollback; +-- -- test that foreign key join estimation performs sanely for outer joins -- begin; diff --git a/src/test/regress/sql/join.sql b/src/test/regress/sql/join.sql index 1403e0ffe7bc..dd60d6a1f3b8 100644 --- a/src/test/regress/sql/join.sql +++ b/src/test/regress/sql/join.sql @@ -1975,6 +1975,35 @@ select t1.b, ss.phv from join_ut1 t1 left join lateral drop table join_pt1; drop table join_ut1; + +-- +-- test estimation behavior with multi-column foreign key and constant qual +-- + +begin; + +create table fkest (x integer, x10 integer, x10b integer, x100 integer); +insert into fkest select x, x/10, x/10, x/100 from generate_series(1,1000) x; +create unique index on fkest(x, x10, x100); +analyze fkest; + +explain (costs off) +select * from fkest f1 + join fkest f2 on (f1.x = f2.x and f1.x10 = f2.x10b and f1.x100 = f2.x100) + join fkest f3 on f1.x = f3.x + where f1.x100 = 2; + +alter table fkest add constraint fk + foreign key (x, x10b, x100) references fkest (x, x10, x100); + +explain (costs off) +select * from fkest f1 + join fkest f2 on (f1.x = f2.x and f1.x10 = f2.x10b and f1.x100 = f2.x100) + join fkest f3 on f1.x = f3.x + where f1.x100 = 2; + +rollback; + -- -- test that foreign key join estimation performs sanely for outer joins -- From 36b93121436cbbf357974144068c23bac75154fa Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 28 Oct 2020 12:18:45 -0400 Subject: [PATCH 392/589] Don't use custom OID symbols in pg_proc.dat. We have a perfectly good convention for OID macros for built-in functions already, so making custom symbols is just introducing unnecessary deviation from the convention. Remove the one case that had snuck in, and add an error check in genbki.pl to discourage future instances. Although this touches pg_proc.dat, there's no need for a catversion bump since the actual catalog data isn't changed. John Naylor Discussion: https://postgr.es/m/CAFBsxsHpCbjfoddNGpnnnY5pHwckWfiYkMYSF74PmP1su0+ZOw@mail.gmail.com --- src/backend/catalog/genbki.pl | 7 +++++++ src/backend/utils/cache/relcache.c | 4 ++-- src/include/catalog/pg_proc.dat | 3 +-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/backend/catalog/genbki.pl b/src/backend/catalog/genbki.pl index ef3105af44bb..47b1a5d79083 100644 --- a/src/backend/catalog/genbki.pl +++ b/src/backend/catalog/genbki.pl @@ -602,6 +602,13 @@ # Emit OID symbol if (defined $bki_values{oid_symbol}) { + # OID symbols for builtin functions are handled automatically + # by utils/Gen_fmgrtab.pl + die sprintf + "custom OID symbols are not allowed for pg_proc entries: '%s'", + $bki_values{oid_symbol} + if $catname eq 'pg_proc'; + printf $def "#define %s %s\n", $bki_values{oid_symbol}, $bki_values{oid}; } diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 9061af81a3e3..9224e2ffeda5 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -1761,7 +1761,7 @@ RelationInitTableAccessMethod(Relation relation) * seem prudent to show that in the catalog. So just overwrite it * here. */ - relation->rd_amhandler = HEAP_TABLE_AM_HANDLER_OID; + relation->rd_amhandler = F_HEAP_TABLEAM_HANDLER; } else if (IsCatalogRelation(relation)) { @@ -1769,7 +1769,7 @@ RelationInitTableAccessMethod(Relation relation) * Avoid doing a syscache lookup for catalog tables. */ Assert(relation->rd_rel->relam == HEAP_TABLE_AM_OID); - relation->rd_amhandler = HEAP_TABLE_AM_HANDLER_OID; + relation->rd_amhandler = F_HEAP_TABLEAM_HANDLER; } else { diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index a66870bcc080..24ec2cfed6a3 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -901,8 +901,7 @@ prosrc => 'ftoi4' }, # Table access method handlers -{ oid => '3', oid_symbol => 'HEAP_TABLE_AM_HANDLER_OID', - descr => 'row-oriented heap table access method handler', +{ oid => '3', descr => 'row-oriented heap table access method handler', proname => 'heap_tableam_handler', provolatile => 'v', prorettype => 'table_am_handler', proargtypes => 'internal', prosrc => 'heap_tableam_handler' }, From ad77039fad0f4128b0e4a05ddbf5dbc3ab5f3fa4 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 28 Oct 2020 13:47:02 -0400 Subject: [PATCH 393/589] Calculate extraUpdatedCols in query rewriter, not parser. It's unsafe to do this at parse time because addition of generated columns to a table would not invalidate stored rules containing UPDATEs on the table ... but there might now be dependent generated columns that were not there when the rule was made. This also fixes an oversight that rewriteTargetView failed to update extraUpdatedCols when transforming an UPDATE on an updatable view. (Since the new calculation is downstream of that, rewriteTargetView doesn't actually need to do anything; but before, there was a demonstrable bug there.) In v13 and HEAD, this leads to easily-visible bugs because (since commit c6679e4fc) we won't recalculate generated columns that aren't listed in extraUpdatedCols. In v12 this bitmap is mostly just used for trigger-firing decisions, so you'd only notice a problem if a trigger cared whether a generated column had been updated. I'd complained about this back in May, but then forgot about it until bug #16671 from Michael Paul Killian revived the issue. Back-patch to v12 where this field was introduced. If existing stored rules contain any extraUpdatedCols values, they'll be ignored because the rewriter will overwrite them, so the bug will be fixed even for existing rules. (But note that if someone were to update to 13.1 or 12.5, store some rules with UPDATEs on tables having generated columns, and then downgrade to a prior minor version, they might observe issues similar to what this patch fixes. That seems unlikely enough to not be worth going to a lot of effort to fix.) Discussion: https://postgr.es/m/10206.1588964727@sss.pgh.pa.us Discussion: https://postgr.es/m/16671-2fa55851859fb166@postgresql.org --- src/backend/optimizer/plan/setrefs.c | 6 +-- src/backend/parser/analyze.c | 33 --------------- src/backend/replication/logical/worker.c | 5 +-- src/backend/rewrite/rewriteHandler.c | 41 +++++++++++++++++++ src/include/nodes/parsenodes.h | 16 +++++--- src/include/parser/analyze.h | 2 - src/include/rewrite/rewriteHandler.h | 3 ++ src/test/regress/expected/updatable_views.out | 35 ++++++++++++++++ src/test/regress/sql/updatable_views.sql | 21 ++++++++++ 9 files changed, 115 insertions(+), 47 deletions(-) diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index 8b4337142594..127ea3d856dc 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -438,9 +438,9 @@ flatten_rtes_walker(Node *node, PlannerGlobal *glob) * In the flat rangetable, we zero out substructure pointers that are not * needed by the executor; this reduces the storage space and copying cost * for cached plans. We keep only the ctename, alias and eref Alias fields, - * which are needed by EXPLAIN, and the selectedCols, insertedCols and - * updatedCols bitmaps, which are needed for executor-startup permissions - * checking and for trigger event checking. + * which are needed by EXPLAIN, and the selectedCols, insertedCols, + * updatedCols, and extraUpdatedCols bitmaps, which are needed for + * executor-startup permissions checking and for trigger event checking. */ static void add_rte_to_flat_rtable(PlannerGlobal *glob, RangeTblEntry *rte) diff --git a/src/backend/parser/analyze.c b/src/backend/parser/analyze.c index 98a83db8b5cf..575e22ce0d2b 100644 --- a/src/backend/parser/analyze.c +++ b/src/backend/parser/analyze.c @@ -2282,7 +2282,6 @@ transformUpdateTargetList(ParseState *pstate, List *origTlist) RangeTblEntry *target_rte; ListCell *orig_tl; ListCell *tl; - TupleDesc tupdesc = pstate->p_target_relation->rd_att; tlist = transformTargetList(pstate, origTlist, EXPR_KIND_UPDATE_SOURCE); @@ -2341,41 +2340,9 @@ transformUpdateTargetList(ParseState *pstate, List *origTlist) if (orig_tl != NULL) elog(ERROR, "UPDATE target count mismatch --- internal error"); - fill_extraUpdatedCols(target_rte, tupdesc); - return tlist; } -/* - * Record in extraUpdatedCols generated columns referencing updated base - * columns. - */ -void -fill_extraUpdatedCols(RangeTblEntry *target_rte, TupleDesc tupdesc) -{ - if (tupdesc->constr && - tupdesc->constr->has_generated_stored) - { - for (int i = 0; i < tupdesc->constr->num_defval; i++) - { - AttrDefault defval = tupdesc->constr->defval[i]; - Node *expr; - Bitmapset *attrs_used = NULL; - - /* skip if not generated column */ - if (!TupleDescAttr(tupdesc, defval.adnum - 1)->attgenerated) - continue; - - expr = stringToNode(defval.adbin); - pull_varattnos(expr, 1, &attrs_used); - - if (bms_overlap(target_rte->updatedCols, attrs_used)) - target_rte->extraUpdatedCols = bms_add_member(target_rte->extraUpdatedCols, - defval.adnum - FirstLowInvalidHeapAttributeNumber); - } - } -} - /* * transformReturningList - * handle a RETURNING clause in INSERT/UPDATE/DELETE diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 3a5b733ee38c..b0f27e0af856 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -81,8 +81,6 @@ #include "miscadmin.h" #include "nodes/makefuncs.h" #include "optimizer/optimizer.h" -#include "parser/analyze.h" -#include "parser/parse_relation.h" #include "pgstat.h" #include "postmaster/bgworker.h" #include "postmaster/interrupt.h" @@ -1323,7 +1321,8 @@ apply_handle_update(StringInfo s) } } - fill_extraUpdatedCols(target_rte, RelationGetDescr(rel->localrel)); + /* Also populate extraUpdatedCols, in case we have generated columns */ + fill_extraUpdatedCols(target_rte, rel->localrel); PushActiveSnapshot(GetTransactionSnapshot()); diff --git a/src/backend/rewrite/rewriteHandler.c b/src/backend/rewrite/rewriteHandler.c index 1faaafab08a6..41dd670572d1 100644 --- a/src/backend/rewrite/rewriteHandler.c +++ b/src/backend/rewrite/rewriteHandler.c @@ -30,6 +30,7 @@ #include "miscadmin.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" +#include "optimizer/optimizer.h" #include "parser/analyze.h" #include "parser/parse_coerce.h" #include "parser/parse_relation.h" @@ -1508,6 +1509,42 @@ rewriteTargetListUD(Query *parsetree, RangeTblEntry *target_rte, } } +/* + * Record in target_rte->extraUpdatedCols the indexes of any generated columns + * that depend on any columns mentioned in target_rte->updatedCols. + */ +void +fill_extraUpdatedCols(RangeTblEntry *target_rte, Relation target_relation) +{ + TupleDesc tupdesc = RelationGetDescr(target_relation); + TupleConstr *constr = tupdesc->constr; + + target_rte->extraUpdatedCols = NULL; + + if (constr && constr->has_generated_stored) + { + for (int i = 0; i < constr->num_defval; i++) + { + AttrDefault *defval = &constr->defval[i]; + Node *expr; + Bitmapset *attrs_used = NULL; + + /* skip if not generated column */ + if (!TupleDescAttr(tupdesc, defval->adnum - 1)->attgenerated) + continue; + + /* identify columns this generated column depends on */ + expr = stringToNode(defval->adbin); + pull_varattnos(expr, 1, &attrs_used); + + if (bms_overlap(target_rte->updatedCols, attrs_used)) + target_rte->extraUpdatedCols = + bms_add_member(target_rte->extraUpdatedCols, + defval->adnum - FirstLowInvalidHeapAttributeNumber); + } + } +} + /* * matchLocks - @@ -1639,6 +1676,7 @@ ApplyRetrieveRule(Query *parsetree, rte->selectedCols = NULL; rte->insertedCols = NULL; rte->updatedCols = NULL; + rte->extraUpdatedCols = NULL; /* * For the most part, Vars referencing the view should remain as @@ -3617,6 +3655,9 @@ RewriteQuery(Query *parsetree, List *rewrite_events) parsetree->override, rt_entry_relation, parsetree->resultRelation); + + /* Also populate extraUpdatedCols (for generated columns) */ + fill_extraUpdatedCols(rt_entry, rt_entry_relation); } else if (event == CMD_DELETE) { diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 60c2f4546604..ff584f2955bb 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -940,12 +940,16 @@ typedef struct PartitionCmd * * updatedCols is also used in some other places, for example, to determine * which triggers to fire and in FDWs to know which changed columns they - * need to ship off. Generated columns that are caused to be updated by an - * update to a base column are collected in extraUpdatedCols. This is not - * considered for permission checking, but it is useful in those places - * that want to know the full set of columns being updated as opposed to - * only the ones the user explicitly mentioned in the query. (There is - * currently no need for an extraInsertedCols, but it could exist.) + * need to ship off. + * + * Generated columns that are caused to be updated by an update to a base + * column are listed in extraUpdatedCols. This is not considered for + * permission checking, but it is useful in those places that want to know + * the full set of columns being updated as opposed to only the ones the + * user explicitly mentioned in the query. (There is currently no need for + * an extraInsertedCols, but it could exist.) Note that extraUpdatedCols + * is populated during query rewrite, NOT in the parser, since generated + * columns could be added after a rule has been parsed and stored. * * securityQuals is a list of security barrier quals (boolean expressions), * to be tested in the listed order before returning a row from the diff --git a/src/include/parser/analyze.h b/src/include/parser/analyze.h index 9d09a0214197..d6a467a57287 100644 --- a/src/include/parser/analyze.h +++ b/src/include/parser/analyze.h @@ -46,6 +46,4 @@ extern void applyLockingClause(Query *qry, Index rtindex, extern List *BuildOnConflictExcludedTargetlist(Relation targetrel, Index exclRelIndex); -extern void fill_extraUpdatedCols(RangeTblEntry *target_rte, TupleDesc tupdesc); - #endif /* ANALYZE_H */ diff --git a/src/include/rewrite/rewriteHandler.h b/src/include/rewrite/rewriteHandler.h index eb2e7b1768ac..a18211f4a23c 100644 --- a/src/include/rewrite/rewriteHandler.h +++ b/src/include/rewrite/rewriteHandler.h @@ -26,6 +26,9 @@ extern Node *build_column_default(Relation rel, int attrno); extern void rewriteTargetListUD(Query *parsetree, RangeTblEntry *target_rte, Relation target_relation); +extern void fill_extraUpdatedCols(RangeTblEntry *target_rte, + Relation target_relation); + extern Query *get_view_query(Relation view); extern const char *view_query_is_auto_updatable(Query *viewquery, bool check_cols); diff --git a/src/test/regress/expected/updatable_views.out b/src/test/regress/expected/updatable_views.out index caed1c19ec76..6a977006efc7 100644 --- a/src/test/regress/expected/updatable_views.out +++ b/src/test/regress/expected/updatable_views.out @@ -1467,6 +1467,41 @@ NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to view rw_view1 drop cascades to view rw_view2 drop cascades to view rw_view3 +-- view on table with GENERATED columns +CREATE TABLE base_tbl (id int, idplus1 int GENERATED ALWAYS AS (id + 1) STORED); +CREATE VIEW rw_view1 AS SELECT * FROM base_tbl; +INSERT INTO base_tbl (id) VALUES (1); +INSERT INTO rw_view1 (id) VALUES (2); +INSERT INTO base_tbl (id, idplus1) VALUES (3, DEFAULT); +INSERT INTO rw_view1 (id, idplus1) VALUES (4, DEFAULT); +INSERT INTO base_tbl (id, idplus1) VALUES (5, 6); -- error +ERROR: cannot insert into column "idplus1" +DETAIL: Column "idplus1" is a generated column. +INSERT INTO rw_view1 (id, idplus1) VALUES (6, 7); -- error +ERROR: cannot insert into column "idplus1" +DETAIL: Column "idplus1" is a generated column. +SELECT * FROM base_tbl; + id | idplus1 +----+--------- + 1 | 2 + 2 | 3 + 3 | 4 + 4 | 5 +(4 rows) + +UPDATE base_tbl SET id = 2000 WHERE id = 2; +UPDATE rw_view1 SET id = 3000 WHERE id = 3; +SELECT * FROM base_tbl; + id | idplus1 +------+--------- + 1 | 2 + 4 | 5 + 2000 | 2001 + 3000 | 3001 +(4 rows) + +DROP TABLE base_tbl CASCADE; +NOTICE: drop cascades to view rw_view1 -- inheritance tests CREATE TABLE base_tbl_parent (a int); CREATE TABLE base_tbl_child (CHECK (a > 0)) INHERITS (base_tbl_parent); diff --git a/src/test/regress/sql/updatable_views.sql b/src/test/regress/sql/updatable_views.sql index 64f23d0902ec..09328e582b27 100644 --- a/src/test/regress/sql/updatable_views.sql +++ b/src/test/regress/sql/updatable_views.sql @@ -697,6 +697,27 @@ SELECT events & 4 != 0 AS upd, DROP TABLE base_tbl CASCADE; +-- view on table with GENERATED columns + +CREATE TABLE base_tbl (id int, idplus1 int GENERATED ALWAYS AS (id + 1) STORED); +CREATE VIEW rw_view1 AS SELECT * FROM base_tbl; + +INSERT INTO base_tbl (id) VALUES (1); +INSERT INTO rw_view1 (id) VALUES (2); +INSERT INTO base_tbl (id, idplus1) VALUES (3, DEFAULT); +INSERT INTO rw_view1 (id, idplus1) VALUES (4, DEFAULT); +INSERT INTO base_tbl (id, idplus1) VALUES (5, 6); -- error +INSERT INTO rw_view1 (id, idplus1) VALUES (6, 7); -- error + +SELECT * FROM base_tbl; + +UPDATE base_tbl SET id = 2000 WHERE id = 2; +UPDATE rw_view1 SET id = 3000 WHERE id = 3; + +SELECT * FROM base_tbl; + +DROP TABLE base_tbl CASCADE; + -- inheritance tests CREATE TABLE base_tbl_parent (a int); From 66f8687a8ff867f656de81e367314604d29dbd59 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 28 Oct 2020 14:35:53 -0400 Subject: [PATCH 394/589] Use mode "r" for popen() in psql's evaluate_backtick(). In almost all other places, we use plain "r" or "w" mode in popen() calls (the exceptions being for COPY data). This one has been overlooked (possibly because it's buried in a ".l" flex file?), but it's using PG_BINARY_R. Kensuke Okamura complained in bug #16688 that we fail to strip \r when stripping the trailing newline from a backtick result string. That's true enough, but we'd also fail to convert embedded \r\n cleanly, which also seems undesirable. Fixing the popen() mode seems like the best way to deal with this. It's been like this for a long time, so back-patch to all supported branches. Discussion: https://postgr.es/m/16688-c649c7b69cd7e6f8@postgresql.org --- src/bin/psql/psqlscanslash.l | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bin/psql/psqlscanslash.l b/src/bin/psql/psqlscanslash.l index 2307491ba82c..4dff84d6271a 100644 --- a/src/bin/psql/psqlscanslash.l +++ b/src/bin/psql/psqlscanslash.l @@ -777,7 +777,7 @@ evaluate_backtick(PsqlScanState state) initPQExpBuffer(&cmd_output); - fd = popen(cmd, PG_BINARY_R); + fd = popen(cmd, "r"); if (!fd) { pg_log_error("%s: %m", cmd); @@ -818,7 +818,7 @@ evaluate_backtick(PsqlScanState state) /* If no error, transfer result to output_buf */ if (!error) { - /* strip any trailing newline */ + /* strip any trailing newline (but only one) */ if (cmd_output.len > 0 && cmd_output.data[cmd_output.len - 1] == '\n') cmd_output.len--; From 4c49d8fc15eeb1dc69b0ddb2d986a1884a5d7f5f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 28 Oct 2020 16:31:40 -0400 Subject: [PATCH 395/589] Doc: clean up verify_heapam() documentation. I started with the intention of just suppressing a PDF build warning by removing the example output, but ended up doing more: correcting factual errors in the function's signature, moving a bunch of generalized handwaving into the "Using amcheck Effectively" section which seemed a better place for it, and improving wording and markup a little bit. Discussion: https://postgr.es/m/732904.1603728748@sss.pgh.pa.us --- doc/src/sgml/amcheck.sgml | 157 ++++++++++++++++---------------------- 1 file changed, 65 insertions(+), 92 deletions(-) diff --git a/doc/src/sgml/amcheck.sgml b/doc/src/sgml/amcheck.sgml index 25e4bb2bfec2..99fad708bf73 100644 --- a/doc/src/sgml/amcheck.sgml +++ b/doc/src/sgml/amcheck.sgml @@ -83,7 +83,7 @@ AND c.relpersistence != 't' -- Function may throw an error when this is omitted: AND c.relkind = 'i' AND i.indisready AND i.indisvalid ORDER BY c.relpages DESC LIMIT 10; - bt_index_check | relname | relpages + bt_index_check | relname | relpages ----------------+---------------------------------+---------- | pg_depend_reference_index | 43 | pg_depend_depender_index | 40 @@ -208,14 +208,14 @@ SET client_min_messages = DEBUG1; verify_heapam(relation regclass, on_error_stop boolean, check_toast boolean, - skip cstring, + skip text, startblock bigint, endblock bigint, blkno OUT bigint, offnum OUT integer, attnum OUT integer, msg OUT text) - returns record + returns setof record @@ -223,89 +223,17 @@ SET client_min_messages = DEBUG1; Checks a table for structural corruption, where pages in the relation contain data that is invalidly formatted, and for logical corruption, where pages are structurally valid but inconsistent with the rest of the - database cluster. Example usage: - -test=# select * from verify_heapam('mytable', check_toast := true); - blkno | offnum | attnum | msg --------+--------+--------+-------------------------------------------------------------------------------------------------- - 17 | 12 | | xmin 4294967295 precedes relation freeze threshold 17:1134217582 - 960 | 4 | | data begins at offset 152 beyond the tuple length 58 - 960 | 4 | | tuple data should begin at byte 24, but actually begins at byte 152 (3 attributes, no nulls) - 960 | 5 | | tuple data should begin at byte 24, but actually begins at byte 27 (3 attributes, no nulls) - 960 | 6 | | tuple data should begin at byte 24, but actually begins at byte 16 (3 attributes, no nulls) - 960 | 7 | | tuple data should begin at byte 24, but actually begins at byte 21 (3 attributes, no nulls) - 1147 | 2 | | number of attributes 2047 exceeds maximum expected for table 3 - 1147 | 10 | | tuple data should begin at byte 280, but actually begins at byte 24 (2047 attributes, has nulls) - 1147 | 15 | | number of attributes 67 exceeds maximum expected for table 3 - 1147 | 16 | 1 | attribute 1 with length 4294967295 ends at offset 416848000 beyond total tuple length 58 - 1147 | 18 | 2 | final toast chunk number 0 differs from expected value 6 - 1147 | 19 | 2 | toasted value for attribute 2 missing from toast table - 1147 | 21 | | tuple is marked as only locked, but also claims key columns were updated - 1147 | 22 | | multitransaction ID 1775655 is from before relation cutoff 2355572 -(14 rows) - - As this example shows, the Tuple ID (TID) of the corrupt tuple is given - in the (blkno, offnum) columns, and - for corruptions specific to a particular attribute in the tuple, the - attnum field shows which one. - - - Structural corruption can happen due to faulty storage hardware, or - relation files being overwritten or modified by unrelated software. - This kind of corruption can also be detected with - data page - checksums. - - - Relation pages which are correctly formatted, internally consistent, and - correct relative to their own internal checksums may still contain - logical corruption. As such, this kind of corruption cannot be detected - with checksums. Examples include toasted - values in the main table which lack a corresponding entry in the toast - table, and tuples in the main table with a Transaction ID that is older - than the oldest valid Transaction ID in the database or cluster. - - - Multiple causes of logical corruption have been observed in production - systems, including bugs in the PostgreSQL - server software, faulty and ill-conceived backup and restore tools, and - user error. - - - Corrupt relations are most concerning in live production environments, - precisely the same environments where high risk activities are least - welcome. For this reason, verify_heapam has been - designed to diagnose corruption without undue risk. It cannot guard - against all causes of backend crashes, as even executing the calling - query could be unsafe on a badly corrupted system. Access to catalog tables are performed and could - be problematic if the catalogs themselves are corrupted. - - - The design principle adhered to in verify_heapam is - that, if the rest of the system and server hardware are correct, under - default options, verify_heapam will not crash the - server due merely to structural or logical corruption in the target - table. - - - The check_toast attempts to reconcile the target - table against entries in its corresponding toast table. This option is - disabled by default and is known to be slow. - If the target relation's corresponding toast table or toast index is - corrupt, reconciling the target table against toast values could - conceivably crash the server, although in many cases this would - just produce an error. + database cluster. The following optional arguments are recognized: - on_error_stop + on_error_stop - If true, corruption checking stops at the end of the first block on + If true, corruption checking stops at the end of the first block in which any corruptions are found. @@ -314,23 +242,29 @@ test=# select * from verify_heapam('mytable', check_toast := true); - check_toast + check_toast - If true, toasted values are checked gainst the corresponding + If true, toasted values are checked against the target relation's TOAST table. + + This option is known to be slow. Also, if the toast table or its + index is corrupt, checking it against toast values could conceivably + crash the server, although in many cases this would just produce an + error. + Defaults to false. - skip + skip If not none, corruption checking skips blocks that - are marked as all-visible or all-frozen, as given. + are marked as all-visible or all-frozen, as specified. Valid options are all-visible, all-frozen and none. @@ -340,7 +274,7 @@ test=# select * from verify_heapam('mytable', check_toast := true); - startblock + startblock If specified, corruption checking begins at the specified block, @@ -349,12 +283,12 @@ test=# select * from verify_heapam('mytable', check_toast := true); target table. - By default, does not skip any blocks. + By default, checking begins at the first block. - endblock + endblock If specified, corruption checking ends at the specified block, @@ -363,7 +297,7 @@ test=# select * from verify_heapam('mytable', check_toast := true); table. - By default, does not skip any blocks. + By default, all blocks are checked. @@ -374,7 +308,7 @@ test=# select * from verify_heapam('mytable', check_toast := true); - blkno + blkno The number of the block containing the corrupt page. @@ -382,7 +316,7 @@ test=# select * from verify_heapam('mytable', check_toast := true); - offnum + offnum The OffsetNumber of the corrupt tuple. @@ -390,7 +324,7 @@ test=# select * from verify_heapam('mytable', check_toast := true); - attnum + attnum The attribute number of the corrupt column in the tuple, if the @@ -399,10 +333,10 @@ test=# select * from verify_heapam('mytable', check_toast := true); - msg + msg - A human readable message describing the corruption in the page. + A message describing the problem detected. @@ -460,7 +394,7 @@ test=# select * from verify_heapam('mytable', check_toast := true); amcheck can be effective at detecting various types of failure modes that data page - checksums will always fail to catch. These include: + checksums will fail to catch. These include: @@ -557,6 +491,45 @@ test=# select * from verify_heapam('mytable', check_toast := true); + + + + Structural corruption can happen due to faulty storage hardware, or + relation files being overwritten or modified by unrelated software. + This kind of corruption can also be detected with + data page + checksums. + + + + Relation pages which are correctly formatted, internally consistent, and + correct relative to their own internal checksums may still contain + logical corruption. As such, this kind of corruption cannot be detected + with checksums. Examples include toasted + values in the main table which lack a corresponding entry in the toast + table, and tuples in the main table with a Transaction ID that is older + than the oldest valid Transaction ID in the database or cluster. + + + + Multiple causes of logical corruption have been observed in production + systems, including bugs in the PostgreSQL + server software, faulty and ill-conceived backup and restore tools, and + user error. + + + + Corrupt relations are most concerning in live production environments, + precisely the same environments where high risk activities are least + welcome. For this reason, verify_heapam has been + designed to diagnose corruption without undue risk. It cannot guard + against all causes of backend crashes, as even executing the calling + query could be unsafe on a badly corrupted system. Access to catalog tables are performed and could + be problematic if the catalogs themselves are corrupted. + + + In general, amcheck can only prove the presence of corruption; it cannot prove its absence. From b787d4ce6d910080065025bcd5f968544997271f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 28 Oct 2020 17:03:05 -0400 Subject: [PATCH 396/589] Doc: clean up pg_relation_check_pages() documentation. Commit f2b883969 did not get the memo about the new formatting style for tables documenting built-in functions. I noticed because of a PDF build warning about an overwidth table. --- doc/src/sgml/func.sgml | 60 +++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 7ef2ec997259..d8eee3a82643 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -26192,44 +26192,50 @@ SELECT convert_from(pg_read_binary_file('file_in_utf8.txt'), 'UTF8'); Data Sanity Functions - + - Name Return Type Description + + + Function + + + Description + - - pg_relation_check_pages(relation regclass [, fork text DEFAULT NULL ]) - - setof record - Check the pages of a relation. - + + + pg_relation_check_pages + + pg_relation_check_pages ( relation regclass [, fork text ] ) + setof record + ( path text, + failed_block_num bigint ) + + + Checks the pages of the specified relation to see if they are valid + enough to safely be loaded into the server's shared buffers. If + given, fork specifies that only the pages of + the given fork are to be verified. fork can + be main for the main data + fork, fsm for the free space + map, vm for the visibility map, + or init for the initialization fork. The + default of NULL means that all forks of the + relation should be checked. The function returns a list of block + numbers that appear corrupted along with the path names of their + files. Use of this function is restricted to superusers by + default, but access may be granted to others + using GRANT. +
- - pg_relation_check_pages - - - pg_relation_check_pages iterates over all blocks of a - given relation and verifies if they are in a state where they can safely - be loaded into the shared buffers. If defined, - fork specifies that only the pages of the given - fork are to be verified. Fork can be 'main' for the - main data fork, 'fsm' for the free space map, - 'vm' for the visibility map, or - 'init' for the initialization fork. The default of - NULL means that all the forks of the relation are - checked. The function returns a list of blocks that are considered as - corrupted with the path of the related file. Use of this function is - restricted to superusers by default but access may be granted to others - using GRANT. - - From 60a51c6b32960822d3987ea7d2816c65bdbcb314 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 29 Oct 2020 09:17:34 +0900 Subject: [PATCH 397/589] Fix incorrect placement of pfree() in pg_relation_check_pages() This would cause the function to crash when more than one page is considered as broken and reported in the SRF. Reported-by: Noriyoshi Shinoda Discussion: https://postgr.es/m/TU4PR8401MB11523D42C315AAF822E74275EE170@TU4PR8401MB1152.NAMPRD84.PROD.OUTLOOK.COM --- src/backend/utils/adt/pagefuncs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/utils/adt/pagefuncs.c b/src/backend/utils/adt/pagefuncs.c index b6a23a2a4fa8..368ada515cf7 100644 --- a/src/backend/utils/adt/pagefuncs.c +++ b/src/backend/utils/adt/pagefuncs.c @@ -220,10 +220,10 @@ check_relation_fork(TupleDesc tupdesc, Tuplestorestate *tupstore, /* Save the corrupted blocks in the tuplestore. */ tuplestore_putvalues(tupstore, tupdesc, values, nulls); - - pfree(path); } + pfree(path); + /* Pop the error context stack */ error_context_stack = errcallback.previous; } From 94bc27b57680b4e757577e3f5b65dc32f96d33c1 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Wed, 28 Oct 2020 17:53:41 -0700 Subject: [PATCH 398/589] Centralize horizon determination for temp tables, fixing bug due to skew. This fixes a bug in the edge case where, for a temp table, heap_page_prune() can end up with a different horizon than heap_vacuum_rel(). Which can trigger errors like "ERROR: cannot freeze committed xmax ...". The bug was introduced due to interaction of a7212be8b9e "Set cutoff xmin more aggressively when vacuuming a temporary table." with dc7420c2c92 "snapshot scalability: Don't compute global horizons while building snapshots.". The problem is caused by lazy_scan_heap() assuming that the only reason its HeapTupleSatisfiesVacuum() call would return HEAPTUPLE_DEAD is if the tuple is a HOT tuple, or if the tuple's inserting transaction has aborted since the heap_page_prune() call. But after a7212be8b9e that was also possible in other cases for temp tables, because heap_page_prune() uses a different visibility test after dc7420c2c92. The fix is fairly simple: Move the special case logic for temp tables from vacuum_set_xid_limits() to the infrastructure introduced in dc7420c2c92. That ensures that the horizon used for pruning is at least as aggressive as the one used by lazy_scan_heap(). The concrete horizon used for temp tables is slightly different than the logic in dc7420c2c92, but should always be as aggressive as before (see comments). A significant benefit to centralizing the logic procarray.c is that now the more aggressive horizons for temp tables does not just apply to VACUUM but also to e.g. HOT pruning and the nbtree killtuples logic. Because isTopLevel is not needed by vacuum_set_xid_limits() anymore, I undid the the related changes from a7212be8b9e. This commit also adds an isolation test ensuring that the more aggressive vacuuming and pruning of temp tables keeps working. Debugged-By: Amit Kapila Debugged-By: Tom Lane Debugged-By: Ashutosh Sharma Author: Andres Freund Discussion: https://postgr.es/m/20201014203103.72oke6hqywcyhx7s@alap3.anarazel.de Discussion: https://postgr.es/m/20201015083735.derdzysdtqdvxshp@alap3.anarazel.de --- src/backend/access/heap/vacuumlazy.c | 1 - src/backend/commands/cluster.c | 28 +-- src/backend/commands/vacuum.c | 74 +++--- src/backend/storage/ipc/procarray.c | 59 ++++- src/include/commands/cluster.h | 3 +- src/include/commands/vacuum.h | 1 - src/test/isolation/expected/horizons.out | 281 +++++++++++++++++++++++ src/test/isolation/isolation_schedule | 1 + src/test/isolation/specs/horizons.spec | 169 ++++++++++++++ 9 files changed, 544 insertions(+), 73 deletions(-) create mode 100644 src/test/isolation/expected/horizons.out create mode 100644 src/test/isolation/specs/horizons.spec diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 4f2f38168dc2..be5439dd9d81 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -471,7 +471,6 @@ heap_vacuum_rel(Relation onerel, VacuumParams *params, params->freeze_table_age, params->multixact_freeze_min_age, params->multixact_freeze_table_age, - true, /* we must be a top-level command */ &OldestXmin, &FreezeLimit, &xidFullScanLimit, &MultiXactCutoff, &mxactFullScanLimit); diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index 0d647e912c47..04d12a7ece69 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -67,13 +67,10 @@ typedef struct } RelToCluster; -static void rebuild_relation(Relation OldHeap, Oid indexOid, - bool isTopLevel, bool verbose); +static void rebuild_relation(Relation OldHeap, Oid indexOid, bool verbose); static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, - bool isTopLevel, bool verbose, - bool *pSwapToastByContent, - TransactionId *pFreezeXid, - MultiXactId *pCutoffMulti); + bool verbose, bool *pSwapToastByContent, + TransactionId *pFreezeXid, MultiXactId *pCutoffMulti); static List *get_tables_to_cluster(MemoryContext cluster_context); @@ -173,7 +170,7 @@ cluster(ClusterStmt *stmt, bool isTopLevel) table_close(rel, NoLock); /* Do the job. */ - cluster_rel(tableOid, indexOid, stmt->options, isTopLevel); + cluster_rel(tableOid, indexOid, stmt->options); } else { @@ -222,8 +219,7 @@ cluster(ClusterStmt *stmt, bool isTopLevel) PushActiveSnapshot(GetTransactionSnapshot()); /* Do the job. */ cluster_rel(rvtc->tableOid, rvtc->indexOid, - stmt->options | CLUOPT_RECHECK, - isTopLevel); + stmt->options | CLUOPT_RECHECK); PopActiveSnapshot(); CommitTransactionCommand(); } @@ -254,7 +250,7 @@ cluster(ClusterStmt *stmt, bool isTopLevel) * and error messages should refer to the operation as VACUUM not CLUSTER. */ void -cluster_rel(Oid tableOid, Oid indexOid, int options, bool isTopLevel) +cluster_rel(Oid tableOid, Oid indexOid, int options) { Relation OldHeap; bool verbose = ((options & CLUOPT_VERBOSE) != 0); @@ -404,7 +400,7 @@ cluster_rel(Oid tableOid, Oid indexOid, int options, bool isTopLevel) TransferPredicateLocksToHeapRelation(OldHeap); /* rebuild_relation does all the dirty work */ - rebuild_relation(OldHeap, indexOid, isTopLevel, verbose); + rebuild_relation(OldHeap, indexOid, verbose); /* NB: rebuild_relation does table_close() on OldHeap */ @@ -549,12 +545,11 @@ mark_index_clustered(Relation rel, Oid indexOid, bool is_internal) * * OldHeap: table to rebuild --- must be opened and exclusive-locked! * indexOid: index to cluster by, or InvalidOid to rewrite in physical order. - * isTopLevel: should be passed down from ProcessUtility. * * NB: this routine closes OldHeap at the right time; caller should not. */ static void -rebuild_relation(Relation OldHeap, Oid indexOid, bool isTopLevel, bool verbose) +rebuild_relation(Relation OldHeap, Oid indexOid, bool verbose) { Oid tableOid = RelationGetRelid(OldHeap); Oid tableSpace = OldHeap->rd_rel->reltablespace; @@ -582,7 +577,7 @@ rebuild_relation(Relation OldHeap, Oid indexOid, bool isTopLevel, bool verbose) AccessExclusiveLock); /* Copy the heap data into the new table in the desired order */ - copy_table_data(OIDNewHeap, tableOid, indexOid, isTopLevel, verbose, + copy_table_data(OIDNewHeap, tableOid, indexOid, verbose, &swap_toast_by_content, &frozenXid, &cutoffMulti); /* @@ -733,8 +728,7 @@ make_new_heap(Oid OIDOldHeap, Oid NewTableSpace, char relpersistence, * *pCutoffMulti receives the MultiXactId used as a cutoff point. */ static void -copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, - bool isTopLevel, bool verbose, +copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, bool verbose, bool *pSwapToastByContent, TransactionId *pFreezeXid, MultiXactId *pCutoffMulti) { @@ -832,7 +826,7 @@ copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, * Since we're going to rewrite the whole table anyway, there's no reason * not to be aggressive about this. */ - vacuum_set_xid_limits(OldHeap, 0, 0, 0, 0, isTopLevel, + vacuum_set_xid_limits(OldHeap, 0, 0, 0, 0, &OldestXmin, &FreezeXid, NULL, &MultiXactCutoff, NULL); diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index ddeec870d811..1b6717f727ef 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -907,8 +907,7 @@ get_all_vacuum_rels(int options) /* * vacuum_set_xid_limits() -- compute oldestXmin and freeze cutoff points * - * Input parameters are the target relation, applicable freeze age settings, - * and isTopLevel which should be passed down from ProcessUtility. + * Input parameters are the target relation, applicable freeze age settings. * * The output parameters are: * - oldestXmin is the cutoff value used to distinguish whether tuples are @@ -934,7 +933,6 @@ vacuum_set_xid_limits(Relation rel, int freeze_table_age, int multixact_freeze_min_age, int multixact_freeze_table_age, - bool isTopLevel, TransactionId *oldestXmin, TransactionId *freezeLimit, TransactionId *xidFullScanLimit, @@ -950,53 +948,33 @@ vacuum_set_xid_limits(Relation rel, MultiXactId mxactLimit; MultiXactId safeMxactLimit; - if (RELATION_IS_LOCAL(rel) && !IsInTransactionBlock(isTopLevel)) - { - /* - * If we are processing a temp relation (which by prior checks must be - * one belonging to our session), and we are not inside any - * transaction block, then there can be no tuples in the rel that are - * still in-doubt, nor can there be any that are dead but possibly - * still interesting to some snapshot our session holds. We don't - * need to care whether other sessions could see such tuples, either. - * So we can aggressively set the cutoff xmin to be the nextXid. - */ - *oldestXmin = ReadNewTransactionId(); - } - else + /* + * We can always ignore processes running lazy vacuum. This is because we + * use these values only for deciding which tuples we must keep in the + * tables. Since lazy vacuum doesn't write its XID anywhere (usually no + * XID assigned), it's safe to ignore it. In theory it could be + * problematic to ignore lazy vacuums in a full vacuum, but keep in mind + * that only one vacuum process can be working on a particular table at + * any time, and that each vacuum is always an independent transaction. + */ + *oldestXmin = GetOldestNonRemovableTransactionId(rel); + + if (OldSnapshotThresholdActive()) { - /* - * Otherwise, calculate the cutoff xmin normally. - * - * We can always ignore processes running lazy vacuum. This is - * because we use these values only for deciding which tuples we must - * keep in the tables. Since lazy vacuum doesn't write its XID - * anywhere (usually no XID assigned), it's safe to ignore it. In - * theory it could be problematic to ignore lazy vacuums in a full - * vacuum, but keep in mind that only one vacuum process can be - * working on a particular table at any time, and that each vacuum is - * always an independent transaction. - */ - *oldestXmin = GetOldestNonRemovableTransactionId(rel); + TransactionId limit_xmin; + TimestampTz limit_ts; - if (OldSnapshotThresholdActive()) + if (TransactionIdLimitedForOldSnapshots(*oldestXmin, rel, + &limit_xmin, &limit_ts)) { - TransactionId limit_xmin; - TimestampTz limit_ts; - - if (TransactionIdLimitedForOldSnapshots(*oldestXmin, rel, - &limit_xmin, &limit_ts)) - { - /* - * TODO: We should only set the threshold if we are pruning on - * the basis of the increased limits. Not as crucial here as - * it is for opportunistic pruning (which often happens at a - * much higher frequency), but would still be a significant - * improvement. - */ - SetOldSnapshotThresholdTimestamp(limit_ts, limit_xmin); - *oldestXmin = limit_xmin; - } + /* + * TODO: We should only set the threshold if we are pruning on the + * basis of the increased limits. Not as crucial here as it is + * for opportunistic pruning (which often happens at a much higher + * frequency), but would still be a significant improvement. + */ + SetOldSnapshotThresholdTimestamp(limit_ts, limit_xmin); + *oldestXmin = limit_xmin; } } @@ -1930,7 +1908,7 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params) cluster_options |= CLUOPT_VERBOSE; /* VACUUM FULL is now a variant of CLUSTER; see cluster.c */ - cluster_rel(relid, InvalidOid, cluster_options, true); + cluster_rel(relid, InvalidOid, cluster_options); } else table_relation_vacuum(onerel, params, vac_strategy); diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c index 07c5eeb74951..ee4caa51152c 100644 --- a/src/backend/storage/ipc/procarray.c +++ b/src/backend/storage/ipc/procarray.c @@ -131,7 +131,7 @@ typedef struct ProcArrayStruct * different types of relations. As e.g. a normal user defined table in one * database is inaccessible to backends connected to another database, a test * specific to a relation can be more aggressive than a test for a shared - * relation. Currently we track three different states: + * relation. Currently we track four different states: * * 1) GlobalVisSharedRels, which only considers an XID's * effects visible-to-everyone if neither snapshots in any database, nor a @@ -153,6 +153,9 @@ typedef struct ProcArrayStruct * I.e. the difference to GlobalVisCatalogRels is that * replication slot's catalog_xmin is not taken into account. * + * 4) GlobalVisTempRels, which only considers the current session, as temp + * tables are not visible to other sessions. + * * GlobalVisTestFor(relation) returns the appropriate state * for the relation. * @@ -234,6 +237,13 @@ typedef struct ComputeXidHorizonsResult * defined tables. */ TransactionId data_oldest_nonremovable; + + /* + * Oldest xid for which deleted tuples need to be retained in this + * session's temporary tables. + */ + TransactionId temp_oldest_nonremovable; + } ComputeXidHorizonsResult; @@ -257,12 +267,13 @@ static TransactionId standbySnapshotPendingXmin; /* * State for visibility checks on different types of relations. See struct - * GlobalVisState for details. As shared, catalog, and user defined + * GlobalVisState for details. As shared, catalog, normal and temporary * relations can have different horizons, one such state exists for each. */ static GlobalVisState GlobalVisSharedRels; static GlobalVisState GlobalVisCatalogRels; static GlobalVisState GlobalVisDataRels; +static GlobalVisState GlobalVisTempRels; /* * This backend's RecentXmin at the last time the accurate xmin horizon was @@ -1668,6 +1679,23 @@ ComputeXidHorizons(ComputeXidHorizonsResult *h) h->oldest_considered_running = initial; h->shared_oldest_nonremovable = initial; h->data_oldest_nonremovable = initial; + + /* + * Only modifications made by this backend affect the horizon for + * temporary relations. Instead of a check in each iteration of the + * loop over all PGPROCs it is cheaper to just initialize to the + * current top-level xid any. + * + * Without an assigned xid we could use a horizon as agressive as + * ReadNewTransactionid(), but we can get away with the much cheaper + * latestCompletedXid + 1: If this backend has no xid there, by + * definition, can't be any newer changes in the temp table than + * latestCompletedXid. + */ + if (TransactionIdIsValid(MyProc->xid)) + h->temp_oldest_nonremovable = MyProc->xid; + else + h->temp_oldest_nonremovable = initial; } /* @@ -1760,6 +1788,7 @@ ComputeXidHorizons(ComputeXidHorizonsResult *h) TransactionIdOlder(h->shared_oldest_nonremovable, kaxmin); h->data_oldest_nonremovable = TransactionIdOlder(h->data_oldest_nonremovable, kaxmin); + /* temp relations cannot be accessed in recovery */ } else { @@ -1785,6 +1814,7 @@ ComputeXidHorizons(ComputeXidHorizonsResult *h) h->data_oldest_nonremovable = TransactionIdRetreatedBy(h->data_oldest_nonremovable, vacuum_defer_cleanup_age); + /* defer doesn't apply to temp relations */ } /* @@ -1844,6 +1874,8 @@ ComputeXidHorizons(ComputeXidHorizonsResult *h) h->catalog_oldest_nonremovable)); Assert(TransactionIdPrecedesOrEquals(h->oldest_considered_running, h->data_oldest_nonremovable)); + Assert(TransactionIdPrecedesOrEquals(h->oldest_considered_running, + h->temp_oldest_nonremovable)); Assert(!TransactionIdIsValid(h->slot_xmin) || TransactionIdPrecedesOrEquals(h->oldest_considered_running, h->slot_xmin)); @@ -1878,6 +1910,8 @@ GetOldestNonRemovableTransactionId(Relation rel) return horizons.shared_oldest_nonremovable; else if (RelationIsAccessibleInLogicalDecoding(rel)) return horizons.catalog_oldest_nonremovable; + else if (RELATION_IS_LOCAL(rel)) + return horizons.temp_oldest_nonremovable; else return horizons.data_oldest_nonremovable; } @@ -2054,8 +2088,8 @@ GetSnapshotDataReuse(Snapshot snapshot) * RecentXmin: the xmin computed for the most recent snapshot. XIDs * older than this are known not running any more. * - * And try to advance the bounds of GlobalVisSharedRels, GlobalVisCatalogRels, - * GlobalVisDataRels for the benefit of theGlobalVisTest* family of functions. + * And try to advance the bounds of GlobalVis{Shared,Catalog,Data,Temp}Rels + * for the benefit of theGlobalVisTest* family of functions. * * Note: this function should probably not be called with an argument that's * not statically allocated (see xip allocation below). @@ -2357,6 +2391,15 @@ GetSnapshotData(Snapshot snapshot) GlobalVisDataRels.definitely_needed = FullTransactionIdNewer(def_vis_fxid_data, GlobalVisDataRels.definitely_needed); + /* See temp_oldest_nonremovable computation in ComputeXidHorizons() */ + if (TransactionIdIsNormal(myxid)) + GlobalVisTempRels.definitely_needed = + FullXidRelativeTo(latest_completed, myxid); + else + { + GlobalVisTempRels.definitely_needed = latest_completed; + FullTransactionIdAdvance(&GlobalVisTempRels.definitely_needed); + } /* * Check if we know that we can initialize or increase the lower @@ -2375,6 +2418,8 @@ GetSnapshotData(Snapshot snapshot) GlobalVisDataRels.maybe_needed = FullTransactionIdNewer(GlobalVisDataRels.maybe_needed, oldestfxid); + /* accurate value known */ + GlobalVisTempRels.maybe_needed = GlobalVisTempRels.definitely_needed; } RecentXmin = xmin; @@ -3892,6 +3937,8 @@ GlobalVisTestFor(Relation rel) state = &GlobalVisSharedRels; else if (need_catalog) state = &GlobalVisCatalogRels; + else if (RELATION_IS_LOCAL(rel)) + state = &GlobalVisTempRels; else state = &GlobalVisDataRels; @@ -3942,6 +3989,9 @@ GlobalVisUpdateApply(ComputeXidHorizonsResult *horizons) GlobalVisDataRels.maybe_needed = FullXidRelativeTo(horizons->latest_completed, horizons->data_oldest_nonremovable); + GlobalVisTempRels.maybe_needed = + FullXidRelativeTo(horizons->latest_completed, + horizons->temp_oldest_nonremovable); /* * In longer running transactions it's possible that transactions we @@ -3957,6 +4007,7 @@ GlobalVisUpdateApply(ComputeXidHorizonsResult *horizons) GlobalVisDataRels.definitely_needed = FullTransactionIdNewer(GlobalVisDataRels.maybe_needed, GlobalVisDataRels.definitely_needed); + GlobalVisTempRels.definitely_needed = GlobalVisTempRels.maybe_needed; ComputeXidHorizonsResultLastXmin = RecentXmin; } diff --git a/src/include/commands/cluster.h b/src/include/commands/cluster.h index 1eb144204b6a..e05884781b94 100644 --- a/src/include/commands/cluster.h +++ b/src/include/commands/cluster.h @@ -19,8 +19,7 @@ extern void cluster(ClusterStmt *stmt, bool isTopLevel); -extern void cluster_rel(Oid tableOid, Oid indexOid, int options, - bool isTopLevel); +extern void cluster_rel(Oid tableOid, Oid indexOid, int options); extern void check_index_is_clusterable(Relation OldHeap, Oid indexOid, bool recheck, LOCKMODE lockmode); extern void mark_index_clustered(Relation rel, Oid indexOid, bool is_internal); diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index d9475c99890c..a4cd7214009c 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -267,7 +267,6 @@ extern void vacuum_set_xid_limits(Relation rel, int freeze_min_age, int freeze_table_age, int multixact_freeze_min_age, int multixact_freeze_table_age, - bool isTopLevel, TransactionId *oldestXmin, TransactionId *freezeLimit, TransactionId *xidFullScanLimit, diff --git a/src/test/isolation/expected/horizons.out b/src/test/isolation/expected/horizons.out new file mode 100644 index 000000000000..07bbc9832cdf --- /dev/null +++ b/src/test/isolation/expected/horizons.out @@ -0,0 +1,281 @@ +Parsed test spec with 2 sessions + +starting permutation: pruner_create_perm ll_start pruner_query_plan pruner_query pruner_query pruner_delete pruner_query pruner_query ll_commit pruner_drop +step pruner_create_perm: + CREATE TABLE horizons_tst (data int unique) WITH (autovacuum_enabled = off); + INSERT INTO horizons_tst(data) VALUES(1),(2); + +step ll_start: + BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; + SELECT 1; + +?column? + +1 +step pruner_query_plan: + EXPLAIN (COSTS OFF) SELECT * FROM horizons_tst ORDER BY data; + +QUERY PLAN + +Index Only Scan using horizons_tst_data_key on horizons_tst +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +2 +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +2 +step pruner_delete: + DELETE FROM horizons_tst; + +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +2 +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +2 +step ll_commit: COMMIT; +step pruner_drop: + DROP TABLE horizons_tst; + + +starting permutation: pruner_create_temp ll_start pruner_query_plan pruner_query pruner_query pruner_delete pruner_query pruner_query ll_commit pruner_drop +step pruner_create_temp: + CREATE TEMPORARY TABLE horizons_tst (data int unique) WITH (autovacuum_enabled = off); + INSERT INTO horizons_tst(data) VALUES(1),(2); + +step ll_start: + BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; + SELECT 1; + +?column? + +1 +step pruner_query_plan: + EXPLAIN (COSTS OFF) SELECT * FROM horizons_tst ORDER BY data; + +QUERY PLAN + +Index Only Scan using horizons_tst_data_key on horizons_tst +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +2 +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +2 +step pruner_delete: + DELETE FROM horizons_tst; + +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +2 +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +0 +step ll_commit: COMMIT; +step pruner_drop: + DROP TABLE horizons_tst; + + +starting permutation: pruner_create_temp ll_start pruner_query pruner_query pruner_begin pruner_delete pruner_query pruner_query ll_commit pruner_commit pruner_drop +step pruner_create_temp: + CREATE TEMPORARY TABLE horizons_tst (data int unique) WITH (autovacuum_enabled = off); + INSERT INTO horizons_tst(data) VALUES(1),(2); + +step ll_start: + BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; + SELECT 1; + +?column? + +1 +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +2 +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +2 +step pruner_begin: BEGIN; +step pruner_delete: + DELETE FROM horizons_tst; + +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +2 +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +2 +step ll_commit: COMMIT; +step pruner_commit: COMMIT; +step pruner_drop: + DROP TABLE horizons_tst; + + +starting permutation: pruner_create_perm ll_start pruner_query pruner_query pruner_delete pruner_vacuum pruner_query pruner_query ll_commit pruner_drop +step pruner_create_perm: + CREATE TABLE horizons_tst (data int unique) WITH (autovacuum_enabled = off); + INSERT INTO horizons_tst(data) VALUES(1),(2); + +step ll_start: + BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; + SELECT 1; + +?column? + +1 +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +2 +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +2 +step pruner_delete: + DELETE FROM horizons_tst; + +step pruner_vacuum: + VACUUM horizons_tst; + +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +2 +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +2 +step ll_commit: COMMIT; +step pruner_drop: + DROP TABLE horizons_tst; + + +starting permutation: pruner_create_temp ll_start pruner_query pruner_query pruner_delete pruner_vacuum pruner_query pruner_query ll_commit pruner_drop +step pruner_create_temp: + CREATE TEMPORARY TABLE horizons_tst (data int unique) WITH (autovacuum_enabled = off); + INSERT INTO horizons_tst(data) VALUES(1),(2); + +step ll_start: + BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; + SELECT 1; + +?column? + +1 +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +2 +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +2 +step pruner_delete: + DELETE FROM horizons_tst; + +step pruner_vacuum: + VACUUM horizons_tst; + +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +0 +step pruner_query: + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; + +?column? + +0 +step ll_commit: COMMIT; +step pruner_drop: + DROP TABLE horizons_tst; + diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index aa386ab1a25b..f2e752c4454a 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -78,6 +78,7 @@ test: timeouts test: vacuum-concurrent-drop test: vacuum-conflict test: vacuum-skip-locked +test: horizons test: predicate-hash test: predicate-gist test: predicate-gin diff --git a/src/test/isolation/specs/horizons.spec b/src/test/isolation/specs/horizons.spec new file mode 100644 index 000000000000..f74035c42f4a --- /dev/null +++ b/src/test/isolation/specs/horizons.spec @@ -0,0 +1,169 @@ +# Test that pruning and vacuuming pay attention to concurrent sessions +# in the right way. For normal relations that means that rows cannot +# be pruned away if there's an older snapshot, in contrast to that +# temporary tables should nearly always be prunable. +# +# NB: Think hard before adding a test showing that rows in permanent +# tables get pruned - it's quite likely that it'd be racy, e.g. due to +# an autovacuum worker holding a snapshot. + +setup { + CREATE OR REPLACE FUNCTION explain_json(p_query text) + RETURNS json + LANGUAGE plpgsql AS $$ + DECLARE + v_ret json; + BEGIN + EXECUTE p_query INTO STRICT v_ret; + RETURN v_ret; + END;$$; +} + +teardown { + DROP FUNCTION explain_json(text); +} + +session "lifeline" + +# Start a transaction, force a snapshot to be held +step "ll_start" +{ + BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; + SELECT 1; +} + +step "ll_commit" { COMMIT; } + + +session "pruner" + +setup +{ + SET enable_seqscan = false; + SET enable_indexscan = false; + SET enable_bitmapscan = false; +} + +step "pruner_create_temp" +{ + CREATE TEMPORARY TABLE horizons_tst (data int unique) WITH (autovacuum_enabled = off); + INSERT INTO horizons_tst(data) VALUES(1),(2); +} + +step "pruner_create_perm" +{ + CREATE TABLE horizons_tst (data int unique) WITH (autovacuum_enabled = off); + INSERT INTO horizons_tst(data) VALUES(1),(2); +} + +# Temp tables cannot be dropped in the teardown, so just always do so +# as part of the permutation +step "pruner_drop" +{ + DROP TABLE horizons_tst; +} + +step "pruner_delete" +{ + DELETE FROM horizons_tst; +} + +step "pruner_begin" { BEGIN; } +step "pruner_commit" { COMMIT; } + +step "pruner_vacuum" +{ + VACUUM horizons_tst; +} + +# Show the heap fetches of an ordered index-only-scan (other plans +# have been forbidden above) - that tells us how many non-killed leaf +# entries there are. +step "pruner_query" +{ + SELECT explain_json($$ + EXPLAIN (FORMAT json, BUFFERS, ANALYZE) + SELECT * FROM horizons_tst ORDER BY data;$$)->0->'Plan'->'Heap Fetches'; +} + +# Verify that the query plan still is an IOS +step "pruner_query_plan" +{ + EXPLAIN (COSTS OFF) SELECT * FROM horizons_tst ORDER BY data; +} + + +# Show that with a permanent relation deleted rows cannot be pruned +# away if there's a concurrent session still seeing the rows. +permutation + "pruner_create_perm" + "ll_start" + "pruner_query_plan" + # Run query that could do pruning twice, first has chance to prune, + # second would not perform heap fetches if first query did. + "pruner_query" + "pruner_query" + "pruner_delete" + "pruner_query" + "pruner_query" + "ll_commit" + "pruner_drop" + +# Show that with a temporary relation deleted rows can be pruned away, +# even if there's a concurrent session with a snapshot from before the +# deletion. That's safe because the session with the older snapshot +# cannot access the temporary table. +permutation + "pruner_create_temp" + "ll_start" + "pruner_query_plan" + "pruner_query" + "pruner_query" + "pruner_delete" + "pruner_query" + "pruner_query" + "ll_commit" + "pruner_drop" + +# Verify that pruning in temporary relations doesn't remove rows still +# visible in the current session +permutation + "pruner_create_temp" + "ll_start" + "pruner_query" + "pruner_query" + "pruner_begin" + "pruner_delete" + "pruner_query" + "pruner_query" + "ll_commit" + "pruner_commit" + "pruner_drop" + +# Show that vacuum cannot remove deleted rows still visible to another +# session's snapshot, when accessing a permanent table. +permutation + "pruner_create_perm" + "ll_start" + "pruner_query" + "pruner_query" + "pruner_delete" + "pruner_vacuum" + "pruner_query" + "pruner_query" + "ll_commit" + "pruner_drop" + +# Show that vacuum can remove deleted rows still visible to another +# session's snapshot, when accessing a temporary table. +permutation + "pruner_create_temp" + "ll_start" + "pruner_query" + "pruner_query" + "pruner_delete" + "pruner_vacuum" + "pruner_query" + "pruner_query" + "ll_commit" + "pruner_drop" From 8e90ec5580d5345fef31005d7cc2215ba2125070 Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Thu, 29 Oct 2020 09:11:51 +0530 Subject: [PATCH 399/589] Track statistics for streaming of changes from ReorderBuffer. This adds the statistics about transactions streamed to the decoding output plugin from ReorderBuffer. Users can query the pg_stat_replication_slots view to check these stats and call pg_stat_reset_replication_slot to reset the stats of a particular slot. Users can pass NULL in pg_stat_reset_replication_slot to reset stats of all the slots. Commit 9868167500 has added the basic infrastructure to capture the stats of slot and this commit extends the statistics collector to track additional information about slots. Bump the catversion as we have added new columns in the catalog entry. Author: Ajin Cherian and Amit Kapila Reviewed-by: Sawada Masahiko and Dilip Kumar Discussion: https://postgr.es/m/CAA4eK1+chpEomLzgSoky-D31qev19AmECNiEAietPQUGEFhtVA@mail.gmail.com --- doc/src/sgml/monitoring.sgml | 38 +++++++++++++++++++ src/backend/catalog/system_views.sql | 3 ++ src/backend/postmaster/pgstat.c | 11 +++++- src/backend/replication/logical/logical.c | 19 +++++++--- .../replication/logical/reorderbuffer.c | 20 ++++++++++ src/backend/replication/slot.c | 2 +- src/backend/utils/adt/pgstatfuncs.c | 9 +++-- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.dat | 6 +-- src/include/pgstat.h | 8 +++- src/include/replication/reorderbuffer.h | 5 +++ src/test/regress/expected/rules.out | 5 ++- 12 files changed, 111 insertions(+), 17 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 313e44ed5498..98e199545388 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -2632,6 +2632,44 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i + + + stream_txns bigint + + + Number of in-progress transactions streamed to the decoding output plugin + after the memory used by logical decoding of changes from WAL for this + slot exceeds logical_decoding_work_mem. Streaming only + works with toplevel transactions (subtransactions can't be streamed + independently), so the counter does not get incremented for subtransactions. + + + + + + stream_countbigint + + + Number of times in-progress transactions were streamed to the decoding + output plugin while decoding changes from WAL for this slot. Transactions + may get streamed repeatedly, and this counter gets incremented on every + such invocation. + + + + + + stream_bytesbigint + + + Amount of decoded in-progress transaction data streamed to the decoding + output plugin while decoding changes from WAL for this slot. This and other + streaming counters for this slot can be used to gauge the network I/O which + occurred during logical decoding and allow tuning logical_decoding_work_mem. + + + + stats_reset timestamp with time zone diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index c6dd084fbccb..5171ea05c7ea 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -802,6 +802,9 @@ CREATE VIEW pg_stat_replication_slots AS s.spill_txns, s.spill_count, s.spill_bytes, + s.stream_txns, + s.stream_count, + s.stream_bytes, s.stats_reset FROM pg_stat_get_replication_slots() AS s; diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 822f0ebc6285..f1dca2f25b75 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -1708,7 +1708,7 @@ pgstat_report_tempfile(size_t filesize) */ void pgstat_report_replslot(const char *slotname, int spilltxns, int spillcount, - int spillbytes) + int spillbytes, int streamtxns, int streamcount, int streambytes) { PgStat_MsgReplSlot msg; @@ -1721,6 +1721,9 @@ pgstat_report_replslot(const char *slotname, int spilltxns, int spillcount, msg.m_spill_txns = spilltxns; msg.m_spill_count = spillcount; msg.m_spill_bytes = spillbytes; + msg.m_stream_txns = streamtxns; + msg.m_stream_count = streamcount; + msg.m_stream_bytes = streambytes; pgstat_send(&msg, sizeof(PgStat_MsgReplSlot)); } @@ -6892,6 +6895,9 @@ pgstat_recv_replslot(PgStat_MsgReplSlot *msg, int len) replSlotStats[idx].spill_txns += msg->m_spill_txns; replSlotStats[idx].spill_count += msg->m_spill_count; replSlotStats[idx].spill_bytes += msg->m_spill_bytes; + replSlotStats[idx].stream_txns += msg->m_stream_txns; + replSlotStats[idx].stream_count += msg->m_stream_count; + replSlotStats[idx].stream_bytes += msg->m_stream_bytes; } } @@ -7125,6 +7131,9 @@ pgstat_reset_replslot(int i, TimestampTz ts) replSlotStats[i].spill_txns = 0; replSlotStats[i].spill_count = 0; replSlotStats[i].spill_bytes = 0; + replSlotStats[i].stream_txns = 0; + replSlotStats[i].stream_count = 0; + replSlotStats[i].stream_bytes = 0; replSlotStats[i].stat_reset_timestamp = ts; } diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c index 8675832f4d6e..d5cfbeaa4aff 100644 --- a/src/backend/replication/logical/logical.c +++ b/src/backend/replication/logical/logical.c @@ -1471,21 +1471,28 @@ UpdateDecodingStats(LogicalDecodingContext *ctx) ReorderBuffer *rb = ctx->reorder; /* - * Nothing to do if we haven't spilled anything since the last time the - * stats has been sent. + * Nothing to do if we haven't spilled or streamed anything since the last + * time the stats has been sent. */ - if (rb->spillBytes <= 0) + if (rb->spillBytes <= 0 && rb->streamBytes <= 0) return; - elog(DEBUG2, "UpdateDecodingStats: updating stats %p %lld %lld %lld", + elog(DEBUG2, "UpdateDecodingStats: updating stats %p %lld %lld %lld %lld %lld %lld", rb, (long long) rb->spillTxns, (long long) rb->spillCount, - (long long) rb->spillBytes); + (long long) rb->spillBytes, + (long long) rb->streamTxns, + (long long) rb->streamCount, + (long long) rb->streamBytes); pgstat_report_replslot(NameStr(ctx->slot->data.name), - rb->spillTxns, rb->spillCount, rb->spillBytes); + rb->spillTxns, rb->spillCount, rb->spillBytes, + rb->streamTxns, rb->streamCount, rb->streamBytes); rb->spillTxns = 0; rb->spillCount = 0; rb->spillBytes = 0; + rb->streamTxns = 0; + rb->streamCount = 0; + rb->streamBytes = 0; } diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index 7a8bf760791c..c1bd68011c59 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -346,6 +346,9 @@ ReorderBufferAllocate(void) buffer->spillTxns = 0; buffer->spillCount = 0; buffer->spillBytes = 0; + buffer->streamTxns = 0; + buffer->streamCount = 0; + buffer->streamBytes = 0; buffer->current_restart_decoding_lsn = InvalidXLogRecPtr; @@ -3482,6 +3485,8 @@ ReorderBufferStreamTXN(ReorderBuffer *rb, ReorderBufferTXN *txn) { Snapshot snapshot_now; CommandId command_id; + Size stream_bytes; + bool txn_is_streamed; /* We can never reach here for a subtransaction. */ Assert(txn->toptxn == NULL); @@ -3562,10 +3567,25 @@ ReorderBufferStreamTXN(ReorderBuffer *rb, ReorderBufferTXN *txn) txn->snapshot_now = NULL; } + /* + * Remember this information to be used later to update stats. We can't + * update the stats here as an error while processing the changes would + * lead to the accumulation of stats even though we haven't streamed all + * the changes. + */ + txn_is_streamed = rbtxn_is_streamed(txn); + stream_bytes = txn->total_size; + /* Process and send the changes to output plugin. */ ReorderBufferProcessTXN(rb, txn, InvalidXLogRecPtr, snapshot_now, command_id, true); + rb->streamCount += 1; + rb->streamBytes += stream_bytes; + + /* Don't consider already streamed transaction. */ + rb->streamTxns += (txn_is_streamed) ? 0 : 1; + Assert(dlist_is_empty(&txn->changes)); Assert(txn->nentries == 0); Assert(txn->nentries_mem == 0); diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 220b4cd6e99c..09be1d8c4851 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -320,7 +320,7 @@ ReplicationSlotCreate(const char *name, bool db_specific, * ReplicationSlotAllocationLock. */ if (SlotIsLogical(slot)) - pgstat_report_replslot(NameStr(slot->data.name), 0, 0, 0); + pgstat_report_replslot(NameStr(slot->data.name), 0, 0, 0, 0, 0, 0); /* * Now that the slot has been marked as in_use and active, it's safe to diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 472fa596e1f8..a210fc93b415 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2153,7 +2153,7 @@ pg_stat_get_archiver(PG_FUNCTION_ARGS) Datum pg_stat_get_replication_slots(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_REPLICATION_SLOT_COLS 5 +#define PG_STAT_GET_REPLICATION_SLOT_COLS 8 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; TupleDesc tupdesc; Tuplestorestate *tupstore; @@ -2201,11 +2201,14 @@ pg_stat_get_replication_slots(PG_FUNCTION_ARGS) values[1] = Int64GetDatum(s->spill_txns); values[2] = Int64GetDatum(s->spill_count); values[3] = Int64GetDatum(s->spill_bytes); + values[4] = Int64GetDatum(s->stream_txns); + values[5] = Int64GetDatum(s->stream_count); + values[6] = Int64GetDatum(s->stream_bytes); if (s->stat_reset_timestamp == 0) - nulls[4] = true; + nulls[7] = true; else - values[4] = TimestampTzGetDatum(s->stat_reset_timestamp); + values[7] = TimestampTzGetDatum(s->stat_reset_timestamp); tuplestore_putvalues(tupstore, tupdesc, values, nulls); } diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 8cf02fc0d852..73650f88e947 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 202010281 +#define CATALOG_VERSION_NO 202010291 #endif diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 24ec2cfed6a3..d9770bbadd8b 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5260,9 +5260,9 @@ proname => 'pg_stat_get_replication_slots', prorows => '10', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => '', - proallargtypes => '{text,int8,int8,int8,timestamptz}', - proargmodes => '{o,o,o,o,o}', - proargnames => '{slot_name,spill_txns,spill_count,spill_bytes,stats_reset}', + proallargtypes => '{text,int8,int8,int8,int8,int8,int8,timestamptz}', + proargmodes => '{o,o,o,o,o,o,o,o}', + proargnames => '{slot_name,spill_txns,spill_count,spill_bytes,stream_txns,stream_count,stream_bytes,stats_reset}', prosrc => 'pg_stat_get_replication_slots' }, { oid => '6118', descr => 'statistics: information about subscription', proname => 'pg_stat_get_subscription', proisstrict => 'f', provolatile => 's', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index a821ff4f158f..257e515bfe75 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -492,6 +492,9 @@ typedef struct PgStat_MsgReplSlot PgStat_Counter m_spill_txns; PgStat_Counter m_spill_count; PgStat_Counter m_spill_bytes; + PgStat_Counter m_stream_txns; + PgStat_Counter m_stream_count; + PgStat_Counter m_stream_bytes; } PgStat_MsgReplSlot; @@ -823,6 +826,9 @@ typedef struct PgStat_ReplSlotStats PgStat_Counter spill_txns; PgStat_Counter spill_count; PgStat_Counter spill_bytes; + PgStat_Counter stream_txns; + PgStat_Counter stream_count; + PgStat_Counter stream_bytes; TimestampTz stat_reset_timestamp; } PgStat_ReplSlotStats; @@ -1387,7 +1393,7 @@ extern void pgstat_report_deadlock(void); extern void pgstat_report_checksum_failures_in_db(Oid dboid, int failurecount); extern void pgstat_report_checksum_failure(void); extern void pgstat_report_replslot(const char *slotname, int spilltxns, int spillcount, - int spillbytes); + int spillbytes, int streamtxns, int streamcount, int streambytes); extern void pgstat_report_replslot_drop(const char *slotname); extern void pgstat_initialize(void); diff --git a/src/include/replication/reorderbuffer.h b/src/include/replication/reorderbuffer.h index 1c77819aad25..dfdda938b2a9 100644 --- a/src/include/replication/reorderbuffer.h +++ b/src/include/replication/reorderbuffer.h @@ -551,6 +551,11 @@ struct ReorderBuffer int64 spillTxns; /* number of transactions spilled to disk */ int64 spillCount; /* spill-to-disk invocation counter */ int64 spillBytes; /* amount of data spilled to disk */ + + /* Statistics about transactions streamed to the decoding output plugin */ + int64 streamTxns; /* number of transactions streamed */ + int64 streamCount; /* streaming invocation counter */ + int64 streamBytes; /* amount of data streamed */ }; diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 492cdcf74c36..097ff5d111f7 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2022,8 +2022,11 @@ pg_stat_replication_slots| SELECT s.slot_name, s.spill_txns, s.spill_count, s.spill_bytes, + s.stream_txns, + s.stream_count, + s.stream_bytes, s.stats_reset - FROM pg_stat_get_replication_slots() s(slot_name, spill_txns, spill_count, spill_bytes, stats_reset); + FROM pg_stat_get_replication_slots() s(slot_name, spill_txns, spill_count, spill_bytes, stream_txns, stream_count, stream_bytes, stats_reset); pg_stat_slru| SELECT s.name, s.blks_zeroed, s.blks_hit, From 1c7675a7a4265064a2c8e1ed02b2c042c2521664 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Wed, 28 Oct 2020 21:48:38 -0700 Subject: [PATCH 400/589] Fix wrong data table horizon computation during backend startup. When ComputeXidHorizons() was called before MyDatabaseOid is set, e.g. because a dead row in a shared relation is encountered during InitPostgres(), the horizon for normal tables was computed too aggressively, ignoring all backends connected to a database. During subsequent pruning in a data table the too aggressive horizon could end up still being used, possibly leading to still needed tuples being removed. Not good. This is a bug in dc7420c2c92, which the test added in 94bc27b5768 made visible, if run with force_parallel_mode set to regress. In that case the bug is reliably triggered, because "pruning_query" is run in a parallel worker and the start of that parallel worker is likely to encounter a dead row in pg_database. The fix is trivial: Compute a more pessimistic data table horizon if MyDatabaseId is not yet known. Author: Andres Freund Discussion: https://postgr.es/m/20201029040030.p4osrmaywhqaesd4@alap3.anarazel.de --- src/backend/storage/ipc/procarray.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c index ee4caa51152c..8da7b1d8a7bb 100644 --- a/src/backend/storage/ipc/procarray.c +++ b/src/backend/storage/ipc/procarray.c @@ -1757,9 +1757,16 @@ ComputeXidHorizons(ComputeXidHorizonsResult *h) * the shared horizon. But in recovery we cannot compute an accurate * per-database horizon as all xids are managed via the * KnownAssignedXids machinery. + * + * Be careful to compute a pessimistic value when MyDatabaseId is not + * set. If this is a backend in the process of starting up, we may not + * use a "too aggressive" horizon (otherwise we could end up using it + * to prune still needed data away). If the current backend never + * connects to a database that is harmless, because + * data_oldest_nonremovable will never be utilized. */ if (in_recovery || - proc->databaseId == MyDatabaseId || + MyDatabaseId == InvalidOid || proc->databaseId == MyDatabaseId || proc->databaseId == 0) /* always include WalSender */ { h->data_oldest_nonremovable = From f90149e6285aaae6b48559afce1bd638ee26c33e Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 29 Oct 2020 13:33:38 -0400 Subject: [PATCH 401/589] Don't use custom OID symbols in pg_type.dat, either. On the same reasoning as in commit 36b931214, forbid using custom oid_symbol macros in pg_type as well as pg_proc, so that we always rely on the predictable macro names generated by genbki.pl. We do continue to grant grandfather status to the names CASHOID and LSNOID, although those are now considered deprecated aliases for the preferred names MONEYOID and PG_LSNOID. This is because there's likely to be client-side code using the old names, and this bout of neatnik-ism doesn't quite seem worth breaking client code. There might be a case for grandfathering EVTTRIGGEROID, too, since externally-maintained PLs may reference that symbol. But renaming such references to EVENT_TRIGGEROID doesn't seem like a particularly heavy lift --- we make far more significant backend API changes in every major release. For now I didn't add that, but we could reconsider if there's pushback. The other names changed here seem pretty unlikely to have any outside uses. Again, we could add alias macros if there are complaints, but for now I didn't. As before, no need for a catversion bump. John Naylor Discussion: https://postgr.es/m/CAFBsxsHpCbjfoddNGpnnnY5pHwckWfiYkMYSF74PmP1su0+ZOw@mail.gmail.com --- src/backend/bootstrap/bootstrap.c | 2 +- src/backend/catalog/genbki.pl | 8 +++++-- src/backend/commands/event_trigger.c | 2 +- src/backend/utils/misc/pg_controldata.c | 10 ++++----- src/fe_utils/print.c | 2 +- src/include/catalog/pg_type.dat | 30 +++++++++---------------- src/include/catalog/pg_type.h | 7 ++++++ src/interfaces/ecpg/ecpglib/execute.c | 2 +- src/pl/plperl/plperl.c | 4 ++-- src/pl/plpgsql/src/pl_comp.c | 2 +- src/pl/plpgsql/src/pl_handler.c | 2 +- src/pl/plpython/plpy_procedure.c | 2 +- src/pl/tcl/pltcl.c | 2 +- 13 files changed, 38 insertions(+), 37 deletions(-) diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index 76b2f5066f68..a7ed93fdc14d 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -133,7 +133,7 @@ static const struct typinfo TypInfo[] = { F_XIDIN, F_XIDOUT}, {"cid", CIDOID, 0, 4, true, TYPALIGN_INT, TYPSTORAGE_PLAIN, InvalidOid, F_CIDIN, F_CIDOUT}, - {"pg_node_tree", PGNODETREEOID, 0, -1, false, TYPALIGN_INT, TYPSTORAGE_EXTENDED, DEFAULT_COLLATION_OID, + {"pg_node_tree", PG_NODE_TREEOID, 0, -1, false, TYPALIGN_INT, TYPSTORAGE_EXTENDED, DEFAULT_COLLATION_OID, F_PG_NODE_TREE_IN, F_PG_NODE_TREE_OUT}, {"int2vector", INT2VECTOROID, INT2OID, -1, false, TYPALIGN_INT, TYPSTORAGE_PLAIN, InvalidOid, F_INT2VECTORIN, F_INT2VECTOROUT}, diff --git a/src/backend/catalog/genbki.pl b/src/backend/catalog/genbki.pl index 47b1a5d79083..66fdaf67b135 100644 --- a/src/backend/catalog/genbki.pl +++ b/src/backend/catalog/genbki.pl @@ -588,9 +588,13 @@ } # Special hack to generate OID symbols for pg_type entries - # that lack one. - if ($catname eq 'pg_type' and !exists $bki_values{oid_symbol}) + if ($catname eq 'pg_type') { + die sprintf + "custom OID symbols are not allowed for pg_type entries: '%s'", + $bki_values{oid_symbol} + if defined $bki_values{oid_symbol}; + my $symbol = form_pg_type_symbol($bki_values{typname}); $bki_values{oid_symbol} = $symbol if defined $symbol; diff --git a/src/backend/commands/event_trigger.c b/src/backend/commands/event_trigger.c index 8bb17c34f5b0..3ffba4e63ec2 100644 --- a/src/backend/commands/event_trigger.c +++ b/src/backend/commands/event_trigger.c @@ -177,7 +177,7 @@ CreateEventTrigger(CreateEventTrigStmt *stmt) /* Find and validate the trigger function. */ funcoid = LookupFuncName(stmt->funcname, 0, NULL, false); funcrettype = get_func_rettype(funcoid); - if (funcrettype != EVTTRIGGEROID) + if (funcrettype != EVENT_TRIGGEROID) ereport(ERROR, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), errmsg("function %s must return type %s", diff --git a/src/backend/utils/misc/pg_controldata.c b/src/backend/utils/misc/pg_controldata.c index 609231275893..d50d87a6021c 100644 --- a/src/backend/utils/misc/pg_controldata.c +++ b/src/backend/utils/misc/pg_controldata.c @@ -94,9 +94,9 @@ pg_control_checkpoint(PG_FUNCTION_ARGS) */ tupdesc = CreateTemplateTupleDesc(18); TupleDescInitEntry(tupdesc, (AttrNumber) 1, "checkpoint_lsn", - LSNOID, -1, 0); + PG_LSNOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "redo_lsn", - LSNOID, -1, 0); + PG_LSNOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 3, "redo_wal_file", TEXTOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 4, "timeline_id", @@ -223,13 +223,13 @@ pg_control_recovery(PG_FUNCTION_ARGS) */ tupdesc = CreateTemplateTupleDesc(5); TupleDescInitEntry(tupdesc, (AttrNumber) 1, "min_recovery_end_lsn", - LSNOID, -1, 0); + PG_LSNOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "min_recovery_end_timeline", INT4OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 3, "backup_start_lsn", - LSNOID, -1, 0); + PG_LSNOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 4, "backup_end_lsn", - LSNOID, -1, 0); + PG_LSNOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 5, "end_of_backup_record_required", BOOLOID, -1, 0); tupdesc = BlessTupleDesc(tupdesc); diff --git a/src/fe_utils/print.c b/src/fe_utils/print.c index 508f537c0c7a..d542792230f8 100644 --- a/src/fe_utils/print.c +++ b/src/fe_utils/print.c @@ -3495,7 +3495,7 @@ column_type_alignment(Oid ftype) case XIDOID: case XID8OID: case CIDOID: - case CASHOID: + case MONEYOID: align = 'r'; break; default: diff --git a/src/include/catalog/pg_type.dat b/src/include/catalog/pg_type.dat index b2cec0741688..21a467a7a7ab 100644 --- a/src/include/catalog/pg_type.dat +++ b/src/include/catalog/pg_type.dat @@ -15,14 +15,10 @@ # For types used in the system catalogs, make sure the values here match # TypInfo[] in bootstrap.c. -# OID symbol macro names for pg_type OIDs are generated by genbki.pl -# according to the following rule, so you don't need to specify them -# here: +# OID symbol macro names for pg_type OIDs are not specified here because +# they are generated by genbki.pl according to the following rule: # foo_bar -> FOO_BAROID # _foo_bar -> FOO_BARARRAYOID -# -# The only oid_symbol entries in this file are for names that don't match -# this rule, and are grandfathered in. # To autogenerate an array type, add 'array_type_oid => 'nnnn' to the element # type, which will instruct genbki.pl to generate a BKI entry for it. @@ -144,35 +140,30 @@ typname => 'xml', typlen => '-1', typbyval => 'f', typcategory => 'U', typinput => 'xml_in', typoutput => 'xml_out', typreceive => 'xml_recv', typsend => 'xml_send', typalign => 'i', typstorage => 'x' }, -{ oid => '194', oid_symbol => 'PGNODETREEOID', - descr => 'string representing an internal node tree', +{ oid => '194', descr => 'string representing an internal node tree', typname => 'pg_node_tree', typlen => '-1', typbyval => 'f', typcategory => 'S', typinput => 'pg_node_tree_in', typoutput => 'pg_node_tree_out', typreceive => 'pg_node_tree_recv', typsend => 'pg_node_tree_send', typalign => 'i', typstorage => 'x', typcollation => 'default' }, -{ oid => '3361', oid_symbol => 'PGNDISTINCTOID', - descr => 'multivariate ndistinct coefficients', +{ oid => '3361', descr => 'multivariate ndistinct coefficients', typname => 'pg_ndistinct', typlen => '-1', typbyval => 'f', typcategory => 'S', typinput => 'pg_ndistinct_in', typoutput => 'pg_ndistinct_out', typreceive => 'pg_ndistinct_recv', typsend => 'pg_ndistinct_send', typalign => 'i', typstorage => 'x', typcollation => 'default' }, -{ oid => '3402', oid_symbol => 'PGDEPENDENCIESOID', - descr => 'multivariate dependencies', +{ oid => '3402', descr => 'multivariate dependencies', typname => 'pg_dependencies', typlen => '-1', typbyval => 'f', typcategory => 'S', typinput => 'pg_dependencies_in', typoutput => 'pg_dependencies_out', typreceive => 'pg_dependencies_recv', typsend => 'pg_dependencies_send', typalign => 'i', typstorage => 'x', typcollation => 'default' }, -{ oid => '5017', oid_symbol => 'PGMCVLISTOID', - descr => 'multivariate MCV list', +{ oid => '5017', descr => 'multivariate MCV list', typname => 'pg_mcv_list', typlen => '-1', typbyval => 'f', typcategory => 'S', typinput => 'pg_mcv_list_in', typoutput => 'pg_mcv_list_out', typreceive => 'pg_mcv_list_recv', typsend => 'pg_mcv_list_send', typalign => 'i', typstorage => 'x', typcollation => 'default' }, -{ oid => '32', oid_symbol => 'PGDDLCOMMANDOID', - descr => 'internal type for passing CollectedCommand', +{ oid => '32', descr => 'internal type for passing CollectedCommand', typname => 'pg_ddl_command', typlen => 'SIZEOF_POINTER', typbyval => 't', typtype => 'p', typcategory => 'P', typinput => 'pg_ddl_command_in', typoutput => 'pg_ddl_command_out', typreceive => 'pg_ddl_command_recv', @@ -237,7 +228,7 @@ typname => 'circle', typlen => '24', typbyval => 'f', typcategory => 'G', typinput => 'circle_in', typoutput => 'circle_out', typreceive => 'circle_recv', typsend => 'circle_send', typalign => 'd' }, -{ oid => '790', oid_symbol => 'CASHOID', array_type_oid => '791', +{ oid => '790', array_type_oid => '791', descr => 'monetary amounts, $d,ddd.cc', typname => 'money', typlen => '8', typbyval => 'FLOAT8PASSBYVAL', typcategory => 'N', typinput => 'cash_in', typoutput => 'cash_out', @@ -409,8 +400,7 @@ typsend => 'uuid_send', typalign => 'c' }, # pg_lsn -{ oid => '3220', oid_symbol => 'LSNOID', array_type_oid => '3221', - descr => 'PostgreSQL LSN datatype', +{ oid => '3220', array_type_oid => '3221', descr => 'PostgreSQL LSN datatype', typname => 'pg_lsn', typlen => '8', typbyval => 'FLOAT8PASSBYVAL', typcategory => 'U', typinput => 'pg_lsn_in', typoutput => 'pg_lsn_out', typreceive => 'pg_lsn_recv', typsend => 'pg_lsn_send', typalign => 'd' }, @@ -542,7 +532,7 @@ typname => 'trigger', typlen => '4', typbyval => 't', typtype => 'p', typcategory => 'P', typinput => 'trigger_in', typoutput => 'trigger_out', typreceive => '-', typsend => '-', typalign => 'i' }, -{ oid => '3838', oid_symbol => 'EVTTRIGGEROID', +{ oid => '3838', descr => 'pseudo-type for the result of an event trigger function', typname => 'event_trigger', typlen => '4', typbyval => 't', typtype => 'p', typcategory => 'P', typinput => 'event_trigger_in', diff --git a/src/include/catalog/pg_type.h b/src/include/catalog/pg_type.h index 7b375626484e..6ae6edf7e0e8 100644 --- a/src/include/catalog/pg_type.h +++ b/src/include/catalog/pg_type.h @@ -312,6 +312,13 @@ typedef FormData_pg_type *Form_pg_type; (typid) == ANYCOMPATIBLENONARRAYOID || \ (typid) == ANYCOMPATIBLERANGEOID) +/* + * Backwards compatibility for ancient random spellings of pg_type OID macros. + * Don't use these names in new code. + */ +#define CASHOID MONEYOID +#define LSNOID PG_LSNOID + #endif /* EXPOSE_TO_CLIENT_CODE */ diff --git a/src/interfaces/ecpg/ecpglib/execute.c b/src/interfaces/ecpg/ecpglib/execute.c index 9d61ae72506f..930b6adbe4fa 100644 --- a/src/interfaces/ecpg/ecpglib/execute.c +++ b/src/interfaces/ecpg/ecpglib/execute.c @@ -230,7 +230,7 @@ ecpg_is_type_an_array(int type, const struct statement *stmt, const struct varia return ECPG_ARRAY_ERROR; if (!ecpg_type_infocache_push(&(stmt->connection->cache_head), CIRCLEOID, ECPG_ARRAY_NONE, stmt->lineno)) return ECPG_ARRAY_ERROR; - if (!ecpg_type_infocache_push(&(stmt->connection->cache_head), CASHOID, ECPG_ARRAY_NONE, stmt->lineno)) + if (!ecpg_type_infocache_push(&(stmt->connection->cache_head), MONEYOID, ECPG_ARRAY_NONE, stmt->lineno)) return ECPG_ARRAY_ERROR; if (!ecpg_type_infocache_push(&(stmt->connection->cache_head), INETOID, ECPG_ARRAY_NONE, stmt->lineno)) return ECPG_ARRAY_ERROR; diff --git a/src/pl/plperl/plperl.c b/src/pl/plperl/plperl.c index 5fdf303fe6b3..7844c500eee8 100644 --- a/src/pl/plperl/plperl.c +++ b/src/pl/plperl/plperl.c @@ -2002,7 +2002,7 @@ plperl_validator(PG_FUNCTION_ARGS) { if (proc->prorettype == TRIGGEROID) is_trigger = true; - else if (proc->prorettype == EVTTRIGGEROID) + else if (proc->prorettype == EVENT_TRIGGEROID) is_event_trigger = true; else if (proc->prorettype != RECORDOID && proc->prorettype != VOIDOID) @@ -2838,7 +2838,7 @@ compile_plperl_function(Oid fn_oid, bool is_trigger, bool is_event_trigger) rettype == RECORDOID) /* okay */ ; else if (rettype == TRIGGEROID || - rettype == EVTTRIGGEROID) + rettype == EVENT_TRIGGEROID) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("trigger functions can only be called " diff --git a/src/pl/plpgsql/src/pl_comp.c b/src/pl/plpgsql/src/pl_comp.c index 344627da956f..6df8e14629d4 100644 --- a/src/pl/plpgsql/src/pl_comp.c +++ b/src/pl/plpgsql/src/pl_comp.c @@ -551,7 +551,7 @@ do_compile(FunctionCallInfo fcinfo, if (rettypeid == VOIDOID || rettypeid == RECORDOID) /* okay */ ; - else if (rettypeid == TRIGGEROID || rettypeid == EVTTRIGGEROID) + else if (rettypeid == TRIGGEROID || rettypeid == EVENT_TRIGGEROID) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("trigger functions can only be called as triggers"))); diff --git a/src/pl/plpgsql/src/pl_handler.c b/src/pl/plpgsql/src/pl_handler.c index b13bd2be31ce..7ece87df48b9 100644 --- a/src/pl/plpgsql/src/pl_handler.c +++ b/src/pl/plpgsql/src/pl_handler.c @@ -445,7 +445,7 @@ plpgsql_validator(PG_FUNCTION_ARGS) { if (proc->prorettype == TRIGGEROID) is_dml_trigger = true; - else if (proc->prorettype == EVTTRIGGEROID) + else if (proc->prorettype == EVENT_TRIGGEROID) is_event_trigger = true; else if (proc->prorettype != RECORDOID && proc->prorettype != VOIDOID && diff --git a/src/pl/plpython/plpy_procedure.c b/src/pl/plpython/plpy_procedure.c index ec47f52e61d6..1f05c633ef2b 100644 --- a/src/pl/plpython/plpy_procedure.c +++ b/src/pl/plpython/plpy_procedure.c @@ -220,7 +220,7 @@ PLy_procedure_create(HeapTuple procTup, Oid fn_oid, bool is_trigger) if (rettype == VOIDOID || rettype == RECORDOID) /* okay */ ; - else if (rettype == TRIGGEROID || rettype == EVTTRIGGEROID) + else if (rettype == TRIGGEROID || rettype == EVENT_TRIGGEROID) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("trigger functions can only be called as triggers"))); diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c index f4eabc8f39c0..a3a2dc8e89fb 100644 --- a/src/pl/tcl/pltcl.c +++ b/src/pl/tcl/pltcl.c @@ -1532,7 +1532,7 @@ compile_pltcl_function(Oid fn_oid, Oid tgreloid, rettype == RECORDOID) /* okay */ ; else if (rettype == TRIGGEROID || - rettype == EVTTRIGGEROID) + rettype == EVENT_TRIGGEROID) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("trigger functions can only be called as triggers"))); From 4a071afbd056282746a5bc9362e87f579a56402d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 29 Oct 2020 15:28:14 -0400 Subject: [PATCH 402/589] Stabilize timetz test across DST transitions. The timetz test cases I added in commit a9632830b were unintentionally sensitive to whether or not DST is active in the PST8PDT time zone. Thus, they'll start failing this coming weekend, as reported by Bernhard M. Wiedemann in bug #16689. Fortunately, DST-awareness is not significant to the purpose of these test cases, so we can just force them all to PDT (DST hours) to preserve stability of the results. Back-patch to v10, as the prior patch was. Discussion: https://postgr.es/m/16689-57701daa23b377bf@postgresql.org --- src/test/regress/expected/timetz.out | 32 ++++++++++++++-------------- src/test/regress/sql/timetz.sql | 16 +++++++------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/test/regress/expected/timetz.out b/src/test/regress/expected/timetz.out index 038bb5fa0943..1ab5ed510547 100644 --- a/src/test/regress/expected/timetz.out +++ b/src/test/regress/expected/timetz.out @@ -91,45 +91,45 @@ SELECT f1 AS "Ten" FROM TIMETZ_TBL WHERE f1 >= '00:00-07'; (12 rows) -- Check edge cases -SELECT '23:59:59.999999'::timetz; +SELECT '23:59:59.999999 PDT'::timetz; timetz -------------------- 23:59:59.999999-07 (1 row) -SELECT '23:59:59.9999999'::timetz; -- rounds up +SELECT '23:59:59.9999999 PDT'::timetz; -- rounds up timetz ------------- 24:00:00-07 (1 row) -SELECT '23:59:60'::timetz; -- rounds up +SELECT '23:59:60 PDT'::timetz; -- rounds up timetz ------------- 24:00:00-07 (1 row) -SELECT '24:00:00'::timetz; -- allowed +SELECT '24:00:00 PDT'::timetz; -- allowed timetz ------------- 24:00:00-07 (1 row) -SELECT '24:00:00.01'::timetz; -- not allowed -ERROR: date/time field value out of range: "24:00:00.01" -LINE 1: SELECT '24:00:00.01'::timetz; +SELECT '24:00:00.01 PDT'::timetz; -- not allowed +ERROR: date/time field value out of range: "24:00:00.01 PDT" +LINE 1: SELECT '24:00:00.01 PDT'::timetz; ^ -SELECT '23:59:60.01'::timetz; -- not allowed -ERROR: date/time field value out of range: "23:59:60.01" -LINE 1: SELECT '23:59:60.01'::timetz; +SELECT '23:59:60.01 PDT'::timetz; -- not allowed +ERROR: date/time field value out of range: "23:59:60.01 PDT" +LINE 1: SELECT '23:59:60.01 PDT'::timetz; ^ -SELECT '24:01:00'::timetz; -- not allowed -ERROR: date/time field value out of range: "24:01:00" -LINE 1: SELECT '24:01:00'::timetz; +SELECT '24:01:00 PDT'::timetz; -- not allowed +ERROR: date/time field value out of range: "24:01:00 PDT" +LINE 1: SELECT '24:01:00 PDT'::timetz; ^ -SELECT '25:00:00'::timetz; -- not allowed -ERROR: date/time field value out of range: "25:00:00" -LINE 1: SELECT '25:00:00'::timetz; +SELECT '25:00:00 PDT'::timetz; -- not allowed +ERROR: date/time field value out of range: "25:00:00 PDT" +LINE 1: SELECT '25:00:00 PDT'::timetz; ^ -- -- TIME simple math diff --git a/src/test/regress/sql/timetz.sql b/src/test/regress/sql/timetz.sql index b699e4b03c40..ce763d89e8bf 100644 --- a/src/test/regress/sql/timetz.sql +++ b/src/test/regress/sql/timetz.sql @@ -36,14 +36,14 @@ SELECT f1 AS "None" FROM TIMETZ_TBL WHERE f1 < '00:00-07'; SELECT f1 AS "Ten" FROM TIMETZ_TBL WHERE f1 >= '00:00-07'; -- Check edge cases -SELECT '23:59:59.999999'::timetz; -SELECT '23:59:59.9999999'::timetz; -- rounds up -SELECT '23:59:60'::timetz; -- rounds up -SELECT '24:00:00'::timetz; -- allowed -SELECT '24:00:00.01'::timetz; -- not allowed -SELECT '23:59:60.01'::timetz; -- not allowed -SELECT '24:01:00'::timetz; -- not allowed -SELECT '25:00:00'::timetz; -- not allowed +SELECT '23:59:59.999999 PDT'::timetz; +SELECT '23:59:59.9999999 PDT'::timetz; -- rounds up +SELECT '23:59:60 PDT'::timetz; -- rounds up +SELECT '24:00:00 PDT'::timetz; -- allowed +SELECT '24:00:00.01 PDT'::timetz; -- not allowed +SELECT '23:59:60.01 PDT'::timetz; -- not allowed +SELECT '24:01:00 PDT'::timetz; -- not allowed +SELECT '25:00:00 PDT'::timetz; -- not allowed -- -- TIME simple math From b401fa206d446f224ec4760f21e0a351816c97b3 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 30 Oct 2020 10:38:49 -0400 Subject: [PATCH 403/589] Doc: clarify description for pg_constraint.convalidated. Jimmy Angelakos Discussion: https://postgr.es/m/CABgVKCW_zPnvFXn24FTF0299_yU6+1p6JRUc0xpiZFWEXH1_jg@mail.gmail.com --- doc/src/sgml/catalogs.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 5bd54cb21832..0ccdff1cda62 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -2508,7 +2508,7 @@ SCRAM-SHA-256$<iteration count>:&l Has the constraint been validated? - Currently, can only be false for foreign keys and CHECK constraints + Currently, can be false only for foreign keys and CHECK constraints From 6f0bc5e1daf09686c526aa161da5336f7c94f4eb Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 30 Oct 2020 19:30:19 +0200 Subject: [PATCH 404/589] Fix missing validation for the new GiST sortsupport functions. Because of this, if you tried to create an operator family with the new sortsupport function, you got an error: ERROR: support function number 11 is invalid for access method gist We missed this in commit 16fa9b2b30 that added the sortsupport function, because it only added sortsupport to a built-in operator family. Author: Andrey Borodin Discussion: https://www.postgresql.org/message-id/3520A18A-5C38-4697-A2E3-F3BDE3496CD5%40yandex-team.ru --- src/backend/access/gist/gistvalidate.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/access/gist/gistvalidate.c b/src/backend/access/gist/gistvalidate.c index 8a14620fab27..e600015b12d6 100644 --- a/src/backend/access/gist/gistvalidate.c +++ b/src/backend/access/gist/gistvalidate.c @@ -338,6 +338,7 @@ gistadjustmembers(Oid opfamilyoid, case GIST_DISTANCE_PROC: case GIST_FETCH_PROC: case GIST_OPTIONS_PROC: + case GIST_SORTSUPPORT_PROC: /* Optional, so force it to be a soft family dependency */ op->ref_is_hard = false; op->ref_is_family = true; From 970c05057593c2f5919a69b43fd917c4fa86f51c Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 30 Oct 2020 17:00:59 -0400 Subject: [PATCH 405/589] Fix assertion failure in check_new_partition_bound(). Commit 6b2c4e59d was overly confident about not being able to see a negative cmpval result from partition_range_bsearch(). Adjust the code to cope with that. Report and patch by Amul Sul; some additional cosmetic changes by me Discussion: https://postgr.es/m/CAAJ_b97WCO=EyVA7fKzc86kKfojHXLU04_zs7-7+yVzm=-1QkQ@mail.gmail.com --- src/backend/partitioning/partbounds.c | 40 +++++++++++----------- src/test/regress/expected/create_table.out | 4 +++ src/test/regress/sql/create_table.sql | 1 + 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/backend/partitioning/partbounds.c b/src/backend/partitioning/partbounds.c index 66c42b58986e..a4f97c10fe7a 100644 --- a/src/backend/partitioning/partbounds.c +++ b/src/backend/partitioning/partbounds.c @@ -2703,10 +2703,10 @@ add_merged_range_bounds(int partnatts, FmgrInfo *partsupfuncs, prev_ub.lower = false; /* - * We pass to partition_rbound_cmp() lower1 as false to prevent it - * from considering the last upper bound to be smaller than the lower - * bound of the merged partition when the values of the two range - * bounds compare equal. + * We pass lower1 = false to partition_rbound_cmp() to prevent it from + * considering the last upper bound to be smaller than the lower bound + * of the merged partition when the values of the two range bounds + * compare equal. */ cmpval = partition_rbound_cmp(partnatts, partsupfuncs, partcollations, merged_lb->datums, merged_lb->kind, @@ -2978,16 +2978,19 @@ check_new_partition_bound(char *relname, Relation parent, /* * First check if the resulting range would be empty with - * specified lower and upper bounds + * specified lower and upper bounds. partition_rbound_cmp + * cannot return zero here, since the lower-bound flags are + * different. */ cmpval = partition_rbound_cmp(key->partnatts, key->partsupfunc, key->partcollation, lower->datums, lower->kind, true, upper); - if (cmpval >= 0) + Assert(cmpval != 0); + if (cmpval > 0) { - /* Fetch the problematic key from the lower datums list. */ + /* Point to problematic key in the lower datums list. */ PartitionRangeDatum *datum = list_nth(spec->lowerdatums, cmpval - 1); @@ -3057,11 +3060,11 @@ check_new_partition_bound(char *relname, Relation parent, if (cmpval < 0) { /* - * Fetch the problematic key from the upper + * Point to problematic key in the upper * datums list. */ PartitionRangeDatum *datum = - list_nth(spec->upperdatums, -cmpval - 1); + list_nth(spec->upperdatums, Abs(cmpval) - 1); /* * The new partition overlaps with the @@ -3083,15 +3086,11 @@ check_new_partition_bound(char *relname, Relation parent, PartitionRangeDatum *datum; /* - * Fetch the problematic key from the lower datums - * list. Given the way partition_range_bsearch() - * works, the new lower bound is certainly >= the - * bound at offset. If the bound matches exactly, we - * flag the 1st key. + * Point to problematic key in the lower datums list; + * if we have equality, point to the first one. */ - Assert(cmpval >= 0); datum = cmpval == 0 ? linitial(spec->lowerdatums) : - list_nth(spec->lowerdatums, cmpval - 1); + list_nth(spec->lowerdatums, Abs(cmpval) - 1); overlap = true; overlap_location = datum->location; with = boundinfo->indexes[offset + 1]; @@ -3393,13 +3392,14 @@ partition_rbound_cmp(int partnatts, FmgrInfo *partsupfunc, else if (kind1[i] > kind2[i]) return colnum; else if (kind1[i] != PARTITION_RANGE_DATUM_VALUE) - + { /* * The column bounds are both MINVALUE or both MAXVALUE. No later * columns should be considered, but we still need to compare * whether they are upper or lower bounds. */ break; + } cmpval = DatumGetInt32(FunctionCall2Coll(&partsupfunc[i], partcollation[i], @@ -3692,9 +3692,9 @@ qsort_partition_rbound_cmp(const void *a, const void *b, void *arg) PartitionRangeBound *b2 = (*(PartitionRangeBound *const *) b); PartitionKey key = (PartitionKey) arg; - return partition_rbound_cmp(key->partnatts, key->partsupfunc, - key->partcollation, b1->datums, b1->kind, - b1->lower, b2); + return compare_range_bounds(key->partnatts, key->partsupfunc, + key->partcollation, + b1, b2); } /* diff --git a/src/test/regress/expected/create_table.out b/src/test/regress/expected/create_table.out index 1fc266dd65cb..ed8c01b8decc 100644 --- a/src/test/regress/expected/create_table.out +++ b/src/test/regress/expected/create_table.out @@ -856,6 +856,10 @@ ERROR: partition "fail_part" would overlap partition "part0" LINE 1: ..._part PARTITION OF range_parted2 FOR VALUES FROM (minvalue) ... ^ CREATE TABLE part1 PARTITION OF range_parted2 FOR VALUES FROM (1) TO (10); +CREATE TABLE fail_part PARTITION OF range_parted2 FOR VALUES FROM (-1) TO (1); +ERROR: partition "fail_part" would overlap partition "part0" +LINE 1: ..._part PARTITION OF range_parted2 FOR VALUES FROM (-1) TO (1)... + ^ CREATE TABLE fail_part PARTITION OF range_parted2 FOR VALUES FROM (9) TO (maxvalue); ERROR: partition "fail_part" would overlap partition "part1" LINE 1: ..._part PARTITION OF range_parted2 FOR VALUES FROM (9) TO (max... diff --git a/src/test/regress/sql/create_table.sql b/src/test/regress/sql/create_table.sql index cee822aa8b6e..d257679ba68c 100644 --- a/src/test/regress/sql/create_table.sql +++ b/src/test/regress/sql/create_table.sql @@ -687,6 +687,7 @@ CREATE TABLE fail_part PARTITION OF range_parted2 FOR VALUES FROM (1) TO (1); CREATE TABLE part0 PARTITION OF range_parted2 FOR VALUES FROM (minvalue) TO (1); CREATE TABLE fail_part PARTITION OF range_parted2 FOR VALUES FROM (minvalue) TO (2); CREATE TABLE part1 PARTITION OF range_parted2 FOR VALUES FROM (1) TO (10); +CREATE TABLE fail_part PARTITION OF range_parted2 FOR VALUES FROM (-1) TO (1); CREATE TABLE fail_part PARTITION OF range_parted2 FOR VALUES FROM (9) TO (maxvalue); CREATE TABLE part2 PARTITION OF range_parted2 FOR VALUES FROM (20) TO (30); CREATE TABLE part3 PARTITION OF range_parted2 FOR VALUES FROM (30) TO (40); From f90e80b9138355a51d2d5b5b63e1f89c4ba53325 Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Sat, 31 Oct 2020 08:43:28 -0700 Subject: [PATCH 406/589] Reproduce debug_query_string==NULL on parallel workers. Certain background workers initiate parallel queries while debug_query_string==NULL, at which point they attempted strlen(NULL) and died to SIGSEGV. Older debug_query_string observers allow NULL, so do likewise in these newer ones. Back-patch to v11, where commit 7de4a1bcc56f494acbd0d6e70781df877dc8ecb5 introduced the first of these. Discussion: https://postgr.es/m/20201014022636.GA1962668@rfd.leadboat.com --- src/backend/access/heap/vacuumlazy.c | 28 +++++++++++++++++++--------- src/backend/access/nbtree/nbtsort.c | 25 +++++++++++++++++-------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index be5439dd9d81..2174fccb1e6f 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -3237,7 +3237,6 @@ begin_parallel_vacuum(Oid relid, Relation *Irel, LVRelStats *vacrelstats, WalUsage *wal_usage; bool *can_parallel_vacuum; long maxtuples; - char *sharedquery; Size est_shared; Size est_deadtuples; int nindexes_mwm = 0; @@ -3334,9 +3333,14 @@ begin_parallel_vacuum(Oid relid, Relation *Irel, LVRelStats *vacrelstats, shm_toc_estimate_keys(&pcxt->estimator, 1); /* Finally, estimate PARALLEL_VACUUM_KEY_QUERY_TEXT space */ - querylen = strlen(debug_query_string); - shm_toc_estimate_chunk(&pcxt->estimator, querylen + 1); - shm_toc_estimate_keys(&pcxt->estimator, 1); + if (debug_query_string) + { + querylen = strlen(debug_query_string); + shm_toc_estimate_chunk(&pcxt->estimator, querylen + 1); + shm_toc_estimate_keys(&pcxt->estimator, 1); + } + else + querylen = 0; /* keep compiler quiet */ InitializeParallelDSM(pcxt); @@ -3381,10 +3385,16 @@ begin_parallel_vacuum(Oid relid, Relation *Irel, LVRelStats *vacrelstats, lps->wal_usage = wal_usage; /* Store query string for workers */ - sharedquery = (char *) shm_toc_allocate(pcxt->toc, querylen + 1); - memcpy(sharedquery, debug_query_string, querylen + 1); - sharedquery[querylen] = '\0'; - shm_toc_insert(pcxt->toc, PARALLEL_VACUUM_KEY_QUERY_TEXT, sharedquery); + if (debug_query_string) + { + char *sharedquery; + + sharedquery = (char *) shm_toc_allocate(pcxt->toc, querylen + 1); + memcpy(sharedquery, debug_query_string, querylen + 1); + sharedquery[querylen] = '\0'; + shm_toc_insert(pcxt->toc, + PARALLEL_VACUUM_KEY_QUERY_TEXT, sharedquery); + } pfree(can_parallel_vacuum); return lps; @@ -3527,7 +3537,7 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) elog(DEBUG1, "starting parallel vacuum worker for bulk delete"); /* Set debug_query_string for individual workers */ - sharedquery = shm_toc_lookup(toc, PARALLEL_VACUUM_KEY_QUERY_TEXT, false); + sharedquery = shm_toc_lookup(toc, PARALLEL_VACUUM_KEY_QUERY_TEXT, true); debug_query_string = sharedquery; pgstat_report_activity(STATE_RUNNING, debug_query_string); diff --git a/src/backend/access/nbtree/nbtsort.c b/src/backend/access/nbtree/nbtsort.c index efee86784bbf..8730de25ed71 100644 --- a/src/backend/access/nbtree/nbtsort.c +++ b/src/backend/access/nbtree/nbtsort.c @@ -1466,7 +1466,6 @@ _bt_begin_parallel(BTBuildState *buildstate, bool isconcurrent, int request) WalUsage *walusage; BufferUsage *bufferusage; bool leaderparticipates = true; - char *sharedquery; int querylen; #ifdef DISABLE_LEADER_PARTICIPATION @@ -1533,9 +1532,14 @@ _bt_begin_parallel(BTBuildState *buildstate, bool isconcurrent, int request) shm_toc_estimate_keys(&pcxt->estimator, 1); /* Finally, estimate PARALLEL_KEY_QUERY_TEXT space */ - querylen = strlen(debug_query_string); - shm_toc_estimate_chunk(&pcxt->estimator, querylen + 1); - shm_toc_estimate_keys(&pcxt->estimator, 1); + if (debug_query_string) + { + querylen = strlen(debug_query_string); + shm_toc_estimate_chunk(&pcxt->estimator, querylen + 1); + shm_toc_estimate_keys(&pcxt->estimator, 1); + } + else + querylen = 0; /* keep compiler quiet */ /* Everyone's had a chance to ask for space, so now create the DSM */ InitializeParallelDSM(pcxt); @@ -1599,9 +1603,14 @@ _bt_begin_parallel(BTBuildState *buildstate, bool isconcurrent, int request) } /* Store query string for workers */ - sharedquery = (char *) shm_toc_allocate(pcxt->toc, querylen + 1); - memcpy(sharedquery, debug_query_string, querylen + 1); - shm_toc_insert(pcxt->toc, PARALLEL_KEY_QUERY_TEXT, sharedquery); + if (debug_query_string) + { + char *sharedquery; + + sharedquery = (char *) shm_toc_allocate(pcxt->toc, querylen + 1); + memcpy(sharedquery, debug_query_string, querylen + 1); + shm_toc_insert(pcxt->toc, PARALLEL_KEY_QUERY_TEXT, sharedquery); + } /* * Allocate space for each worker's WalUsage and BufferUsage; no need to @@ -1806,7 +1815,7 @@ _bt_parallel_build_main(dsm_segment *seg, shm_toc *toc) #endif /* BTREE_BUILD_STATS */ /* Set debug_query_string for individual workers first */ - sharedquery = shm_toc_lookup(toc, PARALLEL_KEY_QUERY_TEXT, false); + sharedquery = shm_toc_lookup(toc, PARALLEL_KEY_QUERY_TEXT, true); debug_query_string = sharedquery; /* Report the query string from leader */ From d2246cde825e4e1a85408390c66367b85b51a233 Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Sat, 31 Oct 2020 08:47:02 -0700 Subject: [PATCH 407/589] Set debug_query_string in worker_spi. This makes elog.c emit the string, which is good practice for a background worker that executes SQL strings. Reviewed by Tom Lane. Discussion: https://postgr.es/m/20201014022636.GA1962668@rfd.leadboat.com --- src/test/modules/worker_spi/worker_spi.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/test/modules/worker_spi/worker_spi.c b/src/test/modules/worker_spi/worker_spi.c index 1c7b17c56fec..258237f9bfe9 100644 --- a/src/test/modules/worker_spi/worker_spi.c +++ b/src/test/modules/worker_spi/worker_spi.c @@ -119,6 +119,7 @@ initialize_worker_spi(worktable *table) appendStringInfo(&buf, "select count(*) from pg_namespace where nspname = '%s'", table->schema); + debug_query_string = buf.data; ret = SPI_execute(buf.data, true, 0); if (ret != SPI_OK_SELECT) elog(FATAL, "SPI_execute failed: error code %d", ret); @@ -134,6 +135,7 @@ initialize_worker_spi(worktable *table) if (ntup == 0) { + debug_query_string = NULL; resetStringInfo(&buf); appendStringInfo(&buf, "CREATE SCHEMA \"%s\" " @@ -147,15 +149,19 @@ initialize_worker_spi(worktable *table) /* set statement start time */ SetCurrentStatementStartTimestamp(); + debug_query_string = buf.data; ret = SPI_execute(buf.data, false, 0); if (ret != SPI_OK_UTILITY) elog(FATAL, "failed to create my schema"); + + debug_query_string = NULL; /* rest is not statement-specific */ } SPI_finish(); PopActiveSnapshot(); CommitTransactionCommand(); + debug_query_string = NULL; pgstat_report_activity(STATE_IDLE, NULL); } @@ -262,6 +268,7 @@ worker_spi_main(Datum main_arg) StartTransactionCommand(); SPI_connect(); PushActiveSnapshot(GetTransactionSnapshot()); + debug_query_string = buf.data; pgstat_report_activity(STATE_RUNNING, buf.data); /* We can now execute queries via SPI */ @@ -291,6 +298,7 @@ worker_spi_main(Datum main_arg) SPI_finish(); PopActiveSnapshot(); CommitTransactionCommand(); + debug_query_string = NULL; pgstat_report_stat(false); pgstat_report_activity(STATE_IDLE, NULL); } From aecaa04418f39c32adb3dbf91c4aa7f6e175f01c Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Sun, 1 Nov 2020 19:22:59 +0900 Subject: [PATCH 408/589] Add error code for encryption failure in pgcrypto PXE_DECRYPT_FAILED exists already for decryption errors, and an equivalent for encryption did not exist. There is one code path that deals with such failures for OpenSSL but it used PXE_ERR_GENERIC, which was inconsistent. This switches this code path to use the new error PXE_ENCRYPT_FAILED instead of PXE_ERR_GENERIC, making the code used for encryption more consistent with the decryption. Author: Daniel Gustafsson Discussion: https://postgr.es/m/03049139-CB7A-436E-B71B-42696D3E2EF7@yesql.se --- contrib/pgcrypto/openssl.c | 2 +- contrib/pgcrypto/px.c | 1 + contrib/pgcrypto/px.h | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/pgcrypto/openssl.c b/contrib/pgcrypto/openssl.c index ed96e4ce5359..5ebe21340693 100644 --- a/contrib/pgcrypto/openssl.c +++ b/contrib/pgcrypto/openssl.c @@ -400,7 +400,7 @@ gen_ossl_encrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, } if (!EVP_EncryptUpdate(od->evp_ctx, res, &outlen, data, dlen)) - return PXE_ERR_GENERIC; + return PXE_ENCRYPT_FAILED; return 0; } diff --git a/contrib/pgcrypto/px.c b/contrib/pgcrypto/px.c index 6a4681dae989..a243f575d3be 100644 --- a/contrib/pgcrypto/px.c +++ b/contrib/pgcrypto/px.c @@ -58,6 +58,7 @@ static const struct error_desc px_err_list[] = { {PXE_MCRYPT_INTERNAL, "mcrypt internal error"}, {PXE_NO_RANDOM, "Failed to generate strong random bits"}, {PXE_DECRYPT_FAILED, "Decryption failed"}, + {PXE_ENCRYPT_FAILED, "Encryption failed"}, {PXE_PGP_CORRUPT_DATA, "Wrong key or corrupt data"}, {PXE_PGP_CORRUPT_ARMOR, "Corrupt ascii-armor"}, {PXE_PGP_UNSUPPORTED_COMPR, "Unsupported compression algorithm"}, diff --git a/contrib/pgcrypto/px.h b/contrib/pgcrypto/px.h index 5487923edb3e..17d6f2249870 100644 --- a/contrib/pgcrypto/px.h +++ b/contrib/pgcrypto/px.h @@ -61,6 +61,7 @@ #define PXE_MCRYPT_INTERNAL -16 #define PXE_NO_RANDOM -17 #define PXE_DECRYPT_FAILED -18 +#define PXE_ENCRYPT_FAILED -19 #define PXE_PGP_CORRUPT_DATA -100 #define PXE_PGP_CORRUPT_ARMOR -101 From b17ff07aa3eb142d2cde2ea00e4a4e8f63686f96 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Sun, 1 Nov 2020 21:22:07 +0900 Subject: [PATCH 409/589] Preserve index data in pg_statistic across REINDEX CONCURRENTLY MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Statistics associated to an index got lost after running REINDEX CONCURRENTLY, while the non-concurrent case preserves these correctly. The concurrent and non-concurrent operations need to be consistent for the end-user, and missing statistics would force to wait for a new analyze to happen, which could take some time depending on the activity of the existing autovacuum workers. This issue is fixed by copying any existing entries in pg_statistic associated to the old index to the new one. Note that this copy is already done with the data of the index in the stats collector. Reported-by: Fabrízio de Royes Mello Author: Michael Paquier, Fabrízio de Royes Mello Reviewed-by: Justin Pryzby Discussion: https://postgr.es/m/CAFcNs+qpFPmiHd1oTXvcPdvAHicJDA9qBUSujgAhUMJyUMb+SA@mail.gmail.com Backpatch-through: 12 --- src/backend/catalog/heap.c | 41 ++++++++++++++++++++++ src/backend/catalog/index.c | 3 ++ src/include/catalog/heap.h | 1 + src/test/regress/expected/create_index.out | 22 ++++++++++++ src/test/regress/sql/create_index.sql | 12 +++++++ 5 files changed, 79 insertions(+) diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 67144aa3c944..9ccf378d45b8 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -3142,6 +3142,47 @@ cookConstraint(ParseState *pstate, return expr; } +/* + * CopyStatistics --- copy entries in pg_statistic from one rel to another + */ +void +CopyStatistics(Oid fromrelid, Oid torelid) +{ + HeapTuple tup; + SysScanDesc scan; + ScanKeyData key[1]; + Relation statrel; + + statrel = table_open(StatisticRelationId, RowExclusiveLock); + + /* Now search for stat records */ + ScanKeyInit(&key[0], + Anum_pg_statistic_starelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(fromrelid)); + + scan = systable_beginscan(statrel, StatisticRelidAttnumInhIndexId, + true, NULL, 1, key); + + while (HeapTupleIsValid((tup = systable_getnext(scan)))) + { + Form_pg_statistic statform; + + /* make a modifiable copy */ + tup = heap_copytuple(tup); + statform = (Form_pg_statistic) GETSTRUCT(tup); + + /* update the copy of the tuple and insert it */ + statform->starelid = torelid; + CatalogTupleInsert(statrel, tup); + + heap_freetuple(tup); + } + + systable_endscan(scan); + + table_close(statrel, RowExclusiveLock); +} /* * RemoveStatistics --- remove entries in pg_statistic for a rel or column diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 0974f3e23a23..a18cc7cad309 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -1711,6 +1711,9 @@ index_concurrently_swap(Oid newIndexId, Oid oldIndexId, const char *oldName) } } + /* Copy data of pg_statistic from the old index to the new one */ + CopyStatistics(oldIndexId, newIndexId); + /* Close relations */ table_close(pg_class, RowExclusiveLock); table_close(pg_index, RowExclusiveLock); diff --git a/src/include/catalog/heap.h b/src/include/catalog/heap.h index d31141c1a218..f94defff3ce8 100644 --- a/src/include/catalog/heap.h +++ b/src/include/catalog/heap.h @@ -134,6 +134,7 @@ extern void RemoveAttributeById(Oid relid, AttrNumber attnum); extern void RemoveAttrDefault(Oid relid, AttrNumber attnum, DropBehavior behavior, bool complain, bool internal); extern void RemoveAttrDefaultById(Oid attrdefId); +extern void CopyStatistics(Oid fromrelid, Oid torelid); extern void RemoveStatistics(Oid relid, AttrNumber attnum); extern const FormData_pg_attribute *SystemAttributeDefinition(AttrNumber attno); diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 6ace7662ee1f..012c1eb0676c 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2551,6 +2551,17 @@ CREATE UNIQUE INDEX concur_exprs_index_pred ON concur_exprs_tab (c1) CREATE UNIQUE INDEX concur_exprs_index_pred_2 ON concur_exprs_tab ((1 / c1)) WHERE ('-H') >= (c2::TEXT) COLLATE "C"; +ANALYZE concur_exprs_tab; +SELECT starelid::regclass, count(*) FROM pg_statistic WHERE starelid IN ( + 'concur_exprs_index_expr'::regclass, + 'concur_exprs_index_pred'::regclass, + 'concur_exprs_index_pred_2'::regclass) + GROUP BY starelid ORDER BY starelid::regclass::text; + starelid | count +-------------------------+------- + concur_exprs_index_expr | 1 +(1 row) + SELECT pg_get_indexdef('concur_exprs_index_expr'::regclass); pg_get_indexdef --------------------------------------------------------------------------------------------------------------- @@ -2608,6 +2619,17 @@ SELECT pg_get_indexdef('concur_exprs_index_pred_2'::regclass); CREATE UNIQUE INDEX concur_exprs_index_pred_2 ON public.concur_exprs_tab USING btree (((1 / c1))) WHERE ('-H'::text >= (c2 COLLATE "C")) (1 row) +-- Statistics should remain intact. +SELECT starelid::regclass, count(*) FROM pg_statistic WHERE starelid IN ( + 'concur_exprs_index_expr'::regclass, + 'concur_exprs_index_pred'::regclass, + 'concur_exprs_index_pred_2'::regclass) + GROUP BY starelid ORDER BY starelid::regclass::text; + starelid | count +-------------------------+------- + concur_exprs_index_expr | 1 +(1 row) + DROP TABLE concur_exprs_tab; -- Temporary tables and on-commit actions, where CONCURRENTLY is ignored. -- ON COMMIT PRESERVE ROWS, the default. diff --git a/src/test/regress/sql/create_index.sql b/src/test/regress/sql/create_index.sql index 37f7259da9e6..4e45b186131b 100644 --- a/src/test/regress/sql/create_index.sql +++ b/src/test/regress/sql/create_index.sql @@ -1079,6 +1079,12 @@ CREATE UNIQUE INDEX concur_exprs_index_pred ON concur_exprs_tab (c1) CREATE UNIQUE INDEX concur_exprs_index_pred_2 ON concur_exprs_tab ((1 / c1)) WHERE ('-H') >= (c2::TEXT) COLLATE "C"; +ANALYZE concur_exprs_tab; +SELECT starelid::regclass, count(*) FROM pg_statistic WHERE starelid IN ( + 'concur_exprs_index_expr'::regclass, + 'concur_exprs_index_pred'::regclass, + 'concur_exprs_index_pred_2'::regclass) + GROUP BY starelid ORDER BY starelid::regclass::text; SELECT pg_get_indexdef('concur_exprs_index_expr'::regclass); SELECT pg_get_indexdef('concur_exprs_index_pred'::regclass); SELECT pg_get_indexdef('concur_exprs_index_pred_2'::regclass); @@ -1091,6 +1097,12 @@ ALTER TABLE concur_exprs_tab ALTER c2 TYPE TEXT; SELECT pg_get_indexdef('concur_exprs_index_expr'::regclass); SELECT pg_get_indexdef('concur_exprs_index_pred'::regclass); SELECT pg_get_indexdef('concur_exprs_index_pred_2'::regclass); +-- Statistics should remain intact. +SELECT starelid::regclass, count(*) FROM pg_statistic WHERE starelid IN ( + 'concur_exprs_index_expr'::regclass, + 'concur_exprs_index_pred'::regclass, + 'concur_exprs_index_pred_2'::regclass) + GROUP BY starelid ORDER BY starelid::regclass::text; DROP TABLE concur_exprs_tab; -- Temporary tables and on-commit actions, where CONCURRENTLY is ignored. From 7f4235032f0d75ea1ad29b192d57fee3d8fe533e Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 1 Nov 2020 11:26:16 -0500 Subject: [PATCH 410/589] Avoid null pointer dereference if error result lacks SQLSTATE. Although error results received from the backend should always have a SQLSTATE field, ones generated by libpq won't, making this code vulnerable to a crash after, say, untimely loss of connection. Noted by Coverity. Oversight in commit 403a3d91c. Back-patch to 9.5, as that was. --- src/bin/pg_dump/pg_backup_db.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/bin/pg_dump/pg_backup_db.c b/src/bin/pg_dump/pg_backup_db.c index 28b2892538ae..6a1e1d3acb26 100644 --- a/src/bin/pg_dump/pg_backup_db.c +++ b/src/bin/pg_dump/pg_backup_db.c @@ -540,9 +540,9 @@ bool IsLockTableGeneric(Archive *AHX) { ArchiveHandle *AH = (ArchiveHandle *) AHX; - PGresult *res; - char *sqlstate; - bool retval; + PGresult *res; + char *sqlstate; + bool retval; if (AHX->remoteVersion >= 140000) return true; @@ -569,13 +569,15 @@ IsLockTableGeneric(Archive *AHX) break; case PGRES_FATAL_ERROR: sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE); - if (strcmp(sqlstate, ERRCODE_WRONG_OBJECT_TYPE) == 0) + if (sqlstate && + strcmp(sqlstate, ERRCODE_WRONG_OBJECT_TYPE) == 0) { retval = false; break; } - else if (strcmp(sqlstate, ERRCODE_LOCK_NOT_AVAILABLE) == 0 || - strcmp(sqlstate, ERRCODE_INSUFFICIENT_PRIVILEGE) == 0) + else if (sqlstate && + (strcmp(sqlstate, ERRCODE_LOCK_NOT_AVAILABLE) == 0 || + strcmp(sqlstate, ERRCODE_INSUFFICIENT_PRIVILEGE) == 0)) { retval = true; break; From dfc797730fc7a07c0e6bd636ad1a564aecab3161 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 1 Nov 2020 18:38:42 -0500 Subject: [PATCH 411/589] Fix two issues in TOAST decompression. pglz_maximum_compressed_size() potentially underestimated the amount of compressed data required to produce N bytes of decompressed data; this is a fault in commit 11a078cf8. Separately from that, pglz_decompress() failed to protect itself against corrupt compressed data, particularly off == 0 in a match tag. Commit c60e520f6 turned such a situation into an infinite loop, where before it'd just have resulted in garbage output. The combination of these two bugs seems like it may explain bug #16694 from Tom Vijlbrief, though it's impossible to be quite sure without direct inspection of the failing session. (One needs to assume that the pglz_maximum_compressed_size() bug caused us to fail to fetch the second byte of a match tag, and what happened to be there instead was a zero. The reported infinite loop is hard to explain without off == 0, though.) Aside from fixing the bugs, rewrite associated comments for more clarity. Back-patch to v13 where both these commits landed. Discussion: https://postgr.es/m/16694-f107871e499ec114@postgresql.org --- src/common/pg_lzcompress.c | 101 ++++++++++++++++++++++++------------- 1 file changed, 66 insertions(+), 35 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index d24d4803a983..79747767ce08 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -710,7 +710,6 @@ pglz_decompress(const char *source, int32 slen, char *dest, for (ctrlc = 0; ctrlc < 8 && sp < srcend && dp < destend; ctrlc++) { - if (ctrl & 1) { /* @@ -732,41 +731,62 @@ pglz_decompress(const char *source, int32 slen, char *dest, len += *sp++; /* - * Now we copy the bytes specified by the tag from OUTPUT to - * OUTPUT (copy len bytes from dp - off to dp). The copied - * areas could overlap, to preven possible uncertainty, we - * copy only non-overlapping regions. + * Check for corrupt data: if we fell off the end of the + * source, or if we obtained off = 0, we have problems. (We + * must check this, else we risk an infinite loop below in the + * face of corrupt data.) + */ + if (sp > srcend || off == 0) + break; + + /* + * Don't emit more data than requested. */ len = Min(len, destend - dp); + + /* + * Now we copy the bytes specified by the tag from OUTPUT to + * OUTPUT (copy len bytes from dp - off to dp). The copied + * areas could overlap, so to avoid undefined behavior in + * memcpy(), be careful to copy only non-overlapping regions. + * + * Note that we cannot use memmove() instead, since while its + * behavior is well-defined, it's also not what we want. + */ while (off < len) { - /*--------- - * When offset is smaller than length - source and - * destination regions overlap. memmove() is resolving - * this overlap in an incompatible way with pglz. Thus we - * resort to memcpy()-ing non-overlapping regions. - * - * Consider input: 112341234123412341234 - * At byte 5 here ^ we have match with length 16 and - * offset 4. 11234M(len=16, off=4) - * We are decoding first period of match and rewrite match - * 112341234M(len=12, off=8) - * - * The same match is now at position 9, it points to the - * same start byte of output, but from another position: - * the offset is doubled. - * - * We iterate through this offset growth until we can - * proceed to usual memcpy(). If we would try to decode - * the match at byte 5 (len=16, off=4) by memmove() we - * would issue memmove(5, 1, 16) which would produce - * 112341234XXXXXXXXXXXX, where series of X is 12 - * undefined bytes, that were at bytes [5:17]. - *--------- + /* + * We can safely copy "off" bytes since that clearly + * results in non-overlapping source and destination. */ memcpy(dp, dp - off, off); len -= off; dp += off; + + /*---------- + * This bit is less obvious: we can double "off" after + * each such step. Consider this raw input: + * 112341234123412341234 + * This will be encoded as 5 literal bytes "11234" and + * then a match tag with length 16 and offset 4. After + * memcpy'ing the first 4 bytes, we will have emitted + * 112341234 + * so we can double "off" to 8, then after the next step + * we have emitted + * 11234123412341234 + * Then we can double "off" again, after which it is more + * than the remaining "len" so we fall out of this loop + * and finish with a non-overlapping copy of the + * remainder. In general, a match tag with off < len + * implies that the decoded data has a repeat length of + * "off". We can handle 1, 2, 4, etc repetitions of the + * repeated string per memcpy until we get to a situation + * where the final copy step is non-overlapping. + * + * (Another way to understand this is that we are keeping + * the copy source point dp - off the same throughout.) + *---------- + */ off += off; } memcpy(dp, dp - off, len); @@ -820,21 +840,32 @@ pglz_decompress(const char *source, int32 slen, char *dest, int32 pglz_maximum_compressed_size(int32 rawsize, int32 total_compressed_size) { - int32 compressed_size; + int64 compressed_size; /* - * pglz uses one control bit per byte, so we need (rawsize * 9) bits. We - * care about bytes though, so we add 7 to make sure we include the last - * incomplete byte (integer division rounds down). + * pglz uses one control bit per byte, so if the entire desired prefix is + * represented as literal bytes, we'll need (rawsize * 9) bits. We care + * about bytes though, so be sure to round up not down. * - * XXX Use int64 to prevent overflow during calculation. + * Use int64 here to prevent overflow during calculation. + */ + compressed_size = ((int64) rawsize * 9 + 7) / 8; + + /* + * The above fails to account for a corner case: we could have compressed + * data that starts with N-1 or N-2 literal bytes and then has a match tag + * of 2 or 3 bytes. It's therefore possible that we need to fetch 1 or 2 + * more bytes in order to have the whole match tag. (Match tags earlier + * in the compressed data don't cause a problem, since they should + * represent more decompressed bytes than they occupy themselves.) */ - compressed_size = (int32) ((int64) rawsize * 9 + 7) / 8; + compressed_size += 2; /* * Maximum compressed size can't be larger than total compressed size. + * (This also ensures that our result fits in int32.) */ compressed_size = Min(compressed_size, total_compressed_size); - return compressed_size; + return (int32) compressed_size; } From a929e17e5a8c9b751b66002c8a89fdebdacfe194 Mon Sep 17 00:00:00 2001 From: David Rowley Date: Mon, 2 Nov 2020 13:46:56 +1300 Subject: [PATCH 412/589] Allow run-time pruning on nested Append/MergeAppend nodes Previously we only tagged on the required information to allow the executor to perform run-time partition pruning for Append/MergeAppend nodes belonging to base relations. It was thought that nested Append/MergeAppend nodes were just about always pulled up into the top-level Append/MergeAppend and that making the run-time pruning info for any sub Append/MergeAppend nodes was a waste of time. However, that was likely badly thought through. Some examples of cases we're unable to pullup nested Append/MergeAppends are: 1) Parallel Append nodes with a mix of parallel and non-parallel paths into a Parallel Append. 2) When planning an ordered Append scan a sub-partition which is unordered may require a nested MergeAppend path to ensure sub-partitions don't mix up the order of tuples being fed into the top-level Append. Unfortunately, it was not just as simple as removing the lines in createplan.c which were purposefully not building the run-time pruning info for anything but RELOPT_BASEREL relations. The code in add_paths_to_append_rel() was far too sloppy about which partitioned_rels it included for the Append/MergeAppend paths. The original code there would always assume accumulate_append_subpath() would pull each sub-Append and sub-MergeAppend path into the top-level path. While it does not appear that there were any actual bugs caused by having the additional partitioned table RT indexes recorded, what it did mean is that later in planning, when we built the run-time pruning info that we wasted effort and built PartitionedRelPruneInfos for partitioned tables that we had no subpaths for the executor to run-time prune. Here we tighten that up so that partitioned_rels only ever contains the RT index for partitioned tables which actually have subpaths in the given Append/MergeAppend. We can now Assert that every PartitionedRelPruneInfo has a non-empty present_parts. That should allow us to catch any weird corner cases that have been missed. In passing, it seems there is no longer a good reason to have the AppendPath and MergeAppendPath's partitioned_rel fields a List of IntList. We can simply have a List of Relids instead. This is more compact in memory and faster to add new members to. We still know which is the root level partition as these always have a lower relid than their children. Previously this field was used for more things, but run-time partition pruning now remains the only user of it and it has no need for a List of IntLists. Here we also get rid of the RelOptInfo partitioned_child_rels field. This is what was previously used to (sometimes incorrectly) set the Append/MergeAppend path's partitioned_rels field. That was the only usage of that field, so we can happily just remove it. I also couldn't resist changing some nearby code to make use of the newly added for_each_from macro so we can skip the first element in the list without checking if the current item was the first one on each iteration. A bug report from Andreas Kretschmer prompted all this work, however, after some consideration, I'm not personally classing this as a bug fix. So no backpatch. In Andreas' test case, it just wasn't that clear that there was a nested Append since the top-level Append just had a single sub-path which was pulled up a level, per 8edd0e794. Author: David Rowley Reviewed-by: Amit Langote Discussion: https://postgr.es/m/flat/CAApHDvqSchs%2BubdybcfFaSPB%2B%2BEA7kqMaoqajtP0GtZvzOOR3g%40mail.gmail.com --- src/backend/nodes/outfuncs.c | 1 - src/backend/optimizer/path/allpaths.c | 234 +++++++++++------- src/backend/optimizer/plan/createplan.c | 2 - src/backend/optimizer/util/relnode.c | 3 - src/backend/partitioning/partprune.c | 29 ++- src/include/nodes/pathnodes.h | 14 +- src/test/regress/expected/partition_prune.out | 102 ++++++++ src/test/regress/sql/partition_prune.sql | 49 ++++ 8 files changed, 319 insertions(+), 115 deletions(-) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 530328af4335..7e324c12e29e 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2310,7 +2310,6 @@ _outRelOptInfo(StringInfo str, const RelOptInfo *node) WRITE_BITMAPSET_FIELD(top_parent_relids); WRITE_BOOL_FIELD(partbounds_merged); WRITE_BITMAPSET_FIELD(all_partrels); - WRITE_NODE_FIELD(partitioned_child_rels); } static void diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index b399592ff815..803d9bae7f11 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -104,8 +104,13 @@ static void generate_orderedappend_paths(PlannerInfo *root, RelOptInfo *rel, static Path *get_cheapest_parameterized_child_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer); +static List *accumulate_partitioned_rels(List *partitioned_rels, + List *sub_partitioned_rels, + bool flatten_partitioned_rels); static void accumulate_append_subpath(Path *path, - List **subpaths, List **special_subpaths); + List **subpaths, List **special_subpaths, + List **partitioned_rels, + bool flatten_partitioned_rels); static Path *get_singleton_append_subpath(Path *path); static void set_dummy_rel_pathlist(RelOptInfo *rel); static void set_subquery_pathlist(PlannerInfo *root, RelOptInfo *rel, @@ -959,17 +964,6 @@ set_append_rel_size(PlannerInfo *root, RelOptInfo *rel, Assert(IS_SIMPLE_REL(rel)); - /* - * Initialize partitioned_child_rels to contain this RT index. - * - * Note that during the set_append_rel_pathlist() phase, we will bubble up - * the indexes of partitioned relations that appear down in the tree, so - * that when we've created Paths for all the children, the root - * partitioned table's list will contain all such indexes. - */ - if (rte->relkind == RELKIND_PARTITIONED_TABLE) - rel->partitioned_child_rels = list_make1_int(rti); - /* * If this is a partitioned baserel, set the consider_partitionwise_join * flag; currently, we only consider partitionwise joins with the baserel @@ -1269,12 +1263,6 @@ set_append_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, if (IS_DUMMY_REL(childrel)) continue; - /* Bubble up childrel's partitioned children. */ - if (rel->part_scheme) - rel->partitioned_child_rels = - list_concat(rel->partitioned_child_rels, - childrel->partitioned_child_rels); - /* * Child is live, so add it to the live_childrels list for use below. */ @@ -1312,56 +1300,35 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, List *all_child_outers = NIL; ListCell *l; List *partitioned_rels = NIL; + List *partial_partitioned_rels = NIL; + List *pa_partitioned_rels = NIL; double partial_rows = -1; + bool flatten_partitioned_rels; /* If appropriate, consider parallel append */ pa_subpaths_valid = enable_parallel_append && rel->consider_parallel; + /* What we do with the partitioned_rels list is different for UNION ALL */ + flatten_partitioned_rels = (rel->rtekind != RTE_SUBQUERY); + /* - * AppendPath generated for partitioned tables must record the RT indexes - * of partitioned tables that are direct or indirect children of this - * Append rel. - * - * AppendPath may be for a sub-query RTE (UNION ALL), in which case, 'rel' - * itself does not represent a partitioned relation, but the child sub- - * queries may contain references to partitioned relations. The loop - * below will look for such children and collect them in a list to be - * passed to the path creation function. (This assumes that we don't need - * to look through multiple levels of subquery RTEs; if we ever do, we - * could consider stuffing the list we generate here into sub-query RTE's - * RelOptInfo, just like we do for partitioned rels, which would be used - * when populating our parent rel with paths. For the present, that - * appears to be unnecessary.) + * For partitioned tables, we accumulate a list of Relids of each + * partitioned table which has at least one of its subpartitions directly + * present as a subpath in this Append. This is used later for run-time + * partition pruning. We must maintain separate lists for each Append + * Path that we create as some paths that we create here can't flatten + * sub-Appends and sub-MergeAppends into the top-level Append. We needn't + * bother doing this for join rels as no run-time pruning is done on + * those. */ - if (rel->part_scheme != NULL) + if (rel->reloptkind != RELOPT_JOINREL && rel->part_scheme != NULL) { - if (IS_SIMPLE_REL(rel)) - partitioned_rels = list_make1(rel->partitioned_child_rels); - else if (IS_JOIN_REL(rel)) - { - int relid = -1; - List *partrels = NIL; - - /* - * For a partitioned joinrel, concatenate the component rels' - * partitioned_child_rels lists. - */ - while ((relid = bms_next_member(rel->relids, relid)) >= 0) - { - RelOptInfo *component; - - Assert(relid >= 1 && relid < root->simple_rel_array_size); - component = root->simple_rel_array[relid]; - Assert(component->part_scheme != NULL); - Assert(list_length(component->partitioned_child_rels) >= 1); - partrels = list_concat(partrels, - component->partitioned_child_rels); - } + partitioned_rels = list_make1(bms_make_singleton(rel->relid)); + partial_partitioned_rels = list_make1(bms_make_singleton(rel->relid)); - partitioned_rels = list_make1(partrels); - } - - Assert(list_length(partitioned_rels) >= 1); + /* skip this one if we're not going to make a Parallel Append path */ + if (pa_subpaths_valid) + pa_partitioned_rels = list_make1(bms_make_singleton(rel->relid)); } /* @@ -1375,14 +1342,6 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, ListCell *lcp; Path *cheapest_partial_path = NULL; - /* - * For UNION ALLs with non-empty partitioned_child_rels, accumulate - * the Lists of child relations. - */ - if (rel->rtekind == RTE_SUBQUERY && childrel->partitioned_child_rels != NIL) - partitioned_rels = lappend(partitioned_rels, - childrel->partitioned_child_rels); - /* * If child has an unparameterized cheapest-total path, add that to * the unparameterized Append path we are constructing for the parent. @@ -1394,7 +1353,8 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, if (childrel->pathlist != NIL && childrel->cheapest_total_path->param_info == NULL) accumulate_append_subpath(childrel->cheapest_total_path, - &subpaths, NULL); + &subpaths, NULL, &partitioned_rels, + flatten_partitioned_rels); else subpaths_valid = false; @@ -1403,7 +1363,9 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, { cheapest_partial_path = linitial(childrel->partial_pathlist); accumulate_append_subpath(cheapest_partial_path, - &partial_subpaths, NULL); + &partial_subpaths, NULL, + &partial_partitioned_rels, + flatten_partitioned_rels); } else partial_subpaths_valid = false; @@ -1432,7 +1394,9 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, Assert(cheapest_partial_path != NULL); accumulate_append_subpath(cheapest_partial_path, &pa_partial_subpaths, - &pa_nonpartial_subpaths); + &pa_nonpartial_subpaths, + &pa_partitioned_rels, + flatten_partitioned_rels); } else @@ -1452,7 +1416,9 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, */ accumulate_append_subpath(nppath, &pa_nonpartial_subpaths, - NULL); + NULL, + &pa_partitioned_rels, + flatten_partitioned_rels); } } @@ -1572,7 +1538,7 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, appendpath = create_append_path(root, rel, NIL, partial_subpaths, NIL, NULL, parallel_workers, enable_parallel_append, - partitioned_rels, -1); + partial_partitioned_rels, -1); /* * Make sure any subsequent partial paths use the same row count @@ -1621,7 +1587,7 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, appendpath = create_append_path(root, rel, pa_nonpartial_subpaths, pa_partial_subpaths, NIL, NULL, parallel_workers, true, - partitioned_rels, partial_rows); + pa_partitioned_rels, partial_rows); add_partial_path(rel, (Path *) appendpath); } @@ -1651,6 +1617,10 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, { Relids required_outer = (Relids) lfirst(l); ListCell *lcr; + List *part_rels = NIL; + + if (rel->reloptkind != RELOPT_JOINREL && rel->part_scheme != NULL) + part_rels = list_make1(bms_make_singleton(rel->relid)); /* Select the child paths for an Append with this parameterization */ subpaths = NIL; @@ -1676,14 +1646,15 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, subpaths_valid = false; break; } - accumulate_append_subpath(subpath, &subpaths, NULL); + accumulate_append_subpath(subpath, &subpaths, NULL, &part_rels, + flatten_partitioned_rels); } if (subpaths_valid) add_path(rel, (Path *) create_append_path(root, rel, subpaths, NIL, NIL, required_outer, 0, false, - partitioned_rels, -1)); + part_rels, -1)); } /* @@ -1697,17 +1668,14 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, { RelOptInfo *childrel = (RelOptInfo *) linitial(live_childrels); - foreach(l, childrel->partial_pathlist) + /* skip the cheapest partial path, since we already used that above */ + for_each_from(l, childrel->partial_pathlist, 1) { Path *path = (Path *) lfirst(l); AppendPath *appendpath; - /* - * Skip paths with no pathkeys. Also skip the cheapest partial - * path, since we already used that above. - */ - if (path->pathkeys == NIL || - path == linitial(childrel->partial_pathlist)) + /* skip paths with no pathkeys. */ + if (path->pathkeys == NIL) continue; appendpath = create_append_path(root, rel, NIL, list_make1(path), @@ -1757,6 +1725,18 @@ generate_orderedappend_paths(PlannerInfo *root, RelOptInfo *rel, List *partition_pathkeys_desc = NIL; bool partition_pathkeys_partial = true; bool partition_pathkeys_desc_partial = true; + List *startup_partitioned_rels = NIL; + List *total_partitioned_rels = NIL; + bool flatten_partitioned_rels; + + /* Set up the method for building the partitioned rels lists */ + flatten_partitioned_rels = (rel->rtekind != RTE_SUBQUERY); + + if (rel->reloptkind != RELOPT_JOINREL && rel->part_scheme != NULL) + { + startup_partitioned_rels = list_make1(bms_make_singleton(rel->relid)); + total_partitioned_rels = list_make1(bms_make_singleton(rel->relid)); + } /* * Some partitioned table setups may allow us to use an Append node @@ -1898,9 +1878,13 @@ generate_orderedappend_paths(PlannerInfo *root, RelOptInfo *rel, * child paths for the MergeAppend. */ accumulate_append_subpath(cheapest_startup, - &startup_subpaths, NULL); + &startup_subpaths, NULL, + &startup_partitioned_rels, + flatten_partitioned_rels); accumulate_append_subpath(cheapest_total, - &total_subpaths, NULL); + &total_subpaths, NULL, + &total_partitioned_rels, + flatten_partitioned_rels); } } @@ -1916,7 +1900,7 @@ generate_orderedappend_paths(PlannerInfo *root, RelOptInfo *rel, NULL, 0, false, - partitioned_rels, + startup_partitioned_rels, -1)); if (startup_neq_total) add_path(rel, (Path *) create_append_path(root, @@ -1927,7 +1911,7 @@ generate_orderedappend_paths(PlannerInfo *root, RelOptInfo *rel, NULL, 0, false, - partitioned_rels, + total_partitioned_rels, -1)); } else @@ -1938,14 +1922,14 @@ generate_orderedappend_paths(PlannerInfo *root, RelOptInfo *rel, startup_subpaths, pathkeys, NULL, - partitioned_rels)); + startup_partitioned_rels)); if (startup_neq_total) add_path(rel, (Path *) create_merge_append_path(root, rel, total_subpaths, pathkeys, NULL, - partitioned_rels)); + total_partitioned_rels)); } } } @@ -2024,6 +2008,54 @@ get_cheapest_parameterized_child_path(PlannerInfo *root, RelOptInfo *rel, return cheapest; } +/* + * accumulate_partitioned_rels + * Record 'sub_partitioned_rels' in the 'partitioned_rels' list, + * flattening as appropriate. + */ +static List * +accumulate_partitioned_rels(List *partitioned_rels, + List *sub_partitioned_rels, + bool flatten) +{ + if (flatten) + { + /* + * We're only called with flatten == true when the partitioned_rels + * list has at most 1 element. So we can just add the members from + * sub list's first element onto the first element of + * partitioned_rels. Only later in planning when doing UNION ALL + * Append processing will we see flatten == false. partitioned_rels + * may end up with more than 1 element then, but we never expect to be + * called with flatten == true again after that, so we needn't bother + * doing anything here for anything but the initial element. + */ + if (partitioned_rels != NIL && sub_partitioned_rels != NIL) + { + Relids partrels = (Relids) linitial(partitioned_rels); + Relids subpartrels = (Relids) linitial(sub_partitioned_rels); + + /* Ensure the above comment holds true */ + Assert(list_length(partitioned_rels) == 1); + Assert(list_length(sub_partitioned_rels) == 1); + + linitial(partitioned_rels) = bms_add_members(partrels, subpartrels); + } + } + else + { + /* + * Handle UNION ALL to partitioned tables. This always occurs after + * we've done the accumulation for sub-partitioned tables, so there's + * no need to consider how adding multiple elements to the top level + * list affects the flatten == true case above. + */ + partitioned_rels = list_concat(partitioned_rels, sub_partitioned_rels); + } + + return partitioned_rels; +} + /* * accumulate_append_subpath * Add a subpath to the list being built for an Append or MergeAppend. @@ -2044,9 +2076,24 @@ get_cheapest_parameterized_child_path(PlannerInfo *root, RelOptInfo *rel, * children to subpaths and the rest to special_subpaths. If the latter is * NULL, we don't flatten the path at all (unless it contains only partial * paths). + * + * When pulling up sub-Appends and sub-Merge Appends, we also gather the + * path's list of partitioned tables and store in 'partitioned_rels'. The + * exact behavior here depends on the value of 'flatten_partitioned_rels'. + * + * When 'flatten_partitioned_rels' is true, 'partitioned_rels' will contain at + * most one element which is a Relids of the partitioned relations which there + * are subpaths for. In this case, we just add the RT indexes for the + * partitioned tables for the subpath we're pulling up to the single entry in + * 'partitioned_rels'. When 'flatten_partitioned_rels' is false we + * concatenate the path's partitioned rel list onto the top-level list. This + * done for UNION ALLs which could have a partitioned table in each union + * branch. */ static void -accumulate_append_subpath(Path *path, List **subpaths, List **special_subpaths) +accumulate_append_subpath(Path *path, List **subpaths, List **special_subpaths, + List **partitioned_rels, + bool flatten_partitioned_rels) { if (IsA(path, AppendPath)) { @@ -2055,6 +2102,9 @@ accumulate_append_subpath(Path *path, List **subpaths, List **special_subpaths) if (!apath->path.parallel_aware || apath->first_partial_path == 0) { *subpaths = list_concat(*subpaths, apath->subpaths); + *partitioned_rels = accumulate_partitioned_rels(*partitioned_rels, + apath->partitioned_rels, + flatten_partitioned_rels); return; } else if (special_subpaths != NULL) @@ -2070,6 +2120,9 @@ accumulate_append_subpath(Path *path, List **subpaths, List **special_subpaths) apath->first_partial_path); *special_subpaths = list_concat(*special_subpaths, new_special_subpaths); + *partitioned_rels = accumulate_partitioned_rels(*partitioned_rels, + apath->partitioned_rels, + flatten_partitioned_rels); return; } } @@ -2078,6 +2131,9 @@ accumulate_append_subpath(Path *path, List **subpaths, List **special_subpaths) MergeAppendPath *mpath = (MergeAppendPath *) path; *subpaths = list_concat(*subpaths, mpath->subpaths); + *partitioned_rels = accumulate_partitioned_rels(*partitioned_rels, + mpath->partitioned_rels, + flatten_partitioned_rels); return; } diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 94280a730c4d..40abe6f9f623 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -1228,7 +1228,6 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags) * do partition pruning. */ if (enable_partition_pruning && - rel->reloptkind == RELOPT_BASEREL && best_path->partitioned_rels != NIL) { List *prunequal; @@ -1395,7 +1394,6 @@ create_merge_append_plan(PlannerInfo *root, MergeAppendPath *best_path, * do partition pruning. */ if (enable_partition_pruning && - rel->reloptkind == RELOPT_BASEREL && best_path->partitioned_rels != NIL) { List *prunequal; diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index a203e6f1ff53..76245c1ff3a1 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -257,7 +257,6 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptInfo *parent) rel->all_partrels = NULL; rel->partexprs = NULL; rel->nullable_partexprs = NULL; - rel->partitioned_child_rels = NIL; /* * Pass assorted information down the inheritance hierarchy. @@ -672,7 +671,6 @@ build_join_rel(PlannerInfo *root, joinrel->all_partrels = NULL; joinrel->partexprs = NULL; joinrel->nullable_partexprs = NULL; - joinrel->partitioned_child_rels = NIL; /* Compute information relevant to the foreign relations. */ set_foreign_rel_properties(joinrel, outer_rel, inner_rel); @@ -850,7 +848,6 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, joinrel->all_partrels = NULL; joinrel->partexprs = NULL; joinrel->nullable_partexprs = NULL; - joinrel->partitioned_child_rels = NIL; joinrel->top_parent_relids = bms_union(outer_rel->top_parent_relids, inner_rel->top_parent_relids); diff --git a/src/backend/partitioning/partprune.c b/src/backend/partitioning/partprune.c index 6268623d5699..8e1187e31f55 100644 --- a/src/backend/partitioning/partprune.c +++ b/src/backend/partitioning/partprune.c @@ -141,7 +141,7 @@ typedef struct PruneStepResult static List *make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, int *relid_subplan_map, - List *partitioned_rels, List *prunequal, + Relids partrelids, List *prunequal, Bitmapset **matchedsubplans); static void gen_partprune_steps(RelOptInfo *rel, List *clauses, PartClauseTarget target, @@ -267,13 +267,13 @@ make_partition_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, prunerelinfos = NIL; foreach(lc, partitioned_rels) { - List *rels = (List *) lfirst(lc); + Relids partrelids = (Relids) lfirst(lc); List *pinfolist; Bitmapset *matchedsubplans = NULL; pinfolist = make_partitionedrel_pruneinfo(root, parentrel, relid_subplan_map, - rels, prunequal, + partrelids, prunequal, &matchedsubplans); /* When pruning is possible, record the matched subplans */ @@ -342,7 +342,7 @@ make_partition_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, static List * make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, int *relid_subplan_map, - List *partitioned_rels, List *prunequal, + Relids partrelids, List *prunequal, Bitmapset **matchedsubplans) { RelOptInfo *targetpart = NULL; @@ -351,6 +351,7 @@ make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, int *relid_subpart_map; Bitmapset *subplansfound = NULL; ListCell *lc; + int rti; int i; /* @@ -364,9 +365,9 @@ make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, relid_subpart_map = palloc0(sizeof(int) * root->simple_rel_array_size); i = 1; - foreach(lc, partitioned_rels) + rti = -1; + while ((rti = bms_next_member(partrelids, rti)) > 0) { - Index rti = lfirst_int(lc); RelOptInfo *subpart = find_base_rel(root, rti); PartitionedRelPruneInfo *pinfo; List *partprunequal; @@ -379,14 +380,11 @@ make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, * Fill the mapping array. * * relid_subpart_map maps relid of a non-leaf partition to the index - * in 'partitioned_rels' of that rel (which will also be the index in - * the returned PartitionedRelPruneInfo list of the info for that - * partition). We use 1-based indexes here, so that zero can - * represent an un-filled array entry. + * in the returned PartitionedRelPruneInfo list of the info for that + * partition. We use 1-based indexes here, so that zero can represent + * an un-filled array entry. */ Assert(rti < root->simple_rel_array_size); - /* No duplicates please */ - Assert(relid_subpart_map[rti] == 0); relid_subpart_map[rti] = i++; /* @@ -582,6 +580,13 @@ make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, present_parts = bms_add_member(present_parts, i); } + /* + * Ensure there were no stray PartitionedRelPruneInfo generated for + * partitioned tables that we have no sub-paths or + * sub-PartitionedRelPruneInfo for. + */ + Assert(!bms_is_empty(present_parts)); + /* Record the maps and other information. */ pinfo->present_parts = present_parts; pinfo->nparts = nparts; diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 45cbf6045a38..8f62d6170289 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -601,9 +601,6 @@ typedef struct PartitionSchemeData *PartitionScheme; * part_rels - RelOptInfos for each partition * all_partrels - Relids set of all partition relids * partexprs, nullable_partexprs - Partition key expressions - * partitioned_child_rels - RT indexes of unpruned partitions of - * this relation that are partitioned tables - * themselves, in hierarchical order * * The partexprs and nullable_partexprs arrays each contain * part_scheme->partnatts elements. Each of the elements is a list of @@ -751,7 +748,6 @@ typedef struct RelOptInfo Relids all_partrels; /* Relids set of all partition relids */ List **partexprs; /* Non-nullable partition key expressions */ List **nullable_partexprs; /* Nullable partition key expressions */ - List *partitioned_child_rels; /* List of RT indexes */ } RelOptInfo; /* @@ -1401,8 +1397,9 @@ typedef struct CustomPath typedef struct AppendPath { Path path; - /* RT indexes of non-leaf tables in a partition tree */ - List *partitioned_rels; + List *partitioned_rels; /* List of Relids containing RT indexes of + * non-leaf tables for each partition + * hierarchy whose paths are in 'subpaths' */ List *subpaths; /* list of component Paths */ /* Index of first partial path in subpaths; list_length(subpaths) if none */ int first_partial_path; @@ -1427,8 +1424,9 @@ extern bool is_dummy_rel(RelOptInfo *rel); typedef struct MergeAppendPath { Path path; - /* RT indexes of non-leaf tables in a partition tree */ - List *partitioned_rels; + List *partitioned_rels; /* List of Relids containing RT indexes of + * non-leaf tables for each partition + * hierarchy whose paths are in 'subpaths' */ List *subpaths; /* list of component Paths */ double limit_tuples; /* hard limit on output tuples, or -1 */ } MergeAppendPath; diff --git a/src/test/regress/expected/partition_prune.out b/src/test/regress/expected/partition_prune.out index 50d2a7e4b975..80e71b8e2b9c 100644 --- a/src/test/regress/expected/partition_prune.out +++ b/src/test/regress/expected/partition_prune.out @@ -3671,6 +3671,108 @@ explain (costs off) update listp1 set a = 1 where a = 2; reset constraint_exclusion; reset enable_partition_pruning; drop table listp; +-- Ensure run-time pruning works correctly for nested Append nodes +set parallel_setup_cost to 0; +set parallel_tuple_cost to 0; +create table listp (a int) partition by list(a); +create table listp_12 partition of listp for values in(1,2) partition by list(a); +create table listp_12_1 partition of listp_12 for values in(1); +create table listp_12_2 partition of listp_12 for values in(2); +-- Force the 2nd subnode of the Append to be non-parallel. This results in +-- a nested Append node because the mixed parallel / non-parallel paths cannot +-- be pulled into the top-level Append. +alter table listp_12_1 set (parallel_workers = 0); +-- Ensure that listp_12_2 is not scanned. (The nested Append is not seen in +-- the plan as it's pulled in setref.c due to having just a single subnode). +explain (analyze on, costs off, timing off, summary off) +select * from listp where a = (select 1); + QUERY PLAN +---------------------------------------------------------------------- + Gather (actual rows=0 loops=1) + Workers Planned: 2 + Params Evaluated: $0 + Workers Launched: 2 + InitPlan 1 (returns $0) + -> Result (actual rows=1 loops=1) + -> Parallel Append (actual rows=0 loops=3) + -> Seq Scan on listp_12_1 listp_1 (actual rows=0 loops=1) + Filter: (a = $0) + -> Parallel Seq Scan on listp_12_2 listp_2 (never executed) + Filter: (a = $0) +(11 rows) + +-- Like the above but throw some more complexity at the planner by adding +-- a UNION ALL. We expect both sides of the union not to scan the +-- non-required partitions. +explain (analyze on, costs off, timing off, summary off) +select * from listp where a = (select 1) + union all +select * from listp where a = (select 2); + QUERY PLAN +----------------------------------------------------------------------------------- + Append (actual rows=0 loops=1) + -> Gather (actual rows=0 loops=1) + Workers Planned: 2 + Params Evaluated: $0 + Workers Launched: 2 + InitPlan 1 (returns $0) + -> Result (actual rows=1 loops=1) + -> Parallel Append (actual rows=0 loops=3) + -> Seq Scan on listp_12_1 listp_1 (actual rows=0 loops=1) + Filter: (a = $0) + -> Parallel Seq Scan on listp_12_2 listp_2 (never executed) + Filter: (a = $0) + -> Gather (actual rows=0 loops=1) + Workers Planned: 2 + Params Evaluated: $1 + Workers Launched: 2 + InitPlan 2 (returns $1) + -> Result (actual rows=1 loops=1) + -> Parallel Append (actual rows=0 loops=3) + -> Seq Scan on listp_12_1 listp_4 (never executed) + Filter: (a = $1) + -> Parallel Seq Scan on listp_12_2 listp_5 (actual rows=0 loops=1) + Filter: (a = $1) +(23 rows) + +drop table listp; +reset parallel_tuple_cost; +reset parallel_setup_cost; +-- Test case for run-time pruning with a nested Merge Append +set enable_sort to 0; +create table rangep (a int, b int) partition by range (a); +create table rangep_0_to_100 partition of rangep for values from (0) to (100) partition by list (b); +-- We need 3 sub-partitions. 1 to validate pruning worked and another two +-- because a single remaining partition would be pulled up to the main Append. +create table rangep_0_to_100_1 partition of rangep_0_to_100 for values in(1); +create table rangep_0_to_100_2 partition of rangep_0_to_100 for values in(2); +create table rangep_0_to_100_3 partition of rangep_0_to_100 for values in(3); +create table rangep_100_to_200 partition of rangep for values from (100) to (200); +create index on rangep (a); +-- Ensure run-time pruning works on the nested Merge Append +explain (analyze on, costs off, timing off, summary off) +select * from rangep where b IN((select 1),(select 2)) order by a; + QUERY PLAN +------------------------------------------------------------------------------------------------------------ + Append (actual rows=0 loops=1) + InitPlan 1 (returns $0) + -> Result (actual rows=1 loops=1) + InitPlan 2 (returns $1) + -> Result (actual rows=1 loops=1) + -> Merge Append (actual rows=0 loops=1) + Sort Key: rangep_2.a + -> Index Scan using rangep_0_to_100_1_a_idx on rangep_0_to_100_1 rangep_2 (actual rows=0 loops=1) + Filter: (b = ANY (ARRAY[$0, $1])) + -> Index Scan using rangep_0_to_100_2_a_idx on rangep_0_to_100_2 rangep_3 (actual rows=0 loops=1) + Filter: (b = ANY (ARRAY[$0, $1])) + -> Index Scan using rangep_0_to_100_3_a_idx on rangep_0_to_100_3 rangep_4 (never executed) + Filter: (b = ANY (ARRAY[$0, $1])) + -> Index Scan using rangep_100_to_200_a_idx on rangep_100_to_200 rangep_5 (actual rows=0 loops=1) + Filter: (b = ANY (ARRAY[$0, $1])) +(15 rows) + +reset enable_sort; +drop table rangep; -- -- Check that gen_prune_steps_from_opexps() works well for various cases of -- clauses for different partition keys diff --git a/src/test/regress/sql/partition_prune.sql b/src/test/regress/sql/partition_prune.sql index 1e904a8c5b7b..939a9b1193ed 100644 --- a/src/test/regress/sql/partition_prune.sql +++ b/src/test/regress/sql/partition_prune.sql @@ -1051,6 +1051,55 @@ reset enable_partition_pruning; drop table listp; +-- Ensure run-time pruning works correctly for nested Append nodes +set parallel_setup_cost to 0; +set parallel_tuple_cost to 0; + +create table listp (a int) partition by list(a); +create table listp_12 partition of listp for values in(1,2) partition by list(a); +create table listp_12_1 partition of listp_12 for values in(1); +create table listp_12_2 partition of listp_12 for values in(2); + +-- Force the 2nd subnode of the Append to be non-parallel. This results in +-- a nested Append node because the mixed parallel / non-parallel paths cannot +-- be pulled into the top-level Append. +alter table listp_12_1 set (parallel_workers = 0); + +-- Ensure that listp_12_2 is not scanned. (The nested Append is not seen in +-- the plan as it's pulled in setref.c due to having just a single subnode). +explain (analyze on, costs off, timing off, summary off) +select * from listp where a = (select 1); + +-- Like the above but throw some more complexity at the planner by adding +-- a UNION ALL. We expect both sides of the union not to scan the +-- non-required partitions. +explain (analyze on, costs off, timing off, summary off) +select * from listp where a = (select 1) + union all +select * from listp where a = (select 2); + +drop table listp; +reset parallel_tuple_cost; +reset parallel_setup_cost; + +-- Test case for run-time pruning with a nested Merge Append +set enable_sort to 0; +create table rangep (a int, b int) partition by range (a); +create table rangep_0_to_100 partition of rangep for values from (0) to (100) partition by list (b); +-- We need 3 sub-partitions. 1 to validate pruning worked and another two +-- because a single remaining partition would be pulled up to the main Append. +create table rangep_0_to_100_1 partition of rangep_0_to_100 for values in(1); +create table rangep_0_to_100_2 partition of rangep_0_to_100 for values in(2); +create table rangep_0_to_100_3 partition of rangep_0_to_100 for values in(3); +create table rangep_100_to_200 partition of rangep for values from (100) to (200); +create index on rangep (a); + +-- Ensure run-time pruning works on the nested Merge Append +explain (analyze on, costs off, timing off, summary off) +select * from rangep where b IN((select 1),(select 2)) order by a; +reset enable_sort; +drop table rangep; + -- -- Check that gen_prune_steps_from_opexps() works well for various cases of -- clauses for different partition keys From 644f0d7cc9c2cb270746f2024c706554e0fbec82 Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Mon, 2 Nov 2020 08:18:18 +0530 Subject: [PATCH 413/589] Use Enum for top level logical replication message types. Logical replication protocol uses a single byte character to identify a message type in logical replication protocol. The code uses string literals for the same. Use Enum so that 1. All the string literals used can be found at a single place. This makes it easy to add more types without the risk of conflicts. 2. It's easy to locate the code handling a given message type. 3. When used with switch statements, it is easy to identify the missing cases using -Wswitch. Author: Ashutosh Bapat Reviewed-by: Kyotaro Horiguchi, Andres Freund, Peter Smith and Amit Kapila Discussion: https://postgr.es/m/CAExHW5uPzQ7L0oAd_ENyvaiYMOPgkrAoJpE+ZY5-obdcVT6NPg@mail.gmail.com --- src/backend/replication/logical/proto.c | 26 +++---- src/backend/replication/logical/worker.c | 87 ++++++++++++------------ src/include/replication/logicalproto.h | 27 ++++++++ 3 files changed, 83 insertions(+), 57 deletions(-) diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c index eb19142b4865..fdb31182d77f 100644 --- a/src/backend/replication/logical/proto.c +++ b/src/backend/replication/logical/proto.c @@ -44,7 +44,7 @@ static const char *logicalrep_read_namespace(StringInfo in); void logicalrep_write_begin(StringInfo out, ReorderBufferTXN *txn) { - pq_sendbyte(out, 'B'); /* BEGIN */ + pq_sendbyte(out, LOGICAL_REP_MSG_BEGIN); /* fixed fields */ pq_sendint64(out, txn->final_lsn); @@ -76,7 +76,7 @@ logicalrep_write_commit(StringInfo out, ReorderBufferTXN *txn, { uint8 flags = 0; - pq_sendbyte(out, 'C'); /* sending COMMIT */ + pq_sendbyte(out, LOGICAL_REP_MSG_COMMIT); /* send the flags field (unused for now) */ pq_sendbyte(out, flags); @@ -112,7 +112,7 @@ void logicalrep_write_origin(StringInfo out, const char *origin, XLogRecPtr origin_lsn) { - pq_sendbyte(out, 'O'); /* ORIGIN */ + pq_sendbyte(out, LOGICAL_REP_MSG_ORIGIN); /* fixed fields */ pq_sendint64(out, origin_lsn); @@ -141,7 +141,7 @@ void logicalrep_write_insert(StringInfo out, TransactionId xid, Relation rel, HeapTuple newtuple, bool binary) { - pq_sendbyte(out, 'I'); /* action INSERT */ + pq_sendbyte(out, LOGICAL_REP_MSG_INSERT); /* transaction ID (if not valid, we're not streaming) */ if (TransactionIdIsValid(xid)) @@ -185,7 +185,7 @@ void logicalrep_write_update(StringInfo out, TransactionId xid, Relation rel, HeapTuple oldtuple, HeapTuple newtuple, bool binary) { - pq_sendbyte(out, 'U'); /* action UPDATE */ + pq_sendbyte(out, LOGICAL_REP_MSG_UPDATE); Assert(rel->rd_rel->relreplident == REPLICA_IDENTITY_DEFAULT || rel->rd_rel->relreplident == REPLICA_IDENTITY_FULL || @@ -263,7 +263,7 @@ logicalrep_write_delete(StringInfo out, TransactionId xid, Relation rel, rel->rd_rel->relreplident == REPLICA_IDENTITY_FULL || rel->rd_rel->relreplident == REPLICA_IDENTITY_INDEX); - pq_sendbyte(out, 'D'); /* action DELETE */ + pq_sendbyte(out, LOGICAL_REP_MSG_DELETE); /* transaction ID (if not valid, we're not streaming) */ if (TransactionIdIsValid(xid)) @@ -317,7 +317,7 @@ logicalrep_write_truncate(StringInfo out, int i; uint8 flags = 0; - pq_sendbyte(out, 'T'); /* action TRUNCATE */ + pq_sendbyte(out, LOGICAL_REP_MSG_TRUNCATE); /* transaction ID (if not valid, we're not streaming) */ if (TransactionIdIsValid(xid)) @@ -369,7 +369,7 @@ logicalrep_write_rel(StringInfo out, TransactionId xid, Relation rel) { char *relname; - pq_sendbyte(out, 'R'); /* sending RELATION */ + pq_sendbyte(out, LOGICAL_REP_MSG_RELATION); /* transaction ID (if not valid, we're not streaming) */ if (TransactionIdIsValid(xid)) @@ -425,7 +425,7 @@ logicalrep_write_typ(StringInfo out, TransactionId xid, Oid typoid) HeapTuple tup; Form_pg_type typtup; - pq_sendbyte(out, 'Y'); /* sending TYPE */ + pq_sendbyte(out, LOGICAL_REP_MSG_TYPE); /* transaction ID (if not valid, we're not streaming) */ if (TransactionIdIsValid(xid)) @@ -755,7 +755,7 @@ void logicalrep_write_stream_start(StringInfo out, TransactionId xid, bool first_segment) { - pq_sendbyte(out, 'S'); /* action STREAM START */ + pq_sendbyte(out, LOGICAL_REP_MSG_STREAM_START); Assert(TransactionIdIsValid(xid)); @@ -788,7 +788,7 @@ logicalrep_read_stream_start(StringInfo in, bool *first_segment) void logicalrep_write_stream_stop(StringInfo out) { - pq_sendbyte(out, 'E'); /* action STREAM END */ + pq_sendbyte(out, LOGICAL_REP_MSG_STREAM_END); } /* @@ -800,7 +800,7 @@ logicalrep_write_stream_commit(StringInfo out, ReorderBufferTXN *txn, { uint8 flags = 0; - pq_sendbyte(out, 'c'); /* action STREAM COMMIT */ + pq_sendbyte(out, LOGICAL_REP_MSG_STREAM_COMMIT); Assert(TransactionIdIsValid(txn->xid)); @@ -849,7 +849,7 @@ void logicalrep_write_stream_abort(StringInfo out, TransactionId xid, TransactionId subxid) { - pq_sendbyte(out, 'A'); /* action STREAM ABORT */ + pq_sendbyte(out, LOGICAL_REP_MSG_STREAM_ABORT); Assert(TransactionIdIsValid(xid) && TransactionIdIsValid(subxid)); diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index b0f27e0af856..04684912dea3 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -1896,67 +1896,66 @@ apply_handle_truncate(StringInfo s) static void apply_dispatch(StringInfo s) { - char action = pq_getmsgbyte(s); + LogicalRepMsgType action = pq_getmsgbyte(s); switch (action) { - /* BEGIN */ - case 'B': + case LOGICAL_REP_MSG_BEGIN: apply_handle_begin(s); - break; - /* COMMIT */ - case 'C': + return; + + case LOGICAL_REP_MSG_COMMIT: apply_handle_commit(s); - break; - /* INSERT */ - case 'I': + return; + + case LOGICAL_REP_MSG_INSERT: apply_handle_insert(s); - break; - /* UPDATE */ - case 'U': + return; + + case LOGICAL_REP_MSG_UPDATE: apply_handle_update(s); - break; - /* DELETE */ - case 'D': + return; + + case LOGICAL_REP_MSG_DELETE: apply_handle_delete(s); - break; - /* TRUNCATE */ - case 'T': + return; + + case LOGICAL_REP_MSG_TRUNCATE: apply_handle_truncate(s); - break; - /* RELATION */ - case 'R': + return; + + case LOGICAL_REP_MSG_RELATION: apply_handle_relation(s); - break; - /* TYPE */ - case 'Y': + return; + + case LOGICAL_REP_MSG_TYPE: apply_handle_type(s); - break; - /* ORIGIN */ - case 'O': + return; + + case LOGICAL_REP_MSG_ORIGIN: apply_handle_origin(s); - break; - /* STREAM START */ - case 'S': + return; + + case LOGICAL_REP_MSG_STREAM_START: apply_handle_stream_start(s); - break; - /* STREAM END */ - case 'E': + return; + + case LOGICAL_REP_MSG_STREAM_END: apply_handle_stream_stop(s); - break; - /* STREAM ABORT */ - case 'A': + return; + + case LOGICAL_REP_MSG_STREAM_ABORT: apply_handle_stream_abort(s); - break; - /* STREAM COMMIT */ - case 'c': + return; + + case LOGICAL_REP_MSG_STREAM_COMMIT: apply_handle_stream_commit(s); - break; - default: - ereport(ERROR, - (errcode(ERRCODE_PROTOCOL_VIOLATION), - errmsg("invalid logical replication message type \"%c\"", action))); + return; } + + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid logical replication message type \"%c\"", action))); } /* diff --git a/src/include/replication/logicalproto.h b/src/include/replication/logicalproto.h index 0c2cda264e14..cca13dae964c 100644 --- a/src/include/replication/logicalproto.h +++ b/src/include/replication/logicalproto.h @@ -33,6 +33,33 @@ #define LOGICALREP_PROTO_STREAM_VERSION_NUM 2 #define LOGICALREP_PROTO_MAX_VERSION_NUM LOGICALREP_PROTO_STREAM_VERSION_NUM +/* + * Logical message types + * + * Used by logical replication wire protocol. + * + * Note: though this is an enum, the values are used to identify message types + * in logical replication protocol, which uses a single byte to identify a + * message type. Hence the values should be single byte wide and preferrably + * human readable characters. + */ +typedef enum LogicalRepMsgType +{ + LOGICAL_REP_MSG_BEGIN = 'B', + LOGICAL_REP_MSG_COMMIT = 'C', + LOGICAL_REP_MSG_ORIGIN = 'O', + LOGICAL_REP_MSG_INSERT = 'I', + LOGICAL_REP_MSG_UPDATE = 'U', + LOGICAL_REP_MSG_DELETE = 'D', + LOGICAL_REP_MSG_TRUNCATE = 'T', + LOGICAL_REP_MSG_RELATION = 'R', + LOGICAL_REP_MSG_TYPE = 'Y', + LOGICAL_REP_MSG_STREAM_START = 'S', + LOGICAL_REP_MSG_STREAM_END = 'E', + LOGICAL_REP_MSG_STREAM_COMMIT = 'c', + LOGICAL_REP_MSG_STREAM_ABORT = 'A' +} LogicalRepMsgType; + /* * This struct stores a tuple received via logical replication. * Keep in mind that the columns correspond to the *remote* table. From 8a15e735be00f156a7227741c0ce88702e6de099 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 2 Nov 2020 15:14:41 +0900 Subject: [PATCH 414/589] Fix some grammar and typos in comments and docs The documentation fixes are backpatched down to where they apply. Author: Justin Pryzby Discussion: https://postgr.es/m/20201031020801.GD3080@telsasoft.com Backpatch-through: 9.6 --- contrib/amcheck/verify_heapam.c | 4 ++-- contrib/pgcrypto/pgp-compress.c | 2 +- doc/src/sgml/auto-explain.sgml | 2 +- doc/src/sgml/ddl.sgml | 2 +- doc/src/sgml/ref/pg_rewind.sgml | 2 +- src/backend/access/heap/pruneheap.c | 2 +- src/backend/catalog/namespace.c | 2 +- src/backend/catalog/pg_namespace.c | 2 +- src/backend/commands/copy.c | 6 +++--- src/backend/commands/tablecmds.c | 2 +- src/backend/executor/execExpr.c | 2 +- src/backend/executor/nodeIncrementalSort.c | 2 +- src/backend/executor/nodeLimit.c | 2 +- src/backend/optimizer/path/allpaths.c | 2 +- src/backend/optimizer/plan/analyzejoins.c | 2 +- src/backend/partitioning/partbounds.c | 2 +- src/backend/postmaster/interrupt.c | 2 +- src/backend/statistics/dependencies.c | 2 +- src/backend/statistics/extended_stats.c | 2 +- src/backend/storage/buffer/bufmgr.c | 2 +- src/backend/utils/adt/varlena.c | 2 +- src/bin/pg_rewind/parsexlog.c | 2 +- src/bin/pgbench/pgbench.c | 2 +- src/interfaces/libpq/fe-connect.c | 2 +- src/test/modules/dummy_index_am/dummy_index_am.c | 2 +- 25 files changed, 28 insertions(+), 28 deletions(-) diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index 8bb890438aa9..570f44b59ee1 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -560,7 +560,7 @@ verify_heapam_tupdesc(void) * Since we do not hold a snapshot, tuple visibility is not a question of * whether we should be able to see the tuple relative to any particular * snapshot, but rather a question of whether it is safe and reasonable to - * to check the tuple attributes. + * check the tuple attributes. * * Some kinds of corruption make it unsafe to check the tuple attributes, for * example when the line pointer refers to a range of bytes outside the page. @@ -1342,7 +1342,7 @@ fxid_in_cached_range(FullTransactionId fxid, const HeapCheckContext *ctx) } /* - * Checks wheter a multitransaction ID is in the cached valid range, returning + * Checks whether a multitransaction ID is in the cached valid range, returning * the nature of the range violation, if any. */ static XidBoundsViolation diff --git a/contrib/pgcrypto/pgp-compress.c b/contrib/pgcrypto/pgp-compress.c index 7e8ddba18735..086bec31ae2c 100644 --- a/contrib/pgcrypto/pgp-compress.c +++ b/contrib/pgcrypto/pgp-compress.c @@ -291,7 +291,7 @@ decompress_read(void *priv, PullFilter *src, int len, * A stream must be terminated by a normal packet. If the last stream * packet in the source stream is a full packet, a normal empty packet * must follow. Since the underlying packet reader doesn't know that - * the compressed stream has been ended, we need to to consume the + * the compressed stream has been ended, we need to consume the * terminating packet here. This read does not harm even if the * stream has already ended. */ diff --git a/doc/src/sgml/auto-explain.sgml b/doc/src/sgml/auto-explain.sgml index 192d6574c30c..30e35a714a5f 100644 --- a/doc/src/sgml/auto-explain.sgml +++ b/doc/src/sgml/auto-explain.sgml @@ -200,7 +200,7 @@ LOAD 'auto_explain'; auto_explain.log_settings controls whether information - about modified configuration options is printed when execution plan is logged. + about modified configuration options is printed when an execution plan is logged. Only options affecting query planning with value different from the built-in default value are included in the output. This parameter is off by default. Only superusers can change this setting. diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index c4897d68c9b9..872f7a7fac27 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -3919,7 +3919,7 @@ CREATE INDEX ON measurement (logdate); Normally the set of partitions established when initially defining the - table are not intended to remain static. It is common to want to + table is not intended to remain static. It is common to want to remove old partitions of data and periodically add new partitions for new data. One of the most important advantages of partitioning is precisely that it allows this otherwise painful task to be executed diff --git a/doc/src/sgml/ref/pg_rewind.sgml b/doc/src/sgml/ref/pg_rewind.sgml index 688acdcb06af..43282e6016fb 100644 --- a/doc/src/sgml/ref/pg_rewind.sgml +++ b/doc/src/sgml/ref/pg_rewind.sgml @@ -74,7 +74,7 @@ PostgreSQL documentation pg_rewind with the -c option to automatically retrieve them from the WAL archive. The use of pg_rewind is not limited to failover, e.g., a standby - server can be promoted, run some write transactions, and then rewinded + server can be promoted, run some write transactions, and then rewound to become a standby again. diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index bc510e2e9b36..9e04bc712c94 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -385,7 +385,7 @@ heap_page_prune(Relation relation, Buffer buffer, /* - * Perform visiblity checks for heap pruning. + * Perform visibility checks for heap pruning. * * This is more complicated than just using GlobalVisTestIsRemovableXid() * because of old_snapshot_threshold. We only want to increase the threshold diff --git a/src/backend/catalog/namespace.c b/src/backend/catalog/namespace.c index 391a9b225db7..740570c566dd 100644 --- a/src/backend/catalog/namespace.c +++ b/src/backend/catalog/namespace.c @@ -3853,7 +3853,7 @@ recomputeNamespacePath(void) /* * We want to detect the case where the effective value of the base search * path variables didn't change. As long as we're doing so, we can avoid - * copying the OID list unncessarily. + * copying the OID list unnecessarily. */ if (baseCreationNamespace == firstNS && baseTempCreationPending == temp_missing && diff --git a/src/backend/catalog/pg_namespace.c b/src/backend/catalog/pg_namespace.c index ed8527607057..7d2e26fd359b 100644 --- a/src/backend/catalog/pg_namespace.c +++ b/src/backend/catalog/pg_namespace.c @@ -106,7 +106,7 @@ NamespaceCreate(const char *nspName, Oid ownerId, bool isTemp) /* dependency on owner */ recordDependencyOnOwner(NamespaceRelationId, nspoid, ownerId); - /* dependences on roles mentioned in default ACL */ + /* dependencies on roles mentioned in default ACL */ recordDependencyOnNewAcl(NamespaceRelationId, nspoid, 0, ownerId, nspacl); /* dependency on extension ... but not for magic temp schemas */ diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 36ddcdccdb83..115860a9d403 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -2610,9 +2610,9 @@ CopyMultiInsertInfoFlush(CopyMultiInsertInfo *miinfo, ResultRelInfo *curr_rri) /* * Trim the list of tracked buffers down if it exceeds the limit. Here we - * remove buffers starting with the ones we created first. It seems more - * likely that these older ones are less likely to be needed than ones - * that were just created. + * remove buffers starting with the ones we created first. It seems less + * likely that these older ones will be needed than the ones that were + * just created. */ while (list_length(miinfo->multiInsertBuffers) > MAX_PARTITION_BUFFERS) { diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index a29c14bf1cf0..df13b72974ff 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -1790,7 +1790,7 @@ ExecuteTruncateGuts(List *explicit_rels, List *relids, List *relids_logged, * * We put the ResultRelInfos in the es_opened_result_relations list, even * though we don't have a range table and don't populate the - * es_result_relations array. That's a big bogus, but it's enough to make + * es_result_relations array. That's a bit bogus, but it's enough to make * ExecGetTriggerResultRel() find them. */ estate = CreateExecutorState(); diff --git a/src/backend/executor/execExpr.c b/src/backend/executor/execExpr.c index 868f8b0858f8..d76836c09b12 100644 --- a/src/backend/executor/execExpr.c +++ b/src/backend/executor/execExpr.c @@ -3266,7 +3266,7 @@ ExecBuildAggTransCall(ExprState *state, AggState *aggstate, * * For ordered aggregates: * - * Only need to choose between the faster path for a single orderred + * Only need to choose between the faster path for a single ordered * column, and the one between multiple columns. Checking strictness etc * is done when finalizing the aggregate. See * process_ordered_aggregate_{single, multi} and diff --git a/src/backend/executor/nodeIncrementalSort.c b/src/backend/executor/nodeIncrementalSort.c index 6c0d24ee25a5..eb6cc19a60cd 100644 --- a/src/backend/executor/nodeIncrementalSort.c +++ b/src/backend/executor/nodeIncrementalSort.c @@ -1097,7 +1097,7 @@ ExecEndIncrementalSort(IncrementalSortState *node) ExecClearTuple(node->ss.ss_ScanTupleSlot); /* must drop pointer to sort result tuple */ ExecClearTuple(node->ss.ps.ps_ResultTupleSlot); - /* must drop stanalone tuple slots from outer node */ + /* must drop standalone tuple slots from outer node */ ExecDropSingleTupleTableSlot(node->group_pivot); ExecDropSingleTupleTableSlot(node->transfer_tuple); diff --git a/src/backend/executor/nodeLimit.c b/src/backend/executor/nodeLimit.c index d85cf7d93e8f..c5896e579089 100644 --- a/src/backend/executor/nodeLimit.c +++ b/src/backend/executor/nodeLimit.c @@ -105,7 +105,7 @@ ExecLimit(PlanState *pstate) } /* - * Tuple at limit is needed for comparation in subsequent + * Tuple at limit is needed for comparison in subsequent * execution to detect ties. */ if (node->limitOption == LIMIT_OPTION_WITH_TIES && diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 803d9bae7f11..8ad6384c6ae8 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -2915,7 +2915,7 @@ generate_useful_gather_paths(PlannerInfo *root, RelOptInfo *rel, bool override_r /* * If the path has no ordering at all, then we can't use either - * incremental sort or rely on implict sorting with a gather + * incremental sort or rely on implicit sorting with a gather * merge. */ if (subpath->pathkeys == NIL) diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index d0ff6602842c..806629fff210 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -371,7 +371,7 @@ remove_rel_from_query(PlannerInfo *root, int relid, Relids joinrelids) * Likewise remove references from PlaceHolderVar data structures, * removing any no-longer-needed placeholders entirely. * - * Removal is a bit tricker than it might seem: we can remove PHVs that + * Removal is a bit trickier than it might seem: we can remove PHVs that * are used at the target rel and/or in the join qual, but not those that * are used at join partner rels or above the join. It's not that easy to * distinguish PHVs used at partner rels from those used in the join qual, diff --git a/src/backend/partitioning/partbounds.c b/src/backend/partitioning/partbounds.c index a4f97c10fe7a..ac0c49597272 100644 --- a/src/backend/partitioning/partbounds.c +++ b/src/backend/partitioning/partbounds.c @@ -1783,7 +1783,7 @@ merge_matching_partitions(PartitionMap *outer_map, PartitionMap *inner_map, if (outer_merged_index >= 0 && inner_merged_index >= 0) { /* - * If the mereged partitions are the same, no need to do anything; + * If the merged partitions are the same, no need to do anything; * return the index of the merged partitions. Otherwise, if each of * the given partitions has been merged with a dummy partition on the * other side, re-map them to either of the two merged partitions. diff --git a/src/backend/postmaster/interrupt.c b/src/backend/postmaster/interrupt.c index 3d02439b79ce..ee7dbf924ae6 100644 --- a/src/backend/postmaster/interrupt.c +++ b/src/backend/postmaster/interrupt.c @@ -92,7 +92,7 @@ SignalHandlerForCrashExit(SIGNAL_ARGS) * Simple signal handler for triggering a long-running background process to * shut down and exit. * - * Typically, this handler would be used for SIGTERM, but some procesess use + * Typically, this handler would be used for SIGTERM, but some processes use * other signals. In particular, the checkpointer exits on SIGUSR2, the * stats collector on SIGQUIT, and the WAL writer exits on either SIGINT * or SIGTERM. diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c index 4e30abb67437..d950b4eabe00 100644 --- a/src/backend/statistics/dependencies.c +++ b/src/backend/statistics/dependencies.c @@ -1305,7 +1305,7 @@ dependencies_clauselist_selectivity(PlannerInfo *root, /* * Work out which dependencies we can apply, starting with the - * widest/stongest ones, and proceeding to smaller/weaker ones. + * widest/strongest ones, and proceeding to smaller/weaker ones. */ dependencies = (MVDependency **) palloc(sizeof(MVDependency *) * total_ndeps); diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c index 9336f9bc5e90..36326927c6b1 100644 --- a/src/backend/statistics/extended_stats.c +++ b/src/backend/statistics/extended_stats.c @@ -1403,7 +1403,7 @@ statext_mcv_clauselist_selectivity(PlannerInfo *root, List *clauses, int varReli stat_sel = mcv_sel + other_sel; CLAMP_PROBABILITY(stat_sel); - /* Factor the estimate from this MCV to the oveall estimate. */ + /* Factor the estimate from this MCV to the overall estimate. */ sel *= stat_sel; } diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 2fa0b065a287..0adf04814cd9 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -560,7 +560,7 @@ PrefetchSharedBuffer(SMgrRelation smgr_reln, * could be used by the caller to avoid the need for a later buffer lookup, but * it's not pinned, so the caller must recheck it. * - * 2. If the kernel has been asked to initiate I/O, the initated_io member is + * 2. If the kernel has been asked to initiate I/O, the initiated_io member is * true. Currently there is no way to know if the data was already cached by * the kernel and therefore didn't really initiate I/O, and no way to know when * the I/O completes other than using synchronous ReadBuffer(). diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index d7bc33054174..5512e02940c0 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -6180,7 +6180,7 @@ unicode_normalize_func(PG_FUNCTION_ARGS) /* * Check whether the string is in the specified Unicode normalization form. * - * This is done by convering the string to the specified normal form and then + * This is done by converting the string to the specified normal form and then * comparing that to the original string. To speed that up, we also apply the * "quick check" algorithm specified in UAX #15, which can give a yes or no * answer for many strings by just scanning the string once. diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 2229c86f9afb..a2f1ab5422bc 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -207,7 +207,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, /* * Check if it is a checkpoint record. This checkpoint record needs to * be the latest checkpoint before WAL forked and not the checkpoint - * where the primary has been stopped to be rewinded. + * where the primary has been stopped to be rewound. */ info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK; if (searchptr < forkptr && diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index cd39f23d5b93..3057665bbec5 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -4185,7 +4185,7 @@ runInitSteps(const char *initialize_steps) } /* - * Extract pgbench table informations into global variables scale, + * Extract pgbench table information into global variables scale, * partition_method and partitions. */ static void diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index b0ca37c2ed81..e7781d010f0d 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -7088,7 +7088,7 @@ pgpassfileWarning(PGconn *conn) } /* - * Check if the SSL procotol value given in input is valid or not. + * Check if the SSL protocol value given in input is valid or not. * This is used as a sanity check routine for the connection parameters * ssl_min_protocol_version and ssl_max_protocol_version. */ diff --git a/src/test/modules/dummy_index_am/dummy_index_am.c b/src/test/modules/dummy_index_am/dummy_index_am.c index e97a32d5be2a..8f4cdab1b334 100644 --- a/src/test/modules/dummy_index_am/dummy_index_am.c +++ b/src/test/modules/dummy_index_am/dummy_index_am.c @@ -153,7 +153,7 @@ dibuild(Relation heap, Relation index, IndexInfo *indexInfo) } /* - * Build an empty index for the initialiation fork. + * Build an empty index for the initialization fork. */ static void dibuildempty(Relation index) From 90d8f1b1826ce076a502a43fe7c88423b46c6349 Mon Sep 17 00:00:00 2001 From: David Rowley Date: Mon, 2 Nov 2020 19:59:02 +1300 Subject: [PATCH 415/589] Fix unstable partition_prune regression tests This was broken recently by a929e17e5. I'd failed to remember that parallel tests should have their EXPLAIN output run through the explain_parallel_append function so that the output is stable when parallel workers fail to start. fairywren was first to notice. Reported-by: Michael Paquier Discussion: https://postgr.es/m/20201102062951.GB15770@paquier.xyz --- src/test/regress/expected/partition_prune.out | 45 +++++++++---------- src/test/regress/sql/partition_prune.sql | 9 ++-- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/src/test/regress/expected/partition_prune.out b/src/test/regress/expected/partition_prune.out index 80e71b8e2b9c..c72a6d051f1d 100644 --- a/src/test/regress/expected/partition_prune.out +++ b/src/test/regress/expected/partition_prune.out @@ -3684,18 +3684,17 @@ create table listp_12_2 partition of listp_12 for values in(2); alter table listp_12_1 set (parallel_workers = 0); -- Ensure that listp_12_2 is not scanned. (The nested Append is not seen in -- the plan as it's pulled in setref.c due to having just a single subnode). -explain (analyze on, costs off, timing off, summary off) -select * from listp where a = (select 1); - QUERY PLAN +select explain_parallel_append('select * from listp where a = (select 1);'); + explain_parallel_append ---------------------------------------------------------------------- - Gather (actual rows=0 loops=1) + Gather (actual rows=N loops=N) Workers Planned: 2 Params Evaluated: $0 - Workers Launched: 2 + Workers Launched: N InitPlan 1 (returns $0) - -> Result (actual rows=1 loops=1) - -> Parallel Append (actual rows=0 loops=3) - -> Seq Scan on listp_12_1 listp_1 (actual rows=0 loops=1) + -> Result (actual rows=N loops=N) + -> Parallel Append (actual rows=N loops=N) + -> Seq Scan on listp_12_1 listp_1 (actual rows=N loops=N) Filter: (a = $0) -> Parallel Seq Scan on listp_12_2 listp_2 (never executed) Filter: (a = $0) @@ -3704,34 +3703,34 @@ select * from listp where a = (select 1); -- Like the above but throw some more complexity at the planner by adding -- a UNION ALL. We expect both sides of the union not to scan the -- non-required partitions. -explain (analyze on, costs off, timing off, summary off) -select * from listp where a = (select 1) +select explain_parallel_append( +'select * from listp where a = (select 1) union all -select * from listp where a = (select 2); - QUERY PLAN +select * from listp where a = (select 2);'); + explain_parallel_append ----------------------------------------------------------------------------------- - Append (actual rows=0 loops=1) - -> Gather (actual rows=0 loops=1) + Append (actual rows=N loops=N) + -> Gather (actual rows=N loops=N) Workers Planned: 2 Params Evaluated: $0 - Workers Launched: 2 + Workers Launched: N InitPlan 1 (returns $0) - -> Result (actual rows=1 loops=1) - -> Parallel Append (actual rows=0 loops=3) - -> Seq Scan on listp_12_1 listp_1 (actual rows=0 loops=1) + -> Result (actual rows=N loops=N) + -> Parallel Append (actual rows=N loops=N) + -> Seq Scan on listp_12_1 listp_1 (actual rows=N loops=N) Filter: (a = $0) -> Parallel Seq Scan on listp_12_2 listp_2 (never executed) Filter: (a = $0) - -> Gather (actual rows=0 loops=1) + -> Gather (actual rows=N loops=N) Workers Planned: 2 Params Evaluated: $1 - Workers Launched: 2 + Workers Launched: N InitPlan 2 (returns $1) - -> Result (actual rows=1 loops=1) - -> Parallel Append (actual rows=0 loops=3) + -> Result (actual rows=N loops=N) + -> Parallel Append (actual rows=N loops=N) -> Seq Scan on listp_12_1 listp_4 (never executed) Filter: (a = $1) - -> Parallel Seq Scan on listp_12_2 listp_5 (actual rows=0 loops=1) + -> Parallel Seq Scan on listp_12_2 listp_5 (actual rows=N loops=N) Filter: (a = $1) (23 rows) diff --git a/src/test/regress/sql/partition_prune.sql b/src/test/regress/sql/partition_prune.sql index 939a9b1193ed..ffd5fe8b0dc5 100644 --- a/src/test/regress/sql/partition_prune.sql +++ b/src/test/regress/sql/partition_prune.sql @@ -1067,16 +1067,15 @@ alter table listp_12_1 set (parallel_workers = 0); -- Ensure that listp_12_2 is not scanned. (The nested Append is not seen in -- the plan as it's pulled in setref.c due to having just a single subnode). -explain (analyze on, costs off, timing off, summary off) -select * from listp where a = (select 1); +select explain_parallel_append('select * from listp where a = (select 1);'); -- Like the above but throw some more complexity at the planner by adding -- a UNION ALL. We expect both sides of the union not to scan the -- non-required partitions. -explain (analyze on, costs off, timing off, summary off) -select * from listp where a = (select 1) +select explain_parallel_append( +'select * from listp where a = (select 1) union all -select * from listp where a = (select 2); +select * from listp where a = (select 2);'); drop table listp; reset parallel_tuple_cost; From 8ef2a5afdf8ec9e4c8b28a7042c9508eb6161671 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 2 Nov 2020 12:51:46 +0200 Subject: [PATCH 416/589] doc: Mention UNION/ORDER BY etc. keywords in section headers. Most of the section and sub-section headers in the Queries chapter have the keywords literally stated, but neither "Sorting Rows" nor "Combining Rows" did. There's no rule that they must be, but it seems like a good practice. The keywords will ring a bell to anyone with with even a little bit of SQL experience. David G. Johnston, per suggestion by bilge@scriptfusion.com Discussion: https://www.postgresql.org/message-id/159981394174.31338.7014519396749859167%40wrigleys.postgresql.org --- doc/src/sgml/queries.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/queries.sgml b/doc/src/sgml/queries.sgml index f06afe2c3fb1..dedb5684e631 100644 --- a/doc/src/sgml/queries.sgml +++ b/doc/src/sgml/queries.sgml @@ -1621,7 +1621,7 @@ SELECT DISTINCT ON (expression , - Combining Queries + Combining Queries (<literal>UNION</literal>, <literal>INTERSECT</literal>, <literal>EXCEPT</literal>) UNION @@ -1701,7 +1701,7 @@ SELECT DISTINCT ON (expression , - Sorting Rows + Sorting Rows (<literal>ORDER BY</literal>) sorting From 7d1297df0830725d4434ba7dbf71a9e8221ad49e Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Mon, 2 Nov 2020 19:36:09 +1300 Subject: [PATCH 417/589] Remove pg_collation.collversion. This model couldn't be extended to cover the default collation, and didn't have any information about the affected database objects when the version changed. Remove, in preparation for a follow-up commit that will add a new mechanism. Author: Thomas Munro Reviewed-by: Julien Rouhaud Reviewed-by: Peter Eisentraut Discussion: https://postgr.es/m/CAEepm%3D0uEQCpfq_%2BLYFBdArCe4Ot98t1aR4eYiYTe%3DyavQygiQ%40mail.gmail.com --- doc/src/sgml/catalogs.sgml | 11 --- doc/src/sgml/func.sgml | 6 +- doc/src/sgml/ref/alter_collation.sgml | 63 ------------- doc/src/sgml/ref/create_collation.sgml | 21 ----- src/backend/catalog/pg_collation.c | 5 -- src/backend/commands/collationcmds.c | 88 ------------------- src/backend/nodes/copyfuncs.c | 13 --- src/backend/nodes/equalfuncs.c | 11 --- src/backend/parser/gram.y | 18 +--- src/backend/tcop/utility.c | 12 --- src/backend/utils/adt/pg_locale.c | 37 -------- src/bin/pg_dump/pg_dump.c | 24 +---- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_collation.dat | 7 +- src/include/catalog/pg_collation.h | 5 -- src/include/catalog/toasting.h | 1 - src/include/commands/collationcmds.h | 1 - src/include/nodes/parsenodes.h | 11 --- .../regress/expected/collate.icu.utf8.out | 3 - .../regress/expected/collate.linux.utf8.out | 3 - src/test/regress/sql/collate.icu.utf8.sql | 5 -- src/test/regress/sql/collate.linux.utf8.sql | 5 -- 22 files changed, 8 insertions(+), 344 deletions(-) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 0ccdff1cda62..d0d2598290e1 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -2361,17 +2361,6 @@ SCRAM-SHA-256$<iteration count>:&l LC_CTYPE for this collation object - - - - collversion text - - - Provider-specific version of the collation. This is recorded when the - collation is created and then checked when it is used, to detect - changes in the collation definition that could lead to data corruption. - - diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index d8eee3a82643..0398b4909bf8 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25444,11 +25444,7 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); Returns the actual version of the collation object as it is currently - installed in the operating system. If this is different from the - value in - pg_collation.collversion, - then objects depending on the collation might need to be rebuilt. See - also . + installed in the operating system. diff --git a/doc/src/sgml/ref/alter_collation.sgml b/doc/src/sgml/ref/alter_collation.sgml index af9ff2867b72..65429aabe28b 100644 --- a/doc/src/sgml/ref/alter_collation.sgml +++ b/doc/src/sgml/ref/alter_collation.sgml @@ -21,8 +21,6 @@ PostgreSQL documentation -ALTER COLLATION name REFRESH VERSION - ALTER COLLATION name RENAME TO new_name ALTER COLLATION name OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER } ALTER COLLATION name SET SCHEMA new_schema @@ -88,70 +86,9 @@ ALTER COLLATION name SET SCHEMA new_sche - - REFRESH VERSION - - - Update the collation's version. - See below. - - - - - Notes - - - When using collations provided by the ICU library, the ICU-specific version - of the collator is recorded in the system catalog when the collation object - is created. When the collation is used, the current version is - checked against the recorded version, and a warning is issued when there is - a mismatch, for example: - -WARNING: collation "xx-x-icu" has version mismatch -DETAIL: The collation in the database was created using version 1.2.3.4, but the operating system provides version 2.3.4.5. -HINT: Rebuild all objects affected by this collation and run ALTER COLLATION pg_catalog."xx-x-icu" REFRESH VERSION, or build PostgreSQL with the right library version. - - A change in collation definitions can lead to corrupt indexes and other - problems because the database system relies on stored objects having a - certain sort order. Generally, this should be avoided, but it can happen - in legitimate circumstances, such as when - using pg_upgrade to upgrade to server binaries linked - with a newer version of ICU. When this happens, all objects depending on - the collation should be rebuilt, for example, - using REINDEX. When that is done, the collation version - can be refreshed using the command ALTER COLLATION ... REFRESH - VERSION. This will update the system catalog to record the - current collator version and will make the warning go away. Note that this - does not actually check whether all affected objects have been rebuilt - correctly. - - - When using collations provided by libc and - PostgreSQL was built with the GNU C library, the - C library's version is used as a collation version. Since collation - definitions typically change only with GNU C library releases, this provides - some defense against corruption, but it is not completely reliable. - - - Currently, there is no version tracking for the database default collation. - - - - The following query can be used to identify all collations in the current - database that need to be refreshed and the objects that depend on them: - pg_collation_actual_version(c.oid) - ORDER BY 1, 2; -]]> - - Examples diff --git a/doc/src/sgml/ref/create_collation.sgml b/doc/src/sgml/ref/create_collation.sgml index 58f5f0cd63a2..b97842071f92 100644 --- a/doc/src/sgml/ref/create_collation.sgml +++ b/doc/src/sgml/ref/create_collation.sgml @@ -27,7 +27,6 @@ CREATE COLLATION [ IF NOT EXISTS ] name ( [ LC_CTYPE = lc_ctype, ] [ PROVIDER = provider, ] [ DETERMINISTIC = boolean, ] - [ VERSION = version ] ) CREATE COLLATION [ IF NOT EXISTS ] name FROM existing_collation @@ -149,26 +148,6 @@ CREATE COLLATION [ IF NOT EXISTS ] name FROM - - version - - - - Specifies the version string to store with the collation. Normally, - this should be omitted, which will cause the version to be computed - from the actual version of the collation as provided by the operating - system. This option is intended to be used - by pg_upgrade for copying the version from an - existing installation. - - - - See also for how to handle - collation version mismatches. - - - - existing_collation diff --git a/src/backend/catalog/pg_collation.c b/src/backend/catalog/pg_collation.c index 5fdf1acb7e5b..3c84378d0255 100644 --- a/src/backend/catalog/pg_collation.c +++ b/src/backend/catalog/pg_collation.c @@ -49,7 +49,6 @@ CollationCreate(const char *collname, Oid collnamespace, bool collisdeterministic, int32 collencoding, const char *collcollate, const char *collctype, - const char *collversion, bool if_not_exists, bool quiet) { @@ -167,10 +166,6 @@ CollationCreate(const char *collname, Oid collnamespace, values[Anum_pg_collation_collcollate - 1] = NameGetDatum(&name_collate); namestrcpy(&name_ctype, collctype); values[Anum_pg_collation_collctype - 1] = NameGetDatum(&name_ctype); - if (collversion) - values[Anum_pg_collation_collversion - 1] = CStringGetTextDatum(collversion); - else - nulls[Anum_pg_collation_collversion - 1] = true; tup = heap_form_tuple(tupDesc, values, nulls); diff --git a/src/backend/commands/collationcmds.c b/src/backend/commands/collationcmds.c index 9f6582c530cb..5ad8886e60a4 100644 --- a/src/backend/commands/collationcmds.c +++ b/src/backend/commands/collationcmds.c @@ -61,14 +61,12 @@ DefineCollation(ParseState *pstate, List *names, List *parameters, bool if_not_e DefElem *lcctypeEl = NULL; DefElem *providerEl = NULL; DefElem *deterministicEl = NULL; - DefElem *versionEl = NULL; char *collcollate = NULL; char *collctype = NULL; char *collproviderstr = NULL; bool collisdeterministic = true; int collencoding = 0; char collprovider = 0; - char *collversion = NULL; Oid newoid; ObjectAddress address; @@ -96,8 +94,6 @@ DefineCollation(ParseState *pstate, List *names, List *parameters, bool if_not_e defelp = &providerEl; else if (strcmp(defel->defname, "deterministic") == 0) defelp = &deterministicEl; - else if (strcmp(defel->defname, "version") == 0) - defelp = &versionEl; else { ereport(ERROR, @@ -166,9 +162,6 @@ DefineCollation(ParseState *pstate, List *names, List *parameters, bool if_not_e if (deterministicEl) collisdeterministic = defGetBoolean(deterministicEl); - if (versionEl) - collversion = defGetString(versionEl); - if (collproviderstr) { if (pg_strcasecmp(collproviderstr, "icu") == 0) @@ -215,9 +208,6 @@ DefineCollation(ParseState *pstate, List *names, List *parameters, bool if_not_e } } - if (!collversion) - collversion = get_collation_actual_version(collprovider, collcollate); - newoid = CollationCreate(collName, collNamespace, GetUserId(), @@ -226,7 +216,6 @@ DefineCollation(ParseState *pstate, List *names, List *parameters, bool if_not_e collencoding, collcollate, collctype, - collversion, if_not_exists, false); /* not quiet */ @@ -277,80 +266,6 @@ IsThereCollationInNamespace(const char *collname, Oid nspOid) collname, get_namespace_name(nspOid)))); } -/* - * ALTER COLLATION - */ -ObjectAddress -AlterCollation(AlterCollationStmt *stmt) -{ - Relation rel; - Oid collOid; - HeapTuple tup; - Form_pg_collation collForm; - Datum collversion; - bool isnull; - char *oldversion; - char *newversion; - ObjectAddress address; - - rel = table_open(CollationRelationId, RowExclusiveLock); - collOid = get_collation_oid(stmt->collname, false); - - if (!pg_collation_ownercheck(collOid, GetUserId())) - aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_COLLATION, - NameListToString(stmt->collname)); - - tup = SearchSysCacheCopy1(COLLOID, ObjectIdGetDatum(collOid)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for collation %u", collOid); - - collForm = (Form_pg_collation) GETSTRUCT(tup); - collversion = SysCacheGetAttr(COLLOID, tup, Anum_pg_collation_collversion, - &isnull); - oldversion = isnull ? NULL : TextDatumGetCString(collversion); - - newversion = get_collation_actual_version(collForm->collprovider, NameStr(collForm->collcollate)); - - /* cannot change from NULL to non-NULL or vice versa */ - if ((!oldversion && newversion) || (oldversion && !newversion)) - elog(ERROR, "invalid collation version change"); - else if (oldversion && newversion && strcmp(newversion, oldversion) != 0) - { - bool nulls[Natts_pg_collation]; - bool replaces[Natts_pg_collation]; - Datum values[Natts_pg_collation]; - - ereport(NOTICE, - (errmsg("changing version from %s to %s", - oldversion, newversion))); - - memset(values, 0, sizeof(values)); - memset(nulls, false, sizeof(nulls)); - memset(replaces, false, sizeof(replaces)); - - values[Anum_pg_collation_collversion - 1] = CStringGetTextDatum(newversion); - replaces[Anum_pg_collation_collversion - 1] = true; - - tup = heap_modify_tuple(tup, RelationGetDescr(rel), - values, nulls, replaces); - } - else - ereport(NOTICE, - (errmsg("version has not changed"))); - - CatalogTupleUpdate(rel, &tup->t_self, tup); - - InvokeObjectPostAlterHook(CollationRelationId, collOid, 0); - - ObjectAddressSet(address, CollationRelationId, collOid); - - heap_freetuple(tup); - table_close(rel, NoLock); - - return address; -} - - Datum pg_collation_actual_version(PG_FUNCTION_ARGS) { @@ -608,7 +523,6 @@ pg_import_system_collations(PG_FUNCTION_ARGS) collid = CollationCreate(localebuf, nspid, GetUserId(), COLLPROVIDER_LIBC, true, enc, localebuf, localebuf, - get_collation_actual_version(COLLPROVIDER_LIBC, localebuf), true, true); if (OidIsValid(collid)) { @@ -669,7 +583,6 @@ pg_import_system_collations(PG_FUNCTION_ARGS) collid = CollationCreate(alias, nspid, GetUserId(), COLLPROVIDER_LIBC, true, enc, locale, locale, - get_collation_actual_version(COLLPROVIDER_LIBC, locale), true, true); if (OidIsValid(collid)) { @@ -731,7 +644,6 @@ pg_import_system_collations(PG_FUNCTION_ARGS) nspid, GetUserId(), COLLPROVIDER_ICU, true, -1, collcollate, collcollate, - get_collation_actual_version(COLLPROVIDER_ICU, collcollate), true, true); if (OidIsValid(collid)) { diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 2b4d7654cc71..ac8b57109c56 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -3224,16 +3224,6 @@ _copyAlterTableCmd(const AlterTableCmd *from) return newnode; } -static AlterCollationStmt * -_copyAlterCollationStmt(const AlterCollationStmt *from) -{ - AlterCollationStmt *newnode = makeNode(AlterCollationStmt); - - COPY_NODE_FIELD(collname); - - return newnode; -} - static AlterDomainStmt * _copyAlterDomainStmt(const AlterDomainStmt *from) { @@ -5229,9 +5219,6 @@ copyObjectImpl(const void *from) case T_AlterTableCmd: retval = _copyAlterTableCmd(from); break; - case T_AlterCollationStmt: - retval = _copyAlterCollationStmt(from); - break; case T_AlterDomainStmt: retval = _copyAlterDomainStmt(from); break; diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index e2d1b987bf49..0cf90ef33c31 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -1107,14 +1107,6 @@ _equalAlterTableCmd(const AlterTableCmd *a, const AlterTableCmd *b) return true; } -static bool -_equalAlterCollationStmt(const AlterCollationStmt *a, const AlterCollationStmt *b) -{ - COMPARE_NODE_FIELD(collname); - - return true; -} - static bool _equalAlterDomainStmt(const AlterDomainStmt *a, const AlterDomainStmt *b) { @@ -3283,9 +3275,6 @@ equal(const void *a, const void *b) case T_AlterTableCmd: retval = _equalAlterTableCmd(a, b); break; - case T_AlterCollationStmt: - retval = _equalAlterCollationStmt(a, b); - break; case T_AlterDomainStmt: retval = _equalAlterDomainStmt(a, b); break; diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 480d16834687..60cf7242a308 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -254,7 +254,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); } %type stmt schema_stmt - AlterEventTrigStmt AlterCollationStmt + AlterEventTrigStmt AlterDatabaseStmt AlterDatabaseSetStmt AlterDomainStmt AlterEnumStmt AlterFdwStmt AlterForeignServerStmt AlterGroupStmt AlterObjectDependsStmt AlterObjectSchemaStmt AlterOwnerStmt @@ -835,7 +835,6 @@ stmtmulti: stmtmulti ';' stmt stmt : AlterEventTrigStmt - | AlterCollationStmt | AlterDatabaseStmt | AlterDatabaseSetStmt | AlterDefaultPrivilegesStmt @@ -10169,21 +10168,6 @@ drop_option: } ; -/***************************************************************************** - * - * ALTER COLLATION - * - *****************************************************************************/ - -AlterCollationStmt: ALTER COLLATION any_name REFRESH VERSION_P - { - AlterCollationStmt *n = makeNode(AlterCollationStmt); - n->collname = $3; - $$ = (Node *)n; - } - ; - - /***************************************************************************** * * ALTER SYSTEM diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 9a35147b26af..f398027fa619 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -1842,10 +1842,6 @@ ProcessUtilitySlow(ParseState *pstate, address = AlterStatistics((AlterStatsStmt *) parsetree); break; - case T_AlterCollationStmt: - address = AlterCollation((AlterCollationStmt *) parsetree); - break; - default: elog(ERROR, "unrecognized node type: %d", (int) nodeTag(parsetree)); @@ -2993,10 +2989,6 @@ CreateCommandTag(Node *parsetree) tag = CMDTAG_DROP_SUBSCRIPTION; break; - case T_AlterCollationStmt: - tag = CMDTAG_ALTER_COLLATION; - break; - case T_PrepareStmt: tag = CMDTAG_PREPARE; break; @@ -3609,10 +3601,6 @@ GetCommandLogLevel(Node *parsetree) lev = LOGSTMT_DDL; break; - case T_AlterCollationStmt: - lev = LOGSTMT_DDL; - break; - /* already-planned queries */ case T_PlannedStmt: { diff --git a/src/backend/utils/adt/pg_locale.c b/src/backend/utils/adt/pg_locale.c index 07299dbc0911..514e0fa0af86 100644 --- a/src/backend/utils/adt/pg_locale.c +++ b/src/backend/utils/adt/pg_locale.c @@ -1513,8 +1513,6 @@ pg_newlocale_from_collation(Oid collid) const char *collctype pg_attribute_unused(); struct pg_locale_struct result; pg_locale_t resultp; - Datum collversion; - bool isnull; tp = SearchSysCache1(COLLOID, ObjectIdGetDatum(collid)); if (!HeapTupleIsValid(tp)) @@ -1616,41 +1614,6 @@ pg_newlocale_from_collation(Oid collid) #endif /* not USE_ICU */ } - collversion = SysCacheGetAttr(COLLOID, tp, Anum_pg_collation_collversion, - &isnull); - if (!isnull) - { - char *actual_versionstr; - char *collversionstr; - - actual_versionstr = get_collation_actual_version(collform->collprovider, collcollate); - if (!actual_versionstr) - { - /* - * This could happen when specifying a version in CREATE - * COLLATION for a libc locale, or manually creating a mess in - * the catalogs. - */ - ereport(ERROR, - (errmsg("collation \"%s\" has no actual version, but a version was specified", - NameStr(collform->collname)))); - } - collversionstr = TextDatumGetCString(collversion); - - if (strcmp(actual_versionstr, collversionstr) != 0) - ereport(WARNING, - (errmsg("collation \"%s\" has version mismatch", - NameStr(collform->collname)), - errdetail("The collation in the database was created using version %s, " - "but the operating system provides version %s.", - collversionstr, actual_versionstr), - errhint("Rebuild all objects affected by this collation and run " - "ALTER COLLATION %s REFRESH VERSION, " - "or build PostgreSQL with the right library version.", - quote_qualified_identifier(get_namespace_name(collform->collnamespace), - NameStr(collform->collname))))); - } - ReleaseSysCache(tp); /* We'll keep the pg_locale_t structures in TopMemoryContext */ diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 03f9d4d9e84f..9851022a53d4 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -13589,12 +13589,10 @@ dumpCollation(Archive *fout, CollInfo *collinfo) if (fout->remoteVersion >= 100000) appendPQExpBufferStr(query, - "collprovider, " - "collversion, "); + "collprovider, "); else appendPQExpBufferStr(query, - "'c' AS collprovider, " - "NULL AS collversion, "); + "'c' AS collprovider, "); if (fout->remoteVersion >= 120000) appendPQExpBufferStr(query, @@ -13655,24 +13653,6 @@ dumpCollation(Archive *fout, CollInfo *collinfo) appendStringLiteralAH(q, collctype, fout); } - /* - * For binary upgrade, carry over the collation version. For normal - * dump/restore, omit the version, so that it is computed upon restore. - */ - if (dopt->binary_upgrade) - { - int i_collversion; - - i_collversion = PQfnumber(res, "collversion"); - if (!PQgetisnull(res, 0, i_collversion)) - { - appendPQExpBufferStr(q, ", version = "); - appendStringLiteralAH(q, - PQgetvalue(res, 0, i_collversion), - fout); - } - } - appendPQExpBufferStr(q, ");\n"); if (dopt->binary_upgrade) diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 73650f88e947..fbb729a0b2ef 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 202010291 +#define CATALOG_VERSION_NO 202011011 #endif diff --git a/src/include/catalog/pg_collation.dat b/src/include/catalog/pg_collation.dat index ba1b3e201b67..45301ccdd7d1 100644 --- a/src/include/catalog/pg_collation.dat +++ b/src/include/catalog/pg_collation.dat @@ -15,17 +15,16 @@ { oid => '100', oid_symbol => 'DEFAULT_COLLATION_OID', descr => 'database\'s default collation', collname => 'default', collnamespace => 'PGNSP', collowner => 'PGUID', - collprovider => 'd', collencoding => '-1', collcollate => '', collctype => '', - collversion => '_null_' }, + collprovider => 'd', collencoding => '-1', collcollate => '', collctype => '' }, { oid => '950', oid_symbol => 'C_COLLATION_OID', descr => 'standard C collation', collname => 'C', collnamespace => 'PGNSP', collowner => 'PGUID', collprovider => 'c', collencoding => '-1', collcollate => 'C', - collctype => 'C', collversion => '_null_' }, + collctype => 'C' }, { oid => '951', oid_symbol => 'POSIX_COLLATION_OID', descr => 'standard POSIX collation', collname => 'POSIX', collnamespace => 'PGNSP', collowner => 'PGUID', collprovider => 'c', collencoding => '-1', collcollate => 'POSIX', - collctype => 'POSIX', collversion => '_null_' }, + collctype => 'POSIX' }, ] diff --git a/src/include/catalog/pg_collation.h b/src/include/catalog/pg_collation.h index 27618b324d3f..e7e958b80898 100644 --- a/src/include/catalog/pg_collation.h +++ b/src/include/catalog/pg_collation.h @@ -37,10 +37,6 @@ CATALOG(pg_collation,3456,CollationRelationId) int32 collencoding; /* encoding for this collation; -1 = "all" */ NameData collcollate; /* LC_COLLATE setting */ NameData collctype; /* LC_CTYPE setting */ -#ifdef CATALOG_VARLEN /* variable-length fields start here */ - text collversion; /* provider-dependent version of collation - * data */ -#endif } FormData_pg_collation; /* ---------------- @@ -65,7 +61,6 @@ extern Oid CollationCreate(const char *collname, Oid collnamespace, bool collisdeterministic, int32 collencoding, const char *collcollate, const char *collctype, - const char *collversion, bool if_not_exists, bool quiet); diff --git a/src/include/catalog/toasting.h b/src/include/catalog/toasting.h index 51491c451314..8f131893dc4b 100644 --- a/src/include/catalog/toasting.h +++ b/src/include/catalog/toasting.h @@ -51,7 +51,6 @@ extern void BootstrapToastTable(char *relName, /* normal catalogs */ DECLARE_TOAST(pg_aggregate, 4159, 4160); DECLARE_TOAST(pg_attrdef, 2830, 2831); -DECLARE_TOAST(pg_collation, 4161, 4162); DECLARE_TOAST(pg_constraint, 2832, 2833); DECLARE_TOAST(pg_default_acl, 4143, 4144); DECLARE_TOAST(pg_description, 2834, 2835); diff --git a/src/include/commands/collationcmds.h b/src/include/commands/collationcmds.h index 373b85374c42..3e1c16ac7f05 100644 --- a/src/include/commands/collationcmds.h +++ b/src/include/commands/collationcmds.h @@ -20,6 +20,5 @@ extern ObjectAddress DefineCollation(ParseState *pstate, List *names, List *parameters, bool if_not_exists); extern void IsThereCollationInNamespace(const char *collname, Oid nspOid); -extern ObjectAddress AlterCollation(AlterCollationStmt *stmt); #endif /* COLLATIONCMDS_H */ diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index ff584f2955bb..319f77013f4d 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1879,17 +1879,6 @@ typedef struct AlterTableCmd /* one subcommand of an ALTER TABLE */ } AlterTableCmd; -/* ---------------------- - * Alter Collation - * ---------------------- - */ -typedef struct AlterCollationStmt -{ - NodeTag type; - List *collname; -} AlterCollationStmt; - - /* ---------------------- * Alter Domain * diff --git a/src/test/regress/expected/collate.icu.utf8.out b/src/test/regress/expected/collate.icu.utf8.out index 2b86ce90286d..60d9263a2ffc 100644 --- a/src/test/regress/expected/collate.icu.utf8.out +++ b/src/test/regress/expected/collate.icu.utf8.out @@ -1082,9 +1082,6 @@ SELECT collname FROM pg_collation WHERE collname LIKE 'test%'; DROP SCHEMA test_schema; DROP ROLE regress_test_role; --- ALTER -ALTER COLLATION "en-x-icu" REFRESH VERSION; -NOTICE: version has not changed -- dependencies CREATE COLLATION test0 FROM "C"; CREATE TABLE collate_dep_test1 (a int, b text COLLATE test0); diff --git a/src/test/regress/expected/collate.linux.utf8.out b/src/test/regress/expected/collate.linux.utf8.out index f06ae543e497..580b00eea79b 100644 --- a/src/test/regress/expected/collate.linux.utf8.out +++ b/src/test/regress/expected/collate.linux.utf8.out @@ -1093,9 +1093,6 @@ SELECT collname FROM pg_collation WHERE collname LIKE 'test%'; DROP SCHEMA test_schema; DROP ROLE regress_test_role; --- ALTER -ALTER COLLATION "en_US" REFRESH VERSION; -NOTICE: version has not changed -- dependencies CREATE COLLATION test0 FROM "C"; CREATE TABLE collate_dep_test1 (a int, b text COLLATE test0); diff --git a/src/test/regress/sql/collate.icu.utf8.sql b/src/test/regress/sql/collate.icu.utf8.sql index 67de7d97949f..35acf91fbf1e 100644 --- a/src/test/regress/sql/collate.icu.utf8.sql +++ b/src/test/regress/sql/collate.icu.utf8.sql @@ -405,11 +405,6 @@ DROP SCHEMA test_schema; DROP ROLE regress_test_role; --- ALTER - -ALTER COLLATION "en-x-icu" REFRESH VERSION; - - -- dependencies CREATE COLLATION test0 FROM "C"; diff --git a/src/test/regress/sql/collate.linux.utf8.sql b/src/test/regress/sql/collate.linux.utf8.sql index cbbd2203e413..c697c9948805 100644 --- a/src/test/regress/sql/collate.linux.utf8.sql +++ b/src/test/regress/sql/collate.linux.utf8.sql @@ -406,11 +406,6 @@ DROP SCHEMA test_schema; DROP ROLE regress_test_role; --- ALTER - -ALTER COLLATION "en_US" REFRESH VERSION; - - -- dependencies CREATE COLLATION test0 FROM "C"; From cd6f479e79f3a33ef7a919c6b6c0c498c790f154 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Mon, 2 Nov 2020 19:40:49 +1300 Subject: [PATCH 418/589] Add pg_depend.refobjversion. Provide a place for the version of referenced database objects to be recorded. A follow-up commit will use this to record dependencies on collation versions for indexes, but similar ideas for other kinds of objects have also been mooted. Author: Thomas Munro Reviewed-by: Julien Rouhaud Reviewed-by: Peter Eisentraut Discussion: https://postgr.es/m/CAEepm%3D0uEQCpfq_%2BLYFBdArCe4Ot98t1aR4eYiYTe%3DyavQygiQ%40mail.gmail.com --- doc/src/sgml/catalogs.sgml | 11 +++++++++++ src/backend/catalog/dependency.c | 14 ++++++++++---- src/backend/catalog/pg_depend.c | 14 ++++++++++---- src/include/catalog/catversion.h | 2 +- src/include/catalog/dependency.h | 1 + src/include/catalog/pg_depend.h | 4 ++++ src/include/catalog/toasting.h | 1 + src/test/regress/expected/misc_sanity.out | 4 ++-- 8 files changed, 40 insertions(+), 11 deletions(-) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index d0d2598290e1..c3f324f05eb4 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -3302,6 +3302,17 @@ SCRAM-SHA-256$<iteration count>:&l A code defining the specific semantics of this dependency relationship; see text + + + + refobjversion text + + + An optional version for the referenced object. + + + + diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c index f515e2c308bc..1a927377e739 100644 --- a/src/backend/catalog/dependency.c +++ b/src/backend/catalog/dependency.c @@ -1600,7 +1600,9 @@ recordDependencyOnExpr(const ObjectAddress *depender, /* And record 'em */ recordMultipleDependencies(depender, - context.addrs->refs, context.addrs->numrefs, + context.addrs->refs, + context.addrs->numrefs, + NULL, behavior); free_object_addresses(context.addrs); @@ -1687,7 +1689,9 @@ recordDependencyOnSingleRelExpr(const ObjectAddress *depender, /* Record the self-dependencies with the appropriate direction */ if (!reverse_self) recordMultipleDependencies(depender, - self_addrs->refs, self_addrs->numrefs, + self_addrs->refs, + self_addrs->numrefs, + NULL, self_behavior); else { @@ -1707,7 +1711,9 @@ recordDependencyOnSingleRelExpr(const ObjectAddress *depender, /* Record the external dependencies */ recordMultipleDependencies(depender, - context.addrs->refs, context.addrs->numrefs, + context.addrs->refs, + context.addrs->numrefs, + NULL, behavior); free_object_addresses(context.addrs); @@ -2679,7 +2685,7 @@ record_object_address_dependencies(const ObjectAddress *depender, { eliminate_duplicate_dependencies(referenced); recordMultipleDependencies(depender, - referenced->refs, referenced->numrefs, + referenced->refs, referenced->numrefs, NULL, behavior); } diff --git a/src/backend/catalog/pg_depend.c b/src/backend/catalog/pg_depend.c index 454e569fa98d..09c30b13e884 100644 --- a/src/backend/catalog/pg_depend.c +++ b/src/backend/catalog/pg_depend.c @@ -24,6 +24,7 @@ #include "catalog/pg_extension.h" #include "commands/extension.h" #include "miscadmin.h" +#include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/rel.h" @@ -44,7 +45,7 @@ recordDependencyOn(const ObjectAddress *depender, const ObjectAddress *referenced, DependencyType behavior) { - recordMultipleDependencies(depender, referenced, 1, behavior); + recordMultipleDependencies(depender, referenced, 1, NULL, behavior); } /* @@ -55,6 +56,7 @@ void recordMultipleDependencies(const ObjectAddress *depender, const ObjectAddress *referenced, int nreferenced, + const char *version, DependencyType behavior) { Relation dependDesc; @@ -115,6 +117,9 @@ recordMultipleDependencies(const ObjectAddress *depender, * Record the dependency. Note we don't bother to check for duplicate * dependencies; there's no harm in them. */ + memset(slot[slot_stored_count]->tts_isnull, false, + slot[slot_stored_count]->tts_tupleDescriptor->natts * sizeof(bool)); + slot[slot_stored_count]->tts_values[Anum_pg_depend_refclassid - 1] = ObjectIdGetDatum(referenced->classId); slot[slot_stored_count]->tts_values[Anum_pg_depend_refobjid - 1] = ObjectIdGetDatum(referenced->objectId); slot[slot_stored_count]->tts_values[Anum_pg_depend_refobjsubid - 1] = Int32GetDatum(referenced->objectSubId); @@ -122,9 +127,10 @@ recordMultipleDependencies(const ObjectAddress *depender, slot[slot_stored_count]->tts_values[Anum_pg_depend_classid - 1] = ObjectIdGetDatum(depender->classId); slot[slot_stored_count]->tts_values[Anum_pg_depend_objid - 1] = ObjectIdGetDatum(depender->objectId); slot[slot_stored_count]->tts_values[Anum_pg_depend_objsubid - 1] = Int32GetDatum(depender->objectSubId); - - memset(slot[slot_stored_count]->tts_isnull, false, - slot[slot_stored_count]->tts_tupleDescriptor->natts * sizeof(bool)); + if (version) + slot[slot_stored_count]->tts_values[Anum_pg_depend_refobjversion - 1] = CStringGetTextDatum(version); + else + slot[slot_stored_count]->tts_isnull[Anum_pg_depend_refobjversion - 1] = true; ExecStoreVirtualTuple(slot[slot_stored_count]); slot_stored_count++; diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index fbb729a0b2ef..6610e3c23f71 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 202011011 +#define CATALOG_VERSION_NO 202011012 #endif diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h index a8f7e9965b7c..3baa5e498aa9 100644 --- a/src/include/catalog/dependency.h +++ b/src/include/catalog/dependency.h @@ -189,6 +189,7 @@ extern void recordDependencyOn(const ObjectAddress *depender, extern void recordMultipleDependencies(const ObjectAddress *depender, const ObjectAddress *referenced, int nreferenced, + const char *version, DependencyType behavior); extern void recordDependencyOnCurrentExtension(const ObjectAddress *object, diff --git a/src/include/catalog/pg_depend.h b/src/include/catalog/pg_depend.h index ccf0a983301c..748902279529 100644 --- a/src/include/catalog/pg_depend.h +++ b/src/include/catalog/pg_depend.h @@ -61,6 +61,10 @@ CATALOG(pg_depend,2608,DependRelationId) * field. See DependencyType in catalog/dependency.h. */ char deptype; /* see codes in dependency.h */ +#ifdef CATALOG_VARLEN + text refobjversion; /* version tracking, NULL if not used or + * unknown */ +#endif } FormData_pg_depend; /* ---------------- diff --git a/src/include/catalog/toasting.h b/src/include/catalog/toasting.h index 8f131893dc4b..e320d822039b 100644 --- a/src/include/catalog/toasting.h +++ b/src/include/catalog/toasting.h @@ -53,6 +53,7 @@ DECLARE_TOAST(pg_aggregate, 4159, 4160); DECLARE_TOAST(pg_attrdef, 2830, 2831); DECLARE_TOAST(pg_constraint, 2832, 2833); DECLARE_TOAST(pg_default_acl, 4143, 4144); +DECLARE_TOAST(pg_depend, 8888, 8889); DECLARE_TOAST(pg_description, 2834, 2835); DECLARE_TOAST(pg_event_trigger, 4145, 4146); DECLARE_TOAST(pg_extension, 4147, 4148); diff --git a/src/test/regress/expected/misc_sanity.out b/src/test/regress/expected/misc_sanity.out index 8538173ff8cd..d40afeef784c 100644 --- a/src/test/regress/expected/misc_sanity.out +++ b/src/test/regress/expected/misc_sanity.out @@ -18,8 +18,8 @@ WHERE refclassid = 0 OR refobjid = 0 OR deptype NOT IN ('a', 'e', 'i', 'n', 'p') OR (deptype != 'p' AND (classid = 0 OR objid = 0)) OR (deptype = 'p' AND (classid != 0 OR objid != 0 OR objsubid != 0)); - classid | objid | objsubid | refclassid | refobjid | refobjsubid | deptype ----------+-------+----------+------------+----------+-------------+--------- + classid | objid | objsubid | refclassid | refobjid | refobjsubid | deptype | refobjversion +---------+-------+----------+------------+----------+-------------+---------+--------------- (0 rows) -- **************** pg_shdepend **************** From 257836a75585934cc05ed7a80bccf8190d41e056 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Mon, 2 Nov 2020 19:50:45 +1300 Subject: [PATCH 419/589] Track collation versions for indexes. Record the current version of dependent collations in pg_depend when creating or rebuilding an index. When accessing the index later, warn that the index may be corrupted if the current version doesn't match. Thanks to Douglas Doole, Peter Eisentraut, Christoph Berg, Laurenz Albe, Michael Paquier, Robert Haas, Tom Lane and others for very helpful discussion. Author: Thomas Munro Author: Julien Rouhaud Reviewed-by: Peter Eisentraut (earlier versions) Discussion: https://postgr.es/m/CAEepm%3D0uEQCpfq_%2BLYFBdArCe4Ot98t1aR4eYiYTe%3DyavQygiQ%40mail.gmail.com --- doc/src/sgml/catalogs.sgml | 3 +- doc/src/sgml/charset.sgml | 38 ++++ doc/src/sgml/func.sgml | 4 +- doc/src/sgml/ref/alter_index.sgml | 15 ++ doc/src/sgml/ref/pgupgrade.sgml | 15 ++ doc/src/sgml/ref/reindex.sgml | 9 + src/backend/catalog/dependency.c | 182 +++++++++++++--- src/backend/catalog/heap.c | 7 +- src/backend/catalog/index.c | 197 +++++++++++++++-- src/backend/catalog/pg_constraint.c | 2 +- src/backend/catalog/pg_depend.c | 46 +++- src/backend/catalog/pg_type.c | 60 ++++++ src/backend/commands/collationcmds.c | 16 +- src/backend/commands/tablecmds.c | 31 +++ src/backend/nodes/copyfuncs.c | 1 + src/backend/optimizer/util/plancat.c | 9 + src/backend/parser/gram.y | 8 + src/backend/utils/adt/pg_locale.c | 46 +++- src/backend/utils/adt/pg_upgrade_support.c | 1 + src/backend/utils/cache/relcache.c | 2 + src/bin/pg_dump/pg_backup.h | 1 + src/bin/pg_dump/pg_dump.c | 182 +++++++++++++++- src/bin/pg_dump/pg_dump.h | 2 + src/bin/pg_upgrade/dump.c | 4 +- src/bin/pg_upgrade/option.c | 7 + src/bin/pg_upgrade/pg_upgrade.h | 1 + src/bin/psql/tab-complete.c | 29 ++- src/include/catalog/catversion.h | 2 +- src/include/catalog/dependency.h | 25 ++- src/include/catalog/index.h | 3 + src/include/catalog/pg_depend.h | 3 +- src/include/catalog/pg_type.h | 2 + src/include/nodes/parsenodes.h | 4 +- src/include/utils/pg_locale.h | 2 +- src/include/utils/rel.h | 1 + src/test/Makefile | 3 +- src/test/locale/.gitignore | 1 + src/test/locale/Makefile | 7 + src/test/locale/t/001_index.pl | 67 ++++++ .../regress/expected/collate.icu.utf8.out | 201 ++++++++++++++++++ src/test/regress/expected/create_index.out | 8 +- src/test/regress/sql/collate.icu.utf8.sql | 132 ++++++++++++ src/tools/pgindent/typedefs.list | 2 + 43 files changed, 1287 insertions(+), 94 deletions(-) create mode 100644 src/test/locale/t/001_index.pl diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index c3f324f05eb4..5fb9dca4258c 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -3308,7 +3308,8 @@ SCRAM-SHA-256$<iteration count>:&l refobjversion text - An optional version for the referenced object. + An optional version for the referenced object. Currently used for + indexes' collations (see ). diff --git a/doc/src/sgml/charset.sgml b/doc/src/sgml/charset.sgml index 2745b4441765..832a701523de 100644 --- a/doc/src/sgml/charset.sgml +++ b/doc/src/sgml/charset.sgml @@ -948,6 +948,44 @@ CREATE COLLATION ignore_accents (provider = icu, locale = 'und-u-ks-level1-kc-tr + + + Collation Versions + + + The sort order defined by a collation is not necessarily fixed over time. + PostgreSQL relies on external libraries that + are subject to operating system upgrades, and can also differ between + servers involved in binary replication and file-system-level migration. + Persistent data structures such as B-trees that depend on sort order might + be corrupted by any resulting change. + PostgreSQL defends against this by recording the + current version of each referenced collation for any index that depends on + it in the + pg_depend + catalog, if the collation provider makes that information available. If the + provider later begins to report a different version, a warning will be + issued when the index is accessed, until either the + command or the + command is used to update the version. + + + Version information is available from the + icu provider on all operating systems. For the + libc provider, versions are currently only available + on systems using the GNU C library (most Linux systems) and Windows. + + + + + When using the GNU C library for collations, the C library's version + is used as a proxy for the collation version. Many Linux distributions + change collation definitions only when upgrading the C library, but this + approach is imperfect as maintainers are free to back-port newer + collation definitions to older C library releases. + + + diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 0398b4909bf8..bf6004f321f6 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25444,7 +25444,9 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); Returns the actual version of the collation object as it is currently - installed in the operating system. + installed in the operating system. null is returned + on operating systems where PostgreSQL + doesn't have support for versions. diff --git a/doc/src/sgml/ref/alter_index.sgml b/doc/src/sgml/ref/alter_index.sgml index 793119d2fc1a..214005a86c5f 100644 --- a/doc/src/sgml/ref/alter_index.sgml +++ b/doc/src/sgml/ref/alter_index.sgml @@ -25,6 +25,7 @@ ALTER INDEX [ IF EXISTS ] name RENA ALTER INDEX [ IF EXISTS ] name SET TABLESPACE tablespace_name ALTER INDEX name ATTACH PARTITION index_name ALTER INDEX name DEPENDS ON EXTENSION extension_name +ALTER INDEX name ALTER COLLATION collation_name REFRESH VERSION ALTER INDEX [ IF EXISTS ] name SET ( storage_parameter [= value] [, ... ] ) ALTER INDEX [ IF EXISTS ] name RESET ( storage_parameter [, ... ] ) ALTER INDEX [ IF EXISTS ] name ALTER [ COLUMN ] column_number @@ -112,6 +113,20 @@ ALTER INDEX ALL IN TABLESPACE name + + ALTER COLLATION collation_name REFRESH VERSION + + + Silences warnings about mismatched collation versions, by declaring + that the index is compatible with the current collation definition. + Be aware that incorrect use of this command can hide index corruption. + If you don't know whether a collation's definition has changed + incompatibly, is a safe alternative. + See for more information. + + + + SET ( storage_parameter [= value] [, ... ] ) diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml index b59c5697a36c..92e1d09a55ca 100644 --- a/doc/src/sgml/ref/pgupgrade.sgml +++ b/doc/src/sgml/ref/pgupgrade.sgml @@ -215,6 +215,21 @@ PostgreSQL documentation + + + + + When upgrading indexes from releases before 14 that didn't track + collation versions, pg_upgrade + assumes by default that the upgraded indexes are compatible with the + currently installed versions of relevant collations (see + ). Specify + to mark + them as needing to be rebuilt instead. + + + + diff --git a/doc/src/sgml/ref/reindex.sgml b/doc/src/sgml/ref/reindex.sgml index fa43e3a97202..f6d425a6910e 100644 --- a/doc/src/sgml/ref/reindex.sgml +++ b/doc/src/sgml/ref/reindex.sgml @@ -38,6 +38,15 @@ REINDEX [ ( option [, ...] ) ] { IN several scenarios in which to use REINDEX: + + + The index depends on the sort order of a collation, and the definition + of the collation has changed. This can cause index scans to fail to + find keys that are present. See for + more information. + + + An index has become corrupted, and no longer contains valid diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c index 1a927377e739..b0d037600e93 100644 --- a/src/backend/catalog/dependency.c +++ b/src/backend/catalog/dependency.c @@ -76,6 +76,7 @@ #include "rewrite/rewriteRemove.h" #include "storage/lmgr.h" #include "utils/acl.h" +#include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/guc.h" #include "utils/lsyscache.h" @@ -434,6 +435,84 @@ performMultipleDeletions(const ObjectAddresses *objects, table_close(depRel, RowExclusiveLock); } +/* + * Call a function for all objects that 'object' depend on. If the function + * returns true, refobjversion will be updated in the catalog. + */ +void +visitDependenciesOf(const ObjectAddress *object, + VisitDependenciesOfCB callback, + void *userdata) +{ + Relation depRel; + ScanKeyData key[3]; + SysScanDesc scan; + HeapTuple tup; + ObjectAddress otherObject; + + ScanKeyInit(&key[0], + Anum_pg_depend_classid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(object->classId)); + ScanKeyInit(&key[1], + Anum_pg_depend_objid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(object->objectId)); + ScanKeyInit(&key[2], + Anum_pg_depend_objsubid, + BTEqualStrategyNumber, F_INT4EQ, + Int32GetDatum(object->objectSubId)); + + depRel = table_open(DependRelationId, RowExclusiveLock); + scan = systable_beginscan(depRel, DependDependerIndexId, true, + NULL, 3, key); + + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + Form_pg_depend foundDep = (Form_pg_depend) GETSTRUCT(tup); + char *new_version; + Datum depversion; + bool isnull; + + otherObject.classId = foundDep->refclassid; + otherObject.objectId = foundDep->refobjid; + otherObject.objectSubId = foundDep->refobjsubid; + + depversion = heap_getattr(tup, Anum_pg_depend_refobjversion, + RelationGetDescr(depRel), &isnull); + + /* Does the callback want to update the version? */ + if (callback(&otherObject, + isnull ? NULL : TextDatumGetCString(depversion), + &new_version, + userdata)) + { + Datum values[Natts_pg_depend]; + bool nulls[Natts_pg_depend]; + bool replaces[Natts_pg_depend]; + + memset(values, 0, sizeof(values)); + memset(nulls, false, sizeof(nulls)); + memset(replaces, false, sizeof(replaces)); + + if (new_version) + values[Anum_pg_depend_refobjversion - 1] = + CStringGetTextDatum(new_version); + else + nulls[Anum_pg_depend_refobjversion - 1] = true; + replaces[Anum_pg_depend_refobjversion - 1] = true; + + tup = heap_modify_tuple(tup, RelationGetDescr(depRel), values, + nulls, replaces); + CatalogTupleUpdate(depRel, &tup->t_self, tup); + + heap_freetuple(tup); + } + } + systable_endscan(scan); + table_close(depRel, RowExclusiveLock); +} + /* * findDependentObjects - find all objects that depend on 'object' * @@ -1566,6 +1645,38 @@ ReleaseDeletionLock(const ObjectAddress *object) AccessExclusiveLock); } +/* + * Record dependencies on a list of collations, optionally with their current + * version. + */ +void +recordDependencyOnCollations(ObjectAddress *myself, + List *collations, + bool record_version) +{ + ObjectAddresses *addrs; + ListCell *lc; + + if (list_length(collations) == 0) + return; + + addrs = new_object_addresses(); + foreach(lc, collations) + { + ObjectAddress referenced; + + ObjectAddressSet(referenced, CollationRelationId, lfirst_oid(lc)); + + add_exact_object_address(&referenced, addrs); + } + + eliminate_duplicate_dependencies(addrs); + recordMultipleDependencies(myself, addrs->refs, addrs->numrefs, + DEPENDENCY_NORMAL, record_version); + + free_object_addresses(addrs); +} + /* * recordDependencyOnExpr - find expression dependencies * @@ -1602,8 +1713,8 @@ recordDependencyOnExpr(const ObjectAddress *depender, recordMultipleDependencies(depender, context.addrs->refs, context.addrs->numrefs, - NULL, - behavior); + behavior, + false); free_object_addresses(context.addrs); } @@ -1630,7 +1741,8 @@ recordDependencyOnSingleRelExpr(const ObjectAddress *depender, Node *expr, Oid relId, DependencyType behavior, DependencyType self_behavior, - bool reverse_self) + bool reverse_self, + bool record_version) { find_expr_references_context context; RangeTblEntry rte; @@ -1691,8 +1803,8 @@ recordDependencyOnSingleRelExpr(const ObjectAddress *depender, recordMultipleDependencies(depender, self_addrs->refs, self_addrs->numrefs, - NULL, - self_behavior); + self_behavior, + record_version); else { /* Can't use recordMultipleDependencies, so do it the hard way */ @@ -1713,8 +1825,8 @@ recordDependencyOnSingleRelExpr(const ObjectAddress *depender, recordMultipleDependencies(depender, context.addrs->refs, context.addrs->numrefs, - NULL, - behavior); + behavior, + record_version); free_object_addresses(context.addrs); } @@ -1770,6 +1882,29 @@ find_expr_references_walker(Node *node, /* If it's a plain relation, reference this column */ add_object_address(OCLASS_CLASS, rte->relid, var->varattno, context->addrs); + + /* Top-level collation if valid */ + if (OidIsValid(var->varcollid)) + add_object_address(OCLASS_COLLATION, var->varcollid, 0, + context->addrs); + /* Otherwise, it may be a type with internal collations */ + else if (var->vartype >= FirstNormalObjectId) + { + List *collations; + ListCell *lc; + + collations = GetTypeCollations(var->vartype); + + foreach(lc, collations) + { + Oid coll = lfirst_oid(lc); + + if (OidIsValid(coll)) + add_object_address(OCLASS_COLLATION, + lfirst_oid(lc), 0, + context->addrs); + } + } } /* @@ -1794,11 +1929,9 @@ find_expr_references_walker(Node *node, /* * We must also depend on the constant's collation: it could be * different from the datatype's, if a CollateExpr was const-folded to - * a simple constant. However we can save work in the most common - * case where the collation is "default", since we know that's pinned. + * a simple constant. */ - if (OidIsValid(con->constcollid) && - con->constcollid != DEFAULT_COLLATION_OID) + if (OidIsValid(con->constcollid)) add_object_address(OCLASS_COLLATION, con->constcollid, 0, context->addrs); @@ -1887,8 +2020,7 @@ find_expr_references_walker(Node *node, add_object_address(OCLASS_TYPE, param->paramtype, 0, context->addrs); /* and its collation, just as for Consts */ - if (OidIsValid(param->paramcollid) && - param->paramcollid != DEFAULT_COLLATION_OID) + if (OidIsValid(param->paramcollid)) add_object_address(OCLASS_COLLATION, param->paramcollid, 0, context->addrs); } @@ -1975,8 +2107,7 @@ find_expr_references_walker(Node *node, add_object_address(OCLASS_TYPE, fselect->resulttype, 0, context->addrs); /* the collation might not be referenced anywhere else, either */ - if (OidIsValid(fselect->resultcollid) && - fselect->resultcollid != DEFAULT_COLLATION_OID) + if (OidIsValid(fselect->resultcollid)) add_object_address(OCLASS_COLLATION, fselect->resultcollid, 0, context->addrs); } @@ -2006,8 +2137,7 @@ find_expr_references_walker(Node *node, add_object_address(OCLASS_TYPE, relab->resulttype, 0, context->addrs); /* the collation might not be referenced anywhere else, either */ - if (OidIsValid(relab->resultcollid) && - relab->resultcollid != DEFAULT_COLLATION_OID) + if (OidIsValid(relab->resultcollid)) add_object_address(OCLASS_COLLATION, relab->resultcollid, 0, context->addrs); } @@ -2019,8 +2149,7 @@ find_expr_references_walker(Node *node, add_object_address(OCLASS_TYPE, iocoerce->resulttype, 0, context->addrs); /* the collation might not be referenced anywhere else, either */ - if (OidIsValid(iocoerce->resultcollid) && - iocoerce->resultcollid != DEFAULT_COLLATION_OID) + if (OidIsValid(iocoerce->resultcollid)) add_object_address(OCLASS_COLLATION, iocoerce->resultcollid, 0, context->addrs); } @@ -2032,8 +2161,7 @@ find_expr_references_walker(Node *node, add_object_address(OCLASS_TYPE, acoerce->resulttype, 0, context->addrs); /* the collation might not be referenced anywhere else, either */ - if (OidIsValid(acoerce->resultcollid) && - acoerce->resultcollid != DEFAULT_COLLATION_OID) + if (OidIsValid(acoerce->resultcollid)) add_object_address(OCLASS_COLLATION, acoerce->resultcollid, 0, context->addrs); /* fall through to examine arguments */ @@ -2121,8 +2249,7 @@ find_expr_references_walker(Node *node, if (OidIsValid(wc->endInRangeFunc)) add_object_address(OCLASS_PROC, wc->endInRangeFunc, 0, context->addrs); - if (OidIsValid(wc->inRangeColl) && - wc->inRangeColl != DEFAULT_COLLATION_OID) + if (OidIsValid(wc->inRangeColl)) add_object_address(OCLASS_COLLATION, wc->inRangeColl, 0, context->addrs); /* fall through to examine substructure */ @@ -2267,7 +2394,7 @@ find_expr_references_walker(Node *node, { Oid collid = lfirst_oid(ct); - if (OidIsValid(collid) && collid != DEFAULT_COLLATION_OID) + if (OidIsValid(collid)) add_object_address(OCLASS_COLLATION, collid, 0, context->addrs); } @@ -2289,7 +2416,7 @@ find_expr_references_walker(Node *node, { Oid collid = lfirst_oid(ct); - if (OidIsValid(collid) && collid != DEFAULT_COLLATION_OID) + if (OidIsValid(collid)) add_object_address(OCLASS_COLLATION, collid, 0, context->addrs); } @@ -2685,8 +2812,9 @@ record_object_address_dependencies(const ObjectAddress *depender, { eliminate_duplicate_dependencies(referenced); recordMultipleDependencies(depender, - referenced->refs, referenced->numrefs, NULL, - behavior); + referenced->refs, referenced->numrefs, + behavior, + false); } /* diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 9ccf378d45b8..4cd7d7693810 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -2336,7 +2336,7 @@ StoreAttrDefault(Relation rel, AttrNumber attnum, */ recordDependencyOnSingleRelExpr(&colobject, expr, RelationGetRelid(rel), DEPENDENCY_AUTO, - DEPENDENCY_AUTO, false); + DEPENDENCY_AUTO, false, false); } else { @@ -2346,7 +2346,7 @@ StoreAttrDefault(Relation rel, AttrNumber attnum, */ recordDependencyOnSingleRelExpr(&defobject, expr, RelationGetRelid(rel), DEPENDENCY_NORMAL, - DEPENDENCY_NORMAL, false); + DEPENDENCY_NORMAL, false, false); } /* @@ -3706,7 +3706,8 @@ StorePartitionKey(Relation rel, RelationGetRelid(rel), DEPENDENCY_NORMAL, DEPENDENCY_INTERNAL, - true /* reverse the self-deps */ ); + true /* reverse the self-deps */ , + false /* don't track versions */ ); /* * We must invalidate the relcache so that the next diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index a18cc7cad309..b88b4a1f12b6 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -53,6 +53,7 @@ #include "catalog/pg_trigger.h" #include "catalog/pg_type.h" #include "catalog/storage.h" +#include "commands/defrem.h" #include "commands/event_trigger.h" #include "commands/progress.h" #include "commands/tablecmds.h" @@ -75,6 +76,7 @@ #include "utils/guc.h" #include "utils/inval.h" #include "utils/lsyscache.h" +#include "utils/pg_locale.h" #include "utils/memutils.h" #include "utils/pg_rusage.h" #include "utils/rel.h" @@ -1020,6 +1022,8 @@ index_create(Relation heapRelation, ObjectAddress myself, referenced; ObjectAddresses *addrs; + List *colls = NIL, + *colls_no_version = NIL; ObjectAddressSet(myself, RelationRelationId, indexRelationId); @@ -1103,30 +1107,65 @@ index_create(Relation heapRelation, recordDependencyOn(&myself, &referenced, DEPENDENCY_PARTITION_SEC); } - /* placeholder for normal dependencies */ - addrs = new_object_addresses(); - - /* Store dependency on collations */ - - /* The default collation is pinned, so don't bother recording it */ + /* Get dependencies on collations for all index keys. */ for (i = 0; i < indexInfo->ii_NumIndexKeyAttrs; i++) { - if (OidIsValid(collationObjectId[i]) && - collationObjectId[i] != DEFAULT_COLLATION_OID) + Oid colloid = collationObjectId[i]; + + if (OidIsValid(colloid)) { - ObjectAddressSet(referenced, CollationRelationId, - collationObjectId[i]); - add_exact_object_address(&referenced, addrs); + Oid opclass = classObjectId[i]; + + /* + * The *_pattern_ops opclasses are special: they explicitly do + * not depend on collation order so we can save some effort. + * + * XXX With more analysis, we could also skip version tracking + * for some cases like hash indexes with deterministic + * collations, because they will never need to order strings. + */ + if (opclass == TEXT_BTREE_PATTERN_OPS_OID || + opclass == VARCHAR_BTREE_PATTERN_OPS_OID || + opclass == BPCHAR_BTREE_PATTERN_OPS_OID) + colls_no_version = lappend_oid(colls_no_version, colloid); + else + colls = lappend_oid(colls, colloid); + } + else + { + Form_pg_attribute att = TupleDescAttr(indexTupDesc, i); + + Assert(i < indexTupDesc->natts); + + /* + * Even though there is no top-level collation, there may be + * collations affecting ordering inside types, so look there + * too. + */ + colls = list_concat(colls, GetTypeCollations(att->atttypid)); } } + /* + * If we have anything in both lists, keep just the versioned one to + * avoid some duplication. + */ + if (colls_no_version != NIL && colls != NIL) + colls_no_version = list_difference_oid(colls_no_version, colls); + + /* Store the versioned and unversioned collation dependencies. */ + if (colls_no_version != NIL) + recordDependencyOnCollations(&myself, colls_no_version, false); + if (colls != NIL) + recordDependencyOnCollations(&myself, colls, true); + /* Store dependency on operator classes */ + addrs = new_object_addresses(); for (i = 0; i < indexInfo->ii_NumIndexKeyAttrs; i++) { ObjectAddressSet(referenced, OperatorClassRelationId, classObjectId[i]); add_exact_object_address(&referenced, addrs); } - record_object_address_dependencies(&myself, addrs, DEPENDENCY_NORMAL); free_object_addresses(addrs); @@ -1137,7 +1176,7 @@ index_create(Relation heapRelation, (Node *) indexInfo->ii_Expressions, heapRelationId, DEPENDENCY_NORMAL, - DEPENDENCY_AUTO, false); + DEPENDENCY_AUTO, false, true); } /* Store dependencies on anything mentioned in predicate */ @@ -1147,7 +1186,7 @@ index_create(Relation heapRelation, (Node *) indexInfo->ii_Predicate, heapRelationId, DEPENDENCY_NORMAL, - DEPENDENCY_AUTO, false); + DEPENDENCY_AUTO, false, true); } } else @@ -1226,6 +1265,129 @@ index_create(Relation heapRelation, return indexRelationId; } +typedef struct do_collation_version_check_context +{ + Oid relid; + List *warned_colls; +} do_collation_version_check_context; + +/* + * Raise a warning if the recorded and current collation version don't match. + * This is a callback for visitDependenciesOf(). + */ +static bool +do_collation_version_check(const ObjectAddress *otherObject, + const char *version, + char **new_version, + void *data) +{ + do_collation_version_check_context *context = data; + char *current_version; + + /* We only care about dependencies on collations with a version. */ + if (!version || otherObject->classId != CollationRelationId) + return false; + + /* Ask the provider for the current version. Give up if unsupported. */ + current_version = get_collation_version_for_oid(otherObject->objectId); + if (!current_version) + return false; + + /* + * We don't expect too many duplicates, but it's possible, and we don't + * want to generate duplicate warnings. + */ + if (list_member_oid(context->warned_colls, otherObject->objectId)) + return false; + + /* Do they match? */ + if (strcmp(current_version, version) != 0) + { + /* + * The version has changed, probably due to an OS/library upgrade or + * streaming replication between different OS/library versions. + */ + ereport(WARNING, + (errmsg("index \"%s\" depends on collation \"%s\" version \"%s\", but the current version is \"%s\"", + get_rel_name(context->relid), + get_collation_name(otherObject->objectId), + version, + current_version), + errdetail("The index may be corrupted due to changes in sort order."), + errhint("REINDEX to avoid the risk of corruption."))); + + /* Remember not to complain about this collation again. */ + context->warned_colls = lappend_oid(context->warned_colls, + otherObject->objectId); + } + + return false; +} + +/* index_check_collation_versions + * Check the collation version for all dependencies on the given index. + */ +void +index_check_collation_versions(Oid relid) +{ + do_collation_version_check_context context; + ObjectAddress object; + + /* + * The callback needs the relid for error messages, and some scratch space + * to avoid duplicate warnings. + */ + context.relid = relid; + context.warned_colls = NIL; + + object.classId = RelationRelationId; + object.objectId = relid; + object.objectSubId = 0; + + visitDependenciesOf(&object, &do_collation_version_check, &context); + + list_free(context.warned_colls); +} + +/* + * Update the version for collations. A callback for visitDependenciesOf(). + */ +static bool +do_collation_version_update(const ObjectAddress *otherObject, + const char *version, + char **new_version, + void *data) +{ + Oid *coll = data; + + /* We only care about dependencies on collations with versions. */ + if (!version || otherObject->classId != CollationRelationId) + return false; + + /* If we're only trying to update one collation, skip others. */ + if (OidIsValid(*coll) && otherObject->objectId != *coll) + return false; + + *new_version = get_collation_version_for_oid(otherObject->objectId); + + return true; +} + +/* + * Record the current versions of one or all collations that an index depends + * on. InvalidOid means all. + */ +void +index_update_collation_versions(Oid relid, Oid coll) +{ + ObjectAddress object; + + object.classId = RelationRelationId; + object.objectId = relid; + object.objectSubId = 0; + visitDependenciesOf(&object, &do_collation_version_update, &coll); +} + /* * index_concurrently_create_copy * @@ -1686,6 +1848,10 @@ index_concurrently_swap(Oid newIndexId, Oid oldIndexId, const char *oldName) changeDependenciesOf(RelationRelationId, oldIndexId, newIndexId); changeDependenciesOn(RelationRelationId, oldIndexId, newIndexId); + /* Now we have the old index's collation versions, so fix that. */ + CommandCounterIncrement(); + index_update_collation_versions(newIndexId, InvalidOid); + /* * Copy over statistics from old to new index */ @@ -3638,6 +3804,9 @@ reindex_index(Oid indexId, bool skip_constraint_checks, char persistence, /* Close rels, but keep locks */ index_close(iRel, NoLock); table_close(heapRelation, NoLock); + + /* Record the current versions of all depended-on collations. */ + index_update_collation_versions(indexId, InvalidOid); } /* diff --git a/src/backend/catalog/pg_constraint.c b/src/backend/catalog/pg_constraint.c index 0d70cb0c3c98..93774c9d21a5 100644 --- a/src/backend/catalog/pg_constraint.c +++ b/src/backend/catalog/pg_constraint.c @@ -362,7 +362,7 @@ CreateConstraintEntry(const char *constraintName, */ recordDependencyOnSingleRelExpr(&conobject, conExpr, relId, DEPENDENCY_NORMAL, - DEPENDENCY_NORMAL, false); + DEPENDENCY_NORMAL, false, true); } /* Post creation hook for new constraint */ diff --git a/src/backend/catalog/pg_depend.c b/src/backend/catalog/pg_depend.c index 09c30b13e884..25290c821fda 100644 --- a/src/backend/catalog/pg_depend.c +++ b/src/backend/catalog/pg_depend.c @@ -19,6 +19,7 @@ #include "access/table.h" #include "catalog/dependency.h" #include "catalog/indexing.h" +#include "catalog/pg_collation.h" #include "catalog/pg_constraint.h" #include "catalog/pg_depend.h" #include "catalog/pg_extension.h" @@ -27,6 +28,7 @@ #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" +#include "utils/pg_locale.h" #include "utils/rel.h" @@ -45,19 +47,24 @@ recordDependencyOn(const ObjectAddress *depender, const ObjectAddress *referenced, DependencyType behavior) { - recordMultipleDependencies(depender, referenced, 1, NULL, behavior); + recordMultipleDependencies(depender, referenced, 1, behavior, false); } /* * Record multiple dependencies (of the same kind) for a single dependent * object. This has a little less overhead than recording each separately. + * + * If record_version is true, then a record is added even if the referenced + * object is pinned, and the dependency version will be retrieved according to + * the referenced object kind. For now, only collation version is + * supported. */ void recordMultipleDependencies(const ObjectAddress *depender, const ObjectAddress *referenced, int nreferenced, - const char *version, - DependencyType behavior) + DependencyType behavior, + bool record_version) { Relation dependDesc; CatalogIndexState indstate; @@ -66,6 +73,7 @@ recordMultipleDependencies(const ObjectAddress *depender, max_slots, slot_init_count, slot_stored_count; + char *version = NULL; if (nreferenced <= 0) return; /* nothing to do */ @@ -96,12 +104,38 @@ recordMultipleDependencies(const ObjectAddress *depender, slot_init_count = 0; for (i = 0; i < nreferenced; i++, referenced++) { + bool ignore_systempin = false; + + if (record_version) + { + /* For now we only know how to deal with collations. */ + if (referenced->classId == CollationRelationId) + { + /* C and POSIX don't need version tracking. */ + if (referenced->objectId == C_COLLATION_OID || + referenced->objectId == POSIX_COLLATION_OID) + continue; + + version = get_collation_version_for_oid(referenced->objectId); + + /* + * Default collation is pinned, so we need to force recording + * the dependency to store the version. + */ + if (referenced->objectId == DEFAULT_COLLATION_OID) + ignore_systempin = true; + } + } + else + Assert(!version); + /* * If the referenced object is pinned by the system, there's no real - * need to record dependencies on it. This saves lots of space in - * pg_depend, so it's worth the time taken to check. + * need to record dependencies on it, unless we need to record a + * version. This saves lots of space in pg_depend, so it's worth the + * time taken to check. */ - if (isObjectPinned(referenced, dependDesc)) + if (!ignore_systempin && isObjectPinned(referenced, dependDesc)) continue; if (slot_init_count < max_slots) diff --git a/src/backend/catalog/pg_type.c b/src/backend/catalog/pg_type.c index 0b04dff7731c..44eed1a0b34a 100644 --- a/src/backend/catalog/pg_type.c +++ b/src/backend/catalog/pg_type.c @@ -15,6 +15,7 @@ #include "postgres.h" #include "access/htup_details.h" +#include "access/relation.h" #include "access/table.h" #include "access/xact.h" #include "catalog/binary_upgrade.h" @@ -512,6 +513,65 @@ TypeCreate(Oid newTypeOid, return address; } +/* + * Get a list of all distinct collations that the given type depends on. + */ +List * +GetTypeCollations(Oid typeoid) +{ + List *result = NIL; + HeapTuple tuple; + Form_pg_type typeTup; + + tuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(typeoid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for type %u", typeoid); + typeTup = (Form_pg_type) GETSTRUCT(tuple); + + if (OidIsValid(typeTup->typcollation)) + result = list_append_unique_oid(result, typeTup->typcollation); + else if (typeTup->typtype == TYPTYPE_COMPOSITE) + { + Relation rel = relation_open(typeTup->typrelid, AccessShareLock); + TupleDesc desc = RelationGetDescr(rel); + + for (int i = 0; i < RelationGetNumberOfAttributes(rel); i++) + { + Form_pg_attribute att = TupleDescAttr(desc, i); + + if (OidIsValid(att->attcollation)) + result = list_append_unique_oid(result, att->attcollation); + else + result = list_concat_unique_oid(result, + GetTypeCollations(att->atttypid)); + } + + relation_close(rel, NoLock); + } + else if (typeTup->typtype == TYPTYPE_DOMAIN) + { + Assert(OidIsValid(typeTup->typbasetype)); + + result = list_concat_unique_oid(result, + GetTypeCollations(typeTup->typbasetype)); + } + else if (typeTup->typtype == TYPTYPE_RANGE) + { + Oid rangeid = get_range_subtype(typeTup->oid); + + Assert(OidIsValid(rangeid)); + + result = list_concat_unique_oid(result, GetTypeCollations(rangeid)); + } + else if (OidIsValid(typeTup->typelem)) + result = list_concat_unique_oid(result, + GetTypeCollations(typeTup->typelem)); + + ReleaseSysCache(tuple); + + return result; +} + /* * GenerateTypeDependencies: build the dependencies needed for a type * diff --git a/src/backend/commands/collationcmds.c b/src/backend/commands/collationcmds.c index 5ad8886e60a4..7b31272734d0 100644 --- a/src/backend/commands/collationcmds.c +++ b/src/backend/commands/collationcmds.c @@ -270,23 +270,9 @@ Datum pg_collation_actual_version(PG_FUNCTION_ARGS) { Oid collid = PG_GETARG_OID(0); - HeapTuple tp; - char *collcollate; - char collprovider; char *version; - tp = SearchSysCache1(COLLOID, ObjectIdGetDatum(collid)); - if (!HeapTupleIsValid(tp)) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("collation with OID %u does not exist", collid))); - - collcollate = pstrdup(NameStr(((Form_pg_collation) GETSTRUCT(tp))->collcollate)); - collprovider = ((Form_pg_collation) GETSTRUCT(tp))->collprovider; - - ReleaseSysCache(tp); - - version = get_collation_actual_version(collprovider, collcollate); + version = get_collation_version_for_oid(collid); if (version) PG_RETURN_TEXT_P(cstring_to_text(version)); diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index df13b72974ff..1b105ba1c428 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -93,6 +93,7 @@ #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/partcache.h" +#include "utils/pg_locale.h" #include "utils/relcache.h" #include "utils/ruleutils.h" #include "utils/snapmgr.h" @@ -559,6 +560,7 @@ static void refuseDupeIndexAttach(Relation parentIdx, Relation partIdx, Relation partitionTbl); static List *GetParentedForeignKeyRefs(Relation partition); static void ATDetachCheckNoForeignKeyRefs(Relation partition); +static void ATExecAlterCollationRefreshVersion(Relation rel, List *coll); /* ---------------------------------------------------------------- @@ -3986,6 +3988,10 @@ AlterTableGetLockLevel(List *cmds) cmd_lockmode = AccessShareLock; break; + case AT_AlterCollationRefreshVersion: + cmd_lockmode = AccessExclusiveLock; + break; + default: /* oops */ elog(ERROR, "unrecognized alter table type: %d", (int) cmd->subtype); @@ -4160,6 +4166,12 @@ ATPrepCmd(List **wqueue, Relation rel, AlterTableCmd *cmd, /* This command never recurses */ pass = AT_PASS_MISC; break; + case AT_AlterCollationRefreshVersion: /* ALTER COLLATION ... REFRESH + * VERSION */ + ATSimplePermissions(rel, ATT_INDEX); + /* This command never recurses */ + pass = AT_PASS_MISC; + break; case AT_SetStorage: /* ALTER COLUMN SET STORAGE */ ATSimplePermissions(rel, ATT_TABLE | ATT_MATVIEW | ATT_FOREIGN_TABLE); ATSimpleRecursion(wqueue, rel, cmd, recurse, lockmode, context); @@ -4738,6 +4750,11 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, Assert(rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); ATExecDetachPartition(rel, ((PartitionCmd *) cmd->def)->name); break; + case AT_AlterCollationRefreshVersion: + /* ATPrepCmd ensured it must be an index */ + Assert(rel->rd_rel->relkind == RELKIND_INDEX); + ATExecAlterCollationRefreshVersion(rel, cmd->object); + break; default: /* oops */ elog(ERROR, "unrecognized alter table type: %d", (int) cmd->subtype); @@ -17582,3 +17599,17 @@ ATDetachCheckNoForeignKeyRefs(Relation partition) table_close(rel, NoLock); } } + +/* + * ALTER INDEX ... ALTER COLLATION ... REFRESH VERSION + * + * Update refobjversion to the current collation version by force. This clears + * warnings about version mismatches without the need to run REINDEX, + * potentially hiding corruption due to ordering changes. + */ +static void +ATExecAlterCollationRefreshVersion(Relation rel, List *coll) +{ + index_update_collation_versions(rel->rd_id, get_collation_oid(coll, false)); + CacheInvalidateRelcache(rel); +} diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index ac8b57109c56..530aac68a762 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -3215,6 +3215,7 @@ _copyAlterTableCmd(const AlterTableCmd *from) COPY_SCALAR_FIELD(subtype); COPY_STRING_FIELD(name); + COPY_NODE_FIELD(object); COPY_SCALAR_FIELD(num); COPY_NODE_FIELD(newowner); COPY_NODE_FIELD(def); diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index 4f0da51c2697..52c01eb86b17 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -28,6 +28,7 @@ #include "catalog/catalog.h" #include "catalog/dependency.h" #include "catalog/heap.h" +#include "catalog/index.h" #include "catalog/pg_am.h" #include "catalog/pg_proc.h" #include "catalog/pg_statistic_ext.h" @@ -198,6 +199,14 @@ get_relation_info(PlannerInfo *root, Oid relationObjectId, bool inhparent, indexRelation = index_open(indexoid, lmode); index = indexRelation->rd_index; + /* Warn if any dependent collations' versions have moved. */ + if (!IsSystemRelation(relation) && + !indexRelation->rd_version_checked) + { + index_check_collation_versions(indexoid); + indexRelation->rd_version_checked = true; + } + /* * Ignore invalid indexes, since they can't safely be used for * queries. Note that this is OK because the data structure we diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 60cf7242a308..357ab93fb650 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -2591,6 +2591,14 @@ alter_table_cmd: n->subtype = AT_NoForceRowSecurity; $$ = (Node *)n; } + /* ALTER INDEX ALTER COLLATION ... REFRESH VERSION */ + | ALTER COLLATION any_name REFRESH VERSION_P + { + AlterTableCmd *n = makeNode(AlterTableCmd); + n->subtype = AT_AlterCollationRefreshVersion; + n->object = $3; + $$ = (Node *)n; + } | alter_generic_options { AlterTableCmd *n = makeNode(AlterTableCmd); diff --git a/src/backend/utils/adt/pg_locale.c b/src/backend/utils/adt/pg_locale.c index 514e0fa0af86..3b0324ce18ab 100644 --- a/src/backend/utils/adt/pg_locale.c +++ b/src/backend/utils/adt/pg_locale.c @@ -57,7 +57,9 @@ #include "access/htup_details.h" #include "catalog/pg_collation.h" #include "catalog/pg_control.h" +#include "catalog/pg_database.h" #include "mb/pg_wchar.h" +#include "miscadmin.h" #include "utils/builtins.h" #include "utils/formatting.h" #include "utils/hsearch.h" @@ -139,6 +141,9 @@ static char *IsoLocaleName(const char *); /* MSVC specific */ static void icu_set_collation_attributes(UCollator *collator, const char *loc); #endif +static char *get_collation_actual_version(char collprovider, + const char *collcollate); + /* * pg_perm_setlocale * @@ -1630,7 +1635,7 @@ pg_newlocale_from_collation(Oid collid) * Get provider-specific collation version string for the given collation from * the operating system/library. */ -char * +static char * get_collation_actual_version(char collprovider, const char *collcollate) { char *collversion = NULL; @@ -1712,6 +1717,45 @@ get_collation_actual_version(char collprovider, const char *collcollate) return collversion; } +/* + * Get provider-specific collation version string for a given collation OID. + * Return NULL if the provider doesn't support versions. + */ +char * +get_collation_version_for_oid(Oid oid) +{ + HeapTuple tp; + char *version = NULL; + + Assert(oid != C_COLLATION_OID && oid != POSIX_COLLATION_OID); + + if (oid == DEFAULT_COLLATION_OID) + { + Form_pg_database dbform; + + tp = SearchSysCache1(DATABASEOID, ObjectIdGetDatum(MyDatabaseId)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for database %u", MyDatabaseId); + dbform = (Form_pg_database) GETSTRUCT(tp); + version = get_collation_actual_version(COLLPROVIDER_LIBC, + NameStr(dbform->datcollate)); + } + else + { + Form_pg_collation collform; + + tp = SearchSysCache1(COLLOID, ObjectIdGetDatum(oid)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for collation %u", oid); + collform = (Form_pg_collation) GETSTRUCT(tp); + version = get_collation_actual_version(collform->collprovider, + NameStr(collform->collcollate)); + } + + ReleaseSysCache(tp); + + return version; +} #ifdef USE_ICU /* diff --git a/src/backend/utils/adt/pg_upgrade_support.c b/src/backend/utils/adt/pg_upgrade_support.c index 14d9eb2b5b3d..8a5953881eeb 100644 --- a/src/backend/utils/adt/pg_upgrade_support.c +++ b/src/backend/utils/adt/pg_upgrade_support.c @@ -13,6 +13,7 @@ #include "catalog/binary_upgrade.h" #include "catalog/heap.h" +#include "catalog/index.h" #include "catalog/namespace.h" #include "catalog/pg_type.h" #include "commands/extension.h" diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 9224e2ffeda5..66393becfb68 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -42,6 +42,7 @@ #include "access/xact.h" #include "access/xlog.h" #include "catalog/catalog.h" +#include "catalog/index.h" #include "catalog/indexing.h" #include "catalog/namespace.h" #include "catalog/partition.h" @@ -5934,6 +5935,7 @@ load_relcache_init_file(bool shared) rel->rd_idattr = NULL; rel->rd_pubactions = NULL; rel->rd_statvalid = false; + rel->rd_version_checked = false; rel->rd_statlist = NIL; rel->rd_fkeyvalid = false; rel->rd_fkeylist = NIL; diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 6c7931916414..4d49d2b96af0 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -179,6 +179,7 @@ typedef struct _dumpOptions int sequence_data; /* dump sequence data even in schema-only mode */ int do_nothing; + int coll_unknown; } DumpOptions; /* diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 9851022a53d4..cf89a7ec6d77 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -46,6 +46,7 @@ #include "catalog/pg_attribute_d.h" #include "catalog/pg_cast_d.h" #include "catalog/pg_class_d.h" +#include "catalog/pg_collation_d.h" #include "catalog/pg_default_acl_d.h" #include "catalog/pg_largeobject_d.h" #include "catalog/pg_largeobject_metadata_d.h" @@ -285,6 +286,9 @@ static void binary_upgrade_extension_member(PQExpBuffer upgrade_buffer, static const char *getAttrName(int attrnum, TableInfo *tblInfo); static const char *fmtCopyColumnList(const TableInfo *ti, PQExpBuffer buffer); static bool nonemptyReloptions(const char *reloptions); +static void appendIndexCollationVersion(PQExpBuffer buffer, IndxInfo *indxinfo, + int enc, bool coll_unknown, + Archive *fount); static void appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions, const char *prefix, Archive *fout); static char *get_synchronized_snapshot(Archive *fout); @@ -385,6 +389,7 @@ main(int argc, char **argv) {"on-conflict-do-nothing", no_argument, &dopt.do_nothing, 1}, {"rows-per-insert", required_argument, NULL, 10}, {"include-foreign-data", required_argument, NULL, 11}, + {"index-collation-versions-unknown", no_argument, &dopt.coll_unknown, 1}, {NULL, 0, NULL, 0} }; @@ -712,6 +717,10 @@ main(int argc, char **argv) if (archiveFormat != archDirectory && numWorkers > 1) fatal("parallel backup only supported by the directory format"); + /* Unknown collation versions only relevant in binary upgrade mode */ + if (dopt.coll_unknown && !dopt.binary_upgrade) + fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); + /* Open the output file */ fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, archiveMode, setupDumpWorker); @@ -7031,7 +7040,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) i_tablespace, i_indreloptions, i_indstatcols, - i_indstatvals; + i_indstatvals, + i_inddependcollnames, + i_inddependcollversions; int ntups; for (i = 0; i < numTables; i++) @@ -7067,7 +7078,64 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) * is not. */ resetPQExpBuffer(query); - if (fout->remoteVersion >= 110000) + if (fout->remoteVersion >= 140000) + { + appendPQExpBuffer(query, + "SELECT t.tableoid, t.oid, " + "t.relname AS indexname, " + "inh.inhparent AS parentidx, " + "pg_catalog.pg_get_indexdef(i.indexrelid) AS indexdef, " + "i.indnkeyatts AS indnkeyatts, " + "i.indnatts AS indnatts, " + "i.indkey, i.indisclustered, " + "i.indisreplident, " + "c.contype, c.conname, " + "c.condeferrable, c.condeferred, " + "c.tableoid AS contableoid, " + "c.oid AS conoid, " + "pg_catalog.pg_get_constraintdef(c.oid, false) AS condef, " + "(SELECT spcname FROM pg_catalog.pg_tablespace s WHERE s.oid = t.reltablespace) AS tablespace, " + "t.reloptions AS indreloptions, " + "(SELECT pg_catalog.array_agg(attnum ORDER BY attnum) " + " FROM pg_catalog.pg_attribute " + " WHERE attrelid = i.indexrelid AND " + " attstattarget >= 0) AS indstatcols," + "(SELECT pg_catalog.array_agg(attstattarget ORDER BY attnum) " + " FROM pg_catalog.pg_attribute " + " WHERE attrelid = i.indexrelid AND " + " attstattarget >= 0) AS indstatvals, " + "(SELECT pg_catalog.array_agg(quote_ident(ns.nspname) || '.' || quote_ident(c.collname) ORDER BY refobjid) " + " FROM pg_catalog.pg_depend d " + " JOIN pg_catalog.pg_collation c ON (c.oid = d.refobjid) " + " JOIN pg_catalog.pg_namespace ns ON (c.collnamespace = ns.oid) " + " WHERE d.classid = 'pg_catalog.pg_class'::regclass AND " + " d.objid = i.indexrelid AND " + " d.objsubid = 0 AND " + " d.refclassid = 'pg_catalog.pg_collation'::regclass AND " + " d.refobjversion IS NOT NULL) AS inddependcollnames, " + "(SELECT pg_catalog.array_agg(quote_literal(refobjversion) ORDER BY refobjid) " + " FROM pg_catalog.pg_depend " + " WHERE classid = 'pg_catalog.pg_class'::regclass AND " + " objid = i.indexrelid AND " + " objsubid = 0 AND " + " refclassid = 'pg_catalog.pg_collation'::regclass AND " + " refobjversion IS NOT NULL) AS inddependcollversions " + "FROM pg_catalog.pg_index i " + "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " + "JOIN pg_catalog.pg_class t2 ON (t2.oid = i.indrelid) " + "LEFT JOIN pg_catalog.pg_constraint c " + "ON (i.indrelid = c.conrelid AND " + "i.indexrelid = c.conindid AND " + "c.contype IN ('p','u','x')) " + "LEFT JOIN pg_catalog.pg_inherits inh " + "ON (inh.inhrelid = indexrelid) " + "WHERE i.indrelid = '%u'::pg_catalog.oid " + "AND (i.indisvalid OR t2.relkind = 'p') " + "AND i.indisready " + "ORDER BY indexname", + tbinfo->dobj.catId.oid); + } + else if (fout->remoteVersion >= 110000) { appendPQExpBuffer(query, "SELECT t.tableoid, t.oid, " @@ -7092,7 +7160,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) "(SELECT pg_catalog.array_agg(attstattarget ORDER BY attnum) " " FROM pg_catalog.pg_attribute " " WHERE attrelid = i.indexrelid AND " - " attstattarget >= 0) AS indstatvals " + " attstattarget >= 0) AS indstatvals, " + "'{}' AS inddependcollnames, " + "'{}' AS inddependcollversions " "FROM pg_catalog.pg_index i " "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " "JOIN pg_catalog.pg_class t2 ON (t2.oid = i.indrelid) " @@ -7131,7 +7201,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) "(SELECT spcname FROM pg_catalog.pg_tablespace s WHERE s.oid = t.reltablespace) AS tablespace, " "t.reloptions AS indreloptions, " "'' AS indstatcols, " - "'' AS indstatvals " + "'' AS indstatvals, " + "'{}' AS inddependcollnames, " + "'{}' AS inddependcollversions " "FROM pg_catalog.pg_index i " "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " "LEFT JOIN pg_catalog.pg_constraint c " @@ -7166,7 +7238,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) "(SELECT spcname FROM pg_catalog.pg_tablespace s WHERE s.oid = t.reltablespace) AS tablespace, " "t.reloptions AS indreloptions, " "'' AS indstatcols, " - "'' AS indstatvals " + "'' AS indstatvals, " + "'{}' AS inddependcollnames, " + "'{}' AS inddependcollversions " "FROM pg_catalog.pg_index i " "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " "LEFT JOIN pg_catalog.pg_constraint c " @@ -7197,7 +7271,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) "(SELECT spcname FROM pg_catalog.pg_tablespace s WHERE s.oid = t.reltablespace) AS tablespace, " "t.reloptions AS indreloptions, " "'' AS indstatcols, " - "'' AS indstatvals " + "'' AS indstatvals, " + "'{}' AS inddependcollnames, " + "'{}' AS inddependcollversions " "FROM pg_catalog.pg_index i " "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " "LEFT JOIN pg_catalog.pg_depend d " @@ -7231,7 +7307,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) "(SELECT spcname FROM pg_catalog.pg_tablespace s WHERE s.oid = t.reltablespace) AS tablespace, " "null AS indreloptions, " "'' AS indstatcols, " - "'' AS indstatvals " + "'' AS indstatvals, " + "'{}' AS inddependcollnames, " + "'{}' AS inddependcollversions " "FROM pg_catalog.pg_index i " "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " "LEFT JOIN pg_catalog.pg_depend d " @@ -7271,6 +7349,8 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) i_indreloptions = PQfnumber(res, "indreloptions"); i_indstatcols = PQfnumber(res, "indstatcols"); i_indstatvals = PQfnumber(res, "indstatvals"); + i_inddependcollnames = PQfnumber(res, "inddependcollnames"); + i_inddependcollversions = PQfnumber(res, "inddependcollversions"); tbinfo->indexes = indxinfo = (IndxInfo *) pg_malloc(ntups * sizeof(IndxInfo)); @@ -7296,6 +7376,8 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) indxinfo[j].indreloptions = pg_strdup(PQgetvalue(res, j, i_indreloptions)); indxinfo[j].indstatcols = pg_strdup(PQgetvalue(res, j, i_indstatcols)); indxinfo[j].indstatvals = pg_strdup(PQgetvalue(res, j, i_indstatvals)); + indxinfo[j].inddependcollnames = pg_strdup(PQgetvalue(res, j, i_inddependcollnames)); + indxinfo[j].inddependcollversions = pg_strdup(PQgetvalue(res, j, i_inddependcollversions)); indxinfo[j].indkeys = (Oid *) pg_malloc(indxinfo[j].indnattrs * sizeof(Oid)); parseOidArray(PQgetvalue(res, j, i_indkey), indxinfo[j].indkeys, indxinfo[j].indnattrs); @@ -16362,7 +16444,8 @@ dumpIndex(Archive *fout, IndxInfo *indxinfo) /* * If there's an associated constraint, don't dump the index per se, but - * do dump any comment for it. (This is safe because dependency ordering + * do dump any comment, or in binary upgrade mode dependency on a + * collation version for it. (This is safe because dependency ordering * will have ensured the constraint is emitted first.) Note that the * emitted comment has to be shown as depending on the constraint, not the * index, in such cases. @@ -16429,6 +16512,10 @@ dumpIndex(Archive *fout, IndxInfo *indxinfo) "pg_catalog.pg_class", "INDEX", qqindxname); + if (dopt->binary_upgrade) + appendIndexCollationVersion(q, indxinfo, fout->encoding, + dopt->coll_unknown, fout); + /* If the index defines identity, we need to record that. */ if (indxinfo->indisreplident) { @@ -16457,6 +16544,21 @@ dumpIndex(Archive *fout, IndxInfo *indxinfo) if (indstatvalsarray) free(indstatvalsarray); } + else if (dopt->binary_upgrade) + { + appendIndexCollationVersion(q, indxinfo, fout->encoding, + dopt->coll_unknown, fout); + + if (indxinfo->dobj.dump & DUMP_COMPONENT_DEFINITION) + ArchiveEntry(fout, indxinfo->dobj.catId, indxinfo->dobj.dumpId, + ARCHIVE_OPTS(.tag = indxinfo->dobj.name, + .namespace = tbinfo->dobj.namespace->dobj.name, + .tablespace = indxinfo->tablespace, + .owner = tbinfo->rolname, + .description = "INDEX", + .section = SECTION_POST_DATA, + .createStmt = q->data)); + } /* Dump Index Comments */ if (indxinfo->dobj.dump & DUMP_COMPONENT_COMMENT) @@ -18441,6 +18543,70 @@ nonemptyReloptions(const char *reloptions) return (reloptions != NULL && strlen(reloptions) > 2); } +/* + * Generate UPDATE statements to import the collation versions into the new + * cluster, during a binary upgrade. + */ +static void +appendIndexCollationVersion(PQExpBuffer buffer, IndxInfo *indxinfo, int enc, + bool coll_unknown, Archive *fout) +{ + char *inddependcollnames = indxinfo->inddependcollnames; + char *inddependcollversions = indxinfo->inddependcollversions; + char **inddependcollnamesarray; + char **inddependcollversionsarray; + int ninddependcollnames; + int ninddependcollversions; + + /* + * By default, the new cluster's index will have pg_depends rows with + * current collation versions, meaning that we assume the index isn't + * corrupted if importing from a release that didn't record versions. + * However, if --index-collation-versions-unknown was passed in, then we + * assume such indexes might be corrupted, and clobber versions with + * 'unknown' to trigger version warnings. + */ + if (coll_unknown) + { + appendPQExpBuffer(buffer, + "\n-- For binary upgrade, clobber new index's collation versions\n"); + appendPQExpBuffer(buffer, + "UPDATE pg_catalog.pg_depend SET refobjversion = 'unknown' WHERE objid = '%u'::pg_catalog.oid AND refclassid = 'pg_catalog.pg_collation'::regclass AND refobjversion IS NOT NULL;\n", + indxinfo->dobj.catId.oid); + } + + /* Restore the versions that were recorded by the old cluster (if any). */ + parsePGArray(inddependcollnames, + &inddependcollnamesarray, + &ninddependcollnames); + parsePGArray(inddependcollversions, + &inddependcollversionsarray, + &ninddependcollversions); + Assert(ninddependcollnames == ninddependcollversions); + + if (ninddependcollnames > 0) + appendPQExpBufferStr(buffer, + "\n-- For binary upgrade, restore old index's collation versions\n"); + for (int i = 0; i < ninddependcollnames; i++) + { + /* + * Import refobjversion from the old cluster, being careful to resolve + * the collation OID by name in the new cluster. + */ + appendPQExpBuffer(buffer, + "UPDATE pg_catalog.pg_depend SET refobjversion = %s WHERE objid = '%u'::pg_catalog.oid AND refclassid = 'pg_catalog.pg_collation'::regclass AND refobjversion IS NOT NULL AND refobjid = ", + inddependcollversionsarray[i], + indxinfo->dobj.catId.oid); + appendStringLiteralAH(buffer,inddependcollnamesarray[i], fout); + appendPQExpBuffer(buffer, "::regcollation;\n"); + } + + if (inddependcollnamesarray) + free(inddependcollnamesarray); + if (inddependcollversionsarray) + free(inddependcollversionsarray); +} + /* * Format a reloptions array and append it to the given buffer. * diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index e0b42e83912f..317bb839702e 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -366,6 +366,8 @@ typedef struct _indxInfo int indnattrs; /* total number of index attributes */ Oid *indkeys; /* In spite of the name 'indkeys' this field * contains both key and nonkey attributes */ + char *inddependcollnames; /* FQ names of depended-on collations */ + char *inddependcollversions; /* versions of the above */ bool indisclustered; bool indisreplident; Oid parentidx; /* if a partition, parent index OID */ diff --git a/src/bin/pg_upgrade/dump.c b/src/bin/pg_upgrade/dump.c index 4d730adfe24e..20e73be36153 100644 --- a/src/bin/pg_upgrade/dump.c +++ b/src/bin/pg_upgrade/dump.c @@ -52,9 +52,11 @@ generate_old_dump(void) parallel_exec_prog(log_file_name, NULL, "\"%s/pg_dump\" %s --schema-only --quote-all-identifiers " - "--binary-upgrade --format=custom %s --file=\"%s\" %s", + "--binary-upgrade --format=custom %s %s --file=\"%s\" %s", new_cluster.bindir, cluster_conn_opts(&old_cluster), log_opts.verbose ? "--verbose" : "", + user_opts.ind_coll_unknown ? + "--index-collation-versions-unknown" : "", sql_file_name, escaped_connstr.data); termPQExpBuffer(&escaped_connstr); diff --git a/src/bin/pg_upgrade/option.c b/src/bin/pg_upgrade/option.c index aca1ee8b48d8..548d648e8c4e 100644 --- a/src/bin/pg_upgrade/option.c +++ b/src/bin/pg_upgrade/option.c @@ -56,6 +56,7 @@ parseCommandLine(int argc, char *argv[]) {"socketdir", required_argument, NULL, 's'}, {"verbose", no_argument, NULL, 'v'}, {"clone", no_argument, NULL, 1}, + {"index-collation-versions-unknown", no_argument, NULL, 2}, {NULL, 0, NULL, 0} }; @@ -203,6 +204,10 @@ parseCommandLine(int argc, char *argv[]) user_opts.transfer_mode = TRANSFER_MODE_CLONE; break; + case 2: + user_opts.ind_coll_unknown = true; + break; + default: fprintf(stderr, _("Try \"%s --help\" for more information.\n"), os_info.progname); @@ -307,6 +312,8 @@ usage(void) printf(_(" -v, --verbose enable verbose internal logging\n")); printf(_(" -V, --version display version information, then exit\n")); printf(_(" --clone clone instead of copying files to new cluster\n")); + printf(_(" --index-collation-versions-unknown\n")); + printf(_(" mark text indexes as needing to be rebuilt\n")); printf(_(" -?, --help show this help, then exit\n")); printf(_("\n" "Before running pg_upgrade you must:\n" diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index 8b90cefbe098..19c64513b06a 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -292,6 +292,7 @@ typedef struct transferMode transfer_mode; /* copy files or link them? */ int jobs; /* number of processes/threads to use */ char *socketdir; /* directory to use for Unix sockets */ + bool ind_coll_unknown; /* mark unknown index collation versions */ } UserOpts; typedef struct diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index b2b4f1fd4d13..5238a960f7e5 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -45,6 +45,7 @@ #include "catalog/pg_am_d.h" #include "catalog/pg_class_d.h" +#include "catalog/pg_collation_d.h" #include "common.h" #include "libpq-fe.h" #include "pqexpbuffer.h" @@ -820,6 +821,20 @@ static const SchemaQuery Query_for_list_of_statistics = { " (SELECT tgrelid FROM pg_catalog.pg_trigger "\ " WHERE pg_catalog.quote_ident(tgname)='%s')" +/* the silly-looking length condition is just to eat up the current word */ +#define Query_for_list_of_colls_for_one_index \ +" SELECT DISTINCT pg_catalog.quote_ident(coll.collname) " \ +" FROM pg_catalog.pg_depend d, pg_catalog.pg_collation coll, " \ +" pg_catalog.pg_class c" \ +" WHERE (%d = pg_catalog.length('%s'))" \ +" AND d.refclassid = " CppAsString2(CollationRelationId) \ +" AND d.refobjid = coll.oid " \ +" AND d.classid = " CppAsString2(RelationRelationId) \ +" AND d.objid = c.oid " \ +" AND c.relkind = " CppAsString2(RELKIND_INDEX) \ +" AND pg_catalog.pg_table_is_visible(c.oid) " \ +" AND c.relname = '%s'" + #define Query_for_list_of_ts_configurations \ "SELECT pg_catalog.quote_ident(cfgname) FROM pg_catalog.pg_ts_config "\ " WHERE substring(pg_catalog.quote_ident(cfgname),1,%d)='%s'" @@ -1715,14 +1730,15 @@ psql_completion(const char *text, int start, int end) /* ALTER INDEX */ else if (Matches("ALTER", "INDEX", MatchAny)) COMPLETE_WITH("ALTER COLUMN", "OWNER TO", "RENAME TO", "SET", - "RESET", "ATTACH PARTITION", "DEPENDS", "NO DEPENDS"); + "RESET", "ATTACH PARTITION", "DEPENDS", "NO DEPENDS", + "ALTER COLLATION"); else if (Matches("ALTER", "INDEX", MatchAny, "ATTACH")) COMPLETE_WITH("PARTITION"); else if (Matches("ALTER", "INDEX", MatchAny, "ATTACH", "PARTITION")) COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_indexes, NULL); /* ALTER INDEX ALTER */ else if (Matches("ALTER", "INDEX", MatchAny, "ALTER")) - COMPLETE_WITH("COLUMN"); + COMPLETE_WITH("COLLATION", "COLUMN"); /* ALTER INDEX ALTER COLUMN */ else if (Matches("ALTER", "INDEX", MatchAny, "ALTER", "COLUMN")) { @@ -1765,6 +1781,15 @@ psql_completion(const char *text, int start, int end) COMPLETE_WITH("ON EXTENSION"); else if (Matches("ALTER", "INDEX", MatchAny, "DEPENDS")) COMPLETE_WITH("ON EXTENSION"); + /* ALTER INDEX ALTER COLLATION */ + else if (Matches("ALTER", "INDEX", MatchAny, "ALTER", "COLLATION")) + { + completion_info_charp = prev3_wd; + COMPLETE_WITH_QUERY(Query_for_list_of_colls_for_one_index); + } + /* ALTER INDEX ALTER COLLATION */ + else if (Matches("ALTER", "INDEX", MatchAny, "ALTER", "COLLATION", MatchAny)) + COMPLETE_WITH("REFRESH VERSION"); /* ALTER LANGUAGE */ else if (Matches("ALTER", "LANGUAGE", MatchAny)) diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 6610e3c23f71..f28f083aca7b 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 202011012 +#define CATALOG_VERSION_NO 202011013 #endif diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h index 3baa5e498aa9..901d5019cdfd 100644 --- a/src/include/catalog/dependency.h +++ b/src/include/catalog/dependency.h @@ -160,7 +160,8 @@ extern void recordDependencyOnSingleRelExpr(const ObjectAddress *depender, Node *expr, Oid relId, DependencyType behavior, DependencyType self_behavior, - bool reverse_self); + bool reverse_self, + bool record_version); extern ObjectClass getObjectClass(const ObjectAddress *object); @@ -180,17 +181,30 @@ extern void sort_object_addresses(ObjectAddresses *addrs); extern void free_object_addresses(ObjectAddresses *addrs); +typedef bool(*VisitDependenciesOfCB) (const ObjectAddress *otherObject, + const char *version, + char **new_version, + void *data); + +extern void visitDependenciesOf(const ObjectAddress *object, + VisitDependenciesOfCB callback, + void *data); + /* in pg_depend.c */ extern void recordDependencyOn(const ObjectAddress *depender, const ObjectAddress *referenced, DependencyType behavior); +extern void recordDependencyOnCollations(ObjectAddress *myself, + List *collations, + bool record_version); + extern void recordMultipleDependencies(const ObjectAddress *depender, const ObjectAddress *referenced, int nreferenced, - const char *version, - DependencyType behavior); + DependencyType behavior, + bool record_version); extern void recordDependencyOnCurrentExtension(const ObjectAddress *object, bool isReplace); @@ -209,10 +223,9 @@ extern long changeDependencyFor(Oid classId, Oid objectId, Oid refClassId, Oid oldRefObjectId, Oid newRefObjectId); -extern long changeDependenciesOf(Oid classId, Oid oldObjectId, +long changeDependenciesOf(Oid classId, Oid oldObjectId, Oid newObjectId); - -extern long changeDependenciesOn(Oid refClassId, Oid oldRefObjectId, +long changeDependenciesOn(Oid refClassId, Oid oldRefObjectId, Oid newRefObjectId); extern Oid getExtensionOfObject(Oid classId, Oid objectId); diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h index f58e8675f32a..f4559b09d735 100644 --- a/src/include/catalog/index.h +++ b/src/include/catalog/index.h @@ -121,6 +121,9 @@ extern void FormIndexDatum(IndexInfo *indexInfo, Datum *values, bool *isnull); +extern void index_check_collation_versions(Oid relid); +extern void index_update_collation_versions(Oid relid, Oid coll); + extern void index_build(Relation heapRelation, Relation indexRelation, IndexInfo *indexInfo, diff --git a/src/include/catalog/pg_depend.h b/src/include/catalog/pg_depend.h index 748902279529..eeafbbe8d7c8 100644 --- a/src/include/catalog/pg_depend.h +++ b/src/include/catalog/pg_depend.h @@ -62,8 +62,7 @@ CATALOG(pg_depend,2608,DependRelationId) */ char deptype; /* see codes in dependency.h */ #ifdef CATALOG_VARLEN - text refobjversion; /* version tracking, NULL if not used or - * unknown */ + text refobjversion; /* version of referenced object */ #endif } FormData_pg_depend; diff --git a/src/include/catalog/pg_type.h b/src/include/catalog/pg_type.h index 6ae6edf7e0e8..d228efffc9b3 100644 --- a/src/include/catalog/pg_type.h +++ b/src/include/catalog/pg_type.h @@ -368,6 +368,8 @@ extern void GenerateTypeDependencies(HeapTuple typeTuple, bool isDependentType, bool rebuild); +extern List *GetTypeCollations(Oid typeObjectid); + extern void RenameTypeInternal(Oid typeOid, const char *newTypeName, Oid typeNamespace); diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 319f77013f4d..e1aeea256062 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1853,7 +1853,8 @@ typedef enum AlterTableType AT_DetachPartition, /* DETACH PARTITION */ AT_AddIdentity, /* ADD IDENTITY */ AT_SetIdentity, /* SET identity column options */ - AT_DropIdentity /* DROP IDENTITY */ + AT_DropIdentity, /* DROP IDENTITY */ + AT_AlterCollationRefreshVersion /* ALTER COLLATION ... REFRESH VERSION */ } AlterTableType; typedef struct ReplicaIdentityStmt @@ -1869,6 +1870,7 @@ typedef struct AlterTableCmd /* one subcommand of an ALTER TABLE */ AlterTableType subtype; /* Type of table alteration to apply */ char *name; /* column, constraint, or trigger to act on, * or tablespace */ + List *object; /* collation to act on if it's a collation */ int16 num; /* attribute number for columns referenced by * number */ RoleSpec *newowner; diff --git a/src/include/utils/pg_locale.h b/src/include/utils/pg_locale.h index 9cb7d91ddfb1..96da132c031b 100644 --- a/src/include/utils/pg_locale.h +++ b/src/include/utils/pg_locale.h @@ -103,7 +103,7 @@ typedef struct pg_locale_struct *pg_locale_t; extern pg_locale_t pg_newlocale_from_collation(Oid collid); -extern char *get_collation_actual_version(char collprovider, const char *collcollate); +extern char *get_collation_version_for_oid(Oid collid); #ifdef USE_ICU extern int32_t icu_to_uchar(UChar **buff_uchar, const char *buff, size_t nbytes); diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index 0b5957ba02c7..c5ffea40f212 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -63,6 +63,7 @@ typedef struct RelationData bool rd_indexvalid; /* is rd_indexlist valid? (also rd_pkindex and * rd_replidindex) */ bool rd_statvalid; /* is rd_statlist valid? */ + bool rd_version_checked; /* has version check been done yet? */ /*---------- * rd_createSubid is the ID of the highest subtransaction the rel has diff --git a/src/test/Makefile b/src/test/Makefile index 9774f534d93f..14cde4f5bacd 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -12,7 +12,8 @@ subdir = src/test top_builddir = ../.. include $(top_builddir)/src/Makefile.global -SUBDIRS = perl regress isolation modules authentication recovery subscription +SUBDIRS = perl regress isolation modules authentication recovery subscription \ + locale # Test suites that are not safe by default but can be run if selected # by the user via the whitespace-separated list in variable diff --git a/src/test/locale/.gitignore b/src/test/locale/.gitignore index 620d3df42542..64e1bf2a8037 100644 --- a/src/test/locale/.gitignore +++ b/src/test/locale/.gitignore @@ -1 +1,2 @@ /test-ctype +/tmp_check/ diff --git a/src/test/locale/Makefile b/src/test/locale/Makefile index 22a45b65f2c7..73495cf16b44 100644 --- a/src/test/locale/Makefile +++ b/src/test/locale/Makefile @@ -4,6 +4,7 @@ subdir = src/test/locale top_builddir = ../../.. include $(top_builddir)/src/Makefile.global +export with_icu PROGS = test-ctype DIRS = de_DE.ISO8859-1 gr_GR.ISO8859-7 koi8-r koi8-to-win1251 @@ -19,3 +20,9 @@ clean distclean maintainer-clean: # These behave like installcheck targets. check-%: all @$(MAKE) -C `echo $@ | sed 's/^check-//'` test + +check: + $(prove_check) + +installcheck: + $(prove_installcheck) diff --git a/src/test/locale/t/001_index.pl b/src/test/locale/t/001_index.pl new file mode 100644 index 000000000000..a67f78cb719c --- /dev/null +++ b/src/test/locale/t/001_index.pl @@ -0,0 +1,67 @@ +use strict; +use warnings; + +use Config; +use PostgresNode; +use TestLib; +use Test::More; + +if ($ENV{with_icu} eq 'yes') +{ + plan tests => 10; +} +else +{ + plan skip_all => 'ICU not supported by this build'; +} + +#### Set up the server + +note "setting up data directory"; +my $node = get_new_node('main'); +$node->init(extra => [ '--encoding=UTF8' ]); + +$ENV{PGHOST} = $node->host; +$ENV{PGPORT} = $node->port; +$node->start; + +sub test_index +{ + my ($err_like, $err_comm) = @_; + my ($ret, $out, $err) = $node->psql('postgres', "SELECT * FROM icu1"); + is($ret, 0, 'SELECT should succeed.'); + like($err, $err_like, $err_comm); +} + +$node->safe_psql('postgres', 'CREATE TABLE icu1(val text);'); +$node->safe_psql('postgres', 'CREATE INDEX icu1_fr ON icu1 (val COLLATE "fr-x-icu");'); + +test_index(qr/^$/, 'No warning should be raised'); + +# Simulate different collation version +$node->safe_psql('postgres', + "UPDATE pg_depend SET refobjversion = 'not_a_version'" + . " WHERE refobjversion IS NOT NULL" + . " AND objid::regclass::text = 'icu1_fr';"); + +test_index(qr/index "icu1_fr" depends on collation "fr-x-icu" version "not_a_version", but the current version is/, + 'Different collation version warning should be raised.'); + +$node->safe_psql('postgres', 'ALTER INDEX icu1_fr ALTER COLLATION "fr-x-icu" REFRESH VERSION;'); + +test_index(qr/^$/, 'No warning should be raised'); + +# Simulate different collation version +$node->safe_psql('postgres', + "UPDATE pg_depend SET refobjversion = 'not_a_version'" + . " WHERE refobjversion IS NOT NULL" + . " AND objid::regclass::text = 'icu1_fr';"); + +test_index(qr/index "icu1_fr" depends on collation "fr-x-icu" version "not_a_version", but the current version is/, + 'Different collation version warning should be raised.'); + +$node->safe_psql('postgres', 'REINDEX TABLE icu1;'); + +test_index(qr/^$/, 'No warning should be raised'); + +$node->stop; diff --git a/src/test/regress/expected/collate.icu.utf8.out b/src/test/regress/expected/collate.icu.utf8.out index 60d9263a2ffc..16b4d9e2cdc6 100644 --- a/src/test/regress/expected/collate.icu.utf8.out +++ b/src/test/regress/expected/collate.icu.utf8.out @@ -1897,6 +1897,207 @@ SELECT (SELECT count(*) FROM test33_0) <> (SELECT count(*) FROM test33_1); t (1 row) +-- collation versioning support +CREATE TYPE t_en_fr AS (fr text COLLATE "fr-x-icu", en text COLLATE "en-x-icu"); +CREATE DOMAIN d_en_fr AS t_en_fr; +CREATE DOMAIN d_es AS text COLLATE "es-x-icu"; +CREATE TYPE t_en_fr_ga AS (en_fr t_en_fr, ga text COLLATE "ga-x-icu"); +CREATE DOMAIN d_en_fr_ga AS t_en_fr_ga; +CREATE TYPE t_custom AS (meh text, meh2 text); +CREATE DOMAIN d_custom AS t_custom; +CREATE COLLATION custom ( + LOCALE = 'fr-x-icu', PROVIDER = 'icu' +); +CREATE TYPE myrange AS range (subtype = text); +CREATE TYPE myrange_en_fr_ga AS range(subtype = t_en_fr_ga); +CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy'); +CREATE TABLE collate_test ( + id integer, + val text COLLATE "fr-x-icu", + t_en_fr t_en_fr, + d_en_fr d_en_fr, + d_es d_es, + t_en_fr_ga t_en_fr_ga, + d_en_fr_ga d_en_fr_ga, + d_en_fr_ga_arr d_en_fr_ga[], + myrange myrange, + myrange_en_fr_ga myrange_en_fr_ga, + mood mood +); +CREATE INDEX icuidx00_val ON collate_test(val); +-- shouldn't get duplicated dependencies +CREATE INDEX icuidx00_val_val ON collate_test(val, val); +-- shouldn't track version +CREATE INDEX icuidx00_val_pattern ON collate_test(val text_pattern_ops); +-- should have single dependency, no version +CREATE INDEX icuidx00_val_pattern_val_pattern ON collate_test(val text_pattern_ops, val text_pattern_ops); +-- should have single dependency, with version +CREATE INDEX icuidx00_val_pattern_val ON collate_test(val text_pattern_ops, val); +-- should have single dependency, with version +CREATE INDEX icuidx00_val_val_pattern ON collate_test(val, val text_pattern_ops); +-- two rows expected, only one a version, because we don't try to merge these yet +CREATE INDEX icuidx00_val_pattern_where ON collate_test(val text_pattern_ops) WHERE val >= val; +-- two rows expected with version, because we don't try to merge these yet +CREATE INDEX icuidx00_val_where ON collate_test(val) WHERE val >= val; +-- two rows expected with version (expression walker + attribute) +CREATE INDEX icuidx00_val_pattern_expr ON collate_test(val varchar_pattern_ops, (val || val)); +-- two rows expected, one with a version (expression walker + attribute) +CREATE INDEX icuidx00_val_pattern_expr_pattern ON collate_test(val varchar_pattern_ops, (val || val) text_pattern_ops); +-- should have single dependency, with version tracked +CREATE INDEX icuidx01_t_en_fr__d_es ON collate_test (t_en_fr, d_es); +CREATE INDEX icuidx02_d_en_fr ON collate_test (d_en_fr); +CREATE INDEX icuidx03_t_en_fr_ga ON collate_test (t_en_fr_ga); +CREATE INDEX icuidx04_d_en_fr_ga ON collate_test (d_en_fr_ga); +CREATE INDEX icuidx05_d_en_fr_ga_arr ON collate_test (d_en_fr_ga_arr); +CREATE INDEX icuidx06_d_en_fr_ga ON collate_test(id) WHERE (d_en_fr_ga).en_fr.fr = 'foo'; +CREATE INDEX icuidx07_d_en_fr_ga ON collate_test(id) WHERE (d_en_fr_ga).ga = 'foo'; +CREATE INDEX icuidx08_d_en_fr_ga ON collate_test(id) WHERE (t_en_fr_ga) = ('foo', 'bar', 'baz'); +CREATE INDEX icuidx09_d_en_fr_ga ON collate_test(id) WHERE (d_en_fr_ga) = ('foo', 'bar', 'baz'); +CREATE INDEX icuidx10_d_en_fr_ga_es ON collate_test(id) WHERE (d_en_fr_ga) = ('foo', 'bar', 'baz' COLLATE "es-x-icu"); +CREATE INDEX icuidx11_d_es ON collate_test(id) WHERE (d_es) = ('foo'); +CREATE INDEX icuidx12_custom ON collate_test(id) WHERE ('foo', 'bar')::d_custom = ('foo', 'bar' COLLATE custom)::d_custom; +CREATE INDEX icuidx13_custom ON collate_test(id) WHERE ('foo' COLLATE custom, 'bar')::d_custom = ('foo', 'bar')::d_custom; +CREATE INDEX icuidx14_myrange ON collate_test(myrange); +CREATE INDEX icuidx15_myrange_en_fr_ga ON collate_test USING gist (myrange_en_fr_ga); +CREATE INDEX icuidx16_mood ON collate_test(id) WHERE mood > 'ok' COLLATE "fr-x-icu"; +CREATE TABLE collate_part(id integer, val text COLLATE "en-x-icu") PARTITION BY range(id); +CREATE TABLE collate_part_0 PARTITION OF collate_part FOR VALUES FROM (0) TO (1); +CREATE TABLE collate_part_1 PARTITION OF collate_part FOR VALUES FROM (1) TO (1000000); +CREATE INDEX icuidx17_part ON collate_part_1 (val); +SELECT objid::regclass::text collate "C", refobjid::regcollation::text collate "C", +CASE +WHEN refobjid = 'default'::regcollation THEN 'XXX' -- depends on libc version support +WHEN refobjversion IS NULL THEN 'version not tracked' +WHEN refobjversion = pg_collation_actual_version(refobjid) THEN 'up to date' +ELSE 'out of date' +END AS version +FROM pg_depend d +LEFT JOIN pg_class c ON c.oid = d.objid +WHERE refclassid = 'pg_collation'::regclass +AND coalesce(relkind, 'i') = 'i' +AND relname LIKE 'icuidx%' +ORDER BY 1, 2, 3; + objid | refobjid | version +-----------------------------------+------------+--------------------- + icuidx00_val | "fr-x-icu" | up to date + icuidx00_val_pattern | "fr-x-icu" | version not tracked + icuidx00_val_pattern_expr | "fr-x-icu" | up to date + icuidx00_val_pattern_expr | "fr-x-icu" | up to date + icuidx00_val_pattern_expr_pattern | "fr-x-icu" | up to date + icuidx00_val_pattern_expr_pattern | "fr-x-icu" | version not tracked + icuidx00_val_pattern_val | "fr-x-icu" | up to date + icuidx00_val_pattern_val_pattern | "fr-x-icu" | version not tracked + icuidx00_val_pattern_where | "fr-x-icu" | up to date + icuidx00_val_pattern_where | "fr-x-icu" | version not tracked + icuidx00_val_val | "fr-x-icu" | up to date + icuidx00_val_val_pattern | "fr-x-icu" | up to date + icuidx00_val_where | "fr-x-icu" | up to date + icuidx00_val_where | "fr-x-icu" | up to date + icuidx01_t_en_fr__d_es | "en-x-icu" | up to date + icuidx01_t_en_fr__d_es | "es-x-icu" | up to date + icuidx01_t_en_fr__d_es | "fr-x-icu" | up to date + icuidx02_d_en_fr | "en-x-icu" | up to date + icuidx02_d_en_fr | "fr-x-icu" | up to date + icuidx03_t_en_fr_ga | "en-x-icu" | up to date + icuidx03_t_en_fr_ga | "fr-x-icu" | up to date + icuidx03_t_en_fr_ga | "ga-x-icu" | up to date + icuidx04_d_en_fr_ga | "en-x-icu" | up to date + icuidx04_d_en_fr_ga | "fr-x-icu" | up to date + icuidx04_d_en_fr_ga | "ga-x-icu" | up to date + icuidx05_d_en_fr_ga_arr | "en-x-icu" | up to date + icuidx05_d_en_fr_ga_arr | "fr-x-icu" | up to date + icuidx05_d_en_fr_ga_arr | "ga-x-icu" | up to date + icuidx06_d_en_fr_ga | "default" | XXX + icuidx06_d_en_fr_ga | "en-x-icu" | up to date + icuidx06_d_en_fr_ga | "fr-x-icu" | up to date + icuidx06_d_en_fr_ga | "ga-x-icu" | up to date + icuidx07_d_en_fr_ga | "default" | XXX + icuidx07_d_en_fr_ga | "en-x-icu" | up to date + icuidx07_d_en_fr_ga | "fr-x-icu" | up to date + icuidx07_d_en_fr_ga | "ga-x-icu" | up to date + icuidx08_d_en_fr_ga | "en-x-icu" | up to date + icuidx08_d_en_fr_ga | "fr-x-icu" | up to date + icuidx08_d_en_fr_ga | "ga-x-icu" | up to date + icuidx09_d_en_fr_ga | "en-x-icu" | up to date + icuidx09_d_en_fr_ga | "fr-x-icu" | up to date + icuidx09_d_en_fr_ga | "ga-x-icu" | up to date + icuidx10_d_en_fr_ga_es | "en-x-icu" | up to date + icuidx10_d_en_fr_ga_es | "es-x-icu" | up to date + icuidx10_d_en_fr_ga_es | "fr-x-icu" | up to date + icuidx10_d_en_fr_ga_es | "ga-x-icu" | up to date + icuidx11_d_es | "default" | XXX + icuidx11_d_es | "es-x-icu" | up to date + icuidx12_custom | "default" | XXX + icuidx12_custom | custom | up to date + icuidx13_custom | "default" | XXX + icuidx13_custom | custom | up to date + icuidx14_myrange | "default" | XXX + icuidx15_myrange_en_fr_ga | "en-x-icu" | up to date + icuidx15_myrange_en_fr_ga | "fr-x-icu" | up to date + icuidx15_myrange_en_fr_ga | "ga-x-icu" | up to date + icuidx16_mood | "fr-x-icu" | up to date + icuidx17_part | "en-x-icu" | up to date +(58 rows) + +-- Validate that REINDEX will update the stored version. +UPDATE pg_depend SET refobjversion = 'not a version' +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text LIKE 'icuidx%' +AND refobjversion IS NOT NULL; +REINDEX TABLE collate_test; +REINDEX TABLE collate_part_0; +REINDEX TABLE collate_part_1; +SELECT objid::regclass FROM pg_depend WHERE refobjversion = 'not a version'; + objid +------- +(0 rows) + +-- Validate that REINDEX CONCURRENTLY will update the stored version. +UPDATE pg_depend SET refobjversion = 'not a version' +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text LIKE 'icuidx%' +AND refobjversion IS NOT NULL; +REINDEX TABLE CONCURRENTLY collate_test; +REINDEX TABLE CONCURRENTLY collate_part_0; +REINDEX INDEX CONCURRENTLY icuidx17_part; +SELECT objid::regclass FROM pg_depend WHERE refobjversion = 'not a version'; + objid +------- +(0 rows) + +-- Validate that VACUUM FULL will update the stored version. +UPDATE pg_depend SET refobjversion = 'not a version' +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text LIKE 'icuidx%' +AND refobjversion IS NOT NULL; +VACUUM FULL collate_test; +VACUUM FULL collate_part_0; +VACUUM FULL collate_part_1; +SELECT objid::regclass FROM pg_depend WHERE refobjversion = 'not a version'; + objid +------- +(0 rows) + +-- Test ALTER INDEX name ALTER COLLATION name REFRESH VERSION +UPDATE pg_depend SET refobjversion = 'not a version' +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text = 'icuidx17_part' +AND refobjversion IS NOT NULL; +SELECT objid::regclass FROM pg_depend WHERE refobjversion = 'not a version'; + objid +--------------- + icuidx17_part +(1 row) + +ALTER INDEX icuidx17_part ALTER COLLATION "en-x-icu" REFRESH VERSION; +SELECT objid::regclass, refobjversion = 'not a version' AS ver FROM pg_depend +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text = 'icuidx17_part'; + objid | ver +---------------+----- + icuidx17_part | f +(1 row) + -- cleanup RESET search_path; SET client_min_messages TO warning; diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 012c1eb0676c..17f1383ea49a 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2065,14 +2065,16 @@ WHERE classid = 'pg_class'::regclass AND obj | objref | deptype ------------------------------------------+------------------------------------------------------------+--------- index concur_reindex_ind1 | constraint concur_reindex_ind1 on table concur_reindex_tab | i + index concur_reindex_ind2 | collation "default" | n index concur_reindex_ind2 | column c2 of table concur_reindex_tab | a index concur_reindex_ind3 | column c1 of table concur_reindex_tab | a index concur_reindex_ind3 | table concur_reindex_tab | a + index concur_reindex_ind4 | collation "default" | n index concur_reindex_ind4 | column c1 of table concur_reindex_tab | a index concur_reindex_ind4 | column c2 of table concur_reindex_tab | a materialized view concur_reindex_matview | schema public | n table concur_reindex_tab | schema public | n -(8 rows) +(10 rows) REINDEX INDEX CONCURRENTLY concur_reindex_ind1; REINDEX TABLE CONCURRENTLY concur_reindex_tab; @@ -2092,14 +2094,16 @@ WHERE classid = 'pg_class'::regclass AND obj | objref | deptype ------------------------------------------+------------------------------------------------------------+--------- index concur_reindex_ind1 | constraint concur_reindex_ind1 on table concur_reindex_tab | i + index concur_reindex_ind2 | collation "default" | n index concur_reindex_ind2 | column c2 of table concur_reindex_tab | a index concur_reindex_ind3 | column c1 of table concur_reindex_tab | a index concur_reindex_ind3 | table concur_reindex_tab | a + index concur_reindex_ind4 | collation "default" | n index concur_reindex_ind4 | column c1 of table concur_reindex_tab | a index concur_reindex_ind4 | column c2 of table concur_reindex_tab | a materialized view concur_reindex_matview | schema public | n table concur_reindex_tab | schema public | n -(8 rows) +(10 rows) -- Check that comments are preserved CREATE TABLE testcomment (i int); diff --git a/src/test/regress/sql/collate.icu.utf8.sql b/src/test/regress/sql/collate.icu.utf8.sql index 35acf91fbf1e..4714c044d531 100644 --- a/src/test/regress/sql/collate.icu.utf8.sql +++ b/src/test/regress/sql/collate.icu.utf8.sql @@ -716,6 +716,138 @@ INSERT INTO test33 VALUES (2, 'DEF'); -- they end up in the same partition (but it's platform-dependent which one) SELECT (SELECT count(*) FROM test33_0) <> (SELECT count(*) FROM test33_1); +-- collation versioning support +CREATE TYPE t_en_fr AS (fr text COLLATE "fr-x-icu", en text COLLATE "en-x-icu"); +CREATE DOMAIN d_en_fr AS t_en_fr; +CREATE DOMAIN d_es AS text COLLATE "es-x-icu"; +CREATE TYPE t_en_fr_ga AS (en_fr t_en_fr, ga text COLLATE "ga-x-icu"); +CREATE DOMAIN d_en_fr_ga AS t_en_fr_ga; +CREATE TYPE t_custom AS (meh text, meh2 text); +CREATE DOMAIN d_custom AS t_custom; + +CREATE COLLATION custom ( + LOCALE = 'fr-x-icu', PROVIDER = 'icu' +); + +CREATE TYPE myrange AS range (subtype = text); +CREATE TYPE myrange_en_fr_ga AS range(subtype = t_en_fr_ga); + +CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy'); + +CREATE TABLE collate_test ( + id integer, + val text COLLATE "fr-x-icu", + t_en_fr t_en_fr, + d_en_fr d_en_fr, + d_es d_es, + t_en_fr_ga t_en_fr_ga, + d_en_fr_ga d_en_fr_ga, + d_en_fr_ga_arr d_en_fr_ga[], + myrange myrange, + myrange_en_fr_ga myrange_en_fr_ga, + mood mood +); + +CREATE INDEX icuidx00_val ON collate_test(val); +-- shouldn't get duplicated dependencies +CREATE INDEX icuidx00_val_val ON collate_test(val, val); +-- shouldn't track version +CREATE INDEX icuidx00_val_pattern ON collate_test(val text_pattern_ops); +-- should have single dependency, no version +CREATE INDEX icuidx00_val_pattern_val_pattern ON collate_test(val text_pattern_ops, val text_pattern_ops); +-- should have single dependency, with version +CREATE INDEX icuidx00_val_pattern_val ON collate_test(val text_pattern_ops, val); +-- should have single dependency, with version +CREATE INDEX icuidx00_val_val_pattern ON collate_test(val, val text_pattern_ops); +-- two rows expected, only one a version, because we don't try to merge these yet +CREATE INDEX icuidx00_val_pattern_where ON collate_test(val text_pattern_ops) WHERE val >= val; +-- two rows expected with version, because we don't try to merge these yet +CREATE INDEX icuidx00_val_where ON collate_test(val) WHERE val >= val; +-- two rows expected with version (expression walker + attribute) +CREATE INDEX icuidx00_val_pattern_expr ON collate_test(val varchar_pattern_ops, (val || val)); +-- two rows expected, one with a version (expression walker + attribute) +CREATE INDEX icuidx00_val_pattern_expr_pattern ON collate_test(val varchar_pattern_ops, (val || val) text_pattern_ops); +-- should have single dependency, with version tracked +CREATE INDEX icuidx01_t_en_fr__d_es ON collate_test (t_en_fr, d_es); +CREATE INDEX icuidx02_d_en_fr ON collate_test (d_en_fr); +CREATE INDEX icuidx03_t_en_fr_ga ON collate_test (t_en_fr_ga); +CREATE INDEX icuidx04_d_en_fr_ga ON collate_test (d_en_fr_ga); +CREATE INDEX icuidx05_d_en_fr_ga_arr ON collate_test (d_en_fr_ga_arr); +CREATE INDEX icuidx06_d_en_fr_ga ON collate_test(id) WHERE (d_en_fr_ga).en_fr.fr = 'foo'; +CREATE INDEX icuidx07_d_en_fr_ga ON collate_test(id) WHERE (d_en_fr_ga).ga = 'foo'; +CREATE INDEX icuidx08_d_en_fr_ga ON collate_test(id) WHERE (t_en_fr_ga) = ('foo', 'bar', 'baz'); +CREATE INDEX icuidx09_d_en_fr_ga ON collate_test(id) WHERE (d_en_fr_ga) = ('foo', 'bar', 'baz'); +CREATE INDEX icuidx10_d_en_fr_ga_es ON collate_test(id) WHERE (d_en_fr_ga) = ('foo', 'bar', 'baz' COLLATE "es-x-icu"); +CREATE INDEX icuidx11_d_es ON collate_test(id) WHERE (d_es) = ('foo'); +CREATE INDEX icuidx12_custom ON collate_test(id) WHERE ('foo', 'bar')::d_custom = ('foo', 'bar' COLLATE custom)::d_custom; +CREATE INDEX icuidx13_custom ON collate_test(id) WHERE ('foo' COLLATE custom, 'bar')::d_custom = ('foo', 'bar')::d_custom; +CREATE INDEX icuidx14_myrange ON collate_test(myrange); +CREATE INDEX icuidx15_myrange_en_fr_ga ON collate_test USING gist (myrange_en_fr_ga); +CREATE INDEX icuidx16_mood ON collate_test(id) WHERE mood > 'ok' COLLATE "fr-x-icu"; + +CREATE TABLE collate_part(id integer, val text COLLATE "en-x-icu") PARTITION BY range(id); +CREATE TABLE collate_part_0 PARTITION OF collate_part FOR VALUES FROM (0) TO (1); +CREATE TABLE collate_part_1 PARTITION OF collate_part FOR VALUES FROM (1) TO (1000000); +CREATE INDEX icuidx17_part ON collate_part_1 (val); + +SELECT objid::regclass::text collate "C", refobjid::regcollation::text collate "C", +CASE +WHEN refobjid = 'default'::regcollation THEN 'XXX' -- depends on libc version support +WHEN refobjversion IS NULL THEN 'version not tracked' +WHEN refobjversion = pg_collation_actual_version(refobjid) THEN 'up to date' +ELSE 'out of date' +END AS version +FROM pg_depend d +LEFT JOIN pg_class c ON c.oid = d.objid +WHERE refclassid = 'pg_collation'::regclass +AND coalesce(relkind, 'i') = 'i' +AND relname LIKE 'icuidx%' +ORDER BY 1, 2, 3; + +-- Validate that REINDEX will update the stored version. +UPDATE pg_depend SET refobjversion = 'not a version' +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text LIKE 'icuidx%' +AND refobjversion IS NOT NULL; + +REINDEX TABLE collate_test; +REINDEX TABLE collate_part_0; +REINDEX TABLE collate_part_1; + +SELECT objid::regclass FROM pg_depend WHERE refobjversion = 'not a version'; + +-- Validate that REINDEX CONCURRENTLY will update the stored version. +UPDATE pg_depend SET refobjversion = 'not a version' +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text LIKE 'icuidx%' +AND refobjversion IS NOT NULL; +REINDEX TABLE CONCURRENTLY collate_test; +REINDEX TABLE CONCURRENTLY collate_part_0; +REINDEX INDEX CONCURRENTLY icuidx17_part; + +SELECT objid::regclass FROM pg_depend WHERE refobjversion = 'not a version'; + +-- Validate that VACUUM FULL will update the stored version. +UPDATE pg_depend SET refobjversion = 'not a version' +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text LIKE 'icuidx%' +AND refobjversion IS NOT NULL; +VACUUM FULL collate_test; +VACUUM FULL collate_part_0; +VACUUM FULL collate_part_1; + +SELECT objid::regclass FROM pg_depend WHERE refobjversion = 'not a version'; + +-- Test ALTER INDEX name ALTER COLLATION name REFRESH VERSION +UPDATE pg_depend SET refobjversion = 'not a version' +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text = 'icuidx17_part' +AND refobjversion IS NOT NULL; +SELECT objid::regclass FROM pg_depend WHERE refobjversion = 'not a version'; +ALTER INDEX icuidx17_part ALTER COLLATION "en-x-icu" REFRESH VERSION; +SELECT objid::regclass, refobjversion = 'not a version' AS ver FROM pg_depend +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text = 'icuidx17_part'; -- cleanup RESET search_path; diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index b6acade6c678..03c4e0fe5ba5 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2914,6 +2914,8 @@ dlist_head dlist_iter dlist_mutable_iter dlist_node +do_collation_version_check_context +do_collation_version_update_context ds_state dsa_area dsa_area_control From 5b3dca096f8e504b73812f68716fb68dd1176d6d Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Mon, 2 Nov 2020 15:00:24 +0100 Subject: [PATCH 420/589] Clarify temporary table name shadowing in CREATE TABLE docs Author: David Johnston --- doc/src/sgml/ref/create_table.sgml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index fd6777ae0192..bc59a2d77ddb 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -171,8 +171,9 @@ WITH ( MODULUS numeric_literal, REM If specified, the table is created as a temporary table. Temporary tables are automatically dropped at the end of a session, or optionally at the end of the current transaction - (see ON COMMIT below). Existing permanent - tables with the same name are not visible to the current session + (see ON COMMIT below). The default + search_path includes the temporary schema first and so identically + named existing permanent tables are not chosen for new plans while the temporary table exists, unless they are referenced with schema-qualified names. Any indexes created on a temporary table are automatically temporary as well. From dd26a0ad760b11237d7ea9cda8ccccc1826c2c64 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 2 Nov 2020 16:48:22 +0100 Subject: [PATCH 421/589] Use PG_GETARG_TRANSACTIONID where appropriate Some places were using PG_GETARG_UINT32 where PG_GETARG_TRANSACTIONID would be more appropriate. (Of course, they are the same internally, so there is no externally visible effect.) To do that, export PG_GETARG_TRANSACTIONID outside of xid.c. We also export PG_RETURN_TRANSACTIONID for symmetry, even though there are currently no external users. Author: Ashutosh Bapat Discussion: https://www.postgresql.org/message-id/flat/d8f6bdd536df403b9b33816e9f7e0b9d@G08CNEXMBPEKD05.g08.fujitsu.local --- src/backend/access/transam/commit_ts.c | 4 ++-- src/backend/access/transam/multixact.c | 2 +- src/backend/utils/adt/xid.c | 3 --- src/include/fmgr.h | 2 ++ 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/backend/access/transam/commit_ts.c b/src/backend/access/transam/commit_ts.c index cb8a96880187..2fe551f17e77 100644 --- a/src/backend/access/transam/commit_ts.c +++ b/src/backend/access/transam/commit_ts.c @@ -404,7 +404,7 @@ error_commit_ts_disabled(void) Datum pg_xact_commit_timestamp(PG_FUNCTION_ARGS) { - TransactionId xid = PG_GETARG_UINT32(0); + TransactionId xid = PG_GETARG_TRANSACTIONID(0); TimestampTz ts; bool found; @@ -481,7 +481,7 @@ pg_last_committed_xact(PG_FUNCTION_ARGS) Datum pg_xact_commit_timestamp_origin(PG_FUNCTION_ARGS) { - TransactionId xid = PG_GETARG_UINT32(0); + TransactionId xid = PG_GETARG_TRANSACTIONID(0); RepOriginId nodeid; TimestampTz ts; Datum values[2]; diff --git a/src/backend/access/transam/multixact.c b/src/backend/access/transam/multixact.c index 43653fe57212..eb8de7cf3293 100644 --- a/src/backend/access/transam/multixact.c +++ b/src/backend/access/transam/multixact.c @@ -3339,7 +3339,7 @@ pg_get_multixact_members(PG_FUNCTION_ARGS) int nmembers; int iter; } mxact; - MultiXactId mxid = PG_GETARG_UINT32(0); + MultiXactId mxid = PG_GETARG_TRANSACTIONID(0); mxact *multi; FuncCallContext *funccxt; diff --git a/src/backend/utils/adt/xid.c b/src/backend/utils/adt/xid.c index 20389aff1d12..a4762014ba1f 100644 --- a/src/backend/utils/adt/xid.c +++ b/src/backend/utils/adt/xid.c @@ -23,9 +23,6 @@ #include "utils/builtins.h" #include "utils/xid8.h" -#define PG_GETARG_TRANSACTIONID(n) DatumGetTransactionId(PG_GETARG_DATUM(n)) -#define PG_RETURN_TRANSACTIONID(x) return TransactionIdGetDatum(x) - #define PG_GETARG_COMMANDID(n) DatumGetCommandId(PG_GETARG_DATUM(n)) #define PG_RETURN_COMMANDID(x) return CommandIdGetDatum(x) diff --git a/src/include/fmgr.h b/src/include/fmgr.h index f25068fae201..ce37e342cd00 100644 --- a/src/include/fmgr.h +++ b/src/include/fmgr.h @@ -276,6 +276,7 @@ extern struct varlena *pg_detoast_datum_packed(struct varlena *datum); #define PG_GETARG_POINTER(n) DatumGetPointer(PG_GETARG_DATUM(n)) #define PG_GETARG_CSTRING(n) DatumGetCString(PG_GETARG_DATUM(n)) #define PG_GETARG_NAME(n) DatumGetName(PG_GETARG_DATUM(n)) +#define PG_GETARG_TRANSACTIONID(n) DatumGetTransactionId(PG_GETARG_DATUM(n)) /* these macros hide the pass-by-reference-ness of the datatype: */ #define PG_GETARG_FLOAT4(n) DatumGetFloat4(PG_GETARG_DATUM(n)) #define PG_GETARG_FLOAT8(n) DatumGetFloat8(PG_GETARG_DATUM(n)) @@ -360,6 +361,7 @@ extern struct varlena *pg_detoast_datum_packed(struct varlena *datum); #define PG_RETURN_POINTER(x) return PointerGetDatum(x) #define PG_RETURN_CSTRING(x) return CStringGetDatum(x) #define PG_RETURN_NAME(x) return NameGetDatum(x) +#define PG_RETURN_TRANSACTIONID(x) return TransactionIdGetDatum(x) /* these macros hide the pass-by-reference-ness of the datatype: */ #define PG_RETURN_FLOAT4(x) return Float4GetDatum(x) #define PG_RETURN_FLOAT8(x) return Float8GetDatum(x) From fd2997565c6f66837440dd57f5e52b56aa964d14 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 2 Nov 2020 11:25:18 -0500 Subject: [PATCH 422/589] Second thoughts on TOAST decompression. On detecting a corrupted match tag, pglz_decompress() should just summarily return -1. Breaking out of the loop, as I did in dfc797730, doesn't quite guarantee that will happen. Also, we can use unlikely() on that check, just in case it helps. Backpatch to v13, like the previous patch. --- src/common/pg_lzcompress.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index 79747767ce08..f9c29820e30b 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -680,9 +680,12 @@ pglz_compress(const char *source, int32 slen, char *dest, * pglz_decompress - * * Decompresses source into dest. Returns the number of bytes - * decompressed in the destination buffer, and *optionally* - * checks that both the source and dest buffers have been - * fully read and written to, respectively. + * decompressed into the destination buffer, or -1 if the + * compressed data is corrupted. + * + * If check_complete is true, the data is considered corrupted + * if we don't exactly fill the destination buffer. Callers that + * are extracting a slice typically can't apply this check. * ---------- */ int32 @@ -736,8 +739,8 @@ pglz_decompress(const char *source, int32 slen, char *dest, * must check this, else we risk an infinite loop below in the * face of corrupt data.) */ - if (sp > srcend || off == 0) - break; + if (unlikely(sp > srcend || off == 0)) + return -1; /* * Don't emit more data than requested. @@ -809,9 +812,7 @@ pglz_decompress(const char *source, int32 slen, char *dest, } /* - * Check we decompressed the right amount. If we are slicing, then we - * won't necessarily be at the end of the source or dest buffers when we - * hit a stop, so we don't test them. + * If requested, check we decompressed the right amount. */ if (check_complete && (dp != destend || sp != srcend)) return -1; From 8e1f37c07aafd4bb7aa6e1e1982010af11f8b5c7 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 2 Nov 2020 11:57:28 -0500 Subject: [PATCH 423/589] Rethink the generation rule for fmgroids.h macros. Traditionally, the names of fmgroids.h macros for pg_proc OIDs have been constructed from the prosrc field. But sometimes the same C function underlies multiple pg_proc entries, forcing us to make an arbitrary choice of which OID to reference; the other entries are then not namable via fmgroids.h. Moreover, we could not have macros at all for pg_proc entries that aren't for C-coded functions. Instead, use the proname field, and append the proargtypes field (replacing inter-argument spaces with underscores) if proname is not unique. Special-casing unique entries such as F_OIDEQ removes the need to change a lot of code. Indeed, I can only find two places in the tree that need to be adjusted; while this changes quite a few existing entries in fmgroids.h, few of them are referenced from C code. With this patch, all entries in pg_proc.dat have macros in fmgroids.h. Discussion: https://postgr.es/m/472274.1604258384@sss.pgh.pa.us --- src/backend/optimizer/util/clauses.c | 2 +- src/backend/utils/Gen_fmgrtab.pl | 51 ++++++++++++++-------------- src/backend/utils/adt/ruleutils.c | 2 +- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c index e7d814651b18..85ef873caaf8 100644 --- a/src/backend/optimizer/util/clauses.c +++ b/src/backend/optimizer/util/clauses.c @@ -779,7 +779,7 @@ contain_volatile_functions_not_nextval(Node *clause) static bool contain_volatile_functions_not_nextval_checker(Oid func_id, void *context) { - return (func_id != F_NEXTVAL_OID && + return (func_id != F_NEXTVAL && func_volatile(func_id) == PROVOLATILE_VOLATILE); } diff --git a/src/backend/utils/Gen_fmgrtab.pl b/src/backend/utils/Gen_fmgrtab.pl index b7c7b4c8fae1..8228ad6db60d 100644 --- a/src/backend/utils/Gen_fmgrtab.pl +++ b/src/backend/utils/Gen_fmgrtab.pl @@ -64,22 +64,26 @@ # Collect certain fields from pg_proc.dat. my @fmgr = (); +my %proname_counts; foreach my $row (@{ $catalog_data{pg_proc} }) { my %bki_values = %$row; - # Select out just the rows for internal-language procedures. - next if $bki_values{prolang} ne 'internal'; - push @fmgr, { oid => $bki_values{oid}, + name => $bki_values{proname}, + lang => $bki_values{prolang}, strict => $bki_values{proisstrict}, retset => $bki_values{proretset}, nargs => $bki_values{pronargs}, + args => $bki_values{proargtypes}, prosrc => $bki_values{prosrc}, }; + + # Count so that we can detect overloaded pronames. + $proname_counts{ $bki_values{proname} }++; } # Emit headers for both files @@ -122,13 +126,10 @@ /* * Constant macros for the OIDs of entries in pg_proc. * - * NOTE: macros are named after the prosrc value, ie the actual C name - * of the implementing function, not the proname which may be overloaded. - * For example, we want to be able to assign different macro names to both - * char_text() and name_text() even though these both appear with proname - * 'text'. If the same C function appears in more than one pg_proc entry, - * its equivalent macro will be defined with the lowest OID among those - * entries. + * F_XXX macros are named after the proname field; if that is not unique, + * we append the proargtypes field, replacing spaces with underscores. + * For example, we have F_OIDEQ because that proname is unique, but + * F_POW_FLOAT8_FLOAT8 (among others) because that proname is not. */ OFH @@ -186,14 +187,20 @@ TFH -# Emit #define's and extern's -- only one per prosrc value +# Emit fmgroids.h and fmgrprotos.h entries in OID order. my %seenit; foreach my $s (sort { $a->{oid} <=> $b->{oid} } @fmgr) { - next if $seenit{ $s->{prosrc} }; - $seenit{ $s->{prosrc} } = 1; - print $ofh "#define F_" . uc $s->{prosrc} . " $s->{oid}\n"; - print $pfh "extern Datum $s->{prosrc}(PG_FUNCTION_ARGS);\n"; + my $sqlname = $s->{name}; + $sqlname .= "_" . $s->{args} if ($proname_counts{ $s->{name} } > 1); + $sqlname =~ s/\s+/_/g; + print $ofh "#define F_" . uc $sqlname . " $s->{oid}\n"; + # We want only one extern per internal-language function + if ($s->{lang} eq 'internal' && !$seenit{ $s->{prosrc} }) + { + $seenit{ $s->{prosrc} } = 1; + print $pfh "extern Datum $s->{prosrc}(PG_FUNCTION_ARGS);\n"; + } } # Create the fmgr_builtins table, collect data for fmgr_builtin_oid_index @@ -206,22 +213,16 @@ my $fmgr_count = 0; foreach my $s (sort { $a->{oid} <=> $b->{oid} } @fmgr) { + next if $s->{lang} ne 'internal'; + + print $tfh ",\n" if ($fmgr_count > 0); print $tfh " { $s->{oid}, $s->{nargs}, $bmap{$s->{strict}}, $bmap{$s->{retset}}, \"$s->{prosrc}\", $s->{prosrc} }"; $fmgr_builtin_oid_index[ $s->{oid} ] = $fmgr_count++; $last_builtin_oid = $s->{oid}; - - if ($fmgr_count <= $#fmgr) - { - print $tfh ",\n"; - } - else - { - print $tfh "\n"; - } } -print $tfh "};\n"; +print $tfh "\n};\n"; printf $tfh qq| const int fmgr_nbuiltins = (sizeof(fmgr_builtins) / sizeof(FmgrBuiltin)); diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index 6c656586e857..744619325294 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -10144,7 +10144,7 @@ get_from_clause_item(Node *jtnode, Query *query, deparse_context *context) RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); if (!IsA(rtfunc->funcexpr, FuncExpr) || - ((FuncExpr *) rtfunc->funcexpr)->funcid != F_ARRAY_UNNEST || + ((FuncExpr *) rtfunc->funcexpr)->funcid != F_UNNEST_ANYARRAY || rtfunc->funccolnames != NIL) { all_unnest = false; From e1339bfc7a2fd4629e1c3f8f919ddd05b4745e13 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 2 Nov 2020 14:34:34 -0500 Subject: [PATCH 424/589] Remove special checks for pg_rewrite.ev_qual and ev_action being NULL. make_ruledef() and make_viewdef() were coded to cope with possible null-ness of these columns, but they've been marked BKI_FORCE_NOT_NULL for some time. So there's not really any need to do more than what we do for the other columns of pg_rewrite, i.e. just Assert that we got non-null results. (There is a school of thought that says Asserts aren't the thing to do to check for corrupt data, but surely here is not the place to start if we want such a policy.) Also, remove long-dead-if-indeed-it-ever-wasn't-dead handling of an empty actions list in make_ruledef(). That's an error case and should be treated as such. (DO INSTEAD NOTHING is represented by a CMD_NOTHING Query, not an empty list; cf transformRuleStmt.) Kyotaro Horiguchi, some changes by me Discussion: https://postgr.es/m/CAEudQApoA=tMTic6xEPYP_hsNZ8XtToVThK_0x7D_aFQYowq3w@mail.gmail.com --- src/backend/utils/adt/ruleutils.c | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index 744619325294..28f56074c0c6 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -4733,7 +4733,7 @@ make_ruledef(StringInfo buf, HeapTuple ruletup, TupleDesc rulettc, bool is_instead; char *ev_qual; char *ev_action; - List *actions = NIL; + List *actions; Relation ev_relation; TupleDesc viewResultDesc = NULL; int fno; @@ -4763,14 +4763,16 @@ make_ruledef(StringInfo buf, HeapTuple ruletup, TupleDesc rulettc, Assert(!isnull); is_instead = DatumGetBool(dat); - /* these could be nulls */ fno = SPI_fnumber(rulettc, "ev_qual"); ev_qual = SPI_getvalue(ruletup, rulettc, fno); + Assert(ev_qual != NULL); fno = SPI_fnumber(rulettc, "ev_action"); ev_action = SPI_getvalue(ruletup, rulettc, fno); - if (ev_action != NULL) - actions = (List *) stringToNode(ev_action); + Assert(ev_action != NULL); + actions = (List *) stringToNode(ev_action); + if (actions == NIL) + elog(ERROR, "invalid empty ev_action list"); ev_relation = table_open(ev_class, AccessShareLock); @@ -4820,9 +4822,7 @@ make_ruledef(StringInfo buf, HeapTuple ruletup, TupleDesc rulettc, generate_qualified_relation_name(ev_class)); /* If the rule has an event qualification, add it */ - if (ev_qual == NULL) - ev_qual = ""; - if (strlen(ev_qual) > 0 && strcmp(ev_qual, "<>") != 0) + if (strcmp(ev_qual, "<>") != 0) { Node *qual; Query *query; @@ -4893,10 +4893,6 @@ make_ruledef(StringInfo buf, HeapTuple ruletup, TupleDesc rulettc, } appendStringInfoString(buf, ");"); } - else if (list_length(actions) == 0) - { - appendStringInfoString(buf, "NOTHING;"); - } else { Query *query; @@ -4926,7 +4922,7 @@ make_viewdef(StringInfo buf, HeapTuple ruletup, TupleDesc rulettc, bool is_instead; char *ev_qual; char *ev_action; - List *actions = NIL; + List *actions; Relation ev_relation; int fno; Datum dat; @@ -4950,14 +4946,14 @@ make_viewdef(StringInfo buf, HeapTuple ruletup, TupleDesc rulettc, Assert(!isnull); is_instead = DatumGetBool(dat); - /* these could be nulls */ fno = SPI_fnumber(rulettc, "ev_qual"); ev_qual = SPI_getvalue(ruletup, rulettc, fno); + Assert(ev_qual != NULL); fno = SPI_fnumber(rulettc, "ev_action"); ev_action = SPI_getvalue(ruletup, rulettc, fno); - if (ev_action != NULL) - actions = (List *) stringToNode(ev_action); + Assert(ev_action != NULL); + actions = (List *) stringToNode(ev_action); if (list_length(actions) != 1) { From 0a4b34031279d938c2e59df8df7159d6c11e39b5 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 2 Nov 2020 21:11:50 -0500 Subject: [PATCH 425/589] Fix unportable use of getnameinfo() in pg_hba_file_rules view. fill_hba_line() thought it could get away with passing sizeof(struct sockaddr_storage) rather than the actual addrlen previously returned by getaddrinfo(). While that appears to work on many platforms, it does not work on FreeBSD 11: you get back a failure, which leads to the view showing NULL for the address and netmask columns in all rows. The POSIX spec for getnameinfo() is pretty clearly on FreeBSD's side here: you should pass the actual address length. So it seems plausible that there are other platforms where this coding also fails, and we just hadn't noticed. Also, IMO the fact that getnameinfo() failure leads to a NULL output is pretty bogus in itself. Our pg_getnameinfo_all() wrapper is careful to emit "???" on failure, and we should use that in such cases. NULL should only be emitted in rows that don't have IP addresses. Per bug #16695 from Peter Vandivier. Back-patch to v10 where this code was added. Discussion: https://postgr.es/m/16695-a665558e2f630be7@postgresql.org --- src/backend/libpq/hba.c | 31 +++++++++++++++++++++---------- src/include/libpq/hba.h | 7 ++++++- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/backend/libpq/hba.c b/src/backend/libpq/hba.c index 4c86fb608748..3a78d2043e57 100644 --- a/src/backend/libpq/hba.c +++ b/src/backend/libpq/hba.c @@ -1188,8 +1188,11 @@ parse_hba_line(TokenizedLine *tok_line, int elevel) ret = pg_getaddrinfo_all(str, NULL, &hints, &gai_result); if (ret == 0 && gai_result) + { memcpy(&parsedline->addr, gai_result->ai_addr, gai_result->ai_addrlen); + parsedline->addrlen = gai_result->ai_addrlen; + } else if (ret == EAI_NONAME) parsedline->hostname = str; else @@ -1238,6 +1241,7 @@ parse_hba_line(TokenizedLine *tok_line, int elevel) token->string); return NULL; } + parsedline->masklen = parsedline->addrlen; pfree(str); } else if (!parsedline->hostname) @@ -1288,6 +1292,7 @@ parse_hba_line(TokenizedLine *tok_line, int elevel) memcpy(&parsedline->mask, gai_result->ai_addr, gai_result->ai_addrlen); + parsedline->masklen = gai_result->ai_addrlen; pg_freeaddrinfo_all(hints.ai_family, gai_result); if (parsedline->addr.ss_family != parsedline->mask.ss_family) @@ -2538,20 +2543,26 @@ fill_hba_line(Tuplestorestate *tuple_store, TupleDesc tupdesc, } else { - if (pg_getnameinfo_all(&hba->addr, sizeof(hba->addr), - buffer, sizeof(buffer), - NULL, 0, - NI_NUMERICHOST) == 0) + /* + * Note: if pg_getnameinfo_all fails, it'll set buffer to + * "???", which we want to return. + */ + if (hba->addrlen > 0) { - clean_ipv6_addr(hba->addr.ss_family, buffer); + if (pg_getnameinfo_all(&hba->addr, hba->addrlen, + buffer, sizeof(buffer), + NULL, 0, + NI_NUMERICHOST) == 0) + clean_ipv6_addr(hba->addr.ss_family, buffer); addrstr = pstrdup(buffer); } - if (pg_getnameinfo_all(&hba->mask, sizeof(hba->mask), - buffer, sizeof(buffer), - NULL, 0, - NI_NUMERICHOST) == 0) + if (hba->masklen > 0) { - clean_ipv6_addr(hba->mask.ss_family, buffer); + if (pg_getnameinfo_all(&hba->mask, hba->masklen, + buffer, sizeof(buffer), + NULL, 0, + NI_NUMERICHOST) == 0) + clean_ipv6_addr(hba->mask.ss_family, buffer); maskstr = pstrdup(buffer); } } diff --git a/src/include/libpq/hba.h b/src/include/libpq/hba.h index d638479d8849..8f09b5638f12 100644 --- a/src/include/libpq/hba.h +++ b/src/include/libpq/hba.h @@ -42,6 +42,10 @@ typedef enum UserAuth #define USER_AUTH_LAST uaPeer /* Must be last value of this enum */ } UserAuth; +/* + * Data structures representing pg_hba.conf entries + */ + typedef enum IPCompareMethod { ipCmpMask, @@ -75,11 +79,12 @@ typedef struct HbaLine List *databases; List *roles; struct sockaddr_storage addr; + int addrlen; /* zero if we don't have a valid addr */ struct sockaddr_storage mask; + int masklen; /* zero if we don't have a valid mask */ IPCompareMethod ip_cmp_method; char *hostname; UserAuth auth_method; - char *usermap; char *pamservice; bool pam_use_hostname; From 8c2d8f6cc4848cf9276dff445bb0f2f664083eca Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Tue, 3 Nov 2020 08:38:27 +0530 Subject: [PATCH 426/589] Fix typos. Author: Hou Zhijie Discussion: https://postgr.es/m/855a9421839d402b8b351d273c89a8f8@G08CNEXMBPEKD05.g08.fujitsu.local --- src/backend/storage/ipc/procarray.c | 2 +- src/include/access/xlogreader.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c index 8da7b1d8a7bb..05661e379e50 100644 --- a/src/backend/storage/ipc/procarray.c +++ b/src/backend/storage/ipc/procarray.c @@ -1600,7 +1600,7 @@ TransactionIdIsActive(TransactionId xid) * See the definition of ComputedXidHorizonsResult for the various computed * horizons. * - * For VACUUM separate horizons (used to to decide which deleted tuples must + * For VACUUM separate horizons (used to decide which deleted tuples must * be preserved), for shared and non-shared tables are computed. For shared * relations backends in all databases must be considered, but for non-shared * relations that's not required, since only backends in my own database could diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index b97688222910..0b6d00dd7dfa 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -17,7 +17,7 @@ * XLogBeginRead() or XLogFindNextRecord(), and call XLogReadRecord() * until it returns NULL. * - * Callers supply a page_read callback if they want to to call + * Callers supply a page_read callback if they want to call * XLogReadRecord or XLogFindNextRecord; it can be passed in as NULL * otherwise. The WALRead function can be used as a helper to write * page_read callbacks, but it is not mandatory; callers that use it, From 3165426e54df02a6199c0a216546e5095e875a0a Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 3 Nov 2020 09:03:22 +0100 Subject: [PATCH 427/589] Remove use of deprecated containment operators in tests Switch @ to <@ and ~ to @> in gist-related tests. The old operator names have been deprecated and will eventually (possibly soon) be removed. Author: Justin Pryzby Discussion: https://www.postgresql.org/message-id/flat/20201027032511.GF9241@telsasoft.com --- src/test/regress/expected/create_am.out | 10 ++++----- src/test/regress/expected/create_index.out | 24 +++++++++++----------- src/test/regress/sql/create_am.sql | 4 ++-- src/test/regress/sql/create_index.sql | 12 +++++------ 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/test/regress/expected/create_am.out b/src/test/regress/expected/create_am.out index b9dc82dd3c3a..730011526a8e 100644 --- a/src/test/regress/expected/create_am.out +++ b/src/test/regress/expected/create_am.out @@ -46,18 +46,18 @@ SET enable_indexscan = ON; SET enable_bitmapscan = OFF; EXPLAIN (COSTS OFF) SELECT * FROM fast_emp4000 - WHERE home_base @ '(200,200),(2000,1000)'::box + WHERE home_base <@ '(200,200),(2000,1000)'::box ORDER BY (home_base[0])[0]; - QUERY PLAN ----------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------- Sort Sort Key: ((home_base[0])[0]) -> Index Only Scan using grect2ind2 on fast_emp4000 - Index Cond: (home_base @ '(2000,1000),(200,200)'::box) + Index Cond: (home_base <@ '(2000,1000),(200,200)'::box) (4 rows) SELECT * FROM fast_emp4000 - WHERE home_base @ '(200,200),(2000,1000)'::box + WHERE home_base <@ '(200,200),(2000,1000)'::box ORDER BY (home_base[0])[0]; home_base ----------------------- diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 17f1383ea49a..93a8736a3f40 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -75,7 +75,7 @@ SET enable_seqscan = ON; SET enable_indexscan = OFF; SET enable_bitmapscan = OFF; SELECT * FROM fast_emp4000 - WHERE home_base @ '(200,200),(2000,1000)'::box + WHERE home_base <@ '(200,200),(2000,1000)'::box ORDER BY (home_base[0])[0]; home_base ----------------------- @@ -95,7 +95,7 @@ SELECT count(*) FROM fast_emp4000 WHERE home_base IS NULL; 278 (1 row) -SELECT * FROM polygon_tbl WHERE f1 ~ '((1,1),(2,2),(2,1))'::polygon +SELECT * FROM polygon_tbl WHERE f1 @> '((1,1),(2,2),(2,1))'::polygon ORDER BY (poly_center(f1))[0]; f1 --------------------- @@ -259,18 +259,18 @@ SET enable_indexscan = ON; SET enable_bitmapscan = OFF; EXPLAIN (COSTS OFF) SELECT * FROM fast_emp4000 - WHERE home_base @ '(200,200),(2000,1000)'::box + WHERE home_base <@ '(200,200),(2000,1000)'::box ORDER BY (home_base[0])[0]; - QUERY PLAN ----------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------- Sort Sort Key: ((home_base[0])[0]) -> Index Only Scan using grect2ind on fast_emp4000 - Index Cond: (home_base @ '(2000,1000),(200,200)'::box) + Index Cond: (home_base <@ '(2000,1000),(200,200)'::box) (4 rows) SELECT * FROM fast_emp4000 - WHERE home_base @ '(200,200),(2000,1000)'::box + WHERE home_base <@ '(200,200),(2000,1000)'::box ORDER BY (home_base[0])[0]; home_base ----------------------- @@ -309,17 +309,17 @@ SELECT count(*) FROM fast_emp4000 WHERE home_base IS NULL; (1 row) EXPLAIN (COSTS OFF) -SELECT * FROM polygon_tbl WHERE f1 ~ '((1,1),(2,2),(2,1))'::polygon +SELECT * FROM polygon_tbl WHERE f1 @> '((1,1),(2,2),(2,1))'::polygon ORDER BY (poly_center(f1))[0]; - QUERY PLAN ------------------------------------------------------------ + QUERY PLAN +------------------------------------------------------------ Sort Sort Key: ((poly_center(f1))[0]) -> Index Scan using gpolygonind on polygon_tbl - Index Cond: (f1 ~ '((1,1),(2,2),(2,1))'::polygon) + Index Cond: (f1 @> '((1,1),(2,2),(2,1))'::polygon) (4 rows) -SELECT * FROM polygon_tbl WHERE f1 ~ '((1,1),(2,2),(2,1))'::polygon +SELECT * FROM polygon_tbl WHERE f1 @> '((1,1),(2,2),(2,1))'::polygon ORDER BY (poly_center(f1))[0]; f1 --------------------- diff --git a/src/test/regress/sql/create_am.sql b/src/test/regress/sql/create_am.sql index 97df244d172a..6dd0de427260 100644 --- a/src/test/regress/sql/create_am.sql +++ b/src/test/regress/sql/create_am.sql @@ -50,10 +50,10 @@ SET enable_bitmapscan = OFF; EXPLAIN (COSTS OFF) SELECT * FROM fast_emp4000 - WHERE home_base @ '(200,200),(2000,1000)'::box + WHERE home_base <@ '(200,200),(2000,1000)'::box ORDER BY (home_base[0])[0]; SELECT * FROM fast_emp4000 - WHERE home_base @ '(200,200),(2000,1000)'::box + WHERE home_base <@ '(200,200),(2000,1000)'::box ORDER BY (home_base[0])[0]; EXPLAIN (COSTS OFF) diff --git a/src/test/regress/sql/create_index.sql b/src/test/regress/sql/create_index.sql index 4e45b186131b..b27643cad68c 100644 --- a/src/test/regress/sql/create_index.sql +++ b/src/test/regress/sql/create_index.sql @@ -107,14 +107,14 @@ SET enable_indexscan = OFF; SET enable_bitmapscan = OFF; SELECT * FROM fast_emp4000 - WHERE home_base @ '(200,200),(2000,1000)'::box + WHERE home_base <@ '(200,200),(2000,1000)'::box ORDER BY (home_base[0])[0]; SELECT count(*) FROM fast_emp4000 WHERE home_base && '(1000,1000,0,0)'::box; SELECT count(*) FROM fast_emp4000 WHERE home_base IS NULL; -SELECT * FROM polygon_tbl WHERE f1 ~ '((1,1),(2,2),(2,1))'::polygon +SELECT * FROM polygon_tbl WHERE f1 @> '((1,1),(2,2),(2,1))'::polygon ORDER BY (poly_center(f1))[0]; SELECT * FROM circle_tbl WHERE f1 && circle(point(1,-2), 1) @@ -161,10 +161,10 @@ SET enable_bitmapscan = OFF; EXPLAIN (COSTS OFF) SELECT * FROM fast_emp4000 - WHERE home_base @ '(200,200),(2000,1000)'::box + WHERE home_base <@ '(200,200),(2000,1000)'::box ORDER BY (home_base[0])[0]; SELECT * FROM fast_emp4000 - WHERE home_base @ '(200,200),(2000,1000)'::box + WHERE home_base <@ '(200,200),(2000,1000)'::box ORDER BY (home_base[0])[0]; EXPLAIN (COSTS OFF) @@ -176,9 +176,9 @@ SELECT count(*) FROM fast_emp4000 WHERE home_base IS NULL; SELECT count(*) FROM fast_emp4000 WHERE home_base IS NULL; EXPLAIN (COSTS OFF) -SELECT * FROM polygon_tbl WHERE f1 ~ '((1,1),(2,2),(2,1))'::polygon +SELECT * FROM polygon_tbl WHERE f1 @> '((1,1),(2,2),(2,1))'::polygon ORDER BY (poly_center(f1))[0]; -SELECT * FROM polygon_tbl WHERE f1 ~ '((1,1),(2,2),(2,1))'::polygon +SELECT * FROM polygon_tbl WHERE f1 @> '((1,1),(2,2),(2,1))'::polygon ORDER BY (poly_center(f1))[0]; EXPLAIN (COSTS OFF) From 5d1833f414973595411779b86e085fb05907a805 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Tue, 3 Nov 2020 09:47:36 +0100 Subject: [PATCH 428/589] Use be_tls_* API for SSL information in sslinfo sslinfo was passing the Port->ssl member directly to OpenSSL in order to extract information regarding the connection. This breaks the API provided by the backend TLS implementation, as well as duplicates code for no benefit. Rewrite to make use of the backend API as much as possible. Author: Daniel Gustafsson --- contrib/sslinfo/sslinfo.c | 139 +++++++++++++------------------------- doc/src/sgml/sslinfo.sgml | 4 +- 2 files changed, 49 insertions(+), 94 deletions(-) diff --git a/contrib/sslinfo/sslinfo.c b/contrib/sslinfo/sslinfo.c index 5ba3988e2704..30cae0bb985e 100644 --- a/contrib/sslinfo/sslinfo.c +++ b/contrib/sslinfo/sslinfo.c @@ -22,7 +22,6 @@ PG_MODULE_MAGIC; static Datum X509_NAME_field_to_text(X509_NAME *name, text *fieldName); -static Datum X509_NAME_to_text(X509_NAME *name); static Datum ASN1_STRING_to_text(ASN1_STRING *str); /* @@ -54,9 +53,16 @@ PG_FUNCTION_INFO_V1(ssl_version); Datum ssl_version(PG_FUNCTION_ARGS) { - if (MyProcPort->ssl == NULL) + const char *version; + + if (!MyProcPort->ssl_in_use) + PG_RETURN_NULL(); + + version = be_tls_get_version(MyProcPort); + if (version == NULL) PG_RETURN_NULL(); - PG_RETURN_TEXT_P(cstring_to_text(SSL_get_version(MyProcPort->ssl))); + + PG_RETURN_TEXT_P(cstring_to_text(version)); } @@ -67,9 +73,16 @@ PG_FUNCTION_INFO_V1(ssl_cipher); Datum ssl_cipher(PG_FUNCTION_ARGS) { - if (MyProcPort->ssl == NULL) + const char *cipher; + + if (!MyProcPort->ssl_in_use) + PG_RETURN_NULL(); + + cipher = be_tls_get_cipher(MyProcPort); + if (cipher == NULL) PG_RETURN_NULL(); - PG_RETURN_TEXT_P(cstring_to_text(SSL_get_cipher(MyProcPort->ssl))); + + PG_RETURN_TEXT_P(cstring_to_text(cipher)); } @@ -83,7 +96,7 @@ PG_FUNCTION_INFO_V1(ssl_client_cert_present); Datum ssl_client_cert_present(PG_FUNCTION_ARGS) { - PG_RETURN_BOOL(MyProcPort->peer != NULL); + PG_RETURN_BOOL(MyProcPort->peer_cert_valid); } @@ -99,25 +112,21 @@ PG_FUNCTION_INFO_V1(ssl_client_serial); Datum ssl_client_serial(PG_FUNCTION_ARGS) { + char decimal[NAMEDATALEN]; Datum result; - Port *port = MyProcPort; - X509 *peer = port->peer; - ASN1_INTEGER *serial = NULL; - BIGNUM *b; - char *decimal; - if (!peer) + if (!MyProcPort->ssl_in_use || !MyProcPort->peer_cert_valid) + PG_RETURN_NULL(); + + be_tls_get_peer_serial(MyProcPort, decimal, NAMEDATALEN); + + if (!*decimal) PG_RETURN_NULL(); - serial = X509_get_serialNumber(peer); - b = ASN1_INTEGER_to_BN(serial, NULL); - decimal = BN_bn2dec(b); - BN_free(b); result = DirectFunctionCall3(numeric_in, CStringGetDatum(decimal), ObjectIdGetDatum(0), Int32GetDatum(-1)); - OPENSSL_free(decimal); return result; } @@ -228,7 +237,7 @@ ssl_client_dn_field(PG_FUNCTION_ARGS) text *fieldname = PG_GETARG_TEXT_PP(0); Datum result; - if (!(MyProcPort->peer)) + if (!MyProcPort->ssl_in_use || !MyProcPort->peer_cert_valid) PG_RETURN_NULL(); result = X509_NAME_field_to_text(X509_get_subject_name(MyProcPort->peer), fieldname); @@ -275,76 +284,6 @@ ssl_issuer_field(PG_FUNCTION_ARGS) } -/* - * Equivalent of X509_NAME_oneline that respects encoding - * - * This function converts X509_NAME structure to the text variable - * converting all textual data into current database encoding. - * - * Parameter: X509_NAME *name X509_NAME structure to be converted - * - * Returns: text datum which contains string representation of - * X509_NAME - */ -static Datum -X509_NAME_to_text(X509_NAME *name) -{ - BIO *membuf = BIO_new(BIO_s_mem()); - int i, - nid, - count = X509_NAME_entry_count(name); - X509_NAME_ENTRY *e; - ASN1_STRING *v; - const char *field_name; - size_t size; - char nullterm; - char *sp; - char *dp; - text *result; - - if (membuf == NULL) - ereport(ERROR, - (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("could not create OpenSSL BIO structure"))); - - (void) BIO_set_close(membuf, BIO_CLOSE); - for (i = 0; i < count; i++) - { - e = X509_NAME_get_entry(name, i); - nid = OBJ_obj2nid(X509_NAME_ENTRY_get_object(e)); - if (nid == NID_undef) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("could not get NID for ASN1_OBJECT object"))); - v = X509_NAME_ENTRY_get_data(e); - field_name = OBJ_nid2sn(nid); - if (field_name == NULL) - field_name = OBJ_nid2ln(nid); - if (field_name == NULL) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("could not convert NID %d to an ASN1_OBJECT structure", nid))); - BIO_printf(membuf, "/%s=", field_name); - ASN1_STRING_print_ex(membuf, v, - ((ASN1_STRFLGS_RFC2253 & ~ASN1_STRFLGS_ESC_MSB) - | ASN1_STRFLGS_UTF8_CONVERT)); - } - - /* ensure null termination of the BIO's content */ - nullterm = '\0'; - BIO_write(membuf, &nullterm, 1); - size = BIO_get_mem_data(membuf, &sp); - dp = pg_any_to_server(sp, size - 1, PG_UTF8); - result = cstring_to_text(dp); - if (dp != sp) - pfree(dp); - if (BIO_free(membuf) != 1) - elog(ERROR, "could not free OpenSSL BIO structure"); - - PG_RETURN_TEXT_P(result); -} - - /* * Returns current client certificate subject as one string * @@ -358,9 +297,17 @@ PG_FUNCTION_INFO_V1(ssl_client_dn); Datum ssl_client_dn(PG_FUNCTION_ARGS) { - if (!(MyProcPort->peer)) + char subject[NAMEDATALEN]; + + if (!MyProcPort->ssl_in_use || !MyProcPort->peer_cert_valid) + PG_RETURN_NULL(); + + be_tls_get_peer_subject_name(MyProcPort, subject, NAMEDATALEN); + + if (!*subject) PG_RETURN_NULL(); - return X509_NAME_to_text(X509_get_subject_name(MyProcPort->peer)); + + PG_RETURN_TEXT_P(cstring_to_text(subject)); } @@ -377,9 +324,17 @@ PG_FUNCTION_INFO_V1(ssl_issuer_dn); Datum ssl_issuer_dn(PG_FUNCTION_ARGS) { - if (!(MyProcPort->peer)) + char issuer[NAMEDATALEN]; + + if (!MyProcPort->ssl_in_use || !MyProcPort->peer_cert_valid) PG_RETURN_NULL(); - return X509_NAME_to_text(X509_get_issuer_name(MyProcPort->peer)); + + be_tls_get_peer_issuer_name(MyProcPort, issuer, NAMEDATALEN); + + if (!*issuer) + PG_RETURN_NULL(); + + PG_RETURN_TEXT_P(cstring_to_text(issuer)); } diff --git a/doc/src/sgml/sslinfo.sgml b/doc/src/sgml/sslinfo.sgml index e16f61b41d72..3213c039ca67 100644 --- a/doc/src/sgml/sslinfo.sgml +++ b/doc/src/sgml/sslinfo.sgml @@ -53,8 +53,8 @@ - Returns the name of the protocol used for the SSL connection (e.g., TLSv1.0 - TLSv1.1, or TLSv1.2). + Returns the name of the protocol used for the SSL connection (e.g., TLSv1.0, + TLSv1.1, TLSv1.2 or TLSv1.3). From 13cfa02f77936895bff6ffabf1fde5d47fd1df07 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Tue, 3 Nov 2020 09:55:51 +0100 Subject: [PATCH 429/589] Improve error handling in backend OpenSSL implementation Commit d94c36a45ab introduced error handling to sslinfo to handle OpenSSL errors gracefully. This ports this errorhandling to the backend TLS implementation. Author: Daniel Gustafsson --- src/backend/libpq/be-secure-openssl.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/backend/libpq/be-secure-openssl.c b/src/backend/libpq/be-secure-openssl.c index 8b21ff4065c5..9231a1470cf4 100644 --- a/src/backend/libpq/be-secure-openssl.c +++ b/src/backend/libpq/be-secure-openssl.c @@ -1298,15 +1298,28 @@ X509_NAME_to_cstring(X509_NAME *name) char *dp; char *result; + if (membuf == NULL) + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("failed to create BIO"))); + (void) BIO_set_close(membuf, BIO_CLOSE); for (i = 0; i < count; i++) { e = X509_NAME_get_entry(name, i); nid = OBJ_obj2nid(X509_NAME_ENTRY_get_object(e)); + if (nid == NID_undef) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("could not get NID for ASN1_OBJECT object"))); v = X509_NAME_ENTRY_get_data(e); field_name = OBJ_nid2sn(nid); - if (!field_name) + if (field_name == NULL) field_name = OBJ_nid2ln(nid); + if (field_name == NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("could not convert NID %d to an ASN1_OBJECT structure", nid))); BIO_printf(membuf, "/%s=", field_name); ASN1_STRING_print_ex(membuf, v, ((ASN1_STRFLGS_RFC2253 & ~ASN1_STRFLGS_ESC_MSB) @@ -1322,7 +1335,8 @@ X509_NAME_to_cstring(X509_NAME *name) result = pstrdup(dp); if (dp != sp) pfree(dp); - BIO_free(membuf); + if (BIO_free(membuf) != 1) + elog(ERROR, "could not free OpenSSL BIO structure"); return result; } From 44a184cb686866b10d63695db344195c239f9374 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Tue, 3 Nov 2020 10:19:55 +0100 Subject: [PATCH 430/589] Use the non-deprecated TG_TABLE_MAME in test trigger Commit 3a9ae3d2068 (back in 2006) deprecated TG_RELNAME in favor of TG_TABLE_NAME, but the existing usage in test cases has remained till today. Change to use TG_TABLE_NAME instead (TG_RELNAME is still covered by a test case). --- src/test/regress/expected/triggers.out | 4 ++-- src/test/regress/sql/triggers.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/regress/expected/triggers.out b/src/test/regress/expected/triggers.out index c19aac967428..027494bc76f9 100644 --- a/src/test/regress/expected/triggers.out +++ b/src/test/regress/expected/triggers.out @@ -701,7 +701,7 @@ create table trigtest (i serial primary key); create table trigtest2 (i int references trigtest(i) on delete cascade); create function trigtest() returns trigger as $$ begin - raise notice '% % % %', TG_RELNAME, TG_OP, TG_WHEN, TG_LEVEL; + raise notice '% % % %', TG_TABLE_NAME, TG_OP, TG_WHEN, TG_LEVEL; return new; end;$$ language plpgsql; create trigger trigtest_b_row_tg before insert or update or delete on trigtest @@ -987,7 +987,7 @@ begin argstr := argstr || TG_argv[i]; end loop; - raise notice '% % % % (%)', TG_RELNAME, TG_WHEN, TG_OP, TG_LEVEL, argstr; + raise notice '% % % % (%)', TG_TABLE_NAME, TG_WHEN, TG_OP, TG_LEVEL, argstr; if TG_LEVEL = 'ROW' then if TG_OP = 'INSERT' then diff --git a/src/test/regress/sql/triggers.sql b/src/test/regress/sql/triggers.sql index bf2e73abf606..212b4f1f959f 100644 --- a/src/test/regress/sql/triggers.sql +++ b/src/test/regress/sql/triggers.sql @@ -444,7 +444,7 @@ create table trigtest2 (i int references trigtest(i) on delete cascade); create function trigtest() returns trigger as $$ begin - raise notice '% % % %', TG_RELNAME, TG_OP, TG_WHEN, TG_LEVEL; + raise notice '% % % %', TG_TABLE_NAME, TG_OP, TG_WHEN, TG_LEVEL; return new; end;$$ language plpgsql; @@ -680,7 +680,7 @@ begin argstr := argstr || TG_argv[i]; end loop; - raise notice '% % % % (%)', TG_RELNAME, TG_WHEN, TG_OP, TG_LEVEL, argstr; + raise notice '% % % % (%)', TG_TABLE_NAME, TG_WHEN, TG_OP, TG_LEVEL, argstr; if TG_LEVEL = 'ROW' then if TG_OP = 'INSERT' then From 2f70fdb0644c32c4154236c2b5c241bec92eac5e Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 3 Nov 2020 10:32:20 +0100 Subject: [PATCH 431/589] Remove deprecated containment operators for built-in types Remove old containment operators @ and ~ for built-in geometry data types. These have been deprecated; use <@ and @> instead. (Some contrib modules still contain the same deprecated operators. That will be dealt with separately.) Author: Justin Pryzby Discussion: https://www.postgresql.org/message-id/flat/20201027032511.GF9241@telsasoft.com --- doc/src/sgml/func.sgml | 9 ---- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_amop.dat | 18 ------- src/include/catalog/pg_operator.dat | 65 ------------------------ src/test/regress/expected/create_am.out | 2 - src/test/regress/expected/opr_sanity.out | 7 +-- src/test/regress/sql/create_am.sql | 2 - 7 files changed, 3 insertions(+), 102 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index bf6004f321f6..ee721d258e00 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -10890,15 +10890,6 @@ CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple - - - Before PostgreSQL 8.2, the containment - operators @> and <@ were respectively - called ~ and @. These names are still - available, but are deprecated and will eventually be removed. - - - Geometric Functions diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index f28f083aca7b..100e6f0a09a9 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 202011013 +#define CATALOG_VERSION_NO 202011031 #endif diff --git a/src/include/catalog/pg_amop.dat b/src/include/catalog/pg_amop.dat index 1dfb6fd37394..bbe357fbc0e8 100644 --- a/src/include/catalog/pg_amop.dat +++ b/src/include/catalog/pg_amop.dat @@ -1100,10 +1100,6 @@ amopstrategy => '11', amopopr => '|>>(box,box)', amopmethod => 'gist' }, { amopfamily => 'gist/box_ops', amoplefttype => 'box', amoprighttype => 'box', amopstrategy => '12', amopopr => '|&>(box,box)', amopmethod => 'gist' }, -{ amopfamily => 'gist/box_ops', amoplefttype => 'box', amoprighttype => 'box', - amopstrategy => '13', amopopr => '~(box,box)', amopmethod => 'gist' }, -{ amopfamily => 'gist/box_ops', amoplefttype => 'box', amoprighttype => 'box', - amopstrategy => '14', amopopr => '@(box,box)', amopmethod => 'gist' }, { amopfamily => 'gist/box_ops', amoplefttype => 'box', amoprighttype => 'point', amopstrategy => '15', amoppurpose => 'o', amopopr => '<->(box,point)', amopmethod => 'gist', amopsortfamily => 'btree/float_ops' }, @@ -1175,12 +1171,6 @@ { amopfamily => 'gist/poly_ops', amoplefttype => 'polygon', amoprighttype => 'polygon', amopstrategy => '12', amopopr => '|&>(polygon,polygon)', amopmethod => 'gist' }, -{ amopfamily => 'gist/poly_ops', amoplefttype => 'polygon', - amoprighttype => 'polygon', amopstrategy => '13', - amopopr => '~(polygon,polygon)', amopmethod => 'gist' }, -{ amopfamily => 'gist/poly_ops', amoplefttype => 'polygon', - amoprighttype => 'polygon', amopstrategy => '14', - amopopr => '@(polygon,polygon)', amopmethod => 'gist' }, { amopfamily => 'gist/poly_ops', amoplefttype => 'polygon', amoprighttype => 'point', amopstrategy => '15', amoppurpose => 'o', amopopr => '<->(polygon,point)', amopmethod => 'gist', @@ -1223,12 +1213,6 @@ { amopfamily => 'gist/circle_ops', amoplefttype => 'circle', amoprighttype => 'circle', amopstrategy => '12', amopopr => '|&>(circle,circle)', amopmethod => 'gist' }, -{ amopfamily => 'gist/circle_ops', amoplefttype => 'circle', - amoprighttype => 'circle', amopstrategy => '13', - amopopr => '~(circle,circle)', amopmethod => 'gist' }, -{ amopfamily => 'gist/circle_ops', amoplefttype => 'circle', - amoprighttype => 'circle', amopstrategy => '14', - amopopr => '@(circle,circle)', amopmethod => 'gist' }, { amopfamily => 'gist/circle_ops', amoplefttype => 'circle', amoprighttype => 'point', amopstrategy => '15', amoppurpose => 'o', amopopr => '<->(circle,point)', amopmethod => 'gist', @@ -2454,8 +2438,6 @@ amoprighttype => 'box', amopstrategy => '12', amopopr => '|&>(box,box)', amopmethod => 'brin' }, -# we could, but choose not to, supply entries for strategies 13 and 14 - { amopfamily => 'brin/box_inclusion_ops', amoplefttype => 'box', amoprighttype => 'point', amopstrategy => '7', amopopr => '@>(box,point)', amopmethod => 'brin' }, diff --git a/src/include/catalog/pg_operator.dat b/src/include/catalog/pg_operator.dat index 7cc812adda63..feb8edece550 100644 --- a/src/include/catalog/pg_operator.dat +++ b/src/include/catalog/pg_operator.dat @@ -2777,71 +2777,6 @@ oprname => '||', oprleft => 'anynonarray', oprright => 'text', oprresult => 'text', oprcode => 'anytextcat' }, -# obsolete names for contains/contained-by operators; remove these someday -{ oid => '2860', descr => 'deprecated, use <@ instead', - oprname => '@', oprleft => 'polygon', oprright => 'polygon', - oprresult => 'bool', oprcom => '~(polygon,polygon)', - oprcode => 'poly_contained', oprrest => 'contsel', oprjoin => 'contjoinsel' }, -{ oid => '2861', descr => 'deprecated, use @> instead', - oprname => '~', oprleft => 'polygon', oprright => 'polygon', - oprresult => 'bool', oprcom => '@(polygon,polygon)', - oprcode => 'poly_contain', oprrest => 'contsel', oprjoin => 'contjoinsel' }, -{ oid => '2862', descr => 'deprecated, use <@ instead', - oprname => '@', oprleft => 'box', oprright => 'box', oprresult => 'bool', - oprcom => '~(box,box)', oprcode => 'box_contained', oprrest => 'contsel', - oprjoin => 'contjoinsel' }, -{ oid => '2863', descr => 'deprecated, use @> instead', - oprname => '~', oprleft => 'box', oprright => 'box', oprresult => 'bool', - oprcom => '@(box,box)', oprcode => 'box_contain', oprrest => 'contsel', - oprjoin => 'contjoinsel' }, -{ oid => '2864', descr => 'deprecated, use <@ instead', - oprname => '@', oprleft => 'circle', oprright => 'circle', - oprresult => 'bool', oprcom => '~(circle,circle)', - oprcode => 'circle_contained', oprrest => 'contsel', - oprjoin => 'contjoinsel' }, -{ oid => '2865', descr => 'deprecated, use @> instead', - oprname => '~', oprleft => 'circle', oprright => 'circle', - oprresult => 'bool', oprcom => '@(circle,circle)', - oprcode => 'circle_contain', oprrest => 'contsel', oprjoin => 'contjoinsel' }, -{ oid => '2866', descr => 'deprecated, use <@ instead', - oprname => '@', oprleft => 'point', oprright => 'box', oprresult => 'bool', - oprcode => 'on_pb' }, -{ oid => '2867', descr => 'deprecated, use <@ instead', - oprname => '@', oprleft => 'point', oprright => 'path', oprresult => 'bool', - oprcom => '~(path,point)', oprcode => 'on_ppath' }, -{ oid => '2868', descr => 'deprecated, use @> instead', - oprname => '~', oprleft => 'path', oprright => 'point', oprresult => 'bool', - oprcom => '@(point,path)', oprcode => 'path_contain_pt' }, -{ oid => '2869', descr => 'deprecated, use <@ instead', - oprname => '@', oprleft => 'point', oprright => 'polygon', - oprresult => 'bool', oprcom => '~(polygon,point)', - oprcode => 'pt_contained_poly' }, -{ oid => '2870', descr => 'deprecated, use @> instead', - oprname => '~', oprleft => 'polygon', oprright => 'point', - oprresult => 'bool', oprcom => '@(point,polygon)', - oprcode => 'poly_contain_pt' }, -{ oid => '2871', descr => 'deprecated, use <@ instead', - oprname => '@', oprleft => 'point', oprright => 'circle', oprresult => 'bool', - oprcom => '~(circle,point)', oprcode => 'pt_contained_circle' }, -{ oid => '2872', descr => 'deprecated, use @> instead', - oprname => '~', oprleft => 'circle', oprright => 'point', oprresult => 'bool', - oprcom => '@(point,circle)', oprcode => 'circle_contain_pt' }, -{ oid => '2873', descr => 'deprecated, use <@ instead', - oprname => '@', oprleft => 'point', oprright => 'line', oprresult => 'bool', - oprcode => 'on_pl' }, -{ oid => '2874', descr => 'deprecated, use <@ instead', - oprname => '@', oprleft => 'point', oprright => 'lseg', oprresult => 'bool', - oprcode => 'on_ps' }, -{ oid => '2875', descr => 'deprecated, use <@ instead', - oprname => '@', oprleft => 'lseg', oprright => 'line', oprresult => 'bool', - oprcode => 'on_sl' }, -{ oid => '2876', descr => 'deprecated, use <@ instead', - oprname => '@', oprleft => 'lseg', oprright => 'box', oprresult => 'bool', - oprcode => 'on_sb' }, -{ oid => '2877', descr => 'deprecated, use @> instead', - oprname => '~', oprleft => '_aclitem', oprright => 'aclitem', - oprresult => 'bool', oprcode => 'aclcontains' }, - # uuid operators { oid => '2972', descr => 'equal', oprname => '=', oprcanmerge => 't', oprcanhash => 't', oprleft => 'uuid', diff --git a/src/test/regress/expected/create_am.out b/src/test/regress/expected/create_am.out index 730011526a8e..0dfb26c30143 100644 --- a/src/test/regress/expected/create_am.out +++ b/src/test/regress/expected/create_am.out @@ -27,8 +27,6 @@ CREATE OPERATOR CLASS box_ops DEFAULT OPERATOR 10 <<|, OPERATOR 11 |>>, OPERATOR 12 |&>, - OPERATOR 13 ~, - OPERATOR 14 @, FUNCTION 1 gist_box_consistent(internal, box, smallint, oid, internal), FUNCTION 2 gist_box_union(internal, internal), -- don't need compress, decompress, or fetch functions diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index 7825a765cd7b..7ed29b49617b 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -1163,14 +1163,13 @@ ORDER BY 1, 2; ?-| | ?-| ?| | ?| ?|| | ?|| - @ | ~ @@ | @@ @@@ | @@@ | | | ~<=~ | ~>=~ ~<~ | ~>~ ~= | ~= -(30 rows) +(29 rows) -- Likewise for negator pairs. SELECT DISTINCT o1.oprname AS op1, o2.oprname AS op2 @@ -1990,8 +1989,6 @@ ORDER BY 1, 2, 3; 783 | 11 | >^ 783 | 11 | |>> 783 | 12 | |&> - 783 | 13 | ~ - 783 | 14 | @ 783 | 15 | <-> 783 | 16 | @> 783 | 18 | = @@ -2084,7 +2081,7 @@ ORDER BY 1, 2, 3; 4000 | 26 | >> 4000 | 27 | >>= 4000 | 28 | ^@ -(125 rows) +(123 rows) -- Check that all opclass search operators have selectivity estimators. -- This is not absolutely required, but it seems a reasonable thing diff --git a/src/test/regress/sql/create_am.sql b/src/test/regress/sql/create_am.sql index 6dd0de427260..9a359466ce49 100644 --- a/src/test/regress/sql/create_am.sql +++ b/src/test/regress/sql/create_am.sql @@ -28,8 +28,6 @@ CREATE OPERATOR CLASS box_ops DEFAULT OPERATOR 10 <<|, OPERATOR 11 |>>, OPERATOR 12 |&>, - OPERATOR 13 ~, - OPERATOR 14 @, FUNCTION 1 gist_box_consistent(internal, box, smallint, oid, internal), FUNCTION 2 gist_box_union(internal, internal), -- don't need compress, decompress, or fetch functions From bf797a8d9768239f5e3204b013044274b2c7c24a Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 3 Nov 2020 15:14:50 +0100 Subject: [PATCH 432/589] Disallow ALTER TABLE ONLY / DROP EXPRESSION The current implementation cannot handle this correctly, so just forbid it for now. GENERATED clauses must be attached to the column definition and cannot be added later like DEFAULT, so if a child table has a generation expression that the parent does not have, the child column will necessarily be an attlocal column. So to implement ALTER TABLE ONLY / DROP EXPRESSION, we'd need extra code to update attislocal of the direct child tables, somewhat similar to how DROP COLUMN does it, so that the resulting state can be properly dumped and restored. Discussion: https://www.postgresql.org/message-id/flat/15830.1575468847%40sss.pgh.pa.us --- src/backend/commands/tablecmds.c | 22 +++++++++++++++++++--- src/test/regress/expected/generated.out | 11 ++++++----- src/test/regress/sql/generated.sql | 2 +- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 1b105ba1c428..4bb33ee03cd1 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -413,7 +413,7 @@ static ObjectAddress ATExecAddIdentity(Relation rel, const char *colName, static ObjectAddress ATExecSetIdentity(Relation rel, const char *colName, Node *def, LOCKMODE lockmode); static ObjectAddress ATExecDropIdentity(Relation rel, const char *colName, bool missing_ok, LOCKMODE lockmode); -static void ATPrepDropExpression(Relation rel, AlterTableCmd *cmd, bool recursing); +static void ATPrepDropExpression(Relation rel, AlterTableCmd *cmd, bool recurse, bool recursing, LOCKMODE lockmode); static ObjectAddress ATExecDropExpression(Relation rel, const char *colName, bool missing_ok, LOCKMODE lockmode); static ObjectAddress ATExecSetStatistics(Relation rel, const char *colName, int16 colNum, Node *newValue, LOCKMODE lockmode); @@ -4151,7 +4151,7 @@ ATPrepCmd(List **wqueue, Relation rel, AlterTableCmd *cmd, case AT_DropExpression: /* ALTER COLUMN DROP EXPRESSION */ ATSimplePermissions(rel, ATT_TABLE | ATT_FOREIGN_TABLE); ATSimpleRecursion(wqueue, rel, cmd, recurse, lockmode, context); - ATPrepDropExpression(rel, cmd, recursing); + ATPrepDropExpression(rel, cmd, recurse, recursing, lockmode); pass = AT_PASS_DROP; break; case AT_SetStatistics: /* ALTER COLUMN SET STATISTICS */ @@ -7262,8 +7262,24 @@ ATExecDropIdentity(Relation rel, const char *colName, bool missing_ok, LOCKMODE * ALTER TABLE ALTER COLUMN DROP EXPRESSION */ static void -ATPrepDropExpression(Relation rel, AlterTableCmd *cmd, bool recursing) +ATPrepDropExpression(Relation rel, AlterTableCmd *cmd, bool recurse, bool recursing, LOCKMODE lockmode) { + /* + * Reject ONLY if there are child tables. We could implement this, but it + * is a bit complicated. GENERATED clauses must be attached to the column + * definition and cannot be added later like DEFAULT, so if a child table + * has a generation expression that the parent does not have, the child + * column will necessarily be an attlocal column. So to implement ONLY + * here, we'd need extra code to update attislocal of the direct child + * tables, somewhat similar to how DROP COLUMN does it, so that the + * resulting state can be properly dumped and restored. + */ + if (!recurse && + find_inheritance_children(RelationGetRelid(rel), lockmode)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ALTER TABLE / DROP EXPRESSION must be applied to child tables too"))); + /* * Cannot drop generation expression from inherited columns. */ diff --git a/src/test/regress/expected/generated.out b/src/test/regress/expected/generated.out index 7ccc3c65ed17..4b0626030442 100644 --- a/src/test/regress/expected/generated.out +++ b/src/test/regress/expected/generated.out @@ -805,13 +805,14 @@ CREATE TABLE gtest30 ( b int GENERATED ALWAYS AS (a * 2) STORED ); CREATE TABLE gtest30_1 () INHERITS (gtest30); -ALTER TABLE ONLY gtest30 ALTER COLUMN b DROP EXPRESSION; +ALTER TABLE ONLY gtest30 ALTER COLUMN b DROP EXPRESSION; -- error +ERROR: ALTER TABLE / DROP EXPRESSION must be applied to child tables too \d gtest30 - Table "public.gtest30" - Column | Type | Collation | Nullable | Default ---------+---------+-----------+----------+--------- + Table "public.gtest30" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+------------------------------------ a | integer | | | - b | integer | | | + b | integer | | | generated always as (a * 2) stored Number of child tables: 1 (Use \d+ to list them.) \d gtest30_1 diff --git a/src/test/regress/sql/generated.sql b/src/test/regress/sql/generated.sql index 4cff1279c779..c86ad34b0064 100644 --- a/src/test/regress/sql/generated.sql +++ b/src/test/regress/sql/generated.sql @@ -411,7 +411,7 @@ CREATE TABLE gtest30 ( b int GENERATED ALWAYS AS (a * 2) STORED ); CREATE TABLE gtest30_1 () INHERITS (gtest30); -ALTER TABLE ONLY gtest30 ALTER COLUMN b DROP EXPRESSION; +ALTER TABLE ONLY gtest30 ALTER COLUMN b DROP EXPRESSION; -- error \d gtest30 \d gtest30_1 ALTER TABLE gtest30_1 ALTER COLUMN b DROP EXPRESSION; -- error From d907bd0543aa63e59653d7345840bed0f8b3a83b Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 3 Nov 2020 15:41:32 -0500 Subject: [PATCH 433/589] Allow users with BYPASSRLS to alter their own passwords. The intention in commit 491c029db was to require superuserness to change the BYPASSRLS property, but the actual effect of the coding in AlterRole() was to require superuserness to change anything at all about a BYPASSRLS role. Other properties of a BYPASSRLS role should be changeable under the same rules as for a normal role, though. Fix that, and also take care of some documentation omissions related to BYPASSRLS and REPLICATION role properties. Tom Lane and Stephen Frost, per bug report from Wolfgang Walther. Back-patch to all supported branches. Discussion: https://postgr.es/m/a5548a9f-89ee-3167-129d-162b5985fcf8@technowledgy.de --- doc/src/sgml/ref/alter_role.sgml | 4 +++- doc/src/sgml/ref/create_role.sgml | 11 +++++++++-- src/backend/commands/user.c | 10 ++++++---- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/doc/src/sgml/ref/alter_role.sgml b/doc/src/sgml/ref/alter_role.sgml index aef30521bcc5..5aa5648ae7b9 100644 --- a/doc/src/sgml/ref/alter_role.sgml +++ b/doc/src/sgml/ref/alter_role.sgml @@ -71,7 +71,9 @@ ALTER ROLE { role_specification | A Attributes not mentioned in the command retain their previous settings. Database superusers can change any of these settings for any role. Roles having CREATEROLE privilege can change any of these - settings, but only for non-superuser and non-replication roles. + settings except SUPERUSER, REPLICATION, + and BYPASSRLS; but only for non-superuser and + non-replication roles. Ordinary roles can only change their own password. diff --git a/doc/src/sgml/ref/create_role.sgml b/doc/src/sgml/ref/create_role.sgml index d23133945db2..73b30d7b23ce 100644 --- a/doc/src/sgml/ref/create_role.sgml +++ b/doc/src/sgml/ref/create_role.sgml @@ -181,6 +181,8 @@ in sync when changing the above synopsis! highly privileged role, and should only be used on roles actually used for replication. If not specified, NOREPLICATION is the default. + You must be a superuser to create a new role having the + REPLICATION attribute. @@ -192,11 +194,16 @@ in sync when changing the above synopsis! These clauses determine whether a role bypasses every row-level security (RLS) policy. NOBYPASSRLS is the default. + You must be a superuser to create a new role having + the BYPASSRLS attribute. + + + Note that pg_dump will set row_security to OFF by default, to ensure all contents of a table are dumped out. If the user running pg_dump does not have appropriate - permissions, an error will be returned. The superuser and owner of the - table being dumped always bypass RLS. + permissions, an error will be returned. However, superusers and the + owner of the table being dumped always bypass RLS. diff --git a/src/backend/commands/user.c b/src/backend/commands/user.c index 9ce9a6692184..293e7e4c0c1c 100644 --- a/src/backend/commands/user.c +++ b/src/backend/commands/user.c @@ -709,8 +709,10 @@ AlterRole(AlterRoleStmt *stmt) roleid = authform->oid; /* - * To mess with a superuser you gotta be superuser; else you need - * createrole, or just want to change your own password + * To mess with a superuser or replication role in any way you gotta be + * superuser. We also insist on superuser to change the BYPASSRLS + * property. Otherwise, if you don't have createrole, you're only allowed + * to change your own password. */ if (authform->rolsuper || issuper >= 0) { @@ -726,7 +728,7 @@ AlterRole(AlterRoleStmt *stmt) (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be superuser to alter replication users"))); } - else if (authform->rolbypassrls || bypassrls >= 0) + else if (bypassrls >= 0) { if (!superuser()) ereport(ERROR, @@ -735,11 +737,11 @@ AlterRole(AlterRoleStmt *stmt) } else if (!have_createrole_privilege()) { + /* We already checked issuper, isreplication, and bypassrls */ if (!(inherit < 0 && createrole < 0 && createdb < 0 && canlogin < 0 && - isreplication < 0 && !dconnlimit && !rolemembers && !validUntil && From 17fb60387ce3fdc2bbb13d9b67bed0e4da77e173 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 3 Nov 2020 15:49:05 -0500 Subject: [PATCH 434/589] Improve error messages around REPLICATION and BYPASSRLS properties. Clarify wording as per suggestion from Wolfgang Walther. No back-patch; this doesn't seem worth thrashing translatable strings in the back branches. Tom Lane and Stephen Frost Discussion: https://postgr.es/m/a5548a9f-89ee-3167-129d-162b5985fcf8@technowledgy.de --- src/backend/commands/user.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/commands/user.c b/src/backend/commands/user.c index 293e7e4c0c1c..0e6800bf3e4f 100644 --- a/src/backend/commands/user.c +++ b/src/backend/commands/user.c @@ -305,7 +305,7 @@ CreateRole(ParseState *pstate, CreateRoleStmt *stmt) if (!superuser()) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("must be superuser to change bypassrls attribute"))); + errmsg("must be superuser to create bypassrls users"))); } else { @@ -719,14 +719,14 @@ AlterRole(AlterRoleStmt *stmt) if (!superuser()) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("must be superuser to alter superusers"))); + errmsg("must be superuser to alter superuser roles or change superuser attribute"))); } else if (authform->rolreplication || isreplication >= 0) { if (!superuser()) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("must be superuser to alter replication users"))); + errmsg("must be superuser to alter replication roles or change replication attribute"))); } else if (bypassrls >= 0) { From 92f87182f2c617fd420832972b6d0ae4527301c8 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 3 Nov 2020 16:16:36 -0500 Subject: [PATCH 435/589] Guard against core dump from uninitialized subplan. If the planner erroneously puts a non-parallel-safe SubPlan into a parallelized portion of the query tree, nodeSubplan.c will fail in the worker processes because it finds a null in es_subplanstates, which it's unable to cope with. It seems worth a test-and-elog to make that an error case rather than a core dump case. This probably should have been included in commit 16ebab688, which was responsible for allowing nulls to appear in es_subplanstates to begin with. So, back-patch to v10 where that came in. Discussion: https://postgr.es/m/924226.1604422326@sss.pgh.pa.us --- src/backend/executor/nodeSubplan.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/backend/executor/nodeSubplan.c b/src/backend/executor/nodeSubplan.c index 9a706df5f061..152c7ae7eb4e 100644 --- a/src/backend/executor/nodeSubplan.c +++ b/src/backend/executor/nodeSubplan.c @@ -797,7 +797,15 @@ ExecInitSubPlan(SubPlan *subplan, PlanState *parent) sstate->planstate = (PlanState *) list_nth(estate->es_subplanstates, subplan->plan_id - 1); - /* ... and to its parent's state */ + /* + * This check can fail if the planner mistakenly puts a parallel-unsafe + * subplan into a parallelized subquery; see ExecSerializePlan. + */ + if (sstate->planstate == NULL) + elog(ERROR, "subplan \"%s\" was not initialized", + subplan->plan_name); + + /* Link to parent's state, too */ sstate->parent = parent; /* Initialize subexpressions */ From ebb7ae839d033d0f279670e249f54646a08b8c48 Mon Sep 17 00:00:00 2001 From: Tomas Vondra Date: Tue, 3 Nov 2020 20:07:23 +0100 Subject: [PATCH 436/589] Fix get_useful_pathkeys_for_relation for volatile expressions When considering Incremental Sort below a Gather Merge, we need to be a bit more careful when matching pathkeys to EC members. It's not enough to find a member whose Vars are all in the current relation's target; volatile expressions in particular need to be contained in the target, otherwise it's too early to use the pathkey. Reported-by: Jaime Casanova Author: James Coleman Reviewed-by: Tomas Vondra Backpatch-through: 13, where the incremental sort code was added Discussion: https://postgr.es/m/CAJGNTeNaxpXgBVcRhJX%2B2vSbq%2BF2kJqGBcvompmpvXb7pq%2BoFA%40mail.gmail.com --- src/backend/optimizer/path/allpaths.c | 13 +-- src/backend/optimizer/path/equivclass.c | 70 +++++++++++++ src/include/optimizer/paths.h | 1 + .../regress/expected/incremental_sort.out | 98 +++++++++++++++++++ src/test/regress/sql/incremental_sort.sql | 31 ++++++ 5 files changed, 207 insertions(+), 6 deletions(-) diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 8ad6384c6ae8..84a69b064a98 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -2816,7 +2816,8 @@ get_useful_pathkeys_for_relation(PlannerInfo *root, RelOptInfo *rel) /* * Considering query_pathkeys is always worth it, because it might allow * us to avoid a total sort when we have a partially presorted path - * available. + * available or to push the total sort into the parallel portion of the + * query. */ if (root->query_pathkeys) { @@ -2829,17 +2830,17 @@ get_useful_pathkeys_for_relation(PlannerInfo *root, RelOptInfo *rel) EquivalenceClass *pathkey_ec = pathkey->pk_eclass; /* - * We can only build an Incremental Sort for pathkeys which - * contain an EC member in the current relation, so ignore any - * suffix of the list as soon as we find a pathkey without an EC - * member the relation. + * We can only build a sort for pathkeys which contain an EC + * member in the current relation's target, so ignore any suffix + * of the list as soon as we find a pathkey without an EC member + * in the relation. * * By still returning the prefix of the pathkeys list that does * meet criteria of EC membership in the current relation, we * enable not just an incremental sort on the entirety of * query_pathkeys but also incremental sort below a JOIN. */ - if (!find_em_expr_for_rel(pathkey_ec, rel)) + if (!find_em_expr_usable_for_sorting_rel(pathkey_ec, rel)) break; npathkeys++; diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index a21b3b4756cd..f98fd7b3eb8b 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -797,6 +797,76 @@ find_em_expr_for_rel(EquivalenceClass *ec, RelOptInfo *rel) return NULL; } +/* + * Find an equivalence class member expression that can be safely used by a + * sort node on top of the provided relation. The rules here must match those + * applied in prepare_sort_from_pathkeys. + */ +Expr * +find_em_expr_usable_for_sorting_rel(EquivalenceClass *ec, RelOptInfo *rel) +{ + ListCell *lc_em; + + /* + * If there is more than one equivalence member matching these + * requirements we'll be content to choose any one of them. + */ + foreach(lc_em, ec->ec_members) + { + EquivalenceMember *em = lfirst(lc_em); + Expr *em_expr = em->em_expr; + PathTarget *target = rel->reltarget; + ListCell *lc_target_expr; + + /* + * We shouldn't be trying to sort by an equivalence class that + * contains a constant, so no need to consider such cases any further. + */ + if (em->em_is_const) + continue; + + /* + * Any Vars in the equivalence class member need to come from this + * relation. This is a superset of prepare_sort_from_pathkeys ignoring + * child members unless they belong to the rel being sorted. + */ + if (!bms_is_subset(em->em_relids, rel->relids)) + continue; + + /* + * As long as the expression isn't volatile then + * prepare_sort_from_pathkeys is able to generate a new target entry, + * so there's no need to verify that one already exists. + */ + if (!ec->ec_has_volatile) + return em->em_expr; + + /* + * If, however, it's volatile, we have to verify that the + * equivalence member's expr is already generated in the + * relation's target (we do strip relabels first from both + * expressions, which is cheap and might allow us to match + * more expressions). + */ + while (em_expr && IsA(em_expr, RelabelType)) + em_expr = ((RelabelType *) em_expr)->arg; + + foreach(lc_target_expr, target->exprs) + { + Expr *target_expr = lfirst(lc_target_expr); + + while (target_expr && IsA(target_expr, RelabelType)) + target_expr = ((RelabelType *) target_expr)->arg; + + if (equal(target_expr, em_expr)) + return em->em_expr; + } + } + + /* We didn't find any suitable equivalence class expression */ + return NULL; +} + /* * generate_base_implied_equalities * Generate any restriction clauses that we can deduce from equivalence diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 2134227ebcbb..8a4c6f8b59ca 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -135,6 +135,7 @@ extern EquivalenceClass *get_eclass_for_sort_expr(PlannerInfo *root, Relids rel, bool create_it); extern Expr *find_em_expr_for_rel(EquivalenceClass *ec, RelOptInfo *rel); +extern Expr *find_em_expr_usable_for_sorting_rel(EquivalenceClass *ec, RelOptInfo *rel); extern void generate_base_implied_equalities(PlannerInfo *root); extern List *generate_join_implied_equalities(PlannerInfo *root, Relids join_relids, diff --git a/src/test/regress/expected/incremental_sort.out b/src/test/regress/expected/incremental_sort.out index e376ea127617..7cf2eee7c14c 100644 --- a/src/test/regress/expected/incremental_sort.out +++ b/src/test/regress/expected/incremental_sort.out @@ -1469,3 +1469,101 @@ explain (costs off) select * from t union select * from t order by 1,3; (13 rows) drop table t; +-- Sort pushdown can't go below where expressions are part of the rel target. +-- In particular this is interesting for volatile expressions which have to +-- go above joins since otherwise we'll incorrectly use expression evaluations +-- across multiple rows. +set enable_hashagg=off; +set enable_seqscan=off; +set enable_incremental_sort = off; +set parallel_tuple_cost=0; +set parallel_setup_cost=0; +set min_parallel_table_scan_size = 0; +set min_parallel_index_scan_size = 0; +-- Parallel sort below join. +explain (costs off) select distinct sub.unique1, stringu1 +from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub; + QUERY PLAN +-------------------------------------------------------------------------- + Unique + -> Nested Loop + -> Gather Merge + Workers Planned: 2 + -> Sort + Sort Key: tenk1.unique1, tenk1.stringu1 + -> Parallel Index Scan using tenk1_unique1 on tenk1 + -> Function Scan on generate_series +(8 rows) + +explain (costs off) select sub.unique1, stringu1 +from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub +order by 1, 2; + QUERY PLAN +-------------------------------------------------------------------- + Nested Loop + -> Gather Merge + Workers Planned: 2 + -> Sort + Sort Key: tenk1.unique1, tenk1.stringu1 + -> Parallel Index Scan using tenk1_unique1 on tenk1 + -> Function Scan on generate_series +(7 rows) + +-- Parallel sort but with expression that can be safely generated at the base rel. +explain (costs off) select distinct sub.unique1, md5(stringu1) +from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub; + QUERY PLAN +---------------------------------------------------------------------------------------- + Unique + -> Nested Loop + -> Gather Merge + Workers Planned: 2 + -> Sort + Sort Key: tenk1.unique1, (md5((tenk1.stringu1)::text)) COLLATE "C" + -> Parallel Index Scan using tenk1_unique1 on tenk1 + -> Function Scan on generate_series +(8 rows) + +explain (costs off) select sub.unique1, md5(stringu1) +from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub +order by 1, 2; + QUERY PLAN +---------------------------------------------------------------------------------- + Nested Loop + -> Gather Merge + Workers Planned: 2 + -> Sort + Sort Key: tenk1.unique1, (md5((tenk1.stringu1)::text)) COLLATE "C" + -> Parallel Index Scan using tenk1_unique1 on tenk1 + -> Function Scan on generate_series +(7 rows) + +-- Parallel sort but with expression not available until the upper rel. +explain (costs off) select distinct sub.unique1, stringu1 || random()::text +from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub; + QUERY PLAN +--------------------------------------------------------------------------------------------- + Unique + -> Sort + Sort Key: tenk1.unique1, (((tenk1.stringu1)::text || (random())::text)) COLLATE "C" + -> Gather + Workers Planned: 2 + -> Nested Loop + -> Parallel Index Scan using tenk1_unique1 on tenk1 + -> Function Scan on generate_series +(8 rows) + +explain (costs off) select sub.unique1, stringu1 || random()::text +from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub +order by 1, 2; + QUERY PLAN +--------------------------------------------------------------------------------------- + Sort + Sort Key: tenk1.unique1, (((tenk1.stringu1)::text || (random())::text)) COLLATE "C" + -> Gather + Workers Planned: 2 + -> Nested Loop + -> Parallel Index Scan using tenk1_unique1 on tenk1 + -> Function Scan on generate_series +(7 rows) + diff --git a/src/test/regress/sql/incremental_sort.sql b/src/test/regress/sql/incremental_sort.sql index 9c040c90e62d..3ee11c394b91 100644 --- a/src/test/regress/sql/incremental_sort.sql +++ b/src/test/regress/sql/incremental_sort.sql @@ -221,3 +221,34 @@ set enable_hashagg to off; explain (costs off) select * from t union select * from t order by 1,3; drop table t; + +-- Sort pushdown can't go below where expressions are part of the rel target. +-- In particular this is interesting for volatile expressions which have to +-- go above joins since otherwise we'll incorrectly use expression evaluations +-- across multiple rows. +set enable_hashagg=off; +set enable_seqscan=off; +set enable_incremental_sort = off; +set parallel_tuple_cost=0; +set parallel_setup_cost=0; +set min_parallel_table_scan_size = 0; +set min_parallel_index_scan_size = 0; + +-- Parallel sort below join. +explain (costs off) select distinct sub.unique1, stringu1 +from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub; +explain (costs off) select sub.unique1, stringu1 +from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub +order by 1, 2; +-- Parallel sort but with expression that can be safely generated at the base rel. +explain (costs off) select distinct sub.unique1, md5(stringu1) +from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub; +explain (costs off) select sub.unique1, md5(stringu1) +from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub +order by 1, 2; +-- Parallel sort but with expression not available until the upper rel. +explain (costs off) select distinct sub.unique1, stringu1 || random()::text +from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub; +explain (costs off) select sub.unique1, stringu1 || random()::text +from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub +order by 1, 2; From 90851d1d26f54ccb4d7b1bc49449138113d6ec83 Mon Sep 17 00:00:00 2001 From: Tomas Vondra Date: Tue, 3 Nov 2020 20:43:12 +0100 Subject: [PATCH 437/589] Use INT64_FORMAT to print int64 variables in sort debug Commit 6ee3b5fb99 cleaned up most of the long/int64 confusion related to incremental sort, but the sort debug messages were still using %ld for int64 variables. So fix that. Author: Haiying Tang Backpatch-through: 13, where the incremental sort code was added Discussion: https://postgr.es/m/4250be9d350c4992abb722a76e288aef%40G08CNEXMBPEKD05.g08.fujitsu.local --- src/backend/executor/nodeIncrementalSort.c | 28 +++++++++++----------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/backend/executor/nodeIncrementalSort.c b/src/backend/executor/nodeIncrementalSort.c index eb6cc19a60cd..eb1c1326dea2 100644 --- a/src/backend/executor/nodeIncrementalSort.c +++ b/src/backend/executor/nodeIncrementalSort.c @@ -333,7 +333,7 @@ switchToPresortedPrefixMode(PlanState *pstate) */ if (node->bounded) { - SO1_printf("Setting bound on presorted prefix tuplesort to: %ld\n", + SO1_printf("Setting bound on presorted prefix tuplesort to: " INT64_FORMAT "\n", node->bound - node->bound_Done); tuplesort_set_bound(node->prefixsort_state, node->bound - node->bound_Done); @@ -417,9 +417,9 @@ switchToPresortedPrefixMode(PlanState *pstate) * remaining in the large single prefix key group we think we've * encountered. */ - SO1_printf("Moving %ld tuples to presorted prefix tuplesort\n", nTuples); + SO1_printf("Moving " INT64_FORMAT " tuples to presorted prefix tuplesort\n", nTuples); node->n_fullsort_remaining -= nTuples; - SO1_printf("Setting n_fullsort_remaining to %ld\n", node->n_fullsort_remaining); + SO1_printf("Setting n_fullsort_remaining to " INT64_FORMAT "\n", node->n_fullsort_remaining); if (lastTuple) { @@ -449,7 +449,7 @@ switchToPresortedPrefixMode(PlanState *pstate) * out all of those tuples, and then come back around to find another * batch. */ - SO1_printf("Sorting presorted prefix tuplesort with %ld tuples\n", nTuples); + SO1_printf("Sorting presorted prefix tuplesort with " INT64_FORMAT " tuples\n", nTuples); tuplesort_performsort(node->prefixsort_state); INSTRUMENT_SORT_GROUP(node, prefixsort); @@ -462,7 +462,7 @@ switchToPresortedPrefixMode(PlanState *pstate) * - n), so store the current number of processed tuples for use * in configuring sorting bound. */ - SO2_printf("Changing bound_Done from %ld to %ld\n", + SO2_printf("Changing bound_Done from " INT64_FORMAT " to " INT64_FORMAT "\n", Min(node->bound, node->bound_Done + nTuples), node->bound_Done); node->bound_Done = Min(node->bound, node->bound_Done + nTuples); } @@ -574,7 +574,7 @@ ExecIncrementalSort(PlanState *pstate) * need to re-execute the prefix mode transition function to pull * out the next prefix key group. */ - SO1_printf("Re-calling switchToPresortedPrefixMode() because n_fullsort_remaining is > 0 (%ld)\n", + SO1_printf("Re-calling switchToPresortedPrefixMode() because n_fullsort_remaining is > 0 (" INT64_FORMAT ")\n", node->n_fullsort_remaining); switchToPresortedPrefixMode(pstate); } @@ -707,7 +707,7 @@ ExecIncrementalSort(PlanState *pstate) */ node->outerNodeDone = true; - SO1_printf("Sorting fullsort with %ld tuples\n", nTuples); + SO1_printf("Sorting fullsort with " INT64_FORMAT " tuples\n", nTuples); tuplesort_performsort(fullsort_state); INSTRUMENT_SORT_GROUP(node, fullsort); @@ -776,7 +776,7 @@ ExecIncrementalSort(PlanState *pstate) * current number of processed tuples for later use * configuring the sort state's bound. */ - SO2_printf("Changing bound_Done from %ld to %ld\n", + SO2_printf("Changing bound_Done from " INT64_FORMAT " to " INT64_FORMAT "\n", node->bound_Done, Min(node->bound, node->bound_Done + nTuples)); node->bound_Done = Min(node->bound, node->bound_Done + nTuples); @@ -787,7 +787,7 @@ ExecIncrementalSort(PlanState *pstate) * sort and transition modes to reading out the sorted * tuples. */ - SO1_printf("Sorting fullsort tuplesort with %ld tuples\n", + SO1_printf("Sorting fullsort tuplesort with " INT64_FORMAT " tuples\n", nTuples); tuplesort_performsort(fullsort_state); @@ -828,7 +828,7 @@ ExecIncrementalSort(PlanState *pstate) * on FIFO retrieval semantics when transferring them to the * presorted prefix tuplesort. */ - SO1_printf("Sorting fullsort tuplesort with %ld tuples\n", nTuples); + SO1_printf("Sorting fullsort tuplesort with " INT64_FORMAT " tuples\n", nTuples); tuplesort_performsort(fullsort_state); INSTRUMENT_SORT_GROUP(node, fullsort); @@ -847,12 +847,12 @@ ExecIncrementalSort(PlanState *pstate) { int64 currentBound = node->bound - node->bound_Done; - SO2_printf("Read %ld tuples, but setting to %ld because we used bounded sort\n", + SO2_printf("Read " INT64_FORMAT " tuples, but setting to " INT64_FORMAT " because we used bounded sort\n", nTuples, Min(currentBound, nTuples)); nTuples = Min(currentBound, nTuples); } - SO1_printf("Setting n_fullsort_remaining to %ld and calling switchToPresortedPrefixMode()\n", + SO1_printf("Setting n_fullsort_remaining to " INT64_FORMAT " and calling switchToPresortedPrefixMode()\n", nTuples); /* @@ -942,7 +942,7 @@ ExecIncrementalSort(PlanState *pstate) * Perform the sort and begin returning the tuples to the parent plan * node. */ - SO1_printf("Sorting presorted prefix tuplesort with >= %ld tuples\n", nTuples); + SO1_printf("Sorting presorted prefix tuplesort with " INT64_FORMAT " tuples\n", nTuples); tuplesort_performsort(node->prefixsort_state); INSTRUMENT_SORT_GROUP(node, prefixsort); @@ -958,7 +958,7 @@ ExecIncrementalSort(PlanState *pstate) * - n), so store the current number of processed tuples for use * in configuring sorting bound. */ - SO2_printf("Changing bound_Done from %ld to %ld\n", + SO2_printf("Changing bound_Done from " INT64_FORMAT " to " INT64_FORMAT "\n", node->bound_Done, Min(node->bound, node->bound_Done + nTuples)); node->bound_Done = Min(node->bound, node->bound_Done + nTuples); From e152506adef4bc503ea7b8ebb4fedc0b8eebda81 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 4 Nov 2020 10:21:46 +0900 Subject: [PATCH 438/589] Revert pg_relation_check_pages() This reverts the following set of commits, following complaints about the lack of portability of the central part of the code in bufmgr.c as well as the use of partition mapping locks during page reads: c780a7a9 f2b88396 b787d4ce ce7f772c 60a51c6b Per discussion with Andres Freund, Robert Haas and myself. Bump catalog version. Discussion: https://postgr.es/m/20201029181729.2nrub47u7yqncsv7@alap3.anarazel.de --- doc/src/sgml/func.sgml | 56 ------ src/backend/catalog/system_views.sql | 9 - src/backend/storage/buffer/bufmgr.c | 92 ---------- src/backend/utils/adt/Makefile | 1 - src/backend/utils/adt/pagefuncs.c | 229 ----------------------- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.dat | 7 - src/include/storage/bufmgr.h | 3 - src/test/recovery/t/022_page_check.pl | 231 ------------------------ src/test/regress/expected/pagefuncs.out | 72 -------- src/test/regress/parallel_schedule | 2 +- src/test/regress/serial_schedule | 1 - src/test/regress/sql/pagefuncs.sql | 41 ----- src/tools/pgindent/typedefs.list | 1 - 14 files changed, 2 insertions(+), 745 deletions(-) delete mode 100644 src/backend/utils/adt/pagefuncs.c delete mode 100644 src/test/recovery/t/022_page_check.pl delete mode 100644 src/test/regress/expected/pagefuncs.out delete mode 100644 src/test/regress/sql/pagefuncs.sql diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index ee721d258e00..87a6ba8d10fe 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -26171,62 +26171,6 @@ SELECT convert_from(pg_read_binary_file('file_in_utf8.txt'), 'UTF8'); - - Data Sanity Functions - - - The functions shown in - provide ways to check the sanity of data files in the cluster. - - -
- Data Sanity Functions - - - - - Function - - - Description - - - - - - - - - pg_relation_check_pages - - pg_relation_check_pages ( relation regclass [, fork text ] ) - setof record - ( path text, - failed_block_num bigint ) - - - Checks the pages of the specified relation to see if they are valid - enough to safely be loaded into the server's shared buffers. If - given, fork specifies that only the pages of - the given fork are to be verified. fork can - be main for the main data - fork, fsm for the free space - map, vm for the visibility map, - or init for the initialization fork. The - default of NULL means that all forks of the - relation should be checked. The function returns a list of block - numbers that appear corrupted along with the path names of their - files. Use of this function is restricted to superusers by - default, but access may be granted to others - using GRANT. - - - - -
- - - diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 5171ea05c7ea..2e4aa1c4b660 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1303,14 +1303,6 @@ LANGUAGE INTERNAL STRICT VOLATILE AS 'pg_create_logical_replication_slot'; -CREATE OR REPLACE FUNCTION pg_relation_check_pages( - IN relation regclass, IN fork text DEFAULT NULL, - OUT path text, OUT failed_block_num bigint) -RETURNS SETOF record -LANGUAGE internal -VOLATILE PARALLEL RESTRICTED -AS 'pg_relation_check_pages'; - CREATE OR REPLACE FUNCTION make_interval(years int4 DEFAULT 0, months int4 DEFAULT 0, weeks int4 DEFAULT 0, days int4 DEFAULT 0, hours int4 DEFAULT 0, mins int4 DEFAULT 0, @@ -1455,7 +1447,6 @@ AS 'unicode_is_normalized'; -- can later change who can access these functions, or leave them as only -- available to superuser / cluster owner, if they choose. -- -REVOKE EXECUTE ON FUNCTION pg_relation_check_pages(regclass, text) FROM public; REVOKE EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_stop_backup() FROM public; REVOKE EXECUTE ON FUNCTION pg_stop_backup(boolean, boolean) FROM public; diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 0adf04814cd9..ad0d1a9abc01 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -4585,95 +4585,3 @@ TestForOldSnapshot_impl(Snapshot snapshot, Relation relation) (errcode(ERRCODE_SNAPSHOT_TOO_OLD), errmsg("snapshot too old"))); } - - -/* - * CheckBuffer - * - * Check the state of a buffer without loading it into the shared buffers. To - * avoid torn pages and possible false positives when reading data, a shared - * LWLock is taken on the target buffer pool partition mapping, and we check - * if the page is in shared buffers or not. An I/O lock is taken on the block - * to prevent any concurrent activity from happening. - * - * If the page is found as dirty in the shared buffers, it is ignored as - * it will be flushed to disk either before the end of the next checkpoint - * or during recovery in the event of an unsafe shutdown. - * - * If the page is found in the shared buffers but is not dirty, we still - * check the state of its data on disk, as it could be possible that the - * page stayed in shared buffers for a rather long time while the on-disk - * data got corrupted. - * - * If the page is not found in shared buffers, the block is read from disk - * while holding the buffer pool partition mapping LWLock. - * - * The page data is stored in a private memory area local to this function - * while running the checks. - */ -bool -CheckBuffer(SMgrRelation smgr, ForkNumber forknum, BlockNumber blkno) -{ - char buffer[BLCKSZ]; - BufferTag buf_tag; /* identity of requested block */ - uint32 buf_hash; /* hash value for buf_tag */ - LWLock *partLock; /* buffer partition lock for the buffer */ - BufferDesc *bufdesc; - int buf_id; - - Assert(smgrexists(smgr, forknum)); - - /* create a tag so we can look after the buffer */ - INIT_BUFFERTAG(buf_tag, smgr->smgr_rnode.node, forknum, blkno); - - /* determine its hash code and partition lock ID */ - buf_hash = BufTableHashCode(&buf_tag); - partLock = BufMappingPartitionLock(buf_hash); - - /* see if the block is in the buffer pool or not */ - LWLockAcquire(partLock, LW_SHARED); - buf_id = BufTableLookup(&buf_tag, buf_hash); - if (buf_id >= 0) - { - uint32 buf_state; - - /* - * Found it. Now, retrieve its state to know what to do with it, and - * release the pin immediately. We do so to limit overhead as much as - * possible. We keep the shared LWLock on the target buffer mapping - * partition for now, so this buffer cannot be evicted, and we acquire - * an I/O Lock on the buffer as we may need to read its contents from - * disk. - */ - bufdesc = GetBufferDescriptor(buf_id); - - LWLockAcquire(BufferDescriptorGetIOLock(bufdesc), LW_SHARED); - buf_state = LockBufHdr(bufdesc); - UnlockBufHdr(bufdesc, buf_state); - - /* If the page is dirty or invalid, skip it */ - if ((buf_state & BM_DIRTY) != 0 || (buf_state & BM_TAG_VALID) == 0) - { - LWLockRelease(BufferDescriptorGetIOLock(bufdesc)); - LWLockRelease(partLock); - return true; - } - - /* Read the buffer from disk, with the I/O lock still held */ - smgrread(smgr, forknum, blkno, buffer); - LWLockRelease(BufferDescriptorGetIOLock(bufdesc)); - } - else - { - /* - * Simply read the buffer. There's no risk of modification on it as - * we are holding the buffer pool partition mapping lock. - */ - smgrread(smgr, forknum, blkno, buffer); - } - - /* buffer lookup done, so now do its check */ - LWLockRelease(partLock); - - return PageIsVerifiedExtended(buffer, blkno, PIV_REPORT_STAT); -} diff --git a/src/backend/utils/adt/Makefile b/src/backend/utils/adt/Makefile index e2279af1e52b..b4d55e849b3d 100644 --- a/src/backend/utils/adt/Makefile +++ b/src/backend/utils/adt/Makefile @@ -69,7 +69,6 @@ OBJS = \ oid.o \ oracle_compat.o \ orderedsetaggs.o \ - pagefuncs.o \ partitionfuncs.o \ pg_locale.o \ pg_lsn.o \ diff --git a/src/backend/utils/adt/pagefuncs.c b/src/backend/utils/adt/pagefuncs.c deleted file mode 100644 index 368ada515cf7..000000000000 --- a/src/backend/utils/adt/pagefuncs.c +++ /dev/null @@ -1,229 +0,0 @@ -/*------------------------------------------------------------------------- - * - * pagefuncs.c - * Functions for features related to relation pages. - * - * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * - * IDENTIFICATION - * src/backend/utils/adt/pagefuncs.c - * - *------------------------------------------------------------------------- - */ - -#include "postgres.h" - -#include "access/relation.h" -#include "funcapi.h" -#include "miscadmin.h" -#include "storage/bufmgr.h" -#include "storage/lmgr.h" -#include "storage/smgr.h" -#include "utils/builtins.h" -#include "utils/syscache.h" - -static void check_one_relation(TupleDesc tupdesc, Tuplestorestate *tupstore, - Oid relid, ForkNumber single_forknum); -static void check_relation_fork(TupleDesc tupdesc, Tuplestorestate *tupstore, - Relation relation, ForkNumber forknum); - -/* - * callback arguments for check_pages_error_callback() - */ -typedef struct CheckPagesErrorInfo -{ - char *path; - BlockNumber blkno; -} CheckPagesErrorInfo; - -/* - * Error callback specific to check_relation_fork(). - */ -static void -check_pages_error_callback(void *arg) -{ - CheckPagesErrorInfo *errinfo = (CheckPagesErrorInfo *) arg; - - errcontext("while checking page %u of path %s", - errinfo->blkno, errinfo->path); -} - -/* - * pg_relation_check_pages - * - * Check the state of all the pages for one or more fork types in the given - * relation. - */ -Datum -pg_relation_check_pages(PG_FUNCTION_ARGS) -{ - Oid relid; - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - MemoryContext per_query_ctx; - MemoryContext oldcontext; - ForkNumber forknum; - - /* Switch into long-lived context to construct returned data structures */ - per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; - oldcontext = MemoryContextSwitchTo(per_query_ctx); - - /* Build a tuple descriptor for our result type */ - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); - - tupstore = tuplestore_begin_heap(true, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - /* handle arguments */ - if (PG_ARGISNULL(0)) - { - /* Just leave if nothing is defined */ - PG_RETURN_VOID(); - } - - /* By default all the forks of a relation are checked */ - if (PG_ARGISNULL(1)) - forknum = InvalidForkNumber; - else - { - const char *forkname = TextDatumGetCString(PG_GETARG_TEXT_PP(1)); - - forknum = forkname_to_number(forkname); - } - - relid = PG_GETARG_OID(0); - - check_one_relation(tupdesc, tupstore, relid, forknum); - tuplestore_donestoring(tupstore); - - return (Datum) 0; -} - -/* - * Perform the check on a single relation, possibly filtered with a single - * fork. This function will check if the given relation exists or not, as - * a relation could be dropped after checking for the list of relations and - * before getting here, and we don't want to error out in this case. - */ -static void -check_one_relation(TupleDesc tupdesc, Tuplestorestate *tupstore, - Oid relid, ForkNumber single_forknum) -{ - Relation relation; - ForkNumber forknum; - - /* Check if relation exists. leaving if there is no such relation */ - if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(relid))) - return; - - relation = relation_open(relid, AccessShareLock); - - /* - * Sanity checks, returning no results if not supported. Temporary - * relations and relations without storage are out of scope. - */ - if (!RELKIND_HAS_STORAGE(relation->rd_rel->relkind) || - relation->rd_rel->relpersistence == RELPERSISTENCE_TEMP) - { - relation_close(relation, AccessShareLock); - return; - } - - RelationOpenSmgr(relation); - - for (forknum = 0; forknum <= MAX_FORKNUM; forknum++) - { - if (single_forknum != InvalidForkNumber && single_forknum != forknum) - continue; - - if (smgrexists(relation->rd_smgr, forknum)) - check_relation_fork(tupdesc, tupstore, relation, forknum); - } - - relation_close(relation, AccessShareLock); -} - -/* - * For a given relation and fork, do the real work of iterating over all pages - * and doing the check. Caller must hold an AccessShareLock lock on the given - * relation. - */ -static void -check_relation_fork(TupleDesc tupdesc, Tuplestorestate *tupstore, - Relation relation, ForkNumber forknum) -{ - BlockNumber blkno, - nblocks; - SMgrRelation smgr = relation->rd_smgr; - char *path; - CheckPagesErrorInfo errinfo; - ErrorContextCallback errcallback; - - /* Number of output arguments in the SRF */ -#define PG_CHECK_RELATION_COLS 2 - - Assert(CheckRelationLockedByMe(relation, AccessShareLock, true)); - - /* - * We remember the number of blocks here. Since caller must hold a lock - * on the relation, we know that it won't be truncated while we are - * iterating over the blocks. Any block added after this function started - * will not be checked. - */ - nblocks = RelationGetNumberOfBlocksInFork(relation, forknum); - - path = relpathbackend(smgr->smgr_rnode.node, - smgr->smgr_rnode.backend, - forknum); - - /* - * Error context to print some information about blocks and relations - * impacted by corruptions. - */ - errinfo.path = pstrdup(path); - errinfo.blkno = 0; - errcallback.callback = check_pages_error_callback; - errcallback.arg = (void *) &errinfo; - errcallback.previous = error_context_stack; - error_context_stack = &errcallback; - - for (blkno = 0; blkno < nblocks; blkno++) - { - Datum values[PG_CHECK_RELATION_COLS]; - bool nulls[PG_CHECK_RELATION_COLS]; - int i = 0; - - /* Update block number for the error context */ - errinfo.blkno = blkno; - - CHECK_FOR_INTERRUPTS(); - - /* Check the given buffer */ - if (CheckBuffer(smgr, forknum, blkno)) - continue; - - memset(values, 0, sizeof(values)); - memset(nulls, 0, sizeof(nulls)); - - values[i++] = CStringGetTextDatum(path); - values[i++] = Int64GetDatum((int64) blkno); - - Assert(i == PG_CHECK_RELATION_COLS); - - /* Save the corrupted blocks in the tuplestore. */ - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - pfree(path); - - /* Pop the error context stack */ - error_context_stack = errcallback.previous; -} diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 100e6f0a09a9..302b9f6b2bd8 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 202011031 +#define CATALOG_VERSION_NO 202011041 #endif diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index d9770bbadd8b..88378953620c 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10957,13 +10957,6 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, -{ oid => '9147', descr => 'check pages of a relation', - proname => 'pg_relation_check_pages', procost => '10000', prorows => '20', - proisstrict => 'f', proretset => 't', provolatile => 'v', proparallel => 'r', - prorettype => 'record', proargtypes => 'regclass text', - proallargtypes => '{regclass,text,text,int8}', proargmodes => '{i,i,o,o}', - proargnames => '{relation,fork,path,failed_block_num}', - prosrc => 'pg_relation_check_pages' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index a21cab2eaf8c..ee91b8fa26c1 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -240,9 +240,6 @@ extern void AtProcExit_LocalBuffers(void); extern void TestForOldSnapshot_impl(Snapshot snapshot, Relation relation); -extern bool CheckBuffer(struct SMgrRelationData *smgr, ForkNumber forknum, - BlockNumber blkno); - /* in freelist.c */ extern BufferAccessStrategy GetAccessStrategy(BufferAccessStrategyType btype); extern void FreeAccessStrategy(BufferAccessStrategy strategy); diff --git a/src/test/recovery/t/022_page_check.pl b/src/test/recovery/t/022_page_check.pl deleted file mode 100644 index 9dd4225c5a0d..000000000000 --- a/src/test/recovery/t/022_page_check.pl +++ /dev/null @@ -1,231 +0,0 @@ -# Emulate on-disk corruptions of relation pages and find such corruptions -# using pg_relation_check_pages(). - -use strict; -use warnings; - -use PostgresNode; -use TestLib; -use Test::More tests => 20; - -our $CHECKSUM_UINT16_OFFSET = 4; -our $PD_UPPER_UINT16_OFFSET = 7; -our $BLOCKSIZE; -our $TOTAL_NB_ERR = 0; - -# Grab a relation page worth a size of BLOCKSIZE from given $filename. -# $blkno is the same block number as for a relation file. -sub read_page -{ - my ($filename, $blkno) = @_; - my $block; - - open(my $infile, '<', $filename) or die; - binmode($infile); - - my $success = read($infile, $block, $BLOCKSIZE, ($blkno * $BLOCKSIZE)); - die($!) if !defined($success); - - close($infile); - - return ($block); -} - -# Update an existing page of size BLOCKSIZE with new contents in given -# $filename. $blkno is the block number assigned in the relation file. -sub write_page -{ - my ($filename, $block, $blkno) = @_; - - open(my $outfile, '>', $filename) or die; - binmode($outfile); - - my $nb = syswrite($outfile, $block, $BLOCKSIZE, ($blkno * $BLOCKSIZE)); - - die($!) if not defined $nb; - die("Write error") if ($nb != $BLOCKSIZE); - - $outfile->flush(); - - close($outfile); - return; -} - -# Read 2 bytes from relation page at a given offset. -sub get_uint16_from_page -{ - my ($block, $offset) = @_; - - return (unpack("S*", $block))[$offset]; -} - -# Write 2 bytes to relation page at a given offset. -sub set_uint16_to_page -{ - my ($block, $data, $offset) = @_; - - my $pack = pack("S", $data); - - # vec with 16B or more won't preserve endianness. - vec($block, 2 * $offset, 8) = (unpack('C*', $pack))[0]; - vec($block, (2 * $offset) + 1, 8) = (unpack('C*', $pack))[1]; - - return $block; -} - -# Sanity check on pg_stat_database looking after the number of checksum -# failures. -sub check_pg_stat_database -{ - my ($node, $test_prefix) = @_; - - my $stdout = $node->safe_psql('postgres', - "SELECT " - . " sum(checksum_failures)" - . " FROM pg_catalog.pg_stat_database"); - is($stdout, $TOTAL_NB_ERR, - "$test_prefix: pg_stat_database should have $TOTAL_NB_ERR error"); - - return; -} - -# Run a round of page checks for any relation present in this test run. -# $expected_broken is the psql output marking all the pages found as -# corrupted using relname|blkno as format for each tuple returned. $nb -# is the new number of checksum errors added to the global counter -# matched with the contents of pg_stat_database. -# -# Note that this has no need to check system relations as these would have -# no corruptions: this test does not manipulate them and should by no mean -# break the cluster. -sub run_page_checks -{ - my ($node, $num_checksum, $expected_broken, $test_prefix) = @_; - - my $stdout = $node->safe_psql('postgres', - "SELECT relname, failed_block_num" - . " FROM (SELECT relname, (pg_catalog.pg_relation_check_pages(oid)).*" - . " FROM pg_class " - . " WHERE relkind in ('r','i', 'm') AND oid >= 16384) AS s"); - - # Check command result - is($stdout, $expected_broken, - "$test_prefix: output mismatch with pg_relation_check_pages()"); - - $TOTAL_NB_ERR += $num_checksum; - return; -} - -# Perform various tests that modify a specified block at the given -# offset, checking that a page corruption is correctly detected. The -# original contents of the page are restored back once done. -# $broken_pages is the set of pages that are expected to be broken -# as of the returned result of pg_relation_check_pages(). $num_checksum -# is the number of checksum failures expected to be added to the contents -# of pg_stat_database after this function is done. -sub corrupt_and_test_block -{ - my ($node, $filename, $blkno, $offset, $broken_pages, $num_checksum, - $test_prefix) - = @_; - my $fake_data = hex '0x0000'; - - # Stop the server cleanly to flush any pages, and to prevent any - # concurrent updates on what is going to be updated. - $node->stop; - my $original_block = read_page($filename, 0); - my $original_data = get_uint16_from_page($original_block, $offset); - - isnt($original_data, $fake_data, - "$test_prefix: fake data at offset $offset should be different from the existing one" - ); - - my $new_block = set_uint16_to_page($original_block, $fake_data, $offset); - isnt( - $original_data, - get_uint16_from_page($new_block, $offset), - "$test_prefix: The data at offset $offset should have been changed in memory" - ); - - write_page($filename, $new_block, 0); - - my $written_data = get_uint16_from_page(read_page($filename, 0), $offset); - - # Some offline checks to validate that the corrupted data is in place. - isnt($original_data, $written_data, - "$test_prefix: data written at offset $offset should be different from the original one" - ); - is( get_uint16_from_page($new_block, $offset), - $written_data, - "$test_prefix: data written at offset $offset should be the same as the one in memory" - ); - is($written_data, $fake_data, - "$test_prefix: The data written at offset $offset should be the one we wanted to write" - ); - - # The corruption is in place, start the server to run the checks. - $node->start; - run_page_checks($node, $num_checksum, $broken_pages, $test_prefix); - - # Stop the server, put the original page back in place. - $node->stop; - - $new_block = set_uint16_to_page($original_block, $original_data, $offset); - is( $original_data, - get_uint16_from_page($new_block, $offset), - "$test_prefix: data at offset $offset should have been restored in memory" - ); - - write_page($filename, $new_block, 0); - is( $original_data, - get_uint16_from_page(read_page($filename, $blkno), $offset), - "$test_prefix: data at offset $offset should have been restored on disk" - ); - - # There should be no errors now that the contents are back in place. - $node->start; - run_page_checks($node, 0, '', $test_prefix); -} - -# Data checksums are necessary for this test. -my $node = get_new_node('main'); -$node->init(extra => ['--data-checksums']); -$node->start; - -my $stdout = - $node->safe_psql('postgres', "SELECT" . " current_setting('block_size')"); - -$BLOCKSIZE = $stdout; - -# Basic schema to corrupt and check -$node->safe_psql( - 'postgres', q| - CREATE TABLE public.t1(id integer); - INSERT INTO public.t1 SELECT generate_series(1, 100); - CHECKPOINT; -|); - -# Get the path to the relation file that will get manipulated by the -# follow-up tests with some on-disk corruptions. -$stdout = $node->safe_psql('postgres', - "SELECT" - . " current_setting('data_directory') || '/' || pg_relation_filepath('t1')" -); - -my $filename = $stdout; - -# Normal case without corruptions, this passes, with pg_stat_database -# reporting no errors. -check_pg_stat_database($node, 'start'); - -# Test with a modified checksum. -corrupt_and_test_block($node, $filename, 0, $CHECKSUM_UINT16_OFFSET, 't1|0', - 1, 'broken checksum'); - -# Test corruption making the block looking like it validates PageIsNew(). -corrupt_and_test_block($node, $filename, 0, $PD_UPPER_UINT16_OFFSET, 't1|0', - 0, 'new page'); - -# Check that the number of errors in pg_stat_database match what we -# expect with the corruptions previously introduced. -check_pg_stat_database($node, 'end'); diff --git a/src/test/regress/expected/pagefuncs.out b/src/test/regress/expected/pagefuncs.out deleted file mode 100644 index 38a72b01b3fd..000000000000 --- a/src/test/regress/expected/pagefuncs.out +++ /dev/null @@ -1,72 +0,0 @@ --- --- Tests for functions related to relation pages --- --- Restricted to superusers by default -CREATE ROLE regress_pgfunc_user; -SET ROLE regress_pgfunc_user; -SELECT pg_relation_check_pages('pg_class'); -- error -ERROR: permission denied for function pg_relation_check_pages -SELECT pg_relation_check_pages('pg_class', 'main'); -- error -ERROR: permission denied for function pg_relation_check_pages -RESET ROLE; -DROP ROLE regress_pgfunc_user; --- NULL and simple sanity checks -SELECT pg_relation_check_pages(NULL); -- empty result - pg_relation_check_pages -------------------------- -(0 rows) - -SELECT pg_relation_check_pages(NULL, NULL); -- empty result - pg_relation_check_pages -------------------------- -(0 rows) - -SELECT pg_relation_check_pages('pg_class', 'invalid_fork'); -- error -ERROR: invalid fork name -HINT: Valid fork names are "main", "fsm", "vm", and "init". --- Relation types that are supported -CREATE TABLE pgfunc_test_tab (id int); -CREATE INDEX pgfunc_test_ind ON pgfunc_test_tab(id); -INSERT INTO pgfunc_test_tab VALUES (generate_series(1,1000)); -SELECT pg_relation_check_pages('pgfunc_test_tab'); - pg_relation_check_pages -------------------------- -(0 rows) - -SELECT pg_relation_check_pages('pgfunc_test_ind'); - pg_relation_check_pages -------------------------- -(0 rows) - -DROP TABLE pgfunc_test_tab; -CREATE MATERIALIZED VIEW pgfunc_test_matview AS SELECT 1; -SELECT pg_relation_check_pages('pgfunc_test_matview'); - pg_relation_check_pages -------------------------- -(0 rows) - -DROP MATERIALIZED VIEW pgfunc_test_matview; -CREATE SEQUENCE pgfunc_test_seq; -SELECT pg_relation_check_pages('pgfunc_test_seq'); - pg_relation_check_pages -------------------------- -(0 rows) - -DROP SEQUENCE pgfunc_test_seq; --- pg_relation_check_pages() returns no results if passed relations that --- do not support the operation, like relations without storage or temporary --- relations. -CREATE TEMPORARY TABLE pgfunc_test_temp AS SELECT generate_series(1,10) AS a; -SELECT pg_relation_check_pages('pgfunc_test_temp'); - pg_relation_check_pages -------------------------- -(0 rows) - -DROP TABLE pgfunc_test_temp; -CREATE VIEW pgfunc_test_view AS SELECT 1; -SELECT pg_relation_check_pages('pgfunc_test_view'); - pg_relation_check_pages -------------------------- -(0 rows) - -DROP VIEW pgfunc_test_view; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 7a46a132525c..ae89ed7f0b40 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -112,7 +112,7 @@ test: plancache limit plpgsql copy2 temp domain rangefuncs prepare conversion tr # ---------- # Another group of parallel tests # ---------- -test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain pagefuncs +test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain # event triggers cannot run concurrently with any test that runs DDL test: event_trigger diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule index 9a80b80f73ad..525bdc804f61 100644 --- a/src/test/regress/serial_schedule +++ b/src/test/regress/serial_schedule @@ -197,7 +197,6 @@ test: hash_part test: indexing test: partition_aggregate test: partition_info -test: pagefuncs test: tuplesort test: explain test: event_trigger diff --git a/src/test/regress/sql/pagefuncs.sql b/src/test/regress/sql/pagefuncs.sql deleted file mode 100644 index 12d32eeae473..000000000000 --- a/src/test/regress/sql/pagefuncs.sql +++ /dev/null @@ -1,41 +0,0 @@ --- --- Tests for functions related to relation pages --- - --- Restricted to superusers by default -CREATE ROLE regress_pgfunc_user; -SET ROLE regress_pgfunc_user; -SELECT pg_relation_check_pages('pg_class'); -- error -SELECT pg_relation_check_pages('pg_class', 'main'); -- error -RESET ROLE; -DROP ROLE regress_pgfunc_user; - --- NULL and simple sanity checks -SELECT pg_relation_check_pages(NULL); -- empty result -SELECT pg_relation_check_pages(NULL, NULL); -- empty result -SELECT pg_relation_check_pages('pg_class', 'invalid_fork'); -- error - --- Relation types that are supported -CREATE TABLE pgfunc_test_tab (id int); -CREATE INDEX pgfunc_test_ind ON pgfunc_test_tab(id); -INSERT INTO pgfunc_test_tab VALUES (generate_series(1,1000)); -SELECT pg_relation_check_pages('pgfunc_test_tab'); -SELECT pg_relation_check_pages('pgfunc_test_ind'); -DROP TABLE pgfunc_test_tab; - -CREATE MATERIALIZED VIEW pgfunc_test_matview AS SELECT 1; -SELECT pg_relation_check_pages('pgfunc_test_matview'); -DROP MATERIALIZED VIEW pgfunc_test_matview; -CREATE SEQUENCE pgfunc_test_seq; -SELECT pg_relation_check_pages('pgfunc_test_seq'); -DROP SEQUENCE pgfunc_test_seq; - --- pg_relation_check_pages() returns no results if passed relations that --- do not support the operation, like relations without storage or temporary --- relations. -CREATE TEMPORARY TABLE pgfunc_test_temp AS SELECT generate_series(1,10) AS a; -SELECT pg_relation_check_pages('pgfunc_test_temp'); -DROP TABLE pgfunc_test_temp; -CREATE VIEW pgfunc_test_view AS SELECT 1; -SELECT pg_relation_check_pages('pgfunc_test_view'); -DROP VIEW pgfunc_test_view; diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 03c4e0fe5ba5..da3e5f73d0f2 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -332,7 +332,6 @@ CatCacheHeader CatalogId CatalogIndexState ChangeVarNodes_context -CheckPagesErrorInfo CheckPoint CheckPointStmt CheckpointStatsData From 9f12a3b95dd56c897f1aa3d756d8fb419e84a187 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Wed, 4 Nov 2020 14:58:34 +1300 Subject: [PATCH 439/589] Tolerate version lookup failure for old style Windows locale names. Accept that we can't get versions for such locale names for now. Users will need to specify the newer language tag format to enable the collation versioning feature. It's not clear that we can do automatic conversion from the old style to the new style reliably enough for this purpose. Unfortunately, this means that collation versioning probably won't work for the default collation unless you provide something like en-US at initdb or CREATE DATABASE time (though, for reasons not yet understood, it does seem to work on some systems). It'd be nice to find a better solution, or document this quirk if we settle on it, but this should unbreak the 3 failing build farm animals in the meantime. Reviewed-by: David Rowley Reviewed-by: Tom Lane Discussion: https://postgr.es/m/CAEepm%3D0uEQCpfq_%2BLYFBdArCe4Ot98t1aR4eYiYTe%3DyavQygiQ%40mail.gmail.com --- src/backend/utils/adt/pg_locale.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/backend/utils/adt/pg_locale.c b/src/backend/utils/adt/pg_locale.c index 3b0324ce18ab..d5a0169420a3 100644 --- a/src/backend/utils/adt/pg_locale.c +++ b/src/backend/utils/adt/pg_locale.c @@ -1702,10 +1702,22 @@ get_collation_actual_version(char collprovider, const char *collcollate) MultiByteToWideChar(CP_ACP, 0, collcollate, -1, wide_collcollate, LOCALE_NAME_MAX_LENGTH); if (!GetNLSVersionEx(COMPARE_STRING, wide_collcollate, &version)) + { + /* + * GetNLSVersionEx() wants a language tag such as "en-US", not a + * locale name like "English_United States.1252". Until those + * values can be prevented from entering the system, or 100% + * reliably converted to the more useful tag format, tolerate the + * resulting error and report that we have no version data. + */ + if (GetLastError() == ERROR_INVALID_PARAMETER) + return NULL; + ereport(ERROR, (errmsg("could not get collation version for locale \"%s\": error code %lu", collcollate, GetLastError()))); + } collversion = psprintf("%d.%d,%d.%d", (version.dwNLSVersion >> 8) & 0xFFFF, version.dwNLSVersion & 0xFF, From 02d332297f9bfe63476d53a439cc8f3b05bb5e67 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Wed, 4 Nov 2020 14:48:02 +0900 Subject: [PATCH 440/589] Use standard SIGHUP handler in syslogger. Commit 1e53fe0e70 changed background processes so that they use standard SIGHUP handler. Like that, this commit makes syslogger use standard SIGHUP handler to simplify the code. Author: Bharath Rupireddy Reviewed-by: Fujii Masao Discussion: https://postgr.es/m/CALj2ACXPorUqePswDtOeM_s82v9RW32E1fYmOPZ5NuE+TWKj_A@mail.gmail.com --- src/backend/postmaster/syslogger.c | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/src/backend/postmaster/syslogger.c b/src/backend/postmaster/syslogger.c index ffcb54968f14..faa82ec48158 100644 --- a/src/backend/postmaster/syslogger.c +++ b/src/backend/postmaster/syslogger.c @@ -39,6 +39,7 @@ #include "pgstat.h" #include "pgtime.h" #include "postmaster/fork_process.h" +#include "postmaster/interrupt.h" #include "postmaster/postmaster.h" #include "postmaster/syslogger.h" #include "storage/dsm.h" @@ -123,7 +124,6 @@ static CRITICAL_SECTION sysloggerSection; /* * Flags set by interrupt handlers for later service in the main loop. */ -static volatile sig_atomic_t got_SIGHUP = false; static volatile sig_atomic_t rotation_requested = false; @@ -144,7 +144,6 @@ static unsigned int __stdcall pipeThread(void *arg); static void logfile_rotate(bool time_based_rotation, int size_rotation_for); static char *logfile_getname(pg_time_t timestamp, const char *suffix); static void set_next_rotation_time(void); -static void sigHupHandler(SIGNAL_ARGS); static void sigUsr1Handler(SIGNAL_ARGS); static void update_metainfo_datafile(void); @@ -240,7 +239,7 @@ SysLoggerMain(int argc, char *argv[]) * broken backends... */ - pqsignal(SIGHUP, sigHupHandler); /* set flag to read config file */ + pqsignal(SIGHUP, SignalHandlerForConfigReload); /* set flag to read config file */ pqsignal(SIGINT, SIG_IGN); pqsignal(SIGTERM, SIG_IGN); pqsignal(SIGQUIT, SIG_IGN); @@ -324,9 +323,9 @@ SysLoggerMain(int argc, char *argv[]) /* * Process any requests or signals received recently. */ - if (got_SIGHUP) + if (ConfigReloadPending) { - got_SIGHUP = false; + ConfigReloadPending = false; ProcessConfigFile(PGC_SIGHUP); /* @@ -1553,18 +1552,6 @@ RemoveLogrotateSignalFiles(void) unlink(LOGROTATE_SIGNAL_FILE); } -/* SIGHUP: set flag to reload config file */ -static void -sigHupHandler(SIGNAL_ARGS) -{ - int save_errno = errno; - - got_SIGHUP = true; - SetLatch(MyLatch); - - errno = save_errno; -} - /* SIGUSR1: set flag to rotate logfile */ static void sigUsr1Handler(SIGNAL_ARGS) From ac22929a2613e122708bd0172508ac863c51c1cc Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Wed, 4 Nov 2020 16:41:29 +0900 Subject: [PATCH 441/589] Get rid of the dedicated latch for signaling the startup process. This commit gets rid of the dedicated latch for signaling the startup process in favor of using its procLatch, since that comports better with possible generic signal handlers using that latch. Commit 1e53fe0e70 changed background processes so that they use standard SIGHUP handler. Like that, this commit also makes the startup process use standard SIGHUP handler to simplify the code. Author: Fujii Masao Reviewed-by: Bharath Rupireddy, Michael Paquier Discussion: https://postgr.es/m/CALj2ACXPorUqePswDtOeM_s82v9RW32E1fYmOPZ5NuE+TWKj_A@mail.gmail.com --- src/backend/access/transam/xlog.c | 29 ++++++++++++++--------------- src/backend/postmaster/startup.c | 24 +++++------------------- 2 files changed, 19 insertions(+), 34 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 52a67b117015..e081bf926089 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -682,7 +682,7 @@ typedef struct XLogCtlData * WAL replay, if it is waiting for WAL to arrive or failover trigger file * to appear. */ - Latch recoveryWakeupLatch; + Latch *recoveryWakeupLatch; /* * During recovery, we keep a copy of the latest checkpoint record here. @@ -5185,7 +5185,6 @@ XLOGShmemInit(void) SpinLockInit(&XLogCtl->Insert.insertpos_lck); SpinLockInit(&XLogCtl->info_lck); SpinLockInit(&XLogCtl->ulsn_lck); - InitSharedLatch(&XLogCtl->recoveryWakeupLatch); } /* @@ -6122,7 +6121,7 @@ recoveryApplyDelay(XLogReaderState *record) while (true) { - ResetLatch(&XLogCtl->recoveryWakeupLatch); + ResetLatch(MyLatch); /* might change the trigger file's location */ HandleStartupProcInterrupts(); @@ -6146,7 +6145,7 @@ recoveryApplyDelay(XLogReaderState *record) elog(DEBUG2, "recovery apply delay %ld seconds, %d milliseconds", secs, microsecs / 1000); - (void) WaitLatch(&XLogCtl->recoveryWakeupLatch, + (void) WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, secs * 1000L + microsecs / 1000, WAIT_EVENT_RECOVERY_APPLY_DELAY); @@ -6474,11 +6473,11 @@ StartupXLOG(void) } /* - * Take ownership of the wakeup latch if we're going to sleep during - * recovery. + * Advertise our latch that other processes can use to wake us up + * if we're going to sleep during recovery. */ if (ArchiveRecoveryRequested) - OwnLatch(&XLogCtl->recoveryWakeupLatch); + XLogCtl->recoveryWakeupLatch = &MyProc->procLatch; /* Set up XLOG reader facility */ MemSet(&private, 0, sizeof(XLogPageReadPrivate)); @@ -7489,11 +7488,11 @@ StartupXLOG(void) ResetUnloggedRelations(UNLOGGED_RELATION_INIT); /* - * We don't need the latch anymore. It's not strictly necessary to disown - * it, but let's do it for the sake of tidiness. + * We don't need the latch anymore. It's not strictly necessary to reset + * it to NULL, but let's do it for the sake of tidiness. */ if (ArchiveRecoveryRequested) - DisownLatch(&XLogCtl->recoveryWakeupLatch); + XLogCtl->recoveryWakeupLatch = NULL; /* * We are now done reading the xlog from stream. Turn off streaming @@ -12242,12 +12241,12 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, wait_time = wal_retrieve_retry_interval - (secs * 1000 + usecs / 1000); - (void) WaitLatch(&XLogCtl->recoveryWakeupLatch, + (void) WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, wait_time, WAIT_EVENT_RECOVERY_RETRIEVE_RETRY_INTERVAL); - ResetLatch(&XLogCtl->recoveryWakeupLatch); + ResetLatch(MyLatch); now = GetCurrentTimestamp(); } last_fail_time = now; @@ -12498,11 +12497,11 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, * to react to a trigger file promptly and to check if the * WAL receiver is still active. */ - (void) WaitLatch(&XLogCtl->recoveryWakeupLatch, + (void) WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, 5000L, WAIT_EVENT_RECOVERY_WAL_STREAM); - ResetLatch(&XLogCtl->recoveryWakeupLatch); + ResetLatch(MyLatch); break; } @@ -12674,7 +12673,7 @@ CheckPromoteSignal(void) void WakeupRecovery(void) { - SetLatch(&XLogCtl->recoveryWakeupLatch); + SetLatch(XLogCtl->recoveryWakeupLatch); } /* diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c index 64af7b8707cc..eab9c8c4ed33 100644 --- a/src/backend/postmaster/startup.c +++ b/src/backend/postmaster/startup.c @@ -37,7 +37,6 @@ /* * Flags set by interrupt handlers for later service in the redo loop. */ -static volatile sig_atomic_t got_SIGHUP = false; static volatile sig_atomic_t shutdown_requested = false; static volatile sig_atomic_t promote_signaled = false; @@ -49,7 +48,6 @@ static volatile sig_atomic_t in_restore_command = false; /* Signal handlers */ static void StartupProcTriggerHandler(SIGNAL_ARGS); -static void StartupProcSigHupHandler(SIGNAL_ARGS); /* -------------------------------- @@ -64,19 +62,7 @@ StartupProcTriggerHandler(SIGNAL_ARGS) int save_errno = errno; promote_signaled = true; - WakeupRecovery(); - - errno = save_errno; -} - -/* SIGHUP: set flag to re-read config file at next convenient time */ -static void -StartupProcSigHupHandler(SIGNAL_ARGS) -{ - int save_errno = errno; - - got_SIGHUP = true; - WakeupRecovery(); + SetLatch(MyLatch); errno = save_errno; } @@ -91,7 +77,7 @@ StartupProcShutdownHandler(SIGNAL_ARGS) proc_exit(1); else shutdown_requested = true; - WakeupRecovery(); + SetLatch(MyLatch); errno = save_errno; } @@ -137,9 +123,9 @@ HandleStartupProcInterrupts(void) /* * Process any requests or signals received recently. */ - if (got_SIGHUP) + if (ConfigReloadPending) { - got_SIGHUP = false; + ConfigReloadPending = false; StartupRereadConfig(); } @@ -172,7 +158,7 @@ StartupProcessMain(void) /* * Properly accept or ignore signals the postmaster might send us. */ - pqsignal(SIGHUP, StartupProcSigHupHandler); /* reload config file */ + pqsignal(SIGHUP, SignalHandlerForConfigReload); /* reload config file */ pqsignal(SIGINT, SIG_IGN); /* ignore query cancel */ pqsignal(SIGTERM, StartupProcShutdownHandler); /* request shutdown */ /* SIGQUIT handler was already set up by InitPostmasterChild */ From ffb4e27e9c5ea87f9fecb7036dfc7cc1f38169b6 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 4 Nov 2020 10:38:39 +0200 Subject: [PATCH 442/589] pg_rewind: Move syncTargetDirectory() to file_ops.c For consistency. All the other low-level functions that operate on the target directory are in file_ops.c. Reviewed-by: Michael Paquier Discussion: https://www.postgresql.org/message-id/0c5b3783-af52-3ee5-f8fa-6e794061f70d%40iki.fi --- src/bin/pg_rewind/file_ops.c | 19 +++++++++++++++++++ src/bin/pg_rewind/file_ops.h | 1 + src/bin/pg_rewind/pg_rewind.c | 22 +--------------------- src/bin/pg_rewind/pg_rewind.h | 1 + 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/bin/pg_rewind/file_ops.c b/src/bin/pg_rewind/file_ops.c index b3bf091c5461..55439db20bac 100644 --- a/src/bin/pg_rewind/file_ops.c +++ b/src/bin/pg_rewind/file_ops.c @@ -19,6 +19,7 @@ #include #include "common/file_perm.h" +#include "common/file_utils.h" #include "file_ops.h" #include "filemap.h" #include "pg_rewind.h" @@ -266,6 +267,24 @@ remove_target_symlink(const char *path) dstpath); } +/* + * Sync target data directory to ensure that modifications are safely on disk. + * + * We do this once, for the whole data directory, for performance reasons. At + * the end of pg_rewind's run, the kernel is likely to already have flushed + * most dirty buffers to disk. Additionally fsync_pgdata uses a two-pass + * approach (only initiating writeback in the first pass), which often reduces + * the overall amount of IO noticeably. + */ +void +sync_target_dir(void) +{ + if (!do_sync || dry_run) + return; + + fsync_pgdata(datadir_target, PG_VERSION_NUM); +} + /* * Read a file into memory. The file to be read is /. diff --git a/src/bin/pg_rewind/file_ops.h b/src/bin/pg_rewind/file_ops.h index 025f24141c98..d8466385cf5c 100644 --- a/src/bin/pg_rewind/file_ops.h +++ b/src/bin/pg_rewind/file_ops.h @@ -19,6 +19,7 @@ extern void remove_target_file(const char *path, bool missing_ok); extern void truncate_target_file(const char *path, off_t newsize); extern void create_target(file_entry_t *t); extern void remove_target(file_entry_t *t); +extern void sync_target_dir(void); extern char *slurpFile(const char *datadir, const char *path, size_t *filesize); diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 0ec52cb03279..5a7ab764db48 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -20,7 +20,6 @@ #include "catalog/pg_control.h" #include "common/controldata_utils.h" #include "common/file_perm.h" -#include "common/file_utils.h" #include "common/restricted_token.h" #include "common/string.h" #include "fe_utils/recovery_gen.h" @@ -38,7 +37,6 @@ static void createBackupLabel(XLogRecPtr startpoint, TimeLineID starttli, static void digestControlFile(ControlFileData *ControlFile, char *source, size_t size); -static void syncTargetDirectory(void); static void getRestoreCommand(const char *argv0); static void sanityChecks(void); static void findCommonAncestorTimeline(XLogRecPtr *recptr, int *tliIndex); @@ -455,7 +453,7 @@ main(int argc, char **argv) if (showprogress) pg_log_info("syncing target data directory"); - syncTargetDirectory(); + sync_target_dir(); if (writerecoveryconf && !dry_run) WriteRecoveryConfig(conn, datadir_target, @@ -803,24 +801,6 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size) checkControlFile(ControlFile); } -/* - * Sync target data directory to ensure that modifications are safely on disk. - * - * We do this once, for the whole data directory, for performance reasons. At - * the end of pg_rewind's run, the kernel is likely to already have flushed - * most dirty buffers to disk. Additionally fsync_pgdata uses a two-pass - * approach (only initiating writeback in the first pass), which often reduces - * the overall amount of IO noticeably. - */ -static void -syncTargetDirectory(void) -{ - if (!do_sync || dry_run) - return; - - fsync_pgdata(datadir_target, PG_VERSION_NUM); -} - /* * Get value of GUC parameter restore_command from the target cluster. * diff --git a/src/bin/pg_rewind/pg_rewind.h b/src/bin/pg_rewind/pg_rewind.h index 8a9319ed6759..67f90c2a38ce 100644 --- a/src/bin/pg_rewind/pg_rewind.h +++ b/src/bin/pg_rewind/pg_rewind.h @@ -24,6 +24,7 @@ extern char *datadir_source; extern char *connstr_source; extern bool showprogress; extern bool dry_run; +extern bool do_sync; extern int WalSegSz; /* Target history */ From eb00f1d4bf96bdba236bcc089f3ae94db9b7c603 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 4 Nov 2020 11:21:09 +0200 Subject: [PATCH 443/589] Refactor pg_rewind for more clear decision making. Deciding what to do with each file is now a separate step after all the necessary information has been gathered. It is more clear that way. Previously, the decision-making was divided between process_source_file() and process_target_file(), and it was a bit hard to piece together what the overall rules were. Reviewed-by: Kyotaro Horiguchi, Soumyadeep Chakraborty Discussion: https://www.postgresql.org/message-id/0c5b3783-af52-3ee5-f8fa-6e794061f70d%40iki.fi --- src/bin/pg_rewind/copy_fetch.c | 14 +- src/bin/pg_rewind/file_ops.c | 16 +- src/bin/pg_rewind/filemap.c | 572 +++++++++++++++++--------------- src/bin/pg_rewind/filemap.h | 76 +++-- src/bin/pg_rewind/libpq_fetch.c | 12 +- src/bin/pg_rewind/parsexlog.c | 2 +- src/bin/pg_rewind/pg_rewind.c | 8 +- 7 files changed, 386 insertions(+), 314 deletions(-) diff --git a/src/bin/pg_rewind/copy_fetch.c b/src/bin/pg_rewind/copy_fetch.c index 1edab5f18670..e4b8ce6aaf41 100644 --- a/src/bin/pg_rewind/copy_fetch.c +++ b/src/bin/pg_rewind/copy_fetch.c @@ -210,7 +210,7 @@ copy_executeFileMap(filemap_t *map) for (i = 0; i < map->narray; i++) { entry = map->array[i]; - execute_pagemap(&entry->pagemap, entry->path); + execute_pagemap(&entry->target_pages_to_overwrite, entry->path); switch (entry->action) { @@ -219,16 +219,16 @@ copy_executeFileMap(filemap_t *map) break; case FILE_ACTION_COPY: - rewind_copy_file_range(entry->path, 0, entry->newsize, true); + rewind_copy_file_range(entry->path, 0, entry->source_size, true); break; case FILE_ACTION_TRUNCATE: - truncate_target_file(entry->path, entry->newsize); + truncate_target_file(entry->path, entry->source_size); break; case FILE_ACTION_COPY_TAIL: - rewind_copy_file_range(entry->path, entry->oldsize, - entry->newsize, false); + rewind_copy_file_range(entry->path, entry->target_size, + entry->source_size, false); break; case FILE_ACTION_CREATE: @@ -238,6 +238,10 @@ copy_executeFileMap(filemap_t *map) case FILE_ACTION_REMOVE: remove_target(entry); break; + + case FILE_ACTION_UNDECIDED: + pg_fatal("no action decided for \"%s\"", entry->path); + break; } } diff --git a/src/bin/pg_rewind/file_ops.c b/src/bin/pg_rewind/file_ops.c index 55439db20bac..ec37d0b2e0df 100644 --- a/src/bin/pg_rewind/file_ops.c +++ b/src/bin/pg_rewind/file_ops.c @@ -126,8 +126,9 @@ void remove_target(file_entry_t *entry) { Assert(entry->action == FILE_ACTION_REMOVE); + Assert(entry->target_exists); - switch (entry->type) + switch (entry->target_type) { case FILE_TYPE_DIRECTORY: remove_target_dir(entry->path); @@ -140,6 +141,10 @@ remove_target(file_entry_t *entry) case FILE_TYPE_SYMLINK: remove_target_symlink(entry->path); break; + + case FILE_TYPE_UNDEFINED: + pg_fatal("undefined file type for \"%s\"", entry->path); + break; } } @@ -147,21 +152,26 @@ void create_target(file_entry_t *entry) { Assert(entry->action == FILE_ACTION_CREATE); + Assert(!entry->target_exists); - switch (entry->type) + switch (entry->source_type) { case FILE_TYPE_DIRECTORY: create_target_dir(entry->path); break; case FILE_TYPE_SYMLINK: - create_target_symlink(entry->path, entry->link_target); + create_target_symlink(entry->path, entry->source_link_target); break; case FILE_TYPE_REGULAR: /* can't happen. Regular files are created with open_target_file. */ pg_fatal("invalid action (CREATE) for regular file"); break; + + case FILE_TYPE_UNDEFINED: + pg_fatal("undefined file type for \"%s\"", entry->path); + break; } } diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index 1abc257177ef..d756c28ca8af 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -26,6 +26,8 @@ static bool isRelDataFile(const char *path); static char *datasegpath(RelFileNode rnode, ForkNumber forknum, BlockNumber segno); static int path_cmp(const void *a, const void *b); + +static file_entry_t *get_filemap_entry(const char *path, bool create); static int final_filemap_cmp(const void *a, const void *b); static void filemap_list_to_array(filemap_t *map); static bool check_file_excluded(const char *path, bool is_source); @@ -146,33 +148,79 @@ filemap_create(void) filemap = map; } +/* Look up or create entry for 'path' */ +static file_entry_t * +get_filemap_entry(const char *path, bool create) +{ + filemap_t *map = filemap; + file_entry_t *entry; + file_entry_t **e; + file_entry_t key; + file_entry_t *key_ptr; + + if (map->array) + { + key.path = (char *) path; + key_ptr = &key; + e = bsearch(&key_ptr, map->array, map->narray, sizeof(file_entry_t *), + path_cmp); + } + else + e = NULL; + + if (e) + entry = *e; + else if (!create) + entry = NULL; + else + { + /* Create a new entry for this file */ + entry = pg_malloc(sizeof(file_entry_t)); + entry->path = pg_strdup(path); + entry->isrelfile = isRelDataFile(path); + entry->action = FILE_ACTION_UNDECIDED; + + entry->target_exists = false; + entry->target_type = FILE_TYPE_UNDEFINED; + entry->target_size = 0; + entry->target_link_target = NULL; + entry->target_pages_to_overwrite.bitmap = NULL; + entry->target_pages_to_overwrite.bitmapsize = 0; + + entry->source_exists = false; + entry->source_type = FILE_TYPE_UNDEFINED; + entry->source_size = 0; + entry->source_link_target = NULL; + + entry->next = NULL; + + if (map->last) + { + map->last->next = entry; + map->last = entry; + } + else + map->first = map->last = entry; + map->nlist++; + } + + return entry; +} + /* * Callback for processing source file list. * - * This is called once for every file in the source server. We decide what - * action needs to be taken for the file, depending on whether the file - * exists in the target and whether the size matches. + * This is called once for every file in the source server. We record the + * type and size of the file, so that decide_file_action() can later decide what + * to do with it. */ void -process_source_file(const char *path, file_type_t type, size_t newsize, +process_source_file(const char *path, file_type_t type, size_t size, const char *link_target) { - bool exists; - char localpath[MAXPGPATH]; - struct stat statbuf; - filemap_t *map = filemap; - file_action_t action = FILE_ACTION_NONE; - size_t oldsize = 0; file_entry_t *entry; - Assert(map->array == NULL); - - /* - * Skip any files matching the exclusion filters. This has the effect to - * remove all those files on the target. - */ - if (check_file_excluded(path, true)) - return; + Assert(filemap->array == NULL); /* * Pretend that pg_wal is a directory, even if it's really a symlink. We @@ -182,16 +230,6 @@ process_source_file(const char *path, file_type_t type, size_t newsize, if (strcmp(path, "pg_wal") == 0 && type == FILE_TYPE_SYMLINK) type = FILE_TYPE_DIRECTORY; - /* - * Skip temporary files, .../pgsql_tmp/... and .../pgsql_tmp.* in source. - * This has the effect that all temporary files in the destination will be - * removed. - */ - if (strstr(path, "/" PG_TEMP_FILE_PREFIX) != NULL) - return; - if (strstr(path, "/" PG_TEMP_FILES_DIR "/") != NULL) - return; - /* * sanity check: a filename that looks like a data file better be a * regular file @@ -199,158 +237,25 @@ process_source_file(const char *path, file_type_t type, size_t newsize, if (type != FILE_TYPE_REGULAR && isRelDataFile(path)) pg_fatal("data file \"%s\" in source is not a regular file", path); - snprintf(localpath, sizeof(localpath), "%s/%s", datadir_target, path); - - /* Does the corresponding file exist in the target data dir? */ - if (lstat(localpath, &statbuf) < 0) - { - if (errno != ENOENT) - pg_fatal("could not stat file \"%s\": %m", - localpath); - - exists = false; - } - else - exists = true; - - switch (type) - { - case FILE_TYPE_DIRECTORY: - if (exists && !S_ISDIR(statbuf.st_mode) && strcmp(path, "pg_wal") != 0) - { - /* it's a directory in source, but not in target. Strange.. */ - pg_fatal("\"%s\" is not a directory", localpath); - } - - if (!exists) - action = FILE_ACTION_CREATE; - else - action = FILE_ACTION_NONE; - oldsize = 0; - break; - - case FILE_TYPE_SYMLINK: - if (exists && -#ifndef WIN32 - !S_ISLNK(statbuf.st_mode) -#else - !pgwin32_is_junction(localpath) -#endif - ) - { - /* - * It's a symbolic link in source, but not in target. - * Strange.. - */ - pg_fatal("\"%s\" is not a symbolic link", localpath); - } - - if (!exists) - action = FILE_ACTION_CREATE; - else - action = FILE_ACTION_NONE; - oldsize = 0; - break; - - case FILE_TYPE_REGULAR: - if (exists && !S_ISREG(statbuf.st_mode)) - pg_fatal("\"%s\" is not a regular file", localpath); - - if (!exists || !isRelDataFile(path)) - { - /* - * File exists in source, but not in target. Or it's a - * non-data file that we have no special processing for. Copy - * it in toto. - * - * An exception: PG_VERSIONs should be identical, but avoid - * overwriting it for paranoia. - */ - if (pg_str_endswith(path, "PG_VERSION")) - { - action = FILE_ACTION_NONE; - oldsize = statbuf.st_size; - } - else - { - action = FILE_ACTION_COPY; - oldsize = 0; - } - } - else - { - /* - * It's a data file that exists in both. - * - * If it's larger in target, we can truncate it. There will - * also be a WAL record of the truncation in the source - * system, so WAL replay would eventually truncate the target - * too, but we might as well do it now. - * - * If it's smaller in the target, it means that it has been - * truncated in the target, or enlarged in the source, or - * both. If it was truncated in the target, we need to copy - * the missing tail from the source system. If it was enlarged - * in the source system, there will be WAL records in the - * source system for the new blocks, so we wouldn't need to - * copy them here. But we don't know which scenario we're - * dealing with, and there's no harm in copying the missing - * blocks now, so do it now. - * - * If it's the same size, do nothing here. Any blocks modified - * in the target will be copied based on parsing the target - * system's WAL, and any blocks modified in the source will be - * updated after rewinding, when the source system's WAL is - * replayed. - */ - oldsize = statbuf.st_size; - if (oldsize < newsize) - action = FILE_ACTION_COPY_TAIL; - else if (oldsize > newsize) - action = FILE_ACTION_TRUNCATE; - else - action = FILE_ACTION_NONE; - } - break; - } - - /* Create a new entry for this file */ - entry = pg_malloc(sizeof(file_entry_t)); - entry->path = pg_strdup(path); - entry->type = type; - entry->action = action; - entry->oldsize = oldsize; - entry->newsize = newsize; - entry->link_target = link_target ? pg_strdup(link_target) : NULL; - entry->next = NULL; - entry->pagemap.bitmap = NULL; - entry->pagemap.bitmapsize = 0; - entry->isrelfile = isRelDataFile(path); - - if (map->last) - { - map->last->next = entry; - map->last = entry; - } - else - map->first = map->last = entry; - map->nlist++; + /* Remember this source file */ + entry = get_filemap_entry(path, true); + entry->source_exists = true; + entry->source_type = type; + entry->source_size = size; + entry->source_link_target = link_target ? pg_strdup(link_target) : NULL; } /* * Callback for processing target file list. * - * All source files must be already processed before calling this. This only - * marks target data directory's files that didn't exist in the source for - * deletion. + * All source files must be already processed before calling this. We record + * the type and size of file, so that decide_file_action() can later decide + * what to do with it. */ void -process_target_file(const char *path, file_type_t type, size_t oldsize, +process_target_file(const char *path, file_type_t type, size_t size, const char *link_target) { - bool exists; - file_entry_t key; - file_entry_t *key_ptr; filemap_t *map = filemap; file_entry_t *entry; @@ -359,7 +264,6 @@ process_target_file(const char *path, file_type_t type, size_t oldsize, * from the target data folder all paths which have been filtered out from * the source data folder when processing the source files. */ - if (map->array == NULL) { /* on first call, initialize lookup array */ @@ -377,120 +281,77 @@ process_target_file(const char *path, file_type_t type, size_t oldsize, } /* - * Like in process_source_file, pretend that xlog is always a directory. + * Like in process_source_file, pretend that pg_wal is always a directory. */ if (strcmp(path, "pg_wal") == 0 && type == FILE_TYPE_SYMLINK) type = FILE_TYPE_DIRECTORY; - key.path = (char *) path; - key_ptr = &key; - exists = (bsearch(&key_ptr, map->array, map->narray, sizeof(file_entry_t *), - path_cmp) != NULL); - - /* Remove any file or folder that doesn't exist in the source system. */ - if (!exists) - { - entry = pg_malloc(sizeof(file_entry_t)); - entry->path = pg_strdup(path); - entry->type = type; - entry->action = FILE_ACTION_REMOVE; - entry->oldsize = oldsize; - entry->newsize = 0; - entry->link_target = link_target ? pg_strdup(link_target) : NULL; - entry->next = NULL; - entry->pagemap.bitmap = NULL; - entry->pagemap.bitmapsize = 0; - entry->isrelfile = isRelDataFile(path); - - if (map->last == NULL) - map->first = entry; - else - map->last->next = entry; - map->last = entry; - map->nlist++; - } - else - { - /* - * We already handled all files that exist in the source system in - * process_source_file(). - */ - } + /* Remember this target file */ + entry = get_filemap_entry(path, true); + entry->target_exists = true; + entry->target_type = type; + entry->target_size = size; + entry->target_link_target = link_target ? pg_strdup(link_target) : NULL; } /* * This callback gets called while we read the WAL in the target, for every - * block that have changed in the target system. It makes note of all the - * changed blocks in the pagemap of the file. + * block that has changed in the target system. It decides if the given + * 'blkno' in the target relfile needs to be overwritten from the source, and + * if so, records it in 'target_pages_to_overwrite' bitmap. + * + * NOTE: All the files on both systems must have already been added to the + * file map! */ void -process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) +process_target_wal_block_change(ForkNumber forknum, RelFileNode rnode, + BlockNumber blkno) { char *path; - file_entry_t key; - file_entry_t *key_ptr; file_entry_t *entry; BlockNumber blkno_inseg; int segno; - filemap_t *map = filemap; - file_entry_t **e; - Assert(map->array); + Assert(filemap->array); segno = blkno / RELSEG_SIZE; blkno_inseg = blkno % RELSEG_SIZE; path = datasegpath(rnode, forknum, segno); - - key.path = (char *) path; - key_ptr = &key; - - e = bsearch(&key_ptr, map->array, map->narray, sizeof(file_entry_t *), - path_cmp); - if (e) - entry = *e; - else - entry = NULL; + entry = get_filemap_entry(path, false); pfree(path); if (entry) { - Assert(entry->isrelfile); + int64 end_offset; - switch (entry->action) - { - case FILE_ACTION_NONE: - case FILE_ACTION_TRUNCATE: - /* skip if we're truncating away the modified block anyway */ - if ((blkno_inseg + 1) * BLCKSZ <= entry->newsize) - datapagemap_add(&entry->pagemap, blkno_inseg); - break; - - case FILE_ACTION_COPY_TAIL: - - /* - * skip the modified block if it is part of the "tail" that - * we're copying anyway. - */ - if ((blkno_inseg + 1) * BLCKSZ <= entry->oldsize) - datapagemap_add(&entry->pagemap, blkno_inseg); - break; + Assert(entry->isrelfile); - case FILE_ACTION_COPY: - case FILE_ACTION_REMOVE: - break; + if (entry->target_type != FILE_TYPE_REGULAR) + pg_fatal("unexpected page modification for non-regular file \"%s\"", + entry->path); - case FILE_ACTION_CREATE: - pg_fatal("unexpected page modification for directory or symbolic link \"%s\"", entry->path); - } + /* + * If the block beyond the EOF in the source system, no need to + * remember it now, because we're going to truncate it away from the + * target anyway. Also no need to remember the block if it's beyond + * the current EOF in the target system; we will copy it over with the + * "tail" from the source system, anyway. + */ + end_offset = (blkno_inseg + 1) * BLCKSZ; + if (end_offset <= entry->source_size && + end_offset <= entry->target_size) + datapagemap_add(&entry->target_pages_to_overwrite, blkno_inseg); } else { /* * If we don't have any record of this file in the file map, it means - * that it's a relation that doesn't exist in the source system, and - * it was subsequently removed in the target system, too. We can - * safely ignore it. + * that it's a relation that doesn't exist in the source system. It + * could exist in the target system; we haven't moved the target-only + * entries from the linked list to the array yet! But in any case, if + * it doesn't exist in the source it will be removed from the target + * too, and we can safely ignore it. */ } } @@ -505,6 +366,15 @@ check_file_excluded(const char *path, bool is_source) int excludeIdx; const char *filename; + /* + * Skip all temporary files, .../pgsql_tmp/... and .../pgsql_tmp.* + */ + if (strstr(path, "/" PG_TEMP_FILE_PREFIX) != NULL || + strstr(path, "/" PG_TEMP_FILES_DIR "/") != NULL) + { + return true; + } + /* check individual files... */ for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++) { @@ -581,16 +451,6 @@ filemap_list_to_array(filemap_t *map) map->first = map->last = NULL; } -void -filemap_finalize(void) -{ - filemap_t *map = filemap; - - filemap_list_to_array(map); - qsort(map->array, map->narray, sizeof(file_entry_t *), - final_filemap_cmp); -} - static const char * action_to_str(file_action_t action) { @@ -631,26 +491,26 @@ calculate_totals(void) { entry = map->array[i]; - if (entry->type != FILE_TYPE_REGULAR) + if (entry->source_type != FILE_TYPE_REGULAR) continue; - map->total_size += entry->newsize; + map->total_size += entry->source_size; if (entry->action == FILE_ACTION_COPY) { - map->fetch_size += entry->newsize; + map->fetch_size += entry->source_size; continue; } if (entry->action == FILE_ACTION_COPY_TAIL) - map->fetch_size += (entry->newsize - entry->oldsize); + map->fetch_size += (entry->source_size - entry->target_size); - if (entry->pagemap.bitmapsize > 0) + if (entry->target_pages_to_overwrite.bitmapsize > 0) { datapagemap_iterator_t *iter; BlockNumber blk; - iter = datapagemap_iterate(&entry->pagemap); + iter = datapagemap_iterate(&entry->target_pages_to_overwrite); while (datapagemap_next(iter, &blk)) map->fetch_size += BLCKSZ; @@ -670,13 +530,13 @@ print_filemap(void) { entry = map->array[i]; if (entry->action != FILE_ACTION_NONE || - entry->pagemap.bitmapsize > 0) + entry->target_pages_to_overwrite.bitmapsize > 0) { pg_log_debug("%s (%s)", entry->path, action_to_str(entry->action)); - if (entry->pagemap.bitmapsize > 0) - datapagemap_print(&entry->pagemap); + if (entry->target_pages_to_overwrite.bitmapsize > 0) + datapagemap_print(&entry->target_pages_to_overwrite); } } fflush(stdout); @@ -825,3 +685,171 @@ final_filemap_cmp(const void *a, const void *b) else return strcmp(fa->path, fb->path); } + +/* + * Decide what action to perform to a file. + */ +static file_action_t +decide_file_action(file_entry_t *entry) +{ + const char *path = entry->path; + + /* + * Don't touch the control file. It is handled specially, after copying + * all the other files. + */ + if (strcmp(path, "global/pg_control") == 0) + return FILE_ACTION_NONE; + + /* + * Remove all files matching the exclusion filters in the target. + */ + if (check_file_excluded(path, true)) + { + if (entry->target_exists) + return FILE_ACTION_REMOVE; + else + return FILE_ACTION_NONE; + } + + /* + * Handle cases where the file is missing from one of the systems. + */ + if (!entry->target_exists && entry->source_exists) + { + /* + * File exists in source, but not in target. Copy it in toto. (If it's + * a relation data file, WAL replay after rewinding should re-create + * it anyway. But there's no harm in copying it now.) + */ + switch (entry->source_type) + { + case FILE_TYPE_DIRECTORY: + case FILE_TYPE_SYMLINK: + return FILE_ACTION_CREATE; + case FILE_TYPE_REGULAR: + return FILE_ACTION_COPY; + case FILE_TYPE_UNDEFINED: + pg_fatal("unknown file type for \"%s\"", entry->path); + break; + } + } + else if (entry->target_exists && !entry->source_exists) + { + /* File exists in target, but not source. Remove it. */ + return FILE_ACTION_REMOVE; + } + else if (!entry->target_exists && !entry->source_exists) + { + /* + * Doesn't exist in either server. Why does it have an entry in the + * first place?? + */ + Assert(false); + return FILE_ACTION_NONE; + } + + /* + * Otherwise, the file exists on both systems + */ + Assert(entry->target_exists && entry->source_exists); + + if (entry->source_type != entry->target_type) + { + /* But it's a different kind of object. Strange.. */ + pg_fatal("file \"%s\" is of different type in source and target", entry->path); + } + + /* + * PG_VERSION files should be identical on both systems, but avoid + * overwriting them for paranoia. + */ + if (pg_str_endswith(entry->path, "PG_VERSION")) + return FILE_ACTION_NONE; + + switch (entry->source_type) + { + case FILE_TYPE_DIRECTORY: + return FILE_ACTION_NONE; + + case FILE_TYPE_SYMLINK: + + /* + * XXX: Should we check if it points to the same target? + */ + return FILE_ACTION_NONE; + + case FILE_TYPE_REGULAR: + if (!entry->isrelfile) + { + /* + * It's a non-data file that we have no special processing + * for. Copy it in toto. + */ + return FILE_ACTION_COPY; + } + else + { + /* + * It's a data file that exists in both systems. + * + * If it's larger in target, we can truncate it. There will + * also be a WAL record of the truncation in the source + * system, so WAL replay would eventually truncate the target + * too, but we might as well do it now. + * + * If it's smaller in the target, it means that it has been + * truncated in the target, or enlarged in the source, or + * both. If it was truncated in the target, we need to copy + * the missing tail from the source system. If it was enlarged + * in the source system, there will be WAL records in the + * source system for the new blocks, so we wouldn't need to + * copy them here. But we don't know which scenario we're + * dealing with, and there's no harm in copying the missing + * blocks now, so do it now. + * + * If it's the same size, do nothing here. Any blocks modified + * in the target will be copied based on parsing the target + * system's WAL, and any blocks modified in the source will be + * updated after rewinding, when the source system's WAL is + * replayed. + */ + if (entry->target_size < entry->source_size) + return FILE_ACTION_COPY_TAIL; + else if (entry->target_size > entry->source_size) + return FILE_ACTION_TRUNCATE; + else + return FILE_ACTION_NONE; + } + break; + + case FILE_TYPE_UNDEFINED: + pg_fatal("unknown file type for \"%s\"", path); + break; + } + + /* unreachable */ + pg_fatal("could not decide what to do with file \"%s\"", path); +} + +/* + * Decide what to do with each file. + */ +void +decide_file_actions(void) +{ + int i; + + filemap_list_to_array(filemap); + + for (i = 0; i < filemap->narray; i++) + { + file_entry_t *entry = filemap->array[i]; + + entry->action = decide_file_action(entry); + } + + /* Sort the actions to the order that they should be performed */ + qsort(filemap->array, filemap->narray, sizeof(file_entry_t *), + final_filemap_cmp); +} diff --git a/src/bin/pg_rewind/filemap.h b/src/bin/pg_rewind/filemap.h index 0cb7425170c9..3d4235587343 100644 --- a/src/bin/pg_rewind/filemap.h +++ b/src/bin/pg_rewind/filemap.h @@ -14,17 +14,22 @@ /* * For every file found in the local or remote system, we have a file entry - * which says what we are going to do with the file. For relation files, - * there is also a page map, marking pages in the file that were changed - * locally. - * - * The enum values are sorted in the order we want actions to be processed. + * that contains information about the file on both systems. For relation + * files, there is also a page map that marks pages in the file that were + * changed in the target after the last common checkpoint. Each entry also + * contains an 'action' field, which says what we are going to do with the + * file. */ + +/* these enum values are sorted in the order we want actions to be processed */ typedef enum { + FILE_ACTION_UNDECIDED = 0, /* not decided yet */ + FILE_ACTION_CREATE, /* create local directory or symbolic link */ FILE_ACTION_COPY, /* copy whole file, overwriting if exists */ - FILE_ACTION_COPY_TAIL, /* copy tail from 'oldsize' to 'newsize' */ + FILE_ACTION_COPY_TAIL, /* copy tail from 'source_size' to + * 'target_size' */ FILE_ACTION_NONE, /* no action (we might still copy modified * blocks based on the parsed WAL) */ FILE_ACTION_TRUNCATE, /* truncate local file to 'newsize' bytes */ @@ -33,6 +38,8 @@ typedef enum typedef enum { + FILE_TYPE_UNDEFINED = 0, + FILE_TYPE_REGULAR, FILE_TYPE_DIRECTORY, FILE_TYPE_SYMLINK @@ -41,19 +48,34 @@ typedef enum typedef struct file_entry_t { char *path; - file_type_t type; + bool isrelfile; /* is it a relation data file? */ - file_action_t action; + /* + * Status of the file in the target. + */ + bool target_exists; + file_type_t target_type; + size_t target_size; /* for a regular file */ + char *target_link_target; /* for a symlink */ - /* for a regular file */ - size_t oldsize; - size_t newsize; - bool isrelfile; /* is it a relation data file? */ + /* + * Pages that were modified in the target and need to be replaced from the + * source. + */ + datapagemap_t target_pages_to_overwrite; - datapagemap_t pagemap; + /* + * Status of the file in the source. + */ + bool source_exists; + file_type_t source_type; + size_t source_size; + char *source_link_target; /* for a symlink */ - /* for a symlink */ - char *link_target; + /* + * What will we do to the file? + */ + file_action_t action; struct file_entry_t *next; } file_entry_t; @@ -71,19 +93,18 @@ typedef struct filemap_t /* * After processing all the remote files, the entries in the linked list * are moved to this array. After processing local files, too, all the - * local entries are added to the array by filemap_finalize, and sorted in - * the final order. After filemap_finalize, all the entries are in the - * array, and the linked list is empty. + * local entries are added to the array by decide_file_actions(), and + * sorted in the final order. After decide_file_actions(), all the entries + * are in the array, and the linked list is empty. */ file_entry_t **array; int narray; /* current length of array */ /* - * Summary information. total_size is the total size of the source - * cluster, and fetch_size is the number of bytes that needs to be copied. + * Summary information. */ - uint64 total_size; - uint64 fetch_size; + uint64 total_size; /* total size of the source cluster */ + uint64 fetch_size; /* number of bytes that needs to be copied */ } filemap_t; extern filemap_t *filemap; @@ -94,11 +115,12 @@ extern void print_filemap(void); /* Functions for populating the filemap */ extern void process_source_file(const char *path, file_type_t type, - size_t newsize, const char *link_target); + size_t size, const char *link_target); extern void process_target_file(const char *path, file_type_t type, - size_t newsize, const char *link_target); -extern void process_block_change(ForkNumber forknum, RelFileNode rnode, - BlockNumber blkno); -extern void filemap_finalize(void); + size_t size, const char *link_target); +extern void process_target_wal_block_change(ForkNumber forknum, + RelFileNode rnode, + BlockNumber blkno); +extern void decide_file_actions(void); #endif /* FILEMAP_H */ diff --git a/src/bin/pg_rewind/libpq_fetch.c b/src/bin/pg_rewind/libpq_fetch.c index bf4dfc23b963..2fc4a784bdb3 100644 --- a/src/bin/pg_rewind/libpq_fetch.c +++ b/src/bin/pg_rewind/libpq_fetch.c @@ -465,7 +465,7 @@ libpq_executeFileMap(filemap_t *map) entry = map->array[i]; /* If this is a relation file, copy the modified blocks */ - execute_pagemap(&entry->pagemap, entry->path); + execute_pagemap(&entry->target_pages_to_overwrite, entry->path); switch (entry->action) { @@ -476,15 +476,15 @@ libpq_executeFileMap(filemap_t *map) case FILE_ACTION_COPY: /* Truncate the old file out of the way, if any */ open_target_file(entry->path, true); - fetch_file_range(entry->path, 0, entry->newsize); + fetch_file_range(entry->path, 0, entry->source_size); break; case FILE_ACTION_TRUNCATE: - truncate_target_file(entry->path, entry->newsize); + truncate_target_file(entry->path, entry->source_size); break; case FILE_ACTION_COPY_TAIL: - fetch_file_range(entry->path, entry->oldsize, entry->newsize); + fetch_file_range(entry->path, entry->target_size, entry->source_size); break; case FILE_ACTION_REMOVE: @@ -494,6 +494,10 @@ libpq_executeFileMap(filemap_t *map) case FILE_ACTION_CREATE: create_target(entry); break; + + case FILE_ACTION_UNDECIDED: + pg_fatal("no action decided for \"%s\"", entry->path); + break; } } diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index a2f1ab5422bc..eae1797f948f 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -436,6 +436,6 @@ extractPageInfo(XLogReaderState *record) if (forknum != MAIN_FORKNUM) continue; - process_block_change(forknum, rnode, blkno); + process_target_wal_block_change(forknum, rnode, blkno); } } diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 5a7ab764db48..4760090d06e4 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -369,7 +369,7 @@ main(int argc, char **argv) chkpttli); /* - * Build the filemap, by comparing the source and target data directories. + * Collect information about all files in the target and source systems. */ filemap_create(); if (showprogress) @@ -390,8 +390,12 @@ main(int argc, char **argv) pg_log_info("reading WAL in target"); extractPageMap(datadir_target, chkptrec, lastcommontliIndex, ControlFile_target.checkPoint, restore_command); - filemap_finalize(); + /* + * We have collected all information we need from both systems. Decide + * what to do with each file. + */ + decide_file_actions(); if (showprogress) calculate_totals(); From f81e97d0475cd4bc597adc23b665bd84fbf79a0d Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 4 Nov 2020 11:21:14 +0200 Subject: [PATCH 444/589] pg_rewind: Replace the hybrid list+array data structure with simplehash. Now that simplehash can be used in frontend code, let's make use of it. Reviewed-by: Kyotaro Horiguchi, Soumyadeep Chakraborty Discussion: https://www.postgresql.org/message-id/0c5b3783-af52-3ee5-f8fa-6e794061f70d%40iki.fi --- src/bin/pg_rewind/copy_fetch.c | 4 +- src/bin/pg_rewind/fetch.c | 2 +- src/bin/pg_rewind/fetch.h | 2 +- src/bin/pg_rewind/filemap.c | 294 +++++++++++++++----------------- src/bin/pg_rewind/filemap.h | 67 +++----- src/bin/pg_rewind/libpq_fetch.c | 4 +- src/bin/pg_rewind/pg_rewind.c | 14 +- 7 files changed, 176 insertions(+), 211 deletions(-) diff --git a/src/bin/pg_rewind/copy_fetch.c b/src/bin/pg_rewind/copy_fetch.c index e4b8ce6aaf41..1cd4449314d1 100644 --- a/src/bin/pg_rewind/copy_fetch.c +++ b/src/bin/pg_rewind/copy_fetch.c @@ -207,9 +207,9 @@ copy_executeFileMap(filemap_t *map) file_entry_t *entry; int i; - for (i = 0; i < map->narray; i++) + for (i = 0; i < map->nentries; i++) { - entry = map->array[i]; + entry = map->entries[i]; execute_pagemap(&entry->target_pages_to_overwrite, entry->path); switch (entry->action) diff --git a/src/bin/pg_rewind/fetch.c b/src/bin/pg_rewind/fetch.c index f18fe5386ed4..f41d0f295ea2 100644 --- a/src/bin/pg_rewind/fetch.c +++ b/src/bin/pg_rewind/fetch.c @@ -37,7 +37,7 @@ fetchSourceFileList(void) * Fetch all relation data files that are marked in the given data page map. */ void -executeFileMap(void) +execute_file_actions(filemap_t *filemap) { if (datadir_source) copy_executeFileMap(filemap); diff --git a/src/bin/pg_rewind/fetch.h b/src/bin/pg_rewind/fetch.h index 7cf8b6ea090d..b20df8b15372 100644 --- a/src/bin/pg_rewind/fetch.h +++ b/src/bin/pg_rewind/fetch.h @@ -25,7 +25,7 @@ */ extern void fetchSourceFileList(void); extern char *fetchFile(const char *filename, size_t *filesize); -extern void executeFileMap(void); +extern void execute_file_actions(filemap_t *filemap); /* in libpq_fetch.c */ extern void libpqProcessFileList(void); diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index d756c28ca8af..314b064b2233 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -3,6 +3,19 @@ * filemap.c * A data structure for keeping track of files that have changed. * + * This source file contains the logic to decide what to do with different + * kinds of files, and the data structure to support it. Before modifying + * anything, pg_rewind collects information about all the files and their + * attributes in the target and source data directories. It also scans the + * WAL log in the target, and collects information about data blocks that + * were changed. All this information is stored in a hash table, using the + * file path relative to the root of the data directory as the key. + * + * After collecting all the information required, the decide_file_actions() + * function scans the hash table and decides what action needs to be taken + * for each file. Finally, it sorts the array to the final order that the + * actions should be executed in. + * * Copyright (c) 2013-2020, PostgreSQL Global Development Group * *------------------------------------------------------------------------- @@ -14,22 +27,41 @@ #include #include "catalog/pg_tablespace_d.h" +#include "common/hashfn.h" #include "common/string.h" #include "datapagemap.h" #include "filemap.h" #include "pg_rewind.h" #include "storage/fd.h" -filemap_t *filemap = NULL; +/* + * Define a hash table which we can use to store information about the files + * appearing in source and target systems. + */ +static uint32 hash_string_pointer(const char *s); +#define SH_PREFIX filehash +#define SH_ELEMENT_TYPE file_entry_t +#define SH_KEY_TYPE const char * +#define SH_KEY path +#define SH_HASH_KEY(tb, key) hash_string_pointer(key) +#define SH_EQUAL(tb, a, b) (strcmp(a, b) == 0) +#define SH_SCOPE static inline +#define SH_RAW_ALLOCATOR pg_malloc0 +#define SH_DECLARE +#define SH_DEFINE +#include "lib/simplehash.h" + +#define FILEHASH_INITIAL_SIZE 1000 + +static filehash_hash *filehash; static bool isRelDataFile(const char *path); static char *datasegpath(RelFileNode rnode, ForkNumber forknum, BlockNumber segno); -static int path_cmp(const void *a, const void *b); -static file_entry_t *get_filemap_entry(const char *path, bool create); +static file_entry_t *insert_filehash_entry(const char *path); +static file_entry_t *lookup_filehash_entry(const char *path); static int final_filemap_cmp(const void *a, const void *b); -static void filemap_list_to_array(filemap_t *map); static bool check_file_excluded(const char *path, bool is_source); /* @@ -131,54 +163,26 @@ static const struct exclude_list_item excludeFiles[] = }; /* - * Create a new file map (stored in the global pointer "filemap"). + * Initialize the hash table for the file map. */ void -filemap_create(void) +filehash_init(void) { - filemap_t *map; - - map = pg_malloc(sizeof(filemap_t)); - map->first = map->last = NULL; - map->nlist = 0; - map->array = NULL; - map->narray = 0; - - Assert(filemap == NULL); - filemap = map; + filehash = filehash_create(FILEHASH_INITIAL_SIZE, NULL); } -/* Look up or create entry for 'path' */ +/* Look up entry for 'path', creating a new one if it doesn't exist */ static file_entry_t * -get_filemap_entry(const char *path, bool create) +insert_filehash_entry(const char *path) { - filemap_t *map = filemap; file_entry_t *entry; - file_entry_t **e; - file_entry_t key; - file_entry_t *key_ptr; - - if (map->array) - { - key.path = (char *) path; - key_ptr = &key; - e = bsearch(&key_ptr, map->array, map->narray, sizeof(file_entry_t *), - path_cmp); - } - else - e = NULL; + bool found; - if (e) - entry = *e; - else if (!create) - entry = NULL; - else + entry = filehash_insert(filehash, path, &found); + if (!found) { - /* Create a new entry for this file */ - entry = pg_malloc(sizeof(file_entry_t)); entry->path = pg_strdup(path); entry->isrelfile = isRelDataFile(path); - entry->action = FILE_ACTION_UNDECIDED; entry->target_exists = false; entry->target_type = FILE_TYPE_UNDEFINED; @@ -192,21 +196,18 @@ get_filemap_entry(const char *path, bool create) entry->source_size = 0; entry->source_link_target = NULL; - entry->next = NULL; - - if (map->last) - { - map->last->next = entry; - map->last = entry; - } - else - map->first = map->last = entry; - map->nlist++; + entry->action = FILE_ACTION_UNDECIDED; } return entry; } +static file_entry_t * +lookup_filehash_entry(const char *path) +{ + return filehash_lookup(filehash, path); +} + /* * Callback for processing source file list. * @@ -220,8 +221,6 @@ process_source_file(const char *path, file_type_t type, size_t size, { file_entry_t *entry; - Assert(filemap->array == NULL); - /* * Pretend that pg_wal is a directory, even if it's really a symlink. We * don't want to mess with the symlink itself, nor complain if it's a @@ -238,7 +237,9 @@ process_source_file(const char *path, file_type_t type, size_t size, pg_fatal("data file \"%s\" in source is not a regular file", path); /* Remember this source file */ - entry = get_filemap_entry(path, true); + entry = insert_filehash_entry(path); + if (entry->source_exists) + pg_fatal("duplicate source file \"%s\"", path); entry->source_exists = true; entry->source_type = type; entry->source_size = size; @@ -248,15 +249,12 @@ process_source_file(const char *path, file_type_t type, size_t size, /* * Callback for processing target file list. * - * All source files must be already processed before calling this. We record - * the type and size of file, so that decide_file_action() can later decide - * what to do with it. + * Record the type and size of the file, like process_source_file() does. */ void process_target_file(const char *path, file_type_t type, size_t size, const char *link_target) { - filemap_t *map = filemap; file_entry_t *entry; /* @@ -264,21 +262,6 @@ process_target_file(const char *path, file_type_t type, size_t size, * from the target data folder all paths which have been filtered out from * the source data folder when processing the source files. */ - if (map->array == NULL) - { - /* on first call, initialize lookup array */ - if (map->nlist == 0) - { - /* should not happen */ - pg_fatal("source file list is empty"); - } - - filemap_list_to_array(map); - - Assert(map->array != NULL); - - qsort(map->array, map->narray, sizeof(file_entry_t *), path_cmp); - } /* * Like in process_source_file, pretend that pg_wal is always a directory. @@ -287,7 +270,9 @@ process_target_file(const char *path, file_type_t type, size_t size, type = FILE_TYPE_DIRECTORY; /* Remember this target file */ - entry = get_filemap_entry(path, true); + entry = insert_filehash_entry(path); + if (entry->target_exists) + pg_fatal("duplicate source file \"%s\"", path); entry->target_exists = true; entry->target_type = type; entry->target_size = size; @@ -301,7 +286,7 @@ process_target_file(const char *path, file_type_t type, size_t size, * if so, records it in 'target_pages_to_overwrite' bitmap. * * NOTE: All the files on both systems must have already been added to the - * file map! + * hash table! */ void process_target_wal_block_change(ForkNumber forknum, RelFileNode rnode, @@ -312,47 +297,45 @@ process_target_wal_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno_inseg; int segno; - Assert(filemap->array); - segno = blkno / RELSEG_SIZE; blkno_inseg = blkno % RELSEG_SIZE; path = datasegpath(rnode, forknum, segno); - entry = get_filemap_entry(path, false); + entry = lookup_filehash_entry(path); pfree(path); + /* + * If the block still exists in both systems, remember it. Otherwise we + * can safely ignore it. + * + * If the block is beyond the EOF in the source system, or the file + * doesn't exist in the source at all, we're going to truncate/remove it + * away from the target anyway. Likewise, if it doesn't exist in the + * target anymore, we will copy it over with the "tail" from the source + * system, anyway. + * + * It is possible to find WAL for a file that doesn't exist on either + * system anymore. It means that the relation was dropped later in the + * target system, and independently on the source system too, or that it + * was created and dropped in the target system and it never existed in + * the source. Either way, we can safely ignore it. + */ if (entry) { - int64 end_offset; - Assert(entry->isrelfile); if (entry->target_type != FILE_TYPE_REGULAR) pg_fatal("unexpected page modification for non-regular file \"%s\"", entry->path); - /* - * If the block beyond the EOF in the source system, no need to - * remember it now, because we're going to truncate it away from the - * target anyway. Also no need to remember the block if it's beyond - * the current EOF in the target system; we will copy it over with the - * "tail" from the source system, anyway. - */ - end_offset = (blkno_inseg + 1) * BLCKSZ; - if (end_offset <= entry->source_size && - end_offset <= entry->target_size) - datapagemap_add(&entry->target_pages_to_overwrite, blkno_inseg); - } - else - { - /* - * If we don't have any record of this file in the file map, it means - * that it's a relation that doesn't exist in the source system. It - * could exist in the target system; we haven't moved the target-only - * entries from the linked list to the array yet! But in any case, if - * it doesn't exist in the source it will be removed from the target - * too, and we can safely ignore it. - */ + if (entry->target_exists && entry->source_exists) + { + off_t end_offset; + + end_offset = (blkno_inseg + 1) * BLCKSZ; + if (end_offset <= entry->source_size && end_offset <= entry->target_size) + datapagemap_add(&entry->target_pages_to_overwrite, blkno_inseg); + } } } @@ -423,34 +406,6 @@ check_file_excluded(const char *path, bool is_source) return false; } -/* - * Convert the linked list of entries in map->first/last to the array, - * map->array. - */ -static void -filemap_list_to_array(filemap_t *map) -{ - int narray; - file_entry_t *entry, - *next; - - map->array = (file_entry_t **) - pg_realloc(map->array, - (map->nlist + map->narray) * sizeof(file_entry_t *)); - - narray = map->narray; - for (entry = map->first; entry != NULL; entry = next) - { - map->array[narray++] = entry; - next = entry->next; - entry->next = NULL; - } - Assert(narray == map->nlist + map->narray); - map->narray = narray; - map->nlist = 0; - map->first = map->last = NULL; -} - static const char * action_to_str(file_action_t action) { @@ -478,32 +433,31 @@ action_to_str(file_action_t action) * Calculate the totals needed for progress reports. */ void -calculate_totals(void) +calculate_totals(filemap_t *filemap) { file_entry_t *entry; int i; - filemap_t *map = filemap; - map->total_size = 0; - map->fetch_size = 0; + filemap->total_size = 0; + filemap->fetch_size = 0; - for (i = 0; i < map->narray; i++) + for (i = 0; i < filemap->nentries; i++) { - entry = map->array[i]; + entry = filemap->entries[i]; if (entry->source_type != FILE_TYPE_REGULAR) continue; - map->total_size += entry->source_size; + filemap->total_size += entry->source_size; if (entry->action == FILE_ACTION_COPY) { - map->fetch_size += entry->source_size; + filemap->fetch_size += entry->source_size; continue; } if (entry->action == FILE_ACTION_COPY_TAIL) - map->fetch_size += (entry->source_size - entry->target_size); + filemap->fetch_size += (entry->source_size - entry->target_size); if (entry->target_pages_to_overwrite.bitmapsize > 0) { @@ -512,7 +466,7 @@ calculate_totals(void) iter = datapagemap_iterate(&entry->target_pages_to_overwrite); while (datapagemap_next(iter, &blk)) - map->fetch_size += BLCKSZ; + filemap->fetch_size += BLCKSZ; pg_free(iter); } @@ -520,15 +474,14 @@ calculate_totals(void) } void -print_filemap(void) +print_filemap(filemap_t *filemap) { - filemap_t *map = filemap; file_entry_t *entry; int i; - for (i = 0; i < map->narray; i++) + for (i = 0; i < filemap->nentries; i++) { - entry = map->array[i]; + entry = filemap->entries[i]; if (entry->action != FILE_ACTION_NONE || entry->target_pages_to_overwrite.bitmapsize > 0) { @@ -650,15 +603,6 @@ datasegpath(RelFileNode rnode, ForkNumber forknum, BlockNumber segno) return path; } -static int -path_cmp(const void *a, const void *b) -{ - file_entry_t *fa = *((file_entry_t **) a); - file_entry_t *fb = *((file_entry_t **) b); - - return strcmp(fa->path, fb->path); -} - /* * In the final stage, the filemap is sorted so that removals come last. * From disk space usage point of view, it would be better to do removals @@ -834,22 +778,52 @@ decide_file_action(file_entry_t *entry) /* * Decide what to do with each file. + * + * Returns a 'filemap' with the entries in the order that their actions + * should be executed. */ -void +filemap_t * decide_file_actions(void) { int i; + filehash_iterator it; + file_entry_t *entry; + filemap_t *filemap; - filemap_list_to_array(filemap); - - for (i = 0; i < filemap->narray; i++) + filehash_start_iterate(filehash, &it); + while ((entry = filehash_iterate(filehash, &it)) != NULL) { - file_entry_t *entry = filemap->array[i]; - entry->action = decide_file_action(entry); } - /* Sort the actions to the order that they should be performed */ - qsort(filemap->array, filemap->narray, sizeof(file_entry_t *), + /* + * Turn the hash table into an array, and sort in the order that the + * actions should be performed. + */ + filemap = pg_malloc(offsetof(filemap_t, entries) + + filehash->members * sizeof(file_entry_t *)); + filemap->nentries = filehash->members; + filehash_start_iterate(filehash, &it); + i = 0; + while ((entry = filehash_iterate(filehash, &it)) != NULL) + { + filemap->entries[i++] = entry; + } + + qsort(&filemap->entries, filemap->nentries, sizeof(file_entry_t *), final_filemap_cmp); + + return filemap; +} + + +/* + * Helper function for filemap hash table. + */ +static uint32 +hash_string_pointer(const char *s) +{ + unsigned char *ss = (unsigned char *) s; + + return hash_bytes(ss, strlen(s)); } diff --git a/src/bin/pg_rewind/filemap.h b/src/bin/pg_rewind/filemap.h index 3d4235587343..6f03447d7ebf 100644 --- a/src/bin/pg_rewind/filemap.h +++ b/src/bin/pg_rewind/filemap.h @@ -12,15 +12,6 @@ #include "storage/block.h" #include "storage/relfilenode.h" -/* - * For every file found in the local or remote system, we have a file entry - * that contains information about the file on both systems. For relation - * files, there is also a page map that marks pages in the file that were - * changed in the target after the last common checkpoint. Each entry also - * contains an 'action' field, which says what we are going to do with the - * file. - */ - /* these enum values are sorted in the order we want actions to be processed */ typedef enum { @@ -45,9 +36,21 @@ typedef enum FILE_TYPE_SYMLINK } file_type_t; +/* + * For every file found in the local or remote system, we have a file entry + * that contains information about the file on both systems. For relation + * files, there is also a page map that marks pages in the file that were + * changed in the target after the last common checkpoint. + * + * When gathering information, these are kept in a hash table, private to + * filemap.c. decide_file_actions() fills in the 'action' field, sorts all + * the entries, and returns them in an array, ready for executing the actions. + */ typedef struct file_entry_t { - char *path; + uint32 status; /* hash status */ + + const char *path; bool isrelfile; /* is it a relation data file? */ /* @@ -76,44 +79,25 @@ typedef struct file_entry_t * What will we do to the file? */ file_action_t action; - - struct file_entry_t *next; } file_entry_t; +/* + * This contains the final decisions on what to do with each file. + * 'entries' array contains an entry for each file, sorted in the order + * that their actions should executed. + */ typedef struct filemap_t { - /* - * New entries are accumulated to a linked list, in process_source_file - * and process_target_file. - */ - file_entry_t *first; - file_entry_t *last; - int nlist; /* number of entries currently in list */ - - /* - * After processing all the remote files, the entries in the linked list - * are moved to this array. After processing local files, too, all the - * local entries are added to the array by decide_file_actions(), and - * sorted in the final order. After decide_file_actions(), all the entries - * are in the array, and the linked list is empty. - */ - file_entry_t **array; - int narray; /* current length of array */ - - /* - * Summary information. - */ + /* Summary information, filled by calculate_totals() */ uint64 total_size; /* total size of the source cluster */ uint64 fetch_size; /* number of bytes that needs to be copied */ -} filemap_t; -extern filemap_t *filemap; - -extern void filemap_create(void); -extern void calculate_totals(void); -extern void print_filemap(void); + int nentries; /* size of 'entries' array */ + file_entry_t *entries[FLEXIBLE_ARRAY_MEMBER]; +} filemap_t; /* Functions for populating the filemap */ +extern void filehash_init(void); extern void process_source_file(const char *path, file_type_t type, size_t size, const char *link_target); extern void process_target_file(const char *path, file_type_t type, @@ -121,6 +105,9 @@ extern void process_target_file(const char *path, file_type_t type, extern void process_target_wal_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno); -extern void decide_file_actions(void); + +extern filemap_t *decide_file_actions(void); +extern void calculate_totals(filemap_t *filemap); +extern void print_filemap(filemap_t *filemap); #endif /* FILEMAP_H */ diff --git a/src/bin/pg_rewind/libpq_fetch.c b/src/bin/pg_rewind/libpq_fetch.c index 2fc4a784bdb3..16d451ae1672 100644 --- a/src/bin/pg_rewind/libpq_fetch.c +++ b/src/bin/pg_rewind/libpq_fetch.c @@ -460,9 +460,9 @@ libpq_executeFileMap(filemap_t *map) PQresultErrorMessage(res)); PQclear(res); - for (i = 0; i < map->narray; i++) + for (i = 0; i < map->nentries; i++) { - entry = map->array[i]; + entry = map->entries[i]; /* If this is a relation file, copy the modified blocks */ execute_pagemap(&entry->target_pages_to_overwrite, entry->path); diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 4760090d06e4..574d7f7163b6 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -129,6 +129,7 @@ main(int argc, char **argv) TimeLineID endtli; ControlFileData ControlFile_new; bool writerecoveryconf = false; + filemap_t *filemap; pg_logging_init(argv[0]); set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_rewind")); @@ -368,13 +369,16 @@ main(int argc, char **argv) (uint32) (chkptrec >> 32), (uint32) chkptrec, chkpttli); + /* Initialize the hash table to track the status of each file */ + filehash_init(); + /* * Collect information about all files in the target and source systems. */ - filemap_create(); if (showprogress) pg_log_info("reading source file list"); fetchSourceFileList(); + if (showprogress) pg_log_info("reading target file list"); traverse_datadir(datadir_target, &process_target_file); @@ -395,13 +399,13 @@ main(int argc, char **argv) * We have collected all information we need from both systems. Decide * what to do with each file. */ - decide_file_actions(); + filemap = decide_file_actions(); if (showprogress) - calculate_totals(); + calculate_totals(filemap); /* this is too verbose even for verbose mode */ if (debug) - print_filemap(); + print_filemap(filemap); /* * Ok, we're ready to start copying things over. @@ -421,7 +425,7 @@ main(int argc, char **argv) * modified the target directory and there is no turning back! */ - executeFileMap(); + execute_file_actions(filemap); progress_report(true); From 41f5985e742af373d569b19f04a969cddaaf0443 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Thu, 21 May 2026 10:56:07 +0300 Subject: [PATCH 445/589] Fix conflicts in src/include/nodes/parsenodes.h (#2547) Commit 257836a75585934cc05ed7a80bccf8190d41e056 added new AlterTableType value AT_AlterCollationRefreshVersion, however there were already GPDB-specific ones nearby. --- src/include/nodes/parsenodes.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index e1d081f5d39f..aeb9e7833091 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -2023,7 +2023,7 @@ typedef enum AlterTableType AT_AddIdentity, /* ADD IDENTITY */ AT_SetIdentity, /* SET identity column options */ AT_DropIdentity, /* DROP IDENTITY */ -<<<<<<< HEAD + AT_AlterCollationRefreshVersion, /* ALTER COLLATION ... REFRESH VERSION */ AT_SetDistributedBy, /* SET DISTRIBUTED BY */ AT_ExpandTable, /* EXPAND DISTRIBUTED */ @@ -2038,9 +2038,6 @@ typedef enum AlterTableType AT_PartSetTemplate, /* Set Subpartition Template */ AT_PartSplit, /* Split */ AT_PartTruncate /* Truncate */ -======= - AT_AlterCollationRefreshVersion /* ALTER COLLATION ... REFRESH VERSION */ ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } AlterTableType; typedef struct ReplicaIdentityStmt From 1e4e90d307a4656636f297b23960736fcad93291 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Thu, 21 May 2026 10:56:28 +0300 Subject: [PATCH 446/589] Fix conflicts in src/include/nodes/pathnodes.h (#2548) Commit a929e17e5a8c9b751b66002c8a89fdebdacfe194 removed partitioned_child_rels field from RelOptInfo, but there were GPDB-specific changes nearby. --- src/include/nodes/pathnodes.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 97472d6b1481..a659b4b7c802 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -926,8 +926,6 @@ typedef struct RelOptInfo Relids all_partrels; /* Relids set of all partition relids */ List **partexprs; /* Non-nullable partition key expressions */ List **nullable_partexprs; /* Nullable partition key expressions */ -<<<<<<< HEAD - List *partitioned_child_rels; /* List of RT indexes */ /* * In a subquery, if this base relation contains quals that must @@ -937,8 +935,6 @@ typedef struct RelOptInfo */ List *upperrestrictinfo; /* RestrictInfo structures (if base * rel) */ -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } RelOptInfo; /* From 195fa0e867bb941e3059d82f694744bb30cb6239 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 21 May 2026 15:07:04 +0500 Subject: [PATCH 447/589] Resolve conflicts in src/Makefile (#2552) Commit 8a21211 in the src/Makefile in clean and in distclean maintainer-clean removed test/thread, while the earlier commit 1c1e362 had already added MOCK_DIR and CMOCKERY_DIR to the same places. --- src/Makefile | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/Makefile b/src/Makefile index 4782076e8d66..0fe4c39a4f29 100644 --- a/src/Makefile +++ b/src/Makefile @@ -69,23 +69,15 @@ clean: $(MAKE) -C test $@ $(MAKE) -C tutorial NO_PGXS=1 $@ $(MAKE) -C test/isolation $@ -<<<<<<< HEAD - $(MAKE) -C test/thread $@ $(MAKE) -C $(MOCK_DIR) $@ $(MAKE) -C $(CMOCKERY_DIR) $@ -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d distclean maintainer-clean: $(MAKE) -C test $@ $(MAKE) -C tutorial NO_PGXS=1 $@ $(MAKE) -C test/isolation $@ -<<<<<<< HEAD - $(MAKE) -C test/thread $@ $(MAKE) -C $(MOCK_DIR) $@ $(MAKE) -C $(CMOCKERY_DIR) $@ -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d rm -f Makefile.port Makefile.global .PHONY: install-local installdirs-local uninstall-local From adc9ab9b830d91544b51c6f57fe6264db1b73f45 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Thu, 21 May 2026 13:20:34 +0300 Subject: [PATCH 448/589] Resolve conflicts in src/backend/storage/sync/sync.c and sync.h (#2553) Commit dee663f7843902535a15ae366cede8b4089f1144 added new SyncRequestHandler values, however there was already GPDB-specific value SYNC_HANDLER_AO = 1. Preserve both changes. --- src/backend/storage/sync/sync.c | 6 ++---- src/include/storage/sync.h | 6 +----- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c index 8bf86b6662b1..980ebbf8840f 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c @@ -105,13 +105,12 @@ static const SyncOps syncsw[] = { .sync_unlinkfiletag = mdunlinkfiletag, .sync_filetagmatches = mdfiletagmatches }, -<<<<<<< HEAD /* append-optimized storage */ - { + [SYNC_HANDLER_AO] = { .sync_syncfiletag = aosyncfiletag, .sync_unlinkfiletag = mdunlinkfiletag, .sync_filetagmatches = mdfiletagmatches -======= + }, /* pg_xact */ [SYNC_HANDLER_CLOG] = { .sync_syncfiletag = clogsyncfiletag @@ -127,7 +126,6 @@ static const SyncOps syncsw[] = { /* pg_multixact/members */ [SYNC_HANDLER_MULTIXACT_MEMBER] = { .sync_syncfiletag = multixactmemberssyncfiletag ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } }; diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h index adc9aeaa145e..498f453310fe 100644 --- a/src/include/storage/sync.h +++ b/src/include/storage/sync.h @@ -34,17 +34,13 @@ typedef enum SyncRequestType */ typedef enum SyncRequestHandler { -<<<<<<< HEAD SYNC_HANDLER_MD = 0, /* md smgr */ - SYNC_HANDLER_AO = 1 -======= - SYNC_HANDLER_MD = 0, + SYNC_HANDLER_AO = 1, SYNC_HANDLER_CLOG, SYNC_HANDLER_COMMIT_TS, SYNC_HANDLER_MULTIXACT_OFFSET, SYNC_HANDLER_MULTIXACT_MEMBER, SYNC_HANDLER_NONE ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } SyncRequestHandler; /* From 57131fd0e5cd08fb08442b78b4d3783251918c01 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Thu, 21 May 2026 13:20:55 +0300 Subject: [PATCH 449/589] Resolve conflicts in src/include/storage/ipc.h (#2551) Commit 6693a96b329ec46f1df916f2a28d640cc9a9977d added new function check_on_shmem_exit_lists_are_empty, however there was already a GPDB-specific function nearby. --- src/include/storage/ipc.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/include/storage/ipc.h b/src/include/storage/ipc.h index be2efb928070..408dcaef7b69 100644 --- a/src/include/storage/ipc.h +++ b/src/include/storage/ipc.h @@ -72,11 +72,8 @@ extern void on_shmem_exit(pg_on_exit_callback function, Datum arg); extern void before_shmem_exit(pg_on_exit_callback function, Datum arg); extern void cancel_before_shmem_exit(pg_on_exit_callback function, Datum arg); extern void on_exit_reset(void); -<<<<<<< HEAD extern void proc_exit_prepare(int code); -======= extern void check_on_shmem_exit_lists_are_empty(void); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* ipci.c */ extern PGDLLIMPORT shmem_startup_hook_type shmem_startup_hook; From 0420d8b92a52cdf27c26dcf8d55ec18d03e0e789 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Thu, 21 May 2026 13:21:14 +0300 Subject: [PATCH 450/589] Resolve conflicts in src/include/parser/parse_coerce.h (#2550) Commit f893e68d761adbee7f888109b1adf76151e3e17a added new function select_common_typmod, however there were already GPDB-specific functions nearby. --- src/include/parser/parse_coerce.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/include/parser/parse_coerce.h b/src/include/parser/parse_coerce.h index 3cca094f7102..1a9ab069d61c 100644 --- a/src/include/parser/parse_coerce.h +++ b/src/include/parser/parse_coerce.h @@ -75,13 +75,11 @@ extern Node *coerce_to_common_type(ParseState *pstate, Node *node, Oid targetTypeId, const char *context); -<<<<<<< HEAD extern void fixup_unknown_vars_in_exprlist(ParseState *pstate, List *exprlist); extern void fixup_unknown_vars_in_targetlist(ParseState *pstate, List *targetlist); -======= + extern int32 select_common_typmod(ParseState *pstate, List *exprs, Oid common_type); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d extern bool check_generic_type_consistency(const Oid *actual_arg_types, const Oid *declared_arg_types, From e1301bc19299a1e98eb8277d1c4ba646e55a8a15 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Thu, 21 May 2026 13:39:19 +0300 Subject: [PATCH 451/589] Resolve conflicts src/include/utils/logtape.h (#2554) Commit 07589649639410032df281e98469db88a0b86271 added a new argument to LogicalTapeSetCreate, however GPDB-specific commit e2f2312c2295b357bb770666789ddd3ae8ba54a8 added LogicalTapeGetBufFilename nearby. Also fix GPDB-specific logicaltape_test. --- src/include/utils/logtape.h | 4 ---- src/test/isolation2/workfile_mgr_test.c | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h index 9fb22108709a..831a17925fbe 100644 --- a/src/include/utils/logtape.h +++ b/src/include/utils/logtape.h @@ -54,13 +54,9 @@ typedef struct TapeShare * prototypes for functions in logtape.c */ -<<<<<<< HEAD extern char * LogicalTapeGetBufFilename(const LogicalTapeSet *lts); -extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, TapeShare *shared, -======= extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared, ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d SharedFileSet *fileset, int worker); extern void LogicalTapeSetClose(LogicalTapeSet *lts); extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts); diff --git a/src/test/isolation2/workfile_mgr_test.c b/src/test/isolation2/workfile_mgr_test.c index fe64e27cc5e7..3e9cb01abf1a 100644 --- a/src/test/isolation2/workfile_mgr_test.c +++ b/src/test/isolation2/workfile_mgr_test.c @@ -524,7 +524,7 @@ logicaltape_test(void) int test_tape = 5; int test_entry = 45000; - LogicalTapeSet *tape_set = LogicalTapeSetCreate(max_tapes, NULL, NULL, -1); + LogicalTapeSet *tape_set = LogicalTapeSetCreate(max_tapes, false, NULL, NULL, -1); int work_tape = 0; long blocknum = 0; From b6425ed827b262c404553f043f3febceb54da766 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 21 May 2026 15:40:07 +0500 Subject: [PATCH 452/589] Resolve conflicts in src/include/port.h (#2555) Commit fe27009 in the file src/include/port.h added a new macro ALL_CONNECTION_FAILURE_ERRNOS with a comment, while the earlier commit 5a3a39b already added an empty line in the same place. --- src/include/port.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/include/port.h b/src/include/port.h index da045c31f892..39fbea0d8d1b 100644 --- a/src/include/port.h +++ b/src/include/port.h @@ -99,8 +99,6 @@ extern void pgfnames_cleanup(char **filenames); ) #endif -<<<<<<< HEAD -======= /* * This macro provides a centralized list of all errnos that identify * hard failure of a previously-established network connection. @@ -122,7 +120,6 @@ extern void pgfnames_cleanup(char **filenames); case ENETDOWN: \ case ENETRESET: \ case ENETUNREACH ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* Portable locale initialization (in exec.c) */ extern void set_pglocale_pgservice(const char *argv0, const char *app); From 7fefbf403303b3943f8ac410ccd1fdad92245f9e Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Thu, 21 May 2026 14:01:30 +0300 Subject: [PATCH 453/589] Resolve conflicts in src/include/executor/executor.h (#2556) Commit a04daa97a4339c38e304cd6164d37da540d665a8 added new argument resultRelInfo to ExecSimpleRelationInsert and ExecSimpleRelationUpdate, however GPDB code is missing an empty line before it. --- src/include/executor/executor.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/include/executor/executor.h b/src/include/executor/executor.h index 3a381721f0a0..daadf189ab04 100644 --- a/src/include/executor/executor.h +++ b/src/include/executor/executor.h @@ -646,16 +646,11 @@ extern bool RelationFindReplTupleByIndex(Relation rel, Oid idxoid, TupleTableSlot *outslot); extern bool RelationFindReplTupleSeq(Relation rel, LockTupleMode lockmode, TupleTableSlot *searchslot, TupleTableSlot *outslot); -<<<<<<< HEAD -extern void ExecSimpleRelationInsert(EState *estate, TupleTableSlot *slot); -extern void ExecSimpleRelationUpdate(EState *estate, EPQState *epqstate, -======= extern void ExecSimpleRelationInsert(ResultRelInfo *resultRelInfo, EState *estate, TupleTableSlot *slot); extern void ExecSimpleRelationUpdate(ResultRelInfo *resultRelInfo, EState *estate, EPQState *epqstate, ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d TupleTableSlot *searchslot, TupleTableSlot *slot); extern void ExecSimpleRelationDelete(ResultRelInfo *resultRelInfo, EState *estate, EPQState *epqstate, From e17f2b90026676150282c1cf8b44191ab6d1c57c Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 21 May 2026 16:19:49 +0500 Subject: [PATCH 454/589] Resolve conflicts in src/include/catalog/catversion.h (#2559) Commit e152506 updated the catalog version after the incompatible changes, so we should do the same for the current date - the date of the incompatible changes. --- src/include/catalog/catversion.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 839f3956c1e4..7c3d7c8999ca 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -55,12 +55,7 @@ * catalog versions from Greenplum. */ -<<<<<<< HEAD /* 3yyymmddN */ -#define CATALOG_VERSION_NO 302605041 -======= -/* yyyymmddN */ -#define CATALOG_VERSION_NO 202011041 ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d +#define CATALOG_VERSION_NO 302605211 #endif From c46cb94124c668c371b9dc9acc4fe10801f52162 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 21 May 2026 16:20:22 +0500 Subject: [PATCH 455/589] Resolve conflicts in table.h and index.c (#2557) 1) Commit 1d65416 in the file src/include/access/table.h added a declaration of a new function try_table_open with two arguments, while the earlier commit 19cd1cf already added it but with three arguments. 2) Commit a6642b3 in the file src/backend/catalog/index.c in the reindex_relation function changed the warning to an error, while earlier commits 25a9039 and 958a672 had already added GPDB-specific code to the same place. --- src/backend/access/table/table.c | 34 ++-------------------- src/backend/catalog/index.c | 7 ++--- src/backend/commands/indexcmds.c | 6 ++-- src/backend/replication/logical/relation.c | 2 +- src/include/access/table.h | 4 --- 5 files changed, 9 insertions(+), 44 deletions(-) diff --git a/src/backend/access/table/table.c b/src/backend/access/table/table.c index 5a232d365330..4cb640d1f880 100644 --- a/src/backend/access/table/table.c +++ b/src/backend/access/table/table.c @@ -71,11 +71,11 @@ table_open(Oid relationId, LOCKMODE lockmode) * ---------------- */ Relation -try_table_open(Oid relationId, LOCKMODE lockmode) +try_table_open(Oid relationId, LOCKMODE lockmode, bool noWait) { Relation r; - r = try_relation_open(relationId, lockmode); + r = try_relation_open(relationId, lockmode, noWait); /* leave if table does not exist */ if (!r) @@ -159,36 +159,6 @@ table_openrv_extended(const RangeVar *relation, LOCKMODE lockmode, return r; } -/* ---------------- - * try_table_open - open a heap relation by relation OID - * - * As above, but relation return NULL for relation-not-found - * ---------------- - */ -Relation -try_table_open(Oid relationId, LOCKMODE lockmode, bool noWait) -{ - Relation r; - - r = try_relation_open(relationId, lockmode, noWait); - - if (!RelationIsValid(r)) - return NULL; - - if (r->rd_rel->relkind == RELKIND_INDEX) - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("\"%s\" is an index", - RelationGetRelationName(r)))); - else if (r->rd_rel->relkind == RELKIND_COMPOSITE_TYPE) - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("\"%s\" is a composite type", - RelationGetRelationName(r)))); - - return r; -} - /* ---------------- * table_close - close a table * diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index faa155ed02de..77d13e750a16 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -3714,7 +3714,7 @@ reindex_index(Oid indexId, bool skip_constraint_checks, char persistence, return; if ((options & REINDEXOPT_MISSING_OK) != 0) - heapRelation = try_table_open(heapId, ShareLock); + heapRelation = try_table_open(heapId, ShareLock, false); else heapRelation = table_open(heapId, ShareLock); @@ -4015,7 +4015,7 @@ reindex_relation(Oid relid, int flags, int options) * should match ReindexTable(). */ if ((options & REINDEXOPT_MISSING_OK) != 0) - rel = try_table_open(relid, ShareLock); + rel = try_table_open(relid, ShareLock, false); else rel = table_open(relid, ShareLock); @@ -4031,11 +4031,8 @@ reindex_relation(Oid relid, int flags, int options) elog(ERROR, "cannot reindex partitioned table \"%s.%s\"", get_namespace_name(RelationGetNamespace(rel)), RelationGetRelationName(rel)); -<<<<<<< HEAD relIsAO = RelationIsAppendOptimized(rel); -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d toast_relid = rel->rd_rel->reltoastrelid; diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 706d0f6a4f6f..848a8b3a66aa 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -3618,7 +3618,8 @@ ReindexRelationConcurrently(Oid relationOid, int options) if ((options & REINDEXOPT_MISSING_OK) != 0) { heapRelation = try_table_open(relationOid, - ShareUpdateExclusiveLock); + ShareUpdateExclusiveLock, + false); /* leave if relation does not exist */ if (!heapRelation) break; @@ -3742,7 +3743,8 @@ ReindexRelationConcurrently(Oid relationOid, int options) if ((options & REINDEXOPT_MISSING_OK) != 0) { heapRelation = try_table_open(heapId, - ShareUpdateExclusiveLock); + ShareUpdateExclusiveLock, + false); /* leave if relation does not exist */ if (!heapRelation) break; diff --git a/src/backend/replication/logical/relation.c b/src/backend/replication/logical/relation.c index 07aa52977f96..25c7a804fad1 100644 --- a/src/backend/replication/logical/relation.c +++ b/src/backend/replication/logical/relation.c @@ -302,7 +302,7 @@ logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode) */ if (entry->localrelvalid) { - entry->localrel = try_table_open(entry->localreloid, lockmode); + entry->localrel = try_table_open(entry->localreloid, lockmode, false); if (!entry->localrel) { /* Table was renamed or dropped. */ diff --git a/src/include/access/table.h b/src/include/access/table.h index f710739c5a74..5c29dbf59558 100644 --- a/src/include/access/table.h +++ b/src/include/access/table.h @@ -22,11 +22,7 @@ extern Relation table_open(Oid relationId, LOCKMODE lockmode); extern Relation table_openrv(const RangeVar *relation, LOCKMODE lockmode); extern Relation table_openrv_extended(const RangeVar *relation, LOCKMODE lockmode, bool missing_ok); -<<<<<<< HEAD extern Relation try_table_open(Oid relationId, LOCKMODE lockmode, bool noWait); -======= -extern Relation try_table_open(Oid relationId, LOCKMODE lockmode); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d extern void table_close(Relation relation, LOCKMODE lockmode); /* CDB */ From 83259789239bd8c54d8a62b89daa5845959bda2c Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 21 May 2026 16:20:47 +0500 Subject: [PATCH 456/589] Resolve conflicts in src/include/catalog/index.h (#2560) Commit 257836a in src/include/catalog/index.h added new function declarations index_check_collation_versions and index_update_collation_versions, while an earlier commit a453004 had already added a function declaration setNewRelfilenodeToOid in the same place. --- src/include/catalog/index.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h index 2ede27ed4452..ff947c8ebc91 100644 --- a/src/include/catalog/index.h +++ b/src/include/catalog/index.h @@ -127,13 +127,11 @@ extern void FormIndexDatum(IndexInfo *indexInfo, Datum *values, bool *isnull); -<<<<<<< HEAD extern Oid setNewRelfilenodeToOid(Relation relation, TransactionId freezeXid, Oid newrelfilenode); -======= + extern void index_check_collation_versions(Oid relid); extern void index_update_collation_versions(Oid relid, Oid coll); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d extern void index_build(Relation heapRelation, Relation indexRelation, From 0795e20af1bf6ecc13e58cb6a4e48dafd81e97e0 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Thu, 21 May 2026 15:34:49 +0300 Subject: [PATCH 457/589] Resolve conflicts in src/include/common/string.h (#2558) Commits 67a472d71c98c3d2fa322a1b4013080b20720b98, 8e3c58e6e459b285d37edb6129e412ed25cd90c1 and 931487018c409a3102452f965ccaa48367244a41 have added new definitions, however GPDB has previously cherry-picked commit b4c36a1dc37fffb2d7e795d0f77aaaf035ad3e9f, so some of the changes were already present. --- src/include/common/string.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/include/common/string.h b/src/include/common/string.h index ba66ca7dab46..6a4baa6f3590 100644 --- a/src/include/common/string.h +++ b/src/include/common/string.h @@ -12,10 +12,7 @@ struct StringInfoData; /* avoid including stringinfo.h here */ -<<<<<<< HEAD -======= /* functions in src/common/string.c */ ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d extern bool pg_str_endswith(const char *str, const char *end); extern int strtoint(const char *pg_restrict str, char **pg_restrict endptr, int base); @@ -23,11 +20,6 @@ extern void pg_clean_ascii(char *str); extern int pg_strip_crlf(char *str); /* functions in src/common/pg_get_line.c */ -<<<<<<< HEAD -extern bool pg_get_line_buf(FILE *stream, struct StringInfoData *buf); -extern bool pg_get_line_append(FILE *stream, struct StringInfoData *buf); - -======= extern char *pg_get_line(FILE *stream); extern bool pg_get_line_buf(FILE *stream, struct StringInfoData *buf); extern bool pg_get_line_append(FILE *stream, struct StringInfoData *buf); @@ -35,5 +27,4 @@ extern bool pg_get_line_append(FILE *stream, struct StringInfoData *buf); /* functions in src/common/sprompt.c */ extern char *simple_prompt(const char *prompt, bool echo); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d #endif /* COMMON_STRING_H */ From 154284663247ed24d8126a121170d7ca5d60d64e Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Fri, 22 May 2026 15:42:36 +0300 Subject: [PATCH 458/589] Resolve conflicts in src/backend/catalog/pg_type.c (#2565) Commit 257836a75585934cc05ed7a80bccf8190d41e056 added new include, however commit 07c2f0112adc8c682fcf330b6beef0197d210834 added several GPDB-specific includes in the same place. Separate all the GPDB-specific includes to avoid future conflicts. --- src/backend/catalog/pg_type.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/backend/catalog/pg_type.c b/src/backend/catalog/pg_type.c index e045800eb5c7..e6e4d092fc59 100644 --- a/src/backend/catalog/pg_type.c +++ b/src/backend/catalog/pg_type.c @@ -14,14 +14,8 @@ */ #include "postgres.h" -#include "access/genam.h" -#include "access/heapam.h" #include "access/htup_details.h" -<<<<<<< HEAD -#include "access/reloptions.h" -======= #include "access/relation.h" ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d #include "access/table.h" #include "access/xact.h" #include "catalog/binary_upgrade.h" @@ -37,7 +31,6 @@ #include "commands/typecmds.h" #include "miscadmin.h" #include "parser/scansup.h" -#include "parser/parse_type.h" #include "utils/acl.h" #include "utils/builtins.h" #include "utils/fmgroids.h" @@ -45,8 +38,12 @@ #include "utils/rel.h" #include "utils/syscache.h" +#include "access/genam.h" +#include "access/heapam.h" +#include "access/reloptions.h" #include "catalog/pg_type_encoding.h" #include "cdb/cdbvars.h" +#include "parser/parse_type.h" /* * Record a type's default encoding clause in the catalog. From cd2786a77a4d982625db36b0c294ca4d1c72928a Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 22 May 2026 20:52:40 +0500 Subject: [PATCH 459/589] Resolve conflicts in src/backend/catalog/system_views.sql (#2566) Commit 9868167 added a new view, pg_stat_replication_slots, to src/backend/catalog/system_views.sql, while earlier commit 1546ec3 had already added the view, gp_stat_replication, in the same location. --- src/backend/catalog/system_views.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 5882b0d7885c..c29bcfc4eadc 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -913,7 +913,6 @@ CREATE VIEW pg_stat_replication AS JOIN pg_stat_get_wal_senders() AS W ON (S.pid = W.pid) LEFT JOIN pg_authid AS U ON (S.usesysid = U.oid); -<<<<<<< HEAD CREATE FUNCTION gp_stat_get_master_replication() RETURNS SETOF RECORD AS $$ SELECT pg_catalog.gp_execution_segment() AS gp_segment_id, * @@ -970,7 +969,7 @@ CREATE VIEW gp_stat_replication AS sync_priority int4, sync_state text, reply_time timestamptz) ON G.gp_segment_id = R.gp_segment_id ); -======= + CREATE VIEW pg_stat_replication_slots AS SELECT s.slot_name, @@ -982,7 +981,6 @@ CREATE VIEW pg_stat_replication_slots AS s.stream_bytes, s.stats_reset FROM pg_stat_get_replication_slots() AS s; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d CREATE VIEW pg_stat_slru AS SELECT From 13f5e045abb4de0a86cdf10d8a999f06cf2bca2b Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 22 May 2026 20:53:01 +0500 Subject: [PATCH 460/589] Resolve conflicts in src/backend/commands/copy.c (#2567) 1) Commit 1375422 in src/backend/commands/copy.c replaced the call to InitResultRelInfo with ExecInitResultRelation in the CopyFrom function, while earlier commit 25a9039 had already added an empty line in the same location. 2) Commit 1375422 in src/backend/commands/copy.c removed the call to ExecCloseIndices in the CopyFrom function, while earlier commit d2461dd had already added a conditional call to table_dml_finish in the same location. --- src/backend/commands/copy.c | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 66658bd02d67..5f671b9115bf 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -3911,20 +3911,9 @@ CopyFrom(CopyState cstate) * index-entry-making machinery. (There used to be a huge amount of code * here that basically duplicated execUtils.c ...) */ -<<<<<<< HEAD - resultRelInfo = makeNode(ResultRelInfo); - InitResultRelInfo(resultRelInfo, - cstate->rel, - 1, /* must match rel's position in range_table */ - NULL, - 0); - - target_resultRelInfo = resultRelInfo; -======= ExecInitRangeTable(estate, cstate->range_table); resultRelInfo = target_resultRelInfo = makeNode(ResultRelInfo); ExecInitResultRelation(estate, resultRelInfo, 1); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* Verify the named relation is a valid target for INSERT */ CheckValidResultRel(resultRelInfo, CMD_INSERT); @@ -4703,14 +4692,9 @@ CopyFrom(CopyState cstate) if (insertMethod != CIM_SINGLE) CopyMultiInsertInfoCleanup(&multiInsertInfo); -<<<<<<< HEAD - ExecCloseIndices(target_resultRelInfo); - if (target_resultRelInfo->ri_RelationDesc->rd_tableam) table_dml_finish(target_resultRelInfo->ri_RelationDesc); -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* Close all the partitioned tables, leaf partitions, and their indices */ if (proute) ExecCleanupTupleRouting(mtstate, proute); From 8268c0b3c37484c1a73ac4faccac27cfcfce9636 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 22 May 2026 20:53:41 +0500 Subject: [PATCH 461/589] Resolve conflicts in src/backend/commands/explain.c (#2568) 1) Commit 9d701e6 in src/backend/commands/explain.c in the ExplainOnePlan function replaced the es->summary && (planning || bufusage) condition with the bufusage condition. However, the earlier commit 7a1ce37 had already added a conditional call to the ExplainParallelRetrieveCursor function in the same location. 2) Commit 9d701e6 in src/backend/commands/explain.c in the ExplainOnePlan function removed the es->summary && bufusage and es->summary && (planation || bufusage) conditions along with the code, while the earlier commit 08d8dba had already added a conditional call to the ExplainPrintSliceTable function in the same place. --- src/backend/commands/explain.c | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index d621566f6bd6..3fd724430c78 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -717,16 +717,12 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, /* Create textual dump of plan tree */ ExplainPrintPlan(es, queryDesc); -<<<<<<< HEAD if (cursorOptions & CURSOR_OPT_PARALLEL_RETRIEVE) ExplainParallelRetrieveCursor(es, queryDesc); - if (es->summary && (planduration || bufusage)) -======= /* Show buffer usage in planning */ if (bufusage) { ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d ExplainOpenGroup("Planning", "Planning", true, es); show_buffer_usage(es, bufusage, true); ExplainCloseGroup("Planning", "Planning", true, es); @@ -739,26 +735,10 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, ExplainPropertyFloat("Planning Time", "ms", 1000.0 * plantime, 3, es); } -<<<<<<< HEAD /* Print slice table */ if (es->slicetable) ExplainPrintSliceTable(es, queryDesc); - /* Show buffer usage */ - if (es->summary && bufusage) - { - if (es->format == EXPLAIN_FORMAT_TEXT) - es->indent++; - show_buffer_usage(es, bufusage); - if (es->format == EXPLAIN_FORMAT_TEXT) - es->indent--; - } - - if (es->summary && (planduration || bufusage)) - ExplainCloseGroup("Planning", "Planning", true, es); - -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* Print info about runtime of triggers */ if (es->analyze) ExplainPrintTriggers(es, queryDesc); From 6477b96f762f3ce47c8968d3b7a7d14ecb48dbfe Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 22 May 2026 20:56:23 +0500 Subject: [PATCH 462/589] Resolve conflicts in src/backend/commands/functioncmds.c (#2569) 1) Commit 2453ea1 in src/backend/commands/functioncmds.c in the interpret_function_parameter_list function renamed the local variable inTypes to sigArgTypes. However, the earlier commit 6b0e52b had already added a pre-allocation comment in the same location. 2) Commit 2453ea1 in src/backend/commands/functioncmds.c in the interpret_function_parameter_list function renamed the local variable inTypes to sigArgTypes, changed the word input to signature in the error message, and removed the assignment to isinput. However, the earlier commit fa287d0 had already added a conditional increment of multisetCount in the same location. --- src/backend/commands/functioncmds.c | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/backend/commands/functioncmds.c b/src/backend/commands/functioncmds.c index a661c7cdb256..1b5217ce9b29 100644 --- a/src/backend/commands/functioncmds.c +++ b/src/backend/commands/functioncmds.c @@ -234,12 +234,7 @@ interpret_function_parameter_list(ParseState *pstate, *allParameterTypes = NULL; *parameterModes = NULL; -<<<<<<< HEAD - /* Allocate local memory */ - inTypes = (Oid *) palloc(parameterCount * sizeof(Oid)); -======= sigArgTypes = (Oid *) palloc(parameterCount * sizeof(Oid)); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d allTypes = (Datum *) palloc(parameterCount * sizeof(Datum)); paramModes = (Datum *) palloc(parameterCount * sizeof(Datum)); paramNames = (Datum *) palloc0(parameterCount * sizeof(Datum)); @@ -324,18 +319,12 @@ interpret_function_parameter_list(ParseState *pstate, if (varCount > 0) ereport(ERROR, (errcode(ERRCODE_INVALID_FUNCTION_DEFINITION), -<<<<<<< HEAD - errmsg("VARIADIC parameter must be the last input parameter"))); - inTypes[inCount++] = toid; - isinput = true; + errmsg("VARIADIC parameter must be the last signature parameter"))); + sigArgTypes[sigArgCount++] = toid; /* Keep track of the number of anytable arguments */ if (toid == ANYTABLEOID) multisetCount++; -======= - errmsg("VARIADIC parameter must be the last signature parameter"))); - sigArgTypes[sigArgCount++] = toid; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } /* handle output parameters */ From a9ad2dce9863f3d4ef75642f103ef772790e6d74 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 22 May 2026 20:57:10 +0500 Subject: [PATCH 463/589] Resolve conflicts in src/backend/commands/lockcmds.c (#2570) Commit 59ab4ac in src/backend/commands/lockcmds.c in the RangeVarCallbackForLockTable function removed the conditional error message, while the earlier commit 19cd1cf had already disabled the code below in the same location. --- src/backend/commands/lockcmds.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/backend/commands/lockcmds.c b/src/backend/commands/lockcmds.c index d6c6843641a1..57d89f9c724b 100644 --- a/src/backend/commands/lockcmds.c +++ b/src/backend/commands/lockcmds.c @@ -105,18 +105,7 @@ RangeVarCallbackForLockTable(const RangeVar *rv, Oid relid, Oid oldrelid, return; /* woops, concurrently dropped; no permissions * check */ -<<<<<<< HEAD - /* Currently, we only allow plain tables or views to be locked */ - if (relkind != RELKIND_RELATION && relkind != RELKIND_PARTITIONED_TABLE && - relkind != RELKIND_VIEW) - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("\"%s\" is not a table or view", - rv->relname))); - #if 0 /* Upstream code not applicable to GPDB */ -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* * Make note if a temporary relation has been accessed in this * transaction. From be99ce680f3a58999b3ffd17803765ae82da1515 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 22 May 2026 20:57:37 +0500 Subject: [PATCH 464/589] Resolve conflicts in src/backend/nodes/equalfuncs.c (#2574) Commit 844c05a in src/backend/nodes/equalfuncs.c removed the use of the concurrent field before the _equalReindexStmt function, while an earlier commit, 6b0e52b, already used the relid field in the same place. --- src/backend/nodes/equalfuncs.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 784f838a1ef1..bf1f71cf6350 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -2310,11 +2310,7 @@ _equalReindexStmt(const ReindexStmt *a, const ReindexStmt *b) COMPARE_NODE_FIELD(relation); COMPARE_STRING_FIELD(name); COMPARE_SCALAR_FIELD(options); -<<<<<<< HEAD - COMPARE_SCALAR_FIELD(concurrent); COMPARE_SCALAR_FIELD(relid); -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d return true; } From 2924b6f6511d2b47dec3efa0e5f42e9768b61f2a Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Mon, 25 May 2026 12:40:10 +0500 Subject: [PATCH 465/589] Resolve conflicts in src/backend/optimizer/path/costsize.c (#2575) 1) Commit a90c950 in src/backend/optimizer/path/costsize.c changed the comment and calculation of nrows in the clamp_row_est function, while earlier commit 6b0e52b had already moved the clamp_row_est function to src/include/optimizer/cost.h. 2) Commit a90c950 in src/backend/optimizer/path/costsize.c in the final_cost_nestloop function removed isnan(outer_path_rows) from the conditions, while earlier commit c5f6dbb had already added the conditional setting of numsegments to the same location. --- src/backend/optimizer/path/costsize.c | 26 -------------------------- src/include/optimizer/cost.h | 21 ++++++++++++++++++--- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index f8cc7f387c81..debb2fdd6602 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -112,14 +112,6 @@ */ #define APPEND_CPU_COST_MULTIPLIER 0.5 -/* - * Maximum value for row estimates. We cap row estimates to this to help - * ensure that costs based on these estimates remain within the range of what - * double can represent. add_path() wouldn't act sanely given infinite or NaN - * cost values. - */ -#define MAXIMUM_ROWCOUNT 1e100 - double seq_page_cost = DEFAULT_SEQ_PAGE_COST; double random_page_cost = DEFAULT_RANDOM_PAGE_COST; double cpu_tuple_cost = DEFAULT_CPU_TUPLE_COST; @@ -234,26 +226,11 @@ static Selectivity adjust_selectivity_for_nulltest(Selectivity selec, */ typedef struct { -<<<<<<< HEAD /* Values copied from RelOptInfo as is, for convenience */ Index relid; RTEKind rtekind; /* RELATION, SUBQUERY, or FUNCTION */ Oid reltablespace; /* containing tablespace */ double allvisfrac; -======= - /* - * Avoid infinite and NaN row estimates. Costs derived from such values - * are going to be useless. Also force the estimate to be at least one - * row, to make explain output look better and to avoid possible - * divide-by-zero when interpolating costs. Make it an integer, too. - */ - if (nrows > MAXIMUM_ROWCOUNT || isnan(nrows)) - nrows = MAXIMUM_ROWCOUNT; - else if (nrows <= 1.0) - nrows = 1.0; - else - nrows = rint(nrows); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* Values adjusted from RelOptInfo, by dividing by numsegments */ double rows; @@ -3035,15 +3012,12 @@ final_cost_nestloop(PlannerInfo *root, NestPath *path, outer_path_rows = 1; if (inner_path_rows <= 0) inner_path_rows = 1; -<<<<<<< HEAD if (CdbPathLocus_IsPartitioned(path->path.locus)) numsegments = CdbPathLocus_NumSegments(path->path.locus); else numsegments = 1; -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* Mark the path with the correct row estimate */ if (path->path.param_info) path->path.rows = path->path.param_info->ppi_rows; diff --git a/src/include/optimizer/cost.h b/src/include/optimizer/cost.h index dd1c5c41a21c..a722750f339b 100644 --- a/src/include/optimizer/cost.h +++ b/src/include/optimizer/cost.h @@ -33,6 +33,14 @@ #define DEFAULT_EFFECTIVE_CACHE_SIZE 524288 /* measured in pages */ +/* + * Maximum value for row estimates. We cap row estimates to this to help + * ensure that costs based on these estimates remain within the range of what + * double can represent. add_path() wouldn't act sanely given infinite or NaN + * cost values. + */ +#define MAXIMUM_ROWCOUNT 1e100 + typedef enum { CONSTRAINT_EXCLUSION_OFF, /* do not use c_e */ @@ -49,11 +57,18 @@ static inline double clamp_row_est(double nrows) { /* - * Force estimate to be at least one row, to make explain output look - * better and to avoid possible divide-by-zero when interpolating costs. + * Avoid infinite and NaN row estimates. Costs derived from such values + * are going to be useless. Also force the estimate to be at least one + * row, to make explain output look better and to avoid possible + * divide-by-zero when interpolating costs. * CDB: Don't round to integer. */ - return (nrows < 1.0) ? 1.0 : nrows; + if (nrows > MAXIMUM_ROWCOUNT || isnan(nrows)) + nrows = MAXIMUM_ROWCOUNT; + else if (nrows < 1.0) + nrows = 1.0; + + return nrows; } From c32f9ba68739b891aac0c5267846f55ded421cc5 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Mon, 25 May 2026 18:50:58 +0500 Subject: [PATCH 466/589] Resolve conflicts in src/backend/partitioning/partprune.c (#2583) Commit a929e17 in src/backend/partitioning/partprune.c in the make_partitionedrel_pruneinfo function replaced the List *partitioned_rels argument with Relids partrelids, while the earlier commit 19cd1cf had already added the Relids available_relids argument in the same location. --- src/backend/partitioning/partprune.c | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/backend/partitioning/partprune.c b/src/backend/partitioning/partprune.c index 9a4c9282608a..35468476da6f 100644 --- a/src/backend/partitioning/partprune.c +++ b/src/backend/partitioning/partprune.c @@ -144,12 +144,8 @@ typedef struct PruneStepResult static List *make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, int *relid_subplan_map, -<<<<<<< HEAD Relids available_relids, - List *partitioned_rels, List *prunequal, -======= Relids partrelids, List *prunequal, ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d Bitmapset **matchedsubplans); static void gen_partprune_steps(RelOptInfo *rel, List *clauses, Relids available_relids, @@ -297,12 +293,8 @@ make_partition_pruneinfo_ext(PlannerInfo *root, RelOptInfo *parentrel, pinfolist = make_partitionedrel_pruneinfo(root, parentrel, relid_subplan_map, -<<<<<<< HEAD available_relids, - rels, prunequal, -======= partrelids, prunequal, ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d &matchedsubplans); /* When pruning is possible, record the matched subplans */ @@ -371,12 +363,8 @@ make_partition_pruneinfo_ext(PlannerInfo *root, RelOptInfo *parentrel, static List * make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, int *relid_subplan_map, -<<<<<<< HEAD Relids available_relids, - List *partitioned_rels, List *prunequal, -======= Relids partrelids, List *prunequal, ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d Bitmapset **matchedsubplans) { RelOptInfo *targetpart = NULL; From 28c63668d224c3d680f634f1102edc509627b983 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Mon, 25 May 2026 18:51:51 +0500 Subject: [PATCH 467/589] Resolve conflicts in src/backend/postmaster/bgwriter.c (#2584) Commit 44fc6e2 in src/backend/postmaster/bgwriter.c in the BackgroundWriterMain function removed the SIGQUIT signal setting and added a corresponding comment. However, an earlier commit, 89f85cc, had already changed this signal setting from SignalHandlerForCrashExit to bg_quickdie, which is needed for the fault_in_background_writer_quickdie default injector in the fts_segment_reset test. --- src/backend/postmaster/bgwriter.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c index b3e629dae27e..46785c34b3e0 100644 --- a/src/backend/postmaster/bgwriter.c +++ b/src/backend/postmaster/bgwriter.c @@ -107,11 +107,7 @@ BackgroundWriterMain(void) pqsignal(SIGHUP, SignalHandlerForConfigReload); pqsignal(SIGINT, SIG_IGN); pqsignal(SIGTERM, SignalHandlerForShutdownRequest); -<<<<<<< HEAD pqsignal(SIGQUIT, bg_quickdie); -======= - /* SIGQUIT handler was already set up by InitPostmasterChild */ ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d pqsignal(SIGALRM, SIG_IGN); pqsignal(SIGPIPE, SIG_IGN); pqsignal(SIGUSR1, procsignal_sigusr1_handler); From 34030d1fce2071ec57fc8efad06be0113fcd9fb5 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Tue, 26 May 2026 12:10:58 +0500 Subject: [PATCH 468/589] Resolve conflicts in src/backend/storage/ipc/procarray.c (#2588) Commit 94bc27b in src/backend/storage/ipc/procarray.c added the maybe_needed field setting to the GlobalVisTempRels variable in the GlobalVisUpdateApply function. However, an earlier commit, 0a5cc3f, had already added a call to the GetDistOldestXmin function as an argument to the FullXidRelativeTo function call above. --- src/backend/storage/ipc/procarray.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c index 7b2f5acf74c9..685c094a725d 100644 --- a/src/backend/storage/ipc/procarray.c +++ b/src/backend/storage/ipc/procarray.c @@ -4926,14 +4926,10 @@ GlobalVisUpdateApply(ComputeXidHorizonsResult *horizons) GetDistOldestXmin(horizons->catalog_oldest_nonremovable)); GlobalVisDataRels.maybe_needed = FullXidRelativeTo(horizons->latest_completed, -<<<<<<< HEAD GetDistOldestXmin(horizons->data_oldest_nonremovable)); -======= - horizons->data_oldest_nonremovable); GlobalVisTempRels.maybe_needed = FullXidRelativeTo(horizons->latest_completed, - horizons->temp_oldest_nonremovable); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + GetDistOldestXmin(horizons->temp_oldest_nonremovable)); /* * In longer running transactions it's possible that transactions we From 9b66b65b37e54b39aad605bb1039078a43147d53 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Tue, 26 May 2026 12:41:24 +0500 Subject: [PATCH 469/589] Resolve conflicts in dbsize.c and numeric.c (#2589) 1) Commit 0aa8f76 in src/backend/utils/adt/dbsize.c in the numeric_shift_right function replaced DirectFunctionCall1 int8_numeric with NumericGetDatum int64_to_numeric, while earlier commit 3e39d45 had already replaced 1 with 1LL above this point. 2) Commit 3438c98 in src/backend/utils/adt/numeric.c in the init_var macro replaced the call to the MemSetAligned function with memset, while earlier commit 6b0e52b had already optimized this init_var macro differently. 3) Commit 0aa8f76 in src/backend/utils/adt/numeric.c in the int2_numeric function simplified the content, while the earlier commit 6b0e52b had already replaced the init_var macro call with quick_init_var in the same location. 4) Commit 0aa8f76 in src/backend/utils/adt/numeric.c in the int8_numeric function simplified the content, while the earlier commit 6b0e52b had already replaced the init_var macro call with quick_init_var in the same location. --- src/backend/utils/adt/dbsize.c | 5 ----- src/backend/utils/adt/numeric.c | 28 ---------------------------- 2 files changed, 33 deletions(-) diff --git a/src/backend/utils/adt/dbsize.c b/src/backend/utils/adt/dbsize.c index 9241d5e26413..62decfdbb607 100644 --- a/src/backend/utils/adt/dbsize.c +++ b/src/backend/utils/adt/dbsize.c @@ -879,12 +879,7 @@ numeric_shift_right(Numeric n, unsigned count) Datum divisor_numeric; Datum result; -<<<<<<< HEAD - divisor_int64 = Int64GetDatum((int64) (1LL << count)); - divisor_numeric = DirectFunctionCall1(int8_numeric, divisor_int64); -======= divisor_numeric = NumericGetDatum(int64_to_numeric(((int64) 1) << count)); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d result = DirectFunctionCall2(numeric_div_trunc, d, divisor_numeric); return DatumGetNumeric(result); } diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c index 256e25a2bfb4..affab61a3b2e 100644 --- a/src/backend/utils/adt/numeric.c +++ b/src/backend/utils/adt/numeric.c @@ -484,7 +484,6 @@ static void dump_var(const char *str, NumericVar *var); (v)->neg_digits = NULL; \ } while (0) -<<<<<<< HEAD #define quick_init_var(v) \ do { \ (v)->buf = (v)->ndb; \ @@ -530,9 +529,6 @@ static void dump_var(const char *str, NumericVar *var); (v)->buf[0] = 0; \ (v)->digits = (v)->buf + 1; \ } while (0) -======= -#define init_var(v) memset(v, 0, sizeof(NumericVar)) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d #define NUMERIC_DIGITS(num) (NUMERIC_HEADER_IS_SHORT(num) ? \ (num)->choice.n_short.n_data : (num)->choice.n_long.n_data) @@ -4526,19 +4522,7 @@ int8_numeric(PG_FUNCTION_ARGS) { int64 val = PG_GETARG_INT64(0); -<<<<<<< HEAD - quick_init_var(&result); - - int64_to_numericvar(val, &result); - - res = make_result(&result); - - free_var(&result); - - PG_RETURN_NUMERIC(res); -======= PG_RETURN_NUMERIC(int64_to_numeric(val)); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } @@ -4578,19 +4562,7 @@ int2_numeric(PG_FUNCTION_ARGS) { int16 val = PG_GETARG_INT16(0); -<<<<<<< HEAD - quick_init_var(&result); - - int64_to_numericvar((int64) val, &result); - - res = make_result(&result); - - free_var(&result); - - PG_RETURN_NUMERIC(res); -======= PG_RETURN_NUMERIC(int64_to_numeric(val)); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } From 568c2fdd9776642fbacc52b99e2fba364cec1923 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Tue, 26 May 2026 12:50:48 +0500 Subject: [PATCH 470/589] Resolve conflicts in src/backend/utils/adt/pg_locale.c (#2590) Commit 257836a added a new include, miscadmin.h, to src/backend/utils/adt/pg_locale.c, while the earlier commit, e084591, already added a GPDB-specific include, utils/faultinjector.h, in the same location. Move the GPDB-specific include below. --- src/backend/utils/adt/pg_locale.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/backend/utils/adt/pg_locale.c b/src/backend/utils/adt/pg_locale.c index de54246600ed..a6d29be5f9a9 100644 --- a/src/backend/utils/adt/pg_locale.c +++ b/src/backend/utils/adt/pg_locale.c @@ -59,11 +59,7 @@ #include "catalog/pg_control.h" #include "catalog/pg_database.h" #include "mb/pg_wchar.h" -<<<<<<< HEAD -#include "utils/faultinjector.h" -======= #include "miscadmin.h" ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d #include "utils/builtins.h" #include "utils/formatting.h" #include "utils/hsearch.h" @@ -72,6 +68,8 @@ #include "utils/pg_locale.h" #include "utils/syscache.h" +#include "utils/faultinjector.h" + #ifdef USE_ICU #include #endif From f57cef90a96489628cecf8fb008afcb43752df7e Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Tue, 26 May 2026 12:51:10 +0500 Subject: [PATCH 471/589] Resolve conflicts in src/backend/utils/error/assert.c (#2591) Commit 18c170a in src/backend/utils/error/assert.c added a PID to the assert messages in the ExceptionalCondition function. However, an earlier commit, 6b0e52b, had already replaced the write_stderr function calls with ereport FATAL in the same location. --- src/backend/utils/error/assert.c | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/src/backend/utils/error/assert.c b/src/backend/utils/error/assert.c index 8b169d06905d..f8d517cac86d 100644 --- a/src/backend/utils/error/assert.c +++ b/src/backend/utils/error/assert.c @@ -37,34 +37,22 @@ ExceptionalCondition(const char *conditionName, const char *fileName, int lineNumber) { -<<<<<<< HEAD /* CDB: Try to tell the QD or client what happened. */ if (!PointerIsValid(conditionName) || !PointerIsValid(fileName) || !PointerIsValid(errorType)) ereport(FATAL, errFatalReturn(gp_reraise_signal), - errmsg("TRAP: ExceptionalCondition: bad arguments")); + errmsg("TRAP: ExceptionalCondition: bad arguments in PID %d")), + (int) getpid(); else ereport(FATAL, errFatalReturn(gp_reraise_signal), errmsg("Unexpected internal error"), - errdetail("%s(\"%s\", File: \"%s\", Line: %d)\n", - errorType, conditionName, fileName, lineNumber)); + errdetail("%s(\"%s\", File: \"%s\", Line: %d, PID: %d)\n", + errorType, conditionName, fileName, lineNumber, + (int) getpid())); -======= - /* Report the failure on stderr (or local equivalent) */ - if (!PointerIsValid(conditionName) - || !PointerIsValid(fileName) - || !PointerIsValid(errorType)) - write_stderr("TRAP: ExceptionalCondition: bad arguments in PID %d\n", - (int) getpid()); - else - write_stderr("TRAP: %s(\"%s\", File: \"%s\", Line: %d, PID: %d)\n", - errorType, conditionName, - fileName, lineNumber, (int) getpid()); - ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* Usually this shouldn't be needed, but make sure the msg went out */ fflush(stderr); From 36b88335ad10988a15426d3446654d56fa4aaae6 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Tue, 26 May 2026 12:56:53 +0500 Subject: [PATCH 472/589] Resolve conflicts in src/backend/utils/init/miscinit.c (#2592) Commit 44fc6e2 added a new include postmaster/interrupt.h to src/backend/utils/init/miscinit.c, while earlier commit 4d9c789 had already added a GPDB-specific include postmaster/fts.h in the same location. Move the GPDB-specific includes below separately. --- src/backend/utils/init/miscinit.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/backend/utils/init/miscinit.c b/src/backend/utils/init/miscinit.c index 7c4ec7d34627..935a69630aa5 100644 --- a/src/backend/utils/init/miscinit.c +++ b/src/backend/utils/init/miscinit.c @@ -37,11 +37,7 @@ #include "miscadmin.h" #include "pgstat.h" #include "postmaster/autovacuum.h" -<<<<<<< HEAD -#include "postmaster/fts.h" -======= #include "postmaster/interrupt.h" ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d #include "postmaster/postmaster.h" #include "postmaster/startup.h" #include "replication/walsender.h" @@ -53,8 +49,6 @@ #include "storage/proc.h" #include "storage/procarray.h" #include "utils/builtins.h" -#include "utils/faultinjector.h" -#include "utils/gdd.h" #include "utils/guc.h" #include "utils/inval.h" #include "utils/memutils.h" @@ -63,6 +57,9 @@ #include "utils/varlena.h" #include "cdb/cdbvars.h" +#include "postmaster/fts.h" +#include "utils/faultinjector.h" +#include "utils/gdd.h" #include "utils/resgroup.h" #include "utils/resource_manager.h" #include "utils/resscheduler.h" From 0ae88d140ca45432ecacd20cbdb7c9e919d4b05c Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Tue, 26 May 2026 13:09:25 +0500 Subject: [PATCH 473/589] Resolve conflicts in src/bin/pg_dump/common.c (#2593) Commit ac673a1 added a conditional return to the buildIndexArray function in src/bin/pg_dump/common.c, although earlier commit d572563 had already removed this function. --- src/bin/pg_dump/common.c | 107 --------------------------------------- 1 file changed, 107 deletions(-) diff --git a/src/bin/pg_dump/common.c b/src/bin/pg_dump/common.c index 4f0415164c00..ec5f9755d998 100644 --- a/src/bin/pg_dump/common.c +++ b/src/bin/pg_dump/common.c @@ -717,114 +717,7 @@ findObjectByCatalogId(CatalogId catalogId) entry = catalogid_lookup(catalogIdHash, catalogId); if (entry == NULL) return NULL; -<<<<<<< HEAD return entry->dobj; -======= - low = catalogIdMap; - high = catalogIdMap + (numCatalogIds - 1); - while (low <= high) - { - DumpableObject **middle; - int difference; - - middle = low + (high - low) / 2; - /* comparison must match DOCatalogIdCompare, below */ - difference = oidcmp((*middle)->catId.oid, catalogId.oid); - if (difference == 0) - difference = oidcmp((*middle)->catId.tableoid, catalogId.tableoid); - if (difference == 0) - return *middle; - else if (difference < 0) - low = middle + 1; - else - high = middle - 1; - } - return NULL; -} - -/* - * Find a DumpableObject by OID, in a pre-sorted array of one type of object - * - * Returns NULL for unknown OID - */ -static DumpableObject * -findObjectByOid(Oid oid, DumpableObject **indexArray, int numObjs) -{ - DumpableObject **low; - DumpableObject **high; - - /* - * This is the same as findObjectByCatalogId except we assume we need not - * look at table OID because the objects are all the same type. - * - * We could use bsearch() here, but the notational cruft of calling - * bsearch is nearly as bad as doing it ourselves; and the generalized - * bsearch function is noticeably slower as well. - */ - if (numObjs <= 0) - return NULL; - low = indexArray; - high = indexArray + (numObjs - 1); - while (low <= high) - { - DumpableObject **middle; - int difference; - - middle = low + (high - low) / 2; - difference = oidcmp((*middle)->catId.oid, oid); - if (difference == 0) - return *middle; - else if (difference < 0) - low = middle + 1; - else - high = middle - 1; - } - return NULL; -} - -/* - * Build an index array of DumpableObject pointers, sorted by OID - */ -static DumpableObject ** -buildIndexArray(void *objArray, int numObjs, Size objSize) -{ - DumpableObject **ptrs; - int i; - - if (numObjs <= 0) - return NULL; - - ptrs = (DumpableObject **) pg_malloc(numObjs * sizeof(DumpableObject *)); - for (i = 0; i < numObjs; i++) - ptrs[i] = (DumpableObject *) ((char *) objArray + i * objSize); - - /* We can use DOCatalogIdCompare to sort since its first key is OID */ - if (numObjs > 1) - qsort((void *) ptrs, numObjs, sizeof(DumpableObject *), - DOCatalogIdCompare); - - return ptrs; -} - -/* - * qsort comparator for pointers to DumpableObjects - */ -static int -DOCatalogIdCompare(const void *p1, const void *p2) -{ - const DumpableObject *obj1 = *(DumpableObject *const *) p1; - const DumpableObject *obj2 = *(DumpableObject *const *) p2; - int cmpval; - - /* - * Compare OID first since it's usually unique, whereas there will only be - * a few distinct values of tableoid. - */ - cmpval = oidcmp(obj1->catId.oid, obj2->catId.oid); - if (cmpval == 0) - cmpval = oidcmp(obj1->catId.tableoid, obj2->catId.tableoid); - return cmpval; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } /* From 0636ceb99bf836fbae433685d49dc9fd88b1b29e Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Tue, 26 May 2026 13:48:13 +0500 Subject: [PATCH 474/589] Resolve conflicts in src/bin/pg_dump/dumputils.h (#2594) Commit 8354e7b in src/bin/pg_dump/dumputils.h removed the first argument, PGconn *conn, from the definition of the buildShSecLabelQuery function. However, earlier commit 1c56023 had already added the definition of the new quoteAclUserName function above. --- src/bin/pg_dump/dumputils.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/bin/pg_dump/dumputils.h b/src/bin/pg_dump/dumputils.h index ff4bee07398f..c5c0c0e2daae 100644 --- a/src/bin/pg_dump/dumputils.h +++ b/src/bin/pg_dump/dumputils.h @@ -44,12 +44,9 @@ extern bool buildDefaultACLCommands(const char *type, const char *nspname, const char *owner, int remoteVersion, PQExpBuffer sql); -<<<<<<< HEAD extern void quoteAclUserName(PQExpBuffer output, const char *input); -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d extern void buildShSecLabelQuery(const char *catalog_name, Oid objectId, PQExpBuffer sql); extern void emitShSecLabels(PGconn *conn, PGresult *res, From 644a3b6668527009633b2b89d8792aee4a39ce76 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Wed, 27 May 2026 10:29:55 +0300 Subject: [PATCH 475/589] Resolve conflicts in src/include/catalog/pg_type.h (#2563) Commit f90149e6285aaae6b48559afce1bd638ee26c33e added new defines, however there is a GPDB-specific macro nearby. Also replace CASHOID and LSNOID in GPDB-specific code. --- src/backend/access/transam/xlogfuncs_gp.c | 4 ++-- src/backend/cdb/cdblegacyhash.c | 2 +- src/include/catalog/pg_type.h | 4 +--- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/backend/access/transam/xlogfuncs_gp.c b/src/backend/access/transam/xlogfuncs_gp.c index 781056324eb2..d95c3fdd39c6 100644 --- a/src/backend/access/transam/xlogfuncs_gp.c +++ b/src/backend/access/transam/xlogfuncs_gp.c @@ -64,7 +64,7 @@ gp_create_restore_point(PG_FUNCTION_ARGS) TupleDescInitEntry(tupdesc, (AttrNumber) 1, "gp_segment_id", INT2OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "restore_lsn", - LSNOID, -1, 0); + PG_LSNOID, -1, 0); funcctx->tuple_desc = BlessTupleDesc(tupdesc); @@ -198,7 +198,7 @@ gp_switch_wal(PG_FUNCTION_ARGS) TupleDescInitEntry(tupdesc, (AttrNumber) 1, "segment_id", INT2OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "switch_lsn", - LSNOID, -1, 0); + PG_LSNOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 3, "switch_walfilename", TEXTOID, -1, 0); diff --git a/src/backend/cdb/cdblegacyhash.c b/src/backend/cdb/cdblegacyhash.c index 5a4b89f831e6..dd7f0d515160 100644 --- a/src/backend/cdb/cdblegacyhash.c +++ b/src/backend/cdb/cdblegacyhash.c @@ -234,7 +234,7 @@ get_legacy_cdbhash_opclass_for_base_type(Oid orig_typid) case OIDVECTOROID: opclass_name = "cdbhash_oidvector_ops"; break; - case CASHOID: + case MONEYOID: opclass_name = "cdbhash_cash_ops"; break; case UUIDOID: diff --git a/src/include/catalog/pg_type.h b/src/include/catalog/pg_type.h index 5b4587cdd263..98bc19c00360 100644 --- a/src/include/catalog/pg_type.h +++ b/src/include/catalog/pg_type.h @@ -324,18 +324,16 @@ typedef FormData_pg_type *Form_pg_type; (typid) == ANYCOMPATIBLENONARRAYOID || \ (typid) == ANYCOMPATIBLERANGEOID) -<<<<<<< HEAD /* Is a type OID suitable for describe callback functions? */ #define TypeSupportsDescribe(typid) \ ((typid) == RECORDOID) -======= + /* * Backwards compatibility for ancient random spellings of pg_type OID macros. * Don't use these names in new code. */ #define CASHOID MONEYOID #define LSNOID PG_LSNOID ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d #endif /* EXPOSE_TO_CLIENT_CODE */ From a9e7e9c38d859ab82b9561a8fe729d294bde4212 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Wed, 27 May 2026 13:40:35 +0500 Subject: [PATCH 476/589] Resolve conflicts in src/bin/pg_rewind/copy_fetch.c (#2599) Commit f81e97d in src/bin/pg_rewind/copy_fetch.c in the copy_executeFileMap function changed the narray and array fields to nentries and entries, while the earlier commit 1a770cf had already changed the pagemap field to target_pages_to_overwrite below. --- src/bin/pg_rewind/copy_fetch.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/bin/pg_rewind/copy_fetch.c b/src/bin/pg_rewind/copy_fetch.c index 20273f8bb334..bee33102d415 100644 --- a/src/bin/pg_rewind/copy_fetch.c +++ b/src/bin/pg_rewind/copy_fetch.c @@ -221,11 +221,7 @@ copy_executeFileMap(filemap_t *map) for (i = 0; i < map->nentries; i++) { -<<<<<<< HEAD - entry = map->array[i]; -======= entry = map->entries[i]; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d execute_pagemap(&entry->target_pages_to_overwrite, entry->path); switch (entry->action) From 9fab257550e7f1944026a5afca2fa82d956d9d04 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Wed, 27 May 2026 19:17:37 +0500 Subject: [PATCH 477/589] Resolve conflicts in src/bin/pg_upgrade/check.c (#2601) Commit 1ed6b89 in src/bin/pg_upgrade/check.c added a conditional call to the new function check_for_user_defined_postfix_ops in the function check_and_dump_old_cluster. However, an earlier commit, 5a1676e, had already added a conditional call to the function old_GPDB6_check_for_unsupported_sha256_password_hashes in the same location. --- src/bin/pg_upgrade/check.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index d2341a86f97f..c7c26585abfa 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -105,7 +105,6 @@ check_and_dump_old_cluster(bool live_check, char **sequence_script_file_name) check_for_isn_and_int8_passing_mismatch(&old_cluster); /* -<<<<<<< HEAD * Check for various Greenplum failure cases */ check_greenplum(); @@ -113,13 +112,13 @@ check_and_dump_old_cluster(bool live_check, char **sequence_script_file_name) /* GPDB 7 removed support for SHA-256 hashed passwords */ if (GET_MAJOR_VERSION(old_cluster.major_version) <= 905) old_GPDB6_check_for_unsupported_sha256_password_hashes(); -======= + + /* * Pre-PG 14 allowed user defined postfix operators, which are not * supported anymore. Verify there are none, iff applicable. */ if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1300) check_for_user_defined_postfix_ops(&old_cluster); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* * Pre-PG 12 allowed tables to be declared WITH OIDS, which is not From 336d0803e3929383e2222d871be01a2da82530c1 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Wed, 27 May 2026 19:17:56 +0500 Subject: [PATCH 478/589] Resolve conflicts in src/bin/pg_upgrade/dump.c (#2602) Commit 257836a added new collation options to the generate_old_dump function in src/bin/pg_upgrade/dump.c, while earlier commit 7d0121c had already added utility options in the same location. --- src/bin/pg_upgrade/dump.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/bin/pg_upgrade/dump.c b/src/bin/pg_upgrade/dump.c index 7f9dbf79c469..e4a36974e02e 100644 --- a/src/bin/pg_upgrade/dump.c +++ b/src/bin/pg_upgrade/dump.c @@ -53,14 +53,9 @@ generate_old_dump(void) snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_LOG_FILE_MASK, old_db->db_oid); parallel_exec_prog(log_file_name, NULL, -<<<<<<< HEAD "%s \"%s/pg_dump\" %s --schema-only --quote-all-identifiers " - "--binary-upgrade --format=custom %s --file=\"%s\" %s", - PG_OPTIONS_UTILITY_MODE_VERSION(old_cluster.major_version), -======= - "\"%s/pg_dump\" %s --schema-only --quote-all-identifiers " "--binary-upgrade --format=custom %s %s --file=\"%s\" %s", ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + PG_OPTIONS_UTILITY_MODE_VERSION(old_cluster.major_version), new_cluster.bindir, cluster_conn_opts(&old_cluster), log_opts.verbose ? "--verbose" : "", user_opts.ind_coll_unknown ? From 292631959a8e4c500dee3801b3f426c1f3bfaa94 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Wed, 27 May 2026 19:18:16 +0500 Subject: [PATCH 479/589] Resolve conflicts in src/bin/pg_upgrade/option.c (#2603) Commit 257836a added new index-collation-versions-unknown options to the generate_old_dump function in src/bin/pg_upgrade/option.c. Earlier commits 9e89719 and e99317c had already added continue-check-on-fatal and skip-target-check options in the same location. --- src/bin/pg_upgrade/option.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/bin/pg_upgrade/option.c b/src/bin/pg_upgrade/option.c index e000f4a5e534..4ff2e08f7dee 100644 --- a/src/bin/pg_upgrade/option.c +++ b/src/bin/pg_upgrade/option.c @@ -324,13 +324,10 @@ usage(void) printf(_(" -v, --verbose enable verbose internal logging\n")); printf(_(" -V, --version display version information, then exit\n")); printf(_(" --clone clone instead of copying files to new cluster\n")); -<<<<<<< HEAD printf(_(" --continue-check-on-fatal goes through all pg_upgrade checks; should be used with -c\n")); printf(_(" --skip-target-check skip all checks and comparisons of new cluster; should be used with -c\n")); -======= printf(_(" --index-collation-versions-unknown\n")); printf(_(" mark text indexes as needing to be rebuilt\n")); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d printf(_(" -?, --help show this help, then exit\n")); printf(_("\n" "Before running pg_upgrade you must:\n" From 6d78b263bb4787b89058943f7f53eb42e4ade9d4 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Wed, 27 May 2026 19:18:34 +0500 Subject: [PATCH 480/589] Resolve conflicts in src/bin/pg_upgrade/relfilenode.c (#2604) Commit 0691797 in src/bin/pg_upgrade/relfilenode.c in the transfer_single_new_db function removed the 8.4 check, although earlier commits had already added handling of AO tables in the same location. --- src/bin/pg_upgrade/relfilenode.c | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/src/bin/pg_upgrade/relfilenode.c b/src/bin/pg_upgrade/relfilenode.c index 3621d26704d4..25aa549281e5 100644 --- a/src/bin/pg_upgrade/relfilenode.c +++ b/src/bin/pg_upgrade/relfilenode.c @@ -169,7 +169,6 @@ transfer_single_new_db(FileNameMap *maps, int size, char *old_tablespace) { RelType type = maps[mapnum].type; -<<<<<<< HEAD if (type == AO || type == AOCS) { transfer_ao(&maps[mapnum]); @@ -179,25 +178,13 @@ transfer_single_new_db(FileNameMap *maps, int size, char *old_tablespace) /* transfer primary file */ transfer_relfile(&maps[mapnum], "", vm_must_add_frozenbit); - /* fsm/vm files added in PG 8.4 */ - if (GET_MAJOR_VERSION(old_cluster.major_version) >= 804) - { - /* - * Copy/link any fsm and vm files, if they exist - */ - transfer_relfile(&maps[mapnum], "_fsm", vm_must_add_frozenbit); - if (vm_crashsafe_match) - transfer_relfile(&maps[mapnum], "_vm", vm_must_add_frozenbit); - } + /* + * Copy/link any fsm and vm files, if they exist + */ + transfer_relfile(&maps[mapnum], "_fsm", vm_must_add_frozenbit); + if (vm_crashsafe_match) + transfer_relfile(&maps[mapnum], "_vm", vm_must_add_frozenbit); } -======= - /* - * Copy/link any fsm and vm files, if they exist - */ - transfer_relfile(&maps[mapnum], "_fsm", vm_must_add_frozenbit); - if (vm_crashsafe_match) - transfer_relfile(&maps[mapnum], "_vm", vm_must_add_frozenbit); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } } } From db3786e3dfdedd6b71f0e7d97384ad570f38b36d Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Wed, 27 May 2026 19:18:58 +0500 Subject: [PATCH 481/589] Resolve conflicts in src/bin/pgbench/pgbench.c (#2605) 1) Commit 9796f45 in the file src/bin/pgbench/pgbench.c in the initCreateTables function removed the call to the append_fillfactor function, while the earlier commit f6ec65f had already added storage_clause in the same place. 2) Commit 9796f45 in the file src/bin/pgbench/pgbench.c removed the append_fillfactor function, while the earlier commit 71e0c39 had already added storage_clause in the same place. 3) Commit 9796f45 in the file src/bin/pgbench/pgbench.c in the initCreatePKeys function replaced the call to the strlcpy function with appendPQExpBufferStr, while the earlier commit 598f4b0 already added a condition when calling the strlcpy function in the same place. --- src/bin/pgbench/pgbench.c | 37 +++++-------------------------------- 1 file changed, 5 insertions(+), 32 deletions(-) diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index abbaf3407908..9684cf8bf226 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -3777,16 +3777,10 @@ initCreateTables(PGconn *con) else if (ddl->declare_fillfactor) { /* fillfactor is only expected on actual tables */ -<<<<<<< HEAD - append_fillfactor(opts, sizeof(opts)); - else - snprintf(opts + strlen(opts), sizeof(opts) - strlen(opts), - " with (%s)", - storage_clause); -======= appendPQExpBuffer(&query, " with (fillfactor=%d)", fillfactor); } ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + else + appendPQExpBuffer(&query, " with (%s)", storage_clause); if (tablespace != NULL) { @@ -3810,22 +3804,6 @@ initCreateTables(PGconn *con) } /* -<<<<<<< HEAD - * add fillfactor percent option. - * - * XXX - As default is 100, it could be removed in this case. - */ -static void -append_fillfactor(char *opts, int len) -{ - snprintf(opts + strlen(opts), len - strlen(opts), - " with (fillfactor=%d, %s)", - fillfactor, storage_clause); -} - -/* -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d * Truncate away any old data, in one command in case there are foreign keys */ static void @@ -4070,17 +4048,12 @@ initCreatePKeys(PGconn *con) for (i = 0; i < lengthof(DDLINDEXes); i++) { -<<<<<<< HEAD - char buffer[256]; + resetPQExpBuffer(&query); if (use_unique_key) - strlcpy(buffer, DDLINDEXes[i], sizeof(buffer)); + appendPQExpBufferStr(&query, DDLINDEXes[i]); else - strlcpy(buffer, NON_UNIQUE_INDEX_DDLINDEXes[i], sizeof(buffer)); -======= - resetPQExpBuffer(&query); - appendPQExpBufferStr(&query, DDLINDEXes[i]); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + appendPQExpBufferStr(&query, NON_UNIQUE_INDEX_DDLINDEXes[i]); if (index_tablespace != NULL) { From 42cfcf0b6ec964f13244e7df5ef53ab3ffe34207 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Thu, 28 May 2026 10:01:28 +0300 Subject: [PATCH 482/589] Resolve conflicts in src/backend/access/transam/multixact.c and xlog.c (#2564) Commit dee663f7843902535a15ae366cede8b4089f1144 removed ShutdownMultiXact which had an extra empty line in GPDB code. It also removed several other Shutdown* functions, and replaced SimpleLruFlush with SimpleLruWriteAll. Apply the same changes to DistributedLog functions as well: remove Shutdown and replace the use of SimpleLruFlush. --- src/backend/access/transam/distributedlog.c | 20 ++------------------ src/backend/access/transam/multixact.c | 17 ----------------- src/backend/access/transam/xlog.c | 8 -------- src/include/access/distributedlog.h | 1 - 4 files changed, 2 insertions(+), 44 deletions(-) diff --git a/src/backend/access/transam/distributedlog.c b/src/backend/access/transam/distributedlog.c index 6d083c30a75b..4dfa99cb9d39 100644 --- a/src/backend/access/transam/distributedlog.c +++ b/src/backend/access/transam/distributedlog.c @@ -836,22 +836,6 @@ DistributedLog_Startup(TransactionId oldestActiveXid, LWLockRelease(DistributedLogControlLock); } -/* - * This must be called ONCE during postmaster or standalone-backend shutdown - */ -void -DistributedLog_Shutdown(void) -{ - if (IS_QUERY_DISPATCHER()) - return; - - elog((Debug_print_full_dtm ? LOG : DEBUG5), - "DistributedLog_Shutdown"); - - /* Flush dirty DistributedLog pages to disk */ - SimpleLruFlush(DistributedLogCtl, false); -} - /* * Perform a checkpoint --- either during shutdown, or on-the-fly */ @@ -864,8 +848,8 @@ DistributedLog_CheckPoint(void) elog((Debug_print_full_dtm ? LOG : DEBUG5), "DistributedLog_CheckPoint"); - /* Flush dirty DistributedLog pages to disk */ - SimpleLruFlush(DistributedLogCtl, true); + /* Write dirty DistributedLog pages to disk */ + SimpleLruWriteAll(DistributedLogCtl, true); } diff --git a/src/backend/access/transam/multixact.c b/src/backend/access/transam/multixact.c index 44df8de9a9cc..d8f7c7db2fdd 100644 --- a/src/backend/access/transam/multixact.c +++ b/src/backend/access/transam/multixact.c @@ -2121,23 +2121,6 @@ TrimMultiXact(void) } /* -<<<<<<< HEAD - * This must be called ONCE during postmaster or standalone-backend shutdown - */ -void -ShutdownMultiXact(void) -{ - /* Flush dirty MultiXact pages to disk */ - TRACE_POSTGRESQL_MULTIXACT_CHECKPOINT_START(false); - SimpleLruFlush(MultiXactOffsetCtl, false); - SimpleLruFlush(MultiXactMemberCtl, false); - - TRACE_POSTGRESQL_MULTIXACT_CHECKPOINT_DONE(false); -} - -/* -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d * Get the MultiXact data to save in a checkpoint record */ void diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8482c48c5737..888ca8874a84 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -8965,14 +8965,6 @@ ShutdownXLOG(int code pg_attribute_unused() , Datum arg pg_attribute_unused() ) CreateCheckPoint(CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_IMMEDIATE); } -<<<<<<< HEAD - ShutdownCLOG(); - ShutdownCommitTs(); - ShutdownSUBTRANS(); - ShutdownMultiXact(); - DistributedLog_Shutdown(); -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } /* diff --git a/src/include/access/distributedlog.h b/src/include/access/distributedlog.h index cbd25344f8fe..7c2a7ff9f8a3 100644 --- a/src/include/access/distributedlog.h +++ b/src/include/access/distributedlog.h @@ -61,7 +61,6 @@ extern void DistributedLog_BootStrap(void); extern void DistributedLog_Startup( TransactionId oldestActiveXid, TransactionId nextXid); -extern void DistributedLog_Shutdown(void); extern void DistributedLog_CheckPoint(void); extern void DistributedLog_Extend(TransactionId newestXid); extern bool DistributedLog_GetLowWaterXid( From 9424faec5914cb5eb053a52951898120d657147f Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 28 May 2026 12:47:01 +0500 Subject: [PATCH 483/589] Resolve conflicts in src/backend/commands/tablecmds.c (#2571) 1) Commit afc7e0a in src/backend/commands/tablecmds.c added a conditional error message in the RemoveRelations function, while earlier commit 64b63f6(d8d378d) had already added a conditional call to the find_all_inheritors function in the same location. 2) Commit 257836a in src/backend/commands/tablecmds.c added handling for the AT_AlterCollationRefreshVersion case in the AlterTableGetLockLevel function, while earlier GPDB-specific commits had already added handling for GPDB-specific cases in the same location. 3) Commit 5b02d68 in src/backend/commands/tablecmds.c added a condition to the ATParseTransformCmd function call for the AT_AddConstraint and AT_AddConstraintRecurse cases and changed the comments. However, the earlier commit 89f85cc had already added a new argument to this function call. 4) Commit 5b02d68 in src/backend/commands/tablecmds.c moved the large switch (cmd2->subtype) in the ATParseTransformCmd function. However, the earlier commit 89f85cc had already added a conditional definition save in the same place. --- src/backend/commands/tablecmds.c | 67 ++++++++++++-------------------- 1 file changed, 24 insertions(+), 43 deletions(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index dc13a05b30b7..1f7c40aa81ba 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -1779,7 +1779,17 @@ RemoveRelations(DropStmt *drop) } /* -<<<<<<< HEAD + * Concurrent index drop cannot be used with partitioned indexes, + * either. + */ + if ((flags & PERFORM_DELETION_CONCURRENTLY) != 0 && + get_rel_relkind(relOid) == RELKIND_PARTITIONED_INDEX) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot drop partitioned index \"%s\" concurrently", + rel->relname))); + + /* * If we're told to drop a partitioned index, we must acquire lock on * all the children of its parent partitioned table before proceeding. * Otherwise we'd try to lock the child index partitions before their @@ -1790,17 +1800,6 @@ RemoveRelations(DropStmt *drop) (void) find_all_inheritors(state.heapOid, state.heap_lockmode, NULL); -======= - * Concurrent index drop cannot be used with partitioned indexes, - * either. - */ - if ((flags & PERFORM_DELETION_CONCURRENTLY) != 0 && - get_rel_relkind(relOid) == RELKIND_PARTITIONED_INDEX) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot drop partitioned index \"%s\" concurrently", - rel->relname))); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* OK, we're ready to delete this one */ obj.classId = RelationRelationId; @@ -4687,7 +4686,10 @@ AlterTableGetLockLevel(List *cmds) cmd_lockmode = AccessShareLock; break; -<<<<<<< HEAD + case AT_AlterCollationRefreshVersion: + cmd_lockmode = AccessExclusiveLock; + break; + /* GPDB additions */ case AT_ExpandTable: case AT_ExpandPartitionTablePrepare: @@ -4711,9 +4713,6 @@ AlterTableGetLockLevel(List *cmds) case AT_PartRename: case AT_PartExchange: case AT_PartSetTemplate: -======= - case AT_AlterCollationRefreshVersion: ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d cmd_lockmode = AccessExclusiveLock; break; @@ -5531,18 +5530,12 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, lockmode); break; case AT_AddConstraint: /* ADD CONSTRAINT */ -<<<<<<< HEAD - cmd = ATParseTransformCmd(wqueue, tab, rel, cmd, false, lockmode, - cur_pass, context, &cmd->execStmts); - /* Might not have gotten AddConstraint back from parse transform */ -======= /* Transform the command only during initial examination */ if (cur_pass == AT_PASS_ADD_CONSTR) cmd = ATParseTransformCmd(wqueue, tab, rel, cmd, false, lockmode, - cur_pass, context); + cur_pass, context, &cmd->execStmts); /* Depending on constraint type, might be no more work to do now */ ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d if (cmd != NULL) address = ATExecAddConstraint(wqueue, tab, rel, @@ -5550,18 +5543,12 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, false, false, lockmode); break; case AT_AddConstraintRecurse: /* ADD CONSTRAINT with recursion */ -<<<<<<< HEAD - cmd = ATParseTransformCmd(wqueue, tab, rel, cmd, true, lockmode, - cur_pass, context, &cmd->execStmts); - /* Might not have gotten AddConstraint back from parse transform */ -======= /* Transform the command only during initial examination */ if (cur_pass == AT_PASS_ADD_CONSTR) cmd = ATParseTransformCmd(wqueue, tab, rel, cmd, true, lockmode, - cur_pass, context); + cur_pass, context, &cmd->execStmts); /* Depending on constraint type, might be no more work to do now */ ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d if (cmd != NULL) address = ATExecAddConstraint(wqueue, tab, rel, @@ -5915,18 +5902,6 @@ ATParseTransformCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, */ switch (cmd2->subtype) { -<<<<<<< HEAD - /* Found the transformed version of our subcommand */ - cmd2->subtype = cmd->subtype; /* copy recursion flag */ - newcmd = cmd2; - - /* - * In the QD save transformed version of definition for executing - * in the QE - */ - if (Gp_role == GP_ROLE_DISPATCH) - cmd->def = newcmd->def; -======= case AT_SetNotNull: /* Need command-specific recursion decision */ ATPrepSetNotNull(wqueue, rel, cmd2, @@ -5980,7 +5955,6 @@ ATParseTransformCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, { /* OK, queue it up for later */ tab->subcmds[pass] = lappend(tab->subcmds[pass], cmd2); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } else { @@ -5992,6 +5966,13 @@ ATParseTransformCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, { /* Found the transformed version of our subcommand */ newcmd = cmd2; + + /* + * In the QD save transformed version of definition for + * executing in the QE + */ + if (Gp_role == GP_ROLE_DISPATCH) + cmd->def = newcmd->def; } else elog(ERROR, "ALTER TABLE scheduling failure: bogus item for pass %d", From 4212fa8fff37e4f7dc57f7bbc1d5b96b4496551f Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 28 May 2026 14:41:28 +0500 Subject: [PATCH 484/589] Resolve conflicts in src/backend/executor/execMain.c (#2572) 1) Commit 1375422 in src/backend/executor/execMain.c removed the conditional initialization of ResultRelInfo in the InitPlan function, while the earlier commit 6b0e52b had already added a comment before this same location. 2) Commit 1375422 in src/backend/executor/execMain.c separated part of the ExecEndPlan function into a new function, ExecCloseResultRelations, while the earlier commit 6b0e52b had already reversed the loop for calling ExecCloseIndices in this same location. 3) Commit a04daa9 removed the es_result_relation_info field from the EState structure in src/include/nodes/execnodes.h. Remove it in GPDB-specific locations. 4) Commit 1375422 removed the es_num_result_relations field from the EState structure in src/include/nodes/execnodes.h, replacing it with the new es_opened_result_relations field. Replace it in GPDB-specific locations. --- src/backend/access/aocs/aocs_compaction.c | 2 - .../access/appendonly/appendonly_compaction.c | 2 - src/backend/commands/copy.c | 6 -- src/backend/executor/execMain.c | 102 +----------------- 4 files changed, 3 insertions(+), 109 deletions(-) diff --git a/src/backend/access/aocs/aocs_compaction.c b/src/backend/access/aocs/aocs_compaction.c index 2cb52b79eb4d..af5231107ae8 100644 --- a/src/backend/access/aocs/aocs_compaction.c +++ b/src/backend/access/aocs/aocs_compaction.c @@ -264,8 +264,6 @@ AOCSSegmentFileFullCompaction(Relation aorel, resultRelInfo->ri_TrigDesc = NULL; /* we don't fire triggers */ ExecOpenIndices(resultRelInfo, false); estate->es_result_relations = resultRelInfo; - estate->es_num_result_relations = 1; - estate->es_result_relation_info = resultRelInfo; /* * We don't want uniqueness checks to be performed while "insert"ing tuples diff --git a/src/backend/access/appendonly/appendonly_compaction.c b/src/backend/access/appendonly/appendonly_compaction.c index a9ca318d06d1..189bc126269c 100644 --- a/src/backend/access/appendonly/appendonly_compaction.c +++ b/src/backend/access/appendonly/appendonly_compaction.c @@ -445,8 +445,6 @@ AppendOnlySegmentFileFullCompaction(Relation aorel, resultRelInfo->ri_TrigDesc = NULL; /* we don't fire triggers */ ExecOpenIndices(resultRelInfo, false); estate->es_result_relations = resultRelInfo; - estate->es_num_result_relations = 1; - estate->es_result_relation_info = resultRelInfo; /* * We don't want uniqueness checks to be performed while "insert"ing tuples diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 5f671b9115bf..f556b77ca8a0 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -4232,12 +4232,6 @@ CopyFrom(CopyState cstate) { if (!NextCopyFromExecute(cstate, econtext, myslot->tts_values, myslot->tts_isnull)) break; - - /* - * NextCopyFromExecute set up estate->es_result_relation_info, - * and stored the tuple in the correct slot. - */ - resultRelInfo = estate->es_result_relation_info; } else { diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index 284db49baf08..e91a1410ee5d 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -1710,94 +1710,6 @@ InitPlan(QueryDesc *queryDesc, int eflags) estate->es_plannedstmt = plannedstmt; /* -<<<<<<< HEAD - * Initialize ResultRelInfo data structures, and open the result rels. - * - * CDB: Note that we need this info even if we aren't the slice that will be doing - * the actual updating, since it's where we learn things, such as if the row needs to - * contain OIDs or not. - */ - if (plannedstmt->resultRelations) - { - List *resultRelations = plannedstmt->resultRelations; - int numResultRelations = list_length(resultRelations); - ResultRelInfo *resultRelInfos; - ResultRelInfo *resultRelInfo; - - resultRelInfos = (ResultRelInfo *) - palloc(numResultRelations * sizeof(ResultRelInfo)); - resultRelInfo = resultRelInfos; - foreach(l, plannedstmt->resultRelations) - { - Index resultRelationIndex = lfirst_int(l); - Relation resultRelation; - - resultRelation = ExecGetRangeTableRelation(estate, - resultRelationIndex); - InitResultRelInfo(resultRelInfo, - resultRelation, - resultRelationIndex, - NULL, - estate->es_instrument); - resultRelInfo++; - } - estate->es_result_relations = resultRelInfos; - estate->es_num_result_relations = numResultRelations; - - /* es_result_relation_info is NULL except when within ModifyTable */ - estate->es_result_relation_info = NULL; - - /* - * In the partitioned result relation case, also build ResultRelInfos - * for all the partitioned table roots, because we will need them to - * fire statement-level triggers, if any. - */ - if (plannedstmt->rootResultRelations) - { - int num_roots = list_length(plannedstmt->rootResultRelations); - - resultRelInfos = (ResultRelInfo *) - palloc(num_roots * sizeof(ResultRelInfo)); - resultRelInfo = resultRelInfos; - foreach(l, plannedstmt->rootResultRelations) - { - Index resultRelIndex = lfirst_int(l); - Relation resultRelDesc; - - resultRelDesc = ExecGetRangeTableRelation(estate, - resultRelIndex); - InitResultRelInfo(resultRelInfo, - resultRelDesc, - resultRelIndex, - NULL, - estate->es_instrument); - resultRelInfo++; - } - - estate->es_root_result_relations = resultRelInfos; - estate->es_num_root_result_relations = num_roots; - } - else - { - estate->es_root_result_relations = NULL; - estate->es_num_root_result_relations = 0; - } - } - else - { - /* - * if no result relation, then set state appropriately - */ - estate->es_result_relations = NULL; - estate->es_num_result_relations = 0; - estate->es_result_relation_info = NULL; - estate->es_root_result_relations = NULL; - estate->es_num_root_result_relations = 0; - } - - /* -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d * Next, build the ExecRowMark array from the PlanRowMark(s), if any. */ if (plannedstmt->rowMarks) @@ -2543,14 +2455,6 @@ ExecEndPlan(PlanState *planstate, EState *estate) * Close any Relations that have been opened for range table entries or * result relations. */ -<<<<<<< HEAD - resultRelInfo = estate->es_result_relations; - for (i = 0; i < estate->es_num_result_relations; i++) - { - ExecCloseIndices(resultRelInfo); - resultRelInfo++; - } -======= ExecCloseResultRelations(estate); ExecCloseRangeTableRelations(estate); } @@ -2562,7 +2466,6 @@ void ExecCloseResultRelations(EState *estate) { ListCell *l; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* * close indexes of result relation(s) if any. (Rels themselves are @@ -4116,14 +4019,15 @@ AdjustReplicatedTableCounts(EState *estate) ResultRelInfo *resultRelInfo; bool containReplicatedTable = false; int numsegments = 1; + ListCell *l; if (Gp_role != GP_ROLE_DISPATCH) return; /* check if result_relations contain replicated table*/ - for (i = 0; i < estate->es_num_result_relations; i++) + foreach(l, estate->es_opened_result_relations) { - resultRelInfo = estate->es_result_relations + i; + resultRelInfo = lfirst(l); if (!resultRelInfo->ri_RelationDesc->rd_cdbpolicy) continue; From d3c83e16004d1a7c97b7c787309feda7738df70e Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Thu, 28 May 2026 13:23:26 +0300 Subject: [PATCH 485/589] Resolve conflicts in defrem.h, parsenodes.h, indexcmds.c, utility.c (#2561) 1. Commit a6642b3ae060976b42830b7dc8f29ec190ab05e4 removed "concurrent" field from ReindexStmt and switched to using bitflags. Update the rest of the code. 2. Commit a6642b3ae060976b42830b7dc8f29ec190ab05e4 added topLevel argument, however this commit was already cherry-picked as 67a7c0f06be0da1cd466e24fb0c33b51308dc1e0, but using "concurrent" argument, and without REINDEXOPT_MISSING_OK option added in 1d65416661bbb0b165865a521ce038ffb61b12ad, and also without the fix from commit d1b0007639a1cefb5dcecf44999a4461f4c34089. --- src/backend/commands/indexcmds.c | 82 +++----------------------------- src/backend/tcop/utility.c | 9 ---- src/include/commands/defrem.h | 6 --- src/include/nodes/parsenodes.h | 4 -- 4 files changed, 6 insertions(+), 95 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 848a8b3a66aa..244f4d9e267d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -100,13 +100,8 @@ static void RangeVarCallbackForReindexIndex(const RangeVar *relation, Oid relId, Oid oldRelId, void *arg); static bool ReindexRelationConcurrently(Oid relationOid, int options); -<<<<<<< HEAD -static void ReindexPartitions(Oid relid, int options, bool concurrent, bool isTopLevel); -static void ReindexMultipleInternal(List *relids, int options, bool concurrent); -======= static void ReindexPartitions(Oid relid, int options, bool isTopLevel); static void ReindexMultipleInternal(List *relids, int options); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d static void reindex_error_callback(void *args); static void update_relispartition(Oid relationId, bool newval); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); @@ -2773,20 +2768,14 @@ ChooseIndexColumnNames(List *indexElems) * Recreate a specific index. */ void -<<<<<<< HEAD ReindexIndex(ReindexStmt *stmt, bool isTopLevel) -======= -ReindexIndex(RangeVar *indexRelation, int options, bool isTopLevel) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d { RangeVar *indexRelation = stmt->relation; int options = stmt->options; - bool concurrent = stmt->concurrent; struct ReindexIndexCallbackState state; Oid indOid; char persistence; char relkind; -<<<<<<< HEAD /* * On QE, we already know the index relation oid since we set it before @@ -2795,7 +2784,7 @@ ReindexIndex(RangeVar *indexRelation, int options, bool isTopLevel) */ if (Gp_role == GP_ROLE_EXECUTE) { - Assert(OidIsValid(stmt->relid) && !concurrent); + Assert(OidIsValid(stmt->relid) && (options & REINDEXOPT_CONCURRENTLY) == 0); LockRelationOid(stmt->relid, AccessExclusiveLock); persistence = get_rel_persistence(stmt->relid); @@ -2805,8 +2794,6 @@ ReindexIndex(RangeVar *indexRelation, int options, bool isTopLevel) reindex_index(stmt->relid, false, persistence, options); return; } -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* * Find and lock index, and check permissions on table; use callback to @@ -2834,16 +2821,9 @@ ReindexIndex(RangeVar *indexRelation, int options, bool isTopLevel) persistence = get_rel_persistence(indOid); relkind = get_rel_relkind(indOid); -<<<<<<< HEAD - - if (relkind == RELKIND_PARTITIONED_INDEX) - ReindexPartitions(indOid, options, concurrent, isTopLevel); - else if (concurrent && -======= if (relkind == RELKIND_PARTITIONED_INDEX) ReindexPartitions(indOid, options, isTopLevel); else if ((options & REINDEXOPT_CONCURRENTLY) != 0 && ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d persistence != RELPERSISTENCE_TEMP) ReindexRelationConcurrently(indOid, options); else @@ -2862,7 +2842,6 @@ ReindexIndex(RangeVar *indexRelation, int options, bool isTopLevel) qestmt->kind = REINDEX_OBJECT_INDEX; qestmt->relation = NULL; qestmt->options = options; - qestmt->concurrent = concurrent; qestmt->relid = indOid; CdbDispatchUtilityStatement((Node *) qestmt, @@ -2949,15 +2928,10 @@ RangeVarCallbackForReindexIndex(const RangeVar *relation, * Recreate all indexes of a table (and of its toast table, if any) */ Oid -<<<<<<< HEAD ReindexTable(ReindexStmt *stmt, bool isTopLevel) -======= -ReindexTable(RangeVar *relation, int options, bool isTopLevel) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d { RangeVar *relation = stmt->relation; int options = stmt->options; - bool concurrent = stmt->concurrent; Oid heapOid; bool result; @@ -2990,13 +2964,8 @@ ReindexTable(RangeVar *relation, int options, bool isTopLevel) RangeVarCallbackOwnsTable, NULL); if (get_rel_relkind(heapOid) == RELKIND_PARTITIONED_TABLE) -<<<<<<< HEAD - ReindexPartitions(heapOid, options, concurrent, isTopLevel); - else if (concurrent && -======= ReindexPartitions(heapOid, options, isTopLevel); else if ((options & REINDEXOPT_CONCURRENTLY) != 0 && ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d get_rel_persistence(heapOid) != RELPERSISTENCE_TEMP) { result = ReindexRelationConcurrently(heapOid, options); @@ -3030,7 +2999,6 @@ ReindexTable(RangeVar *relation, int options, bool isTopLevel) qestmt->kind = REINDEX_OBJECT_TABLE; qestmt->relation = NULL; qestmt->options = options; - qestmt->concurrent = concurrent; qestmt->relid = heapOid; CdbDispatchUtilityStatement((Node *) qestmt, @@ -3218,11 +3186,7 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, * Process each relation listed in a separate transaction. Note that this * commits and then starts a new transaction immediately. */ -<<<<<<< HEAD - ReindexMultipleInternal(relids, options, concurrent); -======= ReindexMultipleInternal(relids, options); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d MemoryContextDelete(private_context); } @@ -3253,11 +3217,7 @@ reindex_error_callback(void *arg) * by the caller. */ static void -<<<<<<< HEAD -ReindexPartitions(Oid relid, int options, bool concurrent, bool isTopLevel) -======= ReindexPartitions(Oid relid, int options, bool isTopLevel) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d { List *partitions = NIL; char relkind = get_rel_relkind(relid); @@ -3334,11 +3294,7 @@ ReindexPartitions(Oid relid, int options, bool isTopLevel) * Process each partition listed in a separate transaction. Note that * this commits and then starts a new transaction immediately. */ -<<<<<<< HEAD - ReindexMultipleInternal(partitions, options, concurrent); -======= ReindexMultipleInternal(partitions, options); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* * Clean up working storage --- note we must do this after @@ -3356,11 +3312,7 @@ ReindexPartitions(Oid relid, int options, bool isTopLevel) * and starts a new transaction when finished. */ static void -<<<<<<< HEAD -ReindexMultipleInternal(List *relids, int options, bool concurrent) -======= ReindexMultipleInternal(List *relids, int options) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d { ListCell *l; @@ -3370,16 +3322,11 @@ ReindexMultipleInternal(List *relids, int options) foreach(l, relids) { Oid relid = lfirst_oid(l); -<<<<<<< HEAD Oid heapId = InvalidOid; char relkind; char relpersistence; bool result = false; LOCKMODE lockmode; -======= - char relkind; - char relpersistence; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d StartTransactionCommand(); @@ -3396,8 +3343,7 @@ ReindexMultipleInternal(List *relids, int options) relkind = get_rel_relkind(relid); relpersistence = get_rel_persistence(relid); -<<<<<<< HEAD - lockmode = concurrent ? ShareUpdateExclusiveLock : + lockmode = (options & REINDEXOPT_CONCURRENTLY) != 0 ? ShareUpdateExclusiveLock : (relkind == RELKIND_INDEX ? AccessExclusiveLock : ShareLock); /* * If the relation is index, lock the table first to prevent dead lock. @@ -3424,8 +3370,6 @@ ReindexMultipleInternal(List *relids, int options) CommitTransactionCommand(); continue; } -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* * Partitioned tables and indexes can never be processed directly, and @@ -3434,36 +3378,23 @@ ReindexMultipleInternal(List *relids, int options) Assert(relkind != RELKIND_PARTITIONED_INDEX && relkind != RELKIND_PARTITIONED_TABLE); -<<<<<<< HEAD - if (concurrent && - relpersistence != RELPERSISTENCE_TEMP) - { - result = ReindexRelationConcurrently(relid, options); -======= if ((options & REINDEXOPT_CONCURRENTLY) != 0 && relpersistence != RELPERSISTENCE_TEMP) { - (void) ReindexRelationConcurrently(relid, - options | - REINDEXOPT_MISSING_OK); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + result = ReindexRelationConcurrently(relid, + options | + REINDEXOPT_MISSING_OK); /* ReindexRelationConcurrently() does the verbose output */ } else if (relkind == RELKIND_INDEX) { reindex_index(relid, false, relpersistence, -<<<<<<< HEAD - options); - PopActiveSnapshot(); - /* reindex_index() does the verbose output */ - result = true; -======= options | REINDEXOPT_REPORT_PROGRESS | REINDEXOPT_MISSING_OK); PopActiveSnapshot(); /* reindex_index() does the verbose output */ ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + result = true; } else { @@ -3494,7 +3425,6 @@ ReindexMultipleInternal(List *relids, int options) REINDEX_OBJECT_INDEX : REINDEX_OBJECT_TABLE; stmt->relation = NULL; stmt->options = options; - stmt->concurrent = concurrent; stmt->relid = relid; PushActiveSnapshot(GetTransactionSnapshot()); diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index ff1b41588402..f617054c0716 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -1095,19 +1095,10 @@ standard_ProcessUtility(PlannedStmt *pstmt, switch (stmt->kind) { case REINDEX_OBJECT_INDEX: -<<<<<<< HEAD ReindexIndex(stmt, isTopLevel); break; case REINDEX_OBJECT_TABLE: ReindexTable(stmt, isTopLevel); -======= - ReindexIndex(stmt->relation, stmt->options, - isTopLevel); - break; - case REINDEX_OBJECT_TABLE: - ReindexTable(stmt->relation, stmt->options, - isTopLevel); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d break; case REINDEX_OBJECT_SCHEMA: case REINDEX_OBJECT_SYSTEM: diff --git a/src/include/commands/defrem.h b/src/include/commands/defrem.h index 73a4662da3d4..ab9efde87b24 100644 --- a/src/include/commands/defrem.h +++ b/src/include/commands/defrem.h @@ -35,16 +35,10 @@ extern ObjectAddress DefineIndex(Oid relationId, bool check_rights, bool check_not_in_use, bool skip_build, -<<<<<<< HEAD bool quiet, bool is_new_table); extern void ReindexIndex(ReindexStmt *stmt, bool isTopLevel); extern Oid ReindexTable(ReindexStmt *stmt, bool isTopLevel); -======= - bool quiet); -extern void ReindexIndex(RangeVar *indexRelation, int options, bool isTopLevel); -extern Oid ReindexTable(RangeVar *relation, int options, bool isTopLevel); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d extern void ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, int options); extern char *makeObjectName(const char *name1, const char *name2, diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index aeb9e7833091..7e1315c6f614 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -3886,11 +3886,7 @@ typedef struct ReindexStmt RangeVar *relation; /* Table or index to reindex */ const char *name; /* name of database to reindex */ int options; /* Reindex options flags */ -<<<<<<< HEAD - bool concurrent; /* reindex concurrently? */ Oid relid; /* oid of table or index, used by QE */ -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } ReindexStmt; /* ---------------------- From eb136644cb29c20b867dbcc0ed52aac9b8468f5e Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 28 May 2026 17:24:17 +0500 Subject: [PATCH 486/589] Resolve conflicts in src/backend/optimizer/plan/planner.c (#2576) 1) Commit 1375422 in src/backend/optimizer/plan/planner.c in the standard_planner function removed the assertion for the rootResultRelations field, while earlier commit e180c8a had already added an assertion in the same location. 2) Commit 41efb83 in src/backend/optimizer/plan/planner.c in the subquery_planner function added initialization for the hasPseudoConstantQuals and hasAlternativeSubPlans fields, while earlier commits 6b0e52b and 7efe320 had already added initialization for the upd_del_replicated_table and config fields in the same location. 3) Commit 62e221e in the src/backend/optimizer/plan/planner.c file in the create_one_window_path function added support for incremental sorting, although it is temporarily disabled in the GPDB. --- src/backend/optimizer/plan/planner.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 5e085481b108..282bbd0786f3 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -670,11 +670,7 @@ standard_planner(Query *parse, const char *query_string, int cursorOptions, Assert(glob->finalrtable == NIL); Assert(glob->finalrowmarks == NIL); Assert(glob->resultRelations == NIL); -<<<<<<< HEAD Assert(parse == root->parse); - Assert(glob->rootResultRelations == NIL); -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d Assert(glob->appendRelations == NIL); if (Gp_role == GP_ROLE_DISPATCH) @@ -848,16 +844,13 @@ subquery_planner(PlannerGlobal *glob, Query *parse, root->minmax_aggs = NIL; root->qual_security_level = 0; root->inhTargetKind = INHKIND_NONE; -<<<<<<< HEAD root->upd_del_replicated_table = 0; Assert(config); root->config = config; -======= root->hasPseudoConstantQuals = false; root->hasAlternativeSubPlans = false; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d root->hasRecursion = hasRecursion; if (hasRecursion) root->wt_param_id = assign_special_exec_param(root); @@ -5253,14 +5246,15 @@ create_one_window_path(PlannerInfo *root, { WindowClause *wc = lfirst_node(WindowClause, l); List *window_pathkeys; +#if 0 /* GPDB_14_MERGE_FIXME: enable incremental sort */ int presorted_keys; bool is_sorted; +#endif window_pathkeys = make_pathkeys_for_window(root, wc, root->processed_tlist); -<<<<<<< HEAD /* * Unless the PARTITION BY in the window happens to match the * current distribution, we need a motion. Each partition @@ -5282,7 +5276,7 @@ create_one_window_path(PlannerInfo *root, -1.0, wc->partitionClause, NIL); -======= +#if 0 /* GPDB_14_MERGE_FIXME: enable incremental sort */ is_sorted = pathkeys_count_contained_in(window_pathkeys, path->pathkeys, &presorted_keys); @@ -5313,7 +5307,7 @@ create_one_window_path(PlannerInfo *root, -1.0); } } ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d +#endif if (lnext(activeWindows, l)) { From 4b0a2bbd027e313dafac9a7e59fc2b884dd84e43 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 28 May 2026 17:24:47 +0500 Subject: [PATCH 487/589] Resolve conflicts in src/common/pg_get_line.c (#2607) Commit 67a472d added a new file src/common/pg_get_line.c, while the earlier commit b4c36a1 already did the same thing, but in a different way. --- src/common/pg_get_line.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/common/pg_get_line.c b/src/common/pg_get_line.c index 62baa16c4468..9eb1a33bbb36 100644 --- a/src/common/pg_get_line.c +++ b/src/common/pg_get_line.c @@ -23,8 +23,6 @@ /* -<<<<<<< HEAD -======= * pg_get_line() * * This is meant to be equivalent to fgets(), except that instead of @@ -71,7 +69,6 @@ pg_get_line(FILE *stream) } /* ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d * pg_get_line_buf() * * This has similar behavior to pg_get_line(), and thence to fgets(), @@ -140,8 +137,4 @@ pg_get_line_append(FILE *stream, StringInfo buf) /* No newline at EOF, but we did collect some data */ return true; -<<<<<<< HEAD - } -======= } ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d From c36a4caef19c7ba46429ed97309243fd3d6c898b Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 28 May 2026 17:25:07 +0500 Subject: [PATCH 488/589] Resolve conflicts in src/pl/plpgsql/src/pl_comp.c (#2608) Commit 2453ea1 in the file src/pl/plpgsql/src/pl_comp.c added a new condition in the do_compile function, while the earlier commit 6b0e52b had already refactored the same place. --- src/pl/plpgsql/src/pl_comp.c | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/src/pl/plpgsql/src/pl_comp.c b/src/pl/plpgsql/src/pl_comp.c index 795289d18240..6df8e14629d4 100644 --- a/src/pl/plpgsql/src/pl_comp.c +++ b/src/pl/plpgsql/src/pl_comp.c @@ -456,35 +456,6 @@ do_compile(FunctionCallInfo fcinfo, } /* Remember arguments in appropriate arrays */ -<<<<<<< HEAD - switch (argmode) - { - /* input modes */ - case PROARGMODE_IN: - case PROARGMODE_VARIADIC: - in_arg_varnos[num_in_args++] = argvariable->dno; - break; - - /* output modes */ - case PROARGMODE_OUT: - case PROARGMODE_TABLE: - out_arg_variables[num_out_args++] = argvariable; - break; - - /* both */ - case PROARGMODE_INOUT: - in_arg_varnos[num_in_args++] = argvariable->dno; - out_arg_variables[num_out_args++] = argvariable; - break; - - default: - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("plpgsql functions do not support argmode '%c'", - argmode))); - break; - } -======= if (argmode == PROARGMODE_IN || argmode == PROARGMODE_INOUT || (argmode == PROARGMODE_OUT && function->fn_prokind == PROKIND_PROCEDURE) || @@ -494,7 +465,6 @@ do_compile(FunctionCallInfo fcinfo, argmode == PROARGMODE_INOUT || argmode == PROARGMODE_TABLE) out_arg_variables[num_out_args++] = argvariable; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* Add to namespace under the $n name */ add_parameter_name(argitemtype, argvariable->dno, buf); From 2cb064c6d68bfa8201b2a122de48cffc89e48543 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 28 May 2026 17:25:33 +0500 Subject: [PATCH 489/589] Resolve conflicts in src/port/dirmod.c (#2609) Commit bed9075 in src/port/dirmod.c removed the pgwin32_safestat function, while an earlier commit 6b0e52b had already added an additional conditional return. --- src/port/dirmod.c | 65 ----------------------------------------------- 1 file changed, 65 deletions(-) diff --git a/src/port/dirmod.c b/src/port/dirmod.c index fef9b8ead14e..8979f100803b 100644 --- a/src/port/dirmod.c +++ b/src/port/dirmod.c @@ -353,68 +353,3 @@ pgwin32_is_junction(const char *path) return ((attr & FILE_ATTRIBUTE_REPARSE_POINT) == FILE_ATTRIBUTE_REPARSE_POINT); } #endif /* defined(WIN32) && !defined(__CYGWIN__) */ -<<<<<<< HEAD - - -#if defined(WIN32) && !defined(__CYGWIN__) - -#undef stat - -/* - * The stat() function in win32 is not guaranteed to update the st_size - * field when run. So we define our own version that uses the Win32 API - * to update this field. - */ -int -pgwin32_safestat(const char *path, struct stat *buf) -{ - int r; - WIN32_FILE_ATTRIBUTE_DATA attr; - - r = stat(path, buf); - if (r < 0) - { - if (GetLastError() == ERROR_DELETE_PENDING) - { - /* - * File has been deleted, but is not gone from the filesystem yet. - * This can happen when some process with FILE_SHARE_DELETE has it - * open and it will be fully removed once that handle is closed. - * Meanwhile, we can't open it, so indicate that the file just - * doesn't exist. - */ - errno = ENOENT; - return -1; - } - - return r; - } - - // MPP-24774: just return if path refer to a windows named pipe file. - // no need to get size of a windows named pipe file - if (strlen(path) >2) - { - if (path[0] == '\\' && path[1] == '\\') - { - return r; - } - } - - if (!GetFileAttributesEx(path, GetFileExInfoStandard, &attr)) - { - _dosmaperr(GetLastError()); - return -1; - } - - /* - * XXX no support for large files here, but we don't do that in general on - * Win32 yet. - */ - buf->st_size = attr.nFileSizeLow; - - return 0; -} - -#endif -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d From f28eddf5e18e8d2c69c897a3c7f616dfb586124a Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 28 May 2026 17:52:15 +0500 Subject: [PATCH 490/589] Resolve conflicts in src/backend/optimizer/plan/subselect.c (#2579) Commit d6c08e2 in the file src/backend/optimizer/plan/subselect.c in the make_subplan and subplan_is_hashable functions changed work_mem to hash_mem, and commit 41efb83 in the file src/backend/optimizer/plan/subselect.c in the make_subplan function changed the call to the subplan_is_hashable function to call a new function subpath_is_hashable, and also moved the call to the create_plan function under the condition, while the early commit 6b0e52b had already added the PlannerInfo *root argument and its use to the subplan_is_hashable function, and other early GPDB-specific commits had already added initialization of the curSlice and flow fields. Add a similar PlannerInfo *root argument to the subpath_is_hashable function and move the initialization of the curSlice and flow fields under the condition. --- src/backend/optimizer/plan/subselect.c | 34 +++++++++----------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c index d30ae4e7e49e..461de6bad6ba 100644 --- a/src/backend/optimizer/plan/subselect.c +++ b/src/backend/optimizer/plan/subselect.c @@ -87,12 +87,8 @@ static List *generate_subquery_params(PlannerInfo *root, List *tlist, List **paramIds); static Node *convert_testexpr_mutator(Node *node, convert_testexpr_context *context); -<<<<<<< HEAD static bool subplan_is_hashable(PlannerInfo *root, Plan *plan); -======= -static bool subplan_is_hashable(Plan *plan); -static bool subpath_is_hashable(Path *path); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d +static bool subpath_is_hashable(PlannerInfo *root, Path *path); static bool testexpr_is_hashable(Node *testexpr, List *param_ids); static bool test_opexpr_is_hashable(OpExpr *testexpr, List *param_ids); static bool hash_ok_operator(OpExpr *expr); @@ -477,27 +473,20 @@ make_subplan(PlannerInfo *root, Query *orig_subquery, final_rel = fetch_upper_rel(subroot, UPPERREL_FINAL, NULL); best_path = final_rel->cheapest_total_path; -<<<<<<< HEAD - subroot->curSlice = palloc0(sizeof(PlanSlice)); - subroot->curSlice->gangType = GANGTYPE_UNALLOCATED; - - plan = create_plan(subroot, best_path, subroot->curSlice); - /* Decorate the top node of the plan with a Flow node. */ - plan->flow = cdbpathtoplan_create_flow(subroot, best_path->locus); - - /* Now we can check if it'll fit in hash_mem */ - /* XXX can we check this at the Path stage? */ - if (subplan_is_hashable(root, plan)) -======= /* Now we can check if it'll fit in hash_mem */ - if (subpath_is_hashable(best_path)) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + if (subpath_is_hashable(root, best_path)) { SubPlan *hashplan; AlternativeSubPlan *asplan; + subroot->curSlice = palloc0(sizeof(PlanSlice)); + subroot->curSlice->gangType = GANGTYPE_UNALLOCATED; + /* OK, finish planning the ANY subquery */ - plan = create_plan(subroot, best_path); + plan = create_plan(subroot, best_path, subroot->curSlice); + /* Decorate the top node of the plan with a Flow node. */ + plan->flow = cdbpathtoplan_create_flow(subroot, + best_path->locus); /* ... and convert to SubPlan format */ hashplan = castNode(SubPlan, @@ -984,10 +973,9 @@ subplan_is_hashable(PlannerInfo *root, Plan *plan) * Identical to subplan_is_hashable, but work from a Path for the subplan. */ static bool -subpath_is_hashable(Path *path) +subpath_is_hashable(PlannerInfo *root, Path *path) { double subquery_size; - int hash_mem = get_hash_mem(); /* * The estimated size of the subquery result must fit in hash_mem. (Note: @@ -997,7 +985,7 @@ subpath_is_hashable(Path *path) */ subquery_size = path->rows * (MAXALIGN(path->pathtarget->width) + MAXALIGN(SizeofHeapTupleHeader)); - if (subquery_size > hash_mem * 1024L) + if (subquery_size > global_work_mem(root)) return false; return true; From e7dd9545d4304a64651ef3526a6be7e854a7f7e6 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 28 May 2026 17:52:45 +0500 Subject: [PATCH 491/589] Fix compilation of pg_proc.h, transform.c, and cdbmutate.c (#2616) Commit 8e1f37c changed the generation rule for fmgroids.h macros. Change in GPDB-specific places. --- src/backend/cdb/cdbmutate.c | 4 ++-- src/backend/optimizer/plan/transform.c | 3 ++- src/include/catalog/pg_proc.h | 8 ++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index 0562007cdbc4..d9965d480b62 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -1603,8 +1603,8 @@ pre_dispatch_function_evaluation_mutator(Node *node, * xlog which will also flush any xlog writes that the sequence * server might do. */ - if (funcid == F_NEXTVAL_OID || funcid == F_CURRVAL_OID || - funcid == F_SETVAL_OID) + if (funcid == F_NEXTVAL || funcid == F_CURRVAL || + funcid == F_SETVAL_REGCLASS_INT8) { ExecutorMarkTransactionUsesSequences(); is_seq_func = true; diff --git a/src/backend/optimizer/plan/transform.c b/src/backend/optimizer/plan/transform.c index bedf15703ff7..b97e3163bb71 100644 --- a/src/backend/optimizer/plan/transform.c +++ b/src/backend/optimizer/plan/transform.c @@ -213,7 +213,8 @@ is_sirv_funcexpr(FuncExpr *fe) if (fe->funcresulttype == RECORDOID) return false; /* Record types cannot be handled currently */ - if (fe->funcid == F_NEXTVAL_OID || fe->funcid == F_CURRVAL_OID || fe-> funcid == F_SETVAL_OID) + if (fe->funcid == F_NEXTVAL || fe->funcid == F_CURRVAL || + fe->funcid == F_SETVAL_REGCLASS_INT8) return false; /* Function cannot be sequence related */ return true; diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 9d7cba72639a..24df4e58e046 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -248,10 +248,10 @@ extern bool function_parse_error_transpose(const char *prosrc); extern List *oid_array_to_list(Datum datum); -#define IS_MEDIAN_OID(x) ((x) == MEDIAN_FLOAT8_OID || \ - (x) == MEDIAN_INTERVAL_OID || \ - (x) == MEDIAN_TIMESTAMP_OID || \ - (x) == MEDIAN_TIMESTAMPTZ_OID) +#define IS_MEDIAN_OID(x) ((x) == F_MEDIAN_FLOAT8_FLOAT8 || \ + (x) == F_MEDIAN_FLOAT8_INTERVAL || \ + (x) == F_MEDIAN_FLOAT8_TIMESTAMP || \ + (x) == F_MEDIAN_FLOAT8_TIMESTAMPTZ) #endif /* PG_PROC_H */ From 964f1218ab61326fc4f8928681f20b193c03234b Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 28 May 2026 17:53:25 +0500 Subject: [PATCH 492/589] Resolve conflicts in src/backend/parser/analyze.c (#2580) Commit f893e68 in src/backend/parser/analyze.c in the transformSetOperationTree function removed the local variables lcoltypmod and rcoltypmod and their usage, while earlier GPDB-specific commits had already added GPDB-specific code in the same place. --- src/backend/parser/analyze.c | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/backend/parser/analyze.c b/src/backend/parser/analyze.c index 5ad5b8f1920e..fc44c11c7824 100644 --- a/src/backend/parser/analyze.c +++ b/src/backend/parser/analyze.c @@ -2444,19 +2444,12 @@ coerceSetOpTypes(ParseState *pstate, Node *sop, Node *rcolnode = (Node *) rtle->expr; Oid lcoltype = exprType(lcolnode); Oid rcoltype = exprType(rcolnode); -<<<<<<< HEAD - int32 lcoltypmod = exprTypmod(lcolnode); - int32 rcoltypmod = exprTypmod(rcolnode); Node *bestexpr = NULL; -======= - Node *bestexpr; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d int bestlocation; Oid rescoltype = pct ? lfirst_oid(pct) : InvalidOid; int32 rescoltypmod = pcm ? lfirst_int(pcm) : -1; Oid rescolcoll; -<<<<<<< HEAD /* * If the preprocessed coltype is InvalidOid, we fall back * to the old style type resolution for backward @@ -2470,9 +2463,6 @@ coerceSetOpTypes(ParseState *pstate, Node *sop, context, &bestexpr); bestlocation = exprLocation(bestexpr); - /* if same type and same typmod, use typmod; else default */ - if (lcoltype == rcoltype && lcoltypmod == rcoltypmod) - rescoltypmod = lcoltypmod; } else { @@ -2483,14 +2473,6 @@ coerceSetOpTypes(ParseState *pstate, Node *sop, bestexpr = lcolnode; bestlocation = exprLocation(lcolnode); } -======= - /* select common type, same as CASE et al */ - rescoltype = select_common_type(pstate, - list_make2(lcolnode, rcolnode), - context, - &bestexpr); - bestlocation = exprLocation(bestexpr); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* * Verify the coercions are actually possible. If not, we'd fail From 3658dff2e5b278140e36c12a4d68f3ad5cb37286 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 28 May 2026 17:57:06 +0500 Subject: [PATCH 493/589] Fix compilation of src/backend/optimizer/path/costsize.c (#2613) Commit ad1c36b added a call to the clause_selectivity function in the get_foreign_key_join_selectivity function in src/backend/optimizer/path/costsize.c, although the earlier commit 6b0e52b had already added the bool use_damping argument to this function. --- src/backend/optimizer/path/costsize.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index debb2fdd6602..368ba7b9f1ae 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -5680,7 +5680,8 @@ get_foreign_key_join_selectivity(PlannerInfo *root, (Node *) rinfo, 0, jointype, - sjinfo); + sjinfo, + false /* use_damping */); if (s0 > 0) fkselec /= s0; } From dd5f5b332445f71c1d96cf9723f454c8fdef179d Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 28 May 2026 17:57:25 +0500 Subject: [PATCH 494/589] Fix compilation of src/include/optimizer/cost.h (#2614) Commit 2924b6f in src/include/optimizer/cost.h added a call to the isnan function in the clamp_row_est function, but forgot to include math.h. --- src/include/optimizer/cost.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/include/optimizer/cost.h b/src/include/optimizer/cost.h index a722750f339b..8c295520b6a6 100644 --- a/src/include/optimizer/cost.h +++ b/src/include/optimizer/cost.h @@ -16,6 +16,8 @@ #ifndef COST_H #define COST_H +#include + #include "nodes/pathnodes.h" #include "nodes/plannodes.h" From 56d0aa4e6beabcdec5f1b17283d07a47481289f5 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 28 May 2026 17:57:48 +0500 Subject: [PATCH 495/589] Fix compilation of src/interfaces/libpq/fe-connect.c (#2615) Commit c0cb87f added a call to the pfree function in the parseServiceFile function in src/interfaces/libpq/fe-connect.c, while the earlier commit 7e46c18 replaced the postgres_fe.h include with c.h. Add the missing common/fe_memutils.h include. --- src/interfaces/libpq/fe-connect.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 4a270a832a99..f1b25ff26155 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -22,6 +22,7 @@ * which only defines FRONTEND besides including "c.h" */ #include "c.h" +#include "common/fe_memutils.h" #ifndef WIN32 #include From 95b408e04ae112ec8134aa95f9d625989047313c Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 28 May 2026 20:05:56 +0500 Subject: [PATCH 496/589] Resolve conflicts in src/backend/optimizer/plan/setrefs.c (#2577) 1) Commit 41efb83 in src/backend/optimizer/plan/setrefs.c added a new field, num_exec, to the fix_join_expr_context structure. However, the earlier commit 6b0e52b had already added the fields, use_outer_tlist_for_matching_nonvars and use_inner_tlist_for_matching_nonvars, to the same location. 2) Commit 41efb83 in src/backend/optimizer/plan/setrefs.c added the num_exec argument to the initialization of the startOffset and endOffset fields in the fix_scan_expr function call. However, earlier GPDB-specific commits had already added GPDB-specific code to the same location. 3) Commit 41efb83 in src/backend/optimizer/plan/setrefs.c added the num_exec argument to the initialization of the indexqual field in the set_indexonlyscan_references function when calling the fix_scan_list macro. However, commit 598f4b0 had already added the initialization of the indexqualorig field to the same location. 4) Commit 41efb83 in src/backend/optimizer/plan/setrefs.c added the initialization of the num_exec field to the fix_join_expr function. However, commit 6b0e52b had already added the initialization of the use_outer_tlist_for_matching_nonvars and use_inner_tlist_for_matching_nonvars fields to the same location. 5) Commit 41efb83 added a new num_exec argument to the fix_scan_list macro in src/backend/optimizer/plan/setrefs.c. Add this in GPDB-specific locations. 6) Commit 41efb83 added a new double num_exec argument to the fix_upper_expr and fix_join_expr functions in src/backend/optimizer/plan/setrefs.c. Add this in GPDB-specific locations. --- src/backend/optimizer/plan/setrefs.c | 70 ++++++++++++++-------------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index 4ab0e8e3f83b..66eea459f10c 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -69,12 +69,9 @@ typedef struct indexed_tlist *inner_itlist; Index acceptable_rel; int rtoffset; -<<<<<<< HEAD bool use_outer_tlist_for_matching_nonvars; bool use_inner_tlist_for_matching_nonvars; -======= double num_exec; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } fix_join_expr_context; typedef struct @@ -187,13 +184,14 @@ static List *fix_hashclauses(PlannerInfo *root, List *clauses, indexed_tlist *outer_itlist, indexed_tlist *inner_itlist, - Index acceptable_rel, int rtoffset); + Index acceptable_rel, int rtoffset, + double num_exec); static List *fix_child_hashclauses(PlannerInfo *root, List *clauses, indexed_tlist *outer_itlist, indexed_tlist *inner_itlist, Index acceptable_rel, int rtoffset, - Index child); + Index child, double num_exec); static Node *fix_upper_expr(PlannerInfo *root, Node *node, indexed_tlist *subplan_itlist, @@ -824,11 +822,13 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) /* adjust for the new range table offset */ tplan->scan.scanrelid += rtoffset; tplan->scan.plan.targetlist = - fix_scan_list(root, tplan->scan.plan.targetlist, rtoffset); + fix_scan_list(root, tplan->scan.plan.targetlist, + rtoffset, NUM_EXEC_TLIST(plan)); tplan->scan.plan.qual = - fix_scan_list(root, tplan->scan.plan.qual, rtoffset); + fix_scan_list(root, tplan->scan.plan.qual, + rtoffset, NUM_EXEC_QUAL(plan)); tplan->function = (RangeTblFunction *) - fix_scan_expr(root, (Node *) tplan->function, rtoffset); + fix_scan_expr(root, (Node *) tplan->function, rtoffset, 1); return plan; } @@ -1017,10 +1017,11 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) pinfo->initial_pruning_steps = (List *) fix_upper_expr(root, (Node *) pinfo->initial_pruning_steps, - childplan_itlist, OUTER_VAR, rtoffset); + childplan_itlist, OUTER_VAR, rtoffset, 1); pinfo->exec_pruning_steps = (List *) fix_upper_expr(root, (Node *) pinfo->exec_pruning_steps, - childplan_itlist, OUTER_VAR, rtoffset); + childplan_itlist, OUTER_VAR, rtoffset, + NUM_EXEC_TLIST(plan)); } } } @@ -1114,7 +1115,8 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) (Node *)dqaExpr->agg_filter, subplan_itlist, OUTER_VAR, - rtoffset); + rtoffset, + NUM_EXEC_TLIST(plan)); lfirst(lc) = dqaExpr; } @@ -1137,7 +1139,6 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) * in GPDB, we allow the ROWS/RANGE expressions to contain * references to the subplan, so we have to use fix_upper_expr. */ -<<<<<<< HEAD if (wplan->startOffset || wplan->endOffset) { subplan_itlist = @@ -1145,18 +1146,12 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) wplan->startOffset = fix_upper_expr(root, wplan->startOffset, - subplan_itlist, OUTER_VAR, rtoffset); + subplan_itlist, OUTER_VAR, rtoffset, 1); wplan->endOffset = fix_upper_expr(root, wplan->endOffset, - subplan_itlist, OUTER_VAR, rtoffset); + subplan_itlist, OUTER_VAR, rtoffset, 1); pfree(subplan_itlist); } -======= - wplan->startOffset = - fix_scan_expr(root, wplan->startOffset, rtoffset, 1); - wplan->endOffset = - fix_scan_expr(root, wplan->endOffset, rtoffset, 1); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } break; case T_Result: @@ -1360,7 +1355,9 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) build_tlist_index(plan->lefttree->targetlist); motion->hashExprs = (List *) - fix_upper_expr(root, (Node*) motion->hashExprs, childplan_itlist, OUTER_VAR, rtoffset); + fix_upper_expr(root, (Node*) motion->hashExprs, + childplan_itlist, OUTER_VAR, rtoffset, + NUM_EXEC_TLIST(plan)); /* no need to fix targetlist and qual */ Assert(plan->qual == NIL); @@ -1425,15 +1422,12 @@ set_indexonlyscan_references(PlannerInfo *root, INDEX_VAR, rtoffset, NUM_EXEC_QUAL((Plan *) plan)); - /* indexqual is already transformed to reference index columns */ -<<<<<<< HEAD - plan->indexqual = fix_scan_list(root, plan->indexqual, rtoffset); /* indexqualorig is already transformed to reference index columns */ - plan->indexqualorig = fix_scan_list(root, plan->indexqualorig, rtoffset); -======= + plan->indexqualorig = fix_scan_list(root, plan->indexqualorig, + rtoffset, 1); + /* indexqual is already transformed to reference index columns */ plan->indexqual = fix_scan_list(root, plan->indexqual, rtoffset, 1); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* indexorderby is already transformed to reference index columns */ plan->indexorderby = fix_scan_list(root, plan->indexorderby, rtoffset, 1); @@ -2336,7 +2330,8 @@ set_join_references(PlannerInfo *root, Join *join, int rtoffset) outer_itlist, inner_itlist, (Index) 0, - rtoffset); + rtoffset, + NUM_EXEC_QUAL((Plan *) join)); /* * HashJoin's hashkeys are used to look for matching tuples from its * outer plan (not the Hash node!) in the hashtable. @@ -3047,13 +3042,10 @@ fix_join_expr(PlannerInfo *root, context.inner_itlist = inner_itlist; context.acceptable_rel = acceptable_rel; context.rtoffset = rtoffset; -<<<<<<< HEAD context.use_outer_tlist_for_matching_nonvars = true; context.use_inner_tlist_for_matching_nonvars = true; - -======= context.num_exec = num_exec; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + return (List *) fix_join_expr_mutator((Node *) clauses, &context); } @@ -3068,7 +3060,9 @@ static List *fix_hashclauses(PlannerInfo *root, List *clauses, indexed_tlist *outer_itlist, indexed_tlist *inner_itlist, - Index acceptable_rel, int rtoffset) + Index acceptable_rel, + int rtoffset, + double num_exec) { Assert(clauses); ListCell *lc = NULL; @@ -3093,7 +3087,8 @@ static List *fix_hashclauses(PlannerInfo *root, inner_itlist, (Index) 0, rtoffset, - OUTER_VAR); + OUTER_VAR, + num_exec); /* * for inner argument, we cannot refer to target entries * in join's outer child target list, otherwise hash table @@ -3106,7 +3101,8 @@ static List *fix_hashclauses(PlannerInfo *root, inner_itlist, (Index) 0, rtoffset, - INNER_VAR); + INNER_VAR, + num_exec); new_args = lappend(new_args, new_outer_arg); new_args = lappend(new_args, new_inner_arg); /* replace old arguments with the fixed arguments */ @@ -3139,7 +3135,8 @@ fix_child_hashclauses(PlannerInfo *root, indexed_tlist *inner_itlist, Index acceptable_rel, int rtoffset, - Index child) + Index child, + double num_exec) { fix_join_expr_context context; context.root = root; @@ -3147,6 +3144,7 @@ fix_child_hashclauses(PlannerInfo *root, context.inner_itlist = inner_itlist; context.acceptable_rel = acceptable_rel; context.rtoffset = rtoffset; + context.num_exec = num_exec; if (INNER_VAR == child) { /* skips using outer target list when matching non-vars */ From b44df3d018c2a08cf58d7f02c823191c646c51d2 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 29 May 2026 08:20:01 +0500 Subject: [PATCH 497/589] Resolve conflicts in src/bin/psql/describe.c (#2606) 1) Commit 07f386e in the file src/bin/psql/describe.c added an element to the translate_columns array in the listTables function, while the earlier commit 1e11aaf already added two elements and a comment to the same array. 2) Commit 07f386e in the file src/bin/psql/describe.c added a conditional join pg_catalog.pg_am to the listTables function, while the earlier commit 4b98e91 already did this, but for different conditions. --- src/bin/psql/describe.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index cb7ecddb0a49..5e7e99cb1c7b 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -4650,11 +4650,7 @@ listTables(const char *tabtypes, const char *pattern, bool verbose, bool showSys PGresult *res; printQueryOpt myopt = pset.popt; int cols_so_far; -<<<<<<< HEAD - bool translate_columns[] = {false, false, true, false, false /* Storage */, false, false, false, false, false}; -======= - bool translate_columns[] = {false, false, true, false, false, false, false, false, false}; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + bool translate_columns[] = {false, false, true, false, false /* Storage */, false, false, false, false, false, false}; /* If tabtypes is empty, we default to \dtvmsE (but see also command.c) */ if (!(showTables || showIndexes || showViews || showMatViews || showSeq || showForeign)) @@ -4710,7 +4706,7 @@ listTables(const char *tabtypes, const char *pattern, bool verbose, bool showSys if (isGPDB7000OrLater()) { /* In GPDB7, we can have user defined access method, display the access method name directly */ - appendPQExpBuffer(&buf, ", a.amname as \"%s\"\n", gettext_noop("Storage")); + appendPQExpBuffer(&buf, ", am.amname as \"%s\"\n", gettext_noop("Storage")); } else { @@ -4788,18 +4784,15 @@ listTables(const char *tabtypes, const char *pattern, bool verbose, bool showSys appendPQExpBufferStr(&buf, "\nFROM pg_catalog.pg_class c" "\n LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace"); -<<<<<<< HEAD if (showTables && isGPDB7000OrLater()) appendPQExpBufferStr(&buf, - "\n LEFT JOIN pg_catalog.pg_am a ON a.oid = c.relam"); -======= - + "\n LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam"); + else if (pset.sversion >= 120000 && !pset.hide_tableam && (showTables || showMatViews || showIndexes)) appendPQExpBufferStr(&buf, "\n LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam"); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d if (showIndexes) appendPQExpBufferStr(&buf, "\n LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid" From 8f798870426d598fa4f5d6c6f5127afcdeff604a Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 29 May 2026 08:20:47 +0500 Subject: [PATCH 498/589] Resolve conflicts in src/backend/executor/nodeModifyTable.c (#2573) 1) Commit a04daa9 in src/backend/executor/nodeModifyTable.c changed the comment before the ExecInsert function, but earlier GPDB-specific commits had already added a GPDB-specific comment in the same location. 2) Commit a04daa9 in src/backend/executor/nodeModifyTable.c in the ExecDelete function deleted a couple of lines, but earlier GPDB-specific commits had already added a conditional error message before this location. 3) Commit c5b097f in src/backend/executor/nodeModifyTable.c moved some code to the new ExecCrossPartitionUpdate function in the ExecUpdate function. However, the earlier commit 19cd1cf had already added the segid and splitUpdate arguments to the ExecDelete and ExecInsert functions in these same locations. 4) Commit a04daa9 in src/backend/executor/nodeModifyTable.c removed the es_result_relation_info save in the ExecModifyTable function before calling the EvalPlanQualSetPlan function. However, earlier GPDB-specific commits had already added code before this location. 5) Commit a04daa9 in src/backend/executor/nodeModifyTable.c added a new resultRelInfo argument to the ExecModifyTable function when calling ExecInsert ExecDelete. However, the earlier commit 19cd1cf had already added the splitUpdate argument to the ExecInsert function. 6) Commit a04daa9 in src/backend/executor/nodeModifyTable.c added a new resultRelInfo argument to the ExecModifyTable function when calling ExecUpdate. However, earlier GPDB-specific commits had already added the code for split updates. 7) Commit a04daa9 in file src/backend/executor/nodeModifyTable.c in function ExecModifyTable when calling function ExecDelete added new argument resultRelInfo, while earlier commit 19cd1cf already added arguments segid and splitUpdate, and commit 973cf25 already added conditional call of function ExecPrepareTupleRouting before and conditional saving of resultRelInfo after this place. --- src/backend/executor/nodeModifyTable.c | 229 ++++++------------------- 1 file changed, 48 insertions(+), 181 deletions(-) diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index 8518ea3415f4..b99997d9ccef 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -384,7 +384,10 @@ ExecComputeStoredGenerated(ResultRelInfo *resultRelInfo, * * Returns RETURNING result if any, otherwise NULL. * -<<<<<<< HEAD + * This may change the currently active tuple conversion map in + * mtstate->mt_transition_capture, so the callers must take care to + * save the previous value to avoid losing track of it. + * * If the target table is partitioned, the input tuple in 'parentslot' * is in the shape required for the parent table. This function will * look up the ResultRelInfo of the target partition, and form a @@ -396,11 +399,6 @@ ExecComputeStoredGenerated(ResultRelInfo *resultRelInfo, * there is a preceding SplitUpdate node. 'splitUpdate' is true in * that case. * -======= - * This may change the currently active tuple conversion map in - * mtstate->mt_transition_capture, so the callers must take care to - * save the previous value to avoid losing track of it. ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d * ---------------------------------------------------------------- */ static TupleTableSlot * @@ -793,7 +791,6 @@ ExecDelete(ModifyTableState *mtstate, if (tupleDeleted) *tupleDeleted = false; -<<<<<<< HEAD /* * Sanity check the distribution of the tuple to prevent * potential data corruption in case users manipulate data @@ -808,14 +805,6 @@ ExecDelete(ModifyTableState *mtstate, tupleid->ip_posid, segid); - /* - * get information on the (current) result relation - */ - resultRelInfo = estate->es_result_relation_info; - resultRelationDesc = resultRelInfo->ri_RelationDesc; - -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* BEFORE ROW DELETE Triggers */ /* * Disallow DELETE triggers on a split UPDATE. See comments in ExecInsert(). @@ -1226,11 +1215,12 @@ ExecCrossPartitionUpdate(ModifyTableState *mtstate, * Row movement, part 1. Delete the tuple, but skip RETURNING processing. * We want to return rows from INSERT. */ - ExecDelete(mtstate, resultRelInfo, tupleid, oldtuple, planSlot, + ExecDelete(mtstate, resultRelInfo, tupleid, segid, oldtuple, planSlot, epqstate, estate, false, /* processReturning */ false, /* canSetTag */ true, /* changingPart */ + false, /* splitUpdate */ &tuple_deleted, &epqslot); /* @@ -1282,7 +1272,8 @@ ExecCrossPartitionUpdate(ModifyTableState *mtstate, /* Tuple routing starts from the root table. */ *inserted_tuple = ExecInsert(mtstate, mtstate->rootResultRelInfo, slot, - planSlot, estate, canSetTag); + planSlot, estate, canSetTag, + false /* splitUpdate */); /* * Reset the transition state that may possibly have been written by @@ -1474,114 +1465,17 @@ lreplace:; * The first part may have to be repeated if it is detected that * the tuple we're trying to move has been concurrently updated. */ -<<<<<<< HEAD - if (((ModifyTable *) mtstate->ps.plan)->onConflictAction == ONCONFLICT_UPDATE) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("invalid ON UPDATE specification"), - errdetail("The result tuple would appear in a different partition than the original tuple."))); - - /* - * When an UPDATE is run on a leaf partition, we will not have - * partition tuple routing set up. In that case, fail with - * partition constraint violation error. - */ - if (proute == NULL) - ExecPartitionCheckEmitError(resultRelInfo, slot, estate); - - /* - * Row movement, part 1. Delete the tuple, but skip RETURNING - * processing. We want to return rows from INSERT. - */ - ExecDelete(mtstate, tupleid, segid, oldtuple, planSlot, epqstate, - estate, false, false /* canSetTag */ , - true /* changingPart */ , - false /* splitUpdate */ , - &tuple_deleted, &epqslot); - - /* - * For some reason if DELETE didn't happen (e.g. trigger prevented - * it, or it was already deleted by self, or it was concurrently - * deleted by another transaction), then we should skip the insert - * as well; otherwise, an UPDATE could cause an increase in the - * total number of rows across all partitions, which is clearly - * wrong. - * - * For a normal UPDATE, the case where the tuple has been the - * subject of a concurrent UPDATE or DELETE would be handled by - * the EvalPlanQual machinery, but for an UPDATE that we've - * translated into a DELETE from this partition and an INSERT into - * some other partition, that's not available, because CTID chains - * can't span relation boundaries. We mimic the semantics to a - * limited extent by skipping the INSERT if the DELETE fails to - * find a tuple. This ensures that two concurrent attempts to - * UPDATE the same tuple at the same time can't turn one tuple - * into two, and that an UPDATE of a just-deleted tuple can't - * resurrect it. - */ - if (!tuple_deleted) -======= retry = !ExecCrossPartitionUpdate(mtstate, resultRelInfo, tupleid, oldtuple, slot, planSlot, epqstate, canSetTag, &retry_slot, &inserted_tuple); if (retry) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d { slot = retry_slot; goto lreplace; } -<<<<<<< HEAD - /* - * Updates set the transition capture map only when a new subplan - * is chosen. But for inserts, it is set for each row. So after - * INSERT, we need to revert back to the map created for UPDATE; - * otherwise the next UPDATE will incorrectly use the one created - * for INSERT. So first save the one created for UPDATE. - */ - if (mtstate->mt_transition_capture) - saved_tcs_map = mtstate->mt_transition_capture->tcs_map; - - /* - * resultRelInfo is one of the per-subplan resultRelInfos. So we - * should convert the tuple into root's tuple descriptor, since - * ExecInsert() starts the search from root. The tuple conversion - * map list is in the order of mtstate->resultRelInfo[], so to - * retrieve the one for this resultRel, we need to know the - * position of the resultRel in mtstate->resultRelInfo[]. - */ - map_index = resultRelInfo - mtstate->resultRelInfo; - Assert(map_index >= 0 && map_index < mtstate->mt_nplans); - tupconv_map = tupconv_map_for_subplan(mtstate, map_index); - if (tupconv_map != NULL) - slot = execute_attr_map_slot(tupconv_map->attrMap, - slot, - mtstate->mt_root_tuple_slot); - - /* - * Prepare for tuple routing, making it look like we're inserting - * into the root. - */ - Assert(mtstate->rootResultRelInfo != NULL); - slot = ExecPrepareTupleRouting(mtstate, estate, proute, - mtstate->rootResultRelInfo, slot); - - ret_slot = ExecInsert(mtstate, slot, planSlot, - estate, canSetTag, false /* splitUpdate */); - - /* Revert ExecPrepareTupleRouting's node change. */ - estate->es_result_relation_info = resultRelInfo; - if (mtstate->mt_transition_capture) - { - mtstate->mt_transition_capture->tcs_original_insert_tuple = NULL; - mtstate->mt_transition_capture->tcs_map = saved_tcs_map; - } - - return ret_slot; -======= return inserted_tuple; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } /* @@ -1790,12 +1684,12 @@ lreplace:; */ static TupleTableSlot * ExecSplitUpdate_Insert(ModifyTableState *mtstate, + ResultRelInfo *resultRelInfo, TupleTableSlot *slot, TupleTableSlot *planSlot, EState *estate, bool canSetTag) { - ResultRelInfo *resultRelInfo; Relation resultRelationDesc; bool partition_constraint_failed; TupleConversionMap *saved_tcs_map = NULL; @@ -1806,7 +1700,6 @@ ExecSplitUpdate_Insert(ModifyTableState *mtstate, /* * get information on the (current) result relation */ - resultRelInfo = estate->es_result_relation_info; resultRelationDesc = resultRelInfo->ri_RelationDesc; /* ensure slot is independent, consider e.g. EPQ */ @@ -1875,15 +1768,17 @@ ExecSplitUpdate_Insert(ModifyTableState *mtstate, * into the root. */ Assert(mtstate->rootResultRelInfo != NULL); + ResultRelInfo *partRelInfo; slot = ExecPrepareTupleRouting(mtstate, estate, proute, - mtstate->rootResultRelInfo, slot); + mtstate->rootResultRelInfo, slot, + &partRelInfo); + resultRelInfo = partRelInfo; - slot = ExecInsert(mtstate, slot, planSlot, + slot = ExecInsert(mtstate, resultRelInfo, slot, planSlot, estate, mtstate->canSetTag, true /* splitUpdate */); /* Revert ExecPrepareTupleRouting's node change. */ - estate->es_result_relation_info = resultRelInfo; if (mtstate->mt_transition_capture) { mtstate->mt_transition_capture->tcs_original_insert_tuple = NULL; @@ -1892,7 +1787,7 @@ ExecSplitUpdate_Insert(ModifyTableState *mtstate, } else { - slot = ExecInsert(mtstate, slot, planSlot, + slot = ExecInsert(mtstate, resultRelInfo, slot, planSlot, estate, mtstate->canSetTag, true /* splitUpdate */); } @@ -2381,16 +2276,11 @@ ExecModifyTable(PlanState *pstate) node->mt_whichplan++; if (node->mt_whichplan < node->mt_nplans) { - estate->es_result_relation_info = estate->es_result_relations + node->mt_whichplan; - resultRelInfo = estate->es_result_relation_info; + resultRelInfo = estate->es_result_relations + node->mt_whichplan; subplanstate = node->mt_plans[node->mt_whichplan]; -<<<<<<< HEAD - junkfilter = estate->es_result_relation_info->ri_junkFilter; - action_attno = estate->es_result_relation_info->ri_action_attno; - segid_attno = estate->es_result_relation_info->ri_segid_attno; -======= junkfilter = resultRelInfo->ri_junkFilter; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + action_attno = resultRelInfo->ri_action_attno; + segid_attno = resultRelInfo->ri_segid_attno; EvalPlanQualSetPlan(&node->mt_epqstate, subplanstate->plan, node->mt_arowmarks[node->mt_whichplan]); continue; @@ -2536,92 +2426,69 @@ ExecModifyTable(PlanState *pstate) switch (operation) { case CMD_INSERT: -<<<<<<< HEAD - /* Prepare for tuple routing if needed. */ - if (proute) - slot = ExecPrepareTupleRouting(node, estate, proute, - resultRelInfo, slot); - slot = ExecInsert(node, slot, planSlot, + slot = ExecInsert(node, resultRelInfo, slot, planSlot, estate, node->canSetTag, false /* splitUpdate */); - /* Revert ExecPrepareTupleRouting's state change. */ - if (proute) - estate->es_result_relation_info = resultRelInfo; break; case CMD_UPDATE: - /* Prepare for tuple routing if needed. */ if (castNode(ModifyTable, node->ps.plan)->forceTupleRouting) + { + PartitionTupleRouting *proute = node->mt_partition_tuple_routing; + ResultRelInfo *partRelInfo; + slot = ExecPrepareTupleRouting(node, estate, proute, - resultRelInfo, slot); + resultRelInfo, slot, + &partRelInfo); + resultRelInfo = partRelInfo; + } if (!AttributeNumberIsValid(action_attno)) { /* normal non-split UPDATE */ - slot = ExecUpdate(node, tupleid, oldtuple, slot, planSlot, - segid, - &node->mt_epqstate, estate, node->canSetTag); + slot = ExecUpdate(node, resultRelInfo, tupleid, oldtuple, + slot, planSlot, segid, + &node->mt_epqstate, estate, + node->canSetTag); } else if (DML_INSERT == action) { - slot = ExecSplitUpdate_Insert(node, slot, planSlot, + slot = ExecSplitUpdate_Insert(node, resultRelInfo, slot, planSlot, estate, node->canSetTag); } else /* DML_DELETE */ { - slot = ExecDelete(node, tupleid, segid, oldtuple, planSlot, + slot = ExecDelete(node, resultRelInfo, tupleid, segid, + oldtuple, planSlot, &node->mt_epqstate, estate, - false, - false /* canSetTag */, - true /* changingPart */ , - true /* splitUpdate */ , + false, /* processReturning */ + false, /* canSetTag */ + true, /* changingPart */ + true, /* splitUpdate */ NULL, NULL); } - /* Revert ExecPrepareTupleRouting's state change. */ - if (castNode(ModifyTable, node->ps.plan)->forceTupleRouting) - estate->es_result_relation_info = resultRelInfo; break; case CMD_DELETE: if (castNode(ModifyTable, node->ps.plan)->forceTupleRouting) + { + PartitionTupleRouting *proute = node->mt_partition_tuple_routing; + ResultRelInfo *partRelInfo; + planSlot = ExecPrepareTupleRouting(node, estate, proute, - resultRelInfo, slot); - slot = ExecDelete(node, tupleid, segid, oldtuple, planSlot, - &node->mt_epqstate, estate, - true, node->canSetTag, - false /* changingPart */ , - false /* splitUpdate */ , - NULL, NULL); - if (castNode(ModifyTable, node->ps.plan)->forceTupleRouting) - estate->es_result_relation_info = resultRelInfo; -======= - slot = ExecInsert(node, resultRelInfo, slot, planSlot, - estate, node->canSetTag); - break; - case CMD_UPDATE: - slot = ExecUpdate(node, resultRelInfo, tupleid, oldtuple, slot, - planSlot, &node->mt_epqstate, estate, - node->canSetTag); - break; - case CMD_DELETE: - slot = ExecDelete(node, resultRelInfo, tupleid, oldtuple, + resultRelInfo, slot, + &partRelInfo); + resultRelInfo = partRelInfo; + } + slot = ExecDelete(node, resultRelInfo, tupleid, segid, oldtuple, planSlot, &node->mt_epqstate, estate, true, /* processReturning */ node->canSetTag, false, /* changingPart */ + false, /* splitUpdate */ NULL, NULL); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d break; default: elog(ERROR, "unknown operation"); break; } - /* - * If the target is a partitioned table, ExecInsert / ExecUpdate / - * ExecDelete might have changed es_result_relation_info to point to - * a partition, instead of the top-level table. Reset it. (It would - * be more tidy if those functions cleaned up after themselves, but - * it's more robust to do it here just once.) - */ - estate->es_result_relation_info = resultRelInfo; - /* * If we got a RETURNING result, return it to caller. We'll continue * the work on next call. From 7b89d86c25429ce304e9745ae4c48f69367434a5 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 29 May 2026 08:21:49 +0500 Subject: [PATCH 499/589] Resolve conflicts in src/backend/parser/parse_utilcmd.c (#2582) 1) Commit 5028981 in src/backend/parser/parse_utilcmd.c removed the inh_indexes field in the CreateStmtContext structure, while earlier commit 391b78e had already added the attr_encodings field in the same location. 2) Commit e7c2b95 in src/backend/parser/parse_utilcmd.c added a new local variable, nameEl_idx, in the generateSerialExtraStmts function, while earlier commit 19cd1cf had added the local variable, has_cache_option, in the same location. 3) Commit 5028981 in src/backend/parser/parse_utilcmd.c changed the comment before the transformTableLikeClause function, while earlier commit 598f4b0 had already added a GPDB-specific comment at this location. 4) Commit 5028981 in src/backend/parser/parse_utilcmd.c moved some code in the transformTableLikeClause function to the new expandTableLikeClause function, while earlier GPDB-specific commits had already added handling for the CREATE_TABLE_LIKE_STORAGE option. A further modification to the CREATE_TABLE_LIKE_STORAGE option will be made in commit 2e7e4d6. 5) Commit 5028981 in src/backend/parser/parse_utilcmd.c in the transformAlterTableStmt function removed the initialization of the inh_indexes field, while the earlier commit 391b78e had already added the initialization of the attr_encodings field in the same place. --- src/backend/parser/parse_utilcmd.c | 68 +----------------------------- 1 file changed, 2 insertions(+), 66 deletions(-) diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index 89f3a22c2f0e..6a43c017eda8 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -97,11 +97,7 @@ typedef struct List *ckconstraints; /* CHECK constraints */ List *fkconstraints; /* FOREIGN KEY constraints */ List *ixconstraints; /* index-creating constraints */ -<<<<<<< HEAD List *attr_encodings; /* List of ColumnReferenceStorageDirectives */ - List *inh_indexes; /* cloned indexes from INCLUDING INDEXES */ -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d List *extstats; /* cloned extended statistics */ List *blist; /* "before list" of things to do before * creating the table */ @@ -451,11 +447,8 @@ generateSerialExtraStmts(CreateStmtContext *cxt, ColumnDef *column, CreateSeqStmt *seqstmt; AlterSeqStmt *altseqstmt; List *attnamelist; -<<<<<<< HEAD bool has_cache_option = false; -======= int nameEl_idx = -1; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* * Determine namespace and name to use for the sequence. @@ -1027,18 +1020,13 @@ transformTableConstraint(CreateStmtContext *cxt, Constraint *constraint) * transformTableLikeClause * * Change the LIKE portion of a CREATE TABLE statement into -<<<<<<< HEAD - * column definitions which recreate the user defined column portions of - * . - * - * GPDB: if forceBareCol is true we disallow inheriting any indexes/constr/defaults. -======= * column definitions that recreate the user defined column portions of * . Also, if there are any LIKE options that we can't fully * process at this point, add the TableLikeClause to cxt->alist, which * will cause utility.c to call expandTableLikeClause() after the new * table has been created. ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + * + * GPDB: if forceBareCol is true we disallow inheriting any indexes/constr/defaults. */ static void transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_clause, @@ -1218,51 +1206,6 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla CREATE_TABLE_LIKE_INDEXES)) cxt->alist = lappend(cxt->alist, table_like_clause); - /* -<<<<<<< HEAD - * Likewise, copy indexes if requested - */ - if ((table_like_clause->options & CREATE_TABLE_LIKE_INDEXES) && - relation->rd_rel->relhasindex) - { - List *parent_indexes; - ListCell *l; - - parent_indexes = RelationGetIndexList(relation); - - foreach(l, parent_indexes) - { - Oid parent_index_oid = lfirst_oid(l); - Relation parent_index; - IndexStmt *index_stmt; - - parent_index = index_open(parent_index_oid, AccessShareLock); - - /* Build CREATE INDEX statement to recreate the parent_index */ - index_stmt = generateClonedIndexStmt(cxt->relation, - parent_index, - attmap, - NULL); - - /* Copy comment on index, if requested */ - if (table_like_clause->options & CREATE_TABLE_LIKE_COMMENTS) - { - comment = GetComment(parent_index_oid, RelationRelationId, 0); - - /* - * We make use of IndexStmt's idxcomment option, so as not to - * need to know now what name the index will have. - */ - index_stmt->idxcomment = comment; - } - - /* Save it in the inh_indexes list for the time being */ - cxt->inh_indexes = lappend(cxt->inh_indexes, index_stmt); - - index_close(parent_index, AccessShareLock); - } - } - /* * GPDB_12_MERGE_FIXME: * This is wrong and creates unspecified behaviour when multiple like @@ -1317,11 +1260,8 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla } /* - * Likewise, copy extended statistics if requested -======= * We may copy extended statistics if requested, since the representation * of CreateStatsStmt doesn't depend on column numbers. ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d */ if (table_like_clause->options & CREATE_TABLE_LIKE_STATISTICS) { @@ -4366,11 +4306,7 @@ transformAlterTableStmt(Oid relid, AlterTableStmt *stmt, cxt.ckconstraints = NIL; cxt.fkconstraints = NIL; cxt.ixconstraints = NIL; -<<<<<<< HEAD - cxt.inh_indexes = NIL; cxt.attr_encodings = NIL; -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d cxt.extstats = NIL; cxt.blist = NIL; cxt.alist = NIL; From 0fad93379fbf02ce99b4099a8c0b5a35b189e94c Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 29 May 2026 13:21:11 +0500 Subject: [PATCH 500/589] Resolve conflicts in src/backend/parser/gram.y (#2581) 1) Commit 06a7c31 in the file src/backend/parser/gram.y replaced var_name type_function_name param_name with BareColLabel, while the earlier commit 9b88bed had already added ColLabelNoAs in the same place. 2) Commit 06a7c31 in the file src/backend/parser/gram.y added a new definition of bare_label_keyword, while the earlier commit 6b0e52b already added the definition of keywords_ok_in_alias_no_as in the same place. 3) Commit 28a61fc in the file src/backend/parser/gram.y replaced the comment in the UNBOUNDED definition and removed GENERATED NULL_P from the IDENT definition, while the earlier commit 6b0e52b had already added many GPDB-specific definitions below in the same place. 4) Commit 844c05a in the file src/backend/parser/gram.y replaced the initialization of the concurrent field with the conditional setting of the REINDEXOPT_CONCURRENTLY option, while the early commit 19cd1cf already began to generate an error about the non-support of this functionality in the GPDB. 5) Commit 06a7c31 in the file src/backend/parser/gram.y replaced a_expr IDENT with a_expr BareColLabel, while the earlier commit 6b0e52b had already added a large comment before this place. 6) Remove GPDB-specific ColLabelNoAs and keywords_ok_in_alias_no_as. --- src/backend/parser/gram.y | 123 +++++++++++++++++--------------------- 1 file changed, 54 insertions(+), 69 deletions(-) diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 0b67918c0c87..08f0f59a473a 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -601,13 +601,9 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ %type RoleId opt_boolean_or_string %type QueueId %type var_list -<<<<<<< HEAD -%type ColId ColLabel ColLabelNoAs var_name type_function_name param_name +%type ColId ColLabel BareColLabel %type PartitionIdentKeyword %type PartitionColId -======= -%type ColId ColLabel BareColLabel ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d %type NonReservedWord NonReservedWord_or_Sconst %type var_name type_function_name param_name %type createdb_opt_name @@ -616,11 +612,7 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ %type unreserved_keyword type_func_name_keyword %type col_name_keyword reserved_keyword -<<<<<<< HEAD -%type keywords_ok_in_alias_no_as -======= %type bare_label_keyword ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d %type TableConstraint TableLikeClause %type TableLikeOptionList TableLikeOption @@ -917,9 +909,8 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ * rather than reducing a conflicting rule that takes CUBE as a function name. * Using the same precedence as IDENT seems right for the reasons given above. */ -<<<<<<< HEAD -%nonassoc UNBOUNDED /* ideally should have same precedence as IDENT */ -%nonassoc IDENT GENERATED NULL_P PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP +%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */ +%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP /* * This is a bit ugly... To allow these to be column aliases without @@ -1227,10 +1218,6 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ %nonassoc UNKNOWN %nonassoc ZONE -======= -%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */ -%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d %left Op OPERATOR /* multi-character ops and user-defined operators */ %left '+' '-' %left '*' '/' '%' @@ -10825,17 +10812,12 @@ ReindexStmt: n->relation = $4; n->name = NULL; n->options = 0; -<<<<<<< HEAD - if (n->concurrent) + if ($3) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("REINDEX CONCURRENTLY is not supported"))); -======= - if ($3) - n->options |= REINDEXOPT_CONCURRENTLY; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d $$ = (Node *)n; } | REINDEX reindex_target_multitable opt_concurrently name @@ -10845,17 +10827,12 @@ ReindexStmt: n->name = $4; n->relation = NULL; n->options = 0; -<<<<<<< HEAD - if (n->concurrent) + if ($3) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("REINDEX CONCURRENTLY is not supported"))); -======= - if ($3) - n->options |= REINDEXOPT_CONCURRENTLY; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d $$ = (Node *)n; } | REINDEX '(' reindex_option_list ')' reindex_target_type opt_concurrently qualified_name @@ -10865,17 +10842,12 @@ ReindexStmt: n->relation = $7; n->name = NULL; n->options = $3; -<<<<<<< HEAD - if (n->concurrent) + if ($6) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("REINDEX CONCURRENTLY is not supported"))); -======= - if ($6) - n->options |= REINDEXOPT_CONCURRENTLY; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d $$ = (Node *)n; } | REINDEX '(' reindex_option_list ')' reindex_target_multitable opt_concurrently name @@ -10885,17 +10857,12 @@ ReindexStmt: n->name = $7; n->relation = NULL; n->options = $3; -<<<<<<< HEAD - if (n->concurrent) + if ($6) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("REINDEX CONCURRENTLY is not supported"))); -======= - if ($6) - n->options |= REINDEXOPT_CONCURRENTLY; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d $$ = (Node *)n; } ; @@ -17635,7 +17602,6 @@ target_el: a_expr AS ColLabel $$->val = (Node *)$1; $$->location = @1; } -<<<<<<< HEAD /* * Postgres supports omitting AS only for column labels that aren't * any known keyword. There is an ambiguity against postfix @@ -17643,25 +17609,8 @@ target_el: a_expr AS ColLabel * expression and a column label? We prefer to resolve this * as an infix expression, which we accomplish by assigning * IDENT a precedence higher than POSTFIXOP. - * - * In GPDB, we extend this to allow most unreserved_keywords by - * also assigning them a precedence. There are certain keywords - * that can't work without the as: reserved_keywords, the date - * modifier suffixes (DAY, MONTH, YEAR, etc) and a few other - * obscure cases. */ - | a_expr IDENT -======= | a_expr BareColLabel ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d - { - $$ = makeNode(ResTarget); - $$->name = $2; - $$->indirection = NIL; - $$->val = (Node *)$1; - $$->location = @1; - } - | a_expr ColLabelNoAs { $$ = makeNode(ResTarget); $$->name = $2; @@ -18420,16 +18369,6 @@ unreserved_keyword: * the grammar. */ -ColLabelNoAs: keywords_ok_in_alias_no_as { $$=pstrdup($1); } - ; - -keywords_ok_in_alias_no_as: PartitionIdentKeyword - | TABLESPACE - | ADD_P - | ALTER - | AT - ; - PartitionColId: PartitionIdentKeyword { $$ = pstrdup($1); } | IDENT { $$ = pstrdup($1); } ; @@ -18926,6 +18865,7 @@ bare_label_keyword: | ABSOLUTE_P | ACCESS | ACTION + | ACTIVE | ADD_P | ADMIN | AFTER @@ -18980,17 +18920,22 @@ bare_label_keyword: | COMMENTS | COMMIT | COMMITTED + | CONCURRENCY | CONCURRENTLY | CONFIGURATION | CONFLICT | CONNECTION | CONSTRAINT | CONSTRAINTS + | CONTAINS | CONTENT_P | CONTINUE_P | CONVERSION_P | COPY | COST + | CPUSET + | CPU_RATE_LIMIT + | CREATEEXTTABLE | CROSS | CSV | CUBE @@ -19036,9 +18981,12 @@ bare_label_keyword: | ENCODING | ENCRYPTED | END_P + | ENDPOINT | ENUM_P + | ERRORS | ESCAPE | EVENT + | EXCHANGE | EXCLUDE | EXCLUDING | EXCLUSIVE @@ -19051,11 +18999,14 @@ bare_label_keyword: | EXTRACT | FALSE_P | FAMILY + | FIELDS + | FILL | FIRST_P | FLOAT_P | FOLLOWING | FORCE | FOREIGN + | FORMAT | FORWARD | FREEZE | FULL @@ -19068,8 +19019,10 @@ bare_label_keyword: | GROUPING | GROUPS | HANDLER + | HASH | HEADER_P | HOLD + | HOST | IDENTITY_P | IF_P | ILIKE @@ -19080,6 +19033,7 @@ bare_label_keyword: | IN_P | INCLUDE | INCLUDING + | INCLUSIVE | INCREMENT | INDEX | INDEXES @@ -19112,6 +19066,7 @@ bare_label_keyword: | LEFT | LEVEL | LIKE + | LIST | LISTEN | LOAD | LOCAL @@ -19120,14 +19075,22 @@ bare_label_keyword: | LOCATION | LOCK_P | LOCKED + | LOG_P | LOGGED | MAPPING + | MASTER | MATCH | MATERIALIZED | MAXVALUE + | MEDIAN + | MEMORY_LIMIT + | MEMORY_SHARED_QUOTA + | MEMORY_SPILL_RATIO | METHOD | MINVALUE + | MISSING | MODE + | MODIFIES | MOVE | NAME_P | NAMES @@ -19135,13 +19098,16 @@ bare_label_keyword: | NATURAL | NCHAR | NEW + | NEWLINE | NEXT | NFC | NFD | NFKC | NFKD | NO + | NOCREATEEXTTABLE | NONE + | NOOVERCOMMIT | NORMALIZE | NORMALIZED | NOT @@ -19166,6 +19132,7 @@ bare_label_keyword: | OTHERS | OUT_P | OUTER_P + | OVERCOMMIT | OVERLAY | OVERRIDING | OWNED @@ -19173,9 +19140,11 @@ bare_label_keyword: | PARALLEL | PARSER | PARTIAL - | PARTITION + | PARTITIONS | PASSING | PASSWORD + | PERCENT + | PERSISTENTLY | PLACING | PLANS | POLICY @@ -19191,10 +19160,15 @@ bare_label_keyword: | PROCEDURE | PROCEDURES | PROGRAM + | PROTOCOL | PUBLICATION + | QUEUE | QUOTE + | RANDOMLY | RANGE | READ + | READABLE + | READS | REAL | REASSIGN | RECHECK @@ -19204,6 +19178,7 @@ bare_label_keyword: | REFERENCING | REFRESH | REINDEX + | REJECT_P | RELATIVE_P | RELEASE | RENAME @@ -19211,8 +19186,10 @@ bare_label_keyword: | REPLACE | REPLICA | RESET + | RESOURCE | RESTART | RESTRICT + | RETRIEVE | RETURNS | REVOKE | RIGHT @@ -19230,6 +19207,8 @@ bare_label_keyword: | SCROLL | SEARCH | SECURITY + | SEGMENT + | SEGMENTS | SELECT | SEQUENCE | SEQUENCES @@ -19248,6 +19227,7 @@ bare_label_keyword: | SMALLINT | SNAPSHOT | SOME + | SPLIT | SQL_P | STABLE | STANDALONE_P @@ -19260,6 +19240,7 @@ bare_label_keyword: | STORED | STRICT_P | STRIP_P + | SUBPARTITION | SUBSCRIPTION | SUBSTRING | SUPPORT @@ -19275,6 +19256,7 @@ bare_label_keyword: | TEMPORARY | TEXT_P | THEN + | THRESHOLD | TIES | TIME | TIMESTAMP @@ -19304,6 +19286,7 @@ bare_label_keyword: | VACUUM | VALID | VALIDATE + | VALIDATION | VALIDATOR | VALUE_P | VALUES @@ -19314,10 +19297,12 @@ bare_label_keyword: | VIEW | VIEWS | VOLATILE + | WEB | WHEN | WHITESPACE_P | WORK | WRAPPER + | WRITABLE | WRITE | XML_P | XMLATTRIBUTES From f58ea4f7e006d22ac107e298c1e75415d39f87d6 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 29 May 2026 13:30:30 +0500 Subject: [PATCH 501/589] Fix compilation of src/backend/utils/error/assert.c (#2618) Commit f57cef9 in src/backend/utils/error/assert.c introduced a syntax error while fixing conflicts. --- src/backend/utils/error/assert.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/utils/error/assert.c b/src/backend/utils/error/assert.c index f8d517cac86d..c4aa7e51780f 100644 --- a/src/backend/utils/error/assert.c +++ b/src/backend/utils/error/assert.c @@ -43,8 +43,8 @@ ExceptionalCondition(const char *conditionName, || !PointerIsValid(errorType)) ereport(FATAL, errFatalReturn(gp_reraise_signal), - errmsg("TRAP: ExceptionalCondition: bad arguments in PID %d")), - (int) getpid(); + errmsg("TRAP: ExceptionalCondition: bad arguments in PID %d"), + (int) getpid()); else ereport(FATAL, errFatalReturn(gp_reraise_signal), From 047aa3611a8811f15ef32ba91493cd96e270eeb8 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 29 May 2026 13:30:53 +0500 Subject: [PATCH 502/589] Fix compilation of src/backend/commands/indexcmds.c (#2619) Commit a6642b3 added a new ReindexErrorInfo structure to src/backend/commands/indexcmds.c, while earlier commit 67a7c0f had already done the same. --- src/backend/commands/indexcmds.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 244f4d9e267d..1b68706cc7c9 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -212,16 +212,6 @@ typedef struct ReindexErrorInfo char relkind; } ReindexErrorInfo; -/* - * callback arguments for reindex_error_callback() - */ -typedef struct ReindexErrorInfo -{ - char *relname; - char *relnamespace; - char relkind; -} ReindexErrorInfo; - /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a From 8c4a73d98e7ecc70443f50de14166f68228ae377 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 29 May 2026 14:08:54 +0500 Subject: [PATCH 503/589] Fix compilation of analyzefuncs.c and cdbmutate.c (#2621) Commit 8e1f37c changed the generation rule for fmgroids.h macros. Change in GPDB-specific places. --- src/backend/cdb/cdbmutate.c | 2 +- src/backend/commands/analyzefuncs.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c index d9965d480b62..3badfc7819b9 100644 --- a/src/backend/cdb/cdbmutate.c +++ b/src/backend/cdb/cdbmutate.c @@ -1133,7 +1133,7 @@ makeSegmentFilterExpr(int segid) make_opclause(Int4EqualOperator, BOOLOID, false, /* opretset */ - (Expr *) makeFuncExpr(F_MPP_EXECUTION_SEGMENT, + (Expr *) makeFuncExpr(F_GP_EXECUTION_SEGMENT, INT4OID, NIL, /* args */ InvalidOid, diff --git a/src/backend/commands/analyzefuncs.c b/src/backend/commands/analyzefuncs.c index 6a4cdb545fef..6ea5f48d47c2 100644 --- a/src/backend/commands/analyzefuncs.c +++ b/src/backend/commands/analyzefuncs.c @@ -342,7 +342,7 @@ gp_acquire_sample_rows_col_type(Oid typid) */ return OIDOID; - case PGNODETREEOID: + case PG_NODE_TREEOID: /* * Input function of pg_node_tree doesn't allow loading * back values. Treat it as text. From dcbb5087273a8693f425a370ca7182b0503d60c9 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 29 May 2026 14:09:29 +0500 Subject: [PATCH 504/589] Resolve conflicts in src/backend/postmaster/pgstat.c (#2585) 1) Commit 8d9a935 in src/backend/postmaster/pgstat.c added handling for the new PGSTAT_MTYPE_WAL case in the PgstatCollectorMain function, while earlier commit 6b0e52b had already added handling for the PGSTAT_MTYPE_QUEUESTAT case in the same location. 2) Commit 9868167 in src/backend/postmaster/pgstat.c added a fputc for the new 'R' case in the pgstat_write_statsfiles function, while earlier commit c7649f1 had already added a fputc for the 'Q' case in the same location. 3) Commit 9868167 in src/backend/postmaster/pgstat.c added allocate the space for replication slot statistics in the pgstat_read_statsfiles function, while earlier commit 6b0e52b had already added creation of a Queue hashtable in the same location. 4) Commit 9868167 in src/backend/postmaster/pgstat.c added handling for the new 'R' case in the pgstat_read_statsfiles function, while earlier commit c7649f1 had already added handling for the 'Q' case in the same location. 5) Commit 9868167 in src/backend/postmaster/pgstat.c in the pgstat_read_db_statsfile_timestamp function added handling for the new case 'R', while the earlier commit c7649f1 had already added handling for the case 'Q' in the same place. --- src/backend/postmaster/pgstat.c | 56 ++++++++++++++++----------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index e72ab5dab8ca..9727c6965603 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -5231,13 +5231,12 @@ PgstatCollectorMain(int argc, char *argv[]) pgstat_recv_bgwriter(&msg.msg_bgwriter, len); break; -<<<<<<< HEAD case PGSTAT_MTYPE_QUEUESTAT: /* GPDB */ pgstat_recv_queuestat((PgStat_MsgQueuestat *) &msg, len); -======= + break; + case PGSTAT_MTYPE_WAL: pgstat_recv_wal(&msg.msg_wal, len); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d break; case PGSTAT_MTYPE_SLRU: @@ -5556,7 +5555,6 @@ pgstat_write_statsfiles(bool permanent, bool allDbs) } /* -<<<<<<< HEAD * Walk through resource queue stats. */ hash_seq_init(&qstat, pgStatQueueHash); @@ -5564,7 +5562,9 @@ pgstat_write_statsfiles(bool permanent, bool allDbs) { fputc('Q', fpout); fwrite(queueentry, sizeof(PgStat_StatQueueEntry), 1, fpout); -======= + } + + /* * Write replication slot stats struct */ for (i = 0; i < nReplSlotStats; i++) @@ -5572,7 +5572,6 @@ pgstat_write_statsfiles(bool permanent, bool allDbs) fputc('R', fpout); rc = fwrite(&replSlotStats[i], sizeof(PgStat_ReplSlotStats), 1, fpout); (void) rc; /* we'll check for error with ferror */ ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } /* @@ -5803,7 +5802,6 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) dbhash = hash_create("Databases hash", PGSTAT_DB_HASH_SIZE, &hash_ctl, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); -<<<<<<< HEAD /** ** Create the Queue hashtable **/ @@ -5815,11 +5813,10 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) queuehash = hash_create("Queues hash", PGSTAT_QUEUE_HASH_SIZE, &hash_ctl, HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT); pgStatQueueHash = queuehash; -======= + /* Allocate the space for replication slot statistics */ replSlotStats = palloc0(max_replication_slots * sizeof(PgStat_ReplSlotStats)); nReplSlotStats = 0; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* * Clear out global, archiver, WAL and SLRU statistics so they start from @@ -6026,25 +6023,15 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) break; /* -<<<<<<< HEAD * 'Q' A PgStat_StatQueueEntry follows. (GPDB) */ case 'Q': if (fread(&queuebuf, 1, sizeof(PgStat_StatQueueEntry), fpin) != sizeof(PgStat_StatQueueEntry)) -======= - * 'R' A PgStat_ReplSlotStats struct describing a replication - * slot follows. - */ - case 'R': - if (fread(&replSlotStats[nReplSlotStats], 1, sizeof(PgStat_ReplSlotStats), fpin) - != sizeof(PgStat_ReplSlotStats)) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d { ereport(pgStatRunningInCollector ? LOG : WARNING, (errmsg("corrupted statistics file \"%s\"", statfile))); -<<<<<<< HEAD goto done; } @@ -6067,12 +6054,23 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) } memcpy(queueentry, &queuebuf, sizeof(PgStat_StatQueueEntry)); -======= + break; + + /* + * 'R' A PgStat_ReplSlotStats struct describing a replication + * slot follows. + */ + case 'R': + if (fread(&replSlotStats[nReplSlotStats], 1, sizeof(PgStat_ReplSlotStats), fpin) + != sizeof(PgStat_ReplSlotStats)) + { + ereport(pgStatRunningInCollector ? LOG : WARNING, + (errmsg("corrupted statistics file \"%s\"", + statfile))); memset(&replSlotStats[nReplSlotStats], 0, sizeof(PgStat_ReplSlotStats)); goto done; } nReplSlotStats++; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d break; case 'E': @@ -6403,30 +6401,32 @@ pgstat_read_db_statsfile_timestamp(Oid databaseid, bool permanent, break; /* -<<<<<<< HEAD * 'Q' A PgStat_StatQueueEntry follows. (GPDB) */ case 'Q': if (fread(&queuebuf, 1, sizeof(PgStat_StatQueueEntry), fpin) != sizeof(PgStat_StatQueueEntry)) -======= + { + ereport(pgStatRunningInCollector ? LOG : WARNING, + (errmsg("corrupted statistics file \"%s\"", + statfile))); + goto done; + } + break; + + /* * 'R' A PgStat_ReplSlotStats struct describing a replication * slot follows. */ case 'R': if (fread(&myReplSlotStats, 1, sizeof(PgStat_ReplSlotStats), fpin) != sizeof(PgStat_ReplSlotStats)) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d { ereport(pgStatRunningInCollector ? LOG : WARNING, (errmsg("corrupted statistics file \"%s\"", statfile))); -<<<<<<< HEAD - goto done; -======= FreeFile(fpin); return false; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } break; From 9fa92d1f5ef55fcd0aeb98378873458680eb82d4 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 29 May 2026 14:09:55 +0500 Subject: [PATCH 505/589] Fix compilation of src/backend/utils/sort/tuplestore.c (#2623) Commit 808e13b added a new int mode argument to the BufFileOpenShared function definition in src/include/storage/buffile.h. Add this in GPDB-specific places. --- src/backend/utils/sort/tuplestore.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/utils/sort/tuplestore.c b/src/backend/utils/sort/tuplestore.c index 62715b4be1ad..852aac08fb03 100644 --- a/src/backend/utils/sort/tuplestore.c +++ b/src/backend/utils/sort/tuplestore.c @@ -1774,7 +1774,7 @@ tuplestore_open_shared(SharedFileSet *fileset, const char *filename) state->writetup = writetup_forbidden; state->readtup = readtup_heap; - state->myfile = BufFileOpenShared(fileset, filename); + state->myfile = BufFileOpenShared(fileset, filename, O_RDONLY); state->readptrs[0].file = 0; state->readptrs[0].offset = 0L; state->status = TSS_READFILE; From c97c117e02a4080ccde948b388ba6aa7eec452bd Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 29 May 2026 14:10:21 +0500 Subject: [PATCH 506/589] Fix compilation of src/backend/access/transam/distributedlog.c (#2624) Commit dee663f added a new SyncRequestHandler sync_handler argument, to the SimpleLruInit function definition in src/include/access/slru.h. Add this in GPDB-specific places. --- src/backend/access/transam/distributedlog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/access/transam/distributedlog.c b/src/backend/access/transam/distributedlog.c index 4dfa99cb9d39..a330fe4681b1 100644 --- a/src/backend/access/transam/distributedlog.c +++ b/src/backend/access/transam/distributedlog.c @@ -661,7 +661,7 @@ DistributedLog_ShmemInit(void) DistributedLogCtl->PagePrecedes = DistributedLog_PagePrecedes; SimpleLruInit(DistributedLogCtl, "DistributedLogCtl", DistributedLog_ShmemBuffers(), 0, DistributedLogControlLock, "pg_distributedlog", - LWTRANCHE_DISTRIBUTEDLOG_BUFFERS); + LWTRANCHE_DISTRIBUTEDLOG_BUFFERS, SYNC_HANDLER_CLOG); /* Create or attach to the shared structure */ DistributedLogShared = From 640451e852bc5f374d396bdac4d43d970673aa12 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 29 May 2026 14:10:44 +0500 Subject: [PATCH 507/589] Fix compilation of src/backend/commands/tablecmds.c (#2625) Commit 5028981 added a new function, ATExecCookedColumnDefault, to src/backend/commands/tablecmds.c, which calls the StoreAttrDefault function. However, the earlier commit 19cd1cf had already added the new arguments bool *cookedMissingVal, Datum *missingval_p, and bool *missingIsNull_p. --- src/backend/commands/tablecmds.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 1f7c40aa81ba..c42da109c666 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -8852,7 +8852,9 @@ ATExecCookedColumnDefault(Relation rel, AttrNumber attnum, RemoveAttrDefault(RelationGetRelid(rel), attnum, DROP_RESTRICT, false, true); - (void) StoreAttrDefault(rel, attnum, newDefault, true, false); + (void) StoreAttrDefault(rel, attnum, newDefault, + NULL, NULL, NULL, /* missing val stuff */ + true, false); ObjectAddressSubSet(address, RelationRelationId, RelationGetRelid(rel), attnum); From cae4111bd07cd4ec3acb8e335180a2b1f565552a Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 29 May 2026 14:34:39 +0500 Subject: [PATCH 508/589] Fix compilation of gpdbwrappers, COptTasks, CTranslatorRelcacheToDXL (#2626) Commit 8e1f37c changed the generation rule for fmgroids.h macros. Change in GPDB-specific places. --- src/backend/gpopt/gpdbwrappers.cpp | 29 +++++++++---------- .../translate/CTranslatorRelcacheToDXL.cpp | 2 +- src/backend/gpopt/utils/COptTasks.cpp | 2 +- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/backend/gpopt/gpdbwrappers.cpp b/src/backend/gpopt/gpdbwrappers.cpp index a6ee4403cebb..ca9fd915e34e 100644 --- a/src/backend/gpopt/gpdbwrappers.cpp +++ b/src/backend/gpopt/gpdbwrappers.cpp @@ -526,17 +526,16 @@ gpdb::IsFuncAllowedForPartitionSelection(Oid funcid) // For range partition selection, the logic in ORCA checks on bounds of the partition ranges. // Hence these must be increasing functions. case F_TIMESTAMP_DATE: // date(timestamp) -> date - case F_DTOI4: // int4(float8) -> int4 - case F_FTOI4: // int4(float4) -> int4 - case F_INT82: // int2(int8) -> int2 - case F_INT84: // int4(int8) -> int4 - case F_I4TOI2: // int2(int4) -> int2 - case F_FTOI8: // int8(float4) -> int8 - case F_FTOI2: // int2(float4) -> int2 + case F_FLOAT8_INT4: // int4(float8) -> int4 + case F_FLOAT4_INT4: // int4(float4) -> int4 + case F_INT8_INT2: // int2(int8) -> int2 + case F_INT8_INT4: // int4(int8) -> int4 + case F_INT4_INT2: // int2(int4) -> int2 + case F_FLOAT4_INT8: // int8(float4) -> int8 + case F_FLOAT4_INT2: // int2(float4) -> int2 case F_FLOAT4_NUMERIC: // numeric(float4) -> numeric - case F_DTOI8: // int8(float8) -> int8 - case F_DTOI2: // int2(float4) -> int2 - case F_DTOF: // float4(float8) -> float4 + case F_FLOAT8_INT8: // int8(float8) -> int8 + case F_FLOAT8_FLOAT4: // float4(float8) -> float4 case F_FLOAT8_NUMERIC: // numeric(float8) -> numeric case F_NUMERIC_INT8: // int8(numeric) -> int8 case F_NUMERIC_INT2: // int2(numeric) -> int2 @@ -568,11 +567,11 @@ gpdb::IsFuncNDVPreserving(Oid funcid) switch (funcid) { // for now, these are the functions we consider for this optimization - case F_LOWER: - case F_LTRIM1: - case F_BTRIM1: - case F_RTRIM1: - case F_UPPER: + case F_LOWER_TEXT: + case F_LTRIM_TEXT: + case F_BTRIM_TEXT: + case F_RTRIM_TEXT: + case F_UPPER_TEXT: return true; default: return false; diff --git a/src/backend/gpopt/translate/CTranslatorRelcacheToDXL.cpp b/src/backend/gpopt/translate/CTranslatorRelcacheToDXL.cpp index 26e7e9055e47..51f882a9e994 100644 --- a/src/backend/gpopt/translate/CTranslatorRelcacheToDXL.cpp +++ b/src/backend/gpopt/translate/CTranslatorRelcacheToDXL.cpp @@ -1098,7 +1098,7 @@ CTranslatorRelcacheToDXL::RetrieveType(CMemoryPool *mp, IMDId *mdid) // count aggregate is the same for all types CMDIdGPDB *mdid_count = - GPOS_NEW(mp) CMDIdGPDB(IMDId::EmdidGeneral, COUNT_ANY_OID); + GPOS_NEW(mp) CMDIdGPDB(IMDId::EmdidGeneral, GPDB_COUNT_ANY); // check if type is composite CMDIdGPDB *mdid_type_relid = nullptr; diff --git a/src/backend/gpopt/utils/COptTasks.cpp b/src/backend/gpopt/utils/COptTasks.cpp index a8fcd23a98d4..c3af62f9af24 100644 --- a/src/backend/gpopt/utils/COptTasks.cpp +++ b/src/backend/gpopt/utils/COptTasks.cpp @@ -384,7 +384,7 @@ COptTasks::CreateOptimizerConfig(CMemoryPool *mp, ICostModel *cost_model) * enforce them ourselves in the executor */ push_group_by_below_setop_threshold, xform_bind_threshold, skew_factor), - GPOS_NEW(mp) CWindowOids(OID(F_WINDOW_ROW_NUMBER), OID(F_WINDOW_RANK))); + GPOS_NEW(mp) CWindowOids(OID(F_ROW_NUMBER), OID(F_RANK_))); } //--------------------------------------------------------------------------- From 18922523fd6c88778469d9f1b62e7b7d17be3f8d Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 29 May 2026 14:41:07 +0500 Subject: [PATCH 509/589] Fix compilation of src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp (#2627) Commit 1375422 in src/include/nodes/plannodes.h removed the resultRelIndex field from the PlannedStmt structure. Remove this in GPDB-specific code. --- src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp b/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp index 679562ee4735..678b9193b35f 100644 --- a/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp +++ b/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp @@ -4388,7 +4388,6 @@ CTranslatorDXLToPlStmt::TranslateDXLDml( dml->canSetTag = true; // FIXME dml->nominalRelation = index; dml->resultRelations = ListMake1Int(index); - dml->resultRelIndex = list_length(m_result_rel_list) - 1; dml->rootRelation = md_rel->IsPartitioned() ? index : 0; dml->plans = ListMake1(child_plan); From 698e01e2d6fca27ae2721a4c6ffc38a368ebb9d8 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 29 May 2026 14:50:16 +0500 Subject: [PATCH 510/589] Resolve conflicts in src/backend/replication/syncrep.c (#2586) Commit be9788e in src/backend/replication/syncrep.c in the SyncRepWaitForLSN function moved the conditional return higher, adding additional conditions and a large comment. Commit e174f69 added an InterruptHoldoffCount assertion before it. However, the earlier commit 6b0e52b had already added a debug log in the same location, and commit 33a6433 added an assertion before it, and commit bdb1ea1 removed InterruptHoldoffCount assertion. --- src/backend/replication/syncrep.c | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/backend/replication/syncrep.c b/src/backend/replication/syncrep.c index b4d61d7be9d6..26fde93d3457 100644 --- a/src/backend/replication/syncrep.c +++ b/src/backend/replication/syncrep.c @@ -160,15 +160,6 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit) const char *old_status; int mode; -<<<<<<< HEAD -======= - /* - * This should be called while holding interrupts during a transaction - * commit to prevent the follow-up shared memory queue cleanups to be - * influenced by external interruptions. - */ - Assert(InterruptHoldoffCount > 0); - /* * Fast exit if user has not requested sync replication, or there are no * sync replication standby names defined. @@ -187,27 +178,17 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit) !((volatile WalSndCtlData *) WalSndCtl)->sync_standbys_defined) return; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* Cap the level for anything other than commit to remote flush only. */ if (commit) mode = SyncRepWaitMode; else mode = Min(SyncRepWaitMode, SYNC_REP_WAIT_FLUSH); -<<<<<<< HEAD Assert(!am_walsender); elogif(debug_walrepl_syncrep, LOG, "syncrep wait -- This backend's commit LSN for syncrep is %X/%X.", (uint32) (lsn >> 32), (uint32) lsn); - /* - * Fast exit if user has not requested sync replication. - */ - if (!SyncRepRequested()) - return; - -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d Assert(SHMQueueIsDetached(&(MyProc->syncRepLinks))); Assert(WalSndCtl != NULL); From 4e77b1b268da62404b7266a4bbe5ae341a44ef54 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 29 May 2026 15:03:14 +0500 Subject: [PATCH 511/589] Resolve conflicts in src/backend/storage/file/buffile.c (#2587) 1) Commit 808e13b in src/backend/storage/file/buffile.c added a new conditional error message to the BufFileSeek function. However, earlier commit 5028054 had already done the same, but with incorrect indentation, and also added a conditional call to the BufFileFlush function. 2) Commit 808e13b added a new function BufFileTruncateShared to src/backend/storage/file/buffile.c, while earlier GPDB-specific commits had already added the functions BufFileGetFilename, BufFileSuspend, BufFileResume, BufFilePledgeSequential, BufFileStartCompression, BufFileDumpCompressedBuffer, BufFileEndCompression, BufFileLoadCompressedBuffer in the same location. --- src/backend/storage/file/buffile.c | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c index e76154624e0e..c8f09c15daaa 100644 --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c @@ -914,34 +914,22 @@ BufFileSeek(BufFile *file, int fileno, off_t offset, int whence) newOffset = (file->curOffset + file->pos) + offset; break; case SEEK_END: -<<<<<<< HEAD -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* * The file size of the last file gives us the end offset of that * file. */ -<<<<<<< HEAD if (file->curFile == file->numFiles - 1 && file->dirty) BufFileFlush(file); -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d newFile = file->numFiles - 1; newOffset = FileSize(file->files[file->numFiles - 1]); if (newOffset < 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not determine size of temporary file \"%s\" from BufFile \"%s\": %m", -<<<<<<< HEAD - FilePathName(file->files[file->numFiles - 1]), - file->name))); - break; -======= FilePathName(file->files[file->numFiles - 1]), file->name))); break; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d default: elog(ERROR, "invalid whence: %d", whence); return EOF; @@ -1160,7 +1148,6 @@ BufFileAppend(BufFile *target, BufFile *source) } /* -<<<<<<< HEAD * Return filename of the underlying file. * * For debugging purposes only. Returns the filename of the @@ -1509,7 +1496,8 @@ BufFileLoadCompressedBuffer(BufFile *file, void *buffer, size_t bufsize) } #endif /* USE_ZSTD */ -======= + +/* * Truncate a BufFile created by BufFileCreateShared up to the given fileno and * the offset. */ @@ -1603,4 +1591,3 @@ BufFileTruncateShared(BufFile *file, int fileno, off_t offset) } /* Nothing to do, if the truncate point is beyond current file. */ } ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d From 9d5567d92782bf0d15d3da8481b2247e2b71e978 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 29 May 2026 15:51:54 +0500 Subject: [PATCH 512/589] Fix OID compilation (#2610) 1) Commit 16fa9b2 in the file src/include/catalog/pg_proc.dat added a new OID 3435, while the earlier commit 19cd1cf already used this OID in the file src/include/catalog/pg_am.dat. 2) Commit 36b9312 prohibited the use of custom OID symbols in pg_proc.dat. --- src/include/catalog/pg_am.dat | 2 +- src/include/catalog/pg_proc.dat | 20 ++++++++------------ src/include/catalog/pg_type.dat | 6 ++---- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/include/catalog/pg_am.dat b/src/include/catalog/pg_am.dat index c606948dfa6f..cad81556f809 100644 --- a/src/include/catalog/pg_am.dat +++ b/src/include/catalog/pg_am.dat @@ -19,7 +19,7 @@ { oid => '6131', oid_symbol => 'AO_ROW_TABLE_AM_OID', descr => 'row-oriented append-optimized table access method', amname => 'ao_row', amhandler => 'ao_row_tableam_handler', amtype => 't' }, -{ oid => '3435', oid_symbol => 'AO_COLUMN_TABLE_AM_OID', +{ oid => '8435', oid_symbol => 'AO_COLUMN_TABLE_AM_OID', descr => 'column-oriented append-optimized table access method', amname => 'ao_column', amhandler => 'ao_column_tableam_handler', amtype => 't' }, diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 265f06eb1bce..121ba59b80c7 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -906,13 +906,11 @@ prorettype => 'table_am_handler', proargtypes => 'internal', prosrc => 'heap_tableam_handler' }, -{ oid => '4198', oid_symbol => 'AO_ROW_TABLE_AM_HANDLER_OID', - descr => 'row-oriented append-optimized table access method handler', +{ oid => '4198', descr => 'row-oriented append-optimized table access method handler', proname => 'ao_row_tableam_handler', provolatile => 'v', prorettype => 'table_am_handler', proargtypes => 'internal', prosrc => 'ao_row_tableam_handler' }, -{ oid => '4199', oid_symbol => 'AO_COLUMN_TABLE_AM_HANDLER_OID', - descr => 'column-oriented append-optimized table access method handler', +{ oid => '4199', descr => 'column-oriented append-optimized table access method handler', proname => 'ao_column_tableam_handler', provolatile => 'v', prorettype => 'table_am_handler', proargtypes => 'internal', prosrc => 'ao_column_tableam_handler' }, @@ -6534,9 +6532,7 @@ proargtypes => 'pg_lsn', prosrc => 'aggregate_dummy' }, # count has two forms: count(any) and count(*) -{ oid => '2147', - oid_symbol => 'COUNT_ANY_OID', - descr => 'number of input rows for which the input expression is not null', +{ oid => '2147', descr => 'number of input rows for which the input expression is not null', proname => 'count', prokind => 'a', proisstrict => 'f', prorettype => 'int8', proargtypes => 'any', prosrc => 'aggregate_dummy' }, { oid => '2803', descr => 'number of input rows', @@ -11389,7 +11385,7 @@ prosrc => 'gp_percentile_disc_transition' }, { oid => 7194, descr => 'unordered percentile discrete aggregate', proname => 'gp_percentile_disc', prokind => 'a', proisstrict => 'f', prorettype => 'anyelement', proargtypes => 'anyelement float8 int8 int8', prosrc => 'aggregate_dummy' }, -{ oid => 7050, oid_symbol => 'BITMAP_INDEXAM_HANDLER_OID', descr => 'bitmap(internal)', +{ oid => 7050, descr => 'bitmap(internal)', proname => 'bmhandler', provolatile => 'v', prorettype => 'index_am_handler', proargtypes => 'internal', prosrc => 'bmhandler' }, @@ -11887,16 +11883,16 @@ { oid => 6126, descr => 'aggregate final function', proname => 'percentile_cont_timestamptz_multi_final', proisstrict => 'f', prorettype => '_timestamptz', proargtypes => 'internal _float8', prosrc => 'percentile_cont_timestamptz_multi_final' }, -{ oid => 6127, oid_symbol => 'MEDIAN_FLOAT8_OID', descr => 'median', +{ oid => 6127, descr => 'median', proname => 'median', prokind => 'a', proisstrict => 'f', prorettype => 'float8', proargtypes => 'float8 float8', prosrc => 'aggregate_dummy' }, -{ oid => 6128, oid_symbol => 'MEDIAN_INTERVAL_OID', descr => 'median', +{ oid => 6128, descr => 'median', proname => 'median', prokind => 'a', proisstrict => 'f', prorettype => 'interval', proargtypes => 'float8 interval', prosrc => 'aggregate_dummy' }, -{ oid => 6129, oid_symbol => 'MEDIAN_TIMESTAMP_OID', descr => 'median', +{ oid => 6129, descr => 'median', proname => 'median', prokind => 'a', proisstrict => 'f', prorettype => 'timestamp', proargtypes => 'float8 timestamp', prosrc => 'aggregate_dummy' }, -{ oid => 6130, oid_symbol => 'MEDIAN_TIMESTAMPTZ_OID', descr => 'median', +{ oid => 6130, descr => 'median', proname => 'median', prokind => 'a', proisstrict => 'f', prorettype => 'timestamptz', proargtypes => 'float8 timestamptz', prosrc => 'aggregate_dummy' }, { oid => 8066, descr => 'itemwise add two integer arrays', diff --git a/src/include/catalog/pg_type.dat b/src/include/catalog/pg_type.dat index 389c3308d888..48765a88bb3f 100644 --- a/src/include/catalog/pg_type.dat +++ b/src/include/catalog/pg_type.dat @@ -620,13 +620,11 @@ typoutput => 'anycompatiblerange_out', typreceive => '-', typsend => '-', typalign => 'd', typstorage => 'x' }, -{ oid => '7198', oid_symbol => 'COMPLEXOID', array_type_oid => '7199', - descr => 'double-precision floating point complex number, 16-byte storage', +{ oid => '7198', array_type_oid => '7199', descr => 'double-precision floating point complex number, 16-byte storage', typname => 'complex', typlen => '16', typbyval => 'f', typcategory => 'N', typdelim => '\054', typinput => 'complex_in', typoutput => 'complex_out', typreceive => 'complex_recv', typsend => 'complex_send', typalign => 'd' }, -{ oid => '7053', oid_symbol => 'ANYTABLEOID', - descr => 'Represents a generic TABLE value expression', +{ oid => '7053', descr => 'Represents a generic TABLE value expression', typname => 'anytable', typlen => '-1', typbyval => 'f', typtype => 'p', typcategory => 'P', typdelim => '\054', typinput => 'anytable_in', typoutput => 'anytable_out', typreceive => '-', typsend => '-', From a982d2df78a05f1933df9b2e46f8082c274f8955 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Fri, 29 May 2026 14:44:33 +0300 Subject: [PATCH 513/589] Resolve conflicts in src/include/parser/kwlist.h (#2549) Commit 06a7c3154f5bfad65549810cc84f0e3a77b408bf has added a new is-bare-label argument to PG_KEYWORD. Most keywords have BARE_LABEL, allowing them to be used as labels without AS keyword, but 39 are marked AS_LABEL, requiring as. However, there are a lot of GPDB-specific keywords too, most of them without comments. Conservatively mark them AS_LABEL to avoid accidentally breaking the grammar, Also mark all GPDB-specific keywords with comments to simplify future conflict resolutions. However, some of the keywords, specifically "format" and "ignore", are present in later versions of Postgres and "format" is marked as BARE_LABEL, so mark it BARE_LABEL here too. --- src/include/parser/kwlist.h | 590 +++++------------------------------- 1 file changed, 68 insertions(+), 522 deletions(-) diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 929707efae55..b8d7d6ac7844 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -24,526 +24,12 @@ * Note: gen_keywordlist.pl requires the entries to appear in ASCII order. */ -<<<<<<< HEAD -/* name, value, category */ -PG_KEYWORD("abort", ABORT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("absolute", ABSOLUTE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("access", ACCESS, UNRESERVED_KEYWORD) -PG_KEYWORD("action", ACTION, UNRESERVED_KEYWORD) -PG_KEYWORD("active", ACTIVE, UNRESERVED_KEYWORD) -PG_KEYWORD("add", ADD_P, UNRESERVED_KEYWORD) -PG_KEYWORD("admin", ADMIN, UNRESERVED_KEYWORD) -PG_KEYWORD("after", AFTER, UNRESERVED_KEYWORD) -PG_KEYWORD("aggregate", AGGREGATE, UNRESERVED_KEYWORD) -PG_KEYWORD("all", ALL, RESERVED_KEYWORD) -PG_KEYWORD("also", ALSO, UNRESERVED_KEYWORD) -PG_KEYWORD("alter", ALTER, UNRESERVED_KEYWORD) -PG_KEYWORD("always", ALWAYS, UNRESERVED_KEYWORD) -PG_KEYWORD("analyse", ANALYSE, RESERVED_KEYWORD) /* British spelling */ -PG_KEYWORD("analyze", ANALYZE, RESERVED_KEYWORD) -PG_KEYWORD("and", AND, RESERVED_KEYWORD) -PG_KEYWORD("any", ANY, RESERVED_KEYWORD) -PG_KEYWORD("array", ARRAY, RESERVED_KEYWORD) -PG_KEYWORD("as", AS, RESERVED_KEYWORD) -PG_KEYWORD("asc", ASC, RESERVED_KEYWORD) -PG_KEYWORD("assertion", ASSERTION, UNRESERVED_KEYWORD) -PG_KEYWORD("assignment", ASSIGNMENT, UNRESERVED_KEYWORD) -PG_KEYWORD("asymmetric", ASYMMETRIC, RESERVED_KEYWORD) -PG_KEYWORD("at", AT, UNRESERVED_KEYWORD) -PG_KEYWORD("attach", ATTACH, UNRESERVED_KEYWORD) -PG_KEYWORD("attribute", ATTRIBUTE, UNRESERVED_KEYWORD) -PG_KEYWORD("authorization", AUTHORIZATION, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("backward", BACKWARD, UNRESERVED_KEYWORD) -PG_KEYWORD("before", BEFORE, UNRESERVED_KEYWORD) -PG_KEYWORD("begin", BEGIN_P, UNRESERVED_KEYWORD) -PG_KEYWORD("between", BETWEEN, COL_NAME_KEYWORD) -PG_KEYWORD("bigint", BIGINT, COL_NAME_KEYWORD) -PG_KEYWORD("binary", BINARY, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("bit", BIT, COL_NAME_KEYWORD) -PG_KEYWORD("boolean", BOOLEAN_P, COL_NAME_KEYWORD) -PG_KEYWORD("both", BOTH, RESERVED_KEYWORD) -PG_KEYWORD("by", BY, UNRESERVED_KEYWORD) -PG_KEYWORD("cache", CACHE, UNRESERVED_KEYWORD) -PG_KEYWORD("call", CALL, UNRESERVED_KEYWORD) -PG_KEYWORD("called", CALLED, UNRESERVED_KEYWORD) -PG_KEYWORD("cascade", CASCADE, UNRESERVED_KEYWORD) -PG_KEYWORD("cascaded", CASCADED, UNRESERVED_KEYWORD) -PG_KEYWORD("case", CASE, RESERVED_KEYWORD) -PG_KEYWORD("cast", CAST, RESERVED_KEYWORD) -PG_KEYWORD("catalog", CATALOG_P, UNRESERVED_KEYWORD) -PG_KEYWORD("chain", CHAIN, UNRESERVED_KEYWORD) -PG_KEYWORD("char", CHAR_P, COL_NAME_KEYWORD) -PG_KEYWORD("character", CHARACTER, COL_NAME_KEYWORD) -PG_KEYWORD("characteristics", CHARACTERISTICS, UNRESERVED_KEYWORD) -PG_KEYWORD("check", CHECK, RESERVED_KEYWORD) -PG_KEYWORD("checkpoint", CHECKPOINT, UNRESERVED_KEYWORD) -PG_KEYWORD("class", CLASS, UNRESERVED_KEYWORD) -PG_KEYWORD("close", CLOSE, UNRESERVED_KEYWORD) -PG_KEYWORD("cluster", CLUSTER, UNRESERVED_KEYWORD) -PG_KEYWORD("coalesce", COALESCE, COL_NAME_KEYWORD) -PG_KEYWORD("collate", COLLATE, RESERVED_KEYWORD) -PG_KEYWORD("collation", COLLATION, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("column", COLUMN, RESERVED_KEYWORD) -PG_KEYWORD("columns", COLUMNS, UNRESERVED_KEYWORD) -PG_KEYWORD("comment", COMMENT, UNRESERVED_KEYWORD) -PG_KEYWORD("comments", COMMENTS, UNRESERVED_KEYWORD) -PG_KEYWORD("commit", COMMIT, UNRESERVED_KEYWORD) -PG_KEYWORD("committed", COMMITTED, UNRESERVED_KEYWORD) -PG_KEYWORD("concurrency", CONCURRENCY, UNRESERVED_KEYWORD) -PG_KEYWORD("concurrently", CONCURRENTLY, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("configuration", CONFIGURATION, UNRESERVED_KEYWORD) -PG_KEYWORD("conflict", CONFLICT, UNRESERVED_KEYWORD) -PG_KEYWORD("connection", CONNECTION, UNRESERVED_KEYWORD) -PG_KEYWORD("constraint", CONSTRAINT, RESERVED_KEYWORD) -PG_KEYWORD("constraints", CONSTRAINTS, UNRESERVED_KEYWORD) -PG_KEYWORD("contains", CONTAINS, UNRESERVED_KEYWORD) -PG_KEYWORD("content", CONTENT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("continue", CONTINUE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("conversion", CONVERSION_P, UNRESERVED_KEYWORD) -PG_KEYWORD("coordinator", COORDINATOR, UNRESERVED_KEYWORD) /* GPDB */ -PG_KEYWORD("copy", COPY, UNRESERVED_KEYWORD) -PG_KEYWORD("cost", COST, UNRESERVED_KEYWORD) -PG_KEYWORD("cpu_rate_limit", CPU_RATE_LIMIT, UNRESERVED_KEYWORD) -PG_KEYWORD("cpuset", CPUSET, UNRESERVED_KEYWORD) -PG_KEYWORD("create", CREATE, RESERVED_KEYWORD) -PG_KEYWORD("createexttable", CREATEEXTTABLE, UNRESERVED_KEYWORD) -PG_KEYWORD("cross", CROSS, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("csv", CSV, UNRESERVED_KEYWORD) -PG_KEYWORD("cube", CUBE, UNRESERVED_KEYWORD) -PG_KEYWORD("current", CURRENT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("current_catalog", CURRENT_CATALOG, RESERVED_KEYWORD) -PG_KEYWORD("current_date", CURRENT_DATE, RESERVED_KEYWORD) -PG_KEYWORD("current_role", CURRENT_ROLE, RESERVED_KEYWORD) -PG_KEYWORD("current_schema", CURRENT_SCHEMA, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("current_time", CURRENT_TIME, RESERVED_KEYWORD) -PG_KEYWORD("current_timestamp", CURRENT_TIMESTAMP, RESERVED_KEYWORD) -PG_KEYWORD("current_user", CURRENT_USER, RESERVED_KEYWORD) -PG_KEYWORD("cursor", CURSOR, UNRESERVED_KEYWORD) -PG_KEYWORD("cycle", CYCLE, UNRESERVED_KEYWORD) -PG_KEYWORD("data", DATA_P, UNRESERVED_KEYWORD) -PG_KEYWORD("database", DATABASE, UNRESERVED_KEYWORD) -PG_KEYWORD("day", DAY_P, UNRESERVED_KEYWORD) -PG_KEYWORD("deallocate", DEALLOCATE, UNRESERVED_KEYWORD) -PG_KEYWORD("dec", DEC, COL_NAME_KEYWORD) -PG_KEYWORD("decimal", DECIMAL_P, COL_NAME_KEYWORD) -PG_KEYWORD("declare", DECLARE, UNRESERVED_KEYWORD) -PG_KEYWORD("decode", DECODE, RESERVED_KEYWORD) -PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD) -PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD) -PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD) -PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD) -PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD) -PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD) -PG_KEYWORD("delimiters", DELIMITERS, UNRESERVED_KEYWORD) -PG_KEYWORD("deny", DENY, UNRESERVED_KEYWORD) -PG_KEYWORD("depends", DEPENDS, UNRESERVED_KEYWORD) -PG_KEYWORD("desc", DESC, RESERVED_KEYWORD) -PG_KEYWORD("detach", DETACH, UNRESERVED_KEYWORD) -PG_KEYWORD("dictionary", DICTIONARY, UNRESERVED_KEYWORD) -PG_KEYWORD("disable", DISABLE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("discard", DISCARD, UNRESERVED_KEYWORD) -PG_KEYWORD("distinct", DISTINCT, RESERVED_KEYWORD) -PG_KEYWORD("distributed", DISTRIBUTED, RESERVED_KEYWORD) -PG_KEYWORD("do", DO, RESERVED_KEYWORD) -PG_KEYWORD("document", DOCUMENT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("domain", DOMAIN_P, UNRESERVED_KEYWORD) -PG_KEYWORD("double", DOUBLE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("drop", DROP, UNRESERVED_KEYWORD) -PG_KEYWORD("dxl", DXL, UNRESERVED_KEYWORD) -PG_KEYWORD("each", EACH, UNRESERVED_KEYWORD) -PG_KEYWORD("else", ELSE, RESERVED_KEYWORD) -PG_KEYWORD("enable", ENABLE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("encoding", ENCODING, UNRESERVED_KEYWORD) -PG_KEYWORD("encrypted", ENCRYPTED, UNRESERVED_KEYWORD) -PG_KEYWORD("end", END_P, RESERVED_KEYWORD) -PG_KEYWORD("endpoint", ENDPOINT, UNRESERVED_KEYWORD) -PG_KEYWORD("enum", ENUM_P, UNRESERVED_KEYWORD) -PG_KEYWORD("errors", ERRORS, UNRESERVED_KEYWORD) -PG_KEYWORD("escape", ESCAPE, UNRESERVED_KEYWORD) -PG_KEYWORD("event", EVENT, UNRESERVED_KEYWORD) -PG_KEYWORD("every", EVERY, UNRESERVED_KEYWORD) /* Reserved in standard */ -PG_KEYWORD("except", EXCEPT, RESERVED_KEYWORD) -PG_KEYWORD("exchange", EXCHANGE, UNRESERVED_KEYWORD) /* GPDB */ -PG_KEYWORD("exclude", EXCLUDE, RESERVED_KEYWORD) -PG_KEYWORD("excluding", EXCLUDING, UNRESERVED_KEYWORD) -PG_KEYWORD("exclusive", EXCLUSIVE, UNRESERVED_KEYWORD) -PG_KEYWORD("execute", EXECUTE, UNRESERVED_KEYWORD) -PG_KEYWORD("exists", EXISTS, COL_NAME_KEYWORD) -PG_KEYWORD("expand", EXPAND, UNRESERVED_KEYWORD) /* GPDB */ -PG_KEYWORD("explain", EXPLAIN, UNRESERVED_KEYWORD) -PG_KEYWORD("expression", EXPRESSION, UNRESERVED_KEYWORD) -PG_KEYWORD("extension", EXTENSION, UNRESERVED_KEYWORD) -PG_KEYWORD("external", EXTERNAL, UNRESERVED_KEYWORD) -PG_KEYWORD("extract", EXTRACT, COL_NAME_KEYWORD) -PG_KEYWORD("false", FALSE_P, RESERVED_KEYWORD) -PG_KEYWORD("family", FAMILY, UNRESERVED_KEYWORD) -PG_KEYWORD("fetch", FETCH, RESERVED_KEYWORD) -PG_KEYWORD("fields", FIELDS, UNRESERVED_KEYWORD) -PG_KEYWORD("fill", FILL, UNRESERVED_KEYWORD) -PG_KEYWORD("filter", FILTER, UNRESERVED_KEYWORD) -PG_KEYWORD("first", FIRST_P, UNRESERVED_KEYWORD) -PG_KEYWORD("float", FLOAT_P, COL_NAME_KEYWORD) -PG_KEYWORD("following", FOLLOWING, RESERVED_KEYWORD) /* Unreserved in standard */ -PG_KEYWORD("for", FOR, RESERVED_KEYWORD) -PG_KEYWORD("force", FORCE, UNRESERVED_KEYWORD) -PG_KEYWORD("foreign", FOREIGN, RESERVED_KEYWORD) -PG_KEYWORD("format", FORMAT, UNRESERVED_KEYWORD) -PG_KEYWORD("forward", FORWARD, UNRESERVED_KEYWORD) -PG_KEYWORD("freeze", FREEZE, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("from", FROM, RESERVED_KEYWORD) -PG_KEYWORD("full", FULL, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("fullscan", FULLSCAN, UNRESERVED_KEYWORD) -PG_KEYWORD("function", FUNCTION, UNRESERVED_KEYWORD) -PG_KEYWORD("functions", FUNCTIONS, UNRESERVED_KEYWORD) -PG_KEYWORD("generated", GENERATED, UNRESERVED_KEYWORD) -PG_KEYWORD("global", GLOBAL, UNRESERVED_KEYWORD) -PG_KEYWORD("grant", GRANT, RESERVED_KEYWORD) -PG_KEYWORD("granted", GRANTED, UNRESERVED_KEYWORD) -PG_KEYWORD("greatest", GREATEST, COL_NAME_KEYWORD) -PG_KEYWORD("group", GROUP_P, RESERVED_KEYWORD) -PG_KEYWORD("group_id", GROUP_ID, COL_NAME_KEYWORD) -PG_KEYWORD("grouping", GROUPING, COL_NAME_KEYWORD) -PG_KEYWORD("groups", GROUPS, UNRESERVED_KEYWORD) -PG_KEYWORD("handler", HANDLER, UNRESERVED_KEYWORD) -PG_KEYWORD("hash", HASH, UNRESERVED_KEYWORD) -PG_KEYWORD("having", HAVING, RESERVED_KEYWORD) -PG_KEYWORD("header", HEADER_P, UNRESERVED_KEYWORD) -PG_KEYWORD("hold", HOLD, UNRESERVED_KEYWORD) -PG_KEYWORD("host", HOST, UNRESERVED_KEYWORD) /* GPDB */ -PG_KEYWORD("hour", HOUR_P, UNRESERVED_KEYWORD) -PG_KEYWORD("identity", IDENTITY_P, UNRESERVED_KEYWORD) -PG_KEYWORD("if", IF_P, UNRESERVED_KEYWORD) -PG_KEYWORD("ignore", IGNORE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("ilike", ILIKE, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("immediate", IMMEDIATE, UNRESERVED_KEYWORD) -PG_KEYWORD("immutable", IMMUTABLE, UNRESERVED_KEYWORD) -PG_KEYWORD("implicit", IMPLICIT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("import", IMPORT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("in", IN_P, RESERVED_KEYWORD) -PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD) -PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD) -PG_KEYWORD("inclusive", INCLUSIVE, UNRESERVED_KEYWORD) /* GPDB */ -PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD) -PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD) -PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD) -PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD) -PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD) -PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD) -PG_KEYWORD("initplan", INITPLAN, UNRESERVED_KEYWORD) /* GPDB */ -PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("inout", INOUT, COL_NAME_KEYWORD) -PG_KEYWORD("input", INPUT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("insensitive", INSENSITIVE, UNRESERVED_KEYWORD) -PG_KEYWORD("insert", INSERT, UNRESERVED_KEYWORD) -PG_KEYWORD("instead", INSTEAD, UNRESERVED_KEYWORD) -PG_KEYWORD("int", INT_P, COL_NAME_KEYWORD) -PG_KEYWORD("integer", INTEGER, COL_NAME_KEYWORD) -PG_KEYWORD("intersect", INTERSECT, RESERVED_KEYWORD) -PG_KEYWORD("interval", INTERVAL, COL_NAME_KEYWORD) -PG_KEYWORD("into", INTO, RESERVED_KEYWORD) -PG_KEYWORD("invoker", INVOKER, UNRESERVED_KEYWORD) -PG_KEYWORD("is", IS, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("isnull", ISNULL, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("isolation", ISOLATION, UNRESERVED_KEYWORD) -PG_KEYWORD("join", JOIN, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("key", KEY, UNRESERVED_KEYWORD) -PG_KEYWORD("label", LABEL, UNRESERVED_KEYWORD) -PG_KEYWORD("language", LANGUAGE, UNRESERVED_KEYWORD) -PG_KEYWORD("large", LARGE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("last", LAST_P, UNRESERVED_KEYWORD) -PG_KEYWORD("lateral", LATERAL_P, RESERVED_KEYWORD) -PG_KEYWORD("leading", LEADING, RESERVED_KEYWORD) -PG_KEYWORD("leakproof", LEAKPROOF, UNRESERVED_KEYWORD) -PG_KEYWORD("least", LEAST, COL_NAME_KEYWORD) -PG_KEYWORD("left", LEFT, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("level", LEVEL, UNRESERVED_KEYWORD) -PG_KEYWORD("like", LIKE, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("limit", LIMIT, RESERVED_KEYWORD) -PG_KEYWORD("list", LIST, UNRESERVED_KEYWORD) -PG_KEYWORD("listen", LISTEN, UNRESERVED_KEYWORD) -PG_KEYWORD("load", LOAD, UNRESERVED_KEYWORD) -PG_KEYWORD("local", LOCAL, UNRESERVED_KEYWORD) /* Reserved in standard */ -PG_KEYWORD("localtime", LOCALTIME, RESERVED_KEYWORD) -PG_KEYWORD("localtimestamp", LOCALTIMESTAMP, RESERVED_KEYWORD) -PG_KEYWORD("location", LOCATION, UNRESERVED_KEYWORD) -PG_KEYWORD("lock", LOCK_P, UNRESERVED_KEYWORD) -PG_KEYWORD("locked", LOCKED, UNRESERVED_KEYWORD) -PG_KEYWORD("log", LOG_P, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("logged", LOGGED, UNRESERVED_KEYWORD) -PG_KEYWORD("mapping", MAPPING, UNRESERVED_KEYWORD) -PG_KEYWORD("master", MASTER, UNRESERVED_KEYWORD) /* GPDB */ -PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD) -PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD) -PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD) -PG_KEYWORD("median", MEDIAN, COL_NAME_KEYWORD) -PG_KEYWORD("memory_limit", MEMORY_LIMIT, UNRESERVED_KEYWORD) -PG_KEYWORD("memory_shared_quota", MEMORY_SHARED_QUOTA, UNRESERVED_KEYWORD) -PG_KEYWORD("memory_spill_ratio", MEMORY_SPILL_RATIO, UNRESERVED_KEYWORD) -PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD) -PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("minvalue", MINVALUE, UNRESERVED_KEYWORD) -PG_KEYWORD("missing", MISSING, UNRESERVED_KEYWORD) -PG_KEYWORD("mode", MODE, UNRESERVED_KEYWORD) -PG_KEYWORD("modifies", MODIFIES, UNRESERVED_KEYWORD) -PG_KEYWORD("month", MONTH_P, UNRESERVED_KEYWORD) -PG_KEYWORD("move", MOVE, UNRESERVED_KEYWORD) -PG_KEYWORD("name", NAME_P, UNRESERVED_KEYWORD) -PG_KEYWORD("names", NAMES, UNRESERVED_KEYWORD) -PG_KEYWORD("national", NATIONAL, COL_NAME_KEYWORD) -PG_KEYWORD("natural", NATURAL, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("nchar", NCHAR, COL_NAME_KEYWORD) -PG_KEYWORD("new", NEW, UNRESERVED_KEYWORD) -PG_KEYWORD("newline", NEWLINE, UNRESERVED_KEYWORD) -PG_KEYWORD("next", NEXT, UNRESERVED_KEYWORD) -PG_KEYWORD("nfc", NFC, UNRESERVED_KEYWORD) -PG_KEYWORD("nfd", NFD, UNRESERVED_KEYWORD) -PG_KEYWORD("nfkc", NFKC, UNRESERVED_KEYWORD) -PG_KEYWORD("nfkd", NFKD, UNRESERVED_KEYWORD) -PG_KEYWORD("no", NO, UNRESERVED_KEYWORD) -PG_KEYWORD("nocreateexttable", NOCREATEEXTTABLE, UNRESERVED_KEYWORD) -PG_KEYWORD("none", NONE, COL_NAME_KEYWORD) -PG_KEYWORD("noovercommit", NOOVERCOMMIT, UNRESERVED_KEYWORD) -PG_KEYWORD("normalize", NORMALIZE, COL_NAME_KEYWORD) -PG_KEYWORD("normalized", NORMALIZED, UNRESERVED_KEYWORD) -PG_KEYWORD("not", NOT, RESERVED_KEYWORD) -PG_KEYWORD("nothing", NOTHING, UNRESERVED_KEYWORD) -PG_KEYWORD("notify", NOTIFY, UNRESERVED_KEYWORD) -PG_KEYWORD("notnull", NOTNULL, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("nowait", NOWAIT, UNRESERVED_KEYWORD) -PG_KEYWORD("null", NULL_P, RESERVED_KEYWORD) -PG_KEYWORD("nullif", NULLIF, COL_NAME_KEYWORD) -PG_KEYWORD("nulls", NULLS_P, UNRESERVED_KEYWORD) -PG_KEYWORD("numeric", NUMERIC, COL_NAME_KEYWORD) -PG_KEYWORD("object", OBJECT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("of", OF, UNRESERVED_KEYWORD) -PG_KEYWORD("off", OFF, UNRESERVED_KEYWORD) -PG_KEYWORD("offset", OFFSET, RESERVED_KEYWORD) -PG_KEYWORD("oids", OIDS, UNRESERVED_KEYWORD) -PG_KEYWORD("old", OLD, UNRESERVED_KEYWORD) -PG_KEYWORD("on", ON, RESERVED_KEYWORD) -PG_KEYWORD("only", ONLY, RESERVED_KEYWORD) -PG_KEYWORD("operator", OPERATOR, UNRESERVED_KEYWORD) -PG_KEYWORD("option", OPTION, UNRESERVED_KEYWORD) -PG_KEYWORD("options", OPTIONS, UNRESERVED_KEYWORD) -PG_KEYWORD("or", OR, RESERVED_KEYWORD) -PG_KEYWORD("order", ORDER, RESERVED_KEYWORD) -PG_KEYWORD("ordered", ORDERED, UNRESERVED_KEYWORD) -PG_KEYWORD("ordinality", ORDINALITY, UNRESERVED_KEYWORD) -PG_KEYWORD("others", OTHERS, UNRESERVED_KEYWORD) -PG_KEYWORD("out", OUT_P, COL_NAME_KEYWORD) -PG_KEYWORD("outer", OUTER_P, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("over", OVER, UNRESERVED_KEYWORD) -PG_KEYWORD("overcommit", OVERCOMMIT, UNRESERVED_KEYWORD) -PG_KEYWORD("overlaps", OVERLAPS, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("overlay", OVERLAY, COL_NAME_KEYWORD) -PG_KEYWORD("overriding", OVERRIDING, UNRESERVED_KEYWORD) -PG_KEYWORD("owned", OWNED, UNRESERVED_KEYWORD) -PG_KEYWORD("owner", OWNER, UNRESERVED_KEYWORD) -PG_KEYWORD("parallel", PARALLEL, UNRESERVED_KEYWORD) -PG_KEYWORD("parser", PARSER, UNRESERVED_KEYWORD) -PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD) -PG_KEYWORD("partition", PARTITION, RESERVED_KEYWORD) /* GPDB */ -PG_KEYWORD("partitions", PARTITIONS, UNRESERVED_KEYWORD) /* GPDB */ -PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD) -PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD) -PG_KEYWORD("percent", PERCENT, UNRESERVED_KEYWORD) -PG_KEYWORD("persistently", PERSISTENTLY, UNRESERVED_KEYWORD) /* GPDB */ -PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD) -PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD) -PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD) -PG_KEYWORD("position", POSITION, COL_NAME_KEYWORD) -PG_KEYWORD("preceding", PRECEDING, RESERVED_KEYWORD) /* unreserved in standard */ -PG_KEYWORD("precision", PRECISION, COL_NAME_KEYWORD) -PG_KEYWORD("prepare", PREPARE, UNRESERVED_KEYWORD) -PG_KEYWORD("prepared", PREPARED, UNRESERVED_KEYWORD) -PG_KEYWORD("preserve", PRESERVE, UNRESERVED_KEYWORD) -PG_KEYWORD("primary", PRIMARY, RESERVED_KEYWORD) -PG_KEYWORD("prior", PRIOR, UNRESERVED_KEYWORD) -PG_KEYWORD("privileges", PRIVILEGES, UNRESERVED_KEYWORD) -PG_KEYWORD("procedural", PROCEDURAL, UNRESERVED_KEYWORD) -PG_KEYWORD("procedure", PROCEDURE, UNRESERVED_KEYWORD) -PG_KEYWORD("procedures", PROCEDURES, UNRESERVED_KEYWORD) -PG_KEYWORD("program", PROGRAM, UNRESERVED_KEYWORD) -PG_KEYWORD("protocol", PROTOCOL, UNRESERVED_KEYWORD) -PG_KEYWORD("publication", PUBLICATION, UNRESERVED_KEYWORD) -PG_KEYWORD("queue", QUEUE, UNRESERVED_KEYWORD) -PG_KEYWORD("quote", QUOTE, UNRESERVED_KEYWORD) -PG_KEYWORD("randomly", RANDOMLY, UNRESERVED_KEYWORD) /* GPDB */ -PG_KEYWORD("range", RANGE, UNRESERVED_KEYWORD) -PG_KEYWORD("read", READ, UNRESERVED_KEYWORD) -PG_KEYWORD("readable", READABLE, UNRESERVED_KEYWORD) /* GPDB */ -PG_KEYWORD("reads", READS, UNRESERVED_KEYWORD) -PG_KEYWORD("real", REAL, COL_NAME_KEYWORD) -PG_KEYWORD("reassign", REASSIGN, UNRESERVED_KEYWORD) -PG_KEYWORD("recheck", RECHECK, UNRESERVED_KEYWORD) -PG_KEYWORD("recursive", RECURSIVE, UNRESERVED_KEYWORD) -PG_KEYWORD("ref", REF, UNRESERVED_KEYWORD) -PG_KEYWORD("references", REFERENCES, RESERVED_KEYWORD) -PG_KEYWORD("referencing", REFERENCING, UNRESERVED_KEYWORD) -PG_KEYWORD("refresh", REFRESH, UNRESERVED_KEYWORD) -PG_KEYWORD("reindex", REINDEX, UNRESERVED_KEYWORD) -PG_KEYWORD("reject", REJECT_P, UNRESERVED_KEYWORD) /* GPDB */ -PG_KEYWORD("relative", RELATIVE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("release", RELEASE, UNRESERVED_KEYWORD) -PG_KEYWORD("rename", RENAME, UNRESERVED_KEYWORD) -PG_KEYWORD("repeatable", REPEATABLE, UNRESERVED_KEYWORD) -PG_KEYWORD("replace", REPLACE, UNRESERVED_KEYWORD) -PG_KEYWORD("replica", REPLICA, UNRESERVED_KEYWORD) -PG_KEYWORD("replicated", REPLICATED, UNRESERVED_KEYWORD) /* GPDB */ -PG_KEYWORD("reset", RESET, UNRESERVED_KEYWORD) -PG_KEYWORD("resource", RESOURCE, UNRESERVED_KEYWORD) -PG_KEYWORD("restart", RESTART, UNRESERVED_KEYWORD) -PG_KEYWORD("restrict", RESTRICT, UNRESERVED_KEYWORD) -PG_KEYWORD("retrieve", RETRIEVE, UNRESERVED_KEYWORD) -PG_KEYWORD("returning", RETURNING, RESERVED_KEYWORD) -PG_KEYWORD("returns", RETURNS, UNRESERVED_KEYWORD) -PG_KEYWORD("revoke", REVOKE, UNRESERVED_KEYWORD) -PG_KEYWORD("right", RIGHT, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("role", ROLE, UNRESERVED_KEYWORD) -PG_KEYWORD("rollback", ROLLBACK, UNRESERVED_KEYWORD) -PG_KEYWORD("rollup", ROLLUP, UNRESERVED_KEYWORD) -PG_KEYWORD("rootpartition", ROOTPARTITION, UNRESERVED_KEYWORD) -PG_KEYWORD("routine", ROUTINE, UNRESERVED_KEYWORD) -PG_KEYWORD("routines", ROUTINES, UNRESERVED_KEYWORD) -PG_KEYWORD("row", ROW, COL_NAME_KEYWORD) -PG_KEYWORD("rows", ROWS, UNRESERVED_KEYWORD) -PG_KEYWORD("rule", RULE, UNRESERVED_KEYWORD) -PG_KEYWORD("savepoint", SAVEPOINT, UNRESERVED_KEYWORD) -PG_KEYWORD("scatter", SCATTER, RESERVED_KEYWORD) /* GPDB */ -PG_KEYWORD("schema", SCHEMA, UNRESERVED_KEYWORD) -PG_KEYWORD("schemas", SCHEMAS, UNRESERVED_KEYWORD) -PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD) -PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD) -PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD) -PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD) -PG_KEYWORD("segment", SEGMENT, UNRESERVED_KEYWORD) -PG_KEYWORD("segments", SEGMENTS, UNRESERVED_KEYWORD) -PG_KEYWORD("select", SELECT, RESERVED_KEYWORD) -PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD) -PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD) -PG_KEYWORD("serializable", SERIALIZABLE, UNRESERVED_KEYWORD) -PG_KEYWORD("server", SERVER, UNRESERVED_KEYWORD) -PG_KEYWORD("session", SESSION, UNRESERVED_KEYWORD) -PG_KEYWORD("session_user", SESSION_USER, RESERVED_KEYWORD) -PG_KEYWORD("set", SET, UNRESERVED_KEYWORD) -PG_KEYWORD("setof", SETOF, COL_NAME_KEYWORD) -PG_KEYWORD("sets", SETS, UNRESERVED_KEYWORD) -PG_KEYWORD("share", SHARE, UNRESERVED_KEYWORD) -PG_KEYWORD("show", SHOW, UNRESERVED_KEYWORD) -PG_KEYWORD("similar", SIMILAR, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("simple", SIMPLE, UNRESERVED_KEYWORD) -PG_KEYWORD("skip", SKIP, UNRESERVED_KEYWORD) -PG_KEYWORD("smallint", SMALLINT, COL_NAME_KEYWORD) -PG_KEYWORD("snapshot", SNAPSHOT, UNRESERVED_KEYWORD) -PG_KEYWORD("some", SOME, RESERVED_KEYWORD) -PG_KEYWORD("split", SPLIT, UNRESERVED_KEYWORD) /* GPDB */ -PG_KEYWORD("sql", SQL_P, UNRESERVED_KEYWORD) -PG_KEYWORD("stable", STABLE, UNRESERVED_KEYWORD) -PG_KEYWORD("standalone", STANDALONE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("start", START, UNRESERVED_KEYWORD) -PG_KEYWORD("statement", STATEMENT, UNRESERVED_KEYWORD) -PG_KEYWORD("statistics", STATISTICS, UNRESERVED_KEYWORD) -PG_KEYWORD("stdin", STDIN, UNRESERVED_KEYWORD) -PG_KEYWORD("stdout", STDOUT, UNRESERVED_KEYWORD) -PG_KEYWORD("storage", STORAGE, UNRESERVED_KEYWORD) -PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD) -PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD) -PG_KEYWORD("subpartition", SUBPARTITION, UNRESERVED_KEYWORD) /* GPDB */ -PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD) -PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD) -PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD) -PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD) -PG_KEYWORD("sysid", SYSID, UNRESERVED_KEYWORD) -PG_KEYWORD("system", SYSTEM_P, UNRESERVED_KEYWORD) -PG_KEYWORD("table", TABLE, RESERVED_KEYWORD) -PG_KEYWORD("tables", TABLES, UNRESERVED_KEYWORD) -PG_KEYWORD("tablesample", TABLESAMPLE, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("tablespace", TABLESPACE, UNRESERVED_KEYWORD) -PG_KEYWORD("temp", TEMP, UNRESERVED_KEYWORD) -PG_KEYWORD("template", TEMPLATE, UNRESERVED_KEYWORD) -PG_KEYWORD("temporary", TEMPORARY, UNRESERVED_KEYWORD) -PG_KEYWORD("text", TEXT_P, UNRESERVED_KEYWORD) -PG_KEYWORD("then", THEN, RESERVED_KEYWORD) -PG_KEYWORD("threshold", THRESHOLD, UNRESERVED_KEYWORD) /* GPDB */ -PG_KEYWORD("ties", TIES, UNRESERVED_KEYWORD) -PG_KEYWORD("time", TIME, COL_NAME_KEYWORD) -PG_KEYWORD("timestamp", TIMESTAMP, COL_NAME_KEYWORD) -PG_KEYWORD("to", TO, RESERVED_KEYWORD) -PG_KEYWORD("trailing", TRAILING, RESERVED_KEYWORD) -PG_KEYWORD("transaction", TRANSACTION, UNRESERVED_KEYWORD) -PG_KEYWORD("transform", TRANSFORM, UNRESERVED_KEYWORD) -PG_KEYWORD("treat", TREAT, COL_NAME_KEYWORD) -PG_KEYWORD("trigger", TRIGGER, UNRESERVED_KEYWORD) -PG_KEYWORD("trim", TRIM, COL_NAME_KEYWORD) -PG_KEYWORD("true", TRUE_P, RESERVED_KEYWORD) -PG_KEYWORD("truncate", TRUNCATE, UNRESERVED_KEYWORD) -PG_KEYWORD("trusted", TRUSTED, UNRESERVED_KEYWORD) -PG_KEYWORD("type", TYPE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("types", TYPES_P, UNRESERVED_KEYWORD) -PG_KEYWORD("uescape", UESCAPE, UNRESERVED_KEYWORD) -PG_KEYWORD("unbounded", UNBOUNDED, RESERVED_KEYWORD) /* Unreserved in standard */ -PG_KEYWORD("uncommitted", UNCOMMITTED, UNRESERVED_KEYWORD) -PG_KEYWORD("unencrypted", UNENCRYPTED, UNRESERVED_KEYWORD) -PG_KEYWORD("union", UNION, RESERVED_KEYWORD) -PG_KEYWORD("unique", UNIQUE, RESERVED_KEYWORD) -PG_KEYWORD("unknown", UNKNOWN, UNRESERVED_KEYWORD) -PG_KEYWORD("unlisten", UNLISTEN, UNRESERVED_KEYWORD) -PG_KEYWORD("unlogged", UNLOGGED, UNRESERVED_KEYWORD) -PG_KEYWORD("until", UNTIL, UNRESERVED_KEYWORD) -PG_KEYWORD("update", UPDATE, UNRESERVED_KEYWORD) -PG_KEYWORD("user", USER, RESERVED_KEYWORD) -PG_KEYWORD("using", USING, RESERVED_KEYWORD) -PG_KEYWORD("vacuum", VACUUM, UNRESERVED_KEYWORD) -PG_KEYWORD("valid", VALID, UNRESERVED_KEYWORD) -PG_KEYWORD("validate", VALIDATE, UNRESERVED_KEYWORD) -PG_KEYWORD("validation", VALIDATION, UNRESERVED_KEYWORD) -PG_KEYWORD("validator", VALIDATOR, UNRESERVED_KEYWORD) -PG_KEYWORD("value", VALUE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("values", VALUES, COL_NAME_KEYWORD) -PG_KEYWORD("varchar", VARCHAR, COL_NAME_KEYWORD) -PG_KEYWORD("variadic", VARIADIC, RESERVED_KEYWORD) -PG_KEYWORD("varying", VARYING, UNRESERVED_KEYWORD) -PG_KEYWORD("verbose", VERBOSE, TYPE_FUNC_NAME_KEYWORD) -PG_KEYWORD("version", VERSION_P, UNRESERVED_KEYWORD) -PG_KEYWORD("view", VIEW, UNRESERVED_KEYWORD) -PG_KEYWORD("views", VIEWS, UNRESERVED_KEYWORD) -PG_KEYWORD("volatile", VOLATILE, UNRESERVED_KEYWORD) -PG_KEYWORD("web", WEB, UNRESERVED_KEYWORD) -PG_KEYWORD("when", WHEN, RESERVED_KEYWORD) -PG_KEYWORD("where", WHERE, RESERVED_KEYWORD) -PG_KEYWORD("whitespace", WHITESPACE_P, UNRESERVED_KEYWORD) -PG_KEYWORD("window", WINDOW, RESERVED_KEYWORD) -PG_KEYWORD("with", WITH, RESERVED_KEYWORD) -PG_KEYWORD("within", WITHIN, UNRESERVED_KEYWORD) -PG_KEYWORD("without", WITHOUT, UNRESERVED_KEYWORD) -PG_KEYWORD("work", WORK, UNRESERVED_KEYWORD) -PG_KEYWORD("wrapper", WRAPPER, UNRESERVED_KEYWORD) -PG_KEYWORD("writable", WRITABLE, UNRESERVED_KEYWORD) -PG_KEYWORD("write", WRITE, UNRESERVED_KEYWORD) -PG_KEYWORD("xml", XML_P, UNRESERVED_KEYWORD) -PG_KEYWORD("xmlattributes", XMLATTRIBUTES, COL_NAME_KEYWORD) -PG_KEYWORD("xmlconcat", XMLCONCAT, COL_NAME_KEYWORD) -PG_KEYWORD("xmlelement", XMLELEMENT, COL_NAME_KEYWORD) -PG_KEYWORD("xmlexists", XMLEXISTS, COL_NAME_KEYWORD) -PG_KEYWORD("xmlforest", XMLFOREST, COL_NAME_KEYWORD) -PG_KEYWORD("xmlnamespaces", XMLNAMESPACES, COL_NAME_KEYWORD) -PG_KEYWORD("xmlparse", XMLPARSE, COL_NAME_KEYWORD) -PG_KEYWORD("xmlpi", XMLPI, COL_NAME_KEYWORD) -PG_KEYWORD("xmlroot", XMLROOT, COL_NAME_KEYWORD) -PG_KEYWORD("xmlserialize", XMLSERIALIZE, COL_NAME_KEYWORD) -PG_KEYWORD("xmltable", XMLTABLE, COL_NAME_KEYWORD) -PG_KEYWORD("year", YEAR_P, UNRESERVED_KEYWORD) -PG_KEYWORD("yes", YES_P, UNRESERVED_KEYWORD) -PG_KEYWORD("zone", ZONE, UNRESERVED_KEYWORD) -======= /* name, value, category, is-bare-label */ PG_KEYWORD("abort", ABORT_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("absolute", ABSOLUTE_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("access", ACCESS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("action", ACTION, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("active", ACTIVE, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("add", ADD_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("admin", ADMIN, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("after", AFTER, UNRESERVED_KEYWORD, BARE_LABEL) @@ -602,18 +88,24 @@ PG_KEYWORD("comment", COMMENT, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("comments", COMMENTS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("commit", COMMIT, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("committed", COMMITTED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("concurrency", CONCURRENCY, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("concurrently", CONCURRENTLY, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("configuration", CONFIGURATION, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("conflict", CONFLICT, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("connection", CONNECTION, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("constraint", CONSTRAINT, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("constraints", CONSTRAINTS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("contains", CONTAINS, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("content", CONTENT_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("continue", CONTINUE_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("conversion", CONVERSION_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("coordinator", COORDINATOR, UNRESERVED_KEYWORD, AS_LABEL) /* GPDB */ PG_KEYWORD("copy", COPY, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("cost", COST, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("cpu_rate_limit", CPU_RATE_LIMIT, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ +PG_KEYWORD("cpuset", CPUSET, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("create", CREATE, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("createexttable", CREATEEXTTABLE, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("cross", CROSS, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("csv", CSV, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("cube", CUBE, UNRESERVED_KEYWORD, BARE_LABEL) @@ -634,6 +126,7 @@ PG_KEYWORD("deallocate", DEALLOCATE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("dec", DEC, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("decimal", DECIMAL_P, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("declare", DECLARE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("decode", DECODE, RESERVED_KEYWORD, AS_LABEL) /* GPDB */ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL) @@ -642,6 +135,7 @@ PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("delimiters", DELIMITERS, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("deny", DENY, UNRESERVED_KEYWORD, AS_LABEL) /* GPDB */ PG_KEYWORD("depends", DEPENDS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("desc", DESC, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("detach", DETACH, UNRESERVED_KEYWORD, BARE_LABEL) @@ -649,26 +143,33 @@ PG_KEYWORD("dictionary", DICTIONARY, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("disable", DISABLE_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("discard", DISCARD, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("distinct", DISTINCT, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("distributed", DISTRIBUTED, RESERVED_KEYWORD, AS_LABEL) /* GPDB */ PG_KEYWORD("do", DO, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("document", DOCUMENT_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("domain", DOMAIN_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("double", DOUBLE_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("drop", DROP, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("dxl", DXL, UNRESERVED_KEYWORD, AS_LABEL) /* GPDB */ PG_KEYWORD("each", EACH, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("else", ELSE, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("enable", ENABLE_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("encoding", ENCODING, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("encrypted", ENCRYPTED, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("end", END_P, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("endpoint", ENDPOINT, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("enum", ENUM_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("errors", ERRORS, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("escape", ESCAPE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("event", EVENT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("every", EVERY, UNRESERVED_KEYWORD, AS_LABEL) /* Reserved in standard */ PG_KEYWORD("except", EXCEPT, RESERVED_KEYWORD, AS_LABEL) -PG_KEYWORD("exclude", EXCLUDE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("exchange", EXCHANGE, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ +PG_KEYWORD("exclude", EXCLUDE, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("excluding", EXCLUDING, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("exclusive", EXCLUSIVE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("execute", EXECUTE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("exists", EXISTS, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("expand", EXPAND, UNRESERVED_KEYWORD, AS_LABEL) /* GPDB */ PG_KEYWORD("explain", EXPLAIN, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("expression", EXPRESSION, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("extension", EXTENSION, UNRESERVED_KEYWORD, BARE_LABEL) @@ -677,17 +178,21 @@ PG_KEYWORD("extract", EXTRACT, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("false", FALSE_P, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("family", FAMILY, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("fetch", FETCH, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("fields", FIELDS, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ +PG_KEYWORD("fill", FILL, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("filter", FILTER, UNRESERVED_KEYWORD, AS_LABEL) PG_KEYWORD("first", FIRST_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("float", FLOAT_P, COL_NAME_KEYWORD, BARE_LABEL) -PG_KEYWORD("following", FOLLOWING, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("following", FOLLOWING, RESERVED_KEYWORD, BARE_LABEL) /* Unreserved in standard */ PG_KEYWORD("for", FOR, RESERVED_KEYWORD, AS_LABEL) PG_KEYWORD("force", FORCE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("foreign", FOREIGN, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("format", FORMAT, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("forward", FORWARD, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("freeze", FREEZE, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("from", FROM, RESERVED_KEYWORD, AS_LABEL) PG_KEYWORD("full", FULL, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("fullscan", FULLSCAN, UNRESERVED_KEYWORD, AS_LABEL) /* GPDB */ PG_KEYWORD("function", FUNCTION, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("functions", FUNCTIONS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("generated", GENERATED, UNRESERVED_KEYWORD, BARE_LABEL) @@ -696,15 +201,19 @@ PG_KEYWORD("grant", GRANT, RESERVED_KEYWORD, AS_LABEL) PG_KEYWORD("granted", GRANTED, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("greatest", GREATEST, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("group", GROUP_P, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("group_id", GROUP_ID, COL_NAME_KEYWORD, AS_LABEL) /* GPDB */ PG_KEYWORD("grouping", GROUPING, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("groups", GROUPS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("handler", HANDLER, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("hash", HASH, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("having", HAVING, RESERVED_KEYWORD, AS_LABEL) PG_KEYWORD("header", HEADER_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("hold", HOLD, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("host", HOST, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("hour", HOUR_P, UNRESERVED_KEYWORD, AS_LABEL) PG_KEYWORD("identity", IDENTITY_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("if", IF_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("ignore", IGNORE_P, UNRESERVED_KEYWORD, AS_LABEL) PG_KEYWORD("ilike", ILIKE, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("immediate", IMMEDIATE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("immutable", IMMUTABLE, UNRESERVED_KEYWORD, BARE_LABEL) @@ -713,12 +222,14 @@ PG_KEYWORD("import", IMPORT_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("inclusive", INCLUSIVE, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("initplan", INITPLAN, UNRESERVED_KEYWORD, AS_LABEL) /* GPDB */ PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("inout", INOUT, COL_NAME_KEYWORD, BARE_LABEL) @@ -749,23 +260,32 @@ PG_KEYWORD("left", LEFT, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("level", LEVEL, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("like", LIKE, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("limit", LIMIT, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("list", LIST, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("listen", LISTEN, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("load", LOAD, UNRESERVED_KEYWORD, BARE_LABEL) -PG_KEYWORD("local", LOCAL, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("local", LOCAL, UNRESERVED_KEYWORD, BARE_LABEL) /* Reserved in standard */ PG_KEYWORD("localtime", LOCALTIME, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("localtimestamp", LOCALTIMESTAMP, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("location", LOCATION, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("lock", LOCK_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("locked", LOCKED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("log", LOG_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("logged", LOGGED, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("mapping", MAPPING, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("master", MASTER, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("median", MEDIAN, COL_NAME_KEYWORD, BARE_LABEL) /* GPDB */ +PG_KEYWORD("memory_limit", MEMORY_LIMIT, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ +PG_KEYWORD("memory_shared_quota", MEMORY_SHARED_QUOTA, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ +PG_KEYWORD("memory_spill_ratio", MEMORY_SPILL_RATIO, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD, AS_LABEL) PG_KEYWORD("minvalue", MINVALUE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("missing", MISSING, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("mode", MODE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("modifies", MODIFIES, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("month", MONTH_P, UNRESERVED_KEYWORD, AS_LABEL) PG_KEYWORD("move", MOVE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("name", NAME_P, UNRESERVED_KEYWORD, BARE_LABEL) @@ -774,13 +294,16 @@ PG_KEYWORD("national", NATIONAL, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("natural", NATURAL, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("nchar", NCHAR, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("new", NEW, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("newline", NEWLINE, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("next", NEXT, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("nfc", NFC, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("nfd", NFD, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("nfkc", NFKC, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("nfkd", NFKD, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("no", NO, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("nocreateexttable", NOCREATEEXTTABLE, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("none", NONE, COL_NAME_KEYWORD, BARE_LABEL) +PG_KEYWORD("noovercommit", NOOVERCOMMIT, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("normalize", NORMALIZE, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("normalized", NORMALIZED, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("not", NOT, RESERVED_KEYWORD, BARE_LABEL) @@ -805,11 +328,13 @@ PG_KEYWORD("option", OPTION, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("options", OPTIONS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("or", OR, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("order", ORDER, RESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("ordered", ORDERED, UNRESERVED_KEYWORD, AS_LABEL) /* GPDB */ PG_KEYWORD("ordinality", ORDINALITY, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("others", OTHERS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("out", OUT_P, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("outer", OUTER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("over", OVER, UNRESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("overcommit", OVERCOMMIT, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("overlaps", OVERLAPS, TYPE_FUNC_NAME_KEYWORD, AS_LABEL) PG_KEYWORD("overlay", OVERLAY, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("overriding", OVERRIDING, UNRESERVED_KEYWORD, BARE_LABEL) @@ -818,14 +343,17 @@ PG_KEYWORD("owner", OWNER, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("parallel", PARALLEL, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("parser", PARSER, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL) -PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("partition", PARTITION, RESERVED_KEYWORD, AS_LABEL) /* GPDB */ +PG_KEYWORD("partitions", PARTITIONS, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("percent", PERCENT, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ +PG_KEYWORD("persistently", PERSISTENTLY, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("position", POSITION, COL_NAME_KEYWORD, BARE_LABEL) -PG_KEYWORD("preceding", PRECEDING, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("preceding", PRECEDING, RESERVED_KEYWORD, BARE_LABEL) /* unreserved in standard */ PG_KEYWORD("precision", PRECISION, COL_NAME_KEYWORD, AS_LABEL) PG_KEYWORD("prepare", PREPARE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("prepared", PREPARED, UNRESERVED_KEYWORD, BARE_LABEL) @@ -837,10 +365,15 @@ PG_KEYWORD("procedural", PROCEDURAL, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("procedure", PROCEDURE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("procedures", PROCEDURES, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("program", PROGRAM, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("protocol", PROTOCOL, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("publication", PUBLICATION, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("queue", QUEUE, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("quote", QUOTE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("randomly", RANDOMLY, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("range", RANGE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("read", READ, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("readable", READABLE, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ +PG_KEYWORD("reads", READS, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("real", REAL, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("reassign", REASSIGN, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("recheck", RECHECK, UNRESERVED_KEYWORD, BARE_LABEL) @@ -850,15 +383,19 @@ PG_KEYWORD("references", REFERENCES, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("referencing", REFERENCING, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("refresh", REFRESH, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("reindex", REINDEX, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("reject", REJECT_P, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("relative", RELATIVE_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("release", RELEASE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("rename", RENAME, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("repeatable", REPEATABLE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("replace", REPLACE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("replica", REPLICA, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("replicated", REPLICATED, UNRESERVED_KEYWORD, AS_LABEL) /* GPDB */ PG_KEYWORD("reset", RESET, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("resource", RESOURCE, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("restart", RESTART, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("restrict", RESTRICT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("retrieve", RETRIEVE, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("returning", RETURNING, RESERVED_KEYWORD, AS_LABEL) PG_KEYWORD("returns", RETURNS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("revoke", REVOKE, UNRESERVED_KEYWORD, BARE_LABEL) @@ -866,18 +403,22 @@ PG_KEYWORD("right", RIGHT, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("role", ROLE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("rollback", ROLLBACK, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("rollup", ROLLUP, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("rootpartition", ROOTPARTITION, UNRESERVED_KEYWORD, AS_LABEL) /* GPDB */ PG_KEYWORD("routine", ROUTINE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("routines", ROUTINES, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("row", ROW, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("rows", ROWS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("rule", RULE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("savepoint", SAVEPOINT, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("scatter", SCATTER, RESERVED_KEYWORD, AS_LABEL) /* GPDB */ PG_KEYWORD("schema", SCHEMA, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("schemas", SCHEMAS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL) PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("segment", SEGMENT, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ +PG_KEYWORD("segments", SEGMENTS, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL) @@ -896,6 +437,7 @@ PG_KEYWORD("skip", SKIP, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("smallint", SMALLINT, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("snapshot", SNAPSHOT, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("some", SOME, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("split", SPLIT, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("sql", SQL_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("stable", STABLE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("standalone", STANDALONE_P, UNRESERVED_KEYWORD, BARE_LABEL) @@ -908,6 +450,7 @@ PG_KEYWORD("storage", STORAGE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("subpartition", SUBPARTITION, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL) @@ -923,6 +466,7 @@ PG_KEYWORD("template", TEMPLATE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("temporary", TEMPORARY, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("text", TEXT_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("then", THEN, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("threshold", THRESHOLD, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("ties", TIES, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("time", TIME, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("timestamp", TIMESTAMP, COL_NAME_KEYWORD, BARE_LABEL) @@ -939,7 +483,7 @@ PG_KEYWORD("trusted", TRUSTED, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("type", TYPE_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("types", TYPES_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("uescape", UESCAPE, UNRESERVED_KEYWORD, BARE_LABEL) -PG_KEYWORD("unbounded", UNBOUNDED, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("unbounded", UNBOUNDED, RESERVED_KEYWORD, BARE_LABEL) /* Unreserved in standard */ PG_KEYWORD("uncommitted", UNCOMMITTED, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("unencrypted", UNENCRYPTED, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("union", UNION, RESERVED_KEYWORD, AS_LABEL) @@ -954,6 +498,7 @@ PG_KEYWORD("using", USING, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("vacuum", VACUUM, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("valid", VALID, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("validate", VALIDATE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("validation", VALIDATION, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("validator", VALIDATOR, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("value", VALUE_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("values", VALUES, COL_NAME_KEYWORD, BARE_LABEL) @@ -965,6 +510,7 @@ PG_KEYWORD("version", VERSION_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("view", VIEW, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("views", VIEWS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("volatile", VOLATILE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("web", WEB, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("when", WHEN, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("where", WHERE, RESERVED_KEYWORD, AS_LABEL) PG_KEYWORD("whitespace", WHITESPACE_P, UNRESERVED_KEYWORD, BARE_LABEL) @@ -974,6 +520,7 @@ PG_KEYWORD("within", WITHIN, UNRESERVED_KEYWORD, AS_LABEL) PG_KEYWORD("without", WITHOUT, UNRESERVED_KEYWORD, AS_LABEL) PG_KEYWORD("work", WORK, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("wrapper", WRAPPER, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("writable", WRITABLE, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("write", WRITE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("xml", XML_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("xmlattributes", XMLATTRIBUTES, COL_NAME_KEYWORD, BARE_LABEL) @@ -990,4 +537,3 @@ PG_KEYWORD("xmltable", XMLTABLE, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("year", YEAR_P, UNRESERVED_KEYWORD, AS_LABEL) PG_KEYWORD("yes", YES_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("zone", ZONE, UNRESERVED_KEYWORD, BARE_LABEL) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d From 8bddffafee1f57d4a360b4473ab6bf27a660ae91 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Fri, 29 May 2026 14:44:59 +0300 Subject: [PATCH 514/589] Resolve conflicts in src/backend/tcop/utility.c (#2562) Commit 19f5a37b9fc48a12c77edafb732543875da2f4a3 simplified code by replacing type casts with cstmt, however the code nearby was modified by GPDB-specific commits like c7649f182fda1974b15e6572358aa67583e959b2. --- src/backend/tcop/utility.c | 50 ++++++-------------------------------- 1 file changed, 7 insertions(+), 43 deletions(-) diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index f617054c0716..e8dec6cbb16c 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -1311,11 +1311,8 @@ ProcessUtilitySlow(ParseState *pstate, { List *stmts; ListCell *l; -<<<<<<< HEAD List *more_stmts = NIL; -======= RangeVar *table_rv = NULL; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* Run parse analysis ... */ /* @@ -1341,7 +1338,6 @@ ProcessUtilitySlow(ParseState *pstate, if (IsA(stmt, CreateStmt)) { CreateStmt *cstmt = (CreateStmt *) stmt; -<<<<<<< HEAD char relKind = RELKIND_RELATION; Datum toast_options; static char *validnsps[] = HEAP_RELOPT_NAMESPACES; @@ -1359,15 +1355,18 @@ ProcessUtilitySlow(ParseState *pstate, else cstmt->relKind = relKind; + /* Remember transformed RangeVar for LIKE */ + table_rv = cstmt->relation; + /* * GPDB: Don't dispatch it yet, as we haven't * created the toast and other auxiliary tables * yet. */ /* Create the table itself */ - address = DefineRelation((CreateStmt *) stmt, + address = DefineRelation(cstmt, relKind, - ((CreateStmt *) stmt)->ownerid, NULL, + cstmt->ownerid, NULL, queryString, false, true, cstmt->intoPolicy); @@ -1395,19 +1394,6 @@ ProcessUtilitySlow(ParseState *pstate, more_stmts = list_concat(more_stmts, parts); } -======= - Datum toast_options; - static char *validnsps[] = HEAP_RELOPT_NAMESPACES; - - /* Remember transformed RangeVar for LIKE */ - table_rv = cstmt->relation; - - /* Create the table itself */ - address = DefineRelation(cstmt, - RELKIND_RELATION, - InvalidOid, NULL, - queryString); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d EventTriggerCollectSimpleCommand(address, secondaryObject, stmt); @@ -1418,7 +1404,6 @@ ProcessUtilitySlow(ParseState *pstate, */ CommandCounterIncrement(); -<<<<<<< HEAD if (relKind != RELKIND_COMPOSITE_TYPE) { /* @@ -1426,7 +1411,7 @@ ProcessUtilitySlow(ParseState *pstate, * table */ toast_options = transformRelOptions((Datum) 0, - ((CreateStmt *) stmt)->options, + cstmt->options, "toast", validnsps, true, @@ -1434,21 +1419,6 @@ ProcessUtilitySlow(ParseState *pstate, (void) heap_reloptions(RELKIND_TOASTVALUE, toast_options, true); -======= - /* - * parse and validate reloptions for the toast - * table - */ - toast_options = transformRelOptions((Datum) 0, - cstmt->options, - "toast", - validnsps, - true, - false); - (void) heap_reloptions(RELKIND_TOASTVALUE, - toast_options, - true); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d NewRelationCreateToastTable(address.objectId, toast_options); @@ -1493,19 +1463,13 @@ ProcessUtilitySlow(ParseState *pstate, address = DefineRelation(&cstmt->base, RELKIND_FOREIGN_TABLE, InvalidOid, NULL, -<<<<<<< HEAD queryString, true, true, NULL); - CreateForeignTable((CreateForeignTableStmt *) stmt, + CreateForeignTable(cstmt, address.objectId, false /* skip_permission_checks */); -======= - queryString); - CreateForeignTable(cstmt, - address.objectId); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d EventTriggerCollectSimpleCommand(address, secondaryObject, stmt); From abea3f4d81e3e9b61314e567622ebbd10f54c32d Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 29 May 2026 17:16:19 +0500 Subject: [PATCH 515/589] Fix compilation of src/bin/pgbench/pgbench.c (#2632) Commit 9796f45 in src/bin/pgbench/pgbench.c in the initCreateTables function removed the local variable opts, replacing snprintf opts with appendPQExpBuffer query. Replace in GPDB-specific code. --- src/bin/pgbench/pgbench.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 9684cf8bf226..d0cce1d0ba0b 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -3790,9 +3790,7 @@ initCreateTables(PGconn *con) appendPQExpBuffer(&query, " tablespace %s", escape_tablespace); PQfreemem(escape_tablespace); } - snprintf(opts + strlen(opts), sizeof(opts) - strlen(opts), - " distributed by (%s)", - ddl->distributed_col); + appendPQExpBuffer(&query, " distributed by (%s)", ddl->distributed_col); executeStatement(con, query.data); } From a5085e25b371d5409e47352fd63be0dbae0ce7b3 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Fri, 29 May 2026 15:24:11 +0300 Subject: [PATCH 516/589] Resolve conflicts in src/test/Makefile (#2628) Commit 257836a75585934cc05ed7a80bccf8190d41e056 added new test "locale", however test "subscription" was already disabled in GPDB by commit 19cd1cf4b68faff2e29bc2fa884c480e4644cdb4. --- src/test/Makefile | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/test/Makefile b/src/test/Makefile index 08abfc4d02e4..b6201b677781 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -12,16 +12,12 @@ subdir = src/test top_builddir = ../.. include $(top_builddir)/src/Makefile.global -<<<<<<< HEAD # GPDB_12_MERGE_FEATURE_NOT_SUPPORTED Since logical replication doesn't work for GPDB, # removed "subscription" test from below list. -SUBDIRS = perl regress isolation modules authentication recovery +SUBDIRS = perl regress isolation modules authentication recovery \ + locale SUBDIRS += fsync walrep heap_checksum isolation2 fdw -======= -SUBDIRS = perl regress isolation modules authentication recovery subscription \ - locale ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d # Test suites that are not safe by default but can be run if selected # by the user via the whitespace-separated list in variable From b57fc9a94fc61c98c1b4c18eaaf87f67741ab802 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Fri, 29 May 2026 15:24:26 +0300 Subject: [PATCH 517/589] Resolve conflicts in src/test/isolation/Makefile (#2629) Commit 2203ede9ae85b6423f850466122606275ea09b17 added new targets to the Makefile for installing pg_isolation_regress and isolationtester, however GPDB-specific commits have already added several targets for *.pl and *.pm files. --- src/test/isolation/Makefile | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/test/isolation/Makefile b/src/test/isolation/Makefile index ecc58af370ce..53e240734f16 100644 --- a/src/test/isolation/Makefile +++ b/src/test/isolation/Makefile @@ -18,7 +18,6 @@ OBJS = \ all: isolationtester$(X) pg_isolation_regress$(X) -<<<<<<< HEAD gpstringsubs.pl: rm -f $@ && $(LN_S) $(top_builddir)/src/test/regress/gpstringsubs.pl @@ -34,13 +33,6 @@ atmsort.pm: explain.pm: rm -f $@ && $(LN_S) $(top_builddir)/src/test/regress/explain.pm -# Though we don't install these binaries, build them during installation -# (including temp-install). Otherwise, "make -j check-world" and "make -j -# installcheck-world" would spawn multiple, concurrent builds in this -# directory. Later builds would overwrite files while earlier builds are -# reading them, causing occasional failures. -install: | all -======= install: all installdirs $(INSTALL_PROGRAM) pg_isolation_regress$(X) '$(DESTDIR)$(pgxsdir)/$(subdir)/pg_isolation_regress$(X)' $(INSTALL_PROGRAM) isolationtester$(X) '$(DESTDIR)$(pgxsdir)/$(subdir)/isolationtester$(X)' @@ -51,7 +43,6 @@ installdirs: uninstall: rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/pg_isolation_regress$(X)' rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/isolationtester$(X)' ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d submake-regress: $(MAKE) -C $(top_builddir)/src/test/regress pg_regress.o From 0cc755f0a23fa601344051de22dfb716d7332bb8 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Fri, 29 May 2026 15:28:10 +0300 Subject: [PATCH 518/589] Resolve conflicts in src/test/regress/pg_regress.c (#2630) Commit 784b1ba1a2b9306697544bedb2ef9425185dd206 replaced array with StringInfoData, however it was already cherry-picked earlier as commit b4c36a1dc37fffb2d7e795d0f77aaaf035ad3e9f. --- src/test/regress/pg_regress.c | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c index 9b6a87ac574e..80929e7fb1a3 100644 --- a/src/test/regress/pg_regress.c +++ b/src/test/regress/pg_regress.c @@ -845,7 +845,6 @@ convert_sourcefiles_in(const char *source_subdir, const char *dest_dir, const ch FILE *infile, *outfile; StringInfoData line; -<<<<<<< HEAD bool has_tokens = false; struct stat fst; @@ -873,8 +872,6 @@ convert_sourcefiles_in(const char *source_subdir, const char *dest_dir, const ch continue; } -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* reject filenames not finishing in ".source" */ if (strlen(*name) < 8) @@ -909,7 +906,6 @@ convert_sourcefiles_in(const char *source_subdir, const char *dest_dir, const ch while (pg_get_line_buf(infile, &line)) { -<<<<<<< HEAD convert_line(&line, &repls); fputs(line.data, outfile); @@ -919,14 +915,6 @@ convert_sourcefiles_in(const char *source_subdir, const char *dest_dir, const ch */ if (!has_tokens && strstr(line.data, "@gp") != NULL) has_tokens = true; -======= - replace_string(&line, "@abs_srcdir@", inputdir); - replace_string(&line, "@abs_builddir@", outputdir); - replace_string(&line, "@testtablespace@", testtablespace); - replace_string(&line, "@libdir@", dlpath); - replace_string(&line, "@DLSUFFIX@", DLSUFFIX); - fputs(line.data, outfile); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } pfree(line.data); From 57f21bda0a0f761929f18669eb9c8bc12784d0fc Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Mon, 1 Jun 2026 13:45:56 +0500 Subject: [PATCH 519/589] Resolve conflicts in pg_backup.h, pg_backup_db.c, and pg_backup_archiver.c (#2595) 1) Commit a45bc8a in src/bin/pg_dump/pg_backup.h added the definition of the new ConnParams structure, while earlier commit 3040b44 had already done the same, adding the definition of the new _dumpPreparedQueries enumeration above. 2) Commit a45bc8a in src/bin/pg_dump/pg_backup.h changed the arguments in the ConnectDatabase function definition, while earlier commit 19cd1cf had already added the new bool binary_upgrade argument. 3) Commit 257836a added a new field, coll_unknown, to the DumpOptions structure in src/bin/pg_dump/pg_backup.h, while an earlier commit, 77cd915, had already added the GPDB-specific fields, dumpGpPolicy and isGPbackend, in the same location. 4) Commit a45bc8a removed the declaration of the _connectDB function in src/bin/pg_dump/pg_backup_db.c, while an earlier commit, 8ae22d1, had already added pg_attribute_unused to the notice_processor function argument below. 5) Commit a45bc8a in src/bin/pg_dump/pg_backup_db.c refactored the code, including arguments, in the ConnectDatabase function. However, the earlier commit 19cd1cf had already added the new bool binary_upgrade argument and its handling. 6) Commit a45bc8a in src/bin/pg_dump/pg_backup_archiver.c in the RestoreArchive function changed the call to the ConnectDatabase function. However, the earlier commit 19cd1cf had already added the new bool binary_upgrade argument to the ConnectDatabase function. 7) Commit a45bc8a in src/bin/pg_dump/pg_backup_archiver.c changed the call to the ConnectDatabase function in the restore_toc_entries_postfork function, while the earlier commit 19cd1cf had already added a new bool binary_upgrade argument to the ConnectDatabase function. 8) Commit a45bc8a in src/bin/pg_dump/pg_backup_archiver.c simplified the code in the CloneArchive function, while the earlier commit 19cd1cf had already added a new bool binary_upgrade argument to the ConnectDatabase function. --- src/bin/pg_dump/pg_backup.h | 21 ++------- src/bin/pg_dump/pg_backup_archiver.c | 66 ++-------------------------- src/bin/pg_dump/pg_backup_db.c | 50 ++++----------------- 3 files changed, 16 insertions(+), 121 deletions(-) diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 89d6aeba7c8b..0067984ccd7d 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -61,7 +61,6 @@ typedef enum _teSection SECTION_POST_DATA /* stuff to be processed after data */ } teSection; -<<<<<<< HEAD /* We need one enum entry per prepared query in pg_dump */ enum _dumpPreparedQueries { @@ -79,8 +78,6 @@ enum _dumpPreparedQueries NUM_PREP_QUERIES /* must be last */ }; -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* Parameters needed by ConnectDatabase; same for dump and restore */ typedef struct _connParams { @@ -204,14 +201,11 @@ typedef struct _dumpOptions int sequence_data; /* dump sequence data even in schema-only mode */ int do_nothing; -<<<<<<< HEAD + int coll_unknown; /* GPDB */ bool dumpGpPolicy; bool isGPbackend; -======= - int coll_unknown; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } DumpOptions; /* @@ -294,19 +288,10 @@ typedef void (*SetupWorkerPtrType) (Archive *AH); * Main archiver interface. */ -<<<<<<< HEAD -extern void ConnectDatabase(Archive *AH, - const char *dbname, - const char *pghost, - const char *pgport, - const char *username, - trivalue prompt_password, - bool binary_upgrade); -======= extern void ConnectDatabase(Archive *AHX, const ConnParams *cparams, - bool isReconnect); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + bool isReconnect, + bool binary_upgrade); extern void DisconnectDatabase(Archive *AHX); extern PGconn *GetConnection(Archive *AHX); diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index b69f79e3f3c4..e83878b8ac0e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -416,14 +416,7 @@ RestoreArchive(Archive *AHX) AHX->minRemoteVersion = 0; AHX->maxRemoteVersion = 9999999; -<<<<<<< HEAD - ConnectDatabase(AHX, ropt->dbname, - ropt->pghost, ropt->pgport, ropt->username, - ropt->promptPassword, - false); -======= - ConnectDatabase(AHX, &ropt->cparams, false); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + ConnectDatabase(AHX, &ropt->cparams, false, false); /* * If we're talking to the DB directly, don't send comments since they @@ -4241,14 +4234,7 @@ restore_toc_entries_postfork(ArchiveHandle *AH, TocEntry *pending_list) /* * Now reconnect the single parent connection. */ -<<<<<<< HEAD - ConnectDatabase((Archive *) AH, ropt->dbname, - ropt->pghost, ropt->pgport, ropt->username, - ropt->promptPassword, - false); -======= - ConnectDatabase((Archive *) AH, &ropt->cparams, true); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + ConnectDatabase((Archive *) AH, &ropt->cparams, true, false); /* re-establish fixed state */ _doSetFixedOutputState(AH); @@ -4913,57 +4899,13 @@ CloneArchive(ArchiveHandle *AH) * Connect our new clone object to the database, using the same connection * parameters used for the original connection. */ - ConnectDatabase((Archive *) clone, &clone->public.ropt->cparams, true); + ConnectDatabase((Archive *) clone, &clone->public.ropt->cparams, true, + false); /* re-establish fixed state */ if (AH->mode == archModeRead) -<<<<<<< HEAD - { - RestoreOptions *ropt = AH->public.ropt; - - Assert(AH->connection == NULL); - - /* this also sets clone->connection */ - ConnectDatabase((Archive *) clone, ropt->dbname, - ropt->pghost, ropt->pgport, ropt->username, - ropt->promptPassword, false); - - /* re-establish fixed state */ - _doSetFixedOutputState(clone); - } - else - { - PQExpBufferData connstr; - char *pghost; - char *pgport; - char *username; - - Assert(AH->connection != NULL); - - /* - * Even though we are technically accessing the parent's database - * object here, these functions are fine to be called like that - * because all just return a pointer and do not actually send/receive - * any data to/from the database. - */ - initPQExpBuffer(&connstr); - appendPQExpBufferStr(&connstr, "dbname="); - appendConnStrVal(&connstr, PQdb(AH->connection)); - pghost = PQhost(AH->connection); - pgport = PQport(AH->connection); - username = PQuser(AH->connection); - - /* this also sets clone->connection */ - ConnectDatabase((Archive *) clone, connstr.data, - pghost, pgport, username, TRI_NO, false); - - termPQExpBuffer(&connstr); - /* setupDumpWorker will fix up connection state */ - } -======= _doSetFixedOutputState(clone); /* in write case, setupDumpWorker will fix up connection state */ ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* Let the format-specific code have a chance too */ clone->ClonePtr(clone); diff --git a/src/bin/pg_dump/pg_backup_db.c b/src/bin/pg_dump/pg_backup_db.c index 33f8afaf1cd4..4e317af24095 100644 --- a/src/bin/pg_dump/pg_backup_db.c +++ b/src/bin/pg_dump/pg_backup_db.c @@ -27,12 +27,7 @@ #include "pg_backup_utils.h" static void _check_database_version(ArchiveHandle *AH); -<<<<<<< HEAD -static PGconn *_connectDB(ArchiveHandle *AH, const char *newdbname, const char *newUser); static void notice_processor(void *arg pg_attribute_unused(), const char *message); -======= -static void notice_processor(void *arg, const char *message); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d static void _check_database_version(ArchiveHandle *AH) @@ -99,7 +94,7 @@ ReconnectToServer(ArchiveHandle *AH, const char *dbname) */ AH->connection = NULL; /* dodge error check in ConnectDatabase */ - ConnectDatabase((Archive *) AH, &ropt->cparams, true); + ConnectDatabase((Archive *) AH, &ropt->cparams, true, false); PQfinish(oldConn); } @@ -117,17 +112,9 @@ ReconnectToServer(ArchiveHandle *AH, const char *dbname) */ void ConnectDatabase(Archive *AHX, -<<<<<<< HEAD - const char *dbname, - const char *pghost, - const char *pgport, - const char *username, - trivalue prompt_password, - bool binary_upgrade) -======= const ConnParams *cparams, - bool isReconnect) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + bool isReconnect, + bool binary_upgrade) { ArchiveHandle *AH = (ArchiveHandle *) AHX; trivalue prompt_password; @@ -149,28 +136,10 @@ ConnectDatabase(Archive *AHX, * Start the connection. Loop until we have a password if requested by * backend. */ - const char *keywords[8]; - const char *values[8]; + const char *keywords[9]; + const char *values[9]; do { -<<<<<<< HEAD - keywords[0] = "host"; - values[0] = pghost; - keywords[1] = "port"; - values[1] = pgport; - keywords[2] = "user"; - values[2] = username; - keywords[3] = "password"; - values[3] = password; - keywords[4] = "dbname"; - values[4] = dbname; - keywords[5] = "fallback_application_name"; - values[5] = progname; - keywords[6] = NULL; - values[6] = NULL; -======= - const char *keywords[8]; - const char *values[8]; int i = 0; /* @@ -198,7 +167,6 @@ ConnectDatabase(Archive *AHX, keywords[i] = NULL; values[i++] = NULL; Assert(i <= lengthof(keywords)); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d new_pass = false; AH->connection = PQconnectdbParams(keywords, values, true); @@ -259,11 +227,11 @@ ConnectDatabase(Archive *AHX, */ if (binary_upgrade) { - keywords[6] = "options"; - values[6] = AH->public.remoteVersion < GPDB7_MAJOR_PGVERSION ? + keywords[7] = "options"; + values[7] = AH->public.remoteVersion < GPDB7_MAJOR_PGVERSION ? "-c gp_session_role=utility" : "-c gp_role=utility"; - keywords[7] = NULL; - values[7] = NULL; + keywords[8] = NULL; + values[8] = NULL; AH->connection = PQconnectdbParams(keywords, values, true); } PQsetNoticeProcessor(AH->connection, notice_processor, NULL); From 690c530a95fe1d864d1d612bcfebb575cdf7d8c8 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Mon, 1 Jun 2026 11:47:27 +0300 Subject: [PATCH 520/589] Fix NULL pointer free in src/backend/cdb/cdbllize.c (#2633) Commit 41efb8340877e8ffd0023bb6b2ef22ffd1ca014d added the possibility of glob->subplan entries being NULL, when they are deleted. GPDB-specific code attempted to pfree it. Add a NULL check to cdbllize.c, with an additional benefit that all the NULL entries will be replaced with dummy plans, which are handled correctly by the rest of GPDB-specific code. --- src/backend/cdb/cdbllize.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/cdb/cdbllize.c b/src/backend/cdb/cdbllize.c index e1787a07f604..579fee0f52aa 100644 --- a/src/backend/cdb/cdbllize.c +++ b/src/backend/cdb/cdbllize.c @@ -1177,7 +1177,8 @@ cdbllize_build_slice_table(PlannerInfo *root, Plan *top_plan, * list, because that would screw up the plan_id numbering of the * subplans). */ - pfree(lfirst(lc)); + if (lfirst(lc) != NULL) + pfree(lfirst(lc)); dummy_plan = (Plan *) make_result(NIL, (Node *) list_make1(makeBoolConst(false, false)), NULL); From e03e1e17168234a28510b911c2ed8008b5c0b3b3 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Mon, 1 Jun 2026 13:50:53 +0500 Subject: [PATCH 521/589] Resolve conflicts in src/bin/pg_dump/pg_dump.c (#2596) 1) Commit a0fda03 in src/bin/pg_dump/pg_dump.c removed the first argument fout from the findNamespace function definition, while an earlier commit by bdfea29 had already added a const specifier to the second argument of the dumpTableData and refreshMatViewData functions below. 2) Commit a0fda03 in src/bin/pg_dump/pg_dump.c removed the first argument fout from the convertRegProcReference function definition, while an earlier commit by bdfea29 had already added a const specifier to the second argument of the format_function_signature function above. 3) Commit a45bc8a in src/bin/pg_dump/pg_dump.c replaced the arguments in the main function call to ConnectDatabase, while earlier commit 19cd1cf had already added a new bool binary_upgrade argument. 4) Commit 4648243 in src/bin/pg_dump/pg_dump.c added substream to the getSubscriptions function, while earlier commit 1b64fae had already changed rolname to subowner in the same place. 5) Commit a0fda03 in src/bin/pg_dump/pg_dump.c removed the first fout argument when calling findNamespace in the getTypes, getAggregates, getFuncs, and getTables functions. Earlier commit 9e5f1f2 had already refactored the code below. 6) Commit 403a3d9 in src/bin/pg_dump/pg_dump.c added a lock on all relations in the getTables function, although earlier commits 8317413 and 0f66928 had already refactored this. 7) Commit 257836a in src/bin/pg_dump/pg_dump.c added handling for inddependcollnames and inddependcollversions in the getIndexes function, although earlier commit 30ab901 had already refactored the code. 8) Commit 5423853 in src/bin/pg_dump/pg_dump.c added the additional !dopt->dataOnly condition twice to the getTableAttrs function, while earlier commit 30ab901 had already done this differently. 9) Commit a0fda03 in src/bin/pg_dump/pg_dump.c removed the first fout argument when calling the findNamespace function in the getOperators, getCollations, getConversions, getOpclasses, getOpfamilies, getExtendedStatistics, getTSDictionaries, and getTSConfigurations functions. Earlier commit 1b64fae had already changed rolname to cfgowner in the same place. 10) Commit 110d817 in src/bin/pg_dump/pg_dump.c replaced the call to the string function appendPQExpBuffer without formatting with the more efficient appendPQExpBufferStr in the dumpFunc function, although earlier commits had already changed the code. 11) Commit 1ed6b89 in src/bin/pg_dump/pg_dump.c added a conditional warning in the dumpOpr function, although earlier commit a0fda03 had already removed the first fout argument when calling the convertRegProcReference function. 12) Commit 110d817 in src/bin/pg_dump/pg_dump.c replaced the call to the string function appendPQExpBuffer without formatting with the more efficient appendPQExpBufferStr in the dumpAgg function, although earlier commits had already done the same. --- src/bin/pg_dump/pg_dump.c | 660 +++----------------------------------- 1 file changed, 43 insertions(+), 617 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index bae3515a92a4..2e80722d1a02 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -193,13 +193,8 @@ static void expand_table_name_patterns(Archive *fout, SimpleOidList *oids, bool strict_names); static NamespaceInfo *findNamespace(Oid nsoid); -<<<<<<< HEAD static void dumpTableData(Archive *fout, const TableDataInfo *tdinfo); static void refreshMatViewData(Archive *fout, const TableDataInfo *tdinfo); -======= -static void dumpTableData(Archive *fout, TableDataInfo *tdinfo); -static void refreshMatViewData(Archive *fout, TableDataInfo *tdinfo); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d static void guessConstraintInheritance(TableInfo *tblinfo, int numTables); static void dumpComment(Archive *fout, const char *type, const char *name, const char *namespace, const char *owner, @@ -287,11 +282,7 @@ static void getTableDataFKConstraints(void); static char *format_function_arguments(const FuncInfo *finfo, const char *funcargs, bool is_agg); static char *format_function_signature(Archive *fout, -<<<<<<< HEAD const FuncInfo *finfo, bool honor_quotes); -======= - FuncInfo *finfo, bool honor_quotes); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d static char *convertRegProcReference(const char *proc); static char *getFormattedOperatorName(const char *oproid); static char *convertTSFunction(Archive *fout, Oid funcOid); @@ -871,11 +862,7 @@ main(int argc, char **argv) * Open the database using the Archiver, so it knows about it. Errors mean * death. */ -<<<<<<< HEAD - ConnectDatabase(fout, dopt.dbname, dopt.pghost, dopt.pgport, dopt.username, prompt_password, dopt.binary_upgrade); -======= - ConnectDatabase(fout, &dopt.cparams, false); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + ConnectDatabase(fout, &dopt.cparams, false, dopt.binary_upgrade); setup_connection(fout, dumpencoding, dumpsnapshot, use_role); /* @@ -4456,12 +4443,8 @@ getSubscriptions(Archive *fout) int i_tableoid; int i_oid; int i_subname; -<<<<<<< HEAD int i_subowner; -======= - int i_rolname; int i_substream; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d int i_subconninfo; int i_subslotname; int i_subsynccommit; @@ -5388,20 +5371,12 @@ getTypes(Archive *fout, int *numTypes) tyinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_typname)); tyinfo[i].dobj.namespace = findNamespace(atooid(PQgetvalue(res, i, i_typnamespace))); -<<<<<<< HEAD tyinfo[i].dacl.acl = pg_strdup(PQgetvalue(res, i, i_typacl)); tyinfo[i].dacl.acldefault = pg_strdup(PQgetvalue(res, i, i_acldefault)); tyinfo[i].dacl.privtype = 0; tyinfo[i].dacl.initprivs = NULL; tyinfo[i].ftypname = NULL; /* may get filled later */ tyinfo[i].rolname = getRoleName(PQgetvalue(res, i, i_typowner)); -======= - tyinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); - tyinfo[i].typacl = pg_strdup(PQgetvalue(res, i, i_typacl)); - tyinfo[i].rtypacl = pg_strdup(PQgetvalue(res, i, i_rtypacl)); - tyinfo[i].inittypacl = pg_strdup(PQgetvalue(res, i, i_inittypacl)); - tyinfo[i].initrtypacl = pg_strdup(PQgetvalue(res, i, i_initrtypacl)); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d tyinfo[i].typelem = atooid(PQgetvalue(res, i, i_typelem)); tyinfo[i].typrelid = atooid(PQgetvalue(res, i, i_typrelid)); tyinfo[i].typrelkind = *PQgetvalue(res, i, i_typrelkind); @@ -5543,11 +5518,7 @@ getOperators(Archive *fout, int *numOprs) oprinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oprname)); oprinfo[i].dobj.namespace = findNamespace(atooid(PQgetvalue(res, i, i_oprnamespace))); -<<<<<<< HEAD oprinfo[i].rolname = getRoleName(PQgetvalue(res, i, i_oprowner)); -======= - oprinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d oprinfo[i].oprkind = (PQgetvalue(res, i, i_oprkind))[0]; oprinfo[i].oprcode = atooid(PQgetvalue(res, i, i_oprcode)); @@ -5624,11 +5595,7 @@ getCollations(Archive *fout, int *numCollations) collinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_collname)); collinfo[i].dobj.namespace = findNamespace(atooid(PQgetvalue(res, i, i_collnamespace))); -<<<<<<< HEAD collinfo[i].rolname = getRoleName(PQgetvalue(res, i, i_collowner)); -======= - collinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* Decide whether we want to dump it */ selectDumpableObject(&(collinfo[i].dobj), fout); @@ -5696,11 +5663,7 @@ getConversions(Archive *fout, int *numConversions) convinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_conname)); convinfo[i].dobj.namespace = findNamespace(atooid(PQgetvalue(res, i, i_connamespace))); -<<<<<<< HEAD convinfo[i].rolname = getRoleName(PQgetvalue(res, i, i_conowner)); -======= - convinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* Decide whether we want to dump it */ selectDumpableObject(&(convinfo[i].dobj), fout); @@ -5837,11 +5800,7 @@ getOpclasses(Archive *fout, int *numOpclasses) opcinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_opcname)); opcinfo[i].dobj.namespace = findNamespace(atooid(PQgetvalue(res, i, i_opcnamespace))); -<<<<<<< HEAD opcinfo[i].rolname = getRoleName(PQgetvalue(res, i, i_opcowner)); -======= - opcinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* Decide whether we want to dump it */ selectDumpableObject(&(opcinfo[i].dobj), fout); @@ -5909,11 +5868,7 @@ getOpfamilies(Archive *fout, int *numOpfamilies) opfinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_opfname)); opfinfo[i].dobj.namespace = findNamespace(atooid(PQgetvalue(res, i, i_opfnamespace))); -<<<<<<< HEAD opfinfo[i].rolname = getRoleName(PQgetvalue(res, i, i_opfowner)); -======= - opfinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* Decide whether we want to dump it */ selectDumpableObject(&(opfinfo[i].dobj), fout); @@ -6061,18 +6016,11 @@ getAggregates(Archive *fout, int *numAggs) agginfo[i].aggfn.dobj.name = pg_strdup(PQgetvalue(res, i, i_aggname)); agginfo[i].aggfn.dobj.namespace = findNamespace(atooid(PQgetvalue(res, i, i_aggnamespace))); -<<<<<<< HEAD agginfo[i].aggfn.dacl.acl = pg_strdup(PQgetvalue(res, i, i_aggacl)); agginfo[i].aggfn.dacl.acldefault = pg_strdup(PQgetvalue(res, i, i_acldefault)); agginfo[i].aggfn.dacl.privtype = 0; agginfo[i].aggfn.dacl.initprivs = NULL; agginfo[i].aggfn.rolname = getRoleName(PQgetvalue(res, i, i_proowner)); -======= - agginfo[i].aggfn.rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); - if (strlen(agginfo[i].aggfn.rolname) == 0) - pg_log_warning("owner of aggregate function \"%s\" appears to be invalid", - agginfo[i].aggfn.dobj.name); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d agginfo[i].aggfn.lang = InvalidOid; /* not currently interesting */ agginfo[i].aggfn.prorettype = InvalidOid; /* not saved */ agginfo[i].aggfn.nargs = atoi(PQgetvalue(res, i, i_pronargs)); @@ -6402,15 +6350,11 @@ getFuncs(Archive *fout, int *numFuncs) finfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_proname)); finfo[i].dobj.namespace = findNamespace(atooid(PQgetvalue(res, i, i_pronamespace))); -<<<<<<< HEAD finfo[i].dacl.acl = pg_strdup(PQgetvalue(res, i, i_proacl)); finfo[i].dacl.acldefault = pg_strdup(PQgetvalue(res, i, i_acldefault)); finfo[i].dacl.privtype = 0; finfo[i].dacl.initprivs = NULL; finfo[i].rolname = getRoleName(PQgetvalue(res, i, i_proowner)); -======= - finfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d finfo[i].lang = atooid(PQgetvalue(res, i, i_prolang)); finfo[i].prorettype = atooid(PQgetvalue(res, i, i_prorettype)); finfo[i].nargs = atoi(PQgetvalue(res, i, i_pronargs)); @@ -6861,18 +6805,10 @@ getTables(Archive *fout, int *numTables) tblinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_relname)); tblinfo[i].dobj.namespace = findNamespace(atooid(PQgetvalue(res, i, i_relnamespace))); -<<<<<<< HEAD tblinfo[i].dacl.acl = pg_strdup(PQgetvalue(res, i, i_relacl)); tblinfo[i].dacl.acldefault = pg_strdup(PQgetvalue(res, i, i_acldefault)); tblinfo[i].dacl.privtype = 0; tblinfo[i].dacl.initprivs = NULL; -======= - tblinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); - tblinfo[i].relacl = pg_strdup(PQgetvalue(res, i, i_relacl)); - tblinfo[i].rrelacl = pg_strdup(PQgetvalue(res, i, i_rrelacl)); - tblinfo[i].initrelacl = pg_strdup(PQgetvalue(res, i, i_initrelacl)); - tblinfo[i].initrrelacl = pg_strdup(PQgetvalue(res, i, i_initrrelacl)); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d tblinfo[i].relkind = *(PQgetvalue(res, i, i_relkind)); tblinfo[i].reltype = atooid(PQgetvalue(res, i, i_reltype)); tblinfo[i].rolname = getRoleName(PQgetvalue(res, i, i_relowner)); @@ -7009,29 +6945,11 @@ getTables(Archive *fout, int *numTables) * We only need to lock the relation for certain components; see * pg_dump.h * -<<<<<<< HEAD * GPDB: Build a single LOCK TABLE statement to lock all interesting tables. * This is more performant than issuing a separate LOCK TABLE statement for each table, * with considerable savings in FE/BE overhead. It does come at the cost of some increased * memory usage in both FE and BE, which we will be able to tolerate. */ -======= - * On server versions that support it, we lock all relations not just - * plain tables. - */ - if (tblinfo[i].dobj.dump && - (fout->hasGenericLockTable || - tblinfo[i].relkind == RELKIND_PARTITIONED_TABLE || - tblinfo[i].relkind == RELKIND_RELATION) && - (tblinfo[i].dobj.dump & DUMP_COMPONENTS_REQUIRING_LOCK)) - { - resetPQExpBuffer(query); - appendPQExpBuffer(query, - "LOCK TABLE %s IN ACCESS SHARE MODE", - fmtQualifiedDumpable(&tblinfo[i])); - ExecuteSqlStatement(fout, query->data); - } ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d if ((tblinfo[i].dobj.dump & DUMP_COMPONENTS_REQUIRING_LOCK) && (tblinfo[i].relkind == RELKIND_RELATION || @@ -7351,14 +7269,7 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) i_tablespace, i_indreloptions, i_indstatcols, -<<<<<<< HEAD i_indstatvals; -======= - i_indstatvals, - i_inddependcollnames, - i_inddependcollversions; - int ntups; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* * We want to perform just one query against pg_index. However, we @@ -7539,259 +7450,11 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) * Locate the associated TableInfo; we rely on tblinfo[] being in OID * order. */ -<<<<<<< HEAD while (++curtblindx < numTables) { tbinfo = &tblinfo[curtblindx]; if (tbinfo->dobj.catId.oid == indrelid) break; -======= - resetPQExpBuffer(query); - if (fout->remoteVersion >= 140000) - { - appendPQExpBuffer(query, - "SELECT t.tableoid, t.oid, " - "t.relname AS indexname, " - "inh.inhparent AS parentidx, " - "pg_catalog.pg_get_indexdef(i.indexrelid) AS indexdef, " - "i.indnkeyatts AS indnkeyatts, " - "i.indnatts AS indnatts, " - "i.indkey, i.indisclustered, " - "i.indisreplident, " - "c.contype, c.conname, " - "c.condeferrable, c.condeferred, " - "c.tableoid AS contableoid, " - "c.oid AS conoid, " - "pg_catalog.pg_get_constraintdef(c.oid, false) AS condef, " - "(SELECT spcname FROM pg_catalog.pg_tablespace s WHERE s.oid = t.reltablespace) AS tablespace, " - "t.reloptions AS indreloptions, " - "(SELECT pg_catalog.array_agg(attnum ORDER BY attnum) " - " FROM pg_catalog.pg_attribute " - " WHERE attrelid = i.indexrelid AND " - " attstattarget >= 0) AS indstatcols," - "(SELECT pg_catalog.array_agg(attstattarget ORDER BY attnum) " - " FROM pg_catalog.pg_attribute " - " WHERE attrelid = i.indexrelid AND " - " attstattarget >= 0) AS indstatvals, " - "(SELECT pg_catalog.array_agg(quote_ident(ns.nspname) || '.' || quote_ident(c.collname) ORDER BY refobjid) " - " FROM pg_catalog.pg_depend d " - " JOIN pg_catalog.pg_collation c ON (c.oid = d.refobjid) " - " JOIN pg_catalog.pg_namespace ns ON (c.collnamespace = ns.oid) " - " WHERE d.classid = 'pg_catalog.pg_class'::regclass AND " - " d.objid = i.indexrelid AND " - " d.objsubid = 0 AND " - " d.refclassid = 'pg_catalog.pg_collation'::regclass AND " - " d.refobjversion IS NOT NULL) AS inddependcollnames, " - "(SELECT pg_catalog.array_agg(quote_literal(refobjversion) ORDER BY refobjid) " - " FROM pg_catalog.pg_depend " - " WHERE classid = 'pg_catalog.pg_class'::regclass AND " - " objid = i.indexrelid AND " - " objsubid = 0 AND " - " refclassid = 'pg_catalog.pg_collation'::regclass AND " - " refobjversion IS NOT NULL) AS inddependcollversions " - "FROM pg_catalog.pg_index i " - "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " - "JOIN pg_catalog.pg_class t2 ON (t2.oid = i.indrelid) " - "LEFT JOIN pg_catalog.pg_constraint c " - "ON (i.indrelid = c.conrelid AND " - "i.indexrelid = c.conindid AND " - "c.contype IN ('p','u','x')) " - "LEFT JOIN pg_catalog.pg_inherits inh " - "ON (inh.inhrelid = indexrelid) " - "WHERE i.indrelid = '%u'::pg_catalog.oid " - "AND (i.indisvalid OR t2.relkind = 'p') " - "AND i.indisready " - "ORDER BY indexname", - tbinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 110000) - { - appendPQExpBuffer(query, - "SELECT t.tableoid, t.oid, " - "t.relname AS indexname, " - "inh.inhparent AS parentidx, " - "pg_catalog.pg_get_indexdef(i.indexrelid) AS indexdef, " - "i.indnkeyatts AS indnkeyatts, " - "i.indnatts AS indnatts, " - "i.indkey, i.indisclustered, " - "i.indisreplident, " - "c.contype, c.conname, " - "c.condeferrable, c.condeferred, " - "c.tableoid AS contableoid, " - "c.oid AS conoid, " - "pg_catalog.pg_get_constraintdef(c.oid, false) AS condef, " - "(SELECT spcname FROM pg_catalog.pg_tablespace s WHERE s.oid = t.reltablespace) AS tablespace, " - "t.reloptions AS indreloptions, " - "(SELECT pg_catalog.array_agg(attnum ORDER BY attnum) " - " FROM pg_catalog.pg_attribute " - " WHERE attrelid = i.indexrelid AND " - " attstattarget >= 0) AS indstatcols," - "(SELECT pg_catalog.array_agg(attstattarget ORDER BY attnum) " - " FROM pg_catalog.pg_attribute " - " WHERE attrelid = i.indexrelid AND " - " attstattarget >= 0) AS indstatvals, " - "'{}' AS inddependcollnames, " - "'{}' AS inddependcollversions " - "FROM pg_catalog.pg_index i " - "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " - "JOIN pg_catalog.pg_class t2 ON (t2.oid = i.indrelid) " - "LEFT JOIN pg_catalog.pg_constraint c " - "ON (i.indrelid = c.conrelid AND " - "i.indexrelid = c.conindid AND " - "c.contype IN ('p','u','x')) " - "LEFT JOIN pg_catalog.pg_inherits inh " - "ON (inh.inhrelid = indexrelid) " - "WHERE i.indrelid = '%u'::pg_catalog.oid " - "AND (i.indisvalid OR t2.relkind = 'p') " - "AND i.indisready " - "ORDER BY indexname", - tbinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 90400) - { - /* - * the test on indisready is necessary in 9.2, and harmless in - * earlier/later versions - */ - appendPQExpBuffer(query, - "SELECT t.tableoid, t.oid, " - "t.relname AS indexname, " - "0 AS parentidx, " - "pg_catalog.pg_get_indexdef(i.indexrelid) AS indexdef, " - "i.indnatts AS indnkeyatts, " - "i.indnatts AS indnatts, " - "i.indkey, i.indisclustered, " - "i.indisreplident, " - "c.contype, c.conname, " - "c.condeferrable, c.condeferred, " - "c.tableoid AS contableoid, " - "c.oid AS conoid, " - "pg_catalog.pg_get_constraintdef(c.oid, false) AS condef, " - "(SELECT spcname FROM pg_catalog.pg_tablespace s WHERE s.oid = t.reltablespace) AS tablespace, " - "t.reloptions AS indreloptions, " - "'' AS indstatcols, " - "'' AS indstatvals, " - "'{}' AS inddependcollnames, " - "'{}' AS inddependcollversions " - "FROM pg_catalog.pg_index i " - "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " - "LEFT JOIN pg_catalog.pg_constraint c " - "ON (i.indrelid = c.conrelid AND " - "i.indexrelid = c.conindid AND " - "c.contype IN ('p','u','x')) " - "WHERE i.indrelid = '%u'::pg_catalog.oid " - "AND i.indisvalid AND i.indisready " - "ORDER BY indexname", - tbinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 90000) - { - /* - * the test on indisready is necessary in 9.2, and harmless in - * earlier/later versions - */ - appendPQExpBuffer(query, - "SELECT t.tableoid, t.oid, " - "t.relname AS indexname, " - "0 AS parentidx, " - "pg_catalog.pg_get_indexdef(i.indexrelid) AS indexdef, " - "i.indnatts AS indnkeyatts, " - "i.indnatts AS indnatts, " - "i.indkey, i.indisclustered, " - "false AS indisreplident, " - "c.contype, c.conname, " - "c.condeferrable, c.condeferred, " - "c.tableoid AS contableoid, " - "c.oid AS conoid, " - "pg_catalog.pg_get_constraintdef(c.oid, false) AS condef, " - "(SELECT spcname FROM pg_catalog.pg_tablespace s WHERE s.oid = t.reltablespace) AS tablespace, " - "t.reloptions AS indreloptions, " - "'' AS indstatcols, " - "'' AS indstatvals, " - "'{}' AS inddependcollnames, " - "'{}' AS inddependcollversions " - "FROM pg_catalog.pg_index i " - "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " - "LEFT JOIN pg_catalog.pg_constraint c " - "ON (i.indrelid = c.conrelid AND " - "i.indexrelid = c.conindid AND " - "c.contype IN ('p','u','x')) " - "WHERE i.indrelid = '%u'::pg_catalog.oid " - "AND i.indisvalid AND i.indisready " - "ORDER BY indexname", - tbinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 80200) - { - appendPQExpBuffer(query, - "SELECT t.tableoid, t.oid, " - "t.relname AS indexname, " - "0 AS parentidx, " - "pg_catalog.pg_get_indexdef(i.indexrelid) AS indexdef, " - "i.indnatts AS indnkeyatts, " - "i.indnatts AS indnatts, " - "i.indkey, i.indisclustered, " - "false AS indisreplident, " - "c.contype, c.conname, " - "c.condeferrable, c.condeferred, " - "c.tableoid AS contableoid, " - "c.oid AS conoid, " - "null AS condef, " - "(SELECT spcname FROM pg_catalog.pg_tablespace s WHERE s.oid = t.reltablespace) AS tablespace, " - "t.reloptions AS indreloptions, " - "'' AS indstatcols, " - "'' AS indstatvals, " - "'{}' AS inddependcollnames, " - "'{}' AS inddependcollversions " - "FROM pg_catalog.pg_index i " - "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " - "LEFT JOIN pg_catalog.pg_depend d " - "ON (d.classid = t.tableoid " - "AND d.objid = t.oid " - "AND d.deptype = 'i') " - "LEFT JOIN pg_catalog.pg_constraint c " - "ON (d.refclassid = c.tableoid " - "AND d.refobjid = c.oid) " - "WHERE i.indrelid = '%u'::pg_catalog.oid " - "AND i.indisvalid " - "ORDER BY indexname", - tbinfo->dobj.catId.oid); - } - else - { - appendPQExpBuffer(query, - "SELECT t.tableoid, t.oid, " - "t.relname AS indexname, " - "0 AS parentidx, " - "pg_catalog.pg_get_indexdef(i.indexrelid) AS indexdef, " - "t.relnatts AS indnkeyatts, " - "t.relnatts AS indnatts, " - "i.indkey, i.indisclustered, " - "false AS indisreplident, " - "c.contype, c.conname, " - "c.condeferrable, c.condeferred, " - "c.tableoid AS contableoid, " - "c.oid AS conoid, " - "null AS condef, " - "(SELECT spcname FROM pg_catalog.pg_tablespace s WHERE s.oid = t.reltablespace) AS tablespace, " - "null AS indreloptions, " - "'' AS indstatcols, " - "'' AS indstatvals, " - "'{}' AS inddependcollnames, " - "'{}' AS inddependcollversions " - "FROM pg_catalog.pg_index i " - "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " - "LEFT JOIN pg_catalog.pg_depend d " - "ON (d.classid = t.tableoid " - "AND d.objid = t.oid " - "AND d.deptype = 'i') " - "LEFT JOIN pg_catalog.pg_constraint c " - "ON (d.refclassid = c.tableoid " - "AND d.refobjid = c.oid) " - "WHERE i.indrelid = '%u'::pg_catalog.oid " - "ORDER BY indexname", - tbinfo->dobj.catId.oid); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } if (curtblindx >= numTables) fatal("unrecognized table OID %u", indrelid); @@ -7805,42 +7468,7 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) tbinfo->indexes = indxinfo + j; tbinfo->numIndexes = numinds; -<<<<<<< HEAD for (int c = 0; c < numinds; c++, j++) -======= - ntups = PQntuples(res); - - i_tableoid = PQfnumber(res, "tableoid"); - i_oid = PQfnumber(res, "oid"); - i_indexname = PQfnumber(res, "indexname"); - i_parentidx = PQfnumber(res, "parentidx"); - i_indexdef = PQfnumber(res, "indexdef"); - i_indnkeyatts = PQfnumber(res, "indnkeyatts"); - i_indnatts = PQfnumber(res, "indnatts"); - i_indkey = PQfnumber(res, "indkey"); - i_indisclustered = PQfnumber(res, "indisclustered"); - i_indisreplident = PQfnumber(res, "indisreplident"); - i_contype = PQfnumber(res, "contype"); - i_conname = PQfnumber(res, "conname"); - i_condeferrable = PQfnumber(res, "condeferrable"); - i_condeferred = PQfnumber(res, "condeferred"); - i_contableoid = PQfnumber(res, "contableoid"); - i_conoid = PQfnumber(res, "conoid"); - i_condef = PQfnumber(res, "condef"); - i_tablespace = PQfnumber(res, "tablespace"); - i_indreloptions = PQfnumber(res, "indreloptions"); - i_indstatcols = PQfnumber(res, "indstatcols"); - i_indstatvals = PQfnumber(res, "indstatvals"); - i_inddependcollnames = PQfnumber(res, "inddependcollnames"); - i_inddependcollversions = PQfnumber(res, "inddependcollversions"); - - tbinfo->indexes = indxinfo = - (IndxInfo *) pg_malloc(ntups * sizeof(IndxInfo)); - constrinfo = (ConstraintInfo *) pg_malloc(ntups * sizeof(ConstraintInfo)); - tbinfo->numIndexes = ntups; - - for (j = 0; j < ntups; j++) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d { char contype; @@ -7859,8 +7487,6 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) indxinfo[j].indreloptions = pg_strdup(PQgetvalue(res, j, i_indreloptions)); indxinfo[j].indstatcols = pg_strdup(PQgetvalue(res, j, i_indstatcols)); indxinfo[j].indstatvals = pg_strdup(PQgetvalue(res, j, i_indstatvals)); - indxinfo[j].inddependcollnames = pg_strdup(PQgetvalue(res, j, i_inddependcollnames)); - indxinfo[j].inddependcollversions = pg_strdup(PQgetvalue(res, j, i_inddependcollversions)); indxinfo[j].indkeys = (Oid *) pg_malloc(indxinfo[j].indnattrs * sizeof(Oid)); parseOidArray(PQgetvalue(res, j, i_indkey), indxinfo[j].indkeys, indxinfo[j].indnattrs); @@ -7980,11 +7606,7 @@ getExtendedStatistics(Archive *fout) statsextinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_stxname)); statsextinfo[i].dobj.namespace = findNamespace(atooid(PQgetvalue(res, i, i_stxnamespace))); -<<<<<<< HEAD statsextinfo[i].rolname = getRoleName(PQgetvalue(res, i, i_stxowner)); -======= - statsextinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d statsextinfo[i].stattarget = atoi(PQgetvalue(res, i, i_stattarget)); /* Decide whether we want to dump it */ @@ -9309,17 +8931,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) tbinfo->attencoding[j] = NULL; } -<<<<<<< HEAD if (hasdefaults) -======= - PQclear(res); - - /* - * Get info about column defaults. This is skipped for a data-only - * dump, as it is only needed for table schemas. - */ - if (!dopt->dataOnly && hasdefaults) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d { /* Collect OIDs of interesting tables that have defaults */ if (tbloids->len > 1) /* do we have more than the '{'? */ @@ -9438,28 +9050,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) */ attrdefs[j].separate = true; } -<<<<<<< HEAD else if (!shouldPrintColumn(dopt, tbinfo, adnum - 1)) -======= - PQclear(res); - } - - /* - * Get info about table CHECK constraints. This is skipped for a - * data-only dump, as it is only needed for table schemas. - */ - if (tbinfo->ncheck > 0 && !dopt->dataOnly) - { - ConstraintInfo *constrs; - int numConstrs; - - pg_log_info("finding check constraints for table \"%s.%s\"", - tbinfo->dobj.namespace->dobj.name, - tbinfo->dobj.name); - - resetPQExpBuffer(q); - if (fout->remoteVersion >= 90200) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d { /* column will be suppressed, print default separately */ attrdefs[j].separate = true; @@ -9807,11 +9398,7 @@ getTSDictionaries(Archive *fout, int *numTSDicts) dictinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_dictname)); dictinfo[i].dobj.namespace = findNamespace(atooid(PQgetvalue(res, i, i_dictnamespace))); -<<<<<<< HEAD dictinfo[i].rolname = getRoleName(PQgetvalue(res, i, i_dictowner)); -======= - dictinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d dictinfo[i].dicttemplate = atooid(PQgetvalue(res, i, i_dicttemplate)); if (PQgetisnull(res, i, i_dictinitoption)) dictinfo[i].dictinitoption = NULL; @@ -9945,11 +9532,7 @@ getTSConfigurations(Archive *fout, int *numTSConfigs) cfginfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_cfgname)); cfginfo[i].dobj.namespace = findNamespace(atooid(PQgetvalue(res, i, i_cfgnamespace))); -<<<<<<< HEAD cfginfo[i].rolname = getRoleName(PQgetvalue(res, i, i_cfgowner)); -======= - cfginfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d cfginfo[i].cfgparser = atooid(PQgetvalue(res, i, i_cfgparser)); /* Decide whether we want to dump it */ @@ -12810,74 +12393,73 @@ dumpFunc(Archive *fout, const FuncInfo *finfo) delqry = createPQExpBuffer(); asPart = createPQExpBuffer(); -<<<<<<< HEAD if (!fout->is_prepared[PREPQUERY_DUMPFUNC]) { /* Set up query for function-specific details */ appendPQExpBufferStr(query, "PREPARE dumpFunc(pg_catalog.oid) AS\n"); - appendPQExpBuffer(query, - "SELECT\n" - "proretset,\n" - "prosrc,\n" - "probin,\n" - "provolatile,\n" - "proisstrict,\n" - "prosecdef,\n" - "(SELECT lanname FROM pg_catalog.pg_language WHERE oid = prolang) AS lanname,\n " - "proconfig,\n" - "procost,\n" - "prorows,\n" - "prodataaccess,\n" - "pg_catalog.pg_get_function_arguments(p.oid) AS funcargs,\n" - "pg_catalog.pg_get_function_identity_arguments(p.oid) AS funciargs,\n" - "pg_catalog.pg_get_function_result(p.oid) AS funcresult,\n" - "(SELECT procallback FROM pg_catalog.pg_proc_callback WHERE profnoid::pg_catalog.oid = p.oid) as callbackfunc,\n"); + appendPQExpBufferStr(query, + "SELECT\n" + "proretset,\n" + "prosrc,\n" + "probin,\n" + "provolatile,\n" + "proisstrict,\n" + "prosecdef,\n" + "(SELECT lanname FROM pg_catalog.pg_language WHERE oid = prolang) AS lanname,\n " + "proconfig,\n" + "procost,\n" + "prorows,\n" + "prodataaccess,\n" + "pg_catalog.pg_get_function_arguments(p.oid) AS funcargs,\n" + "pg_catalog.pg_get_function_identity_arguments(p.oid) AS funciargs,\n" + "pg_catalog.pg_get_function_result(p.oid) AS funcresult,\n" + "(SELECT procallback FROM pg_catalog.pg_proc_callback WHERE profnoid::pg_catalog.oid = p.oid) as callbackfunc,\n"); if (fout->remoteVersion >= 90200) - appendPQExpBuffer(query, - "proleakproof,\n"); + appendPQExpBufferStr(query, + "proleakproof,\n"); else - appendPQExpBuffer(query, - "false AS proleakproof,\n"); + appendPQExpBufferStr(query, + "false AS proleakproof,\n"); /* GPDB6 added proexeclocation */ if (fout->remoteVersion >= GPDB6_MAJOR_PGVERSION) - appendPQExpBuffer(query, - "proexeclocation,\n"); + appendPQExpBufferStr(query, + "proexeclocation,\n"); else - appendPQExpBuffer(query, - "'a' as proexeclocation,\n"); + appendPQExpBufferStr(query, + "'a' as proexeclocation,\n"); if (fout->remoteVersion >= 90500) - appendPQExpBuffer(query, - "array_to_string(protrftypes, ' ') AS protrftypes,\n"); + appendPQExpBufferStr(query, + "array_to_string(protrftypes, ' ') AS protrftypes,\n"); if (fout->remoteVersion >= 90600) - appendPQExpBuffer(query, - "proparallel,\n"); + appendPQExpBufferStr(query, + "proparallel,\n"); else - appendPQExpBuffer(query, - "'u' AS proparallel,\n"); + appendPQExpBufferStr(query, + "'u' AS proparallel,\n"); if (fout->remoteVersion >= 110000) - appendPQExpBuffer(query, - "prokind,\n"); + appendPQExpBufferStr(query, + "prokind,\n"); else if (fout->remoteVersion >= 80400) - appendPQExpBuffer(query, - "CASE WHEN proiswindow THEN 'w' ELSE 'f' END AS prokind,\n"); + appendPQExpBufferStr(query, + "CASE WHEN proiswindow THEN 'w' ELSE 'f' END AS prokind,\n"); else - appendPQExpBuffer(query, - "CASE WHEN proiswin THEN 'w' ELSE 'f' END AS prokind,\n"); + appendPQExpBufferStr(query, + "CASE WHEN proiswin THEN 'w' ELSE 'f' END AS prokind,\n"); if (fout->remoteVersion >= 120000) - appendPQExpBuffer(query, - "prosupport\n"); + appendPQExpBufferStr(query, + "prosupport\n"); else - appendPQExpBuffer(query, - "'-' AS prosupport\n"); + appendPQExpBufferStr(query, + "'-' AS prosupport\n"); - appendPQExpBuffer(query, + appendPQExpBufferStr(query, "FROM pg_catalog.pg_proc p, pg_catalog.pg_language l\n" "WHERE p.oid = $1 " "AND l.oid = p.prolang"); @@ -12890,90 +12472,6 @@ dumpFunc(Archive *fout, const FuncInfo *finfo) printfPQExpBuffer(query, "EXECUTE dumpFunc('%u')", -======= - /* Fetch function-specific details */ - appendPQExpBufferStr(query, - "SELECT\n" - "proretset,\n" - "prosrc,\n" - "probin,\n" - "provolatile,\n" - "proisstrict,\n" - "prosecdef,\n" - "(SELECT lanname FROM pg_catalog.pg_language WHERE oid = prolang) AS lanname,\n"); - - if (fout->remoteVersion >= 80300) - appendPQExpBufferStr(query, - "proconfig,\n" - "procost,\n" - "prorows,\n"); - else - appendPQExpBufferStr(query, - "null AS proconfig,\n" - "0 AS procost,\n" - "0 AS prorows,\n"); - - if (fout->remoteVersion >= 80400) - { - /* - * In 8.4 and up we rely on pg_get_function_arguments and - * pg_get_function_result instead of examining proallargtypes etc. - */ - appendPQExpBufferStr(query, - "pg_catalog.pg_get_function_arguments(oid) AS funcargs,\n" - "pg_catalog.pg_get_function_identity_arguments(oid) AS funciargs,\n" - "pg_catalog.pg_get_function_result(oid) AS funcresult,\n"); - } - else if (fout->remoteVersion >= 80100) - appendPQExpBufferStr(query, - "proallargtypes,\n" - "proargmodes,\n" - "proargnames,\n"); - else - appendPQExpBufferStr(query, - "null AS proallargtypes,\n" - "null AS proargmodes,\n" - "proargnames,\n"); - - if (fout->remoteVersion >= 90200) - appendPQExpBufferStr(query, - "proleakproof,\n"); - else - appendPQExpBufferStr(query, - "false AS proleakproof,\n"); - - if (fout->remoteVersion >= 90500) - appendPQExpBufferStr(query, - "array_to_string(protrftypes, ' ') AS protrftypes,\n"); - - if (fout->remoteVersion >= 90600) - appendPQExpBufferStr(query, - "proparallel,\n"); - else - appendPQExpBufferStr(query, - "'u' AS proparallel,\n"); - - if (fout->remoteVersion >= 110000) - appendPQExpBufferStr(query, - "prokind,\n"); - else if (fout->remoteVersion >= 80400) - appendPQExpBufferStr(query, - "CASE WHEN proiswindow THEN 'w' ELSE 'f' END AS prokind,\n"); - else - appendPQExpBufferStr(query, - "'f' AS prokind,\n"); - - if (fout->remoteVersion >= 120000) - appendPQExpBufferStr(query, - "prosupport\n"); - else - appendPQExpBufferStr(query, - "'-' AS prosupport\n"); - - appendPQExpBuffer(query, - "FROM pg_catalog.pg_proc " - "WHERE oid = '%u'::pg_catalog.oid", ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d finfo->dobj.catId.oid); res = ExecuteSqlQueryForSingleRow(fout, query->data); @@ -13693,14 +13191,11 @@ dumpOpr(Archive *fout, const OprInfo *oprinfo) oprcanmerge = PQgetvalue(res, 0, i_oprcanmerge); oprcanhash = PQgetvalue(res, 0, i_oprcanhash); -<<<<<<< HEAD -======= /* In PG14 upwards postfix operator support does not exist anymore. */ if (strcmp(oprkind, "r") == 0) pg_log_warning("postfix operators are not supported anymore (operator \"%s\")", oprcode); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d oprregproc = convertRegProcReference(oprcode); if (oprregproc) { @@ -14879,7 +14374,6 @@ dumpAgg(Archive *fout, const AggInfo *agginfo) delq = createPQExpBuffer(); details = createPQExpBuffer(); -<<<<<<< HEAD if (!fout->is_prepared[PREPQUERY_DUMPAGG]) { /* Set up query for aggregate-specific details */ @@ -14942,74 +14436,6 @@ dumpAgg(Archive *fout, const AggInfo *agginfo) "'-' AS aggserialfn,\n" "'-' AS aggdeserialfn,\n" "'u' AS proparallel,\n"); -======= - /* Get aggregate-specific details */ - appendPQExpBufferStr(query, - "SELECT\n" - "aggtransfn,\n" - "aggfinalfn,\n" - "aggtranstype::pg_catalog.regtype,\n" - "agginitval,\n"); - - if (fout->remoteVersion >= 80100) - appendPQExpBufferStr(query, - "aggsortop,\n"); - else - appendPQExpBufferStr(query, - "0 AS aggsortop,\n"); - - if (fout->remoteVersion >= 80400) - appendPQExpBufferStr(query, - "pg_catalog.pg_get_function_arguments(p.oid) AS funcargs,\n" - "pg_catalog.pg_get_function_identity_arguments(p.oid) AS funciargs,\n"); - - if (fout->remoteVersion >= 90400) - appendPQExpBufferStr(query, - "aggkind,\n" - "aggmtransfn,\n" - "aggminvtransfn,\n" - "aggmfinalfn,\n" - "aggmtranstype::pg_catalog.regtype,\n" - "aggfinalextra,\n" - "aggmfinalextra,\n" - "aggtransspace,\n" - "aggmtransspace,\n" - "aggminitval,\n"); - else - appendPQExpBufferStr(query, - "'n' AS aggkind,\n" - "'-' AS aggmtransfn,\n" - "'-' AS aggminvtransfn,\n" - "'-' AS aggmfinalfn,\n" - "0 AS aggmtranstype,\n" - "false AS aggfinalextra,\n" - "false AS aggmfinalextra,\n" - "0 AS aggtransspace,\n" - "0 AS aggmtransspace,\n" - "NULL AS aggminitval,\n"); - - if (fout->remoteVersion >= 90600) - appendPQExpBufferStr(query, - "aggcombinefn,\n" - "aggserialfn,\n" - "aggdeserialfn,\n" - "proparallel,\n"); - else - appendPQExpBufferStr(query, - "'-' AS aggcombinefn,\n" - "'-' AS aggserialfn,\n" - "'-' AS aggdeserialfn,\n" - "'u' AS proparallel,\n"); - - if (fout->remoteVersion >= 110000) - appendPQExpBufferStr(query, - "aggfinalmodify,\n" - "aggmfinalmodify\n"); - else - appendPQExpBufferStr(query, - "'0' AS aggfinalmodify,\n" - "'0' AS aggmfinalmodify\n"); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d if (fout->remoteVersion >= 110000) appendPQExpBufferStr(query, From 4dde9bc083fe43cecc2eb31bae1a622770956d87 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Mon, 1 Jun 2026 14:02:12 +0500 Subject: [PATCH 522/589] Resolve conflicts in src/bin/pg_rewind/filemap.h (#2597) 1) Commit f81e97d in src/bin/pg_rewind/filemap.h removed the comment at the beginning, although earlier commits had already changed it. 2) Commit f81e97d in src/bin/pg_rewind/filemap.h added a new field, "status," to the file_entry_t structure, added the const modifier to the "path" field, and removed the "next" field, although earlier commit 55c5525 had already added the new field, "is_gp_tablespace," to this structure. 3) Commit f81e97d in src/bin/pg_rewind/filemap.h renamed the narray and array fields to nentries and entries in the filemap_t structure, and removed the first, last, and nlist fields. Earlier commits had already changed the comments for the fields in this structure. 4) Commit eb00f1d in src/bin/pg_rewind/filemap.h formatted the arguments in the definition of the process_target_wal_block_change function and added a definition of the new decide_file_actions function. Commit f81e97d changed the return type of the decide_file_actions function from void to filemap_t * and added definitions of the new calculate_totals and print_filemap functions. Earlier commit 1a770cf had already partially done the same and added a definition of the new process_target_wal_aofile_change function. --- src/bin/pg_rewind/filemap.h | 59 +------------------------------------ 1 file changed, 1 insertion(+), 58 deletions(-) diff --git a/src/bin/pg_rewind/filemap.h b/src/bin/pg_rewind/filemap.h index 94a168d93b2f..23bbe0169d02 100644 --- a/src/bin/pg_rewind/filemap.h +++ b/src/bin/pg_rewind/filemap.h @@ -12,18 +12,6 @@ #include "storage/block.h" #include "storage/relfilenode.h" -<<<<<<< HEAD -/* - * For every file found in the local or remote system, we have a file entry - * that contains information about the file on both systems. For relation - * files, there is also a page map that marks pages in the file that were - * changed in the target after the last common checkpoint. Each entry also - * contains an 'action' field, which says what we are going to do with the - * file. - */ - -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* these enum values are sorted in the order we want actions to be processed */ typedef enum { @@ -61,13 +49,9 @@ typedef enum */ typedef struct file_entry_t { -<<<<<<< HEAD - char *path; -======= uint32 status; /* hash status */ const char *path; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d bool isrelfile; /* is it a relation data file? */ /* @@ -83,7 +67,6 @@ typedef struct file_entry_t * source. */ datapagemap_t target_pages_to_overwrite; -<<<<<<< HEAD /* * Status of the file in the source. @@ -93,27 +76,12 @@ typedef struct file_entry_t size_t source_size; char *source_link_target; /* for a symlink */ - bool is_gp_tablespace; - /* * What will we do to the file? */ file_action_t action; -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d - - /* - * Status of the file in the source. - */ - bool source_exists; - file_type_t source_type; - size_t source_size; - char *source_link_target; /* for a symlink */ - /* - * What will we do to the file? - */ - file_action_t action; + bool is_gp_tablespace; } file_entry_t; /* @@ -127,26 +95,8 @@ typedef struct filemap_t uint64 total_size; /* total size of the source cluster */ uint64 fetch_size; /* number of bytes that needs to be copied */ -<<<<<<< HEAD - /* - * After processing all the remote files, the entries in the linked list - * are moved to this array. After processing local files, too, all the - * local entries are added to the array by decide_file_actions(), and - * sorted in the final order. After decide_file_actions(), all the entries - * are in the array, and the linked list is empty. - */ - file_entry_t **array; - int narray; /* current length of array */ - - /* - * Summary information. - */ - uint64 total_size; /* total size of the source cluster */ - uint64 fetch_size; /* number of bytes that needs to be copied */ -======= int nentries; /* size of 'entries' array */ file_entry_t *entries[FLEXIBLE_ARRAY_MEMBER]; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } filemap_t; /* Functions for populating the filemap */ @@ -155,22 +105,15 @@ extern void process_source_file(const char *path, file_type_t type, size_t size, const char *link_target); extern void process_target_file(const char *path, file_type_t type, size_t size, const char *link_target); -<<<<<<< HEAD extern void process_target_wal_aofile_change(RelFileNode rnode, int segno, int64 offset); extern void process_target_wal_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno); -extern void decide_file_actions(void); -======= -extern void process_target_wal_block_change(ForkNumber forknum, - RelFileNode rnode, - BlockNumber blkno); extern filemap_t *decide_file_actions(void); extern void calculate_totals(filemap_t *filemap); extern void print_filemap(filemap_t *filemap); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d #endif /* FILEMAP_H */ From 53495156e779c79540adaa1000327a248a5bb5f9 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Mon, 1 Jun 2026 14:06:39 +0500 Subject: [PATCH 523/589] Resolve conflicts in file_ops.c and pg_rewind.c (#2600) 1) Commit ffb4e27 in src/bin/pg_rewind/file_ops.c in the create_target function replaced the link_target field with source_link_target, while earlier commit 1a770cf had already done the same and also added a condition for is_gp_tablespace. 2) Commit ffb4e27 in src/bin/pg_rewind/file_ops.c in the create_target function added handling for the FILE_TYPE_UNDEFINED case, while earlier commit c122be1 had already added handling for the FILE_TYPE_FIFO case in the same location. 3) Commit ffb4e27 added a new function sync_target_dir to src/bin/pg_rewind/file_ops.c, while earlier commit 55c5525 had already added the create_target_tablespace_layout function in the same place. 4) Commit f81e97d in src/bin/pg_rewind/pg_rewind.c added a new local variable filemap and its usage in the main function. However, earlier commit 1a770cf had already added a call to the decide_file_actions function, and commit 19cd1cf had already added a new local variable replication_slot in these same locations. 5) Commit ffb4e27 in src/bin/pg_rewind/pg_rewind.c removed the syncTargetDirectory function, moving its contents to the sync_target_dir function in src/bin/pg_rewind/file_ops.c. Earlier commits had already significantly changed its contents. --- src/bin/pg_rewind/file_ops.c | 75 ++++++++++++++++++++++++----- src/bin/pg_rewind/file_ops.h | 2 +- src/bin/pg_rewind/pg_rewind.c | 91 +---------------------------------- 3 files changed, 65 insertions(+), 103 deletions(-) diff --git a/src/bin/pg_rewind/file_ops.c b/src/bin/pg_rewind/file_ops.c index a21469c38972..c82ea1080f84 100644 --- a/src/bin/pg_rewind/file_ops.c +++ b/src/bin/pg_rewind/file_ops.c @@ -165,14 +165,10 @@ create_target(file_entry_t *entry) break; case FILE_TYPE_SYMLINK: -<<<<<<< HEAD if (entry->is_gp_tablespace) create_target_tablespace_layout(entry->path, entry->source_link_target); else create_target_symlink(entry->path, entry->source_link_target); -======= - create_target_symlink(entry->path, entry->source_link_target); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d break; case FILE_TYPE_REGULAR: @@ -180,14 +176,11 @@ create_target(file_entry_t *entry) pg_fatal("invalid action (CREATE) for regular file"); break; -<<<<<<< HEAD case FILE_TYPE_FIFO: /* Only pgsql_tmp files are FIFO and they are ignored from source target. */ pg_fatal("invalid action (CREATE) for fifo file"); break; -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d case FILE_TYPE_UNDEFINED: pg_fatal("undefined file type for \"%s\"", entry->path); break; @@ -296,7 +289,6 @@ remove_target_symlink(const char *path) dstpath); } -<<<<<<< HEAD /* Create symlink for tablespace, create tablespace target dir */ static void create_target_tablespace_layout(const char *path, const char *link) @@ -322,7 +314,7 @@ create_target_tablespace_layout(const char *path, const char *link) pfree(newlink); } -======= + /* * Sync target data directory to ensure that modifications are safely on disk. * @@ -331,17 +323,74 @@ create_target_tablespace_layout(const char *path, const char *link) * most dirty buffers to disk. Additionally fsync_pgdata uses a two-pass * approach (only initiating writeback in the first pass), which often reduces * the overall amount of IO noticeably. + * + * gpdb: We assume that all files are synchronized before rewinding and thus we + * just need to synchronize those affected files. This is a resonable + * assumption for gpdb since we've ensured that the db state is clean shutdown + * in pg_rewind by running single mode postgres if needed and also we do not + * copy an unsynchronized dababase without sync as the target base. */ void -sync_target_dir(void) +sync_target_dir(filemap_t *filemap) { if (!do_sync || dry_run) return; - fsync_pgdata(datadir_target, PG_VERSION_NUM); -} + file_entry_t *entry; + int i; + + if (chdir(datadir_target) < 0) + { + pg_log_error("could not change directory to \"%s\": %m", datadir_target); + exit(1); + } + + for (i = 0; i < filemap->nentries; i++) + { + entry = filemap->entries[i]; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + if (entry->target_pages_to_overwrite.bitmapsize > 0) + fsync_fname(entry->path, false); + else + { + switch (entry->action) + { + case FILE_ACTION_COPY: + case FILE_ACTION_TRUNCATE: + case FILE_ACTION_COPY_TAIL: + fsync_fname(entry->path, false); + break; + + case FILE_ACTION_CREATE: + fsync_fname(entry->path, + entry->source_type == FILE_TYPE_DIRECTORY); + /* FALLTHROUGH */ + case FILE_ACTION_REMOVE: + /* + * Fsync the parent directory if we either create or delete + * files/directories in the parent directory. The parent + * directory might be missing as expected, so fsync it could + * fail but we ignore that error. + */ + fsync_parent_path(entry->path); + break; + + case FILE_ACTION_NONE: + break; + + default: + pg_fatal("no action decided for \"%s\"", entry->path); + break; + } + } + } + + /* fsync some files that are (possibly) written by pg_rewind. */ + fsync_fname("global/pg_control", false); + fsync_fname("backup_label", false); + fsync_fname("postgresql.auto.conf", false); + fsync_fname(".", true); /* due to new file backup_label. */ +} /* * Read a file into memory. The file to be read is /. diff --git a/src/bin/pg_rewind/file_ops.h b/src/bin/pg_rewind/file_ops.h index d8466385cf5c..92a428ffe0ee 100644 --- a/src/bin/pg_rewind/file_ops.h +++ b/src/bin/pg_rewind/file_ops.h @@ -19,7 +19,7 @@ extern void remove_target_file(const char *path, bool missing_ok); extern void truncate_target_file(const char *path, off_t newsize); extern void create_target(file_entry_t *t); extern void remove_target(file_entry_t *t); -extern void sync_target_dir(void); +extern void sync_target_dir(filemap_t *filemap); extern char *slurpFile(const char *datadir, const char *path, size_t *filesize); diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 6ed5d18cdaff..a1f4989a4744 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -133,11 +133,8 @@ main(int argc, char **argv) TimeLineID endtli; ControlFileData ControlFile_new; bool writerecoveryconf = false; -<<<<<<< HEAD - char *replication_slot = NULL; -======= + char *replication_slot = NULL; filemap_t *filemap; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d pg_logging_init(argv[0]); set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_rewind")); @@ -428,11 +425,7 @@ main(int argc, char **argv) * We have collected all information we need from both systems. Decide * what to do with each file. */ -<<<<<<< HEAD - decide_file_actions(); -======= filemap = decide_file_actions(); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d if (showprogress) calculate_totals(filemap); @@ -498,7 +491,7 @@ main(int argc, char **argv) if (showprogress) pg_log_info("syncing target data directory"); - sync_target_dir(); + sync_target_dir(filemap); if (writerecoveryconf && !dry_run) WriteRecoveryConfig(conn, datadir_target, @@ -846,84 +839,6 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size) checkControlFile(ControlFile); } -/* -<<<<<<< HEAD - * Sync target data directory to ensure that modifications are safely on disk. - * - * We do this once, for the whole data directory, for performance reasons. At - * the end of pg_rewind's run, the kernel is likely to already have flushed - * most dirty buffers to disk. Additionally fsync_pgdata uses a two-pass - * approach (only initiating writeback in the first pass), which often reduces - * the overall amount of IO noticeably. - * - * gpdb: We assume that all files are synchronized before rewinding and thus we - * just need to synchronize those affected files. This is a resonable - * assumption for gpdb since we've ensured that the db state is clean shutdown - * in pg_rewind by running single mode postgres if needed and also we do not - * copy an unsynchronized dababase without sync as the target base. - */ -static void -syncTargetDirectory(void) -{ - if (!do_sync || dry_run) - return; - - file_entry_t *entry; - int i; - - if (chdir(datadir_target) < 0) - { - pg_log_error("could not change directory to \"%s\": %m", datadir_target); - exit(1); - } - - for (i = 0; i < filemap->narray; i++) - { - entry = filemap->array[i]; - - if (entry->target_pages_to_overwrite.bitmapsize > 0) - fsync_fname(entry->path, false); - else - { - switch (entry->action) - { - case FILE_ACTION_COPY: - case FILE_ACTION_TRUNCATE: - case FILE_ACTION_COPY_TAIL: - fsync_fname(entry->path, false); - break; - - case FILE_ACTION_CREATE: - fsync_fname(entry->path, - entry->source_type == FILE_TYPE_DIRECTORY); - /* FALLTHROUGH */ - case FILE_ACTION_REMOVE: - /* - * Fsync the parent directory if we either create or delete - * files/directories in the parent directory. The parent - * directory might be missing as expected, so fsync it could - * fail but we ignore that error. - */ - fsync_parent_path(entry->path); - break; - - case FILE_ACTION_NONE: - break; - - default: - pg_fatal("no action decided for \"%s\"", entry->path); - break; - } - } - } - - /* fsync some files that are (possibly) written by pg_rewind. */ - fsync_fname("global/pg_control", false); - fsync_fname("backup_label", false); - fsync_fname("postgresql.auto.conf", false); - fsync_fname(".", true); /* due to new file backup_label. */ -} - static int32 get_target_dbid(const char *argv0) { @@ -983,8 +898,6 @@ get_target_dbid(const char *argv0) } /* -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d * Get value of GUC parameter restore_command from the target cluster. * * This uses a logic based on "postgres -C" to get the value from the From dfc0ebbcff632889e2eb12f206a3d914808938ee Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Mon, 1 Jun 2026 14:07:17 +0500 Subject: [PATCH 524/589] Resolve conflicts in src/bin/pg_rewind/filemap.c (#2598) 1) Commit f81e97d in src/bin/pg_rewind/filemap.c removed the definitions of the path_cmp and get_filemap_entry functions and added definitions of the new insert_filehash_entry and lookup_filehash_entry functions. However, the earlier commit 1a770cf had already added the definition of get_filemap_entry. 2) Commit f81e97d in src/bin/pg_rewind/filemap.c refactored the process_source_file function, which had already been modified by earlier commits. 3) Commit f81e97d in src/bin/pg_rewind/filemap.c refactored the process_target_file function, which had already been modified by earlier commits. 4) Commit f81e97d in src/bin/pg_rewind/filemap.c refactored the code in the process_target_wal_block_change function, although earlier commits had already changed it. 5) Commit f81e97d in src/bin/pg_rewind/filemap.c removed the filemap_list_to_array function, and commit eb00f1d removed the filemap_finalize function, although earlier commit 1a770cf had already partially done the same. 6) Commit f81e97d in src/bin/pg_rewind/filemap.c in the calculate_totals function replaced the local variable filemap_t *map with the argument filemap_t *filemap, although earlier commit 1a770cf had already changed this code. 7) Commit eb00f1d added a new function, decide_file_action, to src/bin/pg_rewind/filemap.c. Earlier commit 1a770cf had already done the same, but added handling for the FILE_TYPE_FIFO case and setting the is_gp_tablespace field. 8) Commit eb00f1d added a new function, decide_file_actions, to src/bin/pg_rewind/filemap.c. Commit f81e97d refactored its contents, changing its return type from void to filemap_t *, and added a new function, hash_string_pointer. Earlier commit 1a770cf had already partially done the same. --- src/bin/pg_rewind/filemap.c | 229 +----------------------------------- 1 file changed, 1 insertion(+), 228 deletions(-) diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index 43f0e40efee6..d4048f649210 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -59,15 +59,9 @@ static filehash_hash *filehash; static bool isRelDataFile(const char *path); static char *datasegpath(RelFileNode rnode, ForkNumber forknum, BlockNumber segno); -<<<<<<< HEAD -static int path_cmp(const void *a, const void *b); - -static file_entry_t *get_filemap_entry(const char *path, bool create); -======= static file_entry_t *insert_filehash_entry(const char *path); static file_entry_t *lookup_filehash_entry(const char *path); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d static int final_filemap_cmp(const void *a, const void *b); static bool check_file_excluded(const char *path, bool is_source); @@ -226,67 +220,6 @@ lookup_filehash_entry(const char *path) return filehash_lookup(filehash, path); } -/* Look up or create entry for 'path' */ -static file_entry_t * -get_filemap_entry(const char *path, bool create) -{ - filemap_t *map = filemap; - file_entry_t *entry; - file_entry_t **e; - file_entry_t key; - file_entry_t *key_ptr; - - if (map->array) - { - key.path = (char *) path; - key_ptr = &key; - e = bsearch(&key_ptr, map->array, map->narray, sizeof(file_entry_t *), - path_cmp); - } - else - e = NULL; - - if (e) - entry = *e; - else if (!create) - entry = NULL; - else - { - /* Create a new entry for this file */ - entry = pg_malloc(sizeof(file_entry_t)); - entry->path = pg_strdup(path); - entry->isrelfile = isRelDataFile(path); - entry->action = FILE_ACTION_UNDECIDED; - - entry->target_exists = false; - entry->target_type = FILE_TYPE_UNDEFINED; - entry->target_size = 0; - entry->target_link_target = NULL; - entry->target_pages_to_overwrite.bitmap = NULL; - entry->target_pages_to_overwrite.bitmapsize = 0; - - entry->source_exists = false; - entry->source_type = FILE_TYPE_UNDEFINED; - entry->source_size = 0; - entry->source_link_target = NULL; - - entry->is_gp_tablespace = false; - - entry->next = NULL; - - if (map->last) - { - map->last->next = entry; - map->last = entry; - } - else - map->first = map->last = entry; - map->nlist++; - } - - return entry; -} - /* * Callback for processing source file list. * @@ -300,11 +233,6 @@ process_source_file(const char *path, file_type_t type, size_t size, { file_entry_t *entry; -<<<<<<< HEAD - Assert(filemap->array == NULL); - -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d /* * Pretend that pg_wal is a directory, even if it's really a symlink. We * don't want to mess with the symlink itself, nor complain if it's a @@ -321,13 +249,9 @@ process_source_file(const char *path, file_type_t type, size_t size, pg_fatal("data file \"%s\" in source is not a regular file", path); /* Remember this source file */ -<<<<<<< HEAD - entry = get_filemap_entry(path, true); -======= entry = insert_filehash_entry(path); if (entry->source_exists) pg_fatal("duplicate source file \"%s\"", path); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d entry->source_exists = true; entry->source_type = type; entry->source_size = size; @@ -337,22 +261,12 @@ process_source_file(const char *path, file_type_t type, size_t size, /* * Callback for processing target file list. * -<<<<<<< HEAD - * All source files must be already processed before calling this. We record - * the type and size of file, so that decide_file_action() can later decide - * what to do with it. -======= * Record the type and size of the file, like process_source_file() does. ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d */ void process_target_file(const char *path, file_type_t type, size_t size, const char *link_target) { -<<<<<<< HEAD - filemap_t *map = filemap; -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d file_entry_t *entry; /* @@ -366,7 +280,6 @@ process_target_file(const char *path, file_type_t type, size_t size, * should not be copied but also should not be removed, then a separate * function for those would be better. */ -<<<<<<< HEAD { const char *filename = last_dir_separator(path); if (filename == NULL) @@ -382,24 +295,6 @@ process_target_file(const char *path, file_type_t type, size_t size, return; } - if (map->array == NULL) - { - /* on first call, initialize lookup array */ - if (map->nlist == 0) - { - /* should not happen */ - pg_fatal("source file list is empty"); - } - - filemap_list_to_array(map); - - Assert(map->array != NULL); - - qsort(map->array, map->narray, sizeof(file_entry_t *), path_cmp); - } -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d - /* * Like in process_source_file, pretend that pg_wal is always a directory. */ @@ -407,13 +302,9 @@ process_target_file(const char *path, file_type_t type, size_t size, type = FILE_TYPE_DIRECTORY; /* Remember this target file */ -<<<<<<< HEAD - entry = get_filemap_entry(path, true); -======= entry = insert_filehash_entry(path); if (entry->target_exists) pg_fatal("duplicate source file \"%s\"", path); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d entry->target_exists = true; entry->target_type = type; entry->target_size = size; @@ -427,11 +318,7 @@ process_target_file(const char *path, file_type_t type, size_t size, * if so, records it in 'target_pages_to_overwrite' bitmap. * * NOTE: All the files on both systems must have already been added to the -<<<<<<< HEAD - * file map! -======= * hash table! ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d */ void process_target_wal_block_change(ForkNumber forknum, RelFileNode rnode, @@ -441,22 +328,11 @@ process_target_wal_block_change(ForkNumber forknum, RelFileNode rnode, file_entry_t *entry; BlockNumber blkno_inseg; int segno; -<<<<<<< HEAD - - Assert(filemap->array); -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d segno = blkno / RELSEG_SIZE; blkno_inseg = blkno % RELSEG_SIZE; path = datasegpath(rnode, forknum, segno); -<<<<<<< HEAD - entry = get_filemap_entry(path, false); - pfree(path); - - if (entry && entry->target_exists) -======= entry = lookup_filehash_entry(path); pfree(path); @@ -477,7 +353,6 @@ process_target_wal_block_change(ForkNumber forknum, RelFileNode rnode, * the source. Either way, we can safely ignore it. */ if (entry) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d { int64 end_offset; @@ -486,32 +361,6 @@ process_target_wal_block_change(ForkNumber forknum, RelFileNode rnode, if (entry->target_type != FILE_TYPE_REGULAR) pg_fatal("unexpected page modification for non-regular file \"%s\"", entry->path); -<<<<<<< HEAD - - /* - * If the block beyond the EOF in the source system, no need to - * remember it now, because we're going to truncate it away from the - * target anyway. Also no need to remember the block if it's beyond - * the current EOF in the target system; we will copy it over with the - * "tail" from the source system, anyway. - */ - end_offset = (blkno_inseg + 1) * BLCKSZ; - if (end_offset <= entry->source_size && - end_offset <= entry->target_size) - datapagemap_add(&entry->target_pages_to_overwrite, blkno_inseg); - } - else - { - /* - * If we don't have any record of this file in the file map, it means - * that it's a relation that doesn't exist in the source system. It - * could exist in the target system; we haven't moved the target-only - * entries from the linked list to the array yet! But in any case, if - * it doesn't exist in the source it will be removed from the target - * too, and we can safely ignore it. - */ - } -======= if (entry->target_exists && entry->source_exists) { @@ -522,7 +371,6 @@ process_target_wal_block_change(ForkNumber forknum, RelFileNode rnode, datapagemap_add(&entry->target_pages_to_overwrite, blkno_inseg); } } ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } void @@ -531,10 +379,8 @@ process_target_wal_aofile_change(RelFileNode rnode, int segno, int64 offset) char *path; file_entry_t *entry; - Assert(filemap->array); - path = datasegpath(rnode, MAIN_FORKNUM, segno); - entry = get_filemap_entry(path, false); + entry = lookup_filehash_entry(path); pfree(path); if (entry && entry->target_exists) @@ -642,37 +488,6 @@ check_file_excluded(const char *path, bool is_source) return false; } -<<<<<<< HEAD -/* - * Convert the linked list of entries in map->first/last to the array, - * map->array. - */ -static void -filemap_list_to_array(filemap_t *map) -{ - int narray; - file_entry_t *entry, - *next; - - map->array = (file_entry_t **) - pg_realloc(map->array, - (map->nlist + map->narray) * sizeof(file_entry_t *)); - - narray = map->narray; - for (entry = map->first; entry != NULL; entry = next) - { - map->array[narray++] = entry; - next = entry->next; - entry->next = NULL; - } - Assert(narray == map->nlist + map->narray); - map->narray = narray; - map->nlist = 0; - map->first = map->last = NULL; -} - -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d static const char * action_to_str(file_action_t action) { @@ -715,28 +530,16 @@ calculate_totals(filemap_t *filemap) if (entry->source_type != FILE_TYPE_REGULAR) continue; -<<<<<<< HEAD - map->total_size += entry->source_size; - - if (entry->action == FILE_ACTION_COPY) - { - map->fetch_size += entry->source_size; -======= filemap->total_size += entry->source_size; if (entry->action == FILE_ACTION_COPY) { filemap->fetch_size += entry->source_size; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d continue; } if (entry->action == FILE_ACTION_COPY_TAIL) -<<<<<<< HEAD - map->fetch_size += (entry->source_size - entry->target_size); -======= filemap->fetch_size += (entry->source_size - entry->target_size); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d if (entry->target_pages_to_overwrite.bitmapsize > 0) { @@ -949,18 +752,12 @@ decide_file_action(file_entry_t *entry) { case FILE_TYPE_DIRECTORY: case FILE_TYPE_SYMLINK: -<<<<<<< HEAD entry->is_gp_tablespace = strncmp(entry->path, "pg_tblspc/", strlen("pg_tblspc/")) == 0; return FILE_ACTION_CREATE; case FILE_TYPE_REGULAR: return FILE_ACTION_COPY; case FILE_TYPE_FIFO: return FILE_ACTION_NONE; -======= - return FILE_ACTION_CREATE; - case FILE_TYPE_REGULAR: - return FILE_ACTION_COPY; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d case FILE_TYPE_UNDEFINED: pg_fatal("unknown file type for \"%s\"", entry->path); break; @@ -1055,12 +852,9 @@ decide_file_action(file_entry_t *entry) } break; -<<<<<<< HEAD case FILE_TYPE_FIFO: return FILE_ACTION_NONE; -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d case FILE_TYPE_UNDEFINED: pg_fatal("unknown file type for \"%s\"", path); break; @@ -1072,26 +866,6 @@ decide_file_action(file_entry_t *entry) /* * Decide what to do with each file. -<<<<<<< HEAD - */ -void -decide_file_actions(void) -{ - int i; - - filemap_list_to_array(filemap); - - for (i = 0; i < filemap->narray; i++) - { - file_entry_t *entry = filemap->array[i]; - - entry->action = decide_file_action(entry); - } - - /* Sort the actions to the order that they should be performed */ - qsort(filemap->array, filemap->narray, sizeof(file_entry_t *), - final_filemap_cmp); -======= * * Returns a 'filemap' with the entries in the order that their actions * should be executed. @@ -1140,5 +914,4 @@ hash_string_pointer(const char *s) unsigned char *ss = (unsigned char *) s; return hash_bytes(ss, strlen(s)); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } From 81c076477b150357c0b164a153910ba47cd15d45 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Mon, 1 Jun 2026 14:11:51 +0500 Subject: [PATCH 525/589] Fix compilation of src/backend/replication/logical/worker.c (#2617) Commit 4648243 added new functions subxact_info_write and stream_open_file to src/backend/replication/logical/worker.c, which call the BufFileCreateShared function. However, earlier commit a0c36f2 had already added the new workfile_set *work_set argument. --- src/backend/replication/logical/worker.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index e564be3629d0..f15c9e47e2e0 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -2532,6 +2532,7 @@ subxact_info_write(Oid subid, TransactionId xid) if (ent->subxact_fileset == NULL) { MemoryContext oldctx; + workfile_set *work_set; /* * We need to maintain shared fileset across multiple stream @@ -2542,7 +2543,8 @@ subxact_info_write(Oid subid, TransactionId xid) SharedFileSetInit(ent->subxact_fileset, NULL); MemoryContextSwitchTo(oldctx); - fd = BufFileCreateShared(ent->subxact_fileset, path); + work_set = workfile_mgr_create_set("SubxactInfo", path, false /* hold pin */); + fd = BufFileCreateShared(ent->subxact_fileset, path, work_set); } else fd = BufFileOpenShared(ent->subxact_fileset, path, O_RDWR); @@ -2814,6 +2816,7 @@ stream_open_file(Oid subid, TransactionId xid, bool first_segment) { MemoryContext savectx; SharedFileSet *fileset; + workfile_set *work_set; /* * We need to maintain shared fileset across multiple stream @@ -2825,7 +2828,8 @@ stream_open_file(Oid subid, TransactionId xid, bool first_segment) SharedFileSetInit(fileset, NULL); MemoryContextSwitchTo(savectx); - stream_fd = BufFileCreateShared(fileset, path); + work_set = workfile_mgr_create_set("ChangeInfo", path, false /* hold pin */); + stream_fd = BufFileCreateShared(fileset, path, work_set); /* Remember the fileset for the next stream of the same transaction */ ent->xid = xid; From 2e12c350997a9e91d724b9592660c6b4e4417021 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Mon, 1 Jun 2026 14:16:13 +0500 Subject: [PATCH 526/589] Fix compilation of aocs_compaction.c and appendonly_compaction.c (#2620) 1) Commit a04daa9 in src/include/executor/executor.h added a new argument, ResultRelInfo *resultRelInfo, to the definition of the ExecInsertIndexTuples function. Add this in GPDB-specific places. 2) Commit 1375422 in src/include/nodes/execnodes.h changed the type of the es_result_relations field in the EState structure from ResultRelInfo * to ResultRelInfo **. Change this in GPDB-specific places. --- src/backend/access/aocs/aocs_compaction.c | 7 +++++-- src/backend/access/appendonly/appendonly_compaction.c | 8 ++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/backend/access/aocs/aocs_compaction.c b/src/backend/access/aocs/aocs_compaction.c index af5231107ae8..638458e6adc1 100644 --- a/src/backend/access/aocs/aocs_compaction.c +++ b/src/backend/access/aocs/aocs_compaction.c @@ -191,7 +191,7 @@ AOCSMoveTuple(TupleTableSlot *slot, /* insert index' tuples if needed */ if (resultRelInfo->ri_NumIndices > 0) { - ExecInsertIndexTuples(slot, estate, false, false, NIL); + ExecInsertIndexTuples(resultRelInfo, slot, estate, false, false, NIL); ResetPerTupleExprContext(estate); } @@ -263,7 +263,10 @@ AOCSSegmentFileFullCompaction(Relation aorel, resultRelInfo->ri_RelationDesc = aorel; resultRelInfo->ri_TrigDesc = NULL; /* we don't fire triggers */ ExecOpenIndices(resultRelInfo, false); - estate->es_result_relations = resultRelInfo; + if (estate->es_result_relations == NULL) + estate->es_result_relations = (ResultRelInfo **) + palloc0(estate->es_range_table_size * sizeof(ResultRelInfo *)); + estate->es_result_relations[resultRelInfo->ri_RangeTableIndex - 1] = resultRelInfo; /* * We don't want uniqueness checks to be performed while "insert"ing tuples diff --git a/src/backend/access/appendonly/appendonly_compaction.c b/src/backend/access/appendonly/appendonly_compaction.c index 189bc126269c..22c9f591701e 100644 --- a/src/backend/access/appendonly/appendonly_compaction.c +++ b/src/backend/access/appendonly/appendonly_compaction.c @@ -301,7 +301,8 @@ AppendOnlyMoveTuple(TupleTableSlot *slot, /* insert index' tuples if needed */ if (resultRelInfo->ri_NumIndices > 0) { - ExecInsertIndexTuples(slot, + ExecInsertIndexTuples(resultRelInfo, + slot, estate, false, /* noDupError */ NULL, /* specConflict */ @@ -444,7 +445,10 @@ AppendOnlySegmentFileFullCompaction(Relation aorel, resultRelInfo->ri_RelationDesc = aorel; resultRelInfo->ri_TrigDesc = NULL; /* we don't fire triggers */ ExecOpenIndices(resultRelInfo, false); - estate->es_result_relations = resultRelInfo; + if (estate->es_result_relations == NULL) + estate->es_result_relations = (ResultRelInfo **) + palloc0(estate->es_range_table_size * sizeof(ResultRelInfo *)); + estate->es_result_relations[resultRelInfo->ri_RangeTableIndex - 1] = resultRelInfo; /* * We don't want uniqueness checks to be performed while "insert"ing tuples From 4a157578a121a7f642cdf138c8ecca459bff0a70 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Mon, 1 Jun 2026 14:17:52 +0500 Subject: [PATCH 527/589] Fix compilation of src/backend/executor/nodeModifyTable.c (#2622) 1) Commit ffeba7e added the segid argument to the ExecDelete function call in src/backend/executor/nodeModifyTable.c when resolving conflicts, but no such local variable existed. 2) Commit 2000b6c removed the ri_PartitionCheck field from the ResultRelInfo structure in src/include/nodes/execnodes.h, replacing its use with ri_RelationDesc->rd_rel->relispartition. Replace it in GPDB-specific locations. 3) Commit 6973533 in src/include/commands/trigger.h removed the tcs_map field in the TransitionCaptureState structure. Remove this in GPDB-specific locations. 4) Commit 6973533 in src/backend/executor/nodeModifyTable.c removed the tupconv_map_for_subplan function, replacing its call with resultRelInfo->ri_ChildToRootMap. Replace this in GPDB-specific locations. 5) Commit 1375422 in src/include/nodes/execnodes.h changed the es_result_relations field type in the EState structure from ResultRelInfo * to ResultRelInfo **. Replace this in GPDB-specific locations. --- src/backend/executor/nodeModifyTable.c | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index b99997d9ccef..aadf23c8bf03 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -1177,7 +1177,7 @@ ldelete:; static bool ExecCrossPartitionUpdate(ModifyTableState *mtstate, ResultRelInfo *resultRelInfo, - ItemPointer tupleid, HeapTuple oldtuple, + ItemPointer tupleid, int32 segid, HeapTuple oldtuple, TupleTableSlot *slot, TupleTableSlot *planSlot, EPQState *epqstate, bool canSetTag, TupleTableSlot **retry_slot, @@ -1466,6 +1466,7 @@ lreplace:; * the tuple we're trying to move has been concurrently updated. */ retry = !ExecCrossPartitionUpdate(mtstate, resultRelInfo, tupleid, + segid, oldtuple, slot, planSlot, epqstate, canSetTag, &retry_slot, &inserted_tuple); @@ -1692,7 +1693,6 @@ ExecSplitUpdate_Insert(ModifyTableState *mtstate, { Relation resultRelationDesc; bool partition_constraint_failed; - TupleConversionMap *saved_tcs_map = NULL; PartitionTupleRouting *proute = mtstate->mt_partition_tuple_routing; int map_index; TupleConversionMap *tupconv_map; @@ -1713,7 +1713,7 @@ ExecSplitUpdate_Insert(ModifyTableState *mtstate, * row. So skip the WCO checks if the partition constraint fails. */ partition_constraint_failed = - resultRelInfo->ri_PartitionCheck && + resultRelInfo->ri_RelationDesc->rd_rel->relispartition && !ExecPartitionCheck(resultRelInfo, slot, estate, false); if (!partition_constraint_failed && @@ -1727,16 +1727,6 @@ ExecSplitUpdate_Insert(ModifyTableState *mtstate, resultRelInfo, slot, estate); } - /* - * Updates set the transition capture map only when a new subplan - * is chosen. But for inserts, it is set for each row. So after - * INSERT, we need to revert back to the map created for UPDATE; - * otherwise the next UPDATE will incorrectly use the one created - * for INSERT. So first save the one created for UPDATE. - */ - if (mtstate->mt_transition_capture) - saved_tcs_map = mtstate->mt_transition_capture->tcs_map; - if (partition_constraint_failed) { /* @@ -1757,7 +1747,7 @@ ExecSplitUpdate_Insert(ModifyTableState *mtstate, */ map_index = resultRelInfo - mtstate->resultRelInfo; Assert(map_index >= 0 && map_index < mtstate->mt_nplans); - tupconv_map = tupconv_map_for_subplan(mtstate, map_index); + tupconv_map = resultRelInfo->ri_ChildToRootMap; if (tupconv_map != NULL) slot = execute_attr_map_slot(tupconv_map->attrMap, slot, @@ -1780,10 +1770,7 @@ ExecSplitUpdate_Insert(ModifyTableState *mtstate, /* Revert ExecPrepareTupleRouting's node change. */ if (mtstate->mt_transition_capture) - { mtstate->mt_transition_capture->tcs_original_insert_tuple = NULL; - mtstate->mt_transition_capture->tcs_map = saved_tcs_map; - } } else { @@ -2276,7 +2263,7 @@ ExecModifyTable(PlanState *pstate) node->mt_whichplan++; if (node->mt_whichplan < node->mt_nplans) { - resultRelInfo = estate->es_result_relations + node->mt_whichplan; + resultRelInfo = estate->es_result_relations[node->mt_whichplan]; subplanstate = node->mt_plans[node->mt_whichplan]; junkfilter = resultRelInfo->ri_junkFilter; action_attno = resultRelInfo->ri_action_attno; From 8051f2760e90a3c8873169bc744bc8d749ca94f4 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Tue, 2 Jun 2026 12:31:53 +0300 Subject: [PATCH 528/589] Resolve conflicts is src/test/regress/sql/timestamp.sql (#2636) Commit a094c8ff53523e88ff9dd28ad467618039e27b58 added new tests, however there was already GPDB-specific test before it. --- src/test/regress/expected/timestamp.out | 17 +++++++---------- src/test/regress/sql/timestamp.sql | 8 ++------ 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/test/regress/expected/timestamp.out b/src/test/regress/expected/timestamp.out index cc82b7aca6b1..6a6f91e77621 100644 --- a/src/test/regress/expected/timestamp.out +++ b/src/test/regress/expected/timestamp.out @@ -1834,14 +1834,6 @@ SELECT '' AS to_char_12, to_char(d, 'FF1 FF2 FF3 FF4 FF5 FF6 ff1 ff2 ff3 ff4 ff | 7 78 789 7890 78901 789012 7 78 789 7890 78901 789012 789 789012 (4 rows) --- timestamp numeric fields constructor -SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887); - make_timestamp ------------------------------- - Sun Dec 28 06:30:45.887 2014 -(1 row) - -<<<<<<< HEAD SET DateStyle TO DEFAULT; -- Make sure timeofdate() and current_time() are doing roughly the same thing select timeofday()::date = current_timestamp::date; @@ -1850,7 +1842,13 @@ select timeofday()::date = current_timestamp::date; t (1 row) -======= +-- timestamp numeric fields constructor +SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887); + make_timestamp +------------------------------ + Sun Dec 28 06:30:45.887 2014 +(1 row) + SELECT make_timestamp(-44, 3, 15, 12, 30, 15); make_timestamp ----------------------------- @@ -1860,4 +1858,3 @@ SELECT make_timestamp(-44, 3, 15, 12, 30, 15); -- should fail select make_timestamp(0, 7, 15, 12, 30, 15); ERROR: date field value out of range: 0-07-15 ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d diff --git a/src/test/regress/sql/timestamp.sql b/src/test/regress/sql/timestamp.sql index ccba25fffd5c..f27062c0dbd8 100644 --- a/src/test/regress/sql/timestamp.sql +++ b/src/test/regress/sql/timestamp.sql @@ -284,17 +284,13 @@ SELECT '' AS to_char_12, to_char(d, 'FF1 FF2 FF3 FF4 FF5 FF6 ff1 ff2 ff3 ff4 ff ('2018-11-02 12:34:56.78901234') ) d(d); --- timestamp numeric fields constructor -<<<<<<< HEAD -SELECT make_timestamp(2014,12,28,6,30,45.887); - SET DateStyle TO DEFAULT; -- Make sure timeofdate() and current_time() are doing roughly the same thing select timeofday()::date = current_timestamp::date; -======= + +-- timestamp numeric fields constructor SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887); SELECT make_timestamp(-44, 3, 15, 12, 30, 15); -- should fail select make_timestamp(0, 7, 15, 12, 30, 15); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d From 2bd01ef5398b94b5aed41670c8465a05eb48ebd0 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Tue, 2 Jun 2026 14:58:19 +0300 Subject: [PATCH 529/589] Fix segfault in src/backend/executor/nodeModifyTable.c (#2635) 1. Commit 4a157578a121a7f642cdf138c8ecca459bff0a70 incorrectly fixed nodeModifyTable.c, retaining the use of estate->es_result_relations. However commit 1375422c7826a2bf387be29895e961614f69de4b changed ExecInitModifyTable to use a newly allocated buffer in mtstate->resultRelInfo instead of a reference to estate->es_result_relations. As a result, junkfilter is filled correctly only in mtstate->resultRelInfo, and it should be used instead. 2. Commit a04daa97a4339c38e304cd6164d37da540d665a8 added ExecPrepareTupleRouting() call to ExecInsert(), undoing the target relation change in ExecSplitUpdate_Insert() and causing tuples to be inserted into a wrong partition. As a solution, remove ExecPrepareTupleRouting() from ExecSplitUpdate_Insert() and rely on the one on ExecInsert() instead. --- src/backend/executor/nodeModifyTable.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index aadf23c8bf03..89de2a8f236f 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -1758,13 +1758,8 @@ ExecSplitUpdate_Insert(ModifyTableState *mtstate, * into the root. */ Assert(mtstate->rootResultRelInfo != NULL); - ResultRelInfo *partRelInfo; - slot = ExecPrepareTupleRouting(mtstate, estate, proute, - mtstate->rootResultRelInfo, slot, - &partRelInfo); - resultRelInfo = partRelInfo; - slot = ExecInsert(mtstate, resultRelInfo, slot, planSlot, + slot = ExecInsert(mtstate, mtstate->rootResultRelInfo, slot, planSlot, estate, mtstate->canSetTag, true /* splitUpdate */); @@ -2263,7 +2258,7 @@ ExecModifyTable(PlanState *pstate) node->mt_whichplan++; if (node->mt_whichplan < node->mt_nplans) { - resultRelInfo = estate->es_result_relations[node->mt_whichplan]; + resultRelInfo = &node->resultRelInfo[node->mt_whichplan]; subplanstate = node->mt_plans[node->mt_whichplan]; junkfilter = resultRelInfo->ri_junkFilter; action_attno = resultRelInfo->ri_action_attno; From 3c32e47630557d3d6370dd930ed7a9be103d964f Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Wed, 3 Jun 2026 13:01:45 +0300 Subject: [PATCH 530/589] Resolve conflicts in src/test/regress/expected/create_index.out (#2638) 1. Commit 3165426e54df02a6199c0a216546e5095e875a0a replaced operators in plans, so GPDB-specific plans have to be updated. 2. Commit 16fa9b2b30a357b4aea982bd878ec2e5e002dbcc reordered lines in output, however those lines are missing in GPDB since merge commit 19cd1cf4b68faff2e29bc2fa884c480e4644cdb4. 3. Commit 257836a75585934cc05ed7a80bccf8190d41e056 added new output lines, however GPDB has different lines nearby since 8b305092235fa0879589334837513f22bed339b1. 4. Commit a6642b3ae060976b42830b7dc8f29ec190ab05e4 added more tests for concurrent reindex, however GPDB does not support it. Also, this commit was already cherry-picked as 67a7c0f06be0da1cd466e24fb0c33b51308dc1e0. --- src/test/regress/expected/create_index.out | 84 +------------------ .../expected/create_index_optimizer.out | 52 ++++++++---- 2 files changed, 40 insertions(+), 96 deletions(-) diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index f903ffd18761..e67e836d5128 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -260,7 +260,6 @@ EXPLAIN (COSTS OFF) SELECT * FROM fast_emp4000 WHERE home_base <@ '(200,200),(2000,1000)'::box ORDER BY (home_base[0])[0]; -<<<<<<< HEAD QUERY PLAN ---------------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) @@ -268,18 +267,9 @@ SELECT * FROM fast_emp4000 -> Sort Sort Key: ((home_base[0])[0]) -> Index Only Scan using grect2ind on fast_emp4000 - Index Cond: (home_base @ '(2000,1000),(200,200)'::box) + Index Cond: (home_base <@ '(2000,1000),(200,200)'::box) Optimizer: Postgres query optimizer (7 rows) -======= - QUERY PLAN ------------------------------------------------------------------ - Sort - Sort Key: ((home_base[0])[0]) - -> Index Only Scan using grect2ind on fast_emp4000 - Index Cond: (home_base <@ '(2000,1000),(200,200)'::box) -(4 rows) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d SELECT * FROM fast_emp4000 WHERE home_base <@ '(200,200),(2000,1000)'::box @@ -329,7 +319,6 @@ SELECT count(*) FROM fast_emp4000 WHERE home_base IS NULL; EXPLAIN (COSTS OFF) SELECT * FROM polygon_tbl WHERE f1 @> '((1,1),(2,2),(2,1))'::polygon ORDER BY (poly_center(f1))[0]; -<<<<<<< HEAD QUERY PLAN ----------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) @@ -337,18 +326,9 @@ SELECT * FROM polygon_tbl WHERE f1 @> '((1,1),(2,2),(2,1))'::polygon -> Sort Sort Key: ((poly_center(f1))[0]) -> Index Scan using gpolygonind on polygon_tbl - Index Cond: (f1 ~ '((1,1),(2,2),(2,1))'::polygon) + Index Cond: (f1 @> '((1,1),(2,2),(2,1))'::polygon) Optimizer: Postgres query optimizer (7 rows) -======= - QUERY PLAN ------------------------------------------------------------- - Sort - Sort Key: ((poly_center(f1))[0]) - -> Index Scan using gpolygonind on polygon_tbl - Index Cond: (f1 @> '((1,1),(2,2),(2,1))'::polygon) -(4 rows) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d SELECT * FROM polygon_tbl WHERE f1 @> '((1,1),(2,2),(2,1))'::polygon ORDER BY (poly_center(f1))[0]; @@ -585,11 +565,6 @@ SELECT * FROM point_tbl WHERE NOT f1 ~= '(1e-300, -1e-300)' ORDER BY f1 <-> '0,1 SELECT * FROM point_tbl WHERE NOT f1 ~= '(1e-300, -1e-300)' ORDER BY f1 <-> '0,1'; f1 ------------------- -<<<<<<< HEAD -======= - (1e-300,-1e-300) - (0,0) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d (-3,4) (-10,0) (10,10) @@ -631,11 +606,6 @@ SELECT * FROM point_tbl WHERE NOT f1 ~= '(1e-300, -1e-300)' AND f1 IS NOT NULL O SELECT * FROM point_tbl WHERE NOT f1 ~= '(1e-300, -1e-300)' AND f1 IS NOT NULL ORDER BY f1 <-> '0,1'; f1 ------------------- -<<<<<<< HEAD -======= - (1e-300,-1e-300) - (0,0) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d (-3,4) (-10,0) (10,10) @@ -658,17 +628,9 @@ SELECT * FROM point_tbl WHERE NOT f1 ~= '(1e-300, -1e-300)' AND f1 <@ '(-10,-10) Optimizer: Postgres query optimizer (7 rows) -<<<<<<< HEAD SELECT * FROM point_tbl WHERE NOT f1 ~= '(1e-300, -1e-300)' AND f1 <@ '(-10,-10),(10,10)':: box ORDER BY f1 <-> '0,1'; f1 --------- -======= -SELECT * FROM point_tbl WHERE f1 <@ '(-10,-10),(10,10)':: box ORDER BY f1 <-> '0,1'; - f1 ------------------- - (1e-300,-1e-300) - (0,0) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d (-3,4) (-10,0) (10,10) @@ -2275,13 +2237,8 @@ WHERE classid = 'pg_class'::regclass AND index concur_reindex_ind2 | collation "default" | n index concur_reindex_ind2 | column c2 of table concur_reindex_tab | a index concur_reindex_ind3 | column c1 of table concur_reindex_tab | a -<<<<<<< HEAD index concur_reindex_ind3 | column c1 of table concur_reindex_tab | a - index concur_reindex_ind4 | column c1 of table concur_reindex_tab | a -======= - index concur_reindex_ind3 | table concur_reindex_tab | a index concur_reindex_ind4 | collation "default" | n ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d index concur_reindex_ind4 | column c1 of table concur_reindex_tab | a index concur_reindex_ind4 | column c2 of table concur_reindex_tab | a materialized view concur_reindex_matview | schema public | n @@ -2312,13 +2269,8 @@ WHERE classid = 'pg_class'::regclass AND index concur_reindex_ind2 | collation "default" | n index concur_reindex_ind2 | column c2 of table concur_reindex_tab | a index concur_reindex_ind3 | column c1 of table concur_reindex_tab | a -<<<<<<< HEAD index concur_reindex_ind3 | column c1 of table concur_reindex_tab | a - index concur_reindex_ind4 | column c1 of table concur_reindex_tab | a -======= - index concur_reindex_ind3 | table concur_reindex_tab | a index concur_reindex_ind4 | collation "default" | n ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d index concur_reindex_ind4 | column c1 of table concur_reindex_tab | a index concur_reindex_ind4 | column c2 of table concur_reindex_tab | a materialized view concur_reindex_matview | schema public | n @@ -2543,20 +2495,12 @@ SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_ind REINDEX TABLE concur_reindex_part_index; -- error ERROR: "concur_reindex_part_index" is not a table or materialized view REINDEX TABLE CONCURRENTLY concur_reindex_part_index; -- error -<<<<<<< HEAD ERROR: REINDEX CONCURRENTLY is not supported -======= -ERROR: "concur_reindex_part_index" is not a table or materialized view ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d -- Partitioned index with no leaves REINDEX TABLE concur_reindex_part_index_10; -- error ERROR: "concur_reindex_part_index_10" is not a table or materialized view REINDEX TABLE CONCURRENTLY concur_reindex_part_index_10; -- error -<<<<<<< HEAD ERROR: REINDEX CONCURRENTLY is not supported -======= -ERROR: "concur_reindex_part_index_10" is not a table or materialized view ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d -- Cannot run in a transaction block BEGIN; REINDEX INDEX concur_reindex_part_index; @@ -2620,22 +2564,14 @@ SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_inde (1 row) REINDEX INDEX CONCURRENTLY concur_reindex_part_index; -<<<<<<< HEAD ERROR: REINDEX CONCURRENTLY is not supported -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d SELECT * FROM compare_relfilenode_part('reindex_index_status'); relname | relkind | state -------------------------------+---------+-------------------------- concur_reindex_part_index | I | relfilenode is unchanged concur_reindex_part_index_0 | I | relfilenode is unchanged -<<<<<<< HEAD concur_reindex_part_index_0_1 | i | relfilenode is unchanged concur_reindex_part_index_0_2 | i | relfilenode is unchanged -======= - concur_reindex_part_index_0_1 | i | relfilenode has changed - concur_reindex_part_index_0_2 | i | relfilenode has changed ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d concur_reindex_part_index_10 | I | relfilenode is unchanged (5 rows) @@ -2646,20 +2582,12 @@ DROP TABLE reindex_index_status; REINDEX INDEX concur_reindex_part; -- error ERROR: "concur_reindex_part" is not an index REINDEX INDEX CONCURRENTLY concur_reindex_part; -- error -<<<<<<< HEAD ERROR: REINDEX CONCURRENTLY is not supported -======= -ERROR: "concur_reindex_part" is not an index ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d -- Partitioned with no leaves REINDEX INDEX concur_reindex_part_10; -- error ERROR: "concur_reindex_part_10" is not an index REINDEX INDEX CONCURRENTLY concur_reindex_part_10; -- error -<<<<<<< HEAD ERROR: REINDEX CONCURRENTLY is not supported -======= -ERROR: "concur_reindex_part_10" is not an index ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d -- Cannot run in a transaction block BEGIN; REINDEX TABLE concur_reindex_part; @@ -2694,22 +2622,14 @@ SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_inde (1 row) REINDEX TABLE CONCURRENTLY concur_reindex_part; -<<<<<<< HEAD ERROR: REINDEX CONCURRENTLY is not supported -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d SELECT * FROM compare_relfilenode_part('reindex_index_status'); relname | relkind | state -------------------------------+---------+-------------------------- concur_reindex_part_index | I | relfilenode is unchanged concur_reindex_part_index_0 | I | relfilenode is unchanged -<<<<<<< HEAD concur_reindex_part_index_0_1 | i | relfilenode is unchanged concur_reindex_part_index_0_2 | i | relfilenode is unchanged -======= - concur_reindex_part_index_0_1 | i | relfilenode has changed - concur_reindex_part_index_0_2 | i | relfilenode has changed ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d concur_reindex_part_index_10 | I | relfilenode is unchanged (5 rows) diff --git a/src/test/regress/expected/create_index_optimizer.out b/src/test/regress/expected/create_index_optimizer.out index 14a722f51c45..f904250f66e2 100644 --- a/src/test/regress/expected/create_index_optimizer.out +++ b/src/test/regress/expected/create_index_optimizer.out @@ -78,7 +78,7 @@ SET enable_seqscan = ON; SET enable_indexscan = OFF; SET enable_bitmapscan = OFF; SELECT * FROM fast_emp4000 - WHERE home_base @ '(200,200),(2000,1000)'::box + WHERE home_base <@ '(200,200),(2000,1000)'::box ORDER BY (home_base[0])[0]; home_base ----------------------- @@ -98,7 +98,7 @@ SELECT count(*) FROM fast_emp4000 WHERE home_base IS NULL; 278 (1 row) -SELECT * FROM polygon_tbl WHERE f1 ~ '((1,1),(2,2),(2,1))'::polygon +SELECT * FROM polygon_tbl WHERE f1 @> '((1,1),(2,2),(2,1))'::polygon ORDER BY (poly_center(f1))[0]; f1 --------------------- @@ -258,7 +258,7 @@ SET enable_indexscan = ON; SET enable_bitmapscan = OFF; EXPLAIN (COSTS OFF) SELECT * FROM fast_emp4000 - WHERE home_base @ '(200,200),(2000,1000)'::box + WHERE home_base <@ '(200,200),(2000,1000)'::box ORDER BY (home_base[0])[0]; QUERY PLAN ---------------------------------------------------------------------------- @@ -268,13 +268,13 @@ SELECT * FROM fast_emp4000 -> Sort Sort Key: ((home_base[0])[0]) -> Index Scan using grect2ind on fast_emp4000 - Index Cond: (home_base @ '(2000,1000),(200,200)'::box) - Filter: (home_base @ '(2000,1000),(200,200)'::box) + Index Cond: (home_base <@ '(2000,1000),(200,200)'::box) + Filter: (home_base <@ '(2000,1000),(200,200)'::box) Optimizer: Pivotal Optimizer (GPORCA) version 3.83.0 (9 rows) SELECT * FROM fast_emp4000 - WHERE home_base @ '(200,200),(2000,1000)'::box + WHERE home_base <@ '(200,200),(2000,1000)'::box ORDER BY (home_base[0])[0]; home_base ----------------------- @@ -320,7 +320,7 @@ SELECT count(*) FROM fast_emp4000 WHERE home_base IS NULL; (1 row) EXPLAIN (COSTS OFF) -SELECT * FROM polygon_tbl WHERE f1 ~ '((1,1),(2,2),(2,1))'::polygon +SELECT * FROM polygon_tbl WHERE f1 @> '((1,1),(2,2),(2,1))'::polygon ORDER BY (poly_center(f1))[0]; QUERY PLAN ----------------------------------------------------------------------- @@ -330,12 +330,12 @@ SELECT * FROM polygon_tbl WHERE f1 ~ '((1,1),(2,2),(2,1))'::polygon -> Sort Sort Key: ((poly_center(f1))[0]) -> Index Scan using gpolygonind on polygon_tbl - Index Cond: (f1 ~ '((1,1),(2,2),(2,1))'::polygon) - Filter: (f1 ~ '((1,1),(2,2),(2,1))'::polygon) + Index Cond: (f1 @> '((1,1),(2,2),(2,1))'::polygon) + Filter: (f1 @> '((1,1),(2,2),(2,1))'::polygon) Optimizer: Pivotal Optimizer (GPORCA) version 3.83.0 (9 rows) -SELECT * FROM polygon_tbl WHERE f1 ~ '((1,1),(2,2),(2,1))'::polygon +SELECT * FROM polygon_tbl WHERE f1 @> '((1,1),(2,2),(2,1))'::polygon ORDER BY (poly_center(f1))[0]; f1 --------------------- @@ -2231,15 +2231,16 @@ WHERE classid = 'pg_class'::regclass AND obj | objref | deptype ------------------------------------------+------------------------------------------------------------+--------- index concur_reindex_ind1 | constraint concur_reindex_ind1 on table concur_reindex_tab | i + index concur_reindex_ind2 | collation "default" | n index concur_reindex_ind2 | column c2 of table concur_reindex_tab | a index concur_reindex_ind3 | column c1 of table concur_reindex_tab | a index concur_reindex_ind3 | column c1 of table concur_reindex_tab | a - index concur_reindex_ind4 | column c1 of table concur_reindex_tab | a + index concur_reindex_ind4 | collation "default" | n index concur_reindex_ind4 | column c1 of table concur_reindex_tab | a index concur_reindex_ind4 | column c2 of table concur_reindex_tab | a materialized view concur_reindex_matview | schema public | n table concur_reindex_tab | schema public | n -(9 rows) +(10 rows) REINDEX INDEX CONCURRENTLY concur_reindex_ind1; ERROR: REINDEX CONCURRENTLY is not supported @@ -2262,15 +2263,16 @@ WHERE classid = 'pg_class'::regclass AND obj | objref | deptype ------------------------------------------+------------------------------------------------------------+--------- index concur_reindex_ind1 | constraint concur_reindex_ind1 on table concur_reindex_tab | i + index concur_reindex_ind2 | collation "default" | n index concur_reindex_ind2 | column c2 of table concur_reindex_tab | a index concur_reindex_ind3 | column c1 of table concur_reindex_tab | a index concur_reindex_ind3 | column c1 of table concur_reindex_tab | a - index concur_reindex_ind4 | column c1 of table concur_reindex_tab | a + index concur_reindex_ind4 | collation "default" | n index concur_reindex_ind4 | column c1 of table concur_reindex_tab | a index concur_reindex_ind4 | column c2 of table concur_reindex_tab | a materialized view concur_reindex_matview | schema public | n table concur_reindex_tab | schema public | n -(9 rows) +(10 rows) -- Check that comments are preserved CREATE TABLE testcomment (i int); @@ -2734,6 +2736,17 @@ CREATE UNIQUE INDEX concur_exprs_index_pred ON concur_exprs_tab (c1) CREATE UNIQUE INDEX concur_exprs_index_pred_2 ON concur_exprs_tab (c1, (1 / c1)) WHERE ('-H') >= (c2::TEXT) COLLATE "C"; +ANALYZE concur_exprs_tab; +SELECT starelid::regclass, count(*) FROM pg_statistic WHERE starelid IN ( + 'concur_exprs_index_expr'::regclass, + 'concur_exprs_index_pred'::regclass, + 'concur_exprs_index_pred_2'::regclass) + GROUP BY starelid ORDER BY starelid::regclass::text; + starelid | count +-------------------------+------- + concur_exprs_index_expr | 1 +(1 row) + SELECT pg_get_indexdef('concur_exprs_index_expr'::regclass); pg_get_indexdef ------------------------------------------------------------------------------------------------------------------- @@ -2792,6 +2805,17 @@ SELECT pg_get_indexdef('concur_exprs_index_pred_2'::regclass); CREATE UNIQUE INDEX concur_exprs_index_pred_2 ON public.concur_exprs_tab USING btree (c1, ((1 / c1))) WHERE ('-H'::text >= (c2 COLLATE "C")) (1 row) +-- Statistics should remain intact. +SELECT starelid::regclass, count(*) FROM pg_statistic WHERE starelid IN ( + 'concur_exprs_index_expr'::regclass, + 'concur_exprs_index_pred'::regclass, + 'concur_exprs_index_pred_2'::regclass) + GROUP BY starelid ORDER BY starelid::regclass::text; + starelid | count +-------------------------+------- + concur_exprs_index_expr | 1 +(1 row) + DROP TABLE concur_exprs_tab; -- Temporary tables and on-commit actions, where CONCURRENTLY is ignored. -- ON COMMIT PRESERVE ROWS, the default. From 315148f68f42f28fbe88992f4450353c27aec266 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Wed, 3 Jun 2026 13:02:18 +0300 Subject: [PATCH 531/589] Resolve conflicts in src/test/regress/expected/opr_sanity.out (#2637) Commit 2f70fdb0644c32c4154236c2b5c241bec92eac5e removed two operators, however there were GPDB-specific operators in the same test. --- src/test/regress/expected/opr_sanity.out | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index 8339ddda86c9..714d849d2e6b 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -2109,7 +2109,6 @@ ORDER BY 1, 2, 3; 4000 | 26 | >> 4000 | 27 | >>= 4000 | 28 | ^@ -<<<<<<< HEAD 7013 | 1 | *< 7013 | 1 | < 7013 | 1 | << @@ -2128,10 +2127,7 @@ ORDER BY 1, 2, 3; 7013 | 5 | > 7013 | 5 | >> 7013 | 5 | ~>~ -(147 rows) -======= -(123 rows) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d +(145 rows) -- Check that all opclass search operators have selectivity estimators. -- This is not absolutely required, but it seems a reasonable thing From dacd75d9965d40ae631430c0976032866e3baa1c Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Wed, 3 Jun 2026 16:09:27 +0500 Subject: [PATCH 532/589] Fix OID dispatching during index inheritance. (#2639) Commit 5028981 refactored index inheritance during table creation, although the earlier commit 19cd1cf had already disabled OID dispatching during this process. --- src/backend/tcop/utility.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index e8dec6cbb16c..b78c558428ab 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -1779,7 +1779,7 @@ ProcessUtilitySlow(ParseState *pstate, IndexStmt *stmt = (IndexStmt *) parsetree; Oid relid; LOCKMODE lockmode; - bool is_alter_table; + bool is_alter_table = false; // see below if (stmt->concurrent) PreventInTransactionBlock(isTopLevel, @@ -1849,6 +1849,29 @@ ProcessUtilitySlow(ParseState *pstate, list_free(inheritors); } + /* + * Greengage specific behavior: + * Postgres will pass false for is_alter_table for DefineIndex. + * This argument is only used at two places in DefineIndex (in original postgres code): + * 1. the function index_check_primary_key + * 2. print a debug log on what the statement is + * + * In fact when calling DefineIndex here, we can always pass + * false for is_alter_table when it actually comes from expandTableLikeClause: + * for 1, we are sure relationHasPrimaryKey check will pass because we are + * building a new relation with index here. + * for 2, I do not think it will mislead the user if we print it as CreateStmt. + * + * But for Greengage, is_alter_table matters a lot and has to be set false here: + * DefineIndex need to dispatch, and if it is_alter_table is true, Greengage will + * take this as a sub command of AlterTable stmt, thus it will not dispatch and + * lead to errors. Thus, we comment off the following code and pass false for + * is_alter_table for DefineIndex here. + * + * See following discussion for details: + * https://www.postgresql.org/message-id/CANerzActdrdFO1r4RSqK0M2d0Xtwu5t5bH%3DZOoLsAQ%3DHhZrB%3Dg%40mail.gmail.com + */ +#if 0 /* * If the IndexStmt is already transformed, it must have * come from generateClonedIndexStmt, which in current @@ -1859,6 +1882,7 @@ ProcessUtilitySlow(ParseState *pstate, * worth adding a separate bool field for the purpose.) */ is_alter_table = stmt->transformed; +#endif /* Run parse analysis ... */ stmt = transformIndexStmt(relid, stmt, queryString); From 87bda5325016682a18fbe63c13bf79b3c67d7c21 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 4 Jun 2026 13:55:24 +0500 Subject: [PATCH 533/589] Resolve conflicts in src/test/regress/expected/indexing.out (#2649) Commit 9fc2122 in the file src/test/regress/expected/indexing.out changed the error message, while the earlier commit 19cd1cf had already added GPDB-specific tests in the same place. --- src/test/regress/expected/indexing.out | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index 06fe67847148..73731c17d072 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -1304,7 +1304,6 @@ insert into covidxpart values (4, 1); ERROR: duplicate key value violates unique constraint "covidxpart4_a_b_idx" DETAIL: Key (a)=(4) already exists. create unique index on covidxpart (b) include (a); -- should fail -<<<<<<< HEAD ERROR: UNIQUE index must contain all columns in the table's distribution key DETAIL: Distribution key column "a" is not included in the constraint. -- In GPDB, the previous command fails because the distribution key is not @@ -1312,12 +1311,8 @@ DETAIL: Distribution key column "a" is not included in the constraint. -- in upstream. Run another test to exercise the same check as in upstream. create table covidxpart_x (a int, b int) partition by list (a) distributed by (b); create unique index on covidxpart_x (b) include (a); -- should fail -ERROR: insufficient columns in UNIQUE constraint definition -DETAIL: UNIQUE constraint on table "covidxpart_x" lacks column "a" which is part of the partition key. -======= ERROR: unique constraint on partitioned table must include all partitioning columns -DETAIL: UNIQUE constraint on table "covidxpart" lacks column "a" which is part of the partition key. ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d +DETAIL: UNIQUE constraint on table "covidxpart_x" lacks column "a" which is part of the partition key. -- check that detaching a partition also detaches the primary key constraint create table parted_pk_detach_test (a int primary key) partition by list (a); create table parted_pk_detach_test1 partition of parted_pk_detach_test for values in (1); From b310a10b66b0bf7940f1a51d45f9def62e0c7667 Mon Sep 17 00:00:00 2001 From: dimoffon Date: Fri, 29 May 2026 14:11:14 +0300 Subject: [PATCH 534/589] Fix remaining PG14 unit-test failures; document unit-test phase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two genuine PG14 mock-test failures (the rest of the initial -j failures were build races that pass serially): - mock.mk: re-list the server libpgcommon_srv.a/libpgport_srv.a after the mock objects (MOCK_SRV_LIBS) so PG14 fd.c's get_dirent_type() reference resolves against the server common lib instead of pulling in the FRONTEND libpgcommon.a -- which dragged in frontend file_utils.o/fe_memutils.o and broke cdbappendonlyxlog.t with multiple-definition of fsync_fname/ durable_rename (vs fd_mock.o) and palloc/pfree/... (vs mcxt.o). - ftsmessagehandler_test.c: add expect_value for the new two_phase parameter of PG14 ReplicationSlotCreate() (test_HandleFtsWalRepPromoteMirror). - GG_PG_13_14_MERGE_RULES.md: add the previously-undocumented unit-test phase. §11 covers errstart->errstart_cold EXPECT_EREPORT, the GUC coverage test + the 14 PG14 GUCs added to unsync_guc_name.h, mock linkage (uuid_le, get_dirent_type), mock param expectations, and the -j race caveat. §12 tabulates the other API-shape changes (COPY kept monolithic, ConnParams/ simple_prompt, ExecReindex, ProcedureCreate, cluster_rel, hex_encode, errcontext, nodeModifyTable, ReadNextTransactionId, protocol-v2 removal). Full `make -s unittest-check` (backend + src/bin) now passes serially: 53/53 mock suites. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit a8db552f7b2a5e44158a0e7035a0afdb0ef8be99) Changes from original commit 1) GG_PG_13_14_MERGE_RULES.md removed. 2) Changes in ftsmessagehandler_test removed. --- src/backend/mock.mk | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/backend/mock.mk b/src/backend/mock.mk index 442b474e4f5c..8e67c9fe4494 100644 --- a/src/backend/mock.mk +++ b/src/backend/mock.mk @@ -20,6 +20,21 @@ override CPPFLAGS+= -I$(top_srcdir)/src/backend/libpq \ # postgres in src/backend/Makefile doesn't need this and -pthread. MOCK_LIBS := -ldl $(filter-out -ledit, $(LIBS)) $(LDAP_LIBS_BE) $(ICU_LIBS) $(ZSTD_LIBS) +# The server variants of libpgcommon/libpgport are already part of $(OBJFILES) +# (they sit at the end of objfiles.txt), but they are scanned *before* the mock +# objects in the link line. A mocked backend file can reference a symbol that +# lives only in src/common -- e.g. PG14's fd.c references get_dirent_type(), +# which is defined in common/file_utils.c. That reference only becomes +# unresolved once the linker reaches the mock object, i.e. after the server +# archives have already been scanned, so the symbol would otherwise be resolved +# by the FRONTEND libpgcommon.a in $(LIBS) -- pulling in fe_memutils.o / +# file_utils.o and producing "multiple definition" errors against mcxt.o and +# the mock (palloc, fsync_fname, durable_rename, ...). Re-list the *server* +# archives after the mock objects so such late references resolve against the +# server variant (which omits the FRONTEND-only definitions). +MOCK_SRV_LIBS := $(top_builddir)/src/common/libpgcommon_srv.a \ + $(top_builddir)/src/port/libpgport_srv.a + # These files are not linked into test programs. EXCL_OBJS=\ src/backend/main/main.o \ @@ -121,7 +136,7 @@ WRAP_FUNCS=$(addprefix $(WRAP_FLAGS), \ # The test target depends on $(OBJFILES) which would update files including mocks. %.t: $(OBJFILES) $(CMOCKERY_OBJS) $(MOCK_OBJS) %_test.o - $(CXX) $(CFLAGS) $(LDFLAGS) $(call WRAP_FUNCS, $(top_srcdir)/$(subdir)/test/$*_test.c) $(call BACKEND_OBJS, $(top_srcdir)/$(subdir)/$*.o $(patsubst $(MOCK_DIR)/%_mock.o,$(top_builddir)/src/%.o, $^)) $(filter-out %/objfiles.txt, $^) $(MOCK_LIBS) -o $@ + $(CXX) $(CFLAGS) $(LDFLAGS) $(call WRAP_FUNCS, $(top_srcdir)/$(subdir)/test/$*_test.c) $(call BACKEND_OBJS, $(top_srcdir)/$(subdir)/$*.o $(patsubst $(MOCK_DIR)/%_mock.o,$(top_builddir)/src/%.o, $^)) $(filter-out %/objfiles.txt, $^) $(MOCK_SRV_LIBS) $(MOCK_LIBS) -o $@ # We'd like to call only src/backend, but it seems we should build src/port and # src/timezone before src/backend. This is not the case when main build has finished, From 1fa21883151ad0b42fce418e1c2e1e13540613b9 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 4 Jun 2026 18:12:29 +0500 Subject: [PATCH 535/589] Resolve conflicts in src/test/isolation/isolation_schedule (#2643) 1) Commit 1d65416 in src/test/isolation/isolation_schedule added a new reindex-schema test, while earlier GPDB-specific commits had already disabled the adjacent tests. Concurrent reindex is not supported in GPDB. 2) Commits 94bc27b and f481d28 in src/test/isolation/isolation_schedule added new horizons and partition-concurrent-attach tests, while earlier GPDB-specific commits had already disabled the adjacent tests. Concurrent attach partition is not supported in GPDB. --- src/test/isolation/expected/horizons.out | 2 ++ src/test/isolation/isolation_schedule | 31 +++--------------------- 2 files changed, 5 insertions(+), 28 deletions(-) diff --git a/src/test/isolation/expected/horizons.out b/src/test/isolation/expected/horizons.out index 07bbc9832cdf..113faac716d2 100644 --- a/src/test/isolation/expected/horizons.out +++ b/src/test/isolation/expected/horizons.out @@ -18,6 +18,7 @@ step pruner_query_plan: QUERY PLAN Index Only Scan using horizons_tst_data_key on horizons_tst +Optimizer: Postgres query optimizer step pruner_query: SELECT explain_json($$ EXPLAIN (FORMAT json, BUFFERS, ANALYZE) @@ -76,6 +77,7 @@ step pruner_query_plan: QUERY PLAN Index Only Scan using horizons_tst_data_key on horizons_tst +Optimizer: Postgres query optimizer step pruner_query: SELECT explain_json($$ EXPLAIN (FORMAT json, BUFFERS, ANALYZE) diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index 7dceb6470166..a829dec09d88 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -101,8 +101,8 @@ test: delete-abort-savept #test: lock-committed-update #test: lock-committed-keyupdate test: update-locked-tuple -<<<<<<< HEAD #test: reindex-concurrently +#test: reindex-schema #test: propagate-lock-delete #test: tuplelock-conflict #test: tuplelock-update @@ -117,24 +117,6 @@ test: update-locked-tuple #test: skip-locked-2 #test: skip-locked-3 #test: skip-locked-4 -======= -test: reindex-concurrently -test: reindex-schema -test: propagate-lock-delete -test: tuplelock-conflict -test: tuplelock-update -test: tuplelock-upgrade-no-deadlock -test: freeze-the-dead -test: nowait -test: nowait-2 -test: nowait-3 -test: nowait-4 -test: nowait-5 -test: skip-locked -test: skip-locked-2 -test: skip-locked-3 -test: skip-locked-4 ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d test: drop-index-concurrently-1 #test: multiple-cic test: alter-table-1 @@ -151,10 +133,11 @@ test: timeouts test: vacuum-concurrent-drop test: vacuum-conflict test: vacuum-skip-locked -<<<<<<< HEAD +test: horizons #test: predicate-hash #test: predicate-gist #test: predicate-gin +#test: partition-concurrent-attach test: partition-drop-index-locking @@ -162,14 +145,6 @@ test: partition-drop-index-locking # because it fails when test errors from ExecLockRows. The reason is that # GPDB disabled triggers for foreign keys. #test: partition-key-update-1 -======= -test: horizons -test: predicate-hash -test: predicate-gist -test: predicate-gin -test: partition-concurrent-attach -test: partition-key-update-1 ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d test: partition-key-update-2 test: partition-key-update-3 test: partition-key-update-4 From c8f66b7afcf6b393f11b293a1d4527ac413be731 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 4 Jun 2026 18:17:39 +0500 Subject: [PATCH 536/589] Resolve conflicts in src/interfaces/ecpg/test/pg_regress_ecpg.c (#2642) 1) Commit 784b1ba in src/interfaces/ecpg/test/pg_regress_ecpg.c removed the LINEBUFSIZE define. An earlier commit, b4c36a1, had already done the same but also removed the empty line. 2) Commit 784b1ba in src/interfaces/ecpg/test/pg_regress_ecpg.c in the ecpg_start_test function replaced free(testname_dash) and three others below with free(testname_dash.data). An earlier commit, b4c36a1, had already done the same but also removed the empty line. --- src/interfaces/ecpg/test/pg_regress_ecpg.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/interfaces/ecpg/test/pg_regress_ecpg.c b/src/interfaces/ecpg/test/pg_regress_ecpg.c index e6a1dc68f070..6e1d25b1f4a3 100644 --- a/src/interfaces/ecpg/test/pg_regress_ecpg.c +++ b/src/interfaces/ecpg/test/pg_regress_ecpg.c @@ -21,10 +21,7 @@ #include "pg_regress.h" #include "common/string.h" #include "lib/stringinfo.h" -<<<<<<< HEAD -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d static void ecpg_filter(const char *sourcefile, const char *outfile) @@ -164,10 +161,7 @@ ecpg_start_test(const char *testname, unsetenv("PGAPPNAME"); free(appnameenv); -<<<<<<< HEAD -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d free(testname_dash.data); return pid; From 80d9e511193d6954c83abdeeaa999f2af53cdf6a Mon Sep 17 00:00:00 2001 From: Alexander Kondakov Date: Fri, 5 Jun 2026 09:31:33 +0300 Subject: [PATCH 537/589] Fix conflict in errors.sql (#2654) Commit 3c85535 changed VERBOSITY settings to sqlstate in errors.sql. while commit 5212dba added matchsubs for test output for previous setting terse. --- src/test/regress/expected/errors.out | 10 ---------- src/test/regress/sql/errors.sql | 10 ---------- 2 files changed, 20 deletions(-) diff --git a/src/test/regress/expected/errors.out b/src/test/regress/expected/errors.out index e4970d614a03..376ed327a8ca 100644 --- a/src/test/regress/expected/errors.out +++ b/src/test/regress/expected/errors.out @@ -452,20 +452,12 @@ NULL); ERROR: syntax error at or near "NUL" LINE 16: ...L, id2 TEXT NOT NULL PRIMARY KEY, id3 INTEGER NOT NUL, id4 I... ^ -<<<<<<< HEAD -- Check that stack depth detection mechanism works and -- max_stack_depth is not set too high. The full error report is not -- very stable, so show only SQLSTATE and primary error message. create function infinite_recurse() returns int as 'select infinite_recurse()' language sql CONTAINS SQL; \set VERBOSITY sqlstate --- start_matchsubs --- # mpp-2756 --- m/(ERROR|WARNING|CONTEXT|NOTICE):.*stack depth limit exceeded\s+at\s+character/ --- s/\s+at\s+character.*// --- m/ERROR:.*GPDB exception. Aborting Pivotal Optimizer \(GPORCA\).*/ --- s/ERROR:.*GPDB exception. Aborting Pivotal Optimizer \(GPORCA\).*// --- end_matchsubs -- start_ignore select infinite_recurse(); ERROR: 54001 @@ -478,5 +470,3 @@ select 1; -- test that this works 1 (1 row) -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d diff --git a/src/test/regress/sql/errors.sql b/src/test/regress/sql/errors.sql index 4b9d4a1db60d..8cba18ea6ce8 100644 --- a/src/test/regress/sql/errors.sql +++ b/src/test/regress/sql/errors.sql @@ -378,7 +378,6 @@ INT4 UNIQUE NOT NULL); -<<<<<<< HEAD -- Check that stack depth detection mechanism works and -- max_stack_depth is not set too high. The full error report is not @@ -386,17 +385,8 @@ NULL); create function infinite_recurse() returns int as 'select infinite_recurse()' language sql CONTAINS SQL; \set VERBOSITY sqlstate --- start_matchsubs --- # mpp-2756 --- m/(ERROR|WARNING|CONTEXT|NOTICE):.*stack depth limit exceeded\s+at\s+character/ --- s/\s+at\s+character.*// --- m/ERROR:.*GPDB exception. Aborting Pivotal Optimizer \(GPORCA\).*/ --- s/ERROR:.*GPDB exception. Aborting Pivotal Optimizer \(GPORCA\).*// --- end_matchsubs -- start_ignore select infinite_recurse(); -- end_ignore \echo :LAST_ERROR_MESSAGE select 1; -- test that this works -======= ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d From 649b5309ff07be00e9a752df4bec3c727460e7f3 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 5 Jun 2026 11:45:21 +0500 Subject: [PATCH 538/589] Resolve conflicts in src/test/perl/PostgresNode.pm (#2647) 1) Commit 151c0c5 in the file src/test/perl/PostgresNode.pm in the init function removed the explicit setting of replication values, and commit 4964253 returned them back, but in a different order and with different values, while the earlier commit 39c159b had already increased the max_wal_size parameter from 128 to 512, and commit 19cd1cf had already increased max_connections parameter from 10 to 20. 2) Commit 831611b in the file src/test/perl/PostgresNode.pm added a fast checkpoint to the backup function, while the earlier commit 19cd1cf already added target-gp-dbid in the same place. --- src/test/perl/PostgresNode.pm | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index 17f8f05a6359..7caf59b5ac17 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -510,27 +510,17 @@ sub init { print $conf "wal_level = replica\n"; } -<<<<<<< HEAD - print $conf "max_wal_senders = 5\n"; - print $conf "max_replication_slots = 5\n"; - # PG sets this to 128MB but that makes checkpoint too frequent for GPDB. - # 512MB corresponds to the ratio of GPDB seg size (64) over PG seg size (16). - print $conf "max_wal_size = 512MB\n"; - print $conf "shared_buffers = 1MB\n"; - print $conf "wal_log_hints = on\n"; - print $conf "hot_standby = on\n"; - print $conf "max_connections = 20\n"; -======= print $conf "max_wal_senders = 10\n"; print $conf "max_replication_slots = 10\n"; print $conf "wal_log_hints = on\n"; print $conf "hot_standby = on\n"; # conservative settings to ensure we can run multiple postmasters: print $conf "shared_buffers = 1MB\n"; - print $conf "max_connections = 10\n"; + print $conf "max_connections = 20\n"; # limit disk space consumption, too: - print $conf "max_wal_size = 128MB\n"; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + # PG sets this to 128MB but that makes checkpoint too frequent for GPDB. + # 512MB corresponds to the ratio of GPDB seg size (64) over PG seg size (16). + print $conf "max_wal_size = 512MB\n"; } else { @@ -621,15 +611,10 @@ sub backup my $name = $self->name; print "# Taking pg_basebackup $backup_name from node \"$name\"\n"; -<<<<<<< HEAD - TestLib::system_or_bail('pg_basebackup', '-D', $backup_path, '-h', - $self->host, '-p', $self->port, '--no-sync', '--target-gp-dbid', 99); -======= TestLib::system_or_bail( 'pg_basebackup', '-D', $backup_path, '-h', $self->host, '-p', $self->port, '--checkpoint', - 'fast', '--no-sync'); ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + 'fast', '--no-sync', '--target-gp-dbid', 99); print "# Backup finished\n"; return; } From 40af75eb49f2fd065e47a734272acbe56cc9774c Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 5 Jun 2026 14:15:28 +0500 Subject: [PATCH 539/589] Resolve conflicts in insert_conflict.out and updatable_views.out (#2650) 1) Commit 41efb83 in the file src/test/regress/sql/subselect.sql added new tests. Adapt them for the GPDB and add them to the ORCA. 2) Commit ad77039 in the file src/test/regress/sql/updatable_views.sql added new tests. Add them to ORCA. 3) Commit 41efb83 in src/test/regress/expected/updatable_views.out simplified the plans, while earlier GPDB-specific commits had already added GPDB-specific output to the same places. Adapt ORCA as well. 4) Commit 41efb83 in src/test/regress/expected/insert_conflict.out simplified the plan, while earlier GPDB-specific commits had already added GPDB-specific output to the same location. --- src/test/regress/expected/insert_conflict.out | 10 - src/test/regress/expected/subselect.out | 50 +-- .../regress/expected/subselect_optimizer.out | 65 ++++ src/test/regress/expected/updatable_views.out | 284 +++++------------- .../expected/updatable_views_optimizer.out | 241 +++++++-------- 5 files changed, 268 insertions(+), 382 deletions(-) diff --git a/src/test/regress/expected/insert_conflict.out b/src/test/regress/expected/insert_conflict.out index a729560e8a13..477e4277f88f 100644 --- a/src/test/regress/expected/insert_conflict.out +++ b/src/test/regress/expected/insert_conflict.out @@ -53,22 +53,12 @@ explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on con Conflict Filter: (SubPlan 1) -> Result SubPlan 1 -<<<<<<< HEAD -> Result Filter: (ii.key = excluded.key) -> Materialize -> Broadcast Motion 3:1 (slice1; segments: 3) -> Seq Scan on insertconflicttest ii - SubPlan 2 - -> Broadcast Motion 3:1 (slice2; segments: 3) - -> Seq Scan on insertconflicttest ii_1 - Optimizer: Postgres query optimizer -(15 rows) -======= - -> Index Only Scan using both_index_expr_key on insertconflicttest ii - Index Cond: (key = excluded.key) (8 rows) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d -- Neither collation nor operator class specifications are required -- -- supplying them merely *limits* matches to indexes with matching opclasses diff --git a/src/test/regress/expected/subselect.out b/src/test/regress/expected/subselect.out index 24f6039af168..75925377d39d 100644 --- a/src/test/regress/expected/subselect.out +++ b/src/test/regress/expected/subselect.out @@ -889,14 +889,18 @@ rollback; -- to get rid of the bogus operator explain (costs off) select count(*) from tenk1 t where (exists(select 1 from tenk1 k where k.unique1 = t.unique2) or ten < 0); - QUERY PLAN --------------------------------------------------------------- - Aggregate - -> Seq Scan on tenk1 t - Filter: ((hashed SubPlan 2) OR (ten < 0)) - SubPlan 2 - -> Index Only Scan using tenk1_unique1 on tenk1 k -(5 rows) + QUERY PLAN +----------------------------------------------------------------------- + Finalize Aggregate + -> Gather Motion 3:1 (slice1; segments: 3) + -> Partial Aggregate + -> Seq Scan on tenk1 t + Filter: ((hashed SubPlan 2) OR (ten < 0)) + SubPlan 2 + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Seq Scan on tenk1 k + Optimizer: Postgres query optimizer +(9 rows) select count(*) from tenk1 t where (exists(select 1 from tenk1 k where k.unique1 = t.unique2) or ten < 0); @@ -909,18 +913,24 @@ explain (costs off) select count(*) from tenk1 t where (exists(select 1 from tenk1 k where k.unique1 = t.unique2) or ten < 0) and thousand = 1; - QUERY PLAN --------------------------------------------------------------- - Aggregate - -> Bitmap Heap Scan on tenk1 t - Recheck Cond: (thousand = 1) - Filter: ((SubPlan 1) OR (ten < 0)) - -> Bitmap Index Scan on tenk1_thous_tenthous - Index Cond: (thousand = 1) - SubPlan 1 - -> Index Only Scan using tenk1_unique1 on tenk1 k - Index Cond: (unique1 = t.unique2) -(9 rows) + QUERY PLAN +----------------------------------------------------------------------------------- + Finalize Aggregate + -> Gather Motion 3:1 (slice1; segments: 3) + -> Partial Aggregate + -> Bitmap Heap Scan on tenk1 t + Recheck Cond: (thousand = 1) + Filter: ((SubPlan 1) OR (ten < 0)) + -> Bitmap Index Scan on tenk1_thous_tenthous + Index Cond: (thousand = 1) + SubPlan 1 + -> Result + Filter: (k.unique1 = t.unique2) + -> Materialize + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Seq Scan on tenk1 k + Optimizer: Postgres query optimizer +(15 rows) select count(*) from tenk1 t where (exists(select 1 from tenk1 k where k.unique1 = t.unique2) or ten < 0) diff --git a/src/test/regress/expected/subselect_optimizer.out b/src/test/regress/expected/subselect_optimizer.out index 6387b9cd9efb..4667207b1317 100644 --- a/src/test/regress/expected/subselect_optimizer.out +++ b/src/test/regress/expected/subselect_optimizer.out @@ -926,6 +926,71 @@ select * from int8_tbl where q1 in (select c1 from inner_text); (2 rows) rollback; -- to get rid of the bogus operator +-- +-- Test resolution of hashed vs non-hashed implementation of EXISTS subplan +-- +explain (costs off) +select count(*) from tenk1 t +where (exists(select 1 from tenk1 k where k.unique1 = t.unique2) or ten < 0); + QUERY PLAN +------------------------------------------------------------------------------------------------------------------ + Finalize Aggregate + -> Gather Motion 3:1 (slice1; segments: 3) + -> Partial Aggregate + -> HashAggregate + Group Key: tenk1.unique1, tenk1.unique2, tenk1.ten, tenk1.ctid, tenk1.gp_segment_id, (true) + -> Result + Filter: (CASE WHEN (NOT ((true) IS NULL)) THEN true ELSE false END OR (tenk1.ten < 0)) + -> Hash Left Join + Hash Cond: (tenk1.unique2 = tenk1_1.unique1) + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: tenk1.unique2 + -> Seq Scan on tenk1 + -> Hash + -> Seq Scan on tenk1 tenk1_1 + Optimizer: Pivotal Optimizer (GPORCA) +(15 rows) + +select count(*) from tenk1 t +where (exists(select 1 from tenk1 k where k.unique1 = t.unique2) or ten < 0); + count +------- + 10000 +(1 row) + +explain (costs off) +select count(*) from tenk1 t +where (exists(select 1 from tenk1 k where k.unique1 = t.unique2) or ten < 0) + and thousand = 1; + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------- + Aggregate + -> Gather Motion 3:1 (slice1; segments: 3) + -> GroupAggregate + Group Key: tenk1_1.unique1, tenk1_1.unique2, tenk1_1.ten, tenk1_1.thousand, tenk1_1.ctid, tenk1_1.gp_segment_id, (true) + -> Sort + Sort Key: tenk1_1.unique1, tenk1_1.unique2, tenk1_1.ten, tenk1_1.thousand, tenk1_1.ctid, tenk1_1.gp_segment_id, (true) + -> Result + Filter: (CASE WHEN (NOT ((true) IS NULL)) THEN true ELSE false END OR (tenk1_1.ten < 0)) + -> Hash Right Join + Hash Cond: (tenk1.unique1 = tenk1_1.unique2) + -> Seq Scan on tenk1 + -> Hash + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: tenk1_1.unique2 + -> Index Scan using tenk1_thous_tenthous on tenk1 tenk1_1 + Index Cond: (thousand = 1) + Optimizer: Pivotal Optimizer (GPORCA) +(17 rows) + +select count(*) from tenk1 t +where (exists(select 1 from tenk1 k where k.unique1 = t.unique2) or ten < 0) + and thousand = 1; + count +------- + 10 +(1 row) + -- -- Test case for planner bug with nested EXISTS handling -- diff --git a/src/test/regress/expected/updatable_views.out b/src/test/regress/expected/updatable_views.out index d5b3d79443f6..af48d53185e8 100644 --- a/src/test/regress/expected/updatable_views.out +++ b/src/test/regress/expected/updatable_views.out @@ -1915,22 +1915,13 @@ EXPLAIN (costs off) INSERT INTO rw_view1 VALUES (5); Insert on base_tbl b -> Result SubPlan 1 -<<<<<<< HEAD -> Result Filter: (r.a = b.a) -> Materialize -> Broadcast Motion 3:1 (slice1; segments: 3) -> Seq Scan on ref_tbl r - SubPlan 2 - -> Broadcast Motion 3:1 (slice2; segments: 3) - -> Seq Scan on ref_tbl r_1 Optimizer: Postgres query optimizer -(12 rows) -======= - -> Index Only Scan using ref_tbl_pkey on ref_tbl r - Index Cond: (a = b.a) -(5 rows) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d +(9 rows) ANALYZE base_tbl; EXPLAIN (costs off) UPDATE rw_view1 SET a = a + 5; @@ -1944,22 +1935,13 @@ EXPLAIN (costs off) UPDATE rw_view1 SET a = a + 5; -> Seq Scan on base_tbl b -> Seq Scan on ref_tbl r SubPlan 1 -<<<<<<< HEAD -> Result Filter: (r_1.a = b.a) -> Materialize -> Broadcast Motion 3:3 (slice2; segments: 3) -> Seq Scan on ref_tbl r_1 - SubPlan 2 - -> Broadcast Motion 3:3 (slice3; segments: 3) - -> Seq Scan on ref_tbl r_2 Optimizer: Postgres query optimizer -(17 rows) -======= - -> Index Only Scan using ref_tbl_pkey on ref_tbl r_1 - Index Cond: (a = b.a) -(9 rows) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d +(14 rows) DROP TABLE base_tbl, ref_tbl CASCADE; NOTICE: drop cascades to view rw_view1 @@ -2397,26 +2379,20 @@ SELECT * FROM v1 WHERE a=8; EXPLAIN (VERBOSE, COSTS OFF) UPDATE v1 SET a=100 WHERE snoop(a) AND leakproof(a) AND a < 7 AND a != 6; -<<<<<<< HEAD - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -======= - QUERY PLAN ------------------------------------------------------------------------------------------ ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------- Update on public.t1 Update on public.t1 Update on public.t11 t1_1 Update on public.t12 t1_2 Update on public.t111 t1_3 -<<<<<<< HEAD -> Explicit Redistribute Motion 3:3 (slice1; segments: 3) Output: 100, t1.b, t1.c, t1.ctid, t1.gp_segment_id, (DMLAction) -> Split Output: 100, t1.b, t1.c, t1.ctid, t1.gp_segment_id, DMLAction -> Seq Scan on public.t1 Output: 100, t1.b, t1.c, t1.ctid, t1.gp_segment_id - Filter: ((t1.a > 5) AND (t1.a < 7) AND (t1.a <> 6) AND (alternatives: SubPlan 1 (copy 11) or hashed SubPlan 2 (copy 12)) AND snoop(t1.a) AND leakproof(t1.a)) + Filter: ((t1.a > 5) AND (t1.a < 7) AND (t1.a <> 6) AND (SubPlan 1 (copy 11)) AND snoop(t1.a) AND leakproof(t1.a)) SubPlan 1 (copy 11) -> Result Filter: (t12_1.a = t1.a) @@ -2429,123 +2405,66 @@ UPDATE v1 SET a=100 WHERE snoop(a) AND leakproof(a) AND a < 7 AND a != 6; Output: t12_1.a -> Seq Scan on public.t111 t12_2 Output: t12_2.a - SubPlan 2 (copy 12) - -> Broadcast Motion 3:3 (slice3; segments: 3) - Output: t12_4.a - -> Append - -> Seq Scan on public.t12 t12_4 - Output: t12_4.a - -> Seq Scan on public.t111 t12_5 - Output: t12_5.a - -> Explicit Redistribute Motion 3:3 (slice4; segments: 3) + -> Explicit Redistribute Motion 3:3 (slice3; segments: 3) Output: 100, t1_1.b, t1_1.c, t1_1.d, t1_1.ctid, t1_1.gp_segment_id, (DMLAction) -> Split Output: 100, t1_1.b, t1_1.c, t1_1.d, t1_1.ctid, t1_1.gp_segment_id, DMLAction -> Seq Scan on public.t11 t1_1 Output: 100, t1_1.b, t1_1.c, t1_1.d, t1_1.ctid, t1_1.gp_segment_id - Filter: ((t1_1.a > 5) AND (t1_1.a < 7) AND (t1_1.a <> 6) AND (alternatives: SubPlan 1 (copy 13) or hashed SubPlan 2 (copy 14)) AND snoop(t1_1.a) AND leakproof(t1_1.a)) + Filter: ((t1_1.a > 5) AND (t1_1.a < 7) AND (t1_1.a <> 6) AND (SubPlan 1 (copy 13)) AND snoop(t1_1.a) AND leakproof(t1_1.a)) SubPlan 1 (copy 13) -> Result - Filter: (t12_7.a = t1_1.a) + Filter: (t12_4.a = t1_1.a) -> Materialize - Output: t12_7.a - -> Broadcast Motion 3:3 (slice5; segments: 3) - Output: t12_7.a + Output: t12_4.a + -> Broadcast Motion 3:3 (slice4; segments: 3) + Output: t12_4.a -> Append - -> Seq Scan on public.t12 t12_7 - Output: t12_7.a - -> Seq Scan on public.t111 t12_8 - Output: t12_8.a - SubPlan 2 (copy 14) - -> Broadcast Motion 3:3 (slice6; segments: 3) - Output: t12_10.a - -> Append - -> Seq Scan on public.t12 t12_10 - Output: t12_10.a - -> Seq Scan on public.t111 t12_11 - Output: t12_11.a - -> Explicit Redistribute Motion 3:3 (slice7; segments: 3) + -> Seq Scan on public.t12 t12_4 + Output: t12_4.a + -> Seq Scan on public.t111 t12_5 + Output: t12_5.a + -> Explicit Redistribute Motion 3:3 (slice5; segments: 3) Output: 100, t1_2.b, t1_2.c, t1_2.e, t1_2.ctid, t1_2.gp_segment_id, (DMLAction) -> Split Output: 100, t1_2.b, t1_2.c, t1_2.e, t1_2.ctid, t1_2.gp_segment_id, DMLAction -> Seq Scan on public.t12 t1_2 Output: 100, t1_2.b, t1_2.c, t1_2.e, t1_2.ctid, t1_2.gp_segment_id - Filter: ((t1_2.a > 5) AND (t1_2.a < 7) AND (t1_2.a <> 6) AND (alternatives: SubPlan 1 (copy 15) or hashed SubPlan 2 (copy 16)) AND snoop(t1_2.a) AND leakproof(t1_2.a)) + Filter: ((t1_2.a > 5) AND (t1_2.a < 7) AND (t1_2.a <> 6) AND (SubPlan 1 (copy 15)) AND snoop(t1_2.a) AND leakproof(t1_2.a)) SubPlan 1 (copy 15) -> Result - Filter: (t12_13.a = t1_2.a) + Filter: (t12_7.a = t1_2.a) -> Materialize - Output: t12_13.a - -> Broadcast Motion 3:3 (slice8; segments: 3) - Output: t12_13.a + Output: t12_7.a + -> Broadcast Motion 3:3 (slice6; segments: 3) + Output: t12_7.a -> Append - -> Seq Scan on public.t12 t12_13 - Output: t12_13.a - -> Seq Scan on public.t111 t12_14 - Output: t12_14.a - SubPlan 2 (copy 16) - -> Broadcast Motion 3:3 (slice9; segments: 3) - Output: t12_16.a - -> Append - -> Seq Scan on public.t12 t12_16 - Output: t12_16.a - -> Seq Scan on public.t111 t12_17 - Output: t12_17.a - -> Explicit Redistribute Motion 3:3 (slice10; segments: 3) + -> Seq Scan on public.t12 t12_7 + Output: t12_7.a + -> Seq Scan on public.t111 t12_8 + Output: t12_8.a + -> Explicit Redistribute Motion 3:3 (slice7; segments: 3) Output: 100, t1_3.b, t1_3.c, t1_3.d, t1_3.e, t1_3.ctid, t1_3.gp_segment_id, (DMLAction) -> Split Output: 100, t1_3.b, t1_3.c, t1_3.d, t1_3.e, t1_3.ctid, t1_3.gp_segment_id, DMLAction -> Seq Scan on public.t111 t1_3 Output: 100, t1_3.b, t1_3.c, t1_3.d, t1_3.e, t1_3.ctid, t1_3.gp_segment_id - Filter: ((t1_3.a > 5) AND (t1_3.a < 7) AND (t1_3.a <> 6) AND (alternatives: SubPlan 1 (copy 17) or hashed SubPlan 2 (copy 18)) AND snoop(t1_3.a) AND leakproof(t1_3.a)) + Filter: ((t1_3.a > 5) AND (t1_3.a < 7) AND (t1_3.a <> 6) AND (SubPlan 1 (copy 17)) AND snoop(t1_3.a) AND leakproof(t1_3.a)) SubPlan 1 (copy 17) -> Result - Filter: (t12_19.a = t1_3.a) + Filter: (t12_10.a = t1_3.a) -> Materialize - Output: t12_19.a - -> Broadcast Motion 3:3 (slice11; segments: 3) - Output: t12_19.a + Output: t12_10.a + -> Broadcast Motion 3:3 (slice8; segments: 3) + Output: t12_10.a -> Append - -> Seq Scan on public.t12 t12_19 - Output: t12_19.a - -> Seq Scan on public.t111 t12_20 - Output: t12_20.a - SubPlan 2 (copy 18) - -> Broadcast Motion 3:3 (slice12; segments: 3) - Output: t12_22.a - -> Append - -> Seq Scan on public.t12 t12_22 - Output: t12_22.a - -> Seq Scan on public.t111 t12_23 - Output: t12_23.a + -> Seq Scan on public.t12 t12_10 + Output: t12_10.a + -> Seq Scan on public.t111 t12_11 + Output: t12_11.a Optimizer: Postgres query optimizer Settings: enable_mergejoin = 'on', enable_nestloop = 'on', optimizer = 'off' -(115 rows) -======= - -> Index Scan using t1_a_idx on public.t1 - Output: 100, t1.b, t1.c, t1.ctid - Index Cond: ((t1.a > 5) AND (t1.a < 7)) - Filter: ((t1.a <> 6) AND (SubPlan 1) AND snoop(t1.a) AND leakproof(t1.a)) - SubPlan 1 - -> Append - -> Seq Scan on public.t12 t12_1 - Filter: (t12_1.a = t1.a) - -> Seq Scan on public.t111 t12_2 - Filter: (t12_2.a = t1.a) - -> Index Scan using t11_a_idx on public.t11 t1_1 - Output: 100, t1_1.b, t1_1.c, t1_1.d, t1_1.ctid - Index Cond: ((t1_1.a > 5) AND (t1_1.a < 7)) - Filter: ((t1_1.a <> 6) AND (SubPlan 1) AND snoop(t1_1.a) AND leakproof(t1_1.a)) - -> Index Scan using t12_a_idx on public.t12 t1_2 - Output: 100, t1_2.b, t1_2.c, t1_2.e, t1_2.ctid - Index Cond: ((t1_2.a > 5) AND (t1_2.a < 7)) - Filter: ((t1_2.a <> 6) AND (SubPlan 1) AND snoop(t1_2.a) AND leakproof(t1_2.a)) - -> Index Scan using t111_a_idx on public.t111 t1_3 - Output: 100, t1_3.b, t1_3.c, t1_3.d, t1_3.e, t1_3.ctid - Index Cond: ((t1_3.a > 5) AND (t1_3.a < 7)) - Filter: ((t1_3.a <> 6) AND (SubPlan 1) AND snoop(t1_3.a) AND leakproof(t1_3.a)) -(27 rows) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d +(83 rows) UPDATE v1 SET a=100 WHERE snoop(a) AND leakproof(a) AND a < 7 AND a != 6; SELECT * FROM v1 WHERE a=100; -- Nothing should have been changed to 100 @@ -2560,26 +2479,20 @@ SELECT * FROM t1 WHERE a=100; -- Nothing should have been changed to 100 EXPLAIN (VERBOSE, COSTS OFF) UPDATE v1 SET a=a+1 WHERE snoop(a) AND leakproof(a) AND a = 8; -<<<<<<< HEAD - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -======= - QUERY PLAN -------------------------------------------------------------------------- ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------- Update on public.t1 Update on public.t1 Update on public.t11 t1_1 Update on public.t12 t1_2 Update on public.t111 t1_3 -<<<<<<< HEAD -> Explicit Redistribute Motion 1:3 (slice1; segments: 1) Output: ((t1.a + 1)), t1.b, t1.c, t1.ctid, t1.gp_segment_id, (DMLAction) -> Split Output: ((t1.a + 1)), t1.b, t1.c, t1.ctid, t1.gp_segment_id, DMLAction -> Seq Scan on public.t1 Output: (t1.a + 1), t1.b, t1.c, t1.ctid, t1.gp_segment_id - Filter: ((t1.a > 5) AND (t1.a = 8) AND (alternatives: SubPlan 1 (copy 11) or hashed SubPlan 2 (copy 12)) AND snoop(t1.a) AND leakproof(t1.a)) + Filter: ((t1.a > 5) AND (t1.a = 8) AND (SubPlan 1 (copy 11)) AND snoop(t1.a) AND leakproof(t1.a)) SubPlan 1 (copy 11) -> Result Filter: (t12_1.a = t1.a) @@ -2592,123 +2505,66 @@ UPDATE v1 SET a=a+1 WHERE snoop(a) AND leakproof(a) AND a = 8; Output: t12_1.a -> Seq Scan on public.t111 t12_2 Output: t12_2.a - SubPlan 2 (copy 12) - -> Broadcast Motion 3:1 (slice3; segments: 3) - Output: t12_4.a - -> Append - -> Seq Scan on public.t12 t12_4 - Output: t12_4.a - -> Seq Scan on public.t111 t12_5 - Output: t12_5.a - -> Explicit Redistribute Motion 1:3 (slice4; segments: 1) + -> Explicit Redistribute Motion 1:3 (slice3; segments: 1) Output: ((t1_1.a + 1)), t1_1.b, t1_1.c, t1_1.d, t1_1.ctid, t1_1.gp_segment_id, (DMLAction) -> Split Output: ((t1_1.a + 1)), t1_1.b, t1_1.c, t1_1.d, t1_1.ctid, t1_1.gp_segment_id, DMLAction -> Seq Scan on public.t11 t1_1 Output: (t1_1.a + 1), t1_1.b, t1_1.c, t1_1.d, t1_1.ctid, t1_1.gp_segment_id - Filter: ((t1_1.a > 5) AND (t1_1.a = 8) AND (alternatives: SubPlan 1 (copy 13) or hashed SubPlan 2 (copy 14)) AND snoop(t1_1.a) AND leakproof(t1_1.a)) + Filter: ((t1_1.a > 5) AND (t1_1.a = 8) AND (SubPlan 1 (copy 13)) AND snoop(t1_1.a) AND leakproof(t1_1.a)) SubPlan 1 (copy 13) -> Result - Filter: (t12_7.a = t1_1.a) + Filter: (t12_4.a = t1_1.a) -> Materialize - Output: t12_7.a - -> Broadcast Motion 3:1 (slice5; segments: 3) - Output: t12_7.a + Output: t12_4.a + -> Broadcast Motion 3:1 (slice4; segments: 3) + Output: t12_4.a -> Append - -> Seq Scan on public.t12 t12_7 - Output: t12_7.a - -> Seq Scan on public.t111 t12_8 - Output: t12_8.a - SubPlan 2 (copy 14) - -> Broadcast Motion 3:1 (slice6; segments: 3) - Output: t12_10.a - -> Append - -> Seq Scan on public.t12 t12_10 - Output: t12_10.a - -> Seq Scan on public.t111 t12_11 - Output: t12_11.a - -> Explicit Redistribute Motion 1:3 (slice7; segments: 1) + -> Seq Scan on public.t12 t12_4 + Output: t12_4.a + -> Seq Scan on public.t111 t12_5 + Output: t12_5.a + -> Explicit Redistribute Motion 1:3 (slice5; segments: 1) Output: ((t1_2.a + 1)), t1_2.b, t1_2.c, t1_2.e, t1_2.ctid, t1_2.gp_segment_id, (DMLAction) -> Split Output: ((t1_2.a + 1)), t1_2.b, t1_2.c, t1_2.e, t1_2.ctid, t1_2.gp_segment_id, DMLAction -> Seq Scan on public.t12 t1_2 Output: (t1_2.a + 1), t1_2.b, t1_2.c, t1_2.e, t1_2.ctid, t1_2.gp_segment_id - Filter: ((t1_2.a > 5) AND (t1_2.a = 8) AND (alternatives: SubPlan 1 (copy 15) or hashed SubPlan 2 (copy 16)) AND snoop(t1_2.a) AND leakproof(t1_2.a)) + Filter: ((t1_2.a > 5) AND (t1_2.a = 8) AND (SubPlan 1 (copy 15)) AND snoop(t1_2.a) AND leakproof(t1_2.a)) SubPlan 1 (copy 15) -> Result - Filter: (t12_13.a = t1_2.a) + Filter: (t12_7.a = t1_2.a) -> Materialize - Output: t12_13.a - -> Broadcast Motion 3:1 (slice8; segments: 3) - Output: t12_13.a + Output: t12_7.a + -> Broadcast Motion 3:1 (slice6; segments: 3) + Output: t12_7.a -> Append - -> Seq Scan on public.t12 t12_13 - Output: t12_13.a - -> Seq Scan on public.t111 t12_14 - Output: t12_14.a - SubPlan 2 (copy 16) - -> Broadcast Motion 3:1 (slice9; segments: 3) - Output: t12_16.a - -> Append - -> Seq Scan on public.t12 t12_16 - Output: t12_16.a - -> Seq Scan on public.t111 t12_17 - Output: t12_17.a - -> Explicit Redistribute Motion 1:3 (slice10; segments: 1) + -> Seq Scan on public.t12 t12_7 + Output: t12_7.a + -> Seq Scan on public.t111 t12_8 + Output: t12_8.a + -> Explicit Redistribute Motion 1:3 (slice7; segments: 1) Output: ((t1_3.a + 1)), t1_3.b, t1_3.c, t1_3.d, t1_3.e, t1_3.ctid, t1_3.gp_segment_id, (DMLAction) -> Split Output: ((t1_3.a + 1)), t1_3.b, t1_3.c, t1_3.d, t1_3.e, t1_3.ctid, t1_3.gp_segment_id, DMLAction -> Seq Scan on public.t111 t1_3 Output: (t1_3.a + 1), t1_3.b, t1_3.c, t1_3.d, t1_3.e, t1_3.ctid, t1_3.gp_segment_id - Filter: ((t1_3.a > 5) AND (t1_3.a = 8) AND (alternatives: SubPlan 1 (copy 17) or hashed SubPlan 2 (copy 18)) AND snoop(t1_3.a) AND leakproof(t1_3.a)) + Filter: ((t1_3.a > 5) AND (t1_3.a = 8) AND (SubPlan 1 (copy 17)) AND snoop(t1_3.a) AND leakproof(t1_3.a)) SubPlan 1 (copy 17) -> Result - Filter: (t12_19.a = t1_3.a) + Filter: (t12_10.a = t1_3.a) -> Materialize - Output: t12_19.a - -> Broadcast Motion 3:1 (slice11; segments: 3) - Output: t12_19.a + Output: t12_10.a + -> Broadcast Motion 3:1 (slice8; segments: 3) + Output: t12_10.a -> Append - -> Seq Scan on public.t12 t12_19 - Output: t12_19.a - -> Seq Scan on public.t111 t12_20 - Output: t12_20.a - SubPlan 2 (copy 18) - -> Broadcast Motion 3:1 (slice12; segments: 3) - Output: t12_22.a - -> Append - -> Seq Scan on public.t12 t12_22 - Output: t12_22.a - -> Seq Scan on public.t111 t12_23 - Output: t12_23.a + -> Seq Scan on public.t12 t12_10 + Output: t12_10.a + -> Seq Scan on public.t111 t12_11 + Output: t12_11.a Optimizer: Postgres query optimizer Settings: enable_mergejoin = 'on', enable_nestloop = 'on', optimizer = 'off' -(115 rows) -======= - -> Index Scan using t1_a_idx on public.t1 - Output: (t1.a + 1), t1.b, t1.c, t1.ctid - Index Cond: ((t1.a > 5) AND (t1.a = 8)) - Filter: ((SubPlan 1) AND snoop(t1.a) AND leakproof(t1.a)) - SubPlan 1 - -> Append - -> Seq Scan on public.t12 t12_1 - Filter: (t12_1.a = t1.a) - -> Seq Scan on public.t111 t12_2 - Filter: (t12_2.a = t1.a) - -> Index Scan using t11_a_idx on public.t11 t1_1 - Output: (t1_1.a + 1), t1_1.b, t1_1.c, t1_1.d, t1_1.ctid - Index Cond: ((t1_1.a > 5) AND (t1_1.a = 8)) - Filter: ((SubPlan 1) AND snoop(t1_1.a) AND leakproof(t1_1.a)) - -> Index Scan using t12_a_idx on public.t12 t1_2 - Output: (t1_2.a + 1), t1_2.b, t1_2.c, t1_2.e, t1_2.ctid - Index Cond: ((t1_2.a > 5) AND (t1_2.a = 8)) - Filter: ((SubPlan 1) AND snoop(t1_2.a) AND leakproof(t1_2.a)) - -> Index Scan using t111_a_idx on public.t111 t1_3 - Output: (t1_3.a + 1), t1_3.b, t1_3.c, t1_3.d, t1_3.e, t1_3.ctid - Index Cond: ((t1_3.a > 5) AND (t1_3.a = 8)) - Filter: ((SubPlan 1) AND snoop(t1_3.a) AND leakproof(t1_3.a)) -(27 rows) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d +(83 rows) UPDATE v1 SET a=a+1 WHERE snoop(a) AND leakproof(a) AND a = 8; NOTICE: snooped value: 8 (seg0 slice4 127.0.0.1:40000 pid=26920) diff --git a/src/test/regress/expected/updatable_views_optimizer.out b/src/test/regress/expected/updatable_views_optimizer.out index 3b918543b215..37495bdd145c 100644 --- a/src/test/regress/expected/updatable_views_optimizer.out +++ b/src/test/regress/expected/updatable_views_optimizer.out @@ -1483,6 +1483,41 @@ NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to view rw_view1 drop cascades to view rw_view2 drop cascades to view rw_view3 +-- view on table with GENERATED columns +CREATE TABLE base_tbl (id int, idplus1 int GENERATED ALWAYS AS (id + 1) STORED); +CREATE VIEW rw_view1 AS SELECT * FROM base_tbl; +INSERT INTO base_tbl (id) VALUES (1); +INSERT INTO rw_view1 (id) VALUES (2); +INSERT INTO base_tbl (id, idplus1) VALUES (3, DEFAULT); +INSERT INTO rw_view1 (id, idplus1) VALUES (4, DEFAULT); +INSERT INTO base_tbl (id, idplus1) VALUES (5, 6); -- error +ERROR: cannot insert into column "idplus1" +DETAIL: Column "idplus1" is a generated column. +INSERT INTO rw_view1 (id, idplus1) VALUES (6, 7); -- error +ERROR: cannot insert into column "idplus1" +DETAIL: Column "idplus1" is a generated column. +SELECT * FROM base_tbl; + id | idplus1 +----+--------- + 1 | 2 + 2 | 3 + 3 | 4 + 4 | 5 +(4 rows) + +UPDATE base_tbl SET id = 2000 WHERE id = 2; +UPDATE rw_view1 SET id = 3000 WHERE id = 3; +SELECT * FROM base_tbl; + id | idplus1 +------+--------- + 4 | 5 + 2000 | 2001 + 1 | 2 + 3000 | 3001 +(4 rows) + +DROP TABLE base_tbl CASCADE; +NOTICE: drop cascades to view rw_view1 -- inheritance tests CREATE TABLE base_tbl_parent (a int); CREATE TABLE base_tbl_child (CHECK (a > 0)) INHERITS (base_tbl_parent); @@ -1895,9 +1930,6 @@ EXPLAIN (costs off) INSERT INTO rw_view1 VALUES (5); -> Materialize -> Broadcast Motion 3:1 (slice1; segments: 3) -> Seq Scan on ref_tbl r - SubPlan 2 - -> Broadcast Motion 3:1 (slice2; segments: 3) - -> Seq Scan on ref_tbl r_1 Optimizer: Postgres query optimizer (12 rows) @@ -1918,9 +1950,6 @@ EXPLAIN (costs off) UPDATE rw_view1 SET a = a + 5; -> Materialize -> Broadcast Motion 3:3 (slice2; segments: 3) -> Seq Scan on ref_tbl r_1 - SubPlan 2 - -> Broadcast Motion 3:3 (slice3; segments: 3) - -> Seq Scan on ref_tbl r_2 Optimizer: Postgres query optimizer (18 rows) @@ -2376,8 +2405,8 @@ SELECT * FROM v1 WHERE a=8; EXPLAIN (VERBOSE, COSTS OFF) UPDATE v1 SET a=100 WHERE snoop(a) AND leakproof(a) AND a < 7 AND a != 6; - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------- Update on public.t1 Update on public.t1 Update on public.t11 t1_1 @@ -2389,7 +2418,7 @@ UPDATE v1 SET a=100 WHERE snoop(a) AND leakproof(a) AND a < 7 AND a != 6; Output: 100, t1.b, t1.c, t1.ctid, t1.gp_segment_id, DMLAction -> Seq Scan on public.t1 Output: 100, t1.b, t1.c, t1.ctid, t1.gp_segment_id - Filter: ((t1.a > 5) AND (t1.a < 7) AND (t1.a <> 6) AND (alternatives: SubPlan 1 (copy 11) or hashed SubPlan 2 (copy 12)) AND snoop(t1.a) AND leakproof(t1.a)) + Filter: ((t1.a > 5) AND (t1.a < 7) AND (t1.a <> 6) AND (SubPlan 1 (copy 11)) AND snoop(t1.a) AND leakproof(t1.a)) SubPlan 1 (copy 11) -> Result Filter: (t12_1.a = t1.a) @@ -2402,98 +2431,66 @@ UPDATE v1 SET a=100 WHERE snoop(a) AND leakproof(a) AND a < 7 AND a != 6; Output: t12_1.a -> Seq Scan on public.t111 t12_2 Output: t12_2.a - SubPlan 2 (copy 12) - -> Broadcast Motion 3:3 (slice3; segments: 3) - Output: t12_4.a - -> Append - -> Seq Scan on public.t12 t12_4 - Output: t12_4.a - -> Seq Scan on public.t111 t12_5 - Output: t12_5.a - -> Explicit Redistribute Motion 3:3 (slice4; segments: 3) + -> Explicit Redistribute Motion 3:3 (slice3; segments: 3) Output: 100, t1_1.b, t1_1.c, t1_1.d, t1_1.ctid, t1_1.gp_segment_id, (DMLAction) -> Split Output: 100, t1_1.b, t1_1.c, t1_1.d, t1_1.ctid, t1_1.gp_segment_id, DMLAction -> Seq Scan on public.t11 t1_1 Output: 100, t1_1.b, t1_1.c, t1_1.d, t1_1.ctid, t1_1.gp_segment_id - Filter: ((t1_1.a > 5) AND (t1_1.a < 7) AND (t1_1.a <> 6) AND (alternatives: SubPlan 1 (copy 13) or hashed SubPlan 2 (copy 14)) AND snoop(t1_1.a) AND leakproof(t1_1.a)) + Filter: ((t1_1.a > 5) AND (t1_1.a < 7) AND (t1_1.a <> 6) AND (SubPlan 1 (copy 13)) AND snoop(t1_1.a) AND leakproof(t1_1.a)) SubPlan 1 (copy 13) -> Result - Filter: (t12_7.a = t1_1.a) + Filter: (t12_4.a = t1_1.a) -> Materialize - Output: t12_7.a - -> Broadcast Motion 3:3 (slice5; segments: 3) - Output: t12_7.a + Output: t12_4.a + -> Broadcast Motion 3:3 (slice4; segments: 3) + Output: t12_4.a -> Append - -> Seq Scan on public.t12 t12_7 - Output: t12_7.a - -> Seq Scan on public.t111 t12_8 - Output: t12_8.a - SubPlan 2 (copy 14) - -> Broadcast Motion 3:3 (slice6; segments: 3) - Output: t12_10.a - -> Append - -> Seq Scan on public.t12 t12_10 - Output: t12_10.a - -> Seq Scan on public.t111 t12_11 - Output: t12_11.a - -> Explicit Redistribute Motion 3:3 (slice7; segments: 3) + -> Seq Scan on public.t12 t12_4 + Output: t12_4.a + -> Seq Scan on public.t111 t12_5 + Output: t12_5.a + -> Explicit Redistribute Motion 3:3 (slice5; segments: 3) Output: 100, t1_2.b, t1_2.c, t1_2.e, t1_2.ctid, t1_2.gp_segment_id, (DMLAction) -> Split Output: 100, t1_2.b, t1_2.c, t1_2.e, t1_2.ctid, t1_2.gp_segment_id, DMLAction -> Seq Scan on public.t12 t1_2 Output: 100, t1_2.b, t1_2.c, t1_2.e, t1_2.ctid, t1_2.gp_segment_id - Filter: ((t1_2.a > 5) AND (t1_2.a < 7) AND (t1_2.a <> 6) AND (alternatives: SubPlan 1 (copy 15) or hashed SubPlan 2 (copy 16)) AND snoop(t1_2.a) AND leakproof(t1_2.a)) + Filter: ((t1_2.a > 5) AND (t1_2.a < 7) AND (t1_2.a <> 6) AND (SubPlan 1 (copy 15)) AND snoop(t1_2.a) AND leakproof(t1_2.a)) SubPlan 1 (copy 15) -> Result - Filter: (t12_13.a = t1_2.a) + Filter: (t12_7.a = t1_2.a) -> Materialize - Output: t12_13.a - -> Broadcast Motion 3:3 (slice8; segments: 3) - Output: t12_13.a + Output: t12_7.a + -> Broadcast Motion 3:3 (slice6; segments: 3) + Output: t12_7.a -> Append - -> Seq Scan on public.t12 t12_13 - Output: t12_13.a - -> Seq Scan on public.t111 t12_14 - Output: t12_14.a - SubPlan 2 (copy 16) - -> Broadcast Motion 3:3 (slice9; segments: 3) - Output: t12_16.a - -> Append - -> Seq Scan on public.t12 t12_16 - Output: t12_16.a - -> Seq Scan on public.t111 t12_17 - Output: t12_17.a - -> Explicit Redistribute Motion 3:3 (slice10; segments: 3) + -> Seq Scan on public.t12 t12_7 + Output: t12_7.a + -> Seq Scan on public.t111 t12_8 + Output: t12_8.a + -> Explicit Redistribute Motion 3:3 (slice7; segments: 3) Output: 100, t1_3.b, t1_3.c, t1_3.d, t1_3.e, t1_3.ctid, t1_3.gp_segment_id, (DMLAction) -> Split Output: 100, t1_3.b, t1_3.c, t1_3.d, t1_3.e, t1_3.ctid, t1_3.gp_segment_id, DMLAction -> Seq Scan on public.t111 t1_3 Output: 100, t1_3.b, t1_3.c, t1_3.d, t1_3.e, t1_3.ctid, t1_3.gp_segment_id - Filter: ((t1_3.a > 5) AND (t1_3.a < 7) AND (t1_3.a <> 6) AND (alternatives: SubPlan 1 (copy 17) or hashed SubPlan 2 (copy 18)) AND snoop(t1_3.a) AND leakproof(t1_3.a)) + Filter: ((t1_3.a > 5) AND (t1_3.a < 7) AND (t1_3.a <> 6) AND (SubPlan 1 (copy 17)) AND snoop(t1_3.a) AND leakproof(t1_3.a)) SubPlan 1 (copy 17) -> Result - Filter: (t12_19.a = t1_3.a) + Filter: (t12_10.a = t1_3.a) -> Materialize - Output: t12_19.a - -> Broadcast Motion 3:3 (slice11; segments: 3) - Output: t12_19.a + Output: t12_10.a + -> Broadcast Motion 3:3 (slice8; segments: 3) + Output: t12_10.a -> Append - -> Seq Scan on public.t12 t12_19 - Output: t12_19.a - -> Seq Scan on public.t111 t12_20 - Output: t12_20.a - SubPlan 2 (copy 18) - -> Broadcast Motion 3:3 (slice12; segments: 3) - Output: t12_22.a - -> Append - -> Seq Scan on public.t12 t12_22 - Output: t12_22.a - -> Seq Scan on public.t111 t12_23 - Output: t12_23.a + -> Seq Scan on public.t12 t12_10 + Output: t12_10.a + -> Seq Scan on public.t111 t12_11 + Output: t12_11.a Optimizer: Postgres query optimizer Settings: enable_mergejoin = 'on', enable_nestloop = 'on' -(115 rows) +(83 rows) UPDATE v1 SET a=100 WHERE snoop(a) AND leakproof(a) AND a < 7 AND a != 6; SELECT * FROM v1 WHERE a=100; -- Nothing should have been changed to 100 @@ -2508,8 +2505,8 @@ SELECT * FROM t1 WHERE a=100; -- Nothing should have been changed to 100 EXPLAIN (VERBOSE, COSTS OFF) UPDATE v1 SET a=a+1 WHERE snoop(a) AND leakproof(a) AND a = 8; - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------- Update on public.t1 Update on public.t1 Update on public.t11 t1_1 @@ -2521,7 +2518,7 @@ UPDATE v1 SET a=a+1 WHERE snoop(a) AND leakproof(a) AND a = 8; Output: ((t1.a + 1)), t1.b, t1.c, t1.ctid, t1.gp_segment_id, DMLAction -> Seq Scan on public.t1 Output: (t1.a + 1), t1.b, t1.c, t1.ctid, t1.gp_segment_id - Filter: ((t1.a > 5) AND (t1.a = 8) AND (alternatives: SubPlan 1 (copy 11) or hashed SubPlan 2 (copy 12)) AND snoop(t1.a) AND leakproof(t1.a)) + Filter: ((t1.a > 5) AND (t1.a = 8) AND (SubPlan 1 (copy 11)) AND snoop(t1.a) AND leakproof(t1.a)) SubPlan 1 (copy 11) -> Result Filter: (t12_1.a = t1.a) @@ -2534,98 +2531,66 @@ UPDATE v1 SET a=a+1 WHERE snoop(a) AND leakproof(a) AND a = 8; Output: t12_1.a -> Seq Scan on public.t111 t12_2 Output: t12_2.a - SubPlan 2 (copy 12) - -> Broadcast Motion 3:1 (slice3; segments: 3) - Output: t12_4.a - -> Append - -> Seq Scan on public.t12 t12_4 - Output: t12_4.a - -> Seq Scan on public.t111 t12_5 - Output: t12_5.a - -> Explicit Redistribute Motion 1:3 (slice4; segments: 1) + -> Explicit Redistribute Motion 1:3 (slice3; segments: 1) Output: ((t1_1.a + 1)), t1_1.b, t1_1.c, t1_1.d, t1_1.ctid, t1_1.gp_segment_id, (DMLAction) -> Split Output: ((t1_1.a + 1)), t1_1.b, t1_1.c, t1_1.d, t1_1.ctid, t1_1.gp_segment_id, DMLAction -> Seq Scan on public.t11 t1_1 Output: (t1_1.a + 1), t1_1.b, t1_1.c, t1_1.d, t1_1.ctid, t1_1.gp_segment_id - Filter: ((t1_1.a > 5) AND (t1_1.a = 8) AND (alternatives: SubPlan 1 (copy 13) or hashed SubPlan 2 (copy 14)) AND snoop(t1_1.a) AND leakproof(t1_1.a)) + Filter: ((t1_1.a > 5) AND (t1_1.a = 8) AND (SubPlan 1 (copy 13)) AND snoop(t1_1.a) AND leakproof(t1_1.a)) SubPlan 1 (copy 13) -> Result - Filter: (t12_7.a = t1_1.a) + Filter: (t12_4.a = t1_1.a) -> Materialize - Output: t12_7.a - -> Broadcast Motion 3:1 (slice5; segments: 3) - Output: t12_7.a + Output: t12_4.a + -> Broadcast Motion 3:1 (slice4; segments: 3) + Output: t12_4.a -> Append - -> Seq Scan on public.t12 t12_7 - Output: t12_7.a - -> Seq Scan on public.t111 t12_8 - Output: t12_8.a - SubPlan 2 (copy 14) - -> Broadcast Motion 3:1 (slice6; segments: 3) - Output: t12_10.a - -> Append - -> Seq Scan on public.t12 t12_10 - Output: t12_10.a - -> Seq Scan on public.t111 t12_11 - Output: t12_11.a - -> Explicit Redistribute Motion 1:3 (slice7; segments: 1) + -> Seq Scan on public.t12 t12_4 + Output: t12_4.a + -> Seq Scan on public.t111 t12_5 + Output: t12_5.a + -> Explicit Redistribute Motion 1:3 (slice5; segments: 1) Output: ((t1_2.a + 1)), t1_2.b, t1_2.c, t1_2.e, t1_2.ctid, t1_2.gp_segment_id, (DMLAction) -> Split Output: ((t1_2.a + 1)), t1_2.b, t1_2.c, t1_2.e, t1_2.ctid, t1_2.gp_segment_id, DMLAction -> Seq Scan on public.t12 t1_2 Output: (t1_2.a + 1), t1_2.b, t1_2.c, t1_2.e, t1_2.ctid, t1_2.gp_segment_id - Filter: ((t1_2.a > 5) AND (t1_2.a = 8) AND (alternatives: SubPlan 1 (copy 15) or hashed SubPlan 2 (copy 16)) AND snoop(t1_2.a) AND leakproof(t1_2.a)) + Filter: ((t1_2.a > 5) AND (t1_2.a = 8) AND (SubPlan 1 (copy 15)) AND snoop(t1_2.a) AND leakproof(t1_2.a)) SubPlan 1 (copy 15) -> Result - Filter: (t12_13.a = t1_2.a) + Filter: (t12_7.a = t1_2.a) -> Materialize - Output: t12_13.a - -> Broadcast Motion 3:1 (slice8; segments: 3) - Output: t12_13.a + Output: t12_7.a + -> Broadcast Motion 3:1 (slice6; segments: 3) + Output: t12_7.a -> Append - -> Seq Scan on public.t12 t12_13 - Output: t12_13.a - -> Seq Scan on public.t111 t12_14 - Output: t12_14.a - SubPlan 2 (copy 16) - -> Broadcast Motion 3:1 (slice9; segments: 3) - Output: t12_16.a - -> Append - -> Seq Scan on public.t12 t12_16 - Output: t12_16.a - -> Seq Scan on public.t111 t12_17 - Output: t12_17.a - -> Explicit Redistribute Motion 1:3 (slice10; segments: 1) + -> Seq Scan on public.t12 t12_7 + Output: t12_7.a + -> Seq Scan on public.t111 t12_8 + Output: t12_8.a + -> Explicit Redistribute Motion 1:3 (slice7; segments: 1) Output: ((t1_3.a + 1)), t1_3.b, t1_3.c, t1_3.d, t1_3.e, t1_3.ctid, t1_3.gp_segment_id, (DMLAction) -> Split Output: ((t1_3.a + 1)), t1_3.b, t1_3.c, t1_3.d, t1_3.e, t1_3.ctid, t1_3.gp_segment_id, DMLAction -> Seq Scan on public.t111 t1_3 Output: (t1_3.a + 1), t1_3.b, t1_3.c, t1_3.d, t1_3.e, t1_3.ctid, t1_3.gp_segment_id - Filter: ((t1_3.a > 5) AND (t1_3.a = 8) AND (alternatives: SubPlan 1 (copy 17) or hashed SubPlan 2 (copy 18)) AND snoop(t1_3.a) AND leakproof(t1_3.a)) + Filter: ((t1_3.a > 5) AND (t1_3.a = 8) AND (SubPlan 1 (copy 17)) AND snoop(t1_3.a) AND leakproof(t1_3.a)) SubPlan 1 (copy 17) -> Result - Filter: (t12_19.a = t1_3.a) + Filter: (t12_10.a = t1_3.a) -> Materialize - Output: t12_19.a - -> Broadcast Motion 3:1 (slice11; segments: 3) - Output: t12_19.a + Output: t12_10.a + -> Broadcast Motion 3:1 (slice8; segments: 3) + Output: t12_10.a -> Append - -> Seq Scan on public.t12 t12_19 - Output: t12_19.a - -> Seq Scan on public.t111 t12_20 - Output: t12_20.a - SubPlan 2 (copy 18) - -> Broadcast Motion 3:1 (slice12; segments: 3) - Output: t12_22.a - -> Append - -> Seq Scan on public.t12 t12_22 - Output: t12_22.a - -> Seq Scan on public.t111 t12_23 - Output: t12_23.a + -> Seq Scan on public.t12 t12_10 + Output: t12_10.a + -> Seq Scan on public.t111 t12_11 + Output: t12_11.a Optimizer: Postgres query optimizer Settings: enable_mergejoin = 'on', enable_nestloop = 'on' -(115 rows) +(83 rows) UPDATE v1 SET a=a+1 WHERE snoop(a) AND leakproof(a) AND a = 8; NOTICE: snooped value: 8 (seg0 slice4 127.0.0.1:40000 pid=26920) From d82ba72bec414f9d5b9d8f476a335d8f787d61f4 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 5 Jun 2026 14:34:32 +0500 Subject: [PATCH 540/589] Resolve conflicts in src/test/regress/expected/explain.out (#2648) 1) Commit 9d701e6 in the file src/test/regress/sql/explain.sql added new tests. Adapt their output for the GPDB and add them to the ORCA. 2) Commit 9d701e6 in file src/test/regress/expected/explain.out moved "Planning Time" from "Planning" to "Plan", while the earlier commit db2a798 had already added GPDB-specific output to the same place. 3) Commit eabba4a in the file src/test/regress/sql/explain.sql added new tests. Add them to ORCA. 4) Adapt explain_format outputs. --- src/test/regress/expected/explain.out | 263 +++++------------- src/test/regress/expected/explain_format.out | 5 +- .../expected/explain_format_optimizer.out | 5 +- .../regress/expected/explain_optimizer.out | 90 +++++- 4 files changed, 167 insertions(+), 196 deletions(-) diff --git a/src/test/regress/expected/explain.out b/src/test/regress/expected/explain.out index 35983cca5cd9..4576a1c5418b 100644 --- a/src/test/regress/expected/explain.out +++ b/src/test/regress/expected/explain.out @@ -99,7 +99,6 @@ select explain_filter('explain (analyze, buffers, format text) select * from int (8 rows) select explain_filter('explain (analyze, buffers, format json) select * from int8_tbl i8'); -<<<<<<< HEAD explain_filter ---------------------------------------------- [ + @@ -163,7 +162,6 @@ select explain_filter('explain (analyze, buffers, format json) select * from int }, + "Optimizer": "Postgres query optimizer",+ "Planning": { + - "Planning Time": N.N, + "Shared Hit Blocks": N, + "Shared Read Blocks": N, + "Shared Dirtied Blocks": N, + @@ -175,6 +173,7 @@ select explain_filter('explain (analyze, buffers, format json) select * from int "Temp Read Blocks": N, + "Temp Written Blocks": N + }, + + "Planning Time": N.N, + "Triggers": [ + ], + "Slice statistics": [ + @@ -196,58 +195,10 @@ select explain_filter('explain (analyze, buffers, format json) select * from int }, + "Execution Time": N.N + } + -======= - explain_filter ------------------------------------- - [ + - { + - "Plan": { + - "Node Type": "Seq Scan", + - "Parallel Aware": false, + - "Relation Name": "int8_tbl",+ - "Alias": "i8", + - "Startup Cost": N.N, + - "Total Cost": N.N, + - "Plan Rows": N, + - "Plan Width": N, + - "Actual Startup Time": N.N, + - "Actual Total Time": N.N, + - "Actual Rows": N, + - "Actual Loops": N, + - "Shared Hit Blocks": N, + - "Shared Read Blocks": N, + - "Shared Dirtied Blocks": N, + - "Shared Written Blocks": N, + - "Local Hit Blocks": N, + - "Local Read Blocks": N, + - "Local Dirtied Blocks": N, + - "Local Written Blocks": N, + - "Temp Read Blocks": N, + - "Temp Written Blocks": N + - }, + - "Planning": { + - "Shared Hit Blocks": N, + - "Shared Read Blocks": N, + - "Shared Dirtied Blocks": N, + - "Shared Written Blocks": N, + - "Local Hit Blocks": N, + - "Local Read Blocks": N, + - "Local Dirtied Blocks": N, + - "Local Written Blocks": N, + - "Temp Read Blocks": N, + - "Temp Written Blocks": N + - }, + - "Planning Time": N.N, + - "Triggers": [ + - ], + - "Execution Time": N.N + - } + ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d ] (1 row) select explain_filter('explain (analyze, buffers, format xml) select * from int8_tbl i8'); -<<<<<<< HEAD explain_filter ------------------------------------------------------------ + @@ -311,7 +262,6 @@ select explain_filter('explain (analyze, buffers, format xml) select * from int8 + Postgres query optimizer + + - N.N + N + N + N + @@ -323,6 +273,7 @@ select explain_filter('explain (analyze, buffers, format xml) select * from int8 N + N + + + N.N + + + + @@ -344,58 +295,10 @@ select explain_filter('explain (analyze, buffers, format xml) select * from int8 + N.N + + -======= - explain_filter --------------------------------------------------------- - + - + - + - Seq Scan + - false + - int8_tbl + - i8 + - N.N + - N.N + - N + - N + - N.N + - N.N + - N + - N + - N + - N + - N+ - N+ - N + - N + - N + - N + - N + - N + - + - + - N + - N + - N+ - N+ - N + - N + - N + - N + - N + - N + - + - N.N + - + - + - N.N + - + ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d (1 row) select explain_filter('explain (analyze, buffers, format yaml) select * from int8_tbl i8'); -<<<<<<< HEAD explain_filter ----------------------------------------- - Plan: + @@ -453,7 +356,6 @@ select explain_filter('explain (analyze, buffers, format yaml) select * from int Temp Written Blocks: N + Optimizer: "Postgres query optimizer"+ Planning: + - Planning Time: N.N + Shared Hit Blocks: N + Shared Read Blocks: N + Shared Dirtied Blocks: N + @@ -464,6 +366,7 @@ select explain_filter('explain (analyze, buffers, format yaml) select * from int Local Written Blocks: N + Temp Read Blocks: N + Temp Written Blocks: N + + Planning Time: N.N + Triggers: + Slice statistics: + - Slice: N + @@ -475,93 +378,85 @@ select explain_filter('explain (analyze, buffers, format yaml) select * from int Maximum Memory Used: N + Statement statistics: + Memory used: N + -======= - explain_filter -------------------------------- - - Plan: + - Node Type: "Seq Scan" + - Parallel Aware: false + - Relation Name: "int8_tbl"+ - Alias: "i8" + - Startup Cost: N.N + - Total Cost: N.N + - Plan Rows: N + - Plan Width: N + - Actual Startup Time: N.N + - Actual Total Time: N.N + - Actual Rows: N + - Actual Loops: N + - Shared Hit Blocks: N + - Shared Read Blocks: N + - Shared Dirtied Blocks: N + - Shared Written Blocks: N + - Local Hit Blocks: N + - Local Read Blocks: N + - Local Dirtied Blocks: N + - Local Written Blocks: N + - Temp Read Blocks: N + - Temp Written Blocks: N + - Planning: + - Shared Hit Blocks: N + - Shared Read Blocks: N + - Shared Dirtied Blocks: N + - Shared Written Blocks: N + - Local Hit Blocks: N + - Local Read Blocks: N + - Local Dirtied Blocks: N + - Local Written Blocks: N + - Temp Read Blocks: N + - Temp Written Blocks: N + - Planning Time: N.N + - Triggers: + ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d Execution Time: N.N (1 row) select explain_filter('explain (buffers, format text) select * from int8_tbl i8'); - explain_filter ---------------------------------------------------------- - Seq Scan on int8_tbl i8 (cost=N.N..N.N rows=N width=N) -(1 row) + explain_filter +-------------------------------------------------------------------------- + Gather Motion N:N (slice1; segments: N) (cost=N.N..N.N rows=N width=N) + -> Seq Scan on int8_tbl i8 (cost=N.N..N.N rows=N width=N) + Optimizer: Postgres query optimizer +(3 rows) select explain_filter('explain (buffers, format json) select * from int8_tbl i8'); - explain_filter ------------------------------------- - [ + - { + - "Plan": { + - "Node Type": "Seq Scan", + - "Parallel Aware": false, + - "Relation Name": "int8_tbl",+ - "Alias": "i8", + - "Startup Cost": N.N, + - "Total Cost": N.N, + - "Plan Rows": N, + - "Plan Width": N, + - "Shared Hit Blocks": N, + - "Shared Read Blocks": N, + - "Shared Dirtied Blocks": N, + - "Shared Written Blocks": N, + - "Local Hit Blocks": N, + - "Local Read Blocks": N, + - "Local Dirtied Blocks": N, + - "Local Written Blocks": N, + - "Temp Read Blocks": N, + - "Temp Written Blocks": N + - }, + - "Planning": { + - "Shared Hit Blocks": N, + - "Shared Read Blocks": N, + - "Shared Dirtied Blocks": N, + - "Shared Written Blocks": N, + - "Local Hit Blocks": N, + - "Local Read Blocks": N, + - "Local Dirtied Blocks": N, + - "Local Written Blocks": N, + - "Temp Read Blocks": N, + - "Temp Written Blocks": N + - } + - } + + explain_filter +---------------------------------------------- + [ + + { + + "Plan": { + + "Node Type": "Gather Motion", + + "Senders": N, + + "Receivers": N, + + "Slice": N, + + "Segments": N, + + "Gang Type": "primary reader", + + "Parallel Aware": false, + + "Startup Cost": N.N, + + "Total Cost": N.N, + + "Plan Rows": N, + + "Plan Width": N, + + "Shared Hit Blocks": N, + + "Shared Read Blocks": N, + + "Shared Dirtied Blocks": N, + + "Shared Written Blocks": N, + + "Local Hit Blocks": N, + + "Local Read Blocks": N, + + "Local Dirtied Blocks": N, + + "Local Written Blocks": N, + + "Temp Read Blocks": N, + + "Temp Written Blocks": N, + + "Plans": [ + + { + + "Node Type": "Seq Scan", + + "Parent Relationship": "Outer", + + "Slice": N, + + "Segments": N, + + "Gang Type": "primary reader", + + "Parallel Aware": false, + + "Relation Name": "int8_tbl", + + "Alias": "i8", + + "Startup Cost": N.N, + + "Total Cost": N.N, + + "Plan Rows": N, + + "Plan Width": N, + + "Shared Hit Blocks": N, + + "Shared Read Blocks": N, + + "Shared Dirtied Blocks": N, + + "Shared Written Blocks": N, + + "Local Hit Blocks": N, + + "Local Read Blocks": N, + + "Local Dirtied Blocks": N, + + "Local Written Blocks": N, + + "Temp Read Blocks": N, + + "Temp Written Blocks": N + + } + + ] + + }, + + "Optimizer": "Postgres query optimizer",+ + "Planning": { + + "Shared Hit Blocks": N, + + "Shared Read Blocks": N, + + "Shared Dirtied Blocks": N, + + "Shared Written Blocks": N, + + "Local Hit Blocks": N, + + "Local Read Blocks": N, + + "Local Dirtied Blocks": N, + + "Local Written Blocks": N, + + "Temp Read Blocks": N, + + "Temp Written Blocks": N + + } + + } + ] (1 row) @@ -800,8 +695,8 @@ select jsonb_pretty( }, + "Triggers": [ + ], + -<<<<<<< HEAD "Optimizer": "Postgres query optimizer", + + "Planning Time": 0.0, + "Execution Time": 0.0, + "Slice statistics": [ + { + @@ -821,10 +716,6 @@ select jsonb_pretty( "Statement statistics": { + "Memory used": 0 + } + -======= - "Planning Time": 0.0, + - "Execution Time": 0.0 + ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d } + ] (1 row) diff --git a/src/test/regress/expected/explain_format.out b/src/test/regress/expected/explain_format.out index d8d68ede7649..662e8c7d8948 100644 --- a/src/test/regress/expected/explain_format.out +++ b/src/test/regress/expected/explain_format.out @@ -525,8 +525,7 @@ explain_filter Actual Rows: N Actual Loops: N Optimizer: "Postgres query optimizer" - Planning: - Planning Time: N.N + Planning Time: N.N Triggers: Slice statistics: - Slice: N @@ -655,7 +654,7 @@ Execution Time: N.N ms -- Check explain analyze json format output with jit enable select explain_filter_to_json('EXPLAIN (ANALYZE, FORMAT json) SELECT * FROM jit_explain_output LIMIT 10;'); explain_filter_to_json -[{"JIT": {"slice": {"slice": 0, "Timing": {"avg": 0.0, "max": 0.0, "segid": 0, "nworker": 0}, "Functions": {"avg": 0.0, "max": 0.0, "segid": 0, "nworker": 0}}, "Options": {"Inlining": false, "Deforming": true, "Expressions": true, "Optimization": false}}, "Plan": {"Plans": [{"Plans": [{"Plans": [{"Alias": "jit_explain_output", "Slice": 0, "Segments": 0, "Gang Type": "primary reader", "Node Type": "Seq Scan", "Plan Rows": 0, "Plan Width": 0, "Total Cost": 0.0, "Actual Rows": 0, "Actual Loops": 0, "Startup Cost": 0.0, "Relation Name": "jit_explain_output", "Parallel Aware": false, "Actual Total Time": 0.0, "Actual Startup Time": 0.0, "Parent Relationship": "Outer"}], "Slice": 0, "Segments": 0, "Gang Type": "primary reader", "Node Type": "Limit", "Plan Rows": 0, "Plan Width": 0, "Total Cost": 0.0, "Actual Rows": 0, "Actual Loops": 0, "Startup Cost": 0.0, "Parallel Aware": false, "Actual Total Time": 0.0, "Actual Startup Time": 0.0, "Parent Relationship": "Outer"}], "Slice": 0, "Senders": 0, "Segments": 0, "Gang Type": "primary reader", "Node Type": "Gather Motion", "Plan Rows": 0, "Receivers": 0, "Plan Width": 0, "Total Cost": 0.0, "Actual Rows": 0, "Actual Loops": 0, "Startup Cost": 0.0, "Parallel Aware": false, "Actual Total Time": 0.0, "Actual Startup Time": 0.0, "Parent Relationship": "Outer"}], "Slice": 0, "Segments": 0, "Gang Type": "unallocated", "Node Type": "Limit", "Plan Rows": 0, "Plan Width": 0, "Total Cost": 0.0, "Actual Rows": 0, "Actual Loops": 0, "Startup Cost": 0.0, "Parallel Aware": false, "Actual Total Time": 0.0, "Actual Startup Time": 0.0}, "Planning": {"Planning Time": 0.0}, "Triggers": [], "Optimizer": "Postgres query optimizer", "Execution Time": 0.0, "Slice statistics": [{"Slice": 0, "Executor Memory": 0}, {"Slice": 0, "Executor Memory": {"Average": 0, "Workers": 0, "Maximum Memory Used": 0}}], "Statement statistics": {"Memory used": 0}}] +[{"JIT": {"slice": {"slice": 0, "Timing": {"avg": 0.0, "max": 0.0, "segid": 0, "nworker": 0}, "Functions": {"avg": 0.0, "max": 0.0, "segid": 0, "nworker": 0}}, "Options": {"Inlining": false, "Deforming": true, "Expressions": true, "Optimization": false}}, "Plan": {"Plans": [{"Plans": [{"Plans": [{"Alias": "jit_explain_output", "Slice": 0, "Segments": 0, "Gang Type": "primary reader", "Node Type": "Seq Scan", "Plan Rows": 0, "Plan Width": 0, "Total Cost": 0.0, "Actual Rows": 0, "Actual Loops": 0, "Startup Cost": 0.0, "Relation Name": "jit_explain_output", "Parallel Aware": false, "Actual Total Time": 0.0, "Actual Startup Time": 0.0, "Parent Relationship": "Outer"}], "Slice": 0, "Segments": 0, "Gang Type": "primary reader", "Node Type": "Limit", "Plan Rows": 0, "Plan Width": 0, "Total Cost": 0.0, "Actual Rows": 0, "Actual Loops": 0, "Startup Cost": 0.0, "Parallel Aware": false, "Actual Total Time": 0.0, "Actual Startup Time": 0.0, "Parent Relationship": "Outer"}], "Slice": 0, "Senders": 0, "Segments": 0, "Gang Type": "primary reader", "Node Type": "Gather Motion", "Plan Rows": 0, "Receivers": 0, "Plan Width": 0, "Total Cost": 0.0, "Actual Rows": 0, "Actual Loops": 0, "Startup Cost": 0.0, "Parallel Aware": false, "Actual Total Time": 0.0, "Actual Startup Time": 0.0, "Parent Relationship": "Outer"}], "Slice": 0, "Segments": 0, "Gang Type": "unallocated", "Node Type": "Limit", "Plan Rows": 0, "Plan Width": 0, "Total Cost": 0.0, "Actual Rows": 0, "Actual Loops": 0, "Startup Cost": 0.0, "Parallel Aware": false, "Actual Total Time": 0.0, "Actual Startup Time": 0.0}, "Triggers": [], "Optimizer": "Postgres query optimizer", "Planning Time": 0.0, "Execution Time": 0.0, "Slice statistics": [{"Slice": 0, "Executor Memory": 0}, {"Slice": 0, "Executor Memory": {"Average": 0, "Workers": 0, "Maximum Memory Used": 0}}], "Statement statistics": {"Memory used": 0}}] (1 row) RESET jit; RESET jit_above_cost; diff --git a/src/test/regress/expected/explain_format_optimizer.out b/src/test/regress/expected/explain_format_optimizer.out index e1e403f8161f..25c57eca7a02 100644 --- a/src/test/regress/expected/explain_format_optimizer.out +++ b/src/test/regress/expected/explain_format_optimizer.out @@ -475,8 +475,7 @@ explain_filter Index Cond: "(id = boxes.apple_id)" Rows Removed by Index Recheck: N Optimizer: "Pivotal Optimizer (GPORCA)" - Planning: - Planning Time: N.N + Planning Time: N.N Triggers: Slice statistics: - Slice: N @@ -600,7 +599,7 @@ Execution Time: N.N ms -- Check explain analyze json format output with jit enable select explain_filter_to_json('EXPLAIN (ANALYZE, FORMAT json) SELECT * FROM jit_explain_output LIMIT 10;'); explain_filter_to_json -[{"JIT": {"slice": {"slice": 0, "Timing": 0.0, "functions": 0.0}, "Options": {"Inlining": false, "Deforming": true, "Expressions": true, "Optimization": false}}, "Plan": {"Plans": [{"Plans": [{"Alias": "jit_explain_output", "Slice": 0, "Segments": 0, "Gang Type": "primary reader", "Node Type": "Seq Scan", "Plan Rows": 0, "Plan Width": 0, "Total Cost": 0.0, "Actual Rows": 0, "Actual Loops": 0, "Startup Cost": 0.0, "Relation Name": "jit_explain_output", "Parallel Aware": false, "Actual Total Time": 0.0, "Actual Startup Time": 0.0, "Parent Relationship": "Outer"}], "Slice": 0, "Senders": 0, "Segments": 0, "Gang Type": "primary reader", "Node Type": "Gather Motion", "Plan Rows": 0, "Receivers": 0, "Plan Width": 0, "Total Cost": 0.0, "Actual Rows": 0, "Actual Loops": 0, "Startup Cost": 0.0, "Parallel Aware": false, "Actual Total Time": 0.0, "Actual Startup Time": 0.0, "Parent Relationship": "Outer"}], "Slice": 0, "Segments": 0, "Gang Type": "unallocated", "Node Type": "Limit", "Plan Rows": 0, "Plan Width": 0, "Total Cost": 0.0, "Actual Rows": 0, "Actual Loops": 0, "Startup Cost": 0.0, "Parallel Aware": false, "Actual Total Time": 0.0, "Actual Startup Time": 0.0}, "Planning": {"Planning Time": 0.0}, "Triggers": [], "Optimizer": "Pivotal Optimizer (GPORCA)", "Execution Time": 0.0, "Slice statistics": [{"Slice": 0, "Executor Memory": 0}, {"Slice": 0, "Executor Memory": {"Average": 0, "Workers": 0, "Maximum Memory Used": 0}}], "Statement statistics": {"Memory used": 0}}] +[{"JIT": {"slice": {"slice": 0, "Timing": 0.0, "functions": 0.0}, "Options": {"Inlining": false, "Deforming": true, "Expressions": true, "Optimization": false}}, "Plan": {"Plans": [{"Plans": [{"Alias": "jit_explain_output", "Slice": 0, "Segments": 0, "Gang Type": "primary reader", "Node Type": "Seq Scan", "Plan Rows": 0, "Plan Width": 0, "Total Cost": 0.0, "Actual Rows": 0, "Actual Loops": 0, "Startup Cost": 0.0, "Relation Name": "jit_explain_output", "Parallel Aware": false, "Actual Total Time": 0.0, "Actual Startup Time": 0.0, "Parent Relationship": "Outer"}], "Slice": 0, "Senders": 0, "Segments": 0, "Gang Type": "primary reader", "Node Type": "Gather Motion", "Plan Rows": 0, "Receivers": 0, "Plan Width": 0, "Total Cost": 0.0, "Actual Rows": 0, "Actual Loops": 0, "Startup Cost": 0.0, "Parallel Aware": false, "Actual Total Time": 0.0, "Actual Startup Time": 0.0, "Parent Relationship": "Outer"}], "Slice": 0, "Segments": 0, "Gang Type": "unallocated", "Node Type": "Limit", "Plan Rows": 0, "Plan Width": 0, "Total Cost": 0.0, "Actual Rows": 0, "Actual Loops": 0, "Startup Cost": 0.0, "Parallel Aware": false, "Actual Total Time": 0.0, "Actual Startup Time": 0.0}, "Triggers": [], "Optimizer": "Pivotal Optimizer (GPORCA)", "Planning Time": 0.0, "Execution Time": 0.0, "Slice statistics": [{"Slice": 0, "Executor Memory": 0}, {"Slice": 0, "Executor Memory": {"Average": 0, "Workers": 0, "Maximum Memory Used": 0}}], "Statement statistics": {"Memory used": 0}}] (1 row) RESET jit; RESET jit_above_cost; diff --git a/src/test/regress/expected/explain_optimizer.out b/src/test/regress/expected/explain_optimizer.out index b159720312b3..e8e2c61bb1c6 100644 --- a/src/test/regress/expected/explain_optimizer.out +++ b/src/test/regress/expected/explain_optimizer.out @@ -23,6 +23,9 @@ begin -- Ignore text-mode buffers output because it varies depending -- on the system state CONTINUE WHEN (ln ~ ' +Buffers: .*'); + -- Ignore text-mode "Planning:" line because whether it's output + -- varies depending on the system state + CONTINUE WHEN (ln = 'Planning:'); return next ln; end loop; end; @@ -158,7 +161,6 @@ select explain_filter('explain (analyze, buffers, format json) select * from int }, + "Optimizer": "Pivotal Optimizer (GPORCA)",+ "Planning": { + - "Planning Time": N.N, + "Shared Hit Blocks": N, + "Shared Read Blocks": N, + "Shared Dirtied Blocks": N, + @@ -170,6 +172,7 @@ select explain_filter('explain (analyze, buffers, format json) select * from int "Temp Read Blocks": N, + "Temp Written Blocks": N + }, + + "Planning Time": N.N, + "Triggers": [ + ], + "Slice statistics": [ + @@ -258,7 +261,6 @@ select explain_filter('explain (analyze, buffers, format xml) select * from int8 + Pivotal Optimizer (GPORCA) + + - N.N + N + N + N + @@ -270,6 +272,7 @@ select explain_filter('explain (analyze, buffers, format xml) select * from int8 N + N + + + N.N + + + + @@ -352,7 +355,6 @@ select explain_filter('explain (analyze, buffers, format yaml) select * from int Temp Written Blocks: N + Optimizer: "Pivotal Optimizer (GPORCA)"+ Planning: + - Planning Time: N.N + Shared Hit Blocks: N + Shared Read Blocks: N + Shared Dirtied Blocks: N + @@ -363,6 +365,7 @@ select explain_filter('explain (analyze, buffers, format yaml) select * from int Local Written Blocks: N + Temp Read Blocks: N + Temp Written Blocks: N + + Planning Time: N.N + Triggers: + Slice statistics: + - Slice: N + @@ -377,6 +380,85 @@ select explain_filter('explain (analyze, buffers, format yaml) select * from int Execution Time: N.N (1 row) +select explain_filter('explain (buffers, format text) select * from int8_tbl i8'); + explain_filter +-------------------------------------------------------------------------- + Gather Motion N:N (slice1; segments: N) (cost=N.N..N.N rows=N width=N) + -> Seq Scan on int8_tbl (cost=N.N..N.N rows=N width=N) + Optimizer: Pivotal Optimizer (GPORCA) +(3 rows) + +select explain_filter('explain (buffers, format json) select * from int8_tbl i8'); + explain_filter +------------------------------------------------ + [ + + { + + "Plan": { + + "Node Type": "Gather Motion", + + "Senders": N, + + "Receivers": N, + + "Slice": N, + + "Segments": N, + + "Gang Type": "primary reader", + + "Parallel Aware": false, + + "Startup Cost": N.N, + + "Total Cost": N.N, + + "Plan Rows": N, + + "Plan Width": N, + + "Shared Hit Blocks": N, + + "Shared Read Blocks": N, + + "Shared Dirtied Blocks": N, + + "Shared Written Blocks": N, + + "Local Hit Blocks": N, + + "Local Read Blocks": N, + + "Local Dirtied Blocks": N, + + "Local Written Blocks": N, + + "Temp Read Blocks": N, + + "Temp Written Blocks": N, + + "Plans": [ + + { + + "Node Type": "Seq Scan", + + "Parent Relationship": "Outer", + + "Slice": N, + + "Segments": N, + + "Gang Type": "primary reader", + + "Parallel Aware": false, + + "Relation Name": "int8_tbl", + + "Alias": "int8_tbl", + + "Startup Cost": N.N, + + "Total Cost": N.N, + + "Plan Rows": N, + + "Plan Width": N, + + "Shared Hit Blocks": N, + + "Shared Read Blocks": N, + + "Shared Dirtied Blocks": N, + + "Shared Written Blocks": N, + + "Local Hit Blocks": N, + + "Local Read Blocks": N, + + "Local Dirtied Blocks": N, + + "Local Written Blocks": N, + + "Temp Read Blocks": N, + + "Temp Written Blocks": N + + } + + ] + + }, + + "Optimizer": "Pivotal Optimizer (GPORCA)",+ + "Planning": { + + "Shared Hit Blocks": N, + + "Shared Read Blocks": N, + + "Shared Dirtied Blocks": N, + + "Shared Written Blocks": N, + + "Local Hit Blocks": N, + + "Local Read Blocks": N, + + "Local Dirtied Blocks": N, + + "Local Written Blocks": N, + + "Temp Read Blocks": N, + + "Temp Written Blocks": N + + } + + } + + ] +(1 row) + -- SETTINGS option -- We have to ignore other settings that might be imposed by the environment, -- so printing the whole Settings field unfortunately won't do. @@ -593,7 +675,6 @@ select jsonb_pretty( "Shared Written Blocks": 0 + }, + "Planning": { + - "Planning Time": 0.0, + "Local Hit Blocks": 0, + "Temp Read Blocks": 0, + "Local Read Blocks": 0, + @@ -613,6 +694,7 @@ select jsonb_pretty( "Triggers": [ + ], + "Optimizer": "Pivotal Optimizer (GPORCA)", + + "Planning Time": 0.0, + "Execution Time": 0.0, + "Slice statistics": [ + { + From ffead73944b336c21cfa7089c1a9369577cda520 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Fri, 5 Jun 2026 13:11:56 +0300 Subject: [PATCH 541/589] Fix test src/test/regress/expected/gin.out (#2655) Commit 10564ee02ca380f8d614eabc4e80c5d39ea4edad added new test, its output needs to be adapted to GPDB. Also add ORCA output. --- src/test/regress/expected/gin.out | 61 ++++++++------ src/test/regress/expected/gin_optimizer.out | 91 +++++++++++++++++++++ 2 files changed, 127 insertions(+), 25 deletions(-) diff --git a/src/test/regress/expected/gin.out b/src/test/regress/expected/gin.out index 5e754f929cda..ad8bf9925e95 100644 --- a/src/test/regress/expected/gin.out +++ b/src/test/regress/expected/gin.out @@ -228,21 +228,26 @@ select gin_clean_pending_list('t_gin_test_tbl_i_j_idx') is not null; ?column? ---------- t -(1 row) + t + t +(3 rows) analyze t_gin_test_tbl; set enable_seqscan = off; set enable_bitmapscan = on; explain (costs off) select count(*) from t_gin_test_tbl where j @> array[50]; - QUERY PLAN ---------------------------------------------------------- - Aggregate - -> Bitmap Heap Scan on t_gin_test_tbl - Recheck Cond: (j @> '{50}'::integer[]) - -> Bitmap Index Scan on t_gin_test_tbl_i_j_idx - Index Cond: (j @> '{50}'::integer[]) -(5 rows) + QUERY PLAN +--------------------------------------------------------------------- + Finalize Aggregate + -> Gather Motion 3:1 (slice1; segments: 3) + -> Partial Aggregate + -> Bitmap Heap Scan on t_gin_test_tbl + Recheck Cond: (j @> '{50}'::integer[]) + -> Bitmap Index Scan on t_gin_test_tbl_i_j_idx + Index Cond: (j @> '{50}'::integer[]) + Optimizer: Postgres query optimizer +(8 rows) select count(*) from t_gin_test_tbl where j @> array[50]; count @@ -252,14 +257,17 @@ select count(*) from t_gin_test_tbl where j @> array[50]; explain (costs off) select count(*) from t_gin_test_tbl where j @> array[2]; - QUERY PLAN ---------------------------------------------------------- - Aggregate - -> Bitmap Heap Scan on t_gin_test_tbl - Recheck Cond: (j @> '{2}'::integer[]) - -> Bitmap Index Scan on t_gin_test_tbl_i_j_idx - Index Cond: (j @> '{2}'::integer[]) -(5 rows) + QUERY PLAN +--------------------------------------------------------------------- + Finalize Aggregate + -> Gather Motion 3:1 (slice1; segments: 3) + -> Partial Aggregate + -> Bitmap Heap Scan on t_gin_test_tbl + Recheck Cond: (j @> '{2}'::integer[]) + -> Bitmap Index Scan on t_gin_test_tbl_i_j_idx + Index Cond: (j @> '{2}'::integer[]) + Optimizer: Postgres query optimizer +(8 rows) select count(*) from t_gin_test_tbl where j @> array[2]; count @@ -269,14 +277,17 @@ select count(*) from t_gin_test_tbl where j @> array[2]; explain (costs off) select count(*) from t_gin_test_tbl where j @> '{}'::int[]; - QUERY PLAN ---------------------------------------------------------- - Aggregate - -> Bitmap Heap Scan on t_gin_test_tbl - Recheck Cond: (j @> '{}'::integer[]) - -> Bitmap Index Scan on t_gin_test_tbl_i_j_idx - Index Cond: (j @> '{}'::integer[]) -(5 rows) + QUERY PLAN +---------------------------------------------------------------------------------- + Finalize Aggregate + -> Gather Motion 3:1 (slice1; segments: 3) + -> Partial Aggregate + -> Bitmap Heap Scan on t_gin_test_tbl + Recheck Cond: (j OPERATOR(pg_catalog.@>) '{}'::anyarray) + -> Bitmap Index Scan on t_gin_test_tbl_i_j_idx + Index Cond: (j OPERATOR(pg_catalog.@>) '{}'::anyarray) + Optimizer: Postgres query optimizer +(8 rows) select count(*) from t_gin_test_tbl where j @> '{}'::int[]; count diff --git a/src/test/regress/expected/gin_optimizer.out b/src/test/regress/expected/gin_optimizer.out index 3c15bac8c6cf..fdadca31f092 100644 --- a/src/test/regress/expected/gin_optimizer.out +++ b/src/test/regress/expected/gin_optimizer.out @@ -213,6 +213,97 @@ from i @> '{1}' and j @> '{10}' | 2 | 0 | t (10 rows) +reset enable_seqscan; +reset enable_bitmapscan; +-- re-purpose t_gin_test_tbl to test scans involving posting trees +insert into t_gin_test_tbl select array[1, g, g/10], array[2, g, g/10] + from generate_series(1, 20000) g; +select gin_clean_pending_list('t_gin_test_tbl_i_j_idx') is not null; + ?column? +---------- + t + t + t +(3 rows) + +analyze t_gin_test_tbl; +set enable_seqscan = off; +set enable_bitmapscan = on; +explain (costs off) +select count(*) from t_gin_test_tbl where j @> array[50]; + QUERY PLAN +------------------------------------------------------ + Finalize Aggregate + -> Gather Motion 3:1 (slice1; segments: 3) + -> Partial Aggregate + -> Seq Scan on t_gin_test_tbl + Filter: (j @> '{50}'::integer[]) + Optimizer: Pivotal Optimizer (GPORCA) +(6 rows) + +select count(*) from t_gin_test_tbl where j @> array[50]; + count +------- + 11 +(1 row) + +explain (costs off) +select count(*) from t_gin_test_tbl where j @> array[2]; + QUERY PLAN +----------------------------------------------------- + Finalize Aggregate + -> Gather Motion 3:1 (slice1; segments: 3) + -> Partial Aggregate + -> Seq Scan on t_gin_test_tbl + Filter: (j @> '{2}'::integer[]) + Optimizer: Pivotal Optimizer (GPORCA) +(6 rows) + +select count(*) from t_gin_test_tbl where j @> array[2]; + count +------- + 20000 +(1 row) + +explain (costs off) +select count(*) from t_gin_test_tbl where j @> '{}'::int[]; + QUERY PLAN +------------------------------------------------------------------------ + Finalize Aggregate + -> Gather Motion 3:1 (slice1; segments: 3) + -> Partial Aggregate + -> Seq Scan on t_gin_test_tbl + Filter: (j OPERATOR(pg_catalog.@>) '{}'::anyarray) + Optimizer: Pivotal Optimizer (GPORCA) +(6 rows) + +select count(*) from t_gin_test_tbl where j @> '{}'::int[]; + count +------- + 20006 +(1 row) + +-- test vacuuming of posting trees +delete from t_gin_test_tbl where j @> array[2]; +vacuum t_gin_test_tbl; +select count(*) from t_gin_test_tbl where j @> array[50]; + count +------- + 0 +(1 row) + +select count(*) from t_gin_test_tbl where j @> array[2]; + count +------- + 0 +(1 row) + +select count(*) from t_gin_test_tbl where j @> '{}'::int[]; + count +------- + 6 +(1 row) + reset enable_seqscan; reset enable_bitmapscan; drop table t_gin_test_tbl; From 1536691dcf50567c9a4d962ad9cc017c4400e6ef Mon Sep 17 00:00:00 2001 From: Alexander Kondakov Date: Fri, 5 Jun 2026 13:19:20 +0300 Subject: [PATCH 542/589] Fix conflicts in create_table_like.sql (#2641) Commit 19f5a37 added the new test in create_table_like.sql, while upstream has modified the same area in 598f4b0. Resolve the conflict and fix the output. --- src/test/regress/expected/create_table_like.out | 9 +++------ src/test/regress/sql/create_table_like.sql | 6 +----- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/test/regress/expected/create_table_like.out b/src/test/regress/expected/create_table_like.out index e32f5e02f429..7c53f2511355 100644 --- a/src/test/regress/expected/create_table_like.out +++ b/src/test/regress/expected/create_table_like.out @@ -271,7 +271,8 @@ DETAIL: Key (xx)=(10) already exists. DROP TABLE inhg; /* Multiple primary keys creation should fail */ CREATE TABLE inhg (x text, LIKE inhx INCLUDING INDEXES, PRIMARY KEY(x)); /* fails */ -ERROR: multiple primary keys for table "inhg" are not allowed +ERROR: UNIQUE or PRIMARY KEY definitions are incompatible with each other +HINT: When there are multiple PRIMARY KEY / UNIQUE constraints, they must have at least one column in common. CREATE TABLE inhz (xx text DEFAULT 'text', yy int UNIQUE); CREATE UNIQUE INDEX inhz_xx_idx on inhz (xx) WHERE xx <> 'test'; ERROR: UNIQUE index must contain all columns in the table's distribution key @@ -435,9 +436,6 @@ CREATE TABLE inh_error2 (LIKE ctlt4 INCLUDING STORAGE) INHERITS (ctlt1); NOTICE: merging column "a" with inherited definition ERROR: column "a" has a storage parameter conflict DETAIL: MAIN versus EXTENDED -<<<<<<< HEAD -DROP TABLE ctlt12_storage, ctlt12_comments, ctlt1_inh, ctlt13_inh, ctlt13_like, ctlt_all, ctlb, ctla, ctlt1, ctlt2, ctlt3, ctlt4 CASCADE; -======= -- Check that LIKE isn't confused by a system catalog of the same name CREATE TABLE pg_attrdef (LIKE ctlt1 INCLUDING ALL); \d+ public.pg_attrdef @@ -456,8 +454,7 @@ Statistics objects: "public"."pg_attrdef_a_b_stat" (ndistinct, dependencies, mcv) ON a, b FROM public.pg_attrdef DROP TABLE public.pg_attrdef; -DROP TABLE ctlt1, ctlt2, ctlt3, ctlt4, ctlt12_storage, ctlt12_comments, ctlt1_inh, ctlt13_inh, ctlt13_like, ctlt_all, ctla, ctlb CASCADE; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d +DROP TABLE ctlt12_storage, ctlt12_comments, ctlt1_inh, ctlt13_inh, ctlt13_like, ctlt_all, ctlb, ctla, ctlt1, ctlt2, ctlt3, ctlt4 CASCADE; NOTICE: drop cascades to table inhe -- LIKE must respect NO INHERIT property of constraints CREATE TABLE noinh_con_copy (a int CHECK (a > 0) NO INHERIT); diff --git a/src/test/regress/sql/create_table_like.sql b/src/test/regress/sql/create_table_like.sql index b1b710deaf7c..14e23849b1d4 100644 --- a/src/test/regress/sql/create_table_like.sql +++ b/src/test/regress/sql/create_table_like.sql @@ -165,16 +165,12 @@ SELECT s.stxname, objsubid, description FROM pg_description, pg_statistic_ext s CREATE TABLE inh_error1 () INHERITS (ctlt1, ctlt4); CREATE TABLE inh_error2 (LIKE ctlt4 INCLUDING STORAGE) INHERITS (ctlt1); -<<<<<<< HEAD -DROP TABLE ctlt12_storage, ctlt12_comments, ctlt1_inh, ctlt13_inh, ctlt13_like, ctlt_all, ctlb, ctla, ctlt1, ctlt2, ctlt3, ctlt4 CASCADE; -======= -- Check that LIKE isn't confused by a system catalog of the same name CREATE TABLE pg_attrdef (LIKE ctlt1 INCLUDING ALL); \d+ public.pg_attrdef DROP TABLE public.pg_attrdef; -DROP TABLE ctlt1, ctlt2, ctlt3, ctlt4, ctlt12_storage, ctlt12_comments, ctlt1_inh, ctlt13_inh, ctlt13_like, ctlt_all, ctla, ctlb CASCADE; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d +DROP TABLE ctlt12_storage, ctlt12_comments, ctlt1_inh, ctlt13_inh, ctlt13_like, ctlt_all, ctlb, ctla, ctlt1, ctlt2, ctlt3, ctlt4 CASCADE; -- LIKE must respect NO INHERIT property of constraints CREATE TABLE noinh_con_copy (a int CHECK (a > 0) NO INHERIT); From 4b30ae1687e57cbe0952ab6926189d0a5d68fc5f Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Tue, 23 Feb 2021 17:30:21 -0300 Subject: [PATCH 543/589] Reinstate HEAP_XMAX_LOCK_ONLY|HEAP_KEYS_UPDATED as allowed Commit 866e24d47db1 added an assert that HEAP_XMAX_LOCK_ONLY and HEAP_KEYS_UPDATED cannot appear together, on the faulty assumption that the latter necessarily referred to an update and not a tuple lock; but that's wrong, because SELECT FOR UPDATE can use precisely that combination, as evidenced by the amcheck test case added here. Remove the Assert(), and also patch amcheck's verify_heapam.c to not complain if the combination is found. Also, out of overabundance of caution, update (across all branches) README.tuplock to be more explicit about this. Author: Julien Rouhaud Reviewed-by: Mahendra Singh Thalor Reviewed-by: Dilip Kumar Discussion: https://postgr.es/m/20210124061758.GA11756@nol (cherry picked from commit 8deb6b38dc4c7a7fd4719ee45e4b00d62b27dffe) --- contrib/amcheck/t/001_verify_heapam.pl | 14 +++++++++++++- contrib/amcheck/verify_heapam.c | 7 ------- src/backend/access/heap/README.tuplock | 7 ++++--- src/backend/access/heap/hio.c | 8 +++----- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/contrib/amcheck/t/001_verify_heapam.pl b/contrib/amcheck/t/001_verify_heapam.pl index 1581e51f3ca7..8be1069fc20b 100644 --- a/contrib/amcheck/t/001_verify_heapam.pl +++ b/contrib/amcheck/t/001_verify_heapam.pl @@ -4,7 +4,7 @@ use PostgresNode; use TestLib; -use Test::More tests => 79; +use Test::More tests => 80; my ($node, $result); @@ -47,6 +47,9 @@ # fresh_test_table('test'); $node->safe_psql('postgres', q(VACUUM FREEZE test)); +detects_no_corruption( + "verify_heapam('test')", + "all-frozen not corrupted table"); corrupt_first_page('test'); detects_heap_corruption("verify_heapam('test')", "all-frozen corrupted table"); @@ -92,6 +95,15 @@ sub fresh_test_table ALTER TABLE $relname ALTER b SET STORAGE external; INSERT INTO $relname (a, b) (SELECT gs, repeat('b',gs*10) FROM generate_series(1,1000) gs); + BEGIN; + SAVEPOINT s1; + SELECT 1 FROM $relname WHERE a = 42 FOR UPDATE; + UPDATE $relname SET b = b WHERE a = 42; + RELEASE s1; + SAVEPOINT s1; + SELECT 1 FROM $relname WHERE a = 42 FOR UPDATE; + UPDATE $relname SET b = b WHERE a = 42; + COMMIT; )); } diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index 570f44b59ee1..00168bb4292d 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -608,13 +608,6 @@ check_tuple_header_and_visibilty(HeapTupleHeader tuphdr, HeapCheckContext *ctx) ctx->tuphdr->t_hoff, ctx->lp_len)); header_garbled = true; } - if ((ctx->tuphdr->t_infomask & HEAP_XMAX_LOCK_ONLY) && - (ctx->tuphdr->t_infomask2 & HEAP_KEYS_UPDATED)) - { - report_corruption(ctx, - pstrdup("tuple is marked as only locked, but also claims key columns were updated")); - header_garbled = true; - } if ((ctx->tuphdr->t_infomask & HEAP_XMAX_COMMITTED) && (ctx->tuphdr->t_infomask & HEAP_XMAX_IS_MULTI)) diff --git a/src/backend/access/heap/README.tuplock b/src/backend/access/heap/README.tuplock index d03ddf6cdcc8..6441e8baf0e4 100644 --- a/src/backend/access/heap/README.tuplock +++ b/src/backend/access/heap/README.tuplock @@ -146,9 +146,10 @@ The following infomask bits are applicable: FOR UPDATE; this is implemented by the HEAP_KEYS_UPDATED bit. - HEAP_KEYS_UPDATED - This bit lives in t_infomask2. If set, indicates that the XMAX updated - this tuple and changed the key values, or it deleted the tuple. - It's set regardless of whether the XMAX is a TransactionId or a MultiXactId. + This bit lives in t_infomask2. If set, indicates that the operation(s) done + by the XMAX compromise the tuple key, such as a SELECT FOR UPDATE, an UPDATE + that modifies the columns of the key, or a DELETE. It's set regardless of + whether the XMAX is a TransactionId or a MultiXactId. We currently never set the HEAP_XMAX_COMMITTED when the HEAP_XMAX_IS_MULTI bit is set. diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c index b6fbb8550d13..ddf1c5777425 100644 --- a/src/backend/access/heap/hio.c +++ b/src/backend/access/heap/hio.c @@ -49,12 +49,10 @@ RelationPutHeapTuple(Relation relation pg_attribute_unused(), /* * Do not allow tuples with invalid combinations of hint bits to be placed - * on a page. These combinations are detected as corruption by the - * contrib/amcheck logic, so if you disable one or both of these - * assertions, make corresponding changes there. + * on a page. This combination is detected as corruption by the + * contrib/amcheck logic, so if you disable this assertion, make + * corresponding changes there. */ - Assert(!((tuple->t_data->t_infomask & HEAP_XMAX_LOCK_ONLY) && - (tuple->t_data->t_infomask2 & HEAP_KEYS_UPDATED))); Assert(!((tuple->t_data->t_infomask & HEAP_XMAX_COMMITTED) && (tuple->t_data->t_infomask & HEAP_XMAX_IS_MULTI))); From 68a2572cfe67d45c6b085f5f75ff9c546daa8de6 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 5 Jun 2026 21:25:03 +0500 Subject: [PATCH 544/589] Fix tests src/test/regress/expected/query_finish_pending.out (#2653) Commit c8aeaf3 in src/backend/utils/sort/logtape.c added assertions in the LogicalTapeSetBlocks function. The GPDB interrupts the sort and sets QueryFinishPending on purpose in the test query_finish_pending.sql, skipping the assertion for that case. --- src/backend/utils/sort/logtape.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c index 7287c4991f32..0718352b37cd 100644 --- a/src/backend/utils/sort/logtape.c +++ b/src/backend/utils/sort/logtape.c @@ -86,6 +86,9 @@ #include "utils/memdebug.h" #include "utils/memutils.h" +/* GPDB */ +#include "miscadmin.h" + /* * A TapeBlockTrailer is stored at the end of each BLCKSZ block. * @@ -1281,10 +1284,17 @@ long LogicalTapeSetBlocks(LogicalTapeSet *lts) { #ifdef USE_ASSERT_CHECKING - for (int i = 0; i < lts->nTapes; i++) + /* + * GPDB interrupts the sort and set QueryFinishPending on purpose in the + * test query_finish_pending.sql, skipping the assertion for that case. + */ + if (!QueryFinishPending) { - LogicalTape *lt = <s->tapes[i]; - Assert(!lt->writing || lt->buffer == NULL); + for (int i = 0; i < lts->nTapes; i++) + { + LogicalTape *lt = <s->tapes[i]; + Assert(!lt->writing || lt->buffer == NULL); + } } #endif return lts->nBlocksWritten - lts->nHoleBlocks; From 9343e7a35d7574f40cfbd1442e1a14ba99e53192 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Mon, 8 Jun 2026 09:05:31 +0300 Subject: [PATCH 545/589] Resolve conflicts in src/test/regress/expected/create_am.out (#2646) Commit 3165426e54df02a6199c0a216546e5095e875a0a removed the @ operator, so GPDB-specific plan has to be updated. Also update ORCA output for it and also commit d54f99e41541de848a6ca53b3ec060f461e9ab71. --- src/test/regress/expected/create_am.out | 12 +----------- src/test/regress/expected/create_am_optimizer.out | 13 ++++++++----- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/test/regress/expected/create_am.out b/src/test/regress/expected/create_am.out index 98fca3741f74..bc1f1e3d66a0 100644 --- a/src/test/regress/expected/create_am.out +++ b/src/test/regress/expected/create_am.out @@ -46,7 +46,6 @@ EXPLAIN (COSTS OFF) SELECT * FROM fast_emp4000 WHERE home_base <@ '(200,200),(2000,1000)'::box ORDER BY (home_base[0])[0]; -<<<<<<< HEAD QUERY PLAN ---------------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) @@ -54,18 +53,9 @@ SELECT * FROM fast_emp4000 -> Sort Sort Key: ((home_base[0])[0]) -> Index Only Scan using grect2ind2 on fast_emp4000 - Index Cond: (home_base @ '(2000,1000),(200,200)'::box) + Index Cond: (home_base <@ '(2000,1000),(200,200)'::box) Optimizer: Postgres query optimizer (7 rows) -======= - QUERY PLAN ------------------------------------------------------------------ - Sort - Sort Key: ((home_base[0])[0]) - -> Index Only Scan using grect2ind2 on fast_emp4000 - Index Cond: (home_base <@ '(2000,1000),(200,200)'::box) -(4 rows) ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d SELECT * FROM fast_emp4000 WHERE home_base <@ '(200,200),(2000,1000)'::box diff --git a/src/test/regress/expected/create_am_optimizer.out b/src/test/regress/expected/create_am_optimizer.out index a94e800b55c0..c9cd2435095d 100644 --- a/src/test/regress/expected/create_am_optimizer.out +++ b/src/test/regress/expected/create_am_optimizer.out @@ -27,8 +27,6 @@ CREATE OPERATOR CLASS box_ops DEFAULT OPERATOR 10 <<|, OPERATOR 11 |>>, OPERATOR 12 |&>, - OPERATOR 13 ~, - OPERATOR 14 @, FUNCTION 1 gist_box_consistent(internal, box, smallint, oid, internal), FUNCTION 2 gist_box_union(internal, internal), -- don't need compress, decompress, or fetch functions @@ -46,7 +44,7 @@ SET enable_indexscan = ON; SET enable_bitmapscan = OFF; EXPLAIN (COSTS OFF) SELECT * FROM fast_emp4000 - WHERE home_base @ '(200,200),(2000,1000)'::box + WHERE home_base <@ '(200,200),(2000,1000)'::box ORDER BY (home_base[0])[0]; QUERY PLAN ------------------------------------------------------------------------ @@ -56,12 +54,12 @@ SELECT * FROM fast_emp4000 -> Sort Sort Key: ((home_base[0])[0]) -> Seq Scan on fast_emp4000 - Filter: (home_base @ '(2000,1000),(200,200)'::box) + Filter: (home_base <@ '(2000,1000),(200,200)'::box) Optimizer: Pivotal Optimizer (GPORCA) version 3.83.0 (8 rows) SELECT * FROM fast_emp4000 - WHERE home_base @ '(200,200),(2000,1000)'::box + WHERE home_base <@ '(200,200),(2000,1000)'::box ORDER BY (home_base[0])[0]; home_base ----------------------- @@ -110,8 +108,13 @@ ERROR: cannot drop access method gist2 because other objects depend on it DETAIL: index grect2ind2 depends on operator class box_ops for access method gist2 HINT: Use DROP ... CASCADE to drop the dependent objects too. -- Drop access method cascade +-- To prevent a (rare) deadlock against autovacuum, +-- we must lock the table that owns the index that will be dropped +BEGIN; +LOCK TABLE fast_emp4000; DROP ACCESS METHOD gist2 CASCADE; NOTICE: drop cascades to index grect2ind2 +COMMIT; -- -- Test table access methods -- From 602ab3e8e1af76e4dd99bd56f3953e7b3d372b62 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Mon, 8 Jun 2026 09:07:25 +0300 Subject: [PATCH 546/589] Fix test src/test/regress/expected/inherit_optimizer.out (#2644) There is one more issue in nodeModifyTable.c not fixed by commit 2bd01ef5398b94b5aed41670c8465a05eb48ebd0. Tuple routing should normally be done at most once per tuple. ORCA forces tuple routing to be always performed because of dynamic scans. However, INSERT part of split update already does the routing, so we should ignore the forced routing to the wrong partition in case of INSERTs. Also, the routing should only affect the current partition for a single tuple, so resultRelInfo should not be affected. Only update temporary routedResultRelInfo instead, if needed. Also remove outdated assert. --- src/backend/executor/nodeModifyTable.c | 33 +++++++++++++++----------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index 89de2a8f236f..600450533c1a 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -1694,7 +1694,6 @@ ExecSplitUpdate_Insert(ModifyTableState *mtstate, Relation resultRelationDesc; bool partition_constraint_failed; PartitionTupleRouting *proute = mtstate->mt_partition_tuple_routing; - int map_index; TupleConversionMap *tupconv_map; /* @@ -1745,8 +1744,6 @@ ExecSplitUpdate_Insert(ModifyTableState *mtstate, * retrieve the one for this resultRel, we need to know the * position of the resultRel in mtstate->resultRelInfo[]. */ - map_index = resultRelInfo - mtstate->resultRelInfo; - Assert(map_index >= 0 && map_index < mtstate->mt_nplans); tupconv_map = resultRelInfo->ri_ChildToRootMap; if (tupconv_map != NULL) slot = execute_attr_map_slot(tupconv_map->attrMap, @@ -2161,6 +2158,7 @@ ExecModifyTable(PlanState *pstate) EState *estate = node->ps.state; CmdType operation = node->operation; ResultRelInfo *resultRelInfo; + ResultRelInfo *routedResultRelInfo; PlanState *subplanstate; JunkFilter *junkfilter; AttrNumber action_attno; @@ -2412,32 +2410,38 @@ ExecModifyTable(PlanState *pstate) estate, node->canSetTag, false /* splitUpdate */); break; case CMD_UPDATE: - if (castNode(ModifyTable, node->ps.plan)->forceTupleRouting) + + /* + * INSERT part of split update handles the routing by itself, + * no need to force it + */ + if (castNode(ModifyTable, node->ps.plan)->forceTupleRouting && DML_INSERT != action) { PartitionTupleRouting *proute = node->mt_partition_tuple_routing; - ResultRelInfo *partRelInfo; slot = ExecPrepareTupleRouting(node, estate, proute, resultRelInfo, slot, - &partRelInfo); - resultRelInfo = partRelInfo; + &routedResultRelInfo); } + else + routedResultRelInfo = resultRelInfo; + if (!AttributeNumberIsValid(action_attno)) { /* normal non-split UPDATE */ - slot = ExecUpdate(node, resultRelInfo, tupleid, oldtuple, + slot = ExecUpdate(node, routedResultRelInfo, tupleid, oldtuple, slot, planSlot, segid, &node->mt_epqstate, estate, node->canSetTag); } else if (DML_INSERT == action) { - slot = ExecSplitUpdate_Insert(node, resultRelInfo, slot, planSlot, + slot = ExecSplitUpdate_Insert(node, routedResultRelInfo, slot, planSlot, estate, node->canSetTag); } else /* DML_DELETE */ { - slot = ExecDelete(node, resultRelInfo, tupleid, segid, + slot = ExecDelete(node, routedResultRelInfo, tupleid, segid, oldtuple, planSlot, &node->mt_epqstate, estate, false, /* processReturning */ @@ -2451,14 +2455,15 @@ ExecModifyTable(PlanState *pstate) if (castNode(ModifyTable, node->ps.plan)->forceTupleRouting) { PartitionTupleRouting *proute = node->mt_partition_tuple_routing; - ResultRelInfo *partRelInfo; planSlot = ExecPrepareTupleRouting(node, estate, proute, resultRelInfo, slot, - &partRelInfo); - resultRelInfo = partRelInfo; + &routedResultRelInfo); } - slot = ExecDelete(node, resultRelInfo, tupleid, segid, oldtuple, + else + routedResultRelInfo = resultRelInfo; + + slot = ExecDelete(node, routedResultRelInfo, tupleid, segid, oldtuple, planSlot, &node->mt_epqstate, estate, true, /* processReturning */ node->canSetTag, From 28ca034e1a3ad40f882fba28c10c26b297052d5b Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Mon, 8 Jun 2026 09:07:41 +0300 Subject: [PATCH 547/589] Resolve conflicts in src/test/regress/sql/triggers.sql (#2645) Commit ba9f18abd3650e385e9a35df7145a7c38af17e92 added new test, however commit 0f40a2d57e061d1ab01a3d3ac239220af75980fa modified a line nearby. Also modify the test for GPDB by adding a separate distribution key column. --- src/test/regress/expected/triggers.out | 16 ++++++---------- src/test/regress/sql/triggers.sql | 16 ++++++---------- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/src/test/regress/expected/triggers.out b/src/test/regress/expected/triggers.out index 9de644221e60..cfed5f86e724 100644 --- a/src/test/regress/expected/triggers.out +++ b/src/test/regress/expected/triggers.out @@ -227,43 +227,39 @@ select * from trigtest; (0 rows) drop table trigtest; -<<<<<<< HEAD -create sequence ttdummy_seq increment 10 start 0 minvalue 0 cache 1; -======= -- Check behavior with an implicit column default, too (bug #16644) -create table trigtest (a integer); +create table trigtest (a integer, id integer default 0) distributed by (id); create trigger trigger_return_old before insert or delete or update on trigtest for each row execute procedure trigger_return_old(); insert into trigtest values(1); -select * from trigtest; +select a from trigtest; a --- 1 (1 row) alter table trigtest add column b integer default 42 not null; -select * from trigtest; +select a, b from trigtest; a | b ---+---- 1 | 42 (1 row) -update trigtest set a = 2 where a = 1 returning *; +update trigtest set a = 2 where a = 1 returning a, b; a | b ---+---- 1 | 42 (1 row) -select * from trigtest; +select a, b from trigtest; a | b ---+---- 1 | 42 (1 row) drop table trigtest; -create sequence ttdummy_seq increment 10 start 0 minvalue 0; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d +create sequence ttdummy_seq increment 10 start 0 minvalue 0 cache 1; create table tttest ( price_id int4, price_val int4, diff --git a/src/test/regress/sql/triggers.sql b/src/test/regress/sql/triggers.sql index a3bf71991a7b..309772527287 100644 --- a/src/test/regress/sql/triggers.sql +++ b/src/test/regress/sql/triggers.sql @@ -154,29 +154,25 @@ select * from trigtest; drop table trigtest; -<<<<<<< HEAD -create sequence ttdummy_seq increment 10 start 0 minvalue 0 cache 1; -======= -- Check behavior with an implicit column default, too (bug #16644) -create table trigtest (a integer); +create table trigtest (a integer, id integer default 0) distributed by (id); create trigger trigger_return_old before insert or delete or update on trigtest for each row execute procedure trigger_return_old(); insert into trigtest values(1); -select * from trigtest; +select a from trigtest; alter table trigtest add column b integer default 42 not null; -select * from trigtest; -update trigtest set a = 2 where a = 1 returning *; -select * from trigtest; +select a, b from trigtest; +update trigtest set a = 2 where a = 1 returning a, b; +select a, b from trigtest; drop table trigtest; -create sequence ttdummy_seq increment 10 start 0 minvalue 0; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d +create sequence ttdummy_seq increment 10 start 0 minvalue 0 cache 1; create table tttest ( price_id int4, From c6bcf1a7f0f14317adae04891271c9323379e8bd Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Mon, 8 Jun 2026 14:04:17 +0500 Subject: [PATCH 548/589] Fix tests in src/test/regress/expected/limit_optimizer.out (#2661) Commit 3e0242b in src/test/regress/expected/limit.out changed error messages. Change this in ORCA. --- src/test/regress/expected/limit_optimizer.out | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/regress/expected/limit_optimizer.out b/src/test/regress/expected/limit_optimizer.out index 3ce09444e2dc..8dde8c811009 100644 --- a/src/test/regress/expected/limit_optimizer.out +++ b/src/test/regress/expected/limit_optimizer.out @@ -471,7 +471,7 @@ SELECT thousand SELECT ''::text AS two, unique1, unique2, stringu1 FROM onek WHERE unique1 > 50 FETCH FIRST 2 ROW WITH TIES; -ERROR: WITH TIES options can not be specified without ORDER BY clause +ERROR: WITH TIES cannot be specified without ORDER BY clause -- test ruleutils CREATE VIEW limit_thousand_v_1 AS SELECT thousand FROM onek WHERE thousand < 995 ORDER BY thousand FETCH FIRST 5 ROWS WITH TIES OFFSET 10; @@ -505,7 +505,7 @@ View definition: CREATE VIEW limit_thousand_v_3 AS SELECT thousand FROM onek WHERE thousand < 995 ORDER BY thousand FETCH FIRST NULL ROWS WITH TIES; -- fails -ERROR: row count cannot be NULL in FETCH FIRST ... WITH TIES clause +ERROR: row count cannot be null in FETCH FIRST ... WITH TIES clause CREATE VIEW limit_thousand_v_3 AS SELECT thousand FROM onek WHERE thousand < 995 ORDER BY thousand FETCH FIRST (NULL+1) ROWS WITH TIES; \d+ limit_thousand_v_3 From d393e8341de55093319b85f5a5132ca1fd4aacee Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Mon, 8 Jun 2026 14:36:08 +0500 Subject: [PATCH 549/589] Fix tests in src/test/regress/expected/select_parallel_optimizer.out (#2663) Commit be4b0c0 in src/test/regress/sql/select_parallel.sql changed the tests. Change them in ORCA. --- src/test/regress/expected/select_parallel_optimizer.out | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/regress/expected/select_parallel_optimizer.out b/src/test/regress/expected/select_parallel_optimizer.out index 8113d54da3d1..aaf723c01f37 100644 --- a/src/test/regress/expected/select_parallel_optimizer.out +++ b/src/test/regress/expected/select_parallel_optimizer.out @@ -1119,11 +1119,11 @@ EXPLAIN (analyze, timing off, summary off, costs off) SELECT * FROM tenk1; ROLLBACK TO SAVEPOINT settings; -- provoke error in worker +-- (make the error message long enough to require multiple bufferloads) SAVEPOINT settings; SET LOCAL force_parallel_mode = 1; -select stringu1::int2 from tenk1 where unique1 = 1; -ERROR: invalid input syntax for type smallint: "BAAAAA" -CONTEXT: parallel worker +select (stringu1 || repeat('abcd', 5000))::int2 from tenk1 where unique1 = 1; +ERROR: invalid input syntax for type smallint: "BAAAAAabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd" (seg1 slice1 172.22.0.2:8003 pid=350892) ROLLBACK TO SAVEPOINT settings; -- test interaction with set-returning functions SAVEPOINT settings; From e415f82202f5df8ed5c427476648c59698c61368 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Mon, 8 Jun 2026 18:34:17 +0500 Subject: [PATCH 550/589] Fix tests in src/pl/plperl/expected/plperl_call.out (#2665) Commit 2453ea1 added new tests to src/pl/plperl/sql/plperl_call.sql. However, the GPDB consumes trailing spaces when logging. --- src/pl/plperl/expected/plperl_call.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pl/plperl/expected/plperl_call.out b/src/pl/plperl/expected/plperl_call.out index a08b9ff795c6..49e13e3fc3dc 100644 --- a/src/pl/plperl/expected/plperl_call.out +++ b/src/pl/plperl/expected/plperl_call.out @@ -64,7 +64,7 @@ BEGIN RAISE NOTICE '_a: %, _b: %', _a, _b; END $$; -NOTICE: a: 10, b: +NOTICE: a: 10, b: NOTICE: _a: 10, _b: 20 DROP PROCEDURE test_proc1; DROP PROCEDURE test_proc2; From b052523a8f5f3ce69fb7e892a7a14a1e0d0fe636 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Tue, 9 Jun 2026 11:27:58 +0500 Subject: [PATCH 551/589] Fix tests after reltuples changes (#2666) Commit 3d351d9 redefined pg_class.reltuples to be -1 before the first VACUUM or ANALYZE. Adapt the output of GPDB-specific tests - src/test/isolation2/expected/autovacuum-analyze.out - src/test/regress/expected/btree_index.out - src/test/regress/expected/vacuum_gp.out - src/test/regress/expected/qp_misc_jiras.out - src/test/regress/expected/uao_compaction/index_stats.out - src/test/regress/expected/uaocs_compaction/index_stats.out - src/test/regress/expected/uao_ddl/analyze_ao_table_every_dml_row.out - src/test/regress/expected/uao_ddl/analyze_ao_table_every_dml_column.out - src/test/regress/expected/vacuum_full_heap.out - src/test/regress/expected/autostats.out --- src/test/isolation2/output/autovacuum-analyze.source | 6 +++--- src/test/regress/expected/autostats.out | 6 +++--- src/test/regress/expected/btree_index.out | 2 +- src/test/regress/expected/qp_misc_jiras.out | 4 ++-- .../regress/expected/qp_misc_jiras_optimizer.out | 4 ++-- .../regress/expected/uao_compaction/index_stats.out | 8 ++++---- .../expected/uaocs_compaction/index_stats.out | 8 ++++---- src/test/regress/expected/vacuum_full_heap.out | 2 +- src/test/regress/expected/vacuum_gp.out | 12 ++++++------ .../output/uao_ddl/analyze_ao_table_every_dml.source | 2 +- 10 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/test/isolation2/output/autovacuum-analyze.source b/src/test/isolation2/output/autovacuum-analyze.source index ecd90669e916..382b443944cd 100644 --- a/src/test/isolation2/output/autovacuum-analyze.source +++ b/src/test/isolation2/output/autovacuum-analyze.source @@ -243,12 +243,12 @@ select relpages, reltuples from pg_class where oid = 'rankpart_1_prt_5'::regclas select relpages, reltuples from pg_class where oid = 'rankpart_1_prt_6'::regclass; relpages | reltuples ----------+----------- - 0 | 0 + 0 | -1 (1 row) select relpages, reltuples from pg_class where oid = 'rankpart'::regclass; relpages | reltuples ----------+----------- - 0 | 0 + 0 | -1 (1 row) @@ -362,7 +362,7 @@ SELECT count(*) FROM pg_statistic where starelid = 'anaabort'::regclass; select relpages, reltuples from pg_class where oid = 'anaabort'::regclass; relpages | reltuples ----------+----------- - 0 | 0 + 0 | -1 (1 row) 1: END; diff --git a/src/test/regress/expected/autostats.out b/src/test/regress/expected/autostats.out index c3e997c12096..ada8f96aa4a3 100644 --- a/src/test/regress/expected/autostats.out +++ b/src/test/regress/expected/autostats.out @@ -30,7 +30,7 @@ select relname, reltuples from pg_class where relname='autostats_test'; LOG: statement: select relname, reltuples from pg_class where relname='autostats_test'; relname | reltuples ----------------+----------- - autostats_test | 0 + autostats_test | -1 (1 row) -- Try it with gp_autostats_allow_nonowner GUC enabled, but as a non-owner @@ -48,7 +48,7 @@ select relname, reltuples from pg_class where relname='autostats_test'; LOG: statement: select relname, reltuples from pg_class where relname='autostats_test'; relname | reltuples ----------------+----------- - autostats_test | 0 + autostats_test | -1 (1 row) reset role; @@ -60,7 +60,7 @@ set role=autostats_nonowner; LOG: statement: set role=autostats_nonowner; insert into autostats_test select generate_series(11, 20); LOG: statement: insert into autostats_test select generate_series(11, 20); -LOG: In mode on_change, command INSERT on (dboid,tableoid)=(XXXXX,XXXXX) modifying 10 tuples caused Auto-ANALYZE. +LOG: In mode on_change, command INSERT on (dboid,tableoid)=(72126,72259) modifying 10 tuples caused Auto-ANALYZE. select relname, reltuples from pg_class where relname='autostats_test'; LOG: statement: select relname, reltuples from pg_class where relname='autostats_test'; relname | reltuples diff --git a/src/test/regress/expected/btree_index.out b/src/test/regress/expected/btree_index.out index eae1d52b3558..48ada51edae8 100644 --- a/src/test/regress/expected/btree_index.out +++ b/src/test/regress/expected/btree_index.out @@ -377,7 +377,7 @@ INSERT INTO btree_stats_tbl VALUES (1, 'aa', 1001, 101), (2, 'bb', 1002, 102); SELECT reltuples FROM pg_class WHERE relname='btree_stats_tbl'; reltuples ----------- - 0 + -1 (1 row) -- inspect the state of the stats on segments diff --git a/src/test/regress/expected/qp_misc_jiras.out b/src/test/regress/expected/qp_misc_jiras.out index 6d3f638eb1e2..1e3b8a64726c 100644 --- a/src/test/regress/expected/qp_misc_jiras.out +++ b/src/test/regress/expected/qp_misc_jiras.out @@ -2967,7 +2967,7 @@ select relname, reltuples, relpages from pg_class where relname like 'tbl6833_an ------------------------+-----------+---------- tbl6833_anl_1_prt_bb_1 | 0 | 1 tbl6833_anl_1_prt_bb_2 | 0 | 1 - tbl6833_anl | 0 | 0 + tbl6833_anl | -1 | 0 (3 rows) create table qp_misc_jiras.tbl6833_vac(a int, b int, c int) distributed by (a) partition by range (b) (partition bb start (0) end (2) every (1)); @@ -2987,7 +2987,7 @@ select relname, reltuples, relpages from pg_class where relname like 'tbl6833_va ------------------------+-----------+---------- tbl6833_vac_1_prt_bb_1 | 0 | 1 tbl6833_vac_1_prt_bb_2 | 0 | 1 - tbl6833_vac | 0 | 0 + tbl6833_vac | -1 | 0 (3 rows) drop table qp_misc_jiras.tbl6833_anl; diff --git a/src/test/regress/expected/qp_misc_jiras_optimizer.out b/src/test/regress/expected/qp_misc_jiras_optimizer.out index 064a8fc26b5c..63ad17804046 100644 --- a/src/test/regress/expected/qp_misc_jiras_optimizer.out +++ b/src/test/regress/expected/qp_misc_jiras_optimizer.out @@ -2963,7 +2963,7 @@ select relname, reltuples, relpages from pg_class where relname like 'tbl6833_an ------------------------+-----------+---------- tbl6833_anl_1_prt_bb_1 | 0 | 1 tbl6833_anl_1_prt_bb_2 | 0 | 1 - tbl6833_anl | 0 | 0 + tbl6833_anl | -1 | 0 (3 rows) create table qp_misc_jiras.tbl6833_vac(a int, b int, c int) distributed by (a) partition by range (b) (partition bb start (0) end (2) every (1)); @@ -2980,7 +2980,7 @@ explain select * from qp_misc_jiras.tbl6833_vac; -- should not hit cdbRelSize(); select relname, reltuples, relpages from pg_class where relname like 'tbl6833_vac%'; -- should show relpages = 1.0 relname | reltuples | relpages ------------------------+-----------+---------- - tbl6833_vac | 0 | 0 + tbl6833_vac | -1 | 0 tbl6833_vac_1_prt_bb_1 | 0 | 1 tbl6833_vac_1_prt_bb_2 | 0 | 1 (3 rows) diff --git a/src/test/regress/expected/uao_compaction/index_stats.out b/src/test/regress/expected/uao_compaction/index_stats.out index 0eb52a03c8ec..5e328ba673d3 100644 --- a/src/test/regress/expected/uao_compaction/index_stats.out +++ b/src/test/regress/expected/uao_compaction/index_stats.out @@ -59,9 +59,9 @@ vacuum mytab2; SELECT gp_segment_id, relname, reltuples FROM gp_dist_random('pg_class') WHERE relname = 'mytab2_int_idx1'; gp_segment_id | relname | reltuples ---------------+-----------------+----------- - 0 | mytab2_int_idx1 | 0 - 1 | mytab2_int_idx1 | 0 - 2 | mytab2_int_idx1 | 0 + 0 | mytab2_int_idx1 | -1 + 1 | mytab2_int_idx1 | -1 + 2 | mytab2_int_idx1 | -1 (3 rows) -- second vacuum update index stat with table stat @@ -93,7 +93,7 @@ insert into mytab3 values(1,'aa',1001,101),(2,'bb',1002,102); select reltuples from pg_class where relname='mytab3'; reltuples ----------- - 0 + -1 (1 row) -- inspect the state of the stats on segments diff --git a/src/test/regress/expected/uaocs_compaction/index_stats.out b/src/test/regress/expected/uaocs_compaction/index_stats.out index 11ff27fa10c1..63fa84e2f26a 100644 --- a/src/test/regress/expected/uaocs_compaction/index_stats.out +++ b/src/test/regress/expected/uaocs_compaction/index_stats.out @@ -64,9 +64,9 @@ vacuum uaocs_index_stats2; SELECT gp_segment_id, relname, reltuples FROM gp_dist_random('pg_class') WHERE relname = 'uaocs_index_stats2_int_idx1'; gp_segment_id | relname | reltuples ---------------+-----------------------------+----------- - 0 | uaocs_index_stats2_int_idx1 | 0 - 1 | uaocs_index_stats2_int_idx1 | 0 - 2 | uaocs_index_stats2_int_idx1 | 0 + 1 | uaocs_index_stats2_int_idx1 | -1 + 0 | uaocs_index_stats2_int_idx1 | -1 + 2 | uaocs_index_stats2_int_idx1 | -1 (3 rows) -- second vacuum update index stat with table stat @@ -98,7 +98,7 @@ insert into uaocs_index_stats3 values(1,'aa',1001,101),(2,'bb',1002,102); select reltuples from pg_class where relname='uaocs_index_stats3'; reltuples ----------- - 0 + -1 (1 row) -- inspect the state of the stats on segments diff --git a/src/test/regress/expected/vacuum_full_heap.out b/src/test/regress/expected/vacuum_full_heap.out index 234f0e7252f9..d7842a8bc282 100644 --- a/src/test/regress/expected/vacuum_full_heap.out +++ b/src/test/regress/expected/vacuum_full_heap.out @@ -84,7 +84,7 @@ select relname, relpages, reltuples from gp_dist_random('pg_class') where (oid = relname | relpages | reltuples ---------+----------+----------- vfheap | 0 | 0 - ivfheap | 1 | 0 + ivfheap | 0 | -1 (2 rows) -- again, but delete second half diff --git a/src/test/regress/expected/vacuum_gp.out b/src/test/regress/expected/vacuum_gp.out index e7cab8704ec5..77e71f245397 100644 --- a/src/test/regress/expected/vacuum_gp.out +++ b/src/test/regress/expected/vacuum_gp.out @@ -289,16 +289,16 @@ INSERT INTO vacuum_gp_pt SELECT 0, 6 FROM generate_series(1, 12); SELECT relname, reltuples, relpages FROM pg_catalog.pg_class WHERE relname like 'vacuum_gp_pt%'; relname | reltuples | relpages ----------------------+-----------+---------- - vacuum_gp_pt | 0 | 0 - vacuum_gp_pt_1_prt_1 | 0 | 0 - vacuum_gp_pt_1_prt_2 | 0 | 0 + vacuum_gp_pt | -1 | 0 + vacuum_gp_pt_1_prt_1 | -1 | 0 + vacuum_gp_pt_1_prt_2 | -1 | 0 (3 rows) ANALYZE vacuum_gp_pt; SELECT relname, reltuples, relpages FROM pg_catalog.pg_class WHERE relname like 'vacuum_gp_pt%'; relname | reltuples | relpages ----------------------+-----------+---------- - vacuum_gp_pt | 0 | 0 + vacuum_gp_pt | -1 | 0 vacuum_gp_pt_1_prt_1 | 0 | 1 vacuum_gp_pt_1_prt_2 | 12 | 1 (3 rows) @@ -307,7 +307,7 @@ VACUUM vacuum_gp_pt; SELECT relname, reltuples, relpages FROM pg_catalog.pg_class WHERE relname like 'vacuum_gp_pt%'; relname | reltuples | relpages ----------------------+-----------+---------- - vacuum_gp_pt | 0 | 0 + vacuum_gp_pt | -1 | 0 vacuum_gp_pt_1_prt_1 | 0 | 1 vacuum_gp_pt_1_prt_2 | 12 | 1 (3 rows) @@ -316,7 +316,7 @@ VACUUM ANALYZE vacuum_gp_pt; SELECT relname, reltuples, relpages FROM pg_catalog.pg_class WHERE relname like 'vacuum_gp_pt%'; relname | reltuples | relpages ----------------------+-----------+---------- - vacuum_gp_pt | 0 | 0 + vacuum_gp_pt | -1 | 0 vacuum_gp_pt_1_prt_1 | 0 | 1 vacuum_gp_pt_1_prt_2 | 12 | 1 (3 rows) diff --git a/src/test/regress/output/uao_ddl/analyze_ao_table_every_dml.source b/src/test/regress/output/uao_ddl/analyze_ao_table_every_dml.source index bf4fdffdb00b..e9c95c5d838c 100644 --- a/src/test/regress/output/uao_ddl/analyze_ao_table_every_dml.source +++ b/src/test/regress/output/uao_ddl/analyze_ao_table_every_dml.source @@ -22,7 +22,7 @@ select count(*) from sto_uao_city_analyze_everydml; select relname, reltuples from pg_class where oid='sto_uao_city_analyze_everydml'::regclass; relname | reltuples -------------------------------+----------- - sto_uao_city_analyze_everydml | 0 + sto_uao_city_analyze_everydml | -1 (1 row) SELECT 1 AS VisimapPresent FROM pg_appendonly WHERE visimaprelid is not NULL AND From 0addba892c127767c077393985b8f1524f903ab2 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Tue, 9 Jun 2026 09:31:46 +0300 Subject: [PATCH 552/589] Fix test src/test/regress/expected/psql.out (#2668) Commit 07f386ede026ae8c3f2adeba0c22139df19bf2ff added new column Storage to the output of \d+, update the GPDB-specific output too. --- src/test/regress/expected/psql.out | 38 +++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out index a332feb55f1d..63e59bebfb80 100644 --- a/src/test/regress/expected/psql.out +++ b/src/test/regress/expected/psql.out @@ -2862,21 +2862,21 @@ Access method: heap -- AM is displayed for tables, indexes and materialized views. \d+ - List of relations - Schema | Name | Type | Owner | Persistence | Access Method | Size | Description ------------------+--------------------+-------------------+----------------------+-------------+---------------+---------+------------- - tableam_display | mat_view_heap_psql | materialized view | regress_display_role | permanent | heap_psql | 0 bytes | - tableam_display | tbl_heap | table | regress_display_role | permanent | heap | 0 bytes | - tableam_display | tbl_heap_psql | table | regress_display_role | permanent | heap_psql | 0 bytes | - tableam_display | view_heap_psql | view | regress_display_role | permanent | | 0 bytes | + List of relations + Schema | Name | Type | Owner | Storage | Persistence | Access Method | Size | Description +-----------------+--------------------+-------------------+----------------------+-----------+-------------+---------------+---------+------------- + tableam_display | mat_view_heap_psql | materialized view | regress_display_role | heap_psql | permanent | heap_psql | 0 bytes | + tableam_display | tbl_heap | table | regress_display_role | heap | permanent | heap | 0 bytes | + tableam_display | tbl_heap_psql | table | regress_display_role | heap_psql | permanent | heap_psql | 0 bytes | + tableam_display | view_heap_psql | view | regress_display_role | | permanent | | 0 bytes | (4 rows) \dt+ - List of relations - Schema | Name | Type | Owner | Persistence | Access Method | Size | Description ------------------+---------------+-------+----------------------+-------------+---------------+---------+------------- - tableam_display | tbl_heap | table | regress_display_role | permanent | heap | 0 bytes | - tableam_display | tbl_heap_psql | table | regress_display_role | permanent | heap_psql | 0 bytes | + List of relations + Schema | Name | Type | Owner | Storage | Persistence | Access Method | Size | Description +-----------------+---------------+-------+----------------------+-----------+-------------+---------------+---------+------------- + tableam_display | tbl_heap | table | regress_display_role | heap | permanent | heap | 0 bytes | + tableam_display | tbl_heap_psql | table | regress_display_role | heap_psql | permanent | heap_psql | 0 bytes | (2 rows) \dm+ @@ -2896,13 +2896,13 @@ Access method: heap \set HIDE_TABLEAM on \d+ - List of relations - Schema | Name | Type | Owner | Persistence | Size | Description ------------------+--------------------+-------------------+----------------------+-------------+---------+------------- - tableam_display | mat_view_heap_psql | materialized view | regress_display_role | permanent | 0 bytes | - tableam_display | tbl_heap | table | regress_display_role | permanent | 0 bytes | - tableam_display | tbl_heap_psql | table | regress_display_role | permanent | 0 bytes | - tableam_display | view_heap_psql | view | regress_display_role | permanent | 0 bytes | + List of relations + Schema | Name | Type | Owner | Storage | Persistence | Size | Description +-----------------+--------------------+-------------------+----------------------+-----------+-------------+---------+------------- + tableam_display | mat_view_heap_psql | materialized view | regress_display_role | heap_psql | permanent | 0 bytes | + tableam_display | tbl_heap | table | regress_display_role | heap | permanent | 0 bytes | + tableam_display | tbl_heap_psql | table | regress_display_role | heap_psql | permanent | 0 bytes | + tableam_display | view_heap_psql | view | regress_display_role | | permanent | 0 bytes | (4 rows) RESET ROLE; From f9d681a4dc5a298ebbf6cced1cf693f199ac7734 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Tue, 9 Jun 2026 09:32:04 +0300 Subject: [PATCH 553/589] Fix test src/test/regress/expected/subscription.out (#2669) Commit 464824323e57dc4b397e8b05854d779908b55304 added new column Streaming to output of \dRs+, update GPDB-specific test output too. --- src/test/regress/expected/subscription.out | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out index 89699a329ce5..30225d64e52e 100644 --- a/src/test/regress/expected/subscription.out +++ b/src/test/regress/expected/subscription.out @@ -146,10 +146,10 @@ ERROR: DROP SUBSCRIPTION cannot run inside a transaction block COMMIT; ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Binary | Synchronous commit | Conninfo ------------------+----------------------------+---------+---------------------+--------+--------------------+------------------------------ - regress_testsub | regress_subscription_user2 | f | {testpub2,testpub3} | f | local | dbname=regress_doesnotexist2 + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Synchronous commit | Conninfo +-----------------+----------------------------+---------+---------------------+--------+-----------+--------------------+------------------------------ + regress_testsub | regress_subscription_user2 | f | {testpub2,testpub3} | f | f | local | dbname=regress_doesnotexist2 (1 row) -- now it works From 77006d661a2fe0eceb6309ff1ef5a34cd88e32f8 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Tue, 9 Jun 2026 09:32:27 +0300 Subject: [PATCH 554/589] Fix test src/test/regress/expected/window.out (#2670) Commit 62e221e1c01e3985d2b8e4b68c364f8486c327ab added new test which should be adapted for GPDB, since incremental sort is currently disabled. Also add ORCA output. --- src/test/regress/expected/window.out | 28 ++++++----- .../regress/expected/window_optimizer.out | 49 +++++++++++++++++++ 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/src/test/regress/expected/window.out b/src/test/regress/expected/window.out index f1ce6e0c810f..b49ee54713b7 100644 --- a/src/test/regress/expected/window.out +++ b/src/test/regress/expected/window.out @@ -3292,19 +3292,25 @@ SELECT * FROM row_number() OVER (PARTITION BY depname ORDER BY enroll_date DESC) AS last_emp FROM empsalary) emp WHERE first_emp = 1 OR last_emp = 1; - QUERY PLAN ------------------------------------------------------------------------------------ + QUERY PLAN +----------------------------------------------------------------------------------------- Subquery Scan on emp Filter: ((emp.first_emp = 1) OR (emp.last_emp = 1)) - -> WindowAgg - -> Incremental Sort - Sort Key: empsalary.depname, empsalary.enroll_date - Presorted Key: empsalary.depname - -> WindowAgg - -> Sort - Sort Key: empsalary.depname, empsalary.enroll_date DESC - -> Seq Scan on empsalary -(10 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: empsalary.depname, empsalary.enroll_date + -> WindowAgg + Partition By: empsalary.depname + Order By: empsalary.enroll_date + -> Sort + Sort Key: empsalary.depname, empsalary.enroll_date + -> WindowAgg + Partition By: empsalary.depname + Order By: empsalary.enroll_date + -> Sort + Sort Key: empsalary.depname, empsalary.enroll_date DESC + -> Seq Scan on empsalary + Optimizer: Postgres query optimizer +(16 rows) SELECT * FROM (SELECT depname, diff --git a/src/test/regress/expected/window_optimizer.out b/src/test/regress/expected/window_optimizer.out index 58d406a67da2..63c1fd0b9d2c 100644 --- a/src/test/regress/expected/window_optimizer.out +++ b/src/test/regress/expected/window_optimizer.out @@ -3302,6 +3302,55 @@ FROM empsalary; Optimizer: Pivotal Optimizer (GPORCA) (13 rows) +-- Test incremental sorting +EXPLAIN (COSTS OFF) +SELECT * FROM + (SELECT depname, + empno, + salary, + enroll_date, + row_number() OVER (PARTITION BY depname ORDER BY enroll_date) AS first_emp, + row_number() OVER (PARTITION BY depname ORDER BY enroll_date DESC) AS last_emp + FROM empsalary) emp +WHERE first_emp = 1 OR last_emp = 1; + QUERY PLAN +---------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + -> Result + Filter: (((row_number() OVER (?)) = 1) OR ((row_number() OVER (?)) = 1)) + -> WindowAgg + Partition By: depname + Order By: enroll_date + -> Sort + Sort Key: depname, enroll_date DESC + -> WindowAgg + Partition By: depname + Order By: enroll_date + -> Sort + Sort Key: depname, enroll_date + -> Seq Scan on empsalary + Optimizer: Pivotal Optimizer (GPORCA) +(15 rows) + +SELECT * FROM + (SELECT depname, + empno, + salary, + enroll_date, + row_number() OVER (PARTITION BY depname ORDER BY enroll_date) AS first_emp, + row_number() OVER (PARTITION BY depname ORDER BY enroll_date DESC) AS last_emp + FROM empsalary) emp +WHERE first_emp = 1 OR last_emp = 1; + depname | empno | salary | enroll_date | first_emp | last_emp +-----------+-------+--------+-------------+-----------+---------- + develop | 9 | 4500 | 01-01-2008 | 4 | 1 + develop | 8 | 6000 | 10-01-2006 | 1 | 5 + personnel | 5 | 3500 | 12-10-2007 | 2 | 1 + personnel | 2 | 3900 | 12-23-2006 | 1 | 2 + sales | 4 | 4800 | 08-08-2007 | 3 | 1 + sales | 1 | 5000 | 10-01-2006 | 1 | 3 +(6 rows) + -- cleanup DROP TABLE empsalary; -- test user-defined window function with named args and default args From 56176b690d7ee490f414538f0a9d74fd5d85f272 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Tue, 9 Jun 2026 09:32:58 +0300 Subject: [PATCH 555/589] Fix test src/test/regress/expected/rangefuncs.out (#2671) 1. Commit c8ab9701791e22f7a8e1badf362654db179c9703 added new test, adapt its plan for GPDB and add the output for ORCA. 2. Commit ce90f075f0d831ca4085ba73891b7da2a2f7047e added new test, add the output for ORCA. --- src/test/regress/expected/rangefuncs.out | 11 ++-- .../regress/expected/rangefuncs_optimizer.out | 51 +++++++++++++++++++ 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/src/test/regress/expected/rangefuncs.out b/src/test/regress/expected/rangefuncs.out index 44a78505bec2..3230dbf63c7c 100644 --- a/src/test/regress/expected/rangefuncs.out +++ b/src/test/regress/expected/rangefuncs.out @@ -2094,17 +2094,20 @@ select * from testrngfunc(); ---------------------------------------------------------- Subquery Scan on "*SELECT*" Output: "*SELECT*"."?column?", "*SELECT*"."?column?_1" - -> Unique + -> Sort Output: (1), (2) - -> Sort + Sort Key: (1) + -> HashAggregate Output: (1), (2) - Sort Key: (1), (2) + Group Key: (1), (2) -> Append -> Result Output: 1, 2 -> Result Output: 3, 4 -(12 rows) + Optimizer: Postgres query optimizer + Settings: optimizer = 'off' +(15 rows) select * from testrngfunc(); f1 | f2 diff --git a/src/test/regress/expected/rangefuncs_optimizer.out b/src/test/regress/expected/rangefuncs_optimizer.out index 0d304d4e12a8..089c6f541150 100644 --- a/src/test/regress/expected/rangefuncs_optimizer.out +++ b/src/test/regress/expected/rangefuncs_optimizer.out @@ -2083,6 +2083,57 @@ select * from testrngfunc(); 7.136178 | 7.14 (1 row) +create or replace function testrngfunc() returns setof rngfunc_type as $$ + select 1, 2 union select 3, 4 order by 1; +$$ language sql immutable; +explain (verbose, costs off) +select testrngfunc(); + QUERY PLAN +--------------------------------------- + ProjectSet + Output: testrngfunc() + -> Result + Output: true + Optimizer: Pivotal Optimizer (GPORCA) +(5 rows) + +select testrngfunc(); + testrngfunc +----------------- + (1.000000,2.00) + (3.000000,4.00) +(2 rows) + +explain (verbose, costs off) +select * from testrngfunc(); + QUERY PLAN +--------------------------------------- + Function Scan on public.testrngfunc + Output: f1, f2 + Function Call: testrngfunc() + Optimizer: Pivotal Optimizer (GPORCA) +(4 rows) + +select * from testrngfunc(); + f1 | f2 +----------+------ + 1.000000 | 2.00 + 3.000000 | 4.00 +(2 rows) + +-- Check a couple of error cases while we're here +select * from testrngfunc() as t(f1 int8,f2 int8); -- fail, composite result +ERROR: a column definition list is redundant for a function returning a named composite type +LINE 1: select * from testrngfunc() as t(f1 int8,f2 int8); + ^ +select * from pg_get_keywords() as t(f1 int8,f2 int8); -- fail, OUT params +ERROR: a column definition list is redundant for a function with OUT parameters +LINE 1: select * from pg_get_keywords() as t(f1 int8,f2 int8); + ^ +select * from sin(3) as t(f1 int8,f2 int8); -- fail, scalar result type +ERROR: a column definition list is only allowed for functions returning "record" +LINE 1: select * from sin(3) as t(f1 int8,f2 int8); + ^ drop type rngfunc_type cascade; NOTICE: drop cascades to function testrngfunc() -- From 961d68a8333a727b5282a8ff7d070d542bdc87e9 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Tue, 9 Jun 2026 15:02:19 +0500 Subject: [PATCH 556/589] Fix assert with unregistered callback (#2651) The c9ae5cb commit in src/backend/storage/ipc/ipc.c added a conditional error to the cancel_before_shmem_exit function, while an earlier commit 1617960 had already added a call to this function for RemoveTempRelationsCallback. If this callback has not even been registered yet (which happens conditionally in the AtEOXact_Namespace function), then an error results in an assertion, for example, in the instr_in_shmem_terminate test. Add a global variable before_shmem_exit_callback_registered and check it when canceling the callback. --- src/backend/catalog/namespace.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/backend/catalog/namespace.c b/src/backend/catalog/namespace.c index c40c7f66b65a..c91e11c76116 100644 --- a/src/backend/catalog/namespace.c +++ b/src/backend/catalog/namespace.c @@ -172,6 +172,8 @@ static Oid namespaceUser = InvalidOid; /* The above four values are valid only if baseSearchPathValid */ static bool baseSearchPathValid = true; +static bool before_shmem_exit_callback_registered = false; + /* Override requests are remembered in a stack of OverrideStackEntry structs */ typedef struct @@ -4305,10 +4307,14 @@ ResetTempNamespace(void) /* * MPP-19973: The shmem exit callback to remove a temp - * namespace is registered. We need to remove it here as the + * namespace may be registered. We need to remove it here as the * namespace has already been reseted. */ - cancel_before_shmem_exit(RemoveTempRelationsCallback, 0); + if (before_shmem_exit_callback_registered) + { + before_shmem_exit_callback_registered = false; + cancel_before_shmem_exit(RemoveTempRelationsCallback, 0); + } myTempNamespace = InvalidOid; myTempToastNamespace = InvalidOid; @@ -4335,7 +4341,10 @@ AtEOXact_Namespace(bool isCommit, bool parallel) if (myTempNamespaceSubID != InvalidSubTransactionId && !parallel) { if (isCommit) + { before_shmem_exit(RemoveTempRelationsCallback, 0); + before_shmem_exit_callback_registered = true; + } else { myTempNamespace = InvalidOid; @@ -4518,6 +4527,8 @@ RemoveSchemaById(Oid schemaOid) static void RemoveTempRelationsCallback(int code, Datum arg) { + before_shmem_exit_callback_registered = false; + if (DistributedTransactionContext == DTX_CONTEXT_QE_PREPARED) { /* From 2ab0c1a916b88c90efaf91bc9ef5a65c0275b136 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Tue, 9 Jun 2026 13:27:24 +0300 Subject: [PATCH 557/589] Fix test src/test/regress/expected/join.out (#2667) Commit ad1c36b0709e47cdb3cc4abd6c939fe64279b63f added new test, adapt it for GPDB and add its output to ORCA. Use enable_nestloop to achieve plans that better match upstream. --- src/test/regress/expected/join.out | 48 +++++++++------- src/test/regress/expected/join_optimizer.out | 58 ++++++++++++++++++++ src/test/regress/sql/join.sql | 3 + 3 files changed, 88 insertions(+), 21 deletions(-) diff --git a/src/test/regress/expected/join.out b/src/test/regress/expected/join.out index f320dc60f608..8965772ecac8 100644 --- a/src/test/regress/expected/join.out +++ b/src/test/regress/expected/join.out @@ -6297,6 +6297,8 @@ drop table join_ut1; -- test estimation behavior with multi-column foreign key and constant qual -- begin; +-- GPDB: persuade the planner to choose same plan as in upstream. +set local enable_nestloop=on; create table fkest (x integer, x10 integer, x10b integer, x100 integer); insert into fkest select x, x/10, x/10, x/100 from generate_series(1,1000) x; create unique index on fkest(x, x10, x100); @@ -6306,19 +6308,21 @@ select * from fkest f1 join fkest f2 on (f1.x = f2.x and f1.x10 = f2.x10b and f1.x100 = f2.x100) join fkest f3 on f1.x = f3.x where f1.x100 = 2; - QUERY PLAN ------------------------------------------------------------ - Nested Loop - -> Hash Join - Hash Cond: ((f2.x = f1.x) AND (f2.x10b = f1.x10)) - -> Seq Scan on fkest f2 - Filter: (x100 = 2) - -> Hash + QUERY PLAN +----------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + -> Nested Loop + -> Hash Join + Hash Cond: ((f1.x = f2.x) AND (f1.x10 = f2.x10b)) -> Seq Scan on fkest f1 Filter: (x100 = 2) - -> Index Scan using fkest_x_x10_x100_idx on fkest f3 - Index Cond: (x = f1.x) -(10 rows) + -> Hash + -> Seq Scan on fkest f2 + Filter: (x100 = 2) + -> Index Scan using fkest_x_x10_x100_idx on fkest f3 + Index Cond: (x = f1.x) + Optimizer: Postgres query optimizer +(12 rows) alter table fkest add constraint fk foreign key (x, x10b, x100) references fkest (x, x10, x100); @@ -6327,20 +6331,22 @@ select * from fkest f1 join fkest f2 on (f1.x = f2.x and f1.x10 = f2.x10b and f1.x100 = f2.x100) join fkest f3 on f1.x = f3.x where f1.x100 = 2; - QUERY PLAN ------------------------------------------------------ - Hash Join - Hash Cond: ((f2.x = f1.x) AND (f2.x10b = f1.x10)) + QUERY PLAN +----------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) -> Hash Join - Hash Cond: (f3.x = f2.x) - -> Seq Scan on fkest f3 + Hash Cond: ((f1.x = f2.x) AND (f1.x10 = f2.x10b)) + -> Hash Join + Hash Cond: (f3.x = f1.x) + -> Seq Scan on fkest f3 + -> Hash + -> Seq Scan on fkest f1 + Filter: (x100 = 2) -> Hash -> Seq Scan on fkest f2 Filter: (x100 = 2) - -> Hash - -> Seq Scan on fkest f1 - Filter: (x100 = 2) -(11 rows) + Optimizer: Postgres query optimizer +(13 rows) rollback; -- diff --git a/src/test/regress/expected/join_optimizer.out b/src/test/regress/expected/join_optimizer.out index daa6bde6f218..4742d8e266a3 100644 --- a/src/test/regress/expected/join_optimizer.out +++ b/src/test/regress/expected/join_optimizer.out @@ -6337,6 +6337,64 @@ ERROR: could not devise a query plan for the given query (pathnode.c:275) drop table join_pt1; drop table join_ut1; -- +-- test estimation behavior with multi-column foreign key and constant qual +-- +begin; +-- GPDB: persuade the planner to choose same plan as in upstream. +set local enable_nestloop=on; +create table fkest (x integer, x10 integer, x10b integer, x100 integer); +insert into fkest select x, x/10, x/10, x/100 from generate_series(1,1000) x; +create unique index on fkest(x, x10, x100); +analyze fkest; +explain (costs off) +select * from fkest f1 + join fkest f2 on (f1.x = f2.x and f1.x10 = f2.x10b and f1.x100 = f2.x100) + join fkest f3 on f1.x = f3.x + where f1.x100 = 2; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Join + Hash Cond: ((fkest_1.x = fkest_2.x) AND (fkest_1.x10b = fkest_2.x10) AND (fkest_1.x100 = fkest_2.x100)) + -> Hash Join + Hash Cond: (fkest.x = fkest_1.x) + -> Seq Scan on fkest + -> Hash + -> Index Scan using fkest_x_x10_x100_idx on fkest fkest_1 + Index Cond: (x100 = 2) + -> Hash + -> Index Scan using fkest_x_x10_x100_idx on fkest fkest_2 + Index Cond: (x100 = 2) + Optimizer: Pivotal Optimizer (GPORCA) +(13 rows) + +alter table fkest add constraint fk + foreign key (x, x10b, x100) references fkest (x, x10, x100); +WARNING: referential integrity (FOREIGN KEY) constraints are not supported in Greenplum Database, will not be enforced +explain (costs off) +select * from fkest f1 + join fkest f2 on (f1.x = f2.x and f1.x10 = f2.x10b and f1.x100 = f2.x100) + join fkest f3 on f1.x = f3.x + where f1.x100 = 2; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Join + Hash Cond: ((fkest_1.x = fkest_2.x) AND (fkest_1.x10b = fkest_2.x10) AND (fkest_1.x100 = fkest_2.x100)) + -> Hash Join + Hash Cond: (fkest.x = fkest_1.x) + -> Seq Scan on fkest + -> Hash + -> Index Scan using fkest_x_x10_x100_idx on fkest fkest_1 + Index Cond: (x100 = 2) + -> Hash + -> Index Scan using fkest_x_x10_x100_idx on fkest fkest_2 + Index Cond: (x100 = 2) + Optimizer: Pivotal Optimizer (GPORCA) +(13 rows) + +rollback; +-- -- test that foreign key join estimation performs sanely for outer joins -- begin; diff --git a/src/test/regress/sql/join.sql b/src/test/regress/sql/join.sql index 58d1b35b4879..a6ed867e37e1 100644 --- a/src/test/regress/sql/join.sql +++ b/src/test/regress/sql/join.sql @@ -2043,6 +2043,9 @@ drop table join_ut1; begin; +-- GPDB: persuade the planner to choose same plan as in upstream. +set local enable_nestloop=on; + create table fkest (x integer, x10 integer, x10b integer, x100 integer); insert into fkest select x, x/10, x/10, x/100 from generate_series(1,1000) x; create unique index on fkest(x, x10, x100); From 6ae9c679b5947c061c840d89eb0dd802e601ed4d Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Tue, 9 Jun 2026 16:57:20 +0500 Subject: [PATCH 558/589] Fix tests in src/test/regress/expected/partition_prune.out (#2672) Commit a929e17 added new tests to src/test/regress/sql/partition_prune.sql. Adapt them for GPDB and add them to ORCA. --- src/test/regress/expected/partition_prune.out | 85 ++++++++-------- .../expected/partition_prune_optimizer.out | 96 +++++++++++++++++++ 2 files changed, 136 insertions(+), 45 deletions(-) diff --git a/src/test/regress/expected/partition_prune.out b/src/test/regress/expected/partition_prune.out index f8bb611a0995..6767d9b2e111 100644 --- a/src/test/regress/expected/partition_prune.out +++ b/src/test/regress/expected/partition_prune.out @@ -4215,20 +4215,18 @@ alter table listp_12_1 set (parallel_workers = 0); -- Ensure that listp_12_2 is not scanned. (The nested Append is not seen in -- the plan as it's pulled in setref.c due to having just a single subnode). select explain_parallel_append('select * from listp where a = (select 1);'); - explain_parallel_append ----------------------------------------------------------------------- - Gather (actual rows=N loops=N) - Workers Planned: 2 - Params Evaluated: $0 - Workers Launched: N - InitPlan 1 (returns $0) + explain_parallel_append +-------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (actual rows=N loops=N) + InitPlan 1 (returns $0) (slice2) -> Result (actual rows=N loops=N) - -> Parallel Append (actual rows=N loops=N) + -> Append (actual rows=N loops=N) -> Seq Scan on listp_12_1 listp_1 (actual rows=N loops=N) Filter: (a = $0) - -> Parallel Seq Scan on listp_12_2 listp_2 (never executed) + -> Seq Scan on listp_12_2 listp_2 (never executed) Filter: (a = $0) -(11 rows) + Optimizer: Postgres query optimizer +(9 rows) -- Like the above but throw some more complexity at the planner by adding -- a UNION ALL. We expect both sides of the union not to scan the @@ -4237,32 +4235,26 @@ select explain_parallel_append( 'select * from listp where a = (select 1) union all select * from listp where a = (select 2);'); - explain_parallel_append ------------------------------------------------------------------------------------ - Append (actual rows=N loops=N) - -> Gather (actual rows=N loops=N) - Workers Planned: 2 - Params Evaluated: $0 - Workers Launched: N - InitPlan 1 (returns $0) - -> Result (actual rows=N loops=N) - -> Parallel Append (actual rows=N loops=N) + explain_parallel_append +-------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (actual rows=N loops=N) + -> Append (actual rows=N loops=N) + -> Append (actual rows=N loops=N) + InitPlan 1 (returns $0) (slice2) + -> Result (actual rows=N loops=N) -> Seq Scan on listp_12_1 listp_1 (actual rows=N loops=N) Filter: (a = $0) - -> Parallel Seq Scan on listp_12_2 listp_2 (never executed) + -> Seq Scan on listp_12_2 listp_2 (never executed) Filter: (a = $0) - -> Gather (actual rows=N loops=N) - Workers Planned: 2 - Params Evaluated: $1 - Workers Launched: N - InitPlan 2 (returns $1) - -> Result (actual rows=N loops=N) - -> Parallel Append (actual rows=N loops=N) + -> Append (actual rows=N loops=N) + InitPlan 2 (returns $1) (slice3) + -> Result (actual rows=N loops=N) -> Seq Scan on listp_12_1 listp_4 (never executed) Filter: (a = $1) - -> Parallel Seq Scan on listp_12_2 listp_5 (actual rows=N loops=N) + -> Seq Scan on listp_12_2 listp_5 (actual rows=N loops=N) Filter: (a = $1) -(23 rows) + Optimizer: Postgres query optimizer +(17 rows) drop table listp; reset parallel_tuple_cost; @@ -4281,24 +4273,27 @@ create index on rangep (a); -- Ensure run-time pruning works on the nested Merge Append explain (analyze on, costs off, timing off, summary off) select * from rangep where b IN((select 1),(select 2)) order by a; - QUERY PLAN ------------------------------------------------------------------------------------------------------------- - Append (actual rows=0 loops=1) - InitPlan 1 (returns $0) + QUERY PLAN +------------------------------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (actual rows=0 loops=1) + Merge Key: rangep.a + InitPlan 1 (returns $0) (slice2) -> Result (actual rows=1 loops=1) - InitPlan 2 (returns $1) + InitPlan 2 (returns $1) (slice3) -> Result (actual rows=1 loops=1) - -> Merge Append (actual rows=0 loops=1) - Sort Key: rangep_2.a - -> Index Scan using rangep_0_to_100_1_a_idx on rangep_0_to_100_1 rangep_2 (actual rows=0 loops=1) - Filter: (b = ANY (ARRAY[$0, $1])) - -> Index Scan using rangep_0_to_100_2_a_idx on rangep_0_to_100_2 rangep_3 (actual rows=0 loops=1) - Filter: (b = ANY (ARRAY[$0, $1])) - -> Index Scan using rangep_0_to_100_3_a_idx on rangep_0_to_100_3 rangep_4 (never executed) + -> Append (actual rows=0 loops=1) + -> Merge Append (actual rows=0 loops=1) + Sort Key: rangep_2.a + -> Index Scan using rangep_0_to_100_1_a_idx on rangep_0_to_100_1 rangep_2 (actual rows=0 loops=1) + Filter: (b = ANY (ARRAY[$0, $1])) + -> Index Scan using rangep_0_to_100_2_a_idx on rangep_0_to_100_2 rangep_3 (actual rows=0 loops=1) + Filter: (b = ANY (ARRAY[$0, $1])) + -> Index Scan using rangep_0_to_100_3_a_idx on rangep_0_to_100_3 rangep_4 (never executed) + Filter: (b = ANY (ARRAY[$0, $1])) + -> Index Scan using rangep_100_to_200_a_idx on rangep_100_to_200 rangep_5 (actual rows=0 loops=1) Filter: (b = ANY (ARRAY[$0, $1])) - -> Index Scan using rangep_100_to_200_a_idx on rangep_100_to_200 rangep_5 (actual rows=0 loops=1) - Filter: (b = ANY (ARRAY[$0, $1])) -(15 rows) + Optimizer: Postgres query optimizer +(18 rows) reset enable_sort; drop table rangep; diff --git a/src/test/regress/expected/partition_prune_optimizer.out b/src/test/regress/expected/partition_prune_optimizer.out index cf5c79d72009..0f92c7136679 100644 --- a/src/test/regress/expected/partition_prune_optimizer.out +++ b/src/test/regress/expected/partition_prune_optimizer.out @@ -4111,6 +4111,102 @@ explain (costs off) update listp1 set a = 1 where a = 2; reset constraint_exclusion; reset enable_partition_pruning; drop table listp; +-- Ensure run-time pruning works correctly for nested Append nodes +set parallel_setup_cost to 0; +set parallel_tuple_cost to 0; +create table listp (a int) partition by list(a); +create table listp_12 partition of listp for values in(1,2) partition by list(a); +create table listp_12_1 partition of listp_12 for values in(1); +create table listp_12_2 partition of listp_12 for values in(2); +-- Force the 2nd subnode of the Append to be non-parallel. This results in +-- a nested Append node because the mixed parallel / non-parallel paths cannot +-- be pulled into the top-level Append. +alter table listp_12_1 set (parallel_workers = 0); +-- Ensure that listp_12_2 is not scanned. (The nested Append is not seen in +-- the plan as it's pulled in setref.c due to having just a single subnode). +select explain_parallel_append('select * from listp where a = (select 1);'); + explain_parallel_append +-------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (actual rows=N loops=N) + InitPlan 1 (returns $0) (slice2) + -> Result (actual rows=N loops=N) + -> Append (actual rows=N loops=N) + -> Seq Scan on listp_12_1 listp_1 (actual rows=N loops=N) + Filter: (a = $0) + -> Seq Scan on listp_12_2 listp_2 (never executed) + Filter: (a = $0) + Optimizer: Postgres query optimizer +(9 rows) + +-- Like the above but throw some more complexity at the planner by adding +-- a UNION ALL. We expect both sides of the union not to scan the +-- non-required partitions. +select explain_parallel_append( +'select * from listp where a = (select 1) + union all +select * from listp where a = (select 2);'); + explain_parallel_append +-------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (actual rows=N loops=N) + -> Append (actual rows=N loops=N) + -> Append (actual rows=N loops=N) + InitPlan 1 (returns $0) (slice2) + -> Result (actual rows=N loops=N) + -> Seq Scan on listp_12_1 listp_1 (actual rows=N loops=N) + Filter: (a = $0) + -> Seq Scan on listp_12_2 listp_2 (never executed) + Filter: (a = $0) + -> Append (actual rows=N loops=N) + InitPlan 2 (returns $1) (slice3) + -> Result (actual rows=N loops=N) + -> Seq Scan on listp_12_1 listp_4 (never executed) + Filter: (a = $1) + -> Seq Scan on listp_12_2 listp_5 (actual rows=N loops=N) + Filter: (a = $1) + Optimizer: Postgres query optimizer +(17 rows) + +drop table listp; +reset parallel_tuple_cost; +reset parallel_setup_cost; +-- Test case for run-time pruning with a nested Merge Append +set enable_sort to 0; +create table rangep (a int, b int) partition by range (a); +create table rangep_0_to_100 partition of rangep for values from (0) to (100) partition by list (b); +-- We need 3 sub-partitions. 1 to validate pruning worked and another two +-- because a single remaining partition would be pulled up to the main Append. +create table rangep_0_to_100_1 partition of rangep_0_to_100 for values in(1); +create table rangep_0_to_100_2 partition of rangep_0_to_100 for values in(2); +create table rangep_0_to_100_3 partition of rangep_0_to_100 for values in(3); +create table rangep_100_to_200 partition of rangep for values from (100) to (200); +create index on rangep (a); +-- Ensure run-time pruning works on the nested Merge Append +explain (analyze on, costs off, timing off, summary off) +select * from rangep where b IN((select 1),(select 2)) order by a; + QUERY PLAN +------------------------------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (actual rows=0 loops=1) + Merge Key: rangep.a + InitPlan 1 (returns $0) (slice2) + -> Result (actual rows=1 loops=1) + InitPlan 2 (returns $1) (slice3) + -> Result (actual rows=1 loops=1) + -> Append (actual rows=0 loops=1) + -> Merge Append (actual rows=0 loops=1) + Sort Key: rangep_2.a + -> Index Scan using rangep_0_to_100_1_a_idx on rangep_0_to_100_1 rangep_2 (actual rows=0 loops=1) + Filter: (b = ANY (ARRAY[$0, $1])) + -> Index Scan using rangep_0_to_100_2_a_idx on rangep_0_to_100_2 rangep_3 (actual rows=0 loops=1) + Filter: (b = ANY (ARRAY[$0, $1])) + -> Index Scan using rangep_0_to_100_3_a_idx on rangep_0_to_100_3 rangep_4 (never executed) + Filter: (b = ANY (ARRAY[$0, $1])) + -> Index Scan using rangep_100_to_200_a_idx on rangep_100_to_200 rangep_5 (actual rows=0 loops=1) + Filter: (b = ANY (ARRAY[$0, $1])) + Optimizer: Postgres query optimizer +(18 rows) + +reset enable_sort; +drop table rangep; -- -- Check that gen_prune_steps_from_opexps() works well for various cases of -- clauses for different partition keys From 247b4637ed98d5e02c8ff8f683d31d7a7c46158f Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Tue, 9 Jun 2026 17:14:33 +0500 Subject: [PATCH 559/589] Fix detected write past chunk end warning (#2660) Commit 9ca7bae incorrectly fixed the compilation of the aocs_compaction.c and appendonly_compaction.c files, which resulted in similar warnings being generated for AO tables in many tests: problem in alloc set TopTransactionContext: detected write past chunk end in block 0x615ef0207560, chunk 0x615ef0207dc8. Remove unnecessary allocation and assignment. --- src/backend/access/aocs/aocs_compaction.c | 4 ---- src/backend/access/appendonly/appendonly_compaction.c | 4 ---- 2 files changed, 8 deletions(-) diff --git a/src/backend/access/aocs/aocs_compaction.c b/src/backend/access/aocs/aocs_compaction.c index 638458e6adc1..82a5e3629c0a 100644 --- a/src/backend/access/aocs/aocs_compaction.c +++ b/src/backend/access/aocs/aocs_compaction.c @@ -263,10 +263,6 @@ AOCSSegmentFileFullCompaction(Relation aorel, resultRelInfo->ri_RelationDesc = aorel; resultRelInfo->ri_TrigDesc = NULL; /* we don't fire triggers */ ExecOpenIndices(resultRelInfo, false); - if (estate->es_result_relations == NULL) - estate->es_result_relations = (ResultRelInfo **) - palloc0(estate->es_range_table_size * sizeof(ResultRelInfo *)); - estate->es_result_relations[resultRelInfo->ri_RangeTableIndex - 1] = resultRelInfo; /* * We don't want uniqueness checks to be performed while "insert"ing tuples diff --git a/src/backend/access/appendonly/appendonly_compaction.c b/src/backend/access/appendonly/appendonly_compaction.c index 22c9f591701e..e50f2d59716a 100644 --- a/src/backend/access/appendonly/appendonly_compaction.c +++ b/src/backend/access/appendonly/appendonly_compaction.c @@ -445,10 +445,6 @@ AppendOnlySegmentFileFullCompaction(Relation aorel, resultRelInfo->ri_RelationDesc = aorel; resultRelInfo->ri_TrigDesc = NULL; /* we don't fire triggers */ ExecOpenIndices(resultRelInfo, false); - if (estate->es_result_relations == NULL) - estate->es_result_relations = (ResultRelInfo **) - palloc0(estate->es_range_table_size * sizeof(ResultRelInfo *)); - estate->es_result_relations[resultRelInfo->ri_RangeTableIndex - 1] = resultRelInfo; /* * We don't want uniqueness checks to be performed while "insert"ing tuples From 9057792c30d38767b9fc610689a22ef6bce4f9b1 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Tue, 9 Jun 2026 17:06:28 +0300 Subject: [PATCH 560/589] Fix test src/test/regress/expected/eagerfree.out (#2677) Commit 41efb8340877e8ffd0023bb6b2ef22ffd1ca014d removed unused subplans from various plans, update this GPDB-specific test too. --- src/test/regress/expected/eagerfree.out | 8 ++------ src/test/regress/expected/eagerfree_optimizer.out | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/test/regress/expected/eagerfree.out b/src/test/regress/expected/eagerfree.out index fcef2871f9a2..0c60c1373f13 100644 --- a/src/test/regress/expected/eagerfree.out +++ b/src/test/regress/expected/eagerfree.out @@ -1885,19 +1885,15 @@ explain analyze select *, exists(select 1 from pg_class where oid = c.oid) as du QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------- Seq Scan on pg_class c (cost=10000000000.00..10003029725.25 rows=15124 width=206) (actual time=24.734..41.952 rows=6569 loops=1) - SubPlan 1 - -> Index Only Scan using pg_class_oid_index on pg_class (cost=0.29..200.30 rows=1 width=0) (never executed) - Index Cond: (oid = c.oid) - Heap Fetches: 0 SubPlan 2 - -> Index Only Scan using pg_class_oid_index on pg_class pg_class_1 (cost=0.29..21127.15 rows=15124 width=4) (actual time=0.075..18.903 rows=6569 loops=1) + -> Index Only Scan using pg_class_oid_index on pg_class (cost=0.29..21127.15 rows=15124 width=4) (actual time=0.075..18.903 rows=6569 loops=1) Heap Fetches: 20272 Planning time: 5.919 ms (slice0) Executor memory: 937K bytes. Memory used: 128000kB Optimizer: Postgres query optimizer Execution time: 43.318 ms -(13 rows) +(9 rows) -- BitmapScan -- start_ignore diff --git a/src/test/regress/expected/eagerfree_optimizer.out b/src/test/regress/expected/eagerfree_optimizer.out index bbbe29224bbf..0cd21248e712 100644 --- a/src/test/regress/expected/eagerfree_optimizer.out +++ b/src/test/regress/expected/eagerfree_optimizer.out @@ -1899,19 +1899,15 @@ explain analyze select *, exists(select 1 from pg_class where oid = c.oid) as du QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------- Seq Scan on pg_class c (cost=10000000000.00..10003029725.25 rows=15124 width=206) (actual time=24.734..41.952 rows=6569 loops=1) - SubPlan 1 - -> Index Only Scan using pg_class_oid_index on pg_class (cost=0.29..200.30 rows=1 width=0) (never executed) - Index Cond: (oid = c.oid) - Heap Fetches: 0 SubPlan 2 - -> Index Only Scan using pg_class_oid_index on pg_class pg_class_1 (cost=0.29..21127.15 rows=15124 width=4) (actual time=0.075..18.903 rows=6569 loops=1) + -> Index Only Scan using pg_class_oid_index on pg_class (cost=0.29..21127.15 rows=15124 width=4) (actual time=0.075..18.903 rows=6569 loops=1) Heap Fetches: 20272 Planning time: 5.919 ms (slice0) Executor memory: 937K bytes. Memory used: 128000kB Optimizer: Postgres query optimizer Execution time: 43.318 ms -(13 rows) +(9 rows) -- BitmapScan -- start_ignore From c1326830a299d79733daa90be81e744935735719 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Wed, 10 Jun 2026 11:20:27 +0700 Subject: [PATCH 561/589] Resolve conflicts in the test_extensions (#2656) Commit ced138e8cbac7f5a840de8679e9882665478c680 added a new test to the test_extensions, while earlier commit 782b169922760d0cf46b1ffd86507e4425e0743c added a test to the same place. --- src/test/modules/test_extensions/Makefile | 11 +---------- .../test_extensions/expected/test_extensions.out | 16 +++++++--------- .../test_extensions/sql/test_extensions.sql | 14 ++++++-------- 3 files changed, 14 insertions(+), 27 deletions(-) diff --git a/src/test/modules/test_extensions/Makefile b/src/test/modules/test_extensions/Makefile index b382dd80fcdd..d2014cbb294c 100644 --- a/src/test/modules/test_extensions/Makefile +++ b/src/test/modules/test_extensions/Makefile @@ -4,24 +4,15 @@ MODULE = test_extensions PGFILEDESC = "test_extensions - regression testing for EXTENSION support" EXTENSION = test_ext1 test_ext2 test_ext3 test_ext4 test_ext5 test_ext6 \ -<<<<<<< HEAD test_ext7 test_ext8 test_ext_cine test_ext_cor \ - test_ext_cyclic1 test_ext_cyclic2 + test_ext_cyclic1 test_ext_cyclic2 test_ext_evttrig DATA = test_ext1--1.0.sql test_ext2--1.0.sql test_ext3--1.0.sql \ test_ext4--1.0.sql test_ext5--1.0.sql test_ext6--1.0.sql \ test_ext7--1.0.sql test_ext7--1.0--2.0.sql test_ext8--1.0.sql \ test_ext_cine--1.0.sql test_ext_cine--1.0--1.1.sql \ test_ext_cor--1.0.sql \ - test_ext_cyclic1--1.0.sql test_ext_cyclic2--1.0.sql -======= - test_ext7 test_ext8 test_ext_cyclic1 test_ext_cyclic2 \ - test_ext_evttrig -DATA = test_ext1--1.0.sql test_ext2--1.0.sql test_ext3--1.0.sql \ - test_ext4--1.0.sql test_ext5--1.0.sql test_ext6--1.0.sql \ - test_ext7--1.0.sql test_ext7--1.0--2.0.sql test_ext8--1.0.sql \ test_ext_cyclic1--1.0.sql test_ext_cyclic2--1.0.sql \ test_ext_evttrig--1.0.sql test_ext_evttrig--1.0--2.0.sql ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d REGRESS = test_extensions test_extdepend diff --git a/src/test/modules/test_extensions/expected/test_extensions.out b/src/test/modules/test_extensions/expected/test_extensions.out index c26b9874ee19..3168ce6fb1cb 100644 --- a/src/test/modules/test_extensions/expected/test_extensions.out +++ b/src/test/modules/test_extensions/expected/test_extensions.out @@ -154,7 +154,13 @@ DROP TABLE test_ext4_tab; DROP FUNCTION create_extension_with_temp_schema(); RESET client_min_messages; \unset SHOW_CONTEXT -<<<<<<< HEAD + +-- Test case of an event trigger run in an extension upgrade script. +-- See: https://postgr.es/m/20200902193715.6e0269d4@firost +CREATE EXTENSION test_ext_evttrig; +ALTER EXTENSION test_ext_evttrig UPDATE TO '2.0'; +DROP EXTENSION test_ext_evttrig; + -- It's generally bad style to use CREATE OR REPLACE unnecessarily. -- Test what happens if an extension does it anyway. -- Replacing a shell type or operator is sort of like CREATE OR REPLACE; @@ -307,11 +313,3 @@ Objects in extension "test_ext_cine" table ext_cine_tab2 table ext_cine_tab3 (9 rows) - -======= --- Test case of an event trigger run in an extension upgrade script. --- See: https://postgr.es/m/20200902193715.6e0269d4@firost -CREATE EXTENSION test_ext_evttrig; -ALTER EXTENSION test_ext_evttrig UPDATE TO '2.0'; -DROP EXTENSION test_ext_evttrig; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d diff --git a/src/test/modules/test_extensions/sql/test_extensions.sql b/src/test/modules/test_extensions/sql/test_extensions.sql index 0354a737a3c4..41b6cddf0b55 100644 --- a/src/test/modules/test_extensions/sql/test_extensions.sql +++ b/src/test/modules/test_extensions/sql/test_extensions.sql @@ -94,7 +94,12 @@ DROP FUNCTION create_extension_with_temp_schema(); RESET client_min_messages; \unset SHOW_CONTEXT -<<<<<<< HEAD +-- Test case of an event trigger run in an extension upgrade script. +-- See: https://postgr.es/m/20200902193715.6e0269d4@firost +CREATE EXTENSION test_ext_evttrig; +ALTER EXTENSION test_ext_evttrig UPDATE TO '2.0'; +DROP EXTENSION test_ext_evttrig; + -- It's generally bad style to use CREATE OR REPLACE unnecessarily. -- Test what happens if an extension does it anyway. -- Replacing a shell type or operator is sort of like CREATE OR REPLACE; @@ -204,10 +209,3 @@ CREATE EXTENSION test_ext_cine; ALTER EXTENSION test_ext_cine UPDATE TO '1.1'; \dx+ test_ext_cine -======= --- Test case of an event trigger run in an extension upgrade script. --- See: https://postgr.es/m/20200902193715.6e0269d4@firost -CREATE EXTENSION test_ext_evttrig; -ALTER EXTENSION test_ext_evttrig UPDATE TO '2.0'; -DROP EXTENSION test_ext_evttrig; ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d From 0f3697ad5570b89473a24bfb2ac4399595e32f51 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Wed, 10 Jun 2026 18:25:06 +0700 Subject: [PATCH 562/589] Fix optimizer output for expressions test (#2657) Commit 540612fa469eaae3345ede7a160b146dd903e7ee added a new row to the DATE_TBL table. That change results in some queries in the expressions test. Fix the output of these queries in the optimizer output file. --- src/test/regress/expected/expressions_optimizer.out | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/regress/expected/expressions_optimizer.out b/src/test/regress/expected/expressions_optimizer.out index 191b107e15a7..8e390e019078 100644 --- a/src/test/regress/expected/expressions_optimizer.out +++ b/src/test/regress/expected/expressions_optimizer.out @@ -125,7 +125,7 @@ select count(*) from date_tbl where f1 not between '1997-01-01' and '1998-01-01'; count ------- - 12 + 13 (1 row) explain (costs off) @@ -163,6 +163,6 @@ select count(*) from date_tbl where f1 not between symmetric '1997-01-01' and '1998-01-01'; count ------- - 12 + 13 (1 row) From 17091af0d32f69028800eb3f805615f2a791f570 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Thu, 11 Jun 2026 10:31:51 +0300 Subject: [PATCH 563/589] Fix test src/test/regress/expected/qp_misc_rio.out (#2683) Commit 489c9c3407cbfd473c2f8d7863ffaaf6d2e8fcf8 changed the way negative dates are handled, so GPDB-specific test that was relying on this should be updated as well. --- src/test/regress/expected/qp_misc_rio.out | 6 +++--- src/test/regress/sql/qp_misc_rio.sql | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/regress/expected/qp_misc_rio.out b/src/test/regress/expected/qp_misc_rio.out index 53e20979e31d..514325ea6f53 100644 --- a/src/test/regress/expected/qp_misc_rio.out +++ b/src/test/regress/expected/qp_misc_rio.out @@ -730,9 +730,9 @@ order by a; -- does. The fix was submitted to upstream PostgreSQL and fixed there in -- version 8.4.16 (commit 5c4eb9166e.) -- ---------------------------------------------------------------------- -select to_date('-4713-11-23', 'yyyy-mm-dd'); -ERROR: date out of range: "-4713-11-23" -select to_date('-4713-11-24', 'yyyy-mm-dd'); +select to_date('-4714-11-23', 'yyyy-mm-dd'); +ERROR: date out of range: "-4714-11-23" +select to_date('-4714-11-24', 'yyyy-mm-dd'); to_date --------------- 11-24-4714 BC diff --git a/src/test/regress/sql/qp_misc_rio.sql b/src/test/regress/sql/qp_misc_rio.sql index 86acf6b5afc9..1d328ebc9325 100644 --- a/src/test/regress/sql/qp_misc_rio.sql +++ b/src/test/regress/sql/qp_misc_rio.sql @@ -535,7 +535,7 @@ order by a; -- does. The fix was submitted to upstream PostgreSQL and fixed there in -- version 8.4.16 (commit 5c4eb9166e.) -- ---------------------------------------------------------------------- -select to_date('-4713-11-23', 'yyyy-mm-dd'); -select to_date('-4713-11-24', 'yyyy-mm-dd'); +select to_date('-4714-11-23', 'yyyy-mm-dd'); +select to_date('-4714-11-24', 'yyyy-mm-dd'); select to_date('5874897-12-31', 'yyyy-mm-dd'); select to_date('5874898-01-01', 'yyyy-mm-dd'); From d82f7f554459c999d5b739b3da4b3a9a5d00b756 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Thu, 11 Jun 2026 15:05:09 +0700 Subject: [PATCH 564/589] Fix errors test (#2684) Commit 80d9e511193d6954c83abdeeaa999f2af53cdf6a incorrect fixed test. Commit ae0f7b11f143d9748e2201c3647330893c4c1c5a removed the test from errors.sql and put it into infinite_recurse.sql --- src/test/regress/expected/errors.out | 18 ------------------ src/test/regress/sql/errors.sql | 12 ------------ 2 files changed, 30 deletions(-) diff --git a/src/test/regress/expected/errors.out b/src/test/regress/expected/errors.out index 376ed327a8ca..a55912b38aa0 100644 --- a/src/test/regress/expected/errors.out +++ b/src/test/regress/expected/errors.out @@ -452,21 +452,3 @@ NULL); ERROR: syntax error at or near "NUL" LINE 16: ...L, id2 TEXT NOT NULL PRIMARY KEY, id3 INTEGER NOT NUL, id4 I... ^ --- Check that stack depth detection mechanism works and --- max_stack_depth is not set too high. The full error report is not --- very stable, so show only SQLSTATE and primary error message. -create function infinite_recurse() returns int as -'select infinite_recurse()' language sql CONTAINS SQL; -\set VERBOSITY sqlstate --- start_ignore -select infinite_recurse(); -ERROR: 54001 --- end_ignore -\echo :LAST_ERROR_MESSAGE -stack depth limit exceeded -select 1; -- test that this works - ?column? ----------- - 1 -(1 row) - diff --git a/src/test/regress/sql/errors.sql b/src/test/regress/sql/errors.sql index 8cba18ea6ce8..9cdf82cf3abb 100644 --- a/src/test/regress/sql/errors.sql +++ b/src/test/regress/sql/errors.sql @@ -378,15 +378,3 @@ INT4 UNIQUE NOT NULL); - --- Check that stack depth detection mechanism works and --- max_stack_depth is not set too high. The full error report is not --- very stable, so show only SQLSTATE and primary error message. -create function infinite_recurse() returns int as -'select infinite_recurse()' language sql CONTAINS SQL; -\set VERBOSITY sqlstate --- start_ignore -select infinite_recurse(); --- end_ignore -\echo :LAST_ERROR_MESSAGE -select 1; -- test that this works From 1cb4c2cef5d5ad851bdbeca937f40e47e8c2df38 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Thu, 11 Jun 2026 15:05:38 +0700 Subject: [PATCH 565/589] Fix generated test for optimizer (#2685) Commit bf797a8d9768239f5e3204b013044274b2c7c24a changed generated.sql test. This patch changes output for the optimizer. --- src/test/regress/expected/generated_optimizer.out | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/test/regress/expected/generated_optimizer.out b/src/test/regress/expected/generated_optimizer.out index db7bea1dca15..bc053ffa3246 100644 --- a/src/test/regress/expected/generated_optimizer.out +++ b/src/test/regress/expected/generated_optimizer.out @@ -1121,7 +1121,8 @@ CREATE TABLE gtest30 ( b int GENERATED ALWAYS AS (a * 2) STORED ); CREATE TABLE gtest30_1 () INHERITS (gtest30); -ALTER TABLE ONLY gtest30 ALTER COLUMN b DROP EXPRESSION; +ALTER TABLE ONLY gtest30 ALTER COLUMN b DROP EXPRESSION; -- error +ERROR: ALTER TABLE / DROP EXPRESSION must be applied to child tables too \d gtest30 INFO: GPORCA failed to produce a plan, falling back to planner DETAIL: Feature not supported: Non-default collation @@ -1141,11 +1142,11 @@ INFO: GPORCA failed to produce a plan, falling back to planner DETAIL: Feature not supported: Queries on master-only tables INFO: GPORCA failed to produce a plan, falling back to planner DETAIL: Feature not supported: Non-default collation - Table "public.gtest30" - Column | Type | Collation | Nullable | Default ---------+---------+-----------+----------+--------- + Table "public.gtest30" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+------------------------------------ a | integer | | | - b | integer | | | + b | integer | | | generated always as (a * 2) stored Number of child tables: 1 (Use \d+ to list them.) Distributed by: (a) From 7515e8e59bdd187a6527ad9bb6eaa053b4c76ad9 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Thu, 11 Jun 2026 11:15:37 +0300 Subject: [PATCH 566/589] Fix test src/test/regress/output/external_table.source (#2675) Commit cd6f479e79f3a33ef7a919c6b6c0c498c790f154 added new column refobjversion to pg_depend. However, there is a GPDB-specific test that does SELECT * FROM pg_depend. Since it's the only test doing that, and it doesn't actually need the output data, just replace it with count(*) to avoid future problems. --- src/test/regress/input/external_table.source | 2 +- src/test/regress/output/external_table.source | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/test/regress/input/external_table.source b/src/test/regress/input/external_table.source index 70e7dece0c37..62bc354dfe8b 100644 --- a/src/test/regress/input/external_table.source +++ b/src/test/regress/input/external_table.source @@ -42,7 +42,7 @@ FORMAT 'text' (delimiter '|'); -- Only tables with custom protocol should create dependency, due to a bug there -- used to be entries created for non custom protocol tables with refobjid=0. -SELECT * FROM pg_depend WHERE refclassid = 'pg_extprotocol'::regclass and refobjid = 0; +SELECT count(*) FROM pg_depend WHERE refclassid = 'pg_extprotocol'::regclass and refobjid = 0; -- drop tables diff --git a/src/test/regress/output/external_table.source b/src/test/regress/output/external_table.source index 5ecf46ced1a4..2c4d5ad435d8 100644 --- a/src/test/regress/output/external_table.source +++ b/src/test/regress/output/external_table.source @@ -39,10 +39,11 @@ location ('file://@hostname@@abs_srcdir@/data/region.tbl' ) FORMAT 'text' (delimiter '|'); -- Only tables with custom protocol should create dependency, due to a bug there -- used to be entries created for non custom protocol tables with refobjid=0. -SELECT * FROM pg_depend WHERE refclassid = 'pg_extprotocol'::regclass and refobjid = 0; - classid | objid | objsubid | refclassid | refobjid | refobjsubid | deptype ----------+-------+----------+------------+----------+-------------+--------- -(0 rows) +SELECT count(*) FROM pg_depend WHERE refclassid = 'pg_extprotocol'::regclass and refobjid = 0; + count +------- + 0 +(1 row) -- drop tables DROP EXTERNAL TABLE EXT_NATION; From 94f6b5d4c68aa5889b9265127dfc00dbb1ec3df1 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Thu, 11 Jun 2026 11:16:36 +0300 Subject: [PATCH 567/589] Fix ORCA statistics handling in src/backend/optimizer/util/plancat.c (#2658) Commit 3d351d916b20534f973eda760cde17d96545d4c4 changed the possible values of reltuples column in pg_class. Previously the default value was 0, but after this commit the default value is -1, meaning "unknown", while 0 indicates an actually empty table. Adjust GPDB-specific code for ORCA to account for that change. --- src/backend/optimizer/util/plancat.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index c5ddb6ad3f1b..ed0d9760cc92 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -681,7 +681,7 @@ cdb_estimate_partitioned_numtuples(Relation rel) childtuples = childrel->rd_rel->reltuples; - if (gp_enable_relsize_collection && childtuples == 0) + if (gp_enable_relsize_collection && childtuples <= 0) { RelOptInfo *dummy_reloptinfo; BlockNumber numpages; @@ -698,11 +698,18 @@ cdb_estimate_partitioned_numtuples(Relation rel) &allvisfrac); pfree(dummy_reloptinfo); } - if (childtuples == 0 && rel_is_external_table(RelationGetRelid(childrel))) + if (childtuples <= 0 && rel_is_external_table(RelationGetRelid(childrel))) { childtuples = DEFAULT_EXTERNAL_TABLE_TUPLES; } - totaltuples += childtuples; + + + /* + * reltuples of -1 indicates the relation was never analyzed. + * Treat this the same way as an empty relation. + */ + if (childtuples > 0) + totaltuples += childtuples; if (childrel != rel) table_close(childrel, NoLock); From 353d3eca1f58e5a8874f426234b171bc0a04d929 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Thu, 11 Jun 2026 15:59:11 +0700 Subject: [PATCH 568/589] Fix stats_ext test output for optimizer (#2687) Commit 3c99230b4f0d10c9eac5f4efdd2394eccb2af3a0 added a new test for stats_ext. Add its output to the optimizer output. --- .../regress/expected/stats_ext_optimizer.out | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/test/regress/expected/stats_ext_optimizer.out b/src/test/regress/expected/stats_ext_optimizer.out index 54a4c741c6b5..4f258cc1b34b 100644 --- a/src/test/regress/expected/stats_ext_optimizer.out +++ b/src/test/regress/expected/stats_ext_optimizer.out @@ -113,6 +113,15 @@ WARNING: statistics object "public.ab1_a_b_stats" could not be computed for rel ALTER TABLE ab1 ALTER a SET STATISTICS -1; -- setting statistics target 0 skips the statistics, without printing any message, so check catalog ALTER STATISTICS ab1_a_b_stats SET STATISTICS 0; +\d ab1 + Table "public.ab1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | +Statistics objects: + "public"."ab1_a_b_stats" (ndistinct, dependencies, mcv) ON a, b FROM ab1; STATISTICS 0 + ANALYZE ab1; SELECT stxname, stxdndistinct, stxddependencies, stxdmcv FROM pg_statistic_ext s, pg_statistic_ext_data d @@ -124,6 +133,15 @@ SELECT stxname, stxdndistinct, stxddependencies, stxdmcv (1 row) ALTER STATISTICS ab1_a_b_stats SET STATISTICS -1; +\d+ ab1 + Table "public.ab1" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------+---------+-----------+----------+---------+---------+--------------+------------- + a | integer | | | | plain | | + b | integer | | | | plain | | +Statistics objects: + "public"."ab1_a_b_stats" (ndistinct, dependencies, mcv) ON a, b FROM ab1 + -- partial analyze doesn't build stats either ANALYZE ab1 (a); WARNING: statistics object "public.ab1_a_b_stats" could not be computed for relation "public.ab1" From dd01eaccf9f6e8aa74bb4270982a49f4731b285e Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 11 Jun 2026 14:13:30 +0500 Subject: [PATCH 569/589] Fix tests in src/test/regress/expected/subselect_gp.out (#2682) Commit 41efb83 removed unused subplans. Remove in GPDB-specific tests. --- src/test/regress/expected/subselect_gp.out | 24 ++++++++-------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/src/test/regress/expected/subselect_gp.out b/src/test/regress/expected/subselect_gp.out index 6916a1a41365..4a5009f9c6f0 100644 --- a/src/test/regress/expected/subselect_gp.out +++ b/src/test/regress/expected/subselect_gp.out @@ -1800,23 +1800,17 @@ EXPLAIN select count(distinct ss.ten) from -- we should see 2 subplans in the explain -- EXPLAIN SELECT EXISTS(SELECT * FROM tenk1 WHERE tenk1.unique1 = tenk2.unique1) FROM tenk2 LIMIT 1; - QUERY PLAN --------------------------------------------------------------------------------------------------------------------------- - Limit (cost=0.00..0.08 rows=1 width=4) - -> Gather Motion 3:1 (slice3; segments: 3) (cost=0.00..0.08 rows=1 width=4) - -> Limit (cost=0.00..0.06 rows=1 width=4) - -> Seq Scan on tenk2 (cost=0.00..625.00 rows=3334 width=4) - SubPlan 1 - -> Result (cost=0.00..439.00 rows=10000 width=4) - Filter: (tenk1.unique1 = tenk2.unique1) - -> Materialize (cost=0.00..439.00 rows=10000 width=4) - -> Broadcast Motion 3:3 (slice1; segments: 3) (cost=0.00..389.00 rows=3334 width=4) - -> Seq Scan on tenk1 (cost=0.00..189.00 rows=3334 width=4) + QUERY PLAN +------------------------------------------------------------------------------------------------------------- + Limit (cost=17.76..17.79 rows=1 width=1) + -> Gather Motion 3:1 (slice1; segments: 3) (cost=17.76..17.85 rows=3 width=1) + -> Limit (cost=17.76..17.81 rows=1 width=1) + -> Seq Scan on tenk2 (cost=0.00..177.56 rows=3333 width=1) SubPlan 2 - -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..189.00 rows=3334 width=4) - -> Seq Scan on tenk1 tenk1_1 (cost=0.00..189.00 rows=3334 width=4) + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..62.33 rows=3333 width=4) + -> Seq Scan on tenk1 (cost=0.00..62.33 rows=3333 width=4) Optimizer: Postgres query optimizer -(14 rows) +(8 rows) SELECT EXISTS(SELECT * FROM tenk1 WHERE tenk1.unique1 = tenk2.unique1) FROM tenk2 LIMIT 1; exists From 6fa4e9125380c2ed70c6fe341d349e662b452c63 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 11 Jun 2026 14:13:53 +0500 Subject: [PATCH 570/589] Fix tests in update_gp.out, qp_misc.out and qp_subquery.out (#2681) Commit 92f8718 in src/backend/executor/nodeSubplan.c added a conditional error message in the ExecInitSubPlan function, which is inappropriate for the GPDB query executor. --- src/backend/executor/nodeSubplan.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/executor/nodeSubplan.c b/src/backend/executor/nodeSubplan.c index 279d0225f7a6..d64ee4fb683f 100644 --- a/src/backend/executor/nodeSubplan.c +++ b/src/backend/executor/nodeSubplan.c @@ -820,7 +820,7 @@ ExecInitSubPlan(SubPlan *subplan, PlanState *parent) * This check can fail if the planner mistakenly puts a parallel-unsafe * subplan into a parallelized subquery; see ExecSerializePlan. */ - if (sstate->planstate == NULL) + if (Gp_role != GP_ROLE_EXECUTE && sstate->planstate == NULL) elog(ERROR, "subplan \"%s\" was not initialized", subplan->plan_name); From 1a770a9ba33cc1a053b014c4c77ac3e376520895 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 11 Jun 2026 14:14:36 +0500 Subject: [PATCH 571/589] Fix tests in src/test/isolation2/expected/ao_upgrade.out (#2678) Commit 76f412a removed factorial operators, leaving only the factorial() function. Replace in GPDB-specific tests. --- src/test/isolation2/input/ao_upgrade.source | 12 ++++++------ src/test/isolation2/output/ao_upgrade.source | 12 ++++++------ .../isolation2/output/ao_upgrade_optimizer.source | 12 ++++++------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/test/isolation2/input/ao_upgrade.source b/src/test/isolation2/input/ao_upgrade.source index 30e065501006..7c4ab8535a82 100644 --- a/src/test/isolation2/input/ao_upgrade.source +++ b/src/test/isolation2/input/ao_upgrade.source @@ -72,13 +72,13 @@ SET enable_seqscan TO off; -- Ensure we're using a bitmap scan for our tests. Upgrade note to developers: -- the only thing that this test needs to verify is that a fetch-based scan is -- in use. Other diffs are fine. -EXPLAIN SELECT n FROM ao_upgrade_test WHERE n = (9 !); -EXPLAIN SELECT n FROM aocs_upgrade_test WHERE n = (9 !); -EXPLAIN SELECT n FROM aocs_rle_upgrade_test WHERE n = (9 !); +EXPLAIN SELECT n FROM ao_upgrade_test WHERE n = factorial(9); +EXPLAIN SELECT n FROM aocs_upgrade_test WHERE n = factorial(9); +EXPLAIN SELECT n FROM aocs_rle_upgrade_test WHERE n = factorial(9); -SELECT n FROM ao_upgrade_test WHERE n = (9 !); -SELECT n FROM aocs_upgrade_test WHERE n = (9 !); -SELECT n FROM aocs_rle_upgrade_test WHERE n = (9 !); +SELECT n FROM ao_upgrade_test WHERE n = factorial(9); +SELECT n FROM aocs_upgrade_test WHERE n = factorial(9); +SELECT n FROM aocs_rle_upgrade_test WHERE n = factorial(9); RESET enable_seqscan; diff --git a/src/test/isolation2/output/ao_upgrade.source b/src/test/isolation2/output/ao_upgrade.source index 3044d3a9e072..8cb1cfafd731 100644 --- a/src/test/isolation2/output/ao_upgrade.source +++ b/src/test/isolation2/output/ao_upgrade.source @@ -149,7 +149,7 @@ SET -- Ensure we're using a bitmap scan for our tests. Upgrade note to developers: -- the only thing that this test needs to verify is that a fetch-based scan is -- in use. Other diffs are fine. -EXPLAIN SELECT n FROM ao_upgrade_test WHERE n = (9 !); +EXPLAIN SELECT n FROM ao_upgrade_test WHERE n = factorial(9); QUERY PLAN ------------------------------------------------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) (cost=1000.36..1100.37 rows=1 width=9) @@ -160,7 +160,7 @@ EXPLAIN SELECT n FROM ao_upgrade_test WHERE n = (9 !); Settings: enable_seqscan=off Optimizer status: Postgres query optimizer (7 rows) -EXPLAIN SELECT n FROM aocs_upgrade_test WHERE n = (9 !); +EXPLAIN SELECT n FROM aocs_upgrade_test WHERE n = factorial(9); QUERY PLAN ------------------------------------------------------------------------------------------------------------ Gather Motion 3:1 (slice1; segments: 3) (cost=1000.36..1100.37 rows=1 width=9) @@ -171,7 +171,7 @@ EXPLAIN SELECT n FROM aocs_upgrade_test WHERE n = (9 !); Settings: enable_seqscan=off Optimizer status: Postgres query optimizer (7 rows) -EXPLAIN SELECT n FROM aocs_rle_upgrade_test WHERE n = (9 !); +EXPLAIN SELECT n FROM aocs_rle_upgrade_test WHERE n = factorial(9); QUERY PLAN ---------------------------------------------------------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) (cost=1000.36..1100.37 rows=1 width=9) @@ -183,17 +183,17 @@ EXPLAIN SELECT n FROM aocs_rle_upgrade_test WHERE n = (9 !); Optimizer status: Postgres query optimizer (7 rows) -SELECT n FROM ao_upgrade_test WHERE n = (9 !); +SELECT n FROM ao_upgrade_test WHERE n = factorial(9); n -------- 362880 (1 row) -SELECT n FROM aocs_upgrade_test WHERE n = (9 !); +SELECT n FROM aocs_upgrade_test WHERE n = factorial(9); n -------- 362880 (1 row) -SELECT n FROM aocs_rle_upgrade_test WHERE n = (9 !); +SELECT n FROM aocs_rle_upgrade_test WHERE n = factorial(9); n -------- 362880 diff --git a/src/test/isolation2/output/ao_upgrade_optimizer.source b/src/test/isolation2/output/ao_upgrade_optimizer.source index 3044d3a9e072..8cb1cfafd731 100644 --- a/src/test/isolation2/output/ao_upgrade_optimizer.source +++ b/src/test/isolation2/output/ao_upgrade_optimizer.source @@ -149,7 +149,7 @@ SET -- Ensure we're using a bitmap scan for our tests. Upgrade note to developers: -- the only thing that this test needs to verify is that a fetch-based scan is -- in use. Other diffs are fine. -EXPLAIN SELECT n FROM ao_upgrade_test WHERE n = (9 !); +EXPLAIN SELECT n FROM ao_upgrade_test WHERE n = factorial(9); QUERY PLAN ------------------------------------------------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) (cost=1000.36..1100.37 rows=1 width=9) @@ -160,7 +160,7 @@ EXPLAIN SELECT n FROM ao_upgrade_test WHERE n = (9 !); Settings: enable_seqscan=off Optimizer status: Postgres query optimizer (7 rows) -EXPLAIN SELECT n FROM aocs_upgrade_test WHERE n = (9 !); +EXPLAIN SELECT n FROM aocs_upgrade_test WHERE n = factorial(9); QUERY PLAN ------------------------------------------------------------------------------------------------------------ Gather Motion 3:1 (slice1; segments: 3) (cost=1000.36..1100.37 rows=1 width=9) @@ -171,7 +171,7 @@ EXPLAIN SELECT n FROM aocs_upgrade_test WHERE n = (9 !); Settings: enable_seqscan=off Optimizer status: Postgres query optimizer (7 rows) -EXPLAIN SELECT n FROM aocs_rle_upgrade_test WHERE n = (9 !); +EXPLAIN SELECT n FROM aocs_rle_upgrade_test WHERE n = factorial(9); QUERY PLAN ---------------------------------------------------------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) (cost=1000.36..1100.37 rows=1 width=9) @@ -183,17 +183,17 @@ EXPLAIN SELECT n FROM aocs_rle_upgrade_test WHERE n = (9 !); Optimizer status: Postgres query optimizer (7 rows) -SELECT n FROM ao_upgrade_test WHERE n = (9 !); +SELECT n FROM ao_upgrade_test WHERE n = factorial(9); n -------- 362880 (1 row) -SELECT n FROM aocs_upgrade_test WHERE n = (9 !); +SELECT n FROM aocs_upgrade_test WHERE n = factorial(9); n -------- 362880 (1 row) -SELECT n FROM aocs_rle_upgrade_test WHERE n = (9 !); +SELECT n FROM aocs_rle_upgrade_test WHERE n = factorial(9); n -------- 362880 From 6b1744674588d99e93e024194ada7dfd52934117 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 11 Jun 2026 14:15:15 +0500 Subject: [PATCH 572/589] Fix tests in isolation2 lockmodes.out and regress partition_locking.out (#2676) 1) Commit 2000b6c in src/backend/executor/execMain.c in the InitResultRelInfo function removed the call to the RelationGetPartitionQual function. This prevented the AccessShareLock lock from being acquired on the root partition when working with partitioned tables (in RelationGetPartitionQual -> generate_partition_qual). Remove this from the GPDB-specific tests. 2) Commit 3d351d9 redefined pg_class.reltuples to be -1 before the first VACUUM or ANALYZE. Fix this in the leaf_parts_analyzed function. --- src/backend/commands/analyzeutils.c | 4 +- src/test/isolation2/expected/lockmodes.out | 44 ++++++++----------- .../regress/expected/partition_locking.out | 32 ++++++-------- .../expected/partition_locking_optimizer.out | 32 ++++++-------- 4 files changed, 49 insertions(+), 63 deletions(-) diff --git a/src/backend/commands/analyzeutils.c b/src/backend/commands/analyzeutils.c index 0246037277b8..737734de82c0 100644 --- a/src/backend/commands/analyzeutils.c +++ b/src/backend/commands/analyzeutils.c @@ -1216,7 +1216,7 @@ leaf_parts_analyzed(Oid attrelid, Oid relid_exclude, List *va_cols, int elevel) int32 relpages = get_rel_relpages(partRelid); /* Partition is not analyzed */ - if (relTuples == 0.0 && relpages == 0) + if (relTuples < 0.0) { if (relid_exclude == InvalidOid) ereport(elevel, @@ -1240,7 +1240,7 @@ leaf_parts_analyzed(Oid attrelid, Oid relid_exclude, List *va_cols, int elevel) float4 relTuples = get_rel_reltuples(partRelid); /* Partition is analyzed and we detect it is empty */ - if (relTuples == 0.0) + if (relTuples <= 0.0) continue; all_parts_empty = false; diff --git a/src/test/isolation2/expected/lockmodes.out b/src/test/isolation2/expected/lockmodes.out index 8e64c82a51a9..0517dd9af559 100644 --- a/src/test/isolation2/expected/lockmodes.out +++ b/src/test/isolation2/expected/lockmodes.out @@ -1040,13 +1040,12 @@ BEGIN DELETE 10 -- on QD, there's a lock on the root and the target partition 1: select * from show_locks_lockmodes; - locktype | mode | granted | relation -----------+-----------------+---------+--------------------------------- - relation | AccessShareLock | t | t_lockmods_part_tbl_dml - relation | ExclusiveLock | t | t_lockmods_part_tbl_dml - relation | ExclusiveLock | t | t_lockmods_part_tbl_dml_1_prt_2 - relation | ExclusiveLock | t | t_lockmods_part_tbl_dml_1_prt_1 -(4 rows) + locktype | mode | granted | relation +----------+---------------+---------+--------------------------------- + relation | ExclusiveLock | t | t_lockmods_part_tbl_dml + relation | ExclusiveLock | t | t_lockmods_part_tbl_dml_1_prt_2 + relation | ExclusiveLock | t | t_lockmods_part_tbl_dml_1_prt_1 +(3 rows) 1: ROLLBACK; ROLLBACK @@ -1094,13 +1093,12 @@ BEGIN UPDATE 1 -- on QD, there's a lock on the root and the target partition 1: select * from show_locks_lockmodes; - locktype | mode | granted | relation -----------+-----------------+---------+--------------------------------- - relation | AccessShareLock | t | t_lockmods_part_tbl_dml - relation | ExclusiveLock | t | t_lockmods_part_tbl_dml - relation | ExclusiveLock | t | t_lockmods_part_tbl_dml_1_prt_2 - relation | ExclusiveLock | t | t_lockmods_part_tbl_dml_1_prt_1 -(4 rows) + locktype | mode | granted | relation +----------+---------------+---------+--------------------------------- + relation | ExclusiveLock | t | t_lockmods_part_tbl_dml + relation | ExclusiveLock | t | t_lockmods_part_tbl_dml_1_prt_2 + relation | ExclusiveLock | t | t_lockmods_part_tbl_dml_1_prt_1 +(3 rows) 1: ROLLBACK; ROLLBACK 1q: ... @@ -1111,11 +1109,10 @@ BEGIN DELETE 10 -- since the delete operation is on leaf part, there will be a lock on QD 1: select * from show_locks_lockmodes; - locktype | mode | granted | relation -----------+-----------------+---------+--------------------------------- - relation | AccessShareLock | t | t_lockmods_part_tbl_dml - relation | ExclusiveLock | t | t_lockmods_part_tbl_dml_1_prt_1 -(2 rows) + locktype | mode | granted | relation +----------+---------------+---------+--------------------------------- + relation | ExclusiveLock | t | t_lockmods_part_tbl_dml_1_prt_1 +(1 row) 1: ROLLBACK; ROLLBACK 1q: ... @@ -2148,9 +2145,8 @@ DELETE 10 ----------+------------------+---------+--------------------------------- relation | RowExclusiveLock | t | t_lockmods_part_tbl_dml_1_prt_2 relation | RowExclusiveLock | t | t_lockmods_part_tbl_dml_1_prt_1 - relation | AccessShareLock | t | t_lockmods_part_tbl_dml relation | RowExclusiveLock | t | t_lockmods_part_tbl_dml -(4 rows) +(3 rows) 1: ROLLBACK; ROLLBACK 1q: ... @@ -2165,9 +2161,8 @@ UPDATE 1 ----------+------------------+---------+--------------------------------- relation | RowExclusiveLock | t | t_lockmods_part_tbl_dml_1_prt_2 relation | RowExclusiveLock | t | t_lockmods_part_tbl_dml_1_prt_1 - relation | AccessShareLock | t | t_lockmods_part_tbl_dml relation | RowExclusiveLock | t | t_lockmods_part_tbl_dml -(4 rows) +(3 rows) 1: ROLLBACK; ROLLBACK 1q: ... @@ -2180,9 +2175,8 @@ DELETE 10 1: select * from show_locks_lockmodes; locktype | mode | granted | relation ----------+------------------+---------+--------------------------------- - relation | AccessShareLock | t | t_lockmods_part_tbl_dml relation | RowExclusiveLock | t | t_lockmods_part_tbl_dml_1_prt_1 -(2 rows) +(1 row) 1: ROLLBACK; ROLLBACK 1q: ... diff --git a/src/test/regress/expected/partition_locking.out b/src/test/regress/expected/partition_locking.out index bfc381798151..ed52ad42c65f 100644 --- a/src/test/regress/expected/partition_locking.out +++ b/src/test/regress/expected/partition_locking.out @@ -272,12 +272,11 @@ select * from locktest_master where coalesce not like 'gp_%' and coalesce not li select * from locktest_segments where coalesce not like 'gp_%' and coalesce not like 'pg_%'; coalesce | mode | locktype | node -------------------+------------------+----------+------------ - partlockt | AccessShareLock | relation | n segments partlockt | RowExclusiveLock | relation | n segments partlockt_1_prt_1 | RowExclusiveLock | relation | 1 segment partlockt_1_prt_2 | RowExclusiveLock | relation | 1 segment partlockt_1_prt_3 | RowExclusiveLock | relation | 1 segment -(5 rows) +(4 rows) commit; -- drop @@ -440,10 +439,9 @@ select * from locktest_master where coalesce not like 'gp_%' and coalesce not li select * from locktest_segments where coalesce not like 'gp_%' and coalesce not like 'pg_%'; coalesce | mode | locktype | node -------------------+------------------+----------+----------- - partlockt | AccessShareLock | relation | 1 segment partlockt | RowExclusiveLock | relation | 1 segment partlockt_1_prt_3 | RowExclusiveLock | relation | 1 segment -(3 rows) +(2 rows) commit; -- delete locking @@ -451,22 +449,20 @@ begin; delete from partlockt where i = 4; -- Known_opt_diff: MPP-20936 select * from locktest_master where coalesce not like 'gp_%' and coalesce not like 'pg_%'; - coalesce | mode | locktype | node --------------------------+-----------------+----------+-------- - partlockt | AccessShareLock | relation | master - partlockt | ExclusiveLock | relation | master - partlockt_1_prt_4 | ExclusiveLock | relation | master - partlockt_1_prt_4_i_idx | ExclusiveLock | relation | master -(4 rows) + coalesce | mode | locktype | node +-------------------------+---------------+----------+-------- + partlockt | ExclusiveLock | relation | master + partlockt_1_prt_4 | ExclusiveLock | relation | master + partlockt_1_prt_4_i_idx | ExclusiveLock | relation | master +(3 rows) select * from locktest_segments where coalesce not like 'gp_%' and coalesce not like 'pg_%'; - coalesce | mode | locktype | node --------------------------+-----------------+----------+----------- - partlockt | AccessShareLock | relation | 1 segment - partlockt | ExclusiveLock | relation | 1 segment - partlockt_1_prt_4 | ExclusiveLock | relation | 1 segment - partlockt_1_prt_4_i_idx | ExclusiveLock | relation | 1 segment -(4 rows) + coalesce | mode | locktype | node +-------------------------+---------------+----------+----------- + partlockt | ExclusiveLock | relation | 1 segment + partlockt_1_prt_4 | ExclusiveLock | relation | 1 segment + partlockt_1_prt_4_i_idx | ExclusiveLock | relation | 1 segment +(3 rows) commit; -- drop index diff --git a/src/test/regress/expected/partition_locking_optimizer.out b/src/test/regress/expected/partition_locking_optimizer.out index ab7f426cd910..c19c6e9fdce1 100644 --- a/src/test/regress/expected/partition_locking_optimizer.out +++ b/src/test/regress/expected/partition_locking_optimizer.out @@ -272,12 +272,11 @@ select * from locktest_master where coalesce not like 'gp_%' and coalesce not li select * from locktest_segments where coalesce not like 'gp_%' and coalesce not like 'pg_%'; coalesce | mode | locktype | node -------------------+------------------+----------+------------ - partlockt | AccessShareLock | relation | n segments partlockt | RowExclusiveLock | relation | n segments partlockt_1_prt_1 | RowExclusiveLock | relation | 1 segment partlockt_1_prt_2 | RowExclusiveLock | relation | 1 segment partlockt_1_prt_3 | RowExclusiveLock | relation | 1 segment -(5 rows) +(4 rows) commit; -- drop @@ -443,10 +442,9 @@ select * from locktest_master where coalesce not like 'gp_%' and coalesce not li select * from locktest_segments where coalesce not like 'gp_%' and coalesce not like 'pg_%'; coalesce | mode | locktype | node -------------------+------------------+----------+----------- - partlockt | AccessShareLock | relation | 1 segment partlockt | RowExclusiveLock | relation | 1 segment partlockt_1_prt_3 | RowExclusiveLock | relation | 1 segment -(3 rows) +(2 rows) commit; -- delete locking @@ -457,25 +455,23 @@ delete from partlockt where i = 4; -- GPDB_12_MERGE_FIXME Revisit this post merge and see if we have a chance to unlock the pruned partitions -- end_ignore select * from locktest_master where coalesce not like 'gp_%' and coalesce not like 'pg_%'; - coalesce | mode | locktype | node --------------------------+-----------------+----------+-------- - partlockt | AccessShareLock | relation | master - partlockt | ExclusiveLock | relation | master - partlockt_1_prt_4 | ExclusiveLock | relation | master - partlockt_1_prt_4_i_idx | ExclusiveLock | relation | master -(4 rows) + coalesce | mode | locktype | node +-------------------------+---------------+----------+-------- + partlockt | ExclusiveLock | relation | master + partlockt_1_prt_4 | ExclusiveLock | relation | master + partlockt_1_prt_4_i_idx | ExclusiveLock | relation | master +(3 rows) -- start_ignore -- GPDB_12_MERGE_FIXME Revisit this post merge and see if we have a chance to unlock the root table -- end_ignore select * from locktest_segments where coalesce not like 'gp_%' and coalesce not like 'pg_%'; - coalesce | mode | locktype | node --------------------------+-----------------+----------+----------- - partlockt | AccessShareLock | relation | 1 segment - partlockt | ExclusiveLock | relation | 1 segment - partlockt_1_prt_4 | ExclusiveLock | relation | 1 segment - partlockt_1_prt_4_i_idx | ExclusiveLock | relation | 1 segment -(4 rows) + coalesce | mode | locktype | node +-------------------------+---------------+----------+----------- + partlockt | ExclusiveLock | relation | 1 segment + partlockt_1_prt_4 | ExclusiveLock | relation | 1 segment + partlockt_1_prt_4_i_idx | ExclusiveLock | relation | 1 segment +(3 rows) commit; -- drop index From e7298df242d8bb71f9a874ad8a3c1908e614f6d5 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 11 Jun 2026 14:16:59 +0500 Subject: [PATCH 573/589] Fix tests in src/test/regress/expected/groupingsets.out (#2662) 1) Commit 4d346de added new tests to src/test/regress/sql/groupingsets.sql. Adapt them for the GPDB and add them to the ORCA. 2) Commit d6c08e2 added a new hash_mem_multiplier GUC and changed work_mem to hash_mem in many places. Explicitly increase hash_mem_multiplier to preserve plans in a couple of tests. --- src/test/regress/expected/groupingsets.out | 36 +++++++++++++------ .../expected/groupingsets_optimizer.out | 29 +++++++++++++++ src/test/regress/sql/groupingsets.sql | 2 ++ 3 files changed, 57 insertions(+), 10 deletions(-) diff --git a/src/test/regress/expected/groupingsets.out b/src/test/regress/expected/groupingsets.out index 95c4d02469ee..0e6f7b242292 100644 --- a/src/test/regress/expected/groupingsets.out +++ b/src/test/regress/expected/groupingsets.out @@ -512,19 +512,33 @@ select * from ( group by grouping sets(1, 2) ) ss where x = 1 and q1 = 123; - QUERY PLAN --------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------ Subquery Scan on ss Output: ss.x, ss.q1, ss.sum Filter: ((ss.x = 1) AND (ss.q1 = 123)) - -> GroupAggregate - Output: (1), i1.q1, sum(i1.q2) - Group Key: 1 - Sort Key: i1.q1 - Group Key: i1.q1 - -> Seq Scan on public.int8_tbl i1 - Output: 1, i1.q1, i1.q2 -(10 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Output: 1, i1.q1, (sum(i1.q2)) + Merge Key: i1.q1 + -> Finalize GroupAggregate + Output: (1), i1.q1, sum(i1.q2) + Group Key: 1, i1.q1, (GROUPINGSET_ID()) + -> Sort + Output: 1, i1.q1, (PARTIAL sum(i1.q2)), (GROUPINGSET_ID()) + Sort Key: i1.q1, (GROUPINGSET_ID()) + -> Redistribute Motion 3:3 (slice2; segments: 3) + Output: 1, i1.q1, (PARTIAL sum(i1.q2)), (GROUPINGSET_ID()) + Hash Key: 1, i1.q1, (GROUPINGSET_ID()) + -> Partial GroupAggregate + Output: (1), i1.q1, PARTIAL sum(i1.q2), GROUPINGSET_ID() + Group Key: 1 + Sort Key: i1.q1 + Group Key: i1.q1 + -> Seq Scan on public.int8_tbl i1 + Output: 1, i1.q1, i1.q2 + Optimizer: Postgres query optimizer + Settings: enable_hashagg = 'off', optimizer = 'off' +(24 rows) select * from ( select 1 as x, q1, sum(q2) @@ -1774,6 +1788,7 @@ select array(select row(v.a,s1.*) from (select two,four, count(*) from onek grou set enable_indexscan = false; set work_mem = '64kB'; WARNING: "work_mem": setting is deprecated, and may be removed in a future release. +set hash_mem_multiplier = 2; explain (costs off) select unique1, count(two), count(four), count(ten), @@ -1822,6 +1837,7 @@ explain (costs off) Optimizer: Postgres query optimizer (12 rows) +reset hash_mem_multiplier; set work_mem = '384kB'; WARNING: "work_mem": setting is deprecated, and may be removed in a future release. explain (costs off) diff --git a/src/test/regress/expected/groupingsets_optimizer.out b/src/test/regress/expected/groupingsets_optimizer.out index d8edd5da00a7..44b1cff9a817 100644 --- a/src/test/regress/expected/groupingsets_optimizer.out +++ b/src/test/regress/expected/groupingsets_optimizer.out @@ -541,6 +541,33 @@ select x, not x as not_x, q2 from | | 4567890123456789 (5 rows) +-- check qual push-down rules for a subquery with grouping sets +explain (verbose, costs off) +select * from ( + select 1 as x, q1, sum(q2) + from int8_tbl i1 + group by grouping sets(1, 2) +) ss +where x = 1 and q1 = 123; + QUERY PLAN +------------------------------------------------------ + Result + Output: NULL::integer, NULL::bigint, NULL::numeric + One-Time Filter: false + Optimizer: Pivotal Optimizer (GPORCA) + Settings: enable_hashagg = 'off' +(5 rows) + +select * from ( + select 1 as x, q1, sum(q2) + from int8_tbl i1 + group by grouping sets(1, 2) +) ss +where x = 1 and q1 = 123; + x | q1 | sum +---+----+----- +(0 rows) + -- simple rescan tests select a, b, sum(v.x) from (values (1),(2)) v(x), gstest_data(v.x) @@ -1936,6 +1963,7 @@ DETAIL: Feature not supported: ROW EXPRESSION set enable_indexscan = false; set work_mem = '64kB'; WARNING: "work_mem": setting is deprecated, and may be removed in a future release. +set hash_mem_multiplier = 2; explain (costs off) select unique1, count(two), count(four), count(ten), @@ -2047,6 +2075,7 @@ explain (costs off) Optimizer: Pivotal Optimizer (GPORCA) (39 rows) +reset hash_mem_multiplier; set work_mem = '384kB'; WARNING: "work_mem": setting is deprecated, and may be removed in a future release. explain (costs off) diff --git a/src/test/regress/sql/groupingsets.sql b/src/test/regress/sql/groupingsets.sql index cf4c0282b67c..1142a2bc4e58 100644 --- a/src/test/regress/sql/groupingsets.sql +++ b/src/test/regress/sql/groupingsets.sql @@ -530,6 +530,7 @@ select array(select row(v.a,s1.*) from (select two,four, count(*) from onek grou set enable_indexscan = false; set work_mem = '64kB'; +set hash_mem_multiplier = 2; explain (costs off) select unique1, count(two), count(four), count(ten), @@ -542,6 +543,7 @@ explain (costs off) count(hundred), count(thousand), count(twothousand), count(*) from tenk1 group by grouping sets (unique1,hundred,ten,four,two); +reset hash_mem_multiplier; set work_mem = '384kB'; explain (costs off) From aaa224bc9bc965c58504aaaa5ca274defa71abbb Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Thu, 11 Jun 2026 12:44:59 +0300 Subject: [PATCH 574/589] Fix test src/test/regress/expected/incremental_analyze.out (#2679) Commit 3d351d916b20534f973eda760cde17d96545d4c4 redefined pg_class.reltuples to be -1 before the first VACUUM or ANALYZE. Adapt the output of GPDB-specific tests previously missed. --- src/test/regress/expected/analyze.out | 352 +++++++++--------- .../regress/expected/incremental_analyze.out | 2 +- .../incremental_analyze_optimizer.out | 2 +- 3 files changed, 178 insertions(+), 178 deletions(-) diff --git a/src/test/regress/expected/analyze.out b/src/test/regress/expected/analyze.out index 3fa14b96f63a..4ccea49ce576 100644 --- a/src/test/regress/expected/analyze.out +++ b/src/test/regress/expected/analyze.out @@ -30,20 +30,20 @@ analyze p3_sales; select relname, reltuples, relpages from pg_class where relname like 'p3_sales%' order by relname; relname | reltuples | relpages -----------------------------------------------------------------+-----------+---------- - p3_sales | 0 | 0 - p3_sales_1_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2 | 0 | 0 + p3_sales | -1 | 0 + p3_sales_1_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2 | -1 | 0 p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | 0 | 1 p3_sales_1_prt_2_2_prt_2_3_prt_usa | 2 | 1 - p3_sales_1_prt_2_2_prt_other_months | 0 | 0 + p3_sales_1_prt_2_2_prt_other_months | -1 | 0 p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | 0 | 1 p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | 0 | 1 - p3_sales_1_prt_outlying_years | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2 | 0 | 0 + p3_sales_1_prt_outlying_years | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2 | -1 | 0 p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | 0 | 1 p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | 0 | 1 p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | 0 | 1 - p3_sales_1_prt_outlying_years_2_prt_other_months | 0 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months | -1 | 0 p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | 0 | 1 (15 rows) @@ -82,21 +82,21 @@ WARNING: skipping "p3_sales_1_prt_2" --- cannot analyze a mid-level partition. select relname, reltuples, relpages from pg_class where relname like 'p3_sales%' order by relname; relname | reltuples | relpages -----------------------------------------------------------------+-----------+---------- - p3_sales | 0 | 0 - p3_sales_1_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_2_2_prt_2_3_prt_usa | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | 0 | 0 - p3_sales_1_prt_outlying_years | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2 | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_months | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | 0 | 0 + p3_sales | -1 | 0 + p3_sales_1_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_2_2_prt_2_3_prt_usa | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | -1 | 0 + p3_sales_1_prt_outlying_years | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2 | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | -1 | 0 (15 rows) select * from pg_stats where tablename like 'p3_sales%' order by tablename, attname; @@ -128,21 +128,21 @@ analyze p3_sales_1_prt_2_2_prt_2_3_prt_usa; select relname, reltuples, relpages from pg_class where relname like 'p3_sales%' order by relname; relname | reltuples | relpages -----------------------------------------------------------------+-----------+---------- - p3_sales | 0 | 0 - p3_sales_1_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | 0 | 0 + p3_sales | -1 | 0 + p3_sales_1_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | -1 | 0 p3_sales_1_prt_2_2_prt_2_3_prt_usa | 2 | 1 - p3_sales_1_prt_2_2_prt_other_months | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | 0 | 0 - p3_sales_1_prt_outlying_years | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2 | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_months | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | 0 | 0 + p3_sales_1_prt_2_2_prt_other_months | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | -1 | 0 + p3_sales_1_prt_outlying_years | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2 | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | -1 | 0 (15 rows) select * from pg_stats where tablename like 'p3_sales%' order by tablename, attname; @@ -179,20 +179,20 @@ analyze; select relname, reltuples, relpages from pg_class where relname like 'p3_sales%' order by relname; relname | reltuples | relpages -----------------------------------------------------------------+-----------+---------- - p3_sales | 0 | 0 - p3_sales_1_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2 | 0 | 0 + p3_sales | -1 | 0 + p3_sales_1_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2 | -1 | 0 p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | 0 | 1 p3_sales_1_prt_2_2_prt_2_3_prt_usa | 2 | 1 - p3_sales_1_prt_2_2_prt_other_months | 0 | 0 + p3_sales_1_prt_2_2_prt_other_months | -1 | 0 p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | 0 | 1 p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | 0 | 1 - p3_sales_1_prt_outlying_years | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2 | 0 | 0 + p3_sales_1_prt_outlying_years | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2 | -1 | 0 p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | 0 | 1 p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | 0 | 1 p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | 0 | 1 - p3_sales_1_prt_outlying_years_2_prt_other_months | 0 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months | -1 | 0 p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | 0 | 1 (15 rows) @@ -230,20 +230,20 @@ vacuum analyze; select relname, reltuples, relpages from pg_class where relname like 'p3_sales%' order by relname; relname | reltuples | relpages -----------------------------------------------------------------+-----------+---------- - p3_sales | 0 | 0 - p3_sales_1_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2 | 0 | 0 + p3_sales | -1 | 0 + p3_sales_1_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2 | -1 | 0 p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | 0 | 1 p3_sales_1_prt_2_2_prt_2_3_prt_usa | 2 | 1 - p3_sales_1_prt_2_2_prt_other_months | 0 | 0 + p3_sales_1_prt_2_2_prt_other_months | -1 | 0 p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | 0 | 1 p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | 0 | 1 - p3_sales_1_prt_outlying_years | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2 | 0 | 0 + p3_sales_1_prt_outlying_years | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2 | -1 | 0 p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | 0 | 1 p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | 0 | 1 p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | 0 | 1 - p3_sales_1_prt_outlying_years_2_prt_other_months | 0 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months | -1 | 0 p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | 0 | 1 (15 rows) @@ -287,21 +287,21 @@ analyze rootpartition p3_sales; select relname, reltuples, relpages from pg_class where relname like 'p3_sales%' order by relname; relname | reltuples | relpages -----------------------------------------------------------------+-----------+---------- - p3_sales | 0 | 0 - p3_sales_1_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_2_2_prt_2_3_prt_usa | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | 0 | 0 - p3_sales_1_prt_outlying_years | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2 | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_months | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | 0 | 0 + p3_sales | -1 | 0 + p3_sales_1_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_2_2_prt_2_3_prt_usa | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | -1 | 0 + p3_sales_1_prt_outlying_years | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2 | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | -1 | 0 (15 rows) select * from pg_stats where tablename like 'p3_sales%' order by tablename, attname; @@ -339,21 +339,21 @@ WARNING: skipping "p3_sales_1_prt_2" --- cannot analyze a non-root partition us select relname, reltuples, relpages from pg_class where relname like 'p3_sales%' order by relname; relname | reltuples | relpages -----------------------------------------------------------------+-----------+---------- - p3_sales | 0 | 0 - p3_sales_1_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_2_2_prt_2_3_prt_usa | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | 0 | 0 - p3_sales_1_prt_outlying_years | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2 | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_months | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | 0 | 0 + p3_sales | -1 | 0 + p3_sales_1_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_2_2_prt_2_3_prt_usa | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | -1 | 0 + p3_sales_1_prt_outlying_years | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2 | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | -1 | 0 (15 rows) select * from pg_stats where tablename like 'p3_sales%' order by tablename, attname; @@ -386,21 +386,21 @@ WARNING: skipping "p3_sales_1_prt_2_2_prt_2_3_prt_usa" --- cannot analyze a non select relname, reltuples, relpages from pg_class where relname like 'p3_sales%' order by relname; relname | reltuples | relpages -----------------------------------------------------------------+-----------+---------- - p3_sales | 0 | 0 - p3_sales_1_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_2_2_prt_2_3_prt_usa | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | 0 | 0 - p3_sales_1_prt_outlying_years | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2 | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_months | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | 0 | 0 + p3_sales | -1 | 0 + p3_sales_1_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_2_2_prt_2_3_prt_usa | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | -1 | 0 + p3_sales_1_prt_outlying_years | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2 | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | -1 | 0 (15 rows) select * from pg_stats where tablename like 'p3_sales%' order by tablename, attname; @@ -432,20 +432,20 @@ analyze p3_sales; select relname, reltuples, relpages from pg_class where relname like 'p3_sales%' order by relname; relname | reltuples | relpages -----------------------------------------------------------------+-----------+---------- - p3_sales | 0 | 0 - p3_sales_1_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2 | 0 | 0 + p3_sales | -1 | 0 + p3_sales_1_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2 | -1 | 0 p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | 0 | 1 p3_sales_1_prt_2_2_prt_2_3_prt_usa | 2 | 1 - p3_sales_1_prt_2_2_prt_other_months | 0 | 0 + p3_sales_1_prt_2_2_prt_other_months | -1 | 0 p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | 0 | 1 p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | 0 | 1 - p3_sales_1_prt_outlying_years | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2 | 0 | 0 + p3_sales_1_prt_outlying_years | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2 | -1 | 0 p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | 0 | 1 p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | 0 | 1 p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | 0 | 1 - p3_sales_1_prt_outlying_years_2_prt_other_months | 0 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months | -1 | 0 p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | 0 | 1 (15 rows) @@ -488,21 +488,21 @@ analyze rootpartition p3_sales; select relname, reltuples, relpages from pg_class where relname like 'p3_sales%' order by relname; relname | reltuples | relpages -----------------------------------------------------------------+-----------+---------- - p3_sales | 0 | 0 - p3_sales_1_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_2_2_prt_2_3_prt_usa | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | 0 | 0 - p3_sales_1_prt_outlying_years | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2 | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_months | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | 0 | 0 + p3_sales | -1 | 0 + p3_sales_1_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_2_2_prt_2_3_prt_usa | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | -1 | 0 + p3_sales_1_prt_outlying_years | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2 | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | -1 | 0 (15 rows) select * from pg_stats where tablename like 'p3_sales%' order by tablename, attname; @@ -539,20 +539,20 @@ analyze p3_sales; select relname, reltuples, relpages from pg_class where relname like 'p3_sales%' order by relname; relname | reltuples | relpages -----------------------------------------------------------------+-----------+---------- - p3_sales | 0 | 0 - p3_sales_1_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2 | 0 | 0 + p3_sales | -1 | 0 + p3_sales_1_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2 | -1 | 0 p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | 0 | 1 p3_sales_1_prt_2_2_prt_2_3_prt_usa | 2 | 1 - p3_sales_1_prt_2_2_prt_other_months | 0 | 0 + p3_sales_1_prt_2_2_prt_other_months | -1 | 0 p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | 0 | 1 p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | 0 | 1 - p3_sales_1_prt_outlying_years | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2 | 0 | 0 + p3_sales_1_prt_outlying_years | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2 | -1 | 0 p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | 0 | 1 p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | 0 | 1 p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | 0 | 1 - p3_sales_1_prt_outlying_years_2_prt_other_months | 0 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months | -1 | 0 p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | 0 | 1 (15 rows) @@ -605,21 +605,21 @@ analyze rootpartition p3_sales; select relname, reltuples, relpages from pg_class where relname like 'p3_sales%' order by relname; relname | reltuples | relpages -----------------------------------------------------------------+-----------+---------- - p3_sales | 0 | 0 - p3_sales_1_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_2_2_prt_2_3_prt_usa | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | 0 | 0 - p3_sales_1_prt_outlying_years | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2 | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_months | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | 0 | 0 + p3_sales | -1 | 0 + p3_sales_1_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_2_2_prt_2_3_prt_usa | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | -1 | 0 + p3_sales_1_prt_outlying_years | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2 | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | -1 | 0 (15 rows) select * from pg_stats where tablename like 'p3_sales%' order by tablename, attname; @@ -656,21 +656,21 @@ analyze rootpartition p3_sales; select relname, reltuples, relpages from pg_class where relname like 'p3_sales%' order by relname; relname | reltuples | relpages -----------------------------------------------------------------+-----------+---------- - p3_sales | 0 | 0 - p3_sales_1_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_2_2_prt_2_3_prt_usa | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | 0 | 0 - p3_sales_1_prt_outlying_years | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2 | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_months | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | 0 | 0 + p3_sales | -1 | 0 + p3_sales_1_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_2_2_prt_2_3_prt_usa | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | -1 | 0 + p3_sales_1_prt_outlying_years | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2 | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | -1 | 0 (15 rows) select * from pg_stats where tablename like 'p3_sales%' order by tablename, attname; @@ -707,20 +707,20 @@ analyze p3_sales; select relname, reltuples, relpages from pg_class where relname like 'p3_sales%' order by relname; relname | reltuples | relpages -----------------------------------------------------------------+-----------+---------- - p3_sales | 0 | 0 - p3_sales_1_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2 | 0 | 0 + p3_sales | -1 | 0 + p3_sales_1_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2 | -1 | 0 p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | 0 | 1 p3_sales_1_prt_2_2_prt_2_3_prt_usa | 2 | 1 - p3_sales_1_prt_2_2_prt_other_months | 0 | 0 + p3_sales_1_prt_2_2_prt_other_months | -1 | 0 p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | 0 | 1 p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | 0 | 1 - p3_sales_1_prt_outlying_years | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2 | 0 | 0 + p3_sales_1_prt_outlying_years | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2 | -1 | 0 p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | 0 | 1 p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | 0 | 1 p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | 0 | 1 - p3_sales_1_prt_outlying_years_2_prt_other_months | 0 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months | -1 | 0 p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | 0 | 1 (15 rows) @@ -768,21 +768,21 @@ analyze rootpartition p3_sales; select relname, reltuples, relpages from pg_class where relname like 'p3_sales%' order by relname; relname | reltuples | relpages -----------------------------------------------------------------+-----------+---------- - p3_sales | 0 | 0 - p3_sales_1_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2 | 0 | 0 - p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_2_2_prt_2_3_prt_usa | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | 0 | 0 - p3_sales_1_prt_outlying_years | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2 | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_months | 0 | 0 - p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | 0 | 0 + p3_sales | -1 | 0 + p3_sales_1_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2 | -1 | 0 + p3_sales_1_prt_2_2_prt_2_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_2_2_prt_2_3_prt_usa | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_2_2_prt_other_months_3_prt_usa | -1 | 0 + p3_sales_1_prt_outlying_years | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2 | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_2_3_prt_usa | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_m_3_prt_other_regions | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months | -1 | 0 + p3_sales_1_prt_outlying_years_2_prt_other_months_3_prt_usa | -1 | 0 (15 rows) select * from pg_stats where tablename like 'p3_sales%' order by tablename, attname; diff --git a/src/test/regress/expected/incremental_analyze.out b/src/test/regress/expected/incremental_analyze.out index 43f7c3c6ae9c..8e1e2be2e213 100644 --- a/src/test/regress/expected/incremental_analyze.out +++ b/src/test/regress/expected/incremental_analyze.out @@ -1893,7 +1893,7 @@ SELECT tablename, attname, null_frac, n_distinct, most_common_vals, most_common_ SELECT relname, relpages, reltuples FROM pg_class WHERE relname LIKE 'incr_analyze_test%' ORDER BY relname; relname | relpages | reltuples ---------------------------+----------+----------- - incr_analyze_test | 0 | 0 + incr_analyze_test | 0 | -1 incr_analyze_test_1_prt_1 | 1 | 1 incr_analyze_test_1_prt_2 | 2 | 1000 incr_analyze_test_1_prt_3 | 1 | 0 diff --git a/src/test/regress/expected/incremental_analyze_optimizer.out b/src/test/regress/expected/incremental_analyze_optimizer.out index 96c59715a298..cf8c89578016 100644 --- a/src/test/regress/expected/incremental_analyze_optimizer.out +++ b/src/test/regress/expected/incremental_analyze_optimizer.out @@ -1902,7 +1902,7 @@ SELECT tablename, attname, null_frac, n_distinct, most_common_vals, most_common_ SELECT relname, relpages, reltuples FROM pg_class WHERE relname LIKE 'incr_analyze_test%' ORDER BY relname; relname | relpages | reltuples ---------------------------+----------+----------- - incr_analyze_test | 0 | 0 + incr_analyze_test | 0 | -1 incr_analyze_test_1_prt_1 | 1 | 1 incr_analyze_test_1_prt_2 | 2 | 1000 incr_analyze_test_1_prt_3 | 1 | 0 From 44af126d27e7082e3bbd001948458f1ae21370b2 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 11 Jun 2026 15:53:53 +0500 Subject: [PATCH 575/589] =?UTF-8?q?Fix=20tests=20in=20src/test/regress/exp?= =?UTF-8?q?ected/index=5Fconstraint=5Fnaming=5Fpartit=E2=80=A6=20(#2674)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit cd6f479 added the refobjversion column to the pg_depend catalog table in src/include/catalog/pg_depend.h. Add it to the GPDB-specific tests. --- .../regress/expected/index_constraint_naming_partition.out | 6 ++++-- src/test/regress/sql/index_constraint_naming_partition.sql | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/test/regress/expected/index_constraint_naming_partition.out b/src/test/regress/expected/index_constraint_naming_partition.out index a934d8c406ca..0ca5ad08dc9b 100755 --- a/src/test/regress/expected/index_constraint_naming_partition.out +++ b/src/test/regress/expected/index_constraint_naming_partition.out @@ -45,7 +45,8 @@ NOTICE: function dependencies() does not exist, skipping CREATE FUNCTION dependencies() RETURNS TABLE( depname NAME, classtype "char", depnsoid OID, refname NAME, refclasstype "char", refnsoid OID, classid REGCLASS, objid OID, objsubid INTEGER, - refclassid REGCLASS, refobjid OID, refobjsubid OID, deptype "char" ) + refclassid REGCLASS, refobjid OID, refobjsubid OID, + deptype "char", refobjversion "text" ) LANGUAGE SQL STABLE STRICT AS $fn$ WITH RECURSIVE w AS ( @@ -55,7 +56,8 @@ WITH RECURSIVE refclassid::regclass, refobjid, refobjsubid, - deptype + deptype, + refobjversion FROM pg_depend d WHERE classid IN ('pg_constraint'::regclass, 'pg_class'::regclass) AND (objid > 16384 OR refobjid > 16384) diff --git a/src/test/regress/sql/index_constraint_naming_partition.sql b/src/test/regress/sql/index_constraint_naming_partition.sql index 32698aaa9a42..7493fe22f428 100644 --- a/src/test/regress/sql/index_constraint_naming_partition.sql +++ b/src/test/regress/sql/index_constraint_naming_partition.sql @@ -45,7 +45,8 @@ DROP FUNCTION IF EXISTS dependencies(); CREATE FUNCTION dependencies() RETURNS TABLE( depname NAME, classtype "char", depnsoid OID, refname NAME, refclasstype "char", refnsoid OID, classid REGCLASS, objid OID, objsubid INTEGER, - refclassid REGCLASS, refobjid OID, refobjsubid OID, deptype "char" ) + refclassid REGCLASS, refobjid OID, refobjsubid OID, + deptype "char", refobjversion "text" ) LANGUAGE SQL STABLE STRICT AS $fn$ WITH RECURSIVE w AS ( @@ -55,7 +56,8 @@ WITH RECURSIVE refclassid::regclass, refobjid, refobjsubid, - deptype + deptype, + refobjversion FROM pg_depend d WHERE classid IN ('pg_constraint'::regclass, 'pg_class'::regclass) AND (objid > 16384 OR refobjid > 16384) From c8b469eea71c1e95f5713d52e00342a933105620 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Thu, 11 Jun 2026 15:54:07 +0500 Subject: [PATCH 576/589] Fix tests in partition.out, partition1.out and partition_indexing.out (#2673) 1) Commit 9fc2122 in src/backend/commands/indexcmds.c changed the error message in the DefineIndex function. Change this in the GPDB-specific test outputs. 2) Commit 6b2c4e5 in src/backend/partitioning/partbounds.c changed the error position display in the check_new_partition_bound function. Add the error position display in the GPDB-specific test outputs. --- src/test/regress/expected/partition.out | 10 +++++++--- src/test/regress/expected/partition1.out | 6 ++++-- src/test/regress/expected/partition_indexing.out | 4 ++-- src/test/regress/expected/partition_optimizer.out | 10 +++++++--- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/test/regress/expected/partition.out b/src/test/regress/expected/partition.out index 2efa8d74dd33..c8a160abadf3 100755 --- a/src/test/regress/expected/partition.out +++ b/src/test/regress/expected/partition.out @@ -4530,8 +4530,8 @@ create table tc default partition d, start (0) inclusive end(100) inclusive every (50) ); +ERROR: unique constraint on partitioned table must include all partitioning columns DETAIL: PRIMARY KEY constraint on table "tc" lacks column "b" which is part of the partition key. -ERROR: insufficient columns in PRIMARY KEY constraint definition create table cc (a int primary key, b int, c int) distributed by (a) @@ -4540,8 +4540,8 @@ create table cc default partition d, start (0) inclusive end(100) inclusive every (50) ); +ERROR: unique constraint on partitioned table must include all partitioning columns DETAIL: PRIMARY KEY constraint on table "cc" lacks column "b" which is part of the partition key. -ERROR: insufficient columns in PRIMARY KEY constraint definition create table at (a int, b int, c int) distributed by (a) @@ -4552,8 +4552,8 @@ create table at ); alter table at add primary key (a); +ERROR: unique constraint on partitioned table must include all partitioning columns DETAIL: PRIMARY KEY constraint on table "at" lacks column "b" which is part of the partition key. -ERROR: insufficient columns in PRIMARY KEY constraint definition -- MPP-14471 end -- MPP-17606 (using table "at" from above) alter table at @@ -4619,6 +4619,8 @@ create table plst2 partition p3 values ( CAST('(1,2)' as plst2_partkey) ) ); ERROR: partition "plst2_1_prt_p3" would overlap partition "plst2_1_prt_p1" +LINE 11: partition p3 values ( CAST('(1,2)' as plst2_partkey)... + ^ -- postive; make sure inner part duplicates are accepted and quietly removed. drop table if exists plst2; NOTICE: table "plst2" does not exist, skipping @@ -4655,6 +4657,8 @@ Distributed by: (d) -- negative; make sure conflicting alters fail. alter table plst2 add partition p6 values (CAST('(7,8)' as plst2_partkey), CAST('(2,1)' as plst2_partkey)); ERROR: partition "plst2_1_prt_p6" would overlap partition "plst2_1_prt_p5" +LINE 1: alter table plst2 add partition p6 values (CAST('(7,8)' as p... + ^ drop table if exists plst2; -- MPP-17814 end -- MPP-18441 diff --git a/src/test/regress/expected/partition1.out b/src/test/regress/expected/partition1.out index b42e0feffe37..3cb6fdeaac93 100644 --- a/src/test/regress/expected/partition1.out +++ b/src/test/regress/expected/partition1.out @@ -311,6 +311,8 @@ partition by list (rankgender) partition bb values (CAST ('(1,M)' AS rank_partkey)) ); ERROR: partition "rank_1_prt_bb" would overlap partition "rank_1_prt_1" +LINE 10: partition bb values (CAST ('(1,M)' AS rank_partkey)) + ^ -- RANGE validation -- legal if end of aa not inclusive create table ggg (a char(1), b date, d char(3)) @@ -2411,7 +2413,7 @@ PARTITION BY list (b) PARTITION t2 values (2)); -- the following statement should fail because index cols does not contain part key CREATE UNIQUE INDEX uidx_t_idx_col_contain_partkey on t_idx_col_contain_partkey(a); -ERROR: insufficient columns in UNIQUE constraint definition +ERROR: unique constraint on partitioned table must include all partitioning columns DETAIL: UNIQUE constraint on table "t_idx_col_contain_partkey" lacks column "b" which is part of the partition key. -- the following statement should work CREATE UNIQUE INDEX uidx_t_idx_col_contain_partkey on t_idx_col_contain_partkey(a, b); @@ -2443,7 +2445,7 @@ SUBPARTITION BY LIST (r_name) SUBPARTITION TEMPLATE ); -- should fail, must contain all the partition keys of all levels CREATE UNIQUE INDEX uidx_t_idx_col_contain_partkey on t_idx_col_contain_partkey(r_regionkey); -ERROR: insufficient columns in UNIQUE constraint definition +ERROR: unique constraint on partitioned table must include all partitioning columns DETAIL: UNIQUE constraint on table "t_idx_col_contain_partkey_1_prt_region1" lacks column "r_name" which is part of the partition key. -- should work CREATE UNIQUE INDEX uidx_t_idx_col_contain_partkey on t_idx_col_contain_partkey(r_regionkey, r_name); diff --git a/src/test/regress/expected/partition_indexing.out b/src/test/regress/expected/partition_indexing.out index aec0bd0f1b00..f95879d89553 100644 --- a/src/test/regress/expected/partition_indexing.out +++ b/src/test/regress/expected/partition_indexing.out @@ -167,7 +167,7 @@ CREATE UNIQUE INDEX mpp3033a_stringu1 ON mpp3033a (stringu1); ERROR: UNIQUE index must contain all columns in the table's distribution key DETAIL: Distribution key column "unique1" is not included in the constraint. CREATE UNIQUE INDEX mpp3033b_unique1 ON mpp3033b (unique1); -ERROR: insufficient columns in UNIQUE constraint definition +ERROR: unique constraint on partitioned table must include all partitioning columns DETAIL: UNIQUE constraint on table "mpp3033b_1_prt_aa" lacks column "unique2" which is part of the partition key. CREATE UNIQUE INDEX mpp3033b_unique2 ON mpp3033b (unique2); ERROR: UNIQUE index must contain all columns in the table's distribution key @@ -435,7 +435,7 @@ CREATE UNIQUE INDEX mpp3033a_stringu1 ON mpp3033a (stringu1); ERROR: UNIQUE index must contain all columns in the table's distribution key DETAIL: Distribution key column "unique1" is not included in the constraint. CREATE UNIQUE INDEX mpp3033b_unique1 ON mpp3033b (unique1); -ERROR: insufficient columns in UNIQUE constraint definition +ERROR: unique constraint on partitioned table must include all partitioning columns DETAIL: UNIQUE constraint on table "mpp3033b_1_prt_1" lacks column "unique2" which is part of the partition key. CREATE UNIQUE INDEX mpp3033b_unique2 ON mpp3033b (unique2); ERROR: UNIQUE index must contain all columns in the table's distribution key diff --git a/src/test/regress/expected/partition_optimizer.out b/src/test/regress/expected/partition_optimizer.out index a4cdcfa893af..72ec1285899e 100755 --- a/src/test/regress/expected/partition_optimizer.out +++ b/src/test/regress/expected/partition_optimizer.out @@ -4531,8 +4531,8 @@ create table tc default partition d, start (0) inclusive end(100) inclusive every (50) ); +ERROR: unique constraint on partitioned table must include all partitioning columns DETAIL: PRIMARY KEY constraint on table "tc" lacks column "b" which is part of the partition key. -ERROR: insufficient columns in PRIMARY KEY constraint definition create table cc (a int primary key, b int, c int) distributed by (a) @@ -4541,8 +4541,8 @@ create table cc default partition d, start (0) inclusive end(100) inclusive every (50) ); +ERROR: unique constraint on partitioned table must include all partitioning columns DETAIL: PRIMARY KEY constraint on table "cc" lacks column "b" which is part of the partition key. -ERROR: insufficient columns in PRIMARY KEY constraint definition create table at (a int, b int, c int) distributed by (a) @@ -4553,8 +4553,8 @@ create table at ); alter table at add primary key (a); +ERROR: unique constraint on partitioned table must include all partitioning columns DETAIL: PRIMARY KEY constraint on table "at" lacks column "b" which is part of the partition key. -ERROR: insufficient columns in PRIMARY KEY constraint definition -- MPP-14471 end -- MPP-17606 (using table "at" from above) alter table at @@ -4620,6 +4620,8 @@ create table plst2 partition p3 values ( CAST('(1,2)' as plst2_partkey) ) ); ERROR: partition "plst2_1_prt_p3" would overlap partition "plst2_1_prt_p1" +LINE 11: partition p3 values ( CAST('(1,2)' as plst2_partkey)... + ^ -- postive; make sure inner part duplicates are accepted and quietly removed. drop table if exists plst2; NOTICE: table "plst2" does not exist, skipping @@ -4656,6 +4658,8 @@ Distributed by: (d) -- negative; make sure conflicting alters fail. alter table plst2 add partition p6 values (CAST('(7,8)' as plst2_partkey), CAST('(2,1)' as plst2_partkey)); ERROR: partition "plst2_1_prt_p6" would overlap partition "plst2_1_prt_p5" +LINE 1: alter table plst2 add partition p6 values (CAST('(7,8)' as p... + ^ drop table if exists plst2; -- MPP-17814 end -- MPP-18441 From 35c6444a02426590212fd6da8b52c982f7b329db Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Mon, 15 Jun 2026 19:48:56 +0700 Subject: [PATCH 577/589] Fix distributed clog fsync (#2695) Commit dee663f7843902535a15ae366cede8b4089f1144 refactored clog fsyncing logic. Commit c97c117e02a4080ccde948b388ba6aa7eec452bd uses SYNC_HANDLER_CLOG for distributed clog syncing, which is not valid since distributed clog have their own directory. Fix it by adding a separate handler for distributed clog. --- src/backend/access/transam/distributedlog.c | 11 ++++++++++- src/backend/storage/sync/sync.c | 5 +++++ src/include/access/distributedlog.h | 3 +++ src/include/storage/sync.h | 1 + 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/backend/access/transam/distributedlog.c b/src/backend/access/transam/distributedlog.c index a330fe4681b1..5ab6ebd83441 100644 --- a/src/backend/access/transam/distributedlog.c +++ b/src/backend/access/transam/distributedlog.c @@ -661,7 +661,7 @@ DistributedLog_ShmemInit(void) DistributedLogCtl->PagePrecedes = DistributedLog_PagePrecedes; SimpleLruInit(DistributedLogCtl, "DistributedLogCtl", DistributedLog_ShmemBuffers(), 0, DistributedLogControlLock, "pg_distributedlog", - LWTRANCHE_DISTRIBUTEDLOG_BUFFERS, SYNC_HANDLER_CLOG); + LWTRANCHE_DISTRIBUTEDLOG_BUFFERS, SYNC_HANDLER_DISTRIBUTED_CLOG); /* Create or attach to the shared structure */ DistributedLogShared = @@ -1075,3 +1075,12 @@ DistributedLog_redo(XLogReaderState *record) else elog(PANIC, "DistributedLog_redo: unknown op code %u", info); } + +/* + * Entrypoint for sync.c to sync distributed clog files. + */ +int +DistributedLog_syncfiletag(const FileTag *ftag, char *path) +{ + return SlruSyncFileTag(DistributedLogCtl, ftag, path); +} diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c index 980ebbf8840f..208c660d67cf 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c @@ -21,6 +21,7 @@ #include "access/commit_ts.h" #include "access/clog.h" #include "access/multixact.h" +#include "access/distributedlog.h" #include "access/xlog.h" #include "access/xlogutils.h" #include "commands/tablespace.h" @@ -111,6 +112,10 @@ static const SyncOps syncsw[] = { .sync_unlinkfiletag = mdunlinkfiletag, .sync_filetagmatches = mdfiletagmatches }, + /* pg_distributedlog */ + [SYNC_HANDLER_DISTRIBUTED_CLOG] = { + .sync_syncfiletag = DistributedLog_syncfiletag, + }, /* pg_xact */ [SYNC_HANDLER_CLOG] = { .sync_syncfiletag = clogsyncfiletag diff --git a/src/include/access/distributedlog.h b/src/include/access/distributedlog.h index 7c2a7ff9f8a3..68f0225dd509 100644 --- a/src/include/access/distributedlog.h +++ b/src/include/access/distributedlog.h @@ -30,6 +30,7 @@ #define DISTRIBUTEDLOG_H #include "access/xlog.h" +#include "storage/sync.h" /* * The full binary representation of the distributed transaction id. @@ -79,4 +80,6 @@ extern void DistributedLog_GetDistributedXid( TransactionId localXid, DistributedTransactionId *distribXid); +extern int DistributedLog_syncfiletag(const FileTag *ftag, char *path); + #endif /* DISTRIBUTEDLOG_H */ diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h index 498f453310fe..7a904c5c4ef4 100644 --- a/src/include/storage/sync.h +++ b/src/include/storage/sync.h @@ -36,6 +36,7 @@ typedef enum SyncRequestHandler { SYNC_HANDLER_MD = 0, /* md smgr */ SYNC_HANDLER_AO = 1, + SYNC_HANDLER_DISTRIBUTED_CLOG, /* GGDB */ SYNC_HANDLER_CLOG, SYNC_HANDLER_COMMIT_TS, SYNC_HANDLER_MULTIXACT_OFFSET, From 7ecfd96d54902d92eac100da3d844a5181584e0f Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Mon, 15 Jun 2026 17:19:20 +0300 Subject: [PATCH 578/589] Resolve conflicts in doc/src/sgml/ref/select.sgml and select_into.sgml (#2692) Commit c5f42daa6077a4c309c5280a47d0e114c12dc572 fixed documentation for HAVING clause, however GPDB has different WINDOW clause for some reason: it is missing completely in select_into.sgml and it is missing "[, ...]" in select.sgml. The actual gram.y indicates WINDOW clause is present both in SELECT and SELECT INTO, and that WINDOW specifications can be repeated, so resolve in favor of upstream. --- doc/src/sgml/ref/select.sgml | 5 ----- doc/src/sgml/ref/select_into.sgml | 4 ---- 2 files changed, 9 deletions(-) diff --git a/doc/src/sgml/ref/select.sgml b/doc/src/sgml/ref/select.sgml index ae1f7ef82b93..672d9698e480 100644 --- a/doc/src/sgml/ref/select.sgml +++ b/doc/src/sgml/ref/select.sgml @@ -38,13 +38,8 @@ SELECT [ ALL | DISTINCT [ ON ( expressionfrom_item [, ...] ] [ WHERE condition ] [ GROUP BY grouping_element [, ...] ] -<<<<<<< HEAD - [ HAVING condition [, ...] ] - [ WINDOW window_name AS (window_specification) ] -======= [ HAVING condition ] [ WINDOW window_name AS ( window_definition ) [, ...] ] ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d [ { UNION | INTERSECT | EXCEPT } [ ALL | DISTINCT ] select ] [ ORDER BY expression [ ASC | DESC | USING operator ] [ NULLS { FIRST | LAST } ] [, ...] ] [ LIMIT { count | ALL } ] diff --git a/doc/src/sgml/ref/select_into.sgml b/doc/src/sgml/ref/select_into.sgml index 822d1a76408c..7b327d9eeef3 100644 --- a/doc/src/sgml/ref/select_into.sgml +++ b/doc/src/sgml/ref/select_into.sgml @@ -28,12 +28,8 @@ SELECT [ ALL | DISTINCT [ ON ( expressionfrom_item [, ...] ] [ WHERE condition ] [ GROUP BY expression [, ...] ] -<<<<<<< HEAD - [ HAVING condition [, ...] ] -======= [ HAVING condition ] [ WINDOW window_name AS ( window_definition ) [, ...] ] ->>>>>>> f81e97d0475cd4bc597adc23b665bd84fbf79a0d [ { UNION | INTERSECT | EXCEPT } [ ALL | DISTINCT ] select ] [ ORDER BY expression [ ASC | DESC | USING operator ] [ NULLS { FIRST | LAST } ] [, ...] ] [ LIMIT { count | ALL } ] From d580a89defdb52bf98bce0148aba22cc6e8fa6b3 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Mon, 15 Jun 2026 17:48:59 +0300 Subject: [PATCH 579/589] Fix test src/test/regress/expected/gporca.out (#2691) Commit f0f13a3a08b2757997410f3a1c38bdc22973c525 changed the output rows estimate for UPDATE without RETURNING to 0. Most tests weren't affected, since single-line EXPLAIN ignores cost diffs anyway. This test seems to be the only multi-line EXPLAIN in the test suite where these diffs matter. --- src/test/regress/expected/gporca.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/regress/expected/gporca.out b/src/test/regress/expected/gporca.out index a69ce1614503..d722f1964c70 100644 --- a/src/test/regress/expected/gporca.out +++ b/src/test/regress/expected/gporca.out @@ -14373,7 +14373,7 @@ from (select (min(i) over (order by j)) as i, j from window_agg_test) tt where t.j = tt.j; QUERY PLAN -------------------------------------------------------------------------------------------------------------------------- - Update on window_agg_test t (cost=3699.81..185385.16 rows=2471070 width=50) + Update on window_agg_test t (cost=3699.81..185385.16 rows=0 width=0) -> Explicit Redistribute Motion 3:3 (slice1; segments: 3) (cost=3699.81..185385.16 rows=2471070 width=50) -> Hash Join (cost=3699.81..135963.76 rows=2471070 width=50) Hash Cond: (tt.j = t.j) From 4f6acd6168c278a5c1ee9fe8274a10c4a1914eac Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Thu, 18 Jun 2026 15:11:26 +0700 Subject: [PATCH 580/589] Fix synchronous replication on the coordinator (#2699) Commit be9788e9989a0744ba60ab100153340fd123b786 added a fast path for SyncRepWaitForLSN, but it relies on the synchronous_standby_names GUC, which is always empty on the coordinator. Fix it by disabling the fast path on the coordinator. --- src/backend/replication/syncrep.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/backend/replication/syncrep.c b/src/backend/replication/syncrep.c index 26fde93d3457..b0a0007f2279 100644 --- a/src/backend/replication/syncrep.c +++ b/src/backend/replication/syncrep.c @@ -173,9 +173,13 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit) * described in SyncRepUpdateSyncStandbysDefined(). On the other * hand, if it's false, the lock is not necessary because we don't touch * the queue. + * + * GGDB: the coordinator should be able to commit even if standby is not + * available. Therefore, at the coordinator, we make a separate check below + * about whether synchronous replication is currently available or not. */ - if (!SyncRepRequested() || - !((volatile WalSndCtlData *) WalSndCtl)->sync_standbys_defined) + if (!IS_QUERY_DISPATCHER() && (!SyncRepRequested() || + !((volatile WalSndCtlData *) WalSndCtl)->sync_standbys_defined)) return; /* Cap the level for anything other than commit to remote flush only. */ From e5518b46c8f9b6c2d86b84a2b95e45aa748d957b Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Thu, 18 Jun 2026 13:57:44 +0300 Subject: [PATCH 581/589] Fix test src/test/recovery/t/021_row_visibility.pl (#2700) Commit 7b28913bcab8d1bf3dbf59c9d8fb4b51cef57664 added new TAP test, however GPDB does not support PREPARE TRANSACTION, so disable that part of the test. --- src/test/recovery/t/021_row_visibility.pl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/recovery/t/021_row_visibility.pl b/src/test/recovery/t/021_row_visibility.pl index 8a466e56e0fd..6d5faa059cf2 100644 --- a/src/test/recovery/t/021_row_visibility.pl +++ b/src/test/recovery/t/021_row_visibility.pl @@ -114,6 +114,10 @@ # # 5. Check that changes in prepared xacts is invisible # + +# GPDB: PREPARE TRANSACTION is not supported on GPDB, skip +SKIP: { + skip "PREPARE TRANSACTION not implemented on GPDB", 4; ok(send_query_and_wait(\%psql_primary, q[ DELETE from test_visibility; -- delete old data, so we start with clean slate BEGIN; @@ -150,6 +154,7 @@ q[SELECT * FROM test_visibility ORDER BY data;], qr/will_commit.*\n\(1 row\)$/m), 'finished prepared visible'); +} # end SKIP $node_primary->stop; $node_standby->stop; From 0839c32025eda0cae2a154c604729892d88d1476 Mon Sep 17 00:00:00 2001 From: "Kevin.wyh" Date: Fri, 2 Jun 2023 16:30:45 +0800 Subject: [PATCH 582/589] Fix "cannot execute squelched plan node of type" error. This error is caused by the execution logic of Hash Join Node, when the outer node does not fetch a tuple, it calls the ExecSquelchNode method let the lower nodes not to produce tuples anymore, leaving the inner node "squalched" but "un-executed" state. If the hash join needs to be rescanned, then the inner nodes has to be rescanned too if Hash Node's lefttree does not have chgParam, and finally a node does not have chgParam but "squelched" will cause ERROR. This commit resolves the issue by handling of the rescan method in HashJoin Node properly. Authored-by: wuyuhao28 (cherry picked from commit 73517b3121ba9aead136b25ad9a2d0830f673557) --- src/backend/executor/nodeHashjoin.c | 10 ++++ src/test/regress/expected/bfv_joins.out | 57 +++++++++++++++++++ .../regress/expected/bfv_joins_optimizer.out | 57 +++++++++++++++++++ src/test/regress/sql/bfv_joins.sql | 37 ++++++++++++ 4 files changed, 161 insertions(+) diff --git a/src/backend/executor/nodeHashjoin.c b/src/backend/executor/nodeHashjoin.c index a962f3574235..4d1402744044 100644 --- a/src/backend/executor/nodeHashjoin.c +++ b/src/backend/executor/nodeHashjoin.c @@ -1675,6 +1675,16 @@ ExecReScanHashJoin(HashJoinState *node) ExecReScan(node->js.ps.righttree); } } + else + { + /* + * GPDB: HashTable not built with righttree may be squelched, + * HashJoin need rescan righttree to reset its squelch flag. + */ + if (node->js.ps.righttree->chgParam == NULL && + node->js.ps.righttree->squelched) + ExecReScan(node->js.ps.righttree); + } /* Always reset intra-tuple state */ node->hj_CurHashValue = 0; diff --git a/src/test/regress/expected/bfv_joins.out b/src/test/regress/expected/bfv_joins.out index a83b45fe7f49..4791a1c0a866 100644 --- a/src/test/regress/expected/bfv_joins.out +++ b/src/test/regress/expected/bfv_joins.out @@ -3529,6 +3529,63 @@ explain select * from o1 left join o2 on a1 = a2 left join o3 on a2 is not disti Optimizer: Postgres query optimizer (18 rows) +-- +-- Test case for Hash Join rescan after squelched without hashtable built +-- See https://github.com/greenplum-db/gpdb/pull/15590 +-- +--- Lateral Join +set from_collapse_limit = 1; +set join_collapse_limit = 1; +select 1 from pg_namespace join lateral + (select * from aclexplode(nspacl) x join pg_authid on x.grantee = pg_authid.oid where rolname = current_user) z on true limit 1; + ?column? +---------- + 1 +(1 row) + +reset from_collapse_limit; +reset join_collapse_limit; +--- NestLoop index join +create table l_table (a int, b int) distributed replicated; +create index l_table_idx on l_table(a); +create table r_table1 (ra1 int, rb1 int) distributed replicated; +create table r_table2 (ra2 int, rb2 int) distributed replicated; +insert into l_table select i % 10 , i from generate_series(1, 10000) i; +insert into r_table1 select i, i from generate_series(1, 1000) i; +insert into r_table2 values(11, 11), (1, 1) ; +analyze l_table; +analyze r_table1; +analyze r_table2; +set optimizer to off; +set enable_nestloop to on; +set enable_bitmapscan to off; +explain select * from r_table2 where ra2 in ( select a from l_table join r_table1 on b = rb1); + QUERY PLAN +------------------------------------------------------------------------------------------------- + Gather Motion 1:1 (slice1; segments: 1) (cost=64.56..64.56 rows=10 width=8) + -> Nested Loop Semi Join (cost=24.66..64.56 rows=10 width=8) + -> Seq Scan on r_table2 (cost=0.00..1.02 rows=2 width=8) + -> Hash Join (cost=24.66..62.75 rows=100 width=4) + Hash Cond: (l_table.b = r_table1.rb1) + -> Index Scan using l_table_idx on l_table (cost=0.16..25.62 rows=1000 width=8) + Index Cond: (a = r_table2.ra2) + -> Hash (cost=12.00..12.00 rows=1000 width=4) + -> Seq Scan on r_table1 (cost=0.00..12.00 rows=1000 width=4) + Optimizer: Postgres query optimizer +(10 rows) + +select * from r_table2 where ra2 in ( select a from l_table join r_table1 on b = rb1); + ra2 | rb2 +-----+----- + 1 | 1 +(1 row) + +reset optimizer; +reset enable_nestloop; +reset enable_bitmapscan; +drop table l_table; +drop table r_table1; +drop table r_table2; -- Clean up. None of the objects we create are very interesting to keep around. reset search_path; set client_min_messages='warning'; diff --git a/src/test/regress/expected/bfv_joins_optimizer.out b/src/test/regress/expected/bfv_joins_optimizer.out index ff2db765d455..be9db88f5350 100644 --- a/src/test/regress/expected/bfv_joins_optimizer.out +++ b/src/test/regress/expected/bfv_joins_optimizer.out @@ -3522,6 +3522,63 @@ explain select * from o1 left join o2 on a1 = a2 left join o3 on a2 is not disti Optimizer: Pivotal Optimizer (GPORCA) (13 rows) +-- +-- Test case for Hash Join rescan after squelched without hashtable built +-- See https://github.com/greenplum-db/gpdb/pull/15590 +-- +--- Lateral Join +set from_collapse_limit = 1; +set join_collapse_limit = 1; +select 1 from pg_namespace join lateral + (select * from aclexplode(nspacl) x join pg_authid on x.grantee = pg_authid.oid where rolname = current_user) z on true limit 1; + ?column? +---------- + 1 +(1 row) + +reset from_collapse_limit; +reset join_collapse_limit; +--- NestLoop index join +create table l_table (a int, b int) distributed replicated; +create index l_table_idx on l_table(a); +create table r_table1 (ra1 int, rb1 int) distributed replicated; +create table r_table2 (ra2 int, rb2 int) distributed replicated; +insert into l_table select i % 10 , i from generate_series(1, 10000) i; +insert into r_table1 select i, i from generate_series(1, 1000) i; +insert into r_table2 values(11, 11), (1, 1) ; +analyze l_table; +analyze r_table1; +analyze r_table2; +set optimizer to off; +set enable_nestloop to on; +set enable_bitmapscan to off; +explain select * from r_table2 where ra2 in ( select a from l_table join r_table1 on b = rb1); + QUERY PLAN +------------------------------------------------------------------------------------------------- + Gather Motion 1:1 (slice1; segments: 1) (cost=64.56..64.56 rows=10 width=8) + -> Nested Loop Semi Join (cost=24.66..64.56 rows=10 width=8) + -> Seq Scan on r_table2 (cost=0.00..1.02 rows=2 width=8) + -> Hash Join (cost=24.66..62.75 rows=100 width=4) + Hash Cond: (l_table.b = r_table1.rb1) + -> Index Scan using l_table_idx on l_table (cost=0.16..25.62 rows=1000 width=8) + Index Cond: (a = r_table2.ra2) + -> Hash (cost=12.00..12.00 rows=1000 width=4) + -> Seq Scan on r_table1 (cost=0.00..12.00 rows=1000 width=4) + Optimizer: Postgres query optimizer +(10 rows) + +select * from r_table2 where ra2 in ( select a from l_table join r_table1 on b = rb1); + ra2 | rb2 +-----+----- + 1 | 1 +(1 row) + +reset optimizer; +reset enable_nestloop; +reset enable_bitmapscan; +drop table l_table; +drop table r_table1; +drop table r_table2; -- Clean up. None of the objects we create are very interesting to keep around. reset search_path; set client_min_messages='warning'; diff --git a/src/test/regress/sql/bfv_joins.sql b/src/test/regress/sql/bfv_joins.sql index c2a585b4a99b..971c395cb808 100644 --- a/src/test/regress/sql/bfv_joins.sql +++ b/src/test/regress/sql/bfv_joins.sql @@ -386,6 +386,43 @@ explain select * from o1 left join o2 on a1 = a2 left join o3 on a2 is not disti explain select * from o1 left join o2 on a1 = a2 left join o3 on a2 is not distinct from a3 and b2 = b3; +-- +-- Test case for Hash Join rescan after squelched without hashtable built +-- See https://github.com/greenplum-db/gpdb/pull/15590 +-- +--- Lateral Join +set from_collapse_limit = 1; +set join_collapse_limit = 1; +select 1 from pg_namespace join lateral + (select * from aclexplode(nspacl) x join pg_authid on x.grantee = pg_authid.oid where rolname = current_user) z on true limit 1; +reset from_collapse_limit; +reset join_collapse_limit; + +--- NestLoop index join +create table l_table (a int, b int) distributed replicated; +create index l_table_idx on l_table(a); +create table r_table1 (ra1 int, rb1 int) distributed replicated; +create table r_table2 (ra2 int, rb2 int) distributed replicated; +insert into l_table select i % 10 , i from generate_series(1, 10000) i; +insert into r_table1 select i, i from generate_series(1, 1000) i; +insert into r_table2 values(11, 11), (1, 1) ; +analyze l_table; +analyze r_table1; +analyze r_table2; + +set optimizer to off; +set enable_nestloop to on; +set enable_bitmapscan to off; +explain select * from r_table2 where ra2 in ( select a from l_table join r_table1 on b = rb1); +select * from r_table2 where ra2 in ( select a from l_table join r_table1 on b = rb1); + +reset optimizer; +reset enable_nestloop; +reset enable_bitmapscan; +drop table l_table; +drop table r_table1; +drop table r_table2; + -- Clean up. None of the objects we create are very interesting to keep around. reset search_path; set client_min_messages='warning'; From 38ad13418398eebafd768c86c40ed85ad730932f Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Fri, 19 Jun 2026 13:32:03 +0700 Subject: [PATCH 583/589] Fix "gpcheckcat should report and repair extra entries with non-oid primary keys" scenario (#2701) This test uses a postfix operator that is no longer supported. Fix it by using another operator type. --- .../behave/mgmt_utils/steps/data/gpcheckcat/add_operator.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpMgmt/test/behave/mgmt_utils/steps/data/gpcheckcat/add_operator.sql b/gpMgmt/test/behave/mgmt_utils/steps/data/gpcheckcat/add_operator.sql index d63ff0295dac..08aa13707eb5 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/data/gpcheckcat/add_operator.sql +++ b/gpMgmt/test/behave/mgmt_utils/steps/data/gpcheckcat/add_operator.sql @@ -2,4 +2,4 @@ CREATE FUNCTION my_pk_schema.is_zero(int) RETURNS TABLE(f1 boolean) AS $$ select $1 = 0 $$ LANGUAGE SQL; -CREATE OPERATOR my_pk_schema.!# (PROCEDURE = my_pk_schema.is_zero,LEFTARG = integer); +CREATE OPERATOR my_pk_schema.!# (PROCEDURE = my_pk_schema.is_zero,RIGHTARG = integer); From 782212953749f3e709bc3db6f285a9b938abbc9f Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Fri, 19 Jun 2026 11:15:59 +0300 Subject: [PATCH 584/589] Fix test src/bin/pg_dump/t/002_pg_dump.pl (#2696) Commit e03e1e17168234a28510b911c2ed8008b5c0b3b3 incorrectly resolved conflicts related to dumping indexes, partially reverting changes in getIndexes() function from commit 257836a75585934cc05ed7a80bccf8190d41e056 because of refactoring in commit 30ab901a5d040aae184cabb68ea1928c5c154d01. Adjust these changes to the new code structure. --- src/bin/pg_dump/pg_dump.c | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 2e80722d1a02..4e27f65382d9 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -7269,7 +7269,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) i_tablespace, i_indreloptions, i_indstatcols, - i_indstatvals; + i_indstatvals, + i_inddependcollnames, + i_inddependcollversions; /* * We want to perform just one query against pg_index. However, we @@ -7335,14 +7337,37 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) "(SELECT pg_catalog.array_agg(attstattarget ORDER BY attnum) " " FROM pg_catalog.pg_attribute " " WHERE attrelid = i.indexrelid AND " - " attstattarget >= 0) AS indstatvals "); + " attstattarget >= 0) AS indstatvals, "); else appendPQExpBuffer(query, "0 AS parentidx, " "i.indnatts AS indnkeyatts, " "i.indnatts AS indnatts, " "'' AS indstatcols, " - "'' AS indstatvals "); + "'' AS indstatvals, "); + + if (fout->remoteVersion >= 140000) + appendPQExpBuffer(query, + "(SELECT pg_catalog.array_agg(quote_ident(ns.nspname) || '.' || quote_ident(c.collname) ORDER BY refobjid) " + " FROM pg_catalog.pg_depend d " + " JOIN pg_catalog.pg_collation c ON (c.oid = d.refobjid) " + " JOIN pg_catalog.pg_namespace ns ON (c.collnamespace = ns.oid) " + " WHERE d.classid = 'pg_catalog.pg_class'::regclass AND " + " d.objid = i.indexrelid AND " + " d.objsubid = 0 AND " + " d.refclassid = 'pg_catalog.pg_collation'::regclass AND " + " d.refobjversion IS NOT NULL) AS inddependcollnames, " + "(SELECT pg_catalog.array_agg(quote_literal(refobjversion) ORDER BY refobjid) " + " FROM pg_catalog.pg_depend " + " WHERE classid = 'pg_catalog.pg_class'::regclass AND " + " objid = i.indexrelid AND " + " objsubid = 0 AND " + " refclassid = 'pg_catalog.pg_collation'::regclass AND " + " refobjversion IS NOT NULL) AS inddependcollversions "); + else + appendPQExpBuffer(query, + "'{}' AS inddependcollnames, " + "'{}' AS inddependcollversions "); appendPQExpBuffer(query, "FROM unnest('%s'::pg_catalog.oid[]) AS src(tbloid)\n" @@ -7427,6 +7452,8 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) i_indreloptions = PQfnumber(res, "indreloptions"); i_indstatcols = PQfnumber(res, "indstatcols"); i_indstatvals = PQfnumber(res, "indstatvals"); + i_inddependcollnames = PQfnumber(res, "inddependcollnames"); + i_inddependcollversions = PQfnumber(res, "inddependcollversions"); indxinfo = (IndxInfo *) pg_malloc(ntups * sizeof(IndxInfo)); @@ -7487,6 +7514,8 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) indxinfo[j].indreloptions = pg_strdup(PQgetvalue(res, j, i_indreloptions)); indxinfo[j].indstatcols = pg_strdup(PQgetvalue(res, j, i_indstatcols)); indxinfo[j].indstatvals = pg_strdup(PQgetvalue(res, j, i_indstatvals)); + indxinfo[j].inddependcollnames = pg_strdup(PQgetvalue(res, j, i_inddependcollnames)); + indxinfo[j].inddependcollversions = pg_strdup(PQgetvalue(res, j, i_inddependcollversions)); indxinfo[j].indkeys = (Oid *) pg_malloc(indxinfo[j].indnattrs * sizeof(Oid)); parseOidArray(PQgetvalue(res, j, i_indkey), indxinfo[j].indkeys, indxinfo[j].indnattrs); From 288f04bd92a3d965605e83dce9a982f6832bcc49 Mon Sep 17 00:00:00 2001 From: Georgy Shelkovy Date: Fri, 19 Jun 2026 13:18:06 +0500 Subject: [PATCH 585/589] Fix compilation of src/backend/parser/parse_utilcmd.c (#2631) Commit 5028981 in src/backend/parser/parse_utilcmd.c removed the inh_indexes field from the CreateStmtContext structure, although it was used in the transformDistributedBy function in the GPDB. Restore it for the GPDB and add code to populate it in the transformTableLikeClause function from 7X. --- src/backend/parser/parse_utilcmd.c | 50 ++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index 6a43c017eda8..efa82b2a69bb 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -97,6 +97,10 @@ typedef struct List *ckconstraints; /* CHECK constraints */ List *fkconstraints; /* FOREIGN KEY constraints */ List *ixconstraints; /* index-creating constraints */ + List *inh_indexes; /* cloned indexes from INCLUDING INDEXES + * GPDB: used by transformDistributedBy + * Note: Attribute numbers in expressions + * might not be correct, only use names */ List *attr_encodings; /* List of ColumnReferenceStorageDirectives */ List *extstats; /* cloned extended statistics */ List *blist; /* "before list" of things to do before @@ -291,6 +295,7 @@ transformCreateStmt(CreateStmt *stmt, const char *queryString) cxt.ckconstraints = NIL; cxt.fkconstraints = NIL; cxt.ixconstraints = NIL; + cxt.inh_indexes = NIL; /* GPDB: used by transformDistributedBy */ cxt.extstats = NIL; cxt.attr_encodings = stmt->attr_encodings; cxt.blist = NIL; @@ -1092,6 +1097,13 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla tupleDesc = RelationGetDescr(relation); + /* + * Initialize column number map for map_variable_attnos(). We need this + * since dropped columns in the source table aren't copied, so the new + * table can have different column numbers. + */ + AttrMap *attmap = make_attrmap(tupleDesc->natts); + /* * Insert the copied attributes into the cxt for the new table definition. * We must do this now so that they appear in the table in the relative @@ -1137,6 +1149,7 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla * Add to column list */ cxt->columns = lappend(cxt->columns, def); + attmap->attnums[parent_attno - 1] = list_length(cxt->columns); /* * Although we don't transfer the column's default/generation @@ -1297,6 +1310,41 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla list_free(parent_extstats); } + /* + * Copy indexes for Greengage choosing distributed-by keys. + * PostgreSQL processes index statements after here in expandTableLikeClause(), + * but we need indexes in transformDistributedBy() which is before expandTableLikeClause(), + * So we both retain the index statements processing here and expandTableLikeClause. + * the process here is just used by transformDistributedBy(). + */ + if ((table_like_clause->options & CREATE_TABLE_LIKE_INDEXES) && + relation->rd_rel->relhasindex) + { + List *parent_indexes; + ListCell *l; + + parent_indexes = RelationGetIndexList(relation); + + foreach(l, parent_indexes) + { + Oid parent_index_oid = lfirst_oid(l); + Relation parent_index; + IndexStmt *index_stmt; + + parent_index = index_open(parent_index_oid, AccessShareLock); + + /* Build CREATE INDEX statement to recreate the parent_index */ + index_stmt = generateClonedIndexStmt(stmt->relation, + parent_index, + attmap, + NULL); + + /* Save it in the inh_indexes list for the time being */ + cxt->inh_indexes = lappend(cxt->inh_indexes, index_stmt); + + index_close(parent_index, AccessShareLock); + } + } /* * Close the parent rel, but keep our AccessShareLock on it until xact * commit. That will prevent someone else from deleting or ALTERing the @@ -2176,6 +2224,7 @@ transformCreateExternalStmt(CreateExternalStmt *stmt, const char *queryString) cxt.ckconstraints = NIL; cxt.fkconstraints = NIL; cxt.ixconstraints = NIL; + cxt.inh_indexes = NIL; /* GPDB: used by transformDistributedBy */ cxt.attr_encodings = NIL; cxt.pkey = NULL; cxt.rel = NULL; @@ -4306,6 +4355,7 @@ transformAlterTableStmt(Oid relid, AlterTableStmt *stmt, cxt.ckconstraints = NIL; cxt.fkconstraints = NIL; cxt.ixconstraints = NIL; + cxt.inh_indexes = NIL; /* GPDB: used by transformDistributedBy */ cxt.attr_encodings = NIL; cxt.extstats = NIL; cxt.blist = NIL; From 4d2edf864db8452188a02302ce767420896c9792 Mon Sep 17 00:00:00 2001 From: Alexander Kondakov Date: Fri, 19 Jun 2026 11:27:51 +0300 Subject: [PATCH 586/589] Fix sslinfo test output (#2659) Commit 5d1833f statrted to use be_tls_* API for SSL information in sslinfo in extensions functions. It trims expected string produced by X509_get_subject_name at NAMEDATALEN (64) length. Vanila postgres does not have tests for sslinfo at all, they were introduced in GPDB. Since the behaviour is documented, we adjust GPDB tests. --- contrib/sslinfo/expected/sslinfo.out | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/contrib/sslinfo/expected/sslinfo.out b/contrib/sslinfo/expected/sslinfo.out index ef0dc43a3c74..1cb0202a0fd9 100644 --- a/contrib/sslinfo/expected/sslinfo.out +++ b/contrib/sslinfo/expected/sslinfo.out @@ -38,15 +38,15 @@ SELECT ssl_client_serial(); (1 row) SELECT ssl_client_dn(); - ssl_client_dn ------------------------------------------------------------------------------------- - /CN=client.example.com/C=CN/ST=Qingdao/L=ClientLocality/O=SSLINFO-Client/OU=Client + ssl_client_dn +----------------------------------------------------------------- + /CN=client.example.com/C=CN/ST=Qingdao/L=ClientLocality/O=SSLIN (1 row) SELECT ssl_issuer_dn(); - ssl_issuer_dn ---------------------------------------------------------------------------- - /CN=root.example.com/C=CN/ST=Beijing/L=RootLocality/O=SSLINFO-dev/OU=Test + ssl_issuer_dn +----------------------------------------------------------------- + /CN=root.example.com/C=CN/ST=Beijing/L=RootLocality/O=SSLINFO-d (1 row) SELECT ssl_client_dn_field('CN') AS client_dn_CN; From 1e1a7a388b9d84f95302af6d80d1529d9a559f1f Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Fri, 19 Jun 2026 15:51:38 +0700 Subject: [PATCH 587/589] Fix table_functions test output (#2688) Commit ce90f075f0d831ca4085ba73891b7da2a2f7047e changed error message. Change ggdb specific tests accordingly. --- src/test/regress/output/table_functions.source | 4 ++-- src/test/regress/output/table_functions_optimizer.source | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/regress/output/table_functions.source b/src/test/regress/output/table_functions.source index d9139a14b3ef..22884921f683 100644 --- a/src/test/regress/output/table_functions.source +++ b/src/test/regress/output/table_functions.source @@ -2627,7 +2627,7 @@ FROM sessionize_static( ), '1 minute' ) AS sessionize(id integer, "time" timestamp, sessionnum integer) ORDER BY id, time; -- FAIL, double qualified -ERROR: a column definition list is only allowed for functions returning "record" +ERROR: a column definition list is redundant for a function with OUT parameters LINE 10: '1 minute' ) AS sessionize(id integer, "time" timestamp,... ^ -- Describe with qualification fails @@ -2642,7 +2642,7 @@ FROM sessionize_dynamic( ), '1 minute' ) AS sessionize(id integer, "time" timestamp, sessionnum integer) ORDER BY id, time; -- FAIL, double qualified -ERROR: a column definition list is only allowed for functions returning "record" +ERROR: a column definition list is redundant for a function with OUT parameters LINE 10: '1 minute' ) AS sessionize(id integer, "time" timestamp,... ^ -- Otherwise results should match diff --git a/src/test/regress/output/table_functions_optimizer.source b/src/test/regress/output/table_functions_optimizer.source index d6101512352f..33cd8a461b3a 100644 --- a/src/test/regress/output/table_functions_optimizer.source +++ b/src/test/regress/output/table_functions_optimizer.source @@ -2628,7 +2628,7 @@ FROM sessionize_static( ), '1 minute' ) AS sessionize(id integer, "time" timestamp, sessionnum integer) ORDER BY id, time; -- FAIL, double qualified -ERROR: a column definition list is only allowed for functions returning "record" +ERROR: a column definition list is redundant for a function with OUT parameters LINE 10: '1 minute' ) AS sessionize(id integer, "time" timestamp,... ^ -- Describe with qualification fails @@ -2643,7 +2643,7 @@ FROM sessionize_dynamic( ), '1 minute' ) AS sessionize(id integer, "time" timestamp, sessionnum integer) ORDER BY id, time; -- FAIL, double qualified -ERROR: a column definition list is only allowed for functions returning "record" +ERROR: a column definition list is redundant for a function with OUT parameters LINE 10: '1 minute' ) AS sessionize(id integer, "time" timestamp,... ^ -- Otherwise results should match From 1a96bdfbb3c216c1b50cdfe5fd6947138f59cc31 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Fri, 19 Jun 2026 16:17:51 +0700 Subject: [PATCH 588/589] Fix pg_upgrade collation version restore (#2704) Commit 257836a75585934cc05ed7a80bccf8190d41e056 added restoring of the collation versions, but in the ggdb modification of the catalog tables requires enabling allow_system_table_mods GUC. --- src/bin/pg_dump/pg_dump.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 4e27f65382d9..1e69a24ef282 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -20031,10 +20031,12 @@ appendIndexCollationVersion(PQExpBuffer buffer, IndxInfo *indxinfo, int enc, if (coll_unknown) { appendPQExpBuffer(buffer, - "\n-- For binary upgrade, clobber new index's collation versions\n"); + "\n-- For binary upgrade, clobber new index's collation versions\n" + "SET allow_system_table_mods = true;\n"); appendPQExpBuffer(buffer, "UPDATE pg_catalog.pg_depend SET refobjversion = 'unknown' WHERE objid = '%u'::pg_catalog.oid AND refclassid = 'pg_catalog.pg_collation'::regclass AND refobjversion IS NOT NULL;\n", indxinfo->dobj.catId.oid); + appendPQExpBuffer(buffer, "RESET allow_system_table_mods;\n"); } /* Restore the versions that were recorded by the old cluster (if any). */ @@ -20048,7 +20050,8 @@ appendIndexCollationVersion(PQExpBuffer buffer, IndxInfo *indxinfo, int enc, if (ninddependcollnames > 0) appendPQExpBufferStr(buffer, - "\n-- For binary upgrade, restore old index's collation versions\n"); + "\n-- For binary upgrade, restore old index's collation versions\n" + "SET allow_system_table_mods = true;\n"); for (int i = 0; i < ninddependcollnames; i++) { /* @@ -20063,6 +20066,9 @@ appendIndexCollationVersion(PQExpBuffer buffer, IndxInfo *indxinfo, int enc, appendPQExpBuffer(buffer, "::regcollation;\n"); } + if (ninddependcollnames > 0) + appendPQExpBufferStr(buffer, "RESET allow_system_table_mods;\n"); + if (inddependcollnamesarray) free(inddependcollnamesarray); if (inddependcollversionsarray) From 6e17270e86d27530598f1ee78b7015b4ab073004 Mon Sep 17 00:00:00 2001 From: Maxim Michkov Date: Fri, 19 Jun 2026 13:27:38 +0300 Subject: [PATCH 589/589] Fix test src/test/isolation2/input/resgroup/resgroup_cpuset.source (#2705) Commit 76f412ab310554acb970a0b73c8d1f37f35548c6 removed factorial operators, leaving only the factorial() function. Replace in GPDB-specific tests. --- .../isolation2/input/resgroup/resgroup_cpuset.source | 10 +++++----- .../isolation2/output/resgroup/resgroup_cpuset.source | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/isolation2/input/resgroup/resgroup_cpuset.source b/src/test/isolation2/input/resgroup/resgroup_cpuset.source index 64dde1ebc27c..3c4894325440 100644 --- a/src/test/isolation2/input/resgroup/resgroup_cpuset.source +++ b/src/test/isolation2/input/resgroup/resgroup_cpuset.source @@ -170,11 +170,11 @@ CREATE VIEW busy AS bigtable t3, bigtable t4, bigtable t5 - WHERE 0 = (t1.c1 % 2 + 10000)! - AND 0 = (t2.c1 % 2 + 10000)! - AND 0 = (t3.c1 % 2 + 10000)! - AND 0 = (t4.c1 % 2 + 10000)! - AND 0 = (t5.c1 % 2 + 10000)! + WHERE 0 = factorial(t1.c1 % 2 + 10000) + AND 0 = factorial(t2.c1 % 2 + 10000) + AND 0 = factorial(t3.c1 % 2 + 10000) + AND 0 = factorial(t4.c1 % 2 + 10000) + AND 0 = factorial(t5.c1 % 2 + 10000) ; CREATE VIEW cancel_all AS diff --git a/src/test/isolation2/output/resgroup/resgroup_cpuset.source b/src/test/isolation2/output/resgroup/resgroup_cpuset.source index 45798b865d88..c0e7b7383757 100644 --- a/src/test/isolation2/output/resgroup/resgroup_cpuset.source +++ b/src/test/isolation2/output/resgroup/resgroup_cpuset.source @@ -39,7 +39,7 @@ CREATE CREATE TABLE bigtable AS SELECT i AS c1, 'abc' AS c2 FROM generate_series(1,50000) i; CREATE 50000 -CREATE VIEW busy AS SELECT count(*) FROM bigtable t1, bigtable t2, bigtable t3, bigtable t4, bigtable t5 WHERE 0 = (t1.c1 % 2 + 10000)! AND 0 = (t2.c1 % 2 + 10000)! AND 0 = (t3.c1 % 2 + 10000)! AND 0 = (t4.c1 % 2 + 10000)! AND 0 = (t5.c1 % 2 + 10000)! ; +CREATE VIEW busy AS SELECT count(*) FROM bigtable t1, bigtable t2, bigtable t3, bigtable t4, bigtable t5 WHERE 0 = factorial(t1.c1 % 2 + 10000) AND 0 = factorial(t2.c1 % 2 + 10000) AND 0 = factorial(t3.c1 % 2 + 10000) AND 0 = factorial(t4.c1 % 2 + 10000) AND 0 = factorial(t5.c1 % 2 + 10000) ; CREATE CREATE VIEW cancel_all AS SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE query LIKE 'SELECT * FROM busy%';