diff --git a/src/backend/cdb/cdbgroupingpaths.c b/src/backend/cdb/cdbgroupingpaths.c index 9ffd084afb96..ca408ceece64 100644 --- a/src/backend/cdb/cdbgroupingpaths.c +++ b/src/backend/cdb/cdbgroupingpaths.c @@ -2302,6 +2302,12 @@ fetch_single_dqa_info(PlannerInfo *root, * is already sorted, we prefer to gather it to a single node to make * use of the pre-existing order, instead of redistributing and resorting * it. + * + * The presorted_keys parameter is the number of keys by which the input path + * is known to be sorted. + * If this parameter is greater than 0 (NO_INCREMENTAL_SORT), the input is + * considered to be partially sorted and incremental sort is applied. + * Otherwise, a full sort is used. */ Path * cdb_prepare_path_for_sorted_agg(PlannerInfo *root, @@ -2311,6 +2317,7 @@ cdb_prepare_path_for_sorted_agg(PlannerInfo *root, Path *subpath, PathTarget *target, List *group_pathkeys, + int presorted_keys, double limit_tuples, /* extra arguments */ List *groupClause, @@ -2344,11 +2351,23 @@ cdb_prepare_path_for_sorted_agg(PlannerInfo *root, { if (!is_sorted) { - subpath = (Path *) create_sort_path(root, - rel, - subpath, - group_pathkeys, - -1.0); + if (presorted_keys == NO_INCREMENTAL_SORT) + { + subpath = (Path *) create_sort_path(root, + rel, + subpath, + group_pathkeys, + -1.0); + } + else + { + subpath = (Path *) create_incremental_sort_path(root, + rel, + subpath, + group_pathkeys, + presorted_keys, + -1.0); + } } return subpath; } @@ -2380,14 +2399,42 @@ cdb_prepare_path_for_sorted_agg(PlannerInfo *root, * to merge the inputs. */ if (CdbPathLocus_IsPartitioned(locus)) - subpath = cdbpath_create_motion_path(root, subpath, NIL, - false, locus); + { + Path *motion_path = cdbpath_create_motion_path(root, subpath, subpath->pathkeys, + presorted_keys != NO_INCREMENTAL_SORT, locus); - subpath = (Path *) create_sort_path(root, - rel, - subpath, - group_pathkeys, - -1.0); + + /* + * We can not return NULL here, so make a possibly redundant path with + * no incremental sort + */ + if (motion_path == NULL) + { + presorted_keys = NO_INCREMENTAL_SORT; + motion_path = cdbpath_create_motion_path(root, subpath, NIL, + false, locus); + } + Assert(motion_path != NULL); + subpath = motion_path; + } + + if (presorted_keys == NO_INCREMENTAL_SORT) + { + subpath = (Path *) create_sort_path(root, + rel, + subpath, + group_pathkeys, + -1.0); + } + else + { + subpath = (Path *) create_incremental_sort_path(root, + rel, + subpath, + group_pathkeys, + presorted_keys, + -1.0); + } if (!CdbPathLocus_IsPartitioned(locus)) subpath = cdbpath_create_motion_path(root, subpath, diff --git a/src/backend/cdb/cdbllize.c b/src/backend/cdb/cdbllize.c index 579fee0f52aa..5ad0e7be6cc2 100644 --- a/src/backend/cdb/cdbllize.c +++ b/src/backend/cdb/cdbllize.c @@ -1485,6 +1485,7 @@ motion_sanity_walker(Node *node, sanity_result_t *result) case T_SetOp: case T_Limit: case T_Sort: + case T_IncrementalSort: case T_Material: case T_ForeignScan: if (plan_tree_walker(node, motion_sanity_walker, result, true)) diff --git a/src/backend/cdb/cdbplan.c b/src/backend/cdb/cdbplan.c index d38052183a6e..1c2cd6f60aee 100644 --- a/src/backend/cdb/cdbplan.c +++ b/src/backend/cdb/cdbplan.c @@ -621,6 +621,20 @@ plan_tree_mutator(Node *node, } break; + case T_IncrementalSort: + { + IncrementalSort *sort = (IncrementalSort *) node; + IncrementalSort *newsort; + + FLATCOPY(newsort, sort, IncrementalSort); + PLANMUTATE(newsort, sort); + COPYARRAY(&newsort->sort, &sort->sort, numCols, sortColIdx); + COPYARRAY(&newsort->sort, &sort->sort, numCols, sortOperators); + COPYARRAY(&newsort->sort, &sort->sort, numCols, nullsFirst); + return (Node *) newsort; + } + break; + case T_Agg: { Agg *agg = (Agg *) node; diff --git a/src/backend/cdb/cdbtargeteddispatch.c b/src/backend/cdb/cdbtargeteddispatch.c index 905c7afd59be..cd3243a149ba 100644 --- a/src/backend/cdb/cdbtargeteddispatch.c +++ b/src/backend/cdb/cdbtargeteddispatch.c @@ -521,6 +521,7 @@ DirectDispatchUpdateContentIdsFromPlan(PlannerInfo *root, Plan *plan) break; case T_Material: case T_Sort: + case T_IncrementalSort: case T_Agg: case T_TupleSplit: case T_Unique: diff --git a/src/backend/executor/execAmi.c b/src/backend/executor/execAmi.c index fc4391850fb3..1edb50fd0f8c 100644 --- a/src/backend/executor/execAmi.c +++ b/src/backend/executor/execAmi.c @@ -832,6 +832,10 @@ ExecSquelchNode(PlanState *node) ExecSquelchSort((SortState *) node); break; + case T_IncrementalSortState: + ExecSquelchIncrementalSort((IncrementalSortState *) node); + break; + case T_AggState: ExecSquelchAgg((AggState*) node); break; diff --git a/src/backend/executor/nodeIncrementalSort.c b/src/backend/executor/nodeIncrementalSort.c index eb1c1326dea2..8eabb323705b 100644 --- a/src/backend/executor/nodeIncrementalSort.c +++ b/src/backend/executor/nodeIncrementalSort.c @@ -115,6 +115,7 @@ } \ } while (0) +static void ExecEagerFreeIncrementalSort(IncrementalSortState *node); /* ---------------------------------------------------------------- * instrumentSortedGroup @@ -977,6 +978,12 @@ ExecIncrementalSort(PlanState *pstate) slot = node->ss.ps.ps_ResultTupleSlot; (void) tuplesort_gettupleslot(read_sortstate, ScanDirectionIsForward(dir), false, slot, NULL); + + if (TupIsNull(slot) && !node->delayEagerFree) + { + ExecEagerFreeIncrementalSort(node); + } + return slot; } @@ -1046,6 +1053,11 @@ ExecInitIncrementalSort(IncrementalSort *node, EState *estate, int eflags) * ExecQual or ExecProject. */ + /* + * If eflag contains EXEC_FLAG_REWIND then this node is not eager free safe. + */ + incrsortstate->delayEagerFree = ((eflags & (EXEC_FLAG_REWIND)) != 0); + /* * Initialize child nodes. * @@ -1056,6 +1068,15 @@ ExecInitIncrementalSort(IncrementalSort *node, EState *estate, int eflags) */ outerPlanState(incrsortstate) = ExecInitNode(outerPlan(node), estate, eflags); + + /* + * If the child node is a Motion, then this node is not eager free safe. + */ + if (IsA(outerPlan((Plan *)node), Motion)) + { + incrsortstate->delayEagerFree = true; + } + /* * Initialize scan slot and type. */ @@ -1093,27 +1114,7 @@ ExecEndIncrementalSort(IncrementalSortState *node) { SO_printf("ExecEndIncrementalSort: shutting down sort node\n"); - /* clean out the scan tuple */ - ExecClearTuple(node->ss.ss_ScanTupleSlot); - /* must drop pointer to sort result tuple */ - ExecClearTuple(node->ss.ps.ps_ResultTupleSlot); - /* must drop standalone tuple slots from outer node */ - ExecDropSingleTupleTableSlot(node->group_pivot); - ExecDropSingleTupleTableSlot(node->transfer_tuple); - - /* - * Release tuplesort resources. - */ - if (node->fullsort_state != NULL) - { - tuplesort_end(node->fullsort_state); - node->fullsort_state = NULL; - } - if (node->prefixsort_state != NULL) - { - tuplesort_end(node->prefixsort_state); - node->prefixsort_state = NULL; - } + ExecEagerFreeIncrementalSort(node); /* * Shut down the subplan. @@ -1185,6 +1186,52 @@ ExecReScanIncrementalSort(IncrementalSortState *node) ExecReScan(outerPlan); } +static void +ExecEagerFreeIncrementalSort(IncrementalSortState *node) +{ + /* clean out the scan tuple */ + ExecClearTuple(node->ss.ss_ScanTupleSlot); + /* must drop pointer to sort result tuple */ + ExecClearTuple(node->ss.ps.ps_ResultTupleSlot); + /* must drop standalone tuple slots from outer node */ + if (node->group_pivot != NULL) + { + ExecDropSingleTupleTableSlot(node->group_pivot); + node->group_pivot = NULL; + } + + if (node->transfer_tuple != NULL) + { + ExecDropSingleTupleTableSlot(node->transfer_tuple); + node->transfer_tuple = NULL; + } + + /* + * Release tuplesort resources. + */ + if (node->fullsort_state != NULL) + { + tuplesort_end(node->fullsort_state); + node->fullsort_state = NULL; + } + if (node->prefixsort_state != NULL) + { + tuplesort_end(node->prefixsort_state); + node->prefixsort_state = NULL; + } +} + +void +ExecSquelchIncrementalSort(IncrementalSortState *node) +{ + if (!node->delayEagerFree) + { + ExecEagerFreeIncrementalSort(node); + ExecSquelchNode(outerPlanState(node)); + } +} + + /* ---------------------------------------------------------------- * Parallel Query Support * ---------------------------------------------------------------- diff --git a/src/backend/nodes/outfast.c b/src/backend/nodes/outfast.c index 6e38339fe6a4..7ecf33fd738e 100644 --- a/src/backend/nodes/outfast.c +++ b/src/backend/nodes/outfast.c @@ -1082,6 +1082,9 @@ _outNode(StringInfo str, void *obj) case T_Sort: _outSort(str, obj); break; + case T_IncrementalSort: + _outIncrementalSort(str, obj); + break; case T_Unique: _outUnique(str, obj); break; diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 54c744088e78..a040e6dda374 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2472,6 +2472,17 @@ _outSortPath(StringInfo str, const SortPath *node) WRITE_NODE_FIELD(subpath); } +static void +_outIncrementalSortPath(StringInfo str, const IncrementalSortPath *node) +{ + WRITE_NODE_TYPE("INCREMENTALSORTPATH"); + + _outPathInfo(str, (const Path *) node); + + WRITE_NODE_FIELD(spath.subpath); + WRITE_INT_FIELD(nPresortedCols); +} + static void _outGroupPath(StringInfo str, const GroupPath *node) { @@ -5970,6 +5981,9 @@ outNode(StringInfo str, const void *obj) case T_SortPath: _outSortPath(str, obj); break; + case T_IncrementalSortPath: + _outIncrementalSortPath(str, obj); + break; case T_GroupPath: _outGroupPath(str, obj); break; diff --git a/src/backend/nodes/readfast.c b/src/backend/nodes/readfast.c index 586724451084..afab435418af 100644 --- a/src/backend/nodes/readfast.c +++ b/src/backend/nodes/readfast.c @@ -1854,6 +1854,9 @@ readNodeBinary(void) case T_Sort: return_value = _readSort(); break; + case T_IncrementalSort: + return_value = _readIncrementalSort(); + break; case T_Unique: return_value = _readUnique(); break; diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 368ba7b9f1ae..1e9795935641 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -132,8 +132,7 @@ bool enable_indexonlyscan = true; bool enable_bitmapscan = true; bool enable_tidscan = true; bool enable_sort = true; -/* GPDB_13_MERGE_FIXME: enable incremental sort */ -bool enable_incremental_sort = false; +bool enable_incremental_sort = true; bool enable_hashagg = true; bool enable_groupagg = true; bool enable_nestloop = false; diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 282bbd0786f3..489c1a7bef8b 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -4816,6 +4816,22 @@ create_ordinary_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel, extra); } + +static double +calculate_num_groups(Path *path, double dNumGroupsTotal) +{ + /* + * dNumGroupsTotal is the total number of groups across all segments. If the + * Aggregate is distributed, then the number of groups in one segment + * is only a fraction of the total. + */ + + if (CdbPathLocus_IsPartitioned(path->locus)) + return clamp_row_est(dNumGroupsTotal / + CdbPathLocus_NumSegments(path->locus)); + return dNumGroupsTotal; +} + /* * For a given input path, consider the possible ways of doing grouping sets on * it, by combinations of hashing and sorting. This can be called multiple @@ -4862,16 +4878,7 @@ consider_groupingsets_paths(PlannerInfo *root, parse->groupClause, gd->rollups); - /* - * dNumGroupsTotal is the total number of groups across all segments. If the - * Aggregate is distributed, then the number of groups in one segment - * is only a fraction of the total. - */ - if (CdbPathLocus_IsPartitioned(path->locus)) - dNumGroups = clamp_row_est(dNumGroupsTotal / - CdbPathLocus_NumSegments(path->locus)); - else - dNumGroups = dNumGroupsTotal; + dNumGroups = calculate_num_groups(path, dNumGroupsTotal); srd = make_new_rollups_for_hash_grouping_set(root, path, gd); @@ -4930,20 +4937,12 @@ consider_groupingsets_paths(PlannerInfo *root, path, path->pathtarget, root->group_pathkeys, + NO_INCREMENTAL_SORT, -1.0, parse->groupClause, gd->rollups); - /* - * dNumGroupsTotal is the total number of groups across all segments. If the - * Aggregate is distributed, then the number of groups in one segment - * is only a fraction of the total. - */ - if (CdbPathLocus_IsPartitioned(path->locus)) - dNumGroups = clamp_row_est(dNumGroupsTotal / - CdbPathLocus_NumSegments(path->locus)); - else - dNumGroups = dNumGroupsTotal; + dNumGroups = calculate_num_groups(path, dNumGroupsTotal); /* * Given sorted input, we try and make two paths: one sorted and one mixed @@ -5246,41 +5245,18 @@ create_one_window_path(PlannerInfo *root, { WindowClause *wc = lfirst_node(WindowClause, l); List *window_pathkeys; -#if 0 /* GPDB_14_MERGE_FIXME: enable incremental sort */ int presorted_keys; bool is_sorted; -#endif window_pathkeys = make_pathkeys_for_window(root, wc, root->processed_tlist); - /* - * Unless the PARTITION BY in the window happens to match the - * current distribution, we need a motion. Each partition - * needs to be handled in the same segment. - * - * If there is no PARTITION BY, then all rows form a single - * partition, so we need to gather all the tuples to a single - * node. But we'll do that after the Sort, so that the Sort - * is parallelized. - * - * This is the same logic that is used for sorted Aggregates. - */ - path = cdb_prepare_path_for_sorted_agg(root, - pathkeys_contained_in(window_pathkeys, path->pathkeys), - window_rel, - path, - path->pathtarget, - window_pathkeys, - -1.0, - wc->partitionClause, - NIL); -#if 0 /* GPDB_14_MERGE_FIXME: enable incremental sort */ is_sorted = pathkeys_count_contained_in(window_pathkeys, path->pathkeys, &presorted_keys); +#if 0 /* PostgreSQL */ /* Sort if necessary */ if (!is_sorted) { @@ -5308,6 +5284,28 @@ create_one_window_path(PlannerInfo *root, } } #endif + /* + * Unless the PARTITION BY in the window happens to match the + * current distribution, we need a motion. Each partition + * needs to be handled in the same segment. + * + * If there is no PARTITION BY, then all rows form a single + * partition, so we need to gather all the tuples to a single + * node. But we'll do that after the Sort, so that the Sort + * is parallelized. + * + * This is the same logic that is used for sorted Aggregates. + */ + path = cdb_prepare_path_for_sorted_agg(root, + is_sorted, + window_rel, + path, + path->pathtarget, + window_pathkeys, + enable_incremental_sort ? presorted_keys : NO_INCREMENTAL_SORT, + -1.0, + wc->partitionClause, + NIL); if (lnext(activeWindows, l)) { @@ -5458,6 +5456,7 @@ create_distinct_paths(PlannerInfo *root, distinct_rel, path, path->pathtarget, needed_pathkeys, + NO_INCREMENTAL_SORT, -1.0, parse->distinctClause, NIL); @@ -5489,13 +5488,20 @@ create_distinct_paths(PlannerInfo *root, needed_pathkeys = root->distinct_pathkeys; path = cheapest_input_path; - +#if 0 /* PostgreSQL */ + if (!pathkeys_contained_in(needed_pathkeys, path->pathkeys)) + path = (Path *) create_sort_path(root, distinct_rel, + path, + needed_pathkeys, + -1.0); +#endif path = cdb_prepare_path_for_sorted_agg(root, pathkeys_contained_in(needed_pathkeys, cheapest_input_path->pathkeys), distinct_rel, cheapest_input_path, cheapest_input_path->pathtarget, needed_pathkeys, + NO_INCREMENTAL_SORT, -1.0, parse->distinctClause, NIL); @@ -7374,11 +7380,10 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel, foreach(lc, input_rel->pathlist) { Path *path = (Path *) lfirst(lc); -#if 0 /* GPDB_13_MERGE_FIXME: enable incremental sort */ Path *path_original = path; -#endif bool is_sorted; int presorted_keys; + double dNumGroups; is_sorted = pathkeys_count_contained_in(root->group_pathkeys, path->pathkeys, @@ -7386,8 +7391,15 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel, if (path == cheapest_path || is_sorted) { - double dNumGroups; - +#if 0 /* PostgreSQL */ + /* Sort the cheapest-total path if it isn't already sorted */ + if (!is_sorted) + path = (Path *) create_sort_path(root, + grouped_rel, + path, + root->group_pathkeys, + -1.0); +#endif /* * Sort the cheapest-total path if it isn't already sorted. * This also adds a Motion to redistribute it if needed. @@ -7398,20 +7410,12 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel, path, path->pathtarget, root->group_pathkeys, + NO_INCREMENTAL_SORT, -1.0, parse->groupClause, gd ? gd->rollups : NIL); - /* - * dNumGroupsTotal is the total number of groups across all segments. If the - * Aggregate is distributed, then the number of groups in one segment - * is only a fraction of the total. - */ - if (CdbPathLocus_IsPartitioned(path->locus)) - dNumGroups = clamp_row_est(dNumGroupsTotal / - CdbPathLocus_NumSegments(path->locus)); - else - dNumGroups = dNumGroupsTotal; + dNumGroups = calculate_num_groups(path, dNumGroupsTotal); /* Now decide what to stick atop it */ if (parse->groupingSets) @@ -7468,7 +7472,6 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel, } } -#if 0 /* GPDB_13_MERGE_FIXME: enable incremental sort */ /* * Now we may consider incremental sort on this path, but only * when the path is not already sorted and when incremental sort @@ -7490,12 +7493,28 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel, */ Assert(list_length(root->group_pathkeys) != 1); +/* Use cdb_prepare_path_for_sorted_agg instead */ +#if 0 path = (Path *) create_incremental_sort_path(root, grouped_rel, path, root->group_pathkeys, presorted_keys, -1.0); +#endif + + path = cdb_prepare_path_for_sorted_agg(root, + is_sorted, + grouped_rel, + path, + path->pathtarget, + root->group_pathkeys, + presorted_keys, + -1.0, + parse->groupClause, + gd ? gd->rollups : NIL); + + dNumGroups = calculate_num_groups(path, dNumGroupsTotal); /* Now decide what to stick atop it */ if (parse->groupingSets) @@ -7504,11 +7523,12 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel, path, true, can_hash, gd, agg_costs, dNumGroups); } - else if (parse->hasAggs) + else if (parse->hasAggs || parse->groupClause) { /* * We have aggregation, possibly with plain GROUP BY. Make an * AggPath. + * Since group nodes are not used in GPDB, use just agg_path instead. */ add_path(grouped_rel, (Path *) create_agg_path(root, @@ -7517,11 +7537,14 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel, grouped_rel->reltarget, parse->groupClause ? AGG_SORTED : AGG_PLAIN, AGGSPLIT_SIMPLE, + false, /* streaming */ parse->groupClause, havingQual, agg_costs, dNumGroups)); } + /* Group nodes are not used in GPDB */ +#if 0 else if (parse->groupClause) { /* @@ -7536,12 +7559,12 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel, havingQual, dNumGroups)); } +#endif else { /* Other cases should have been handled above */ Assert(false); } -#endif } /* @@ -7553,9 +7576,7 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel, foreach(lc, partially_grouped_rel->pathlist) { Path *path = (Path *) lfirst(lc); -#if 0 /* GPDB_13_MERGE_FIXME: enable incremental sort */ Path *path_original = path; -#endif bool is_sorted; int presorted_keys; double dNumGroups; @@ -7572,6 +7593,13 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel, { if (path != partially_grouped_rel->cheapest_total_path) continue; +#if 0 /* PostgreSQL */ + path = (Path *) create_sort_path(root, + grouped_rel, + path, + root->group_pathkeys, + -1.0); +#endif } path = cdb_prepare_path_for_sorted_agg(root, @@ -7580,20 +7608,12 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel, path, path->pathtarget, root->group_pathkeys, + NO_INCREMENTAL_SORT, -1.0, parse->groupClause, NIL); - /* - * dNumGroupsTotal is the total number of groups across all segments. If the - * Aggregate is distributed, then the number of groups in one segment - * is only a fraction of the total. - */ - if (CdbPathLocus_IsPartitioned(path->locus)) - dNumGroups = clamp_row_est(dNumGroupsTotal / - CdbPathLocus_NumSegments(path->locus)); - else - dNumGroups = dNumGroupsTotal; + dNumGroups = calculate_num_groups(path, dNumGroupsTotal); //if (parse->hasAggs) { @@ -7622,7 +7642,6 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel, dNumGroups)); #endif -#if 0 /* GPDB_13_MERGE_FIXME: enable incremental sort */ /* * Now we may consider incremental sort on this path, but only * when the path is not already sorted and when incremental @@ -7645,12 +7664,28 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel, */ Assert(list_length(root->group_pathkeys) != 1); +/* Use cdb_prepare_path_for_sorted_agg instead */ +#if 0 path = (Path *) create_incremental_sort_path(root, - grouped_rel, - path, - root->group_pathkeys, - presorted_keys, - -1.0); + grouped_rel, + path, + root->group_pathkeys, + presorted_keys, + -1.0); +#endif + + path = cdb_prepare_path_for_sorted_agg(root, + is_sorted, + grouped_rel, + path, + path->pathtarget, + root->group_pathkeys, + presorted_keys, + -1.0, + parse->groupClause, + NIL); + + dNumGroups = calculate_num_groups(path, dNumGroupsTotal); if (parse->hasAggs) add_path(grouped_rel, (Path *) @@ -7660,10 +7695,13 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel, grouped_rel->reltarget, parse->groupClause ? AGG_SORTED : AGG_PLAIN, AGGSPLIT_FINAL_DESERIAL, + false, /* streaming */ parse->groupClause, havingQual, agg_final_costs, dNumGroups)); + /* Group nodes are not used in GPDB */ +#if 0 else add_path(grouped_rel, (Path *) create_group_path(root, @@ -7700,17 +7738,7 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel, parse->groupClause, NIL); - /* - * dNumGroupsTotal is the total number of groups across all segments. If the - * Aggregate is distributed, then the number of groups in one segment - * is only a fraction of the total. - */ - if (CdbPathLocus_IsPartitioned(path->locus)) - dNumGroups = clamp_row_est(dNumGroupsTotal / - CdbPathLocus_NumSegments(path->locus)); - else - dNumGroups = dNumGroupsTotal; - + dNumGroups = calculate_num_groups(path, dNumGroupsTotal); /* * Generate a HashAgg Path. We just need an Agg over the * cheapest-total input path, since input order won't matter. @@ -7743,16 +7771,7 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel, parse->groupClause, NIL); - /* - * dNumGroupsTotal is the total number of groups across all segments. If the - * Aggregate is distributed, then the number of groups in one segment - * is only a fraction of the total. - */ - if (CdbPathLocus_IsPartitioned(path->locus)) - dNumGroups = clamp_row_est(dNumGroupsTotal / - CdbPathLocus_NumSegments(path->locus)); - else - dNumGroups = dNumGroupsTotal; + dNumGroups = calculate_num_groups(path, dNumGroupsTotal); add_path(grouped_rel, (Path *) create_agg_path(root, @@ -8079,7 +8098,6 @@ create_partial_grouping_paths(PlannerInfo *root, } } -#if 0 /* GPDB_13_MERGE_FIXME: enable incremental sort */ /* * Consider incremental sort on all partial paths, if enabled. * @@ -8122,10 +8140,13 @@ create_partial_grouping_paths(PlannerInfo *root, partially_grouped_rel->reltarget, parse->groupClause ? AGG_SORTED : AGG_PLAIN, AGGSPLIT_INITIAL_SERIAL, + false, /* streaming */ parse->groupClause, NIL, agg_partial_costs, dNumPartialGroups)); + /* Group nodes are not used in GPDB */ +#if 0 else add_path(partially_grouped_rel, (Path *) create_group_path(root, @@ -8134,9 +8155,9 @@ create_partial_grouping_paths(PlannerInfo *root, parse->groupClause, NIL, dNumPartialGroups)); +#endif } } -#endif } if (can_sort && cheapest_partial_path != NULL) @@ -8145,9 +8166,7 @@ create_partial_grouping_paths(PlannerInfo *root, foreach(lc, input_rel->partial_pathlist) { Path *path = (Path *) lfirst(lc); -#if 0 /* GPDB_13_MERGE_FIXME: enable incremental sort */ Path *path_original = path; -#endif bool is_sorted; int presorted_keys; @@ -8193,7 +8212,6 @@ create_partial_grouping_paths(PlannerInfo *root, #endif } -#if 0 /* GPDB_13_MERGE_FIXME: enable incremental sort */ /* * Now we may consider incremental sort on this path, but only * when the path is not already sorted and when incremental sort @@ -8230,10 +8248,13 @@ create_partial_grouping_paths(PlannerInfo *root, partially_grouped_rel->reltarget, parse->groupClause ? AGG_SORTED : AGG_PLAIN, AGGSPLIT_INITIAL_SERIAL, + false, /* streaming */ parse->groupClause, NIL, agg_partial_costs, dNumPartialPartialGroups)); + /* Group nodes are not used in GPDB */ +#if 0 else add_partial_path(partially_grouped_rel, (Path *) create_group_path(root, diff --git a/src/backend/optimizer/util/walkers.c b/src/backend/optimizer/util/walkers.c index e8247bcc128e..8d09af8cef60 100644 --- a/src/backend/optimizer/util/walkers.c +++ b/src/backend/optimizer/util/walkers.c @@ -361,6 +361,7 @@ plan_tree_walker(Node *node, break; case T_Sort: + case T_IncrementalSort: if (walk_plan_node_fields((Plan *) node, walker, context)) return true; /* Other fields are simple counts and lists of indexes and oids. */ diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index d821e8817e19..5c066309cea9 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -1036,8 +1036,7 @@ static struct config_bool ConfigureNamesBool[] = NULL }, &enable_incremental_sort, - /* GPDB_13_MERGE_FIXME: enable incremental sort */ - false, + true, NULL, NULL, NULL }, { diff --git a/src/backend/utils/resource_manager/memquota.c b/src/backend/utils/resource_manager/memquota.c index 7a28c6729899..4fb39fa613f1 100644 --- a/src/backend/utils/resource_manager/memquota.c +++ b/src/backend/utils/resource_manager/memquota.c @@ -246,6 +246,7 @@ IsBlockingOperator(Node *node) case T_BitmapIndexScan: case T_Hash: case T_Sort: + case T_IncrementalSort: return true; case T_Material: @@ -295,6 +296,7 @@ IsMemoryIntensiveOperator(Node *node, PlannedStmt *stmt) { case T_Material: case T_Sort: + case T_IncrementalSort: case T_ShareInputScan: case T_Hash: case T_BitmapIndexScan: diff --git a/src/include/cdb/cdbgroupingpaths.h b/src/include/cdb/cdbgroupingpaths.h index 4b64b211515d..43e9d7118f23 100644 --- a/src/include/cdb/cdbgroupingpaths.h +++ b/src/include/cdb/cdbgroupingpaths.h @@ -16,6 +16,8 @@ #include "nodes/pathnodes.h" +#define NO_INCREMENTAL_SORT 0 + extern void cdb_create_multistage_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel, RelOptInfo *output_rel, @@ -43,6 +45,7 @@ extern Path *cdb_prepare_path_for_sorted_agg(PlannerInfo *root, Path *subpath, PathTarget *target, List *group_pathkeys, + int presorted_keys, double limit_tuples, List *groupClause, List *rollups); diff --git a/src/include/executor/nodeIncrementalSort.h b/src/include/executor/nodeIncrementalSort.h index e62c02a4f300..ede1f8b7dac2 100644 --- a/src/include/executor/nodeIncrementalSort.h +++ b/src/include/executor/nodeIncrementalSort.h @@ -18,6 +18,7 @@ extern IncrementalSortState *ExecInitIncrementalSort(IncrementalSort *node, EState *estate, int eflags); extern void ExecEndIncrementalSort(IncrementalSortState *node); extern void ExecReScanIncrementalSort(IncrementalSortState *node); +extern void ExecSquelchIncrementalSort(IncrementalSortState *node); /* parallel instrumentation support */ extern void ExecIncrementalSortEstimate(IncrementalSortState *node, ParallelContext *pcxt); diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 55d8c52a3abb..ac97152a5420 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -2486,6 +2486,10 @@ typedef struct IncrementalSortState TupleTableSlot *transfer_tuple; bool am_worker; /* are we a worker? */ SharedIncrementalSortInfo *shared_info; /* one entry per worker */ + + bool delayEagerFree; /* is it safe to free memory used by this node, + * when this node has outputted its last row? */ + } IncrementalSortState; /* --------------------- diff --git a/src/test/regress/expected/dpe.out b/src/test/regress/expected/dpe.out index b71d22911c84..aafd1d4912a9 100644 --- a/src/test/regress/expected/dpe.out +++ b/src/test/regress/expected/dpe.out @@ -2934,10 +2934,9 @@ select * from pt, t where t.dist = pt.dist and t.tid = pt.ptid order by t.tid, t ------------------------------------------------------------------------------------------------------------------------------------------------ Gather Motion 3:1 (slice1; segments: 3) (cost=10000000014.65..10000000014.75 rows=7 width=24) (actual rows=5 loops=1) Merge Key: pt.ptid, t.sk - -> Sort (cost=10000000014.65..10000000014.66 rows=2 width=24) (actual rows=2 loops=1) + -> Incremental Sort (cost=10000000008.18..10000000014.74 rows=2 width=24) (actual rows=2 loops=1) Sort Key: pt.ptid, t.sk - Sort Method: quicksort Memory: 75kB - Executor Memory: 76kB Segments: 3 Max: 26kB (segment 0) + Presorted Key: pt.ptid -> Merge Join (cost=10000000003.00..10000000014.64 rows=2 width=24) (actual rows=2 loops=1) Merge Cond: (pt.ptid = t.tid) Join Filter: (pt.dist = t.dist) @@ -2956,5 +2955,5 @@ select * from pt, t where t.dist = pt.dist and t.tid = pt.ptid order by t.tid, t -> Partition Selector (selector id: $0) (cost=10000000000.00..10000000001.33 rows=33 width=12) (actual rows=37 loops=1) -> Seq Scan on t (cost=10000000000.00..10000000001.33 rows=33 width=12) (actual rows=37 loops=1) Optimizer: Postgres query optimizer -(22 rows) +(23 rows) diff --git a/src/test/regress/expected/incremental_sort.out b/src/test/regress/expected/incremental_sort.out index 7cf2eee7c14c..c8b7da40dbc4 100644 --- a/src/test/regress/expected/incremental_sort.out +++ b/src/test/regress/expected/incremental_sort.out @@ -2,47 +2,57 @@ -- be slower than plain sort, so it should not be used. explain (costs off) select * from (select * from tenk1 order by four) t order by four, ten; - QUERY PLAN ------------------------------------ - Sort - Sort Key: tenk1.four, tenk1.ten + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: tenk1.four, tenk1.ten -> Sort - Sort Key: tenk1.four - -> Seq Scan on tenk1 -(5 rows) + Sort Key: tenk1.four, tenk1.ten + -> Sort + Sort Key: tenk1.four + -> Seq Scan on tenk1 + Optimizer: Postgres query optimizer +(8 rows) -- When there is a LIMIT clause, incremental sort is beneficial because -- it only has to sort some of the groups, and not the entire table. explain (costs off) select * from (select * from tenk1 order by four) t order by four, ten limit 1; - QUERY PLAN ------------------------------------------ + QUERY PLAN +----------------------------------------------------- Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: tenk1.four, tenk1.ten + -> Limit + -> Incremental Sort + Sort Key: tenk1.four, tenk1.ten + Presorted Key: tenk1.four + -> Sort + Sort Key: tenk1.four + -> Seq Scan on tenk1 + Optimizer: Postgres query optimizer +(11 rows) + +-- When working memory is not enough to sort the entire table, incremental sort +-- may be faster if individual groups still fit into work_mem. +set planner_work_mem=280; +explain (costs off) +select * from (select * from tenk1 order by four) t order by four, ten; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: tenk1.four, tenk1.ten -> Incremental Sort Sort Key: tenk1.four, tenk1.ten Presorted Key: tenk1.four -> Sort Sort Key: tenk1.four -> Seq Scan on tenk1 -(7 rows) - --- When work_mem is not enough to sort the entire table, incremental sort --- may be faster if individual groups still fit into work_mem. -set work_mem to '2MB'; -explain (costs off) -select * from (select * from tenk1 order by four) t order by four, ten; - QUERY PLAN ------------------------------------ - Incremental Sort - Sort Key: tenk1.four, tenk1.ten - Presorted Key: tenk1.four - -> Sort - Sort Key: tenk1.four - -> Seq Scan on tenk1 -(6 rows) + Optimizer: Postgres query optimizer +(9 rows) -reset work_mem; +reset planner_work_mem; create table t(a integer, b integer); create or replace function explain_analyze_without_memory(query text) returns table (out_line text) language plpgsql @@ -144,16 +154,20 @@ $$; insert into t(a, b) select i/100 + 1, i + 1 from generate_series(0, 999) n(i); analyze t; explain (costs off) select * from (select * from t order by a) s order by a, b limit 31; - QUERY PLAN ---------------------------------- + QUERY PLAN +------------------------------------------------ Limit - -> Incremental Sort - Sort Key: t.a, t.b - Presorted Key: t.a - -> Sort - Sort Key: t.a - -> Seq Scan on t -(7 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a, t.b + -> Limit + -> Incremental Sort + Sort Key: t.a, t.b + Presorted Key: t.a + -> Sort + Sort Key: t.a + -> Seq Scan on t + Optimizer: Postgres query optimizer +(11 rows) select * from (select * from t order by a) s order by a, b limit 31; a | b @@ -192,16 +206,20 @@ select * from (select * from t order by a) s order by a, b limit 31; (31 rows) explain (costs off) select * from (select * from t order by a) s order by a, b limit 32; - QUERY PLAN ---------------------------------- + QUERY PLAN +------------------------------------------------ Limit - -> Incremental Sort - Sort Key: t.a, t.b - Presorted Key: t.a - -> Sort - Sort Key: t.a - -> Seq Scan on t -(7 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a, t.b + -> Limit + -> Incremental Sort + Sort Key: t.a, t.b + Presorted Key: t.a + -> Sort + Sort Key: t.a + -> Seq Scan on t + Optimizer: Postgres query optimizer +(11 rows) select * from (select * from t order by a) s order by a, b limit 32; a | b @@ -241,16 +259,20 @@ select * from (select * from t order by a) s order by a, b limit 32; (32 rows) explain (costs off) select * from (select * from t order by a) s order by a, b limit 33; - QUERY PLAN ---------------------------------- + QUERY PLAN +------------------------------------------------ Limit - -> Incremental Sort - Sort Key: t.a, t.b - Presorted Key: t.a - -> Sort - Sort Key: t.a - -> Seq Scan on t -(7 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a, t.b + -> Limit + -> Incremental Sort + Sort Key: t.a, t.b + Presorted Key: t.a + -> Sort + Sort Key: t.a + -> Seq Scan on t + Optimizer: Postgres query optimizer +(11 rows) select * from (select * from t order by a) s order by a, b limit 33; a | b @@ -291,16 +313,20 @@ select * from (select * from t order by a) s order by a, b limit 33; (33 rows) explain (costs off) select * from (select * from t order by a) s order by a, b limit 65; - QUERY PLAN ---------------------------------- + QUERY PLAN +------------------------------------------------ Limit - -> Incremental Sort - Sort Key: t.a, t.b - Presorted Key: t.a - -> Sort - Sort Key: t.a - -> Seq Scan on t -(7 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a, t.b + -> Limit + -> Incremental Sort + Sort Key: t.a, t.b + Presorted Key: t.a + -> Sort + Sort Key: t.a + -> Seq Scan on t + Optimizer: Postgres query optimizer +(11 rows) select * from (select * from t order by a) s order by a, b limit 65; a | b @@ -373,16 +399,20 @@ select * from (select * from t order by a) s order by a, b limit 65; (65 rows) explain (costs off) select * from (select * from t order by a) s order by a, b limit 66; - QUERY PLAN ---------------------------------- + QUERY PLAN +------------------------------------------------ Limit - -> Incremental Sort - Sort Key: t.a, t.b - Presorted Key: t.a - -> Sort - Sort Key: t.a - -> Seq Scan on t -(7 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a, t.b + -> Limit + -> Incremental Sort + Sort Key: t.a, t.b + Presorted Key: t.a + -> Sort + Sort Key: t.a + -> Seq Scan on t + Optimizer: Postgres query optimizer +(11 rows) select * from (select * from t order by a) s order by a, b limit 66; a | b @@ -460,16 +490,20 @@ delete from t; insert into t(a, b) select i/50 + 1, i + 1 from generate_series(0, 999) n(i); analyze t; explain (costs off) select * from (select * from t order by a) s order by a, b limit 55; - QUERY PLAN ---------------------------------- + QUERY PLAN +------------------------------------------------ Limit - -> Incremental Sort - Sort Key: t.a, t.b - Presorted Key: t.a - -> Sort - Sort Key: t.a - -> Seq Scan on t -(7 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a, t.b + -> Limit + -> Incremental Sort + Sort Key: t.a, t.b + Presorted Key: t.a + -> Sort + Sort Key: t.a + -> Seq Scan on t + Optimizer: Postgres query optimizer +(11 rows) select * from (select * from t order by a) s order by a, b limit 55; a | b @@ -533,48 +567,44 @@ select * from (select * from t order by a) s order by a, b limit 55; -- Test EXPLAIN ANALYZE with only a fullsort group. select explain_analyze_without_memory('select * from (select * from t order by a) s order by a, b limit 55'); - explain_analyze_without_memory ---------------------------------------------------------------------------------------------------------------- + explain_analyze_without_memory +------------------------------------------------------------------------------------- Limit (actual rows=55 loops=1) - -> Incremental Sort (actual rows=55 loops=1) - Sort Key: t.a, t.b - Presorted Key: t.a - Full-sort Groups: 2 Sort Methods: top-N heapsort, quicksort Average Memory: NNkB Peak Memory: NNkB - -> Sort (actual rows=101 loops=1) - Sort Key: t.a - Sort Method: quicksort Memory: NNkB - -> Seq Scan on t (actual rows=1000 loops=1) -(9 rows) + -> Gather Motion 3:1 (slice1; segments: 3) (actual rows=55 loops=1) + Merge Key: t.a, t.b + -> Limit (actual rows=55 loops=1) + -> Incremental Sort (actual rows=55 loops=1) + Sort Key: t.a, t.b + Presorted Key: t.a + -> Sort (actual rows=101 loops=1) + Sort Key: t.a + Sort Method: quicksort Memory: NNkB + Executor Memory: NNkB Segments: 3 Max: NNkB (segment 0) + -> Seq Scan on t (actual rows=400 loops=1) + Optimizer: Postgres query optimizer +(13 rows) select jsonb_pretty(explain_analyze_inc_sort_nodes_without_memory('select * from (select * from t order by a) s order by a, b limit 55')); - jsonb_pretty -------------------------------------------------- - [ + - { + - "Sort Key": [ + - "t.a", + - "t.b" + - ], + - "Node Type": "Incremental Sort", + - "Actual Rows": 55, + - "Actual Loops": 1, + - "Presorted Key": [ + - "t.a" + - ], + - "Parallel Aware": false, + - "Full-sort Groups": { + - "Group Count": 2, + - "Sort Methods Used": [ + - "top-N heapsort", + - "quicksort" + - ], + - "Sort Space Memory": { + - "Peak Sort Space Used": "NN", + - "Average Sort Space Used": "NN"+ - } + - }, + - "Parent Relationship": "Outer" + - } + + jsonb_pretty +------------------------------------------ + [ + + { + + "Slice": 1, + + "Segments": 3, + + "Sort Key": [ + + "t.a", + + "t.b" + + ], + + "Gang Type": "primary reader", + + "Node Type": "Incremental Sort",+ + "Actual Rows": 55, + + "Actual Loops": 1, + + "Presorted Key": [ + + "t.a" + + ], + + "Parallel Aware": false, + + "Parent Relationship": "Outer" + + } + ] (1 row) @@ -589,16 +619,20 @@ delete from t; insert into t(a, b) select (case when i < 5 then i else 9 end), i from generate_series(1, 1000) n(i); analyze t; explain (costs off) select * from (select * from t order by a) s order by a, b limit 70; - QUERY PLAN ---------------------------------- + QUERY PLAN +------------------------------------------------ Limit - -> Incremental Sort - Sort Key: t.a, t.b - Presorted Key: t.a - -> Sort - Sort Key: t.a - -> Seq Scan on t -(7 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a, t.b + -> Limit + -> Incremental Sort + Sort Key: t.a, t.b + Presorted Key: t.a + -> Sort + Sort Key: t.a + -> Seq Scan on t + Optimizer: Postgres query optimizer +(11 rows) select * from (select * from t order by a) s order by a, b limit 70; a | b @@ -684,83 +718,71 @@ set local enable_mergejoin = off; set local enable_material = off; set local enable_sort = off; explain (costs off) select * from t left join (select * from (select * from t order by a) v order by a, b) s on s.a = t.a where t.a in (1, 2); - QUERY PLAN ------------------------------------------------- - Nested Loop Left Join - Join Filter: (t_1.a = t.a) - -> Seq Scan on t - Filter: (a = ANY ('{1,2}'::integer[])) - -> Incremental Sort - Sort Key: t_1.a, t_1.b - Presorted Key: t_1.a - -> Sort - Sort Key: t_1.a - -> Seq Scan on t t_1 -(10 rows) + QUERY PLAN +------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Nested Loop Left Join + Join Filter: (t_1.a = t.a) + -> Seq Scan on t + Filter: (a = ANY ('{1,2}'::integer[])) + -> Materialize + -> Incremental Sort + Sort Key: t_1.a, t_1.b + Presorted Key: t_1.a + -> Sort + Sort Key: t_1.a + -> Seq Scan on t t_1 + Optimizer: Postgres query optimizer +(13 rows) -select * from t left join (select * from (select * from t order by a) v order by a, b) s on s.a = t.a where t.a in (1, 2); +select * from t left join (select * from (select * from t order by a) v order by a, b) s on s.a = t.a where t.a in (1, 2); -- order none a | b | a | b ---+---+---+--- - 1 | 1 | 1 | 1 2 | 2 | 2 | 2 + 1 | 1 | 1 | 1 (2 rows) rollback; -- Test EXPLAIN ANALYZE with both fullsort and presorted groups. select explain_analyze_without_memory('select * from (select * from t order by a) s order by a, b limit 70'); - explain_analyze_without_memory ----------------------------------------------------------------------------------------------------------------- + explain_analyze_without_memory +------------------------------------------------------------------------------------- Limit (actual rows=70 loops=1) - -> Incremental Sort (actual rows=70 loops=1) - Sort Key: t.a, t.b - Presorted Key: t.a - Full-sort Groups: 1 Sort Method: quicksort Average Memory: NNkB Peak Memory: NNkB - Pre-sorted Groups: 5 Sort Methods: top-N heapsort, quicksort Average Memory: NNkB Peak Memory: NNkB - -> Sort (actual rows=1000 loops=1) - Sort Key: t.a - Sort Method: quicksort Memory: NNkB - -> Seq Scan on t (actual rows=1000 loops=1) -(10 rows) + -> Gather Motion 3:1 (slice1; segments: 3) (actual rows=70 loops=1) + Merge Key: t.a, t.b + -> Limit (actual rows=70 loops=1) + -> Incremental Sort (actual rows=70 loops=1) + Sort Key: t.a, t.b + Presorted Key: t.a + -> Sort (actual rows=996 loops=1) + Sort Key: t.a + Sort Method: quicksort Memory: NNkB + Executor Memory: NNkB Segments: 3 Max: NNkB (segment 2) + -> Seq Scan on t (actual rows=996 loops=1) + Optimizer: Postgres query optimizer +(13 rows) select jsonb_pretty(explain_analyze_inc_sort_nodes_without_memory('select * from (select * from t order by a) s order by a, b limit 70')); - jsonb_pretty -------------------------------------------------- - [ + - { + - "Sort Key": [ + - "t.a", + - "t.b" + - ], + - "Node Type": "Incremental Sort", + - "Actual Rows": 70, + - "Actual Loops": 1, + - "Presorted Key": [ + - "t.a" + - ], + - "Parallel Aware": false, + - "Full-sort Groups": { + - "Group Count": 1, + - "Sort Methods Used": [ + - "quicksort" + - ], + - "Sort Space Memory": { + - "Peak Sort Space Used": "NN", + - "Average Sort Space Used": "NN"+ - } + - }, + - "Pre-sorted Groups": { + - "Group Count": 5, + - "Sort Methods Used": [ + - "top-N heapsort", + - "quicksort" + - ], + - "Sort Space Memory": { + - "Peak Sort Space Used": "NN", + - "Average Sort Space Used": "NN"+ - } + - }, + - "Parent Relationship": "Outer" + - } + + jsonb_pretty +------------------------------------------ + [ + + { + + "Slice": 1, + + "Segments": 3, + + "Sort Key": [ + + "t.a", + + "t.b" + + ], + + "Gang Type": "primary reader", + + "Node Type": "Incremental Sort",+ + "Actual Rows": 70, + + "Actual Loops": 1, + + "Presorted Key": [ + + "t.a" + + ], + + "Parallel Aware": false, + + "Parent Relationship": "Outer" + + } + ] (1 row) @@ -775,16 +797,20 @@ delete from t; insert into t(a, b) select i / 10, i from generate_series(1, 1000) n(i); analyze t; explain (costs off) select * from (select * from t order by a) s order by a, b limit 31; - QUERY PLAN ---------------------------------- + QUERY PLAN +------------------------------------------------ Limit - -> Incremental Sort - Sort Key: t.a, t.b - Presorted Key: t.a - -> Sort - Sort Key: t.a - -> Seq Scan on t -(7 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a, t.b + -> Limit + -> Incremental Sort + Sort Key: t.a, t.b + Presorted Key: t.a + -> Sort + Sort Key: t.a + -> Seq Scan on t + Optimizer: Postgres query optimizer +(11 rows) select * from (select * from t order by a) s order by a, b limit 31; a | b @@ -823,16 +849,20 @@ select * from (select * from t order by a) s order by a, b limit 31; (31 rows) explain (costs off) select * from (select * from t order by a) s order by a, b limit 32; - QUERY PLAN ---------------------------------- + QUERY PLAN +------------------------------------------------ Limit - -> Incremental Sort - Sort Key: t.a, t.b - Presorted Key: t.a - -> Sort - Sort Key: t.a - -> Seq Scan on t -(7 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a, t.b + -> Limit + -> Incremental Sort + Sort Key: t.a, t.b + Presorted Key: t.a + -> Sort + Sort Key: t.a + -> Seq Scan on t + Optimizer: Postgres query optimizer +(11 rows) select * from (select * from t order by a) s order by a, b limit 32; a | b @@ -872,16 +902,20 @@ select * from (select * from t order by a) s order by a, b limit 32; (32 rows) explain (costs off) select * from (select * from t order by a) s order by a, b limit 33; - QUERY PLAN ---------------------------------- + QUERY PLAN +------------------------------------------------ Limit - -> Incremental Sort - Sort Key: t.a, t.b - Presorted Key: t.a - -> Sort - Sort Key: t.a - -> Seq Scan on t -(7 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a, t.b + -> Limit + -> Incremental Sort + Sort Key: t.a, t.b + Presorted Key: t.a + -> Sort + Sort Key: t.a + -> Seq Scan on t + Optimizer: Postgres query optimizer +(11 rows) select * from (select * from t order by a) s order by a, b limit 33; a | b @@ -922,16 +956,20 @@ select * from (select * from t order by a) s order by a, b limit 33; (33 rows) explain (costs off) select * from (select * from t order by a) s order by a, b limit 65; - QUERY PLAN ---------------------------------- + QUERY PLAN +------------------------------------------------ Limit - -> Incremental Sort - Sort Key: t.a, t.b - Presorted Key: t.a - -> Sort - Sort Key: t.a - -> Seq Scan on t -(7 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a, t.b + -> Limit + -> Incremental Sort + Sort Key: t.a, t.b + Presorted Key: t.a + -> Sort + Sort Key: t.a + -> Seq Scan on t + Optimizer: Postgres query optimizer +(11 rows) select * from (select * from t order by a) s order by a, b limit 65; a | b @@ -1004,16 +1042,20 @@ select * from (select * from t order by a) s order by a, b limit 65; (65 rows) explain (costs off) select * from (select * from t order by a) s order by a, b limit 66; - QUERY PLAN ---------------------------------- + QUERY PLAN +------------------------------------------------ Limit - -> Incremental Sort - Sort Key: t.a, t.b - Presorted Key: t.a - -> Sort - Sort Key: t.a - -> Seq Scan on t -(7 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a, t.b + -> Limit + -> Incremental Sort + Sort Key: t.a, t.b + Presorted Key: t.a + -> Sort + Sort Key: t.a + -> Seq Scan on t + Optimizer: Postgres query optimizer +(11 rows) select * from (select * from t order by a) s order by a, b limit 66; a | b @@ -1091,16 +1133,20 @@ delete from t; insert into t(a, b) select i, i from generate_series(1, 1000) n(i); analyze t; explain (costs off) select * from (select * from t order by a) s order by a, b limit 31; - QUERY PLAN ---------------------------------- + QUERY PLAN +------------------------------------------------ Limit - -> Incremental Sort - Sort Key: t.a, t.b - Presorted Key: t.a - -> Sort - Sort Key: t.a - -> Seq Scan on t -(7 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a, t.b + -> Limit + -> Incremental Sort + Sort Key: t.a, t.b + Presorted Key: t.a + -> Sort + Sort Key: t.a + -> Seq Scan on t + Optimizer: Postgres query optimizer +(11 rows) select * from (select * from t order by a) s order by a, b limit 31; a | b @@ -1139,16 +1185,20 @@ select * from (select * from t order by a) s order by a, b limit 31; (31 rows) explain (costs off) select * from (select * from t order by a) s order by a, b limit 32; - QUERY PLAN ---------------------------------- + QUERY PLAN +------------------------------------------------ Limit - -> Incremental Sort - Sort Key: t.a, t.b - Presorted Key: t.a - -> Sort - Sort Key: t.a - -> Seq Scan on t -(7 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a, t.b + -> Limit + -> Incremental Sort + Sort Key: t.a, t.b + Presorted Key: t.a + -> Sort + Sort Key: t.a + -> Seq Scan on t + Optimizer: Postgres query optimizer +(11 rows) select * from (select * from t order by a) s order by a, b limit 32; a | b @@ -1188,16 +1238,20 @@ select * from (select * from t order by a) s order by a, b limit 32; (32 rows) explain (costs off) select * from (select * from t order by a) s order by a, b limit 33; - QUERY PLAN ---------------------------------- + QUERY PLAN +------------------------------------------------ Limit - -> Incremental Sort - Sort Key: t.a, t.b - Presorted Key: t.a - -> Sort - Sort Key: t.a - -> Seq Scan on t -(7 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a, t.b + -> Limit + -> Incremental Sort + Sort Key: t.a, t.b + Presorted Key: t.a + -> Sort + Sort Key: t.a + -> Seq Scan on t + Optimizer: Postgres query optimizer +(11 rows) select * from (select * from t order by a) s order by a, b limit 33; a | b @@ -1238,16 +1292,20 @@ select * from (select * from t order by a) s order by a, b limit 33; (33 rows) explain (costs off) select * from (select * from t order by a) s order by a, b limit 65; - QUERY PLAN ---------------------------------- + QUERY PLAN +------------------------------------------------ Limit - -> Incremental Sort - Sort Key: t.a, t.b - Presorted Key: t.a - -> Sort - Sort Key: t.a - -> Seq Scan on t -(7 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a, t.b + -> Limit + -> Incremental Sort + Sort Key: t.a, t.b + Presorted Key: t.a + -> Sort + Sort Key: t.a + -> Seq Scan on t + Optimizer: Postgres query optimizer +(11 rows) select * from (select * from t order by a) s order by a, b limit 65; a | b @@ -1320,16 +1378,20 @@ select * from (select * from t order by a) s order by a, b limit 65; (65 rows) explain (costs off) select * from (select * from t order by a) s order by a, b limit 66; - QUERY PLAN ---------------------------------- + QUERY PLAN +------------------------------------------------ Limit - -> Incremental Sort - Sort Key: t.a, t.b - Presorted Key: t.a - -> Sort - Sort Key: t.a - -> Seq Scan on t -(7 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a, t.b + -> Limit + -> Incremental Sort + Sort Key: t.a, t.b + Presorted Key: t.a + -> Sort + Sort Key: t.a + -> Seq Scan on t + Optimizer: Postgres query optimizer +(11 rows) select * from (select * from t order by a) s order by a, b limit 66; a | b @@ -1411,62 +1473,133 @@ set parallel_setup_cost = 0; set parallel_tuple_cost = 0; set max_parallel_workers_per_gather = 2; create table t (a int, b int, c int); -insert into t select mod(i,10),mod(i,10),i from generate_series(1,10000) s(i); +insert into t select mod(i,30),mod(i,30),i from generate_series(1,30000) s(i); create index on t (a); analyze t; set enable_incremental_sort = off; explain (costs off) select a,b,sum(c) from t group by 1,2 order by 1,2,3 limit 1; - QUERY PLAN ------------------------------------------------------- + QUERY PLAN +------------------------------------------------ Limit - -> Sort - Sort Key: a, b, (sum(c)) - -> Finalize HashAggregate - Group Key: a, b - -> Gather - Workers Planned: 2 - -> Partial HashAggregate + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b, (sum(c)) + -> Limit + -> Sort + Sort Key: a, b, (sum(c)) + -> HashAggregate Group Key: a, b - -> Parallel Seq Scan on t + -> Seq Scan on t + Optimizer: Postgres query optimizer (10 rows) set enable_incremental_sort = on; explain (costs off) select a,b,sum(c) from t group by 1,2 order by 1,2,3 limit 1; - QUERY PLAN ----------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------- Limit - -> Incremental Sort - Sort Key: a, b, (sum(c)) - Presorted Key: a, b - -> GroupAggregate - Group Key: a, b - -> Gather Merge - Workers Planned: 2 - -> Incremental Sort - Sort Key: a, b - Presorted Key: a - -> Parallel Index Scan using t_a_idx on t -(12 rows) + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b, (sum(c)) + -> Limit + -> Incremental Sort + Sort Key: a, b, (sum(c)) + Presorted Key: a, b + -> GroupAggregate + Group Key: a, b + -> Incremental Sort + Sort Key: a, b + Presorted Key: a + -> Index Scan using t_a_idx on t + Optimizer: Postgres query optimizer +(14 rows) -- Incremental sort vs. set operations with varno 0 set enable_hashagg to off; explain (costs off) select * from t union select * from t order by 1,3; - QUERY PLAN ----------------------------------------------------------- - Incremental Sort - Sort Key: t.a, t.c - Presorted Key: t.a - -> Unique - -> Sort - Sort Key: t.a, t.b, t.c - -> Append - -> Gather - Workers Planned: 2 - -> Parallel Seq Scan on t - -> Gather - Workers Planned: 2 - -> Parallel Seq Scan on t t_1 -(13 rows) + QUERY PLAN +------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a, t.c + -> Incremental Sort + Sort Key: t.a, t.c + Presorted Key: t.a + -> Unique + Group Key: t.a, t.b, t.c + -> Sort + Sort Key: t.a, t.b, t.c + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: t.a, t.b, t.c + -> Append + -> Seq Scan on t + -> Seq Scan on t t_1 + Optimizer: Postgres query optimizer +(15 rows) + +drop table t; +-- Incremental sort shall not be an input for redistribute motion +create table t1 (a int, b int) distributed randomly; +create index ON t1 (a); +-- 10 times +insert into t1 select a, a from generate_series(1, 10000) cross join generate_series(1, 10) as gs(a); +analyze t1; +set enable_hashagg = off; +explain select a, b, count(*) as res from t1 group by a, b; + QUERY PLAN +------------------------------------------------------------------------------------------- + Finalize GroupAggregate (cost=2874.45..3216.03 rows=100 width=16) + Group Key: a, b + -> Gather Motion 3:1 (slice1; segments: 3) (cost=2874.45..3212.78 rows=300 width=16) + Merge Key: a, b + -> Partial GroupAggregate (cost=2874.45..3208.78 rows=100 width=16) + Group Key: a, b + -> Sort (cost=2874.45..2957.78 rows=33333 width=8) + Sort Key: a, b + -> Seq Scan on t1 (cost=0.00..370.33 rows=33333 width=8) + Optimizer: Postgres query optimizer +(10 rows) + +drop table t1; +-- Rescan over incremental sort +create table t (a int, b int, c int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +insert into t select mod(i,30),mod(i,30),i from generate_series(1,30000) s(i); +create index on t (a); +analyze t; +explain (costs off) with cte as ( + select * from t order by a +) +select a, b from t where t.a = (select b from cte where cte.b >= t.b order by a, b limit 1) order by a limit 1; + QUERY PLAN +----------------------------------------------------------------------------------------------------- + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a + -> Limit + -> Index Scan using t_a_idx on t + Filter: (a = (SubPlan 1)) + SubPlan 1 + -> Limit + -> Incremental Sort + Sort Key: t_1.a, t_1.b + Presorted Key: t_1.a + -> Result + Filter: (t_1.b >= t.b) + -> Sort + Sort Key: t_1.a + -> Materialize + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Seq Scan on t t_1 + Optimizer: Postgres query optimizer +(19 rows) + +with cte as ( + select * from t order by a +) +select a, b from t where t.a = (select b from cte where cte.b >= t.b order by a, b limit 1) order by a limit 1; + a | b +---+--- + 0 | 0 +(1 row) drop table t; -- Sort pushdown can't go below where expressions are part of the rel target. @@ -1483,87 +1616,96 @@ set min_parallel_index_scan_size = 0; -- Parallel sort below join. explain (costs off) select distinct sub.unique1, stringu1 from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub; - QUERY PLAN --------------------------------------------------------------------------- - Unique - -> Nested Loop - -> Gather Merge - Workers Planned: 2 - -> Sort - Sort Key: tenk1.unique1, tenk1.stringu1 - -> Parallel Index Scan using tenk1_unique1 on tenk1 - -> Function Scan on generate_series -(8 rows) + QUERY PLAN +----------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: tenk1.unique1, tenk1.stringu1 + -> Unique + Group Key: tenk1.unique1, tenk1.stringu1 + -> Sort + Sort Key: tenk1.unique1, tenk1.stringu1 + -> Nested Loop + -> Index Scan using tenk1_unique1 on tenk1 + -> Function Scan on generate_series + Optimizer: Postgres query optimizer +(10 rows) explain (costs off) select sub.unique1, stringu1 from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub order by 1, 2; - QUERY PLAN --------------------------------------------------------------------- - Nested Loop - -> Gather Merge - Workers Planned: 2 - -> Sort - Sort Key: tenk1.unique1, tenk1.stringu1 - -> Parallel Index Scan using tenk1_unique1 on tenk1 - -> Function Scan on generate_series -(7 rows) + QUERY PLAN +----------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: tenk1.unique1, tenk1.stringu1 + -> Sort + Sort Key: tenk1.unique1, tenk1.stringu1 + -> Nested Loop + -> Index Scan using tenk1_unique1 on tenk1 + -> Function Scan on generate_series + Optimizer: Postgres query optimizer +(8 rows) -- Parallel sort but with expression that can be safely generated at the base rel. explain (costs off) select distinct sub.unique1, md5(stringu1) from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub; - QUERY PLAN ----------------------------------------------------------------------------------------- - Unique - -> Nested Loop - -> Gather Merge - Workers Planned: 2 - -> Sort - Sort Key: tenk1.unique1, (md5((tenk1.stringu1)::text)) COLLATE "C" - -> Parallel Index Scan using tenk1_unique1 on tenk1 - -> Function Scan on generate_series -(8 rows) - -explain (costs off) select sub.unique1, md5(stringu1) -from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub -order by 1, 2; QUERY PLAN ---------------------------------------------------------------------------------- - Nested Loop - -> Gather Merge - Workers Planned: 2 + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: tenk1.unique1, (md5((tenk1.stringu1)::text)) + -> Unique + Group Key: tenk1.unique1, (md5((tenk1.stringu1)::text)) -> Sort Sort Key: tenk1.unique1, (md5((tenk1.stringu1)::text)) COLLATE "C" - -> Parallel Index Scan using tenk1_unique1 on tenk1 - -> Function Scan on generate_series -(7 rows) + -> Nested Loop + -> Index Scan using tenk1_unique1 on tenk1 + -> Function Scan on generate_series + Optimizer: Postgres query optimizer +(10 rows) + +explain (costs off) select sub.unique1, md5(stringu1) +from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub +order by 1, 2; + QUERY PLAN +---------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: tenk1.unique1, (md5((tenk1.stringu1)::text)) + -> Sort + Sort Key: tenk1.unique1, (md5((tenk1.stringu1)::text)) COLLATE "C" + -> Nested Loop + -> Index Scan using tenk1_unique1 on tenk1 + -> Function Scan on generate_series + Optimizer: Postgres query optimizer +(8 rows) -- Parallel sort but with expression not available until the upper rel. explain (costs off) select distinct sub.unique1, stringu1 || random()::text from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub; - QUERY PLAN ---------------------------------------------------------------------------------------------- - Unique - -> Sort - Sort Key: tenk1.unique1, (((tenk1.stringu1)::text || (random())::text)) COLLATE "C" - -> Gather - Workers Planned: 2 + QUERY PLAN +--------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: tenk1.unique1, (((tenk1.stringu1)::text || (random())::text)) + -> Unique + Group Key: tenk1.unique1, (((tenk1.stringu1)::text || (random())::text)) + -> Sort + Sort Key: tenk1.unique1, (((tenk1.stringu1)::text || (random())::text)) COLLATE "C" -> Nested Loop - -> Parallel Index Scan using tenk1_unique1 on tenk1 + -> Index Scan using tenk1_unique1 on tenk1 -> Function Scan on generate_series -(8 rows) + Optimizer: Postgres query optimizer +(10 rows) explain (costs off) select sub.unique1, stringu1 || random()::text from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub order by 1, 2; - QUERY PLAN ---------------------------------------------------------------------------------------- - Sort - Sort Key: tenk1.unique1, (((tenk1.stringu1)::text || (random())::text)) COLLATE "C" - -> Gather - Workers Planned: 2 + QUERY PLAN +--------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: tenk1.unique1, (((tenk1.stringu1)::text || (random())::text)) + -> Sort + Sort Key: tenk1.unique1, (((tenk1.stringu1)::text || (random())::text)) COLLATE "C" -> Nested Loop - -> Parallel Index Scan using tenk1_unique1 on tenk1 + -> Index Scan using tenk1_unique1 on tenk1 -> Function Scan on generate_series -(7 rows) + Optimizer: Postgres query optimizer +(8 rows) diff --git a/src/test/regress/expected/incremental_sort_optimizer.out b/src/test/regress/expected/incremental_sort_optimizer.out new file mode 100644 index 000000000000..00de1c72d0db --- /dev/null +++ b/src/test/regress/expected/incremental_sort_optimizer.out @@ -0,0 +1,1601 @@ +-- When we have to sort the entire table, incremental sort will +-- be slower than plain sort, so it should not be used. +explain (costs off) +select * from (select * from tenk1 order by four) t order by four, ten; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: four, ten + -> Sort + Sort Key: four, ten + -> Seq Scan on tenk1 + Optimizer: Pivotal Optimizer (GPORCA) +(6 rows) + +-- When there is a LIMIT clause, incremental sort is beneficial because +-- it only has to sort some of the groups, and not the entire table. +explain (costs off) +select * from (select * from tenk1 order by four) t order by four, ten +limit 1; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: four, ten + -> Limit + -> Sort + Sort Key: four, ten + -> Seq Scan on tenk1 + Optimizer: Pivotal Optimizer (GPORCA) +(8 rows) + +-- When working memory is not enough to sort the entire table, incremental sort +-- may be faster if individual groups still fit into work_mem. +set planner_work_mem=280; +explain (costs off) +select * from (select * from tenk1 order by four) t order by four, ten; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: four, ten + -> Sort + Sort Key: four, ten + -> Seq Scan on tenk1 + Optimizer: Pivotal Optimizer (GPORCA) +(6 rows) + +reset planner_work_mem; +create table t(a integer, b integer); +create or replace function explain_analyze_without_memory(query text) +returns table (out_line text) language plpgsql +as +$$ +declare + line text; +begin + for line in + execute 'explain (analyze, costs off, summary off, timing off) ' || query + loop + out_line := regexp_replace(line, '\d+kB', 'NNkB', 'g'); + return next; + end loop; +end; +$$; +create or replace function explain_analyze_inc_sort_nodes(query text) +returns jsonb language plpgsql +as +$$ +declare + elements jsonb; + element jsonb; + matching_nodes jsonb := '[]'::jsonb; +begin + execute 'explain (analyze, costs off, summary off, timing off, format ''json'') ' || query into strict elements; + while jsonb_array_length(elements) > 0 loop + element := elements->0; + elements := elements - 0; + case jsonb_typeof(element) + when 'array' then + if jsonb_array_length(element) > 0 then + elements := elements || element; + end if; + when 'object' then + if element ? 'Plan' then + elements := elements || jsonb_build_array(element->'Plan'); + element := element - 'Plan'; + else + if element ? 'Plans' then + elements := elements || jsonb_build_array(element->'Plans'); + element := element - 'Plans'; + end if; + if (element->>'Node Type')::text = 'Incremental Sort' then + matching_nodes := matching_nodes || element; + end if; + end if; + end case; + end loop; + return matching_nodes; +end; +$$; +create or replace function explain_analyze_inc_sort_nodes_without_memory(query text) +returns jsonb language plpgsql +as +$$ +declare + nodes jsonb := '[]'::jsonb; + node jsonb; + group_key text; + space_key text; +begin + for node in select * from jsonb_array_elements(explain_analyze_inc_sort_nodes(query)) t loop + for group_key in select unnest(array['Full-sort Groups', 'Pre-sorted Groups']::text[]) t loop + for space_key in select unnest(array['Sort Space Memory', 'Sort Space Disk']::text[]) t loop + node := jsonb_set(node, array[group_key, space_key, 'Average Sort Space Used'], '"NN"', false); + node := jsonb_set(node, array[group_key, space_key, 'Peak Sort Space Used'], '"NN"', false); + end loop; + end loop; + nodes := nodes || node; + end loop; + return nodes; +end; +$$; +create or replace function explain_analyze_inc_sort_nodes_verify_invariants(query text) +returns bool language plpgsql +as +$$ +declare + node jsonb; + group_stats jsonb; + group_key text; + space_key text; +begin + for node in select * from jsonb_array_elements(explain_analyze_inc_sort_nodes(query)) t loop + for group_key in select unnest(array['Full-sort Groups', 'Pre-sorted Groups']::text[]) t loop + group_stats := node->group_key; + for space_key in select unnest(array['Sort Space Memory', 'Sort Space Disk']::text[]) t loop + if (group_stats->space_key->'Peak Sort Space Used')::bigint < (group_stats->space_key->'Peak Sort Space Used')::bigint then + raise exception '% has invalid max space < average space', group_key; + end if; + end loop; + end loop; + end loop; + return true; +end; +$$; +-- A single large group tested around each mode transition point. +insert into t(a, b) select i/100 + 1, i + 1 from generate_series(0, 999) n(i); +analyze t; +explain (costs off) select * from (select * from t order by a) s order by a, b limit 31; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b + -> Limit + -> Sort + Sort Key: a, b + -> Seq Scan on t + Optimizer: Pivotal Optimizer (GPORCA) +(8 rows) + +select * from (select * from t order by a) s order by a, b limit 31; + a | b +---+---- + 1 | 1 + 1 | 2 + 1 | 3 + 1 | 4 + 1 | 5 + 1 | 6 + 1 | 7 + 1 | 8 + 1 | 9 + 1 | 10 + 1 | 11 + 1 | 12 + 1 | 13 + 1 | 14 + 1 | 15 + 1 | 16 + 1 | 17 + 1 | 18 + 1 | 19 + 1 | 20 + 1 | 21 + 1 | 22 + 1 | 23 + 1 | 24 + 1 | 25 + 1 | 26 + 1 | 27 + 1 | 28 + 1 | 29 + 1 | 30 + 1 | 31 +(31 rows) + +explain (costs off) select * from (select * from t order by a) s order by a, b limit 32; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b + -> Limit + -> Sort + Sort Key: a, b + -> Seq Scan on t + Optimizer: Pivotal Optimizer (GPORCA) +(8 rows) + +select * from (select * from t order by a) s order by a, b limit 32; + a | b +---+---- + 1 | 1 + 1 | 2 + 1 | 3 + 1 | 4 + 1 | 5 + 1 | 6 + 1 | 7 + 1 | 8 + 1 | 9 + 1 | 10 + 1 | 11 + 1 | 12 + 1 | 13 + 1 | 14 + 1 | 15 + 1 | 16 + 1 | 17 + 1 | 18 + 1 | 19 + 1 | 20 + 1 | 21 + 1 | 22 + 1 | 23 + 1 | 24 + 1 | 25 + 1 | 26 + 1 | 27 + 1 | 28 + 1 | 29 + 1 | 30 + 1 | 31 + 1 | 32 +(32 rows) + +explain (costs off) select * from (select * from t order by a) s order by a, b limit 33; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b + -> Limit + -> Sort + Sort Key: a, b + -> Seq Scan on t + Optimizer: Pivotal Optimizer (GPORCA) +(8 rows) + +select * from (select * from t order by a) s order by a, b limit 33; + a | b +---+---- + 1 | 1 + 1 | 2 + 1 | 3 + 1 | 4 + 1 | 5 + 1 | 6 + 1 | 7 + 1 | 8 + 1 | 9 + 1 | 10 + 1 | 11 + 1 | 12 + 1 | 13 + 1 | 14 + 1 | 15 + 1 | 16 + 1 | 17 + 1 | 18 + 1 | 19 + 1 | 20 + 1 | 21 + 1 | 22 + 1 | 23 + 1 | 24 + 1 | 25 + 1 | 26 + 1 | 27 + 1 | 28 + 1 | 29 + 1 | 30 + 1 | 31 + 1 | 32 + 1 | 33 +(33 rows) + +explain (costs off) select * from (select * from t order by a) s order by a, b limit 65; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b + -> Limit + -> Sort + Sort Key: a, b + -> Seq Scan on t + Optimizer: Pivotal Optimizer (GPORCA) +(8 rows) + +select * from (select * from t order by a) s order by a, b limit 65; + a | b +---+---- + 1 | 1 + 1 | 2 + 1 | 3 + 1 | 4 + 1 | 5 + 1 | 6 + 1 | 7 + 1 | 8 + 1 | 9 + 1 | 10 + 1 | 11 + 1 | 12 + 1 | 13 + 1 | 14 + 1 | 15 + 1 | 16 + 1 | 17 + 1 | 18 + 1 | 19 + 1 | 20 + 1 | 21 + 1 | 22 + 1 | 23 + 1 | 24 + 1 | 25 + 1 | 26 + 1 | 27 + 1 | 28 + 1 | 29 + 1 | 30 + 1 | 31 + 1 | 32 + 1 | 33 + 1 | 34 + 1 | 35 + 1 | 36 + 1 | 37 + 1 | 38 + 1 | 39 + 1 | 40 + 1 | 41 + 1 | 42 + 1 | 43 + 1 | 44 + 1 | 45 + 1 | 46 + 1 | 47 + 1 | 48 + 1 | 49 + 1 | 50 + 1 | 51 + 1 | 52 + 1 | 53 + 1 | 54 + 1 | 55 + 1 | 56 + 1 | 57 + 1 | 58 + 1 | 59 + 1 | 60 + 1 | 61 + 1 | 62 + 1 | 63 + 1 | 64 + 1 | 65 +(65 rows) + +explain (costs off) select * from (select * from t order by a) s order by a, b limit 66; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b + -> Limit + -> Sort + Sort Key: a, b + -> Seq Scan on t + Optimizer: Pivotal Optimizer (GPORCA) +(8 rows) + +select * from (select * from t order by a) s order by a, b limit 66; + a | b +---+---- + 1 | 1 + 1 | 2 + 1 | 3 + 1 | 4 + 1 | 5 + 1 | 6 + 1 | 7 + 1 | 8 + 1 | 9 + 1 | 10 + 1 | 11 + 1 | 12 + 1 | 13 + 1 | 14 + 1 | 15 + 1 | 16 + 1 | 17 + 1 | 18 + 1 | 19 + 1 | 20 + 1 | 21 + 1 | 22 + 1 | 23 + 1 | 24 + 1 | 25 + 1 | 26 + 1 | 27 + 1 | 28 + 1 | 29 + 1 | 30 + 1 | 31 + 1 | 32 + 1 | 33 + 1 | 34 + 1 | 35 + 1 | 36 + 1 | 37 + 1 | 38 + 1 | 39 + 1 | 40 + 1 | 41 + 1 | 42 + 1 | 43 + 1 | 44 + 1 | 45 + 1 | 46 + 1 | 47 + 1 | 48 + 1 | 49 + 1 | 50 + 1 | 51 + 1 | 52 + 1 | 53 + 1 | 54 + 1 | 55 + 1 | 56 + 1 | 57 + 1 | 58 + 1 | 59 + 1 | 60 + 1 | 61 + 1 | 62 + 1 | 63 + 1 | 64 + 1 | 65 + 1 | 66 +(66 rows) + +delete from t; +-- An initial large group followed by a small group. +insert into t(a, b) select i/50 + 1, i + 1 from generate_series(0, 999) n(i); +analyze t; +explain (costs off) select * from (select * from t order by a) s order by a, b limit 55; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b + -> Limit + -> Sort + Sort Key: a, b + -> Seq Scan on t + Optimizer: Pivotal Optimizer (GPORCA) +(8 rows) + +select * from (select * from t order by a) s order by a, b limit 55; + a | b +---+---- + 1 | 1 + 1 | 2 + 1 | 3 + 1 | 4 + 1 | 5 + 1 | 6 + 1 | 7 + 1 | 8 + 1 | 9 + 1 | 10 + 1 | 11 + 1 | 12 + 1 | 13 + 1 | 14 + 1 | 15 + 1 | 16 + 1 | 17 + 1 | 18 + 1 | 19 + 1 | 20 + 1 | 21 + 1 | 22 + 1 | 23 + 1 | 24 + 1 | 25 + 1 | 26 + 1 | 27 + 1 | 28 + 1 | 29 + 1 | 30 + 1 | 31 + 1 | 32 + 1 | 33 + 1 | 34 + 1 | 35 + 1 | 36 + 1 | 37 + 1 | 38 + 1 | 39 + 1 | 40 + 1 | 41 + 1 | 42 + 1 | 43 + 1 | 44 + 1 | 45 + 1 | 46 + 1 | 47 + 1 | 48 + 1 | 49 + 1 | 50 + 2 | 51 + 2 | 52 + 2 | 53 + 2 | 54 + 2 | 55 +(55 rows) + +-- Test EXPLAIN ANALYZE with only a fullsort group. +select explain_analyze_without_memory('select * from (select * from t order by a) s order by a, b limit 55'); + explain_analyze_without_memory +------------------------------------------------------------------------------- + Limit (actual rows=55 loops=1) + -> Gather Motion 3:1 (slice1; segments: 3) (actual rows=55 loops=1) + Merge Key: a, b + -> Limit (actual rows=55 loops=1) + -> Sort (actual rows=55 loops=1) + Sort Key: a, b + Sort Method: top-N heapsort Memory: NNkB + Executor Memory: NNkB Segments: 3 Max: NNkB (segment 0) + -> Seq Scan on t (actual rows=400 loops=1) + Optimizer: Pivotal Optimizer (GPORCA) +(10 rows) + +select jsonb_pretty(explain_analyze_inc_sort_nodes_without_memory('select * from (select * from t order by a) s order by a, b limit 55')); + jsonb_pretty +-------------- + [ + + ] +(1 row) + +select explain_analyze_inc_sort_nodes_verify_invariants('select * from (select * from t order by a) s order by a, b limit 55'); + explain_analyze_inc_sort_nodes_verify_invariants +-------------------------------------------------- + t +(1 row) + +delete from t; +-- An initial small group followed by a large group. +insert into t(a, b) select (case when i < 5 then i else 9 end), i from generate_series(1, 1000) n(i); +analyze t; +explain (costs off) select * from (select * from t order by a) s order by a, b limit 70; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b + -> Limit + -> Sort + Sort Key: a, b + -> Seq Scan on t + Optimizer: Pivotal Optimizer (GPORCA) +(8 rows) + +select * from (select * from t order by a) s order by a, b limit 70; + a | b +---+---- + 1 | 1 + 2 | 2 + 3 | 3 + 4 | 4 + 9 | 5 + 9 | 6 + 9 | 7 + 9 | 8 + 9 | 9 + 9 | 10 + 9 | 11 + 9 | 12 + 9 | 13 + 9 | 14 + 9 | 15 + 9 | 16 + 9 | 17 + 9 | 18 + 9 | 19 + 9 | 20 + 9 | 21 + 9 | 22 + 9 | 23 + 9 | 24 + 9 | 25 + 9 | 26 + 9 | 27 + 9 | 28 + 9 | 29 + 9 | 30 + 9 | 31 + 9 | 32 + 9 | 33 + 9 | 34 + 9 | 35 + 9 | 36 + 9 | 37 + 9 | 38 + 9 | 39 + 9 | 40 + 9 | 41 + 9 | 42 + 9 | 43 + 9 | 44 + 9 | 45 + 9 | 46 + 9 | 47 + 9 | 48 + 9 | 49 + 9 | 50 + 9 | 51 + 9 | 52 + 9 | 53 + 9 | 54 + 9 | 55 + 9 | 56 + 9 | 57 + 9 | 58 + 9 | 59 + 9 | 60 + 9 | 61 + 9 | 62 + 9 | 63 + 9 | 64 + 9 | 65 + 9 | 66 + 9 | 67 + 9 | 68 + 9 | 69 + 9 | 70 +(70 rows) + +-- Test rescan. +begin; +-- We force the planner to choose a plan with incremental sort on the right side +-- of a nested loop join node. That way we trigger the rescan code path. +set local enable_hashjoin = off; +set local enable_mergejoin = off; +set local enable_material = off; +set local enable_sort = off; +explain (costs off) select * from t left join (select * from (select * from t order by a) v order by a, b) s on s.a = t.a where t.a in (1, 2); + QUERY PLAN +------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Left Join + Hash Cond: (t.a = t_1.a) + -> Seq Scan on t + Filter: (a = ANY ('{1,2}'::integer[])) + -> Hash + -> Seq Scan on t t_1 + Filter: (a = ANY ('{1,2}'::integer[])) + Optimizer: Pivotal Optimizer (GPORCA) +(9 rows) + +select * from t left join (select * from (select * from t order by a) v order by a, b) s on s.a = t.a where t.a in (1, 2); -- order none + a | b | a | b +---+---+---+--- + 2 | 2 | 2 | 2 + 1 | 1 | 1 | 1 +(2 rows) + +rollback; +-- Test EXPLAIN ANALYZE with both fullsort and presorted groups. +select explain_analyze_without_memory('select * from (select * from t order by a) s order by a, b limit 70'); + explain_analyze_without_memory +------------------------------------------------------------------------------- + Limit (actual rows=70 loops=1) + -> Gather Motion 3:1 (slice1; segments: 3) (actual rows=70 loops=1) + Merge Key: a, b + -> Limit (actual rows=70 loops=1) + -> Sort (actual rows=70 loops=1) + Sort Key: a, b + Sort Method: top-N heapsort Memory: NNkB + Sort Method: quicksort Memory: NNkB + Executor Memory: NNkB Segments: 3 Max: NNkB (segment 2) + -> Seq Scan on t (actual rows=996 loops=1) + Optimizer: Pivotal Optimizer (GPORCA) +(11 rows) + +select jsonb_pretty(explain_analyze_inc_sort_nodes_without_memory('select * from (select * from t order by a) s order by a, b limit 70')); + jsonb_pretty +-------------- + [ + + ] +(1 row) + +select explain_analyze_inc_sort_nodes_verify_invariants('select * from (select * from t order by a) s order by a, b limit 70'); + explain_analyze_inc_sort_nodes_verify_invariants +-------------------------------------------------- + t +(1 row) + +delete from t; +-- Small groups of 10 tuples each tested around each mode transition point. +insert into t(a, b) select i / 10, i from generate_series(1, 1000) n(i); +analyze t; +explain (costs off) select * from (select * from t order by a) s order by a, b limit 31; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b + -> Limit + -> Sort + Sort Key: a, b + -> Seq Scan on t + Optimizer: Pivotal Optimizer (GPORCA) +(8 rows) + +select * from (select * from t order by a) s order by a, b limit 31; + a | b +---+---- + 0 | 1 + 0 | 2 + 0 | 3 + 0 | 4 + 0 | 5 + 0 | 6 + 0 | 7 + 0 | 8 + 0 | 9 + 1 | 10 + 1 | 11 + 1 | 12 + 1 | 13 + 1 | 14 + 1 | 15 + 1 | 16 + 1 | 17 + 1 | 18 + 1 | 19 + 2 | 20 + 2 | 21 + 2 | 22 + 2 | 23 + 2 | 24 + 2 | 25 + 2 | 26 + 2 | 27 + 2 | 28 + 2 | 29 + 3 | 30 + 3 | 31 +(31 rows) + +explain (costs off) select * from (select * from t order by a) s order by a, b limit 32; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b + -> Limit + -> Sort + Sort Key: a, b + -> Seq Scan on t + Optimizer: Pivotal Optimizer (GPORCA) +(8 rows) + +select * from (select * from t order by a) s order by a, b limit 32; + a | b +---+---- + 0 | 1 + 0 | 2 + 0 | 3 + 0 | 4 + 0 | 5 + 0 | 6 + 0 | 7 + 0 | 8 + 0 | 9 + 1 | 10 + 1 | 11 + 1 | 12 + 1 | 13 + 1 | 14 + 1 | 15 + 1 | 16 + 1 | 17 + 1 | 18 + 1 | 19 + 2 | 20 + 2 | 21 + 2 | 22 + 2 | 23 + 2 | 24 + 2 | 25 + 2 | 26 + 2 | 27 + 2 | 28 + 2 | 29 + 3 | 30 + 3 | 31 + 3 | 32 +(32 rows) + +explain (costs off) select * from (select * from t order by a) s order by a, b limit 33; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b + -> Limit + -> Sort + Sort Key: a, b + -> Seq Scan on t + Optimizer: Pivotal Optimizer (GPORCA) +(8 rows) + +select * from (select * from t order by a) s order by a, b limit 33; + a | b +---+---- + 0 | 1 + 0 | 2 + 0 | 3 + 0 | 4 + 0 | 5 + 0 | 6 + 0 | 7 + 0 | 8 + 0 | 9 + 1 | 10 + 1 | 11 + 1 | 12 + 1 | 13 + 1 | 14 + 1 | 15 + 1 | 16 + 1 | 17 + 1 | 18 + 1 | 19 + 2 | 20 + 2 | 21 + 2 | 22 + 2 | 23 + 2 | 24 + 2 | 25 + 2 | 26 + 2 | 27 + 2 | 28 + 2 | 29 + 3 | 30 + 3 | 31 + 3 | 32 + 3 | 33 +(33 rows) + +explain (costs off) select * from (select * from t order by a) s order by a, b limit 65; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b + -> Limit + -> Sort + Sort Key: a, b + -> Seq Scan on t + Optimizer: Pivotal Optimizer (GPORCA) +(8 rows) + +select * from (select * from t order by a) s order by a, b limit 65; + a | b +---+---- + 0 | 1 + 0 | 2 + 0 | 3 + 0 | 4 + 0 | 5 + 0 | 6 + 0 | 7 + 0 | 8 + 0 | 9 + 1 | 10 + 1 | 11 + 1 | 12 + 1 | 13 + 1 | 14 + 1 | 15 + 1 | 16 + 1 | 17 + 1 | 18 + 1 | 19 + 2 | 20 + 2 | 21 + 2 | 22 + 2 | 23 + 2 | 24 + 2 | 25 + 2 | 26 + 2 | 27 + 2 | 28 + 2 | 29 + 3 | 30 + 3 | 31 + 3 | 32 + 3 | 33 + 3 | 34 + 3 | 35 + 3 | 36 + 3 | 37 + 3 | 38 + 3 | 39 + 4 | 40 + 4 | 41 + 4 | 42 + 4 | 43 + 4 | 44 + 4 | 45 + 4 | 46 + 4 | 47 + 4 | 48 + 4 | 49 + 5 | 50 + 5 | 51 + 5 | 52 + 5 | 53 + 5 | 54 + 5 | 55 + 5 | 56 + 5 | 57 + 5 | 58 + 5 | 59 + 6 | 60 + 6 | 61 + 6 | 62 + 6 | 63 + 6 | 64 + 6 | 65 +(65 rows) + +explain (costs off) select * from (select * from t order by a) s order by a, b limit 66; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b + -> Limit + -> Sort + Sort Key: a, b + -> Seq Scan on t + Optimizer: Pivotal Optimizer (GPORCA) +(8 rows) + +select * from (select * from t order by a) s order by a, b limit 66; + a | b +---+---- + 0 | 1 + 0 | 2 + 0 | 3 + 0 | 4 + 0 | 5 + 0 | 6 + 0 | 7 + 0 | 8 + 0 | 9 + 1 | 10 + 1 | 11 + 1 | 12 + 1 | 13 + 1 | 14 + 1 | 15 + 1 | 16 + 1 | 17 + 1 | 18 + 1 | 19 + 2 | 20 + 2 | 21 + 2 | 22 + 2 | 23 + 2 | 24 + 2 | 25 + 2 | 26 + 2 | 27 + 2 | 28 + 2 | 29 + 3 | 30 + 3 | 31 + 3 | 32 + 3 | 33 + 3 | 34 + 3 | 35 + 3 | 36 + 3 | 37 + 3 | 38 + 3 | 39 + 4 | 40 + 4 | 41 + 4 | 42 + 4 | 43 + 4 | 44 + 4 | 45 + 4 | 46 + 4 | 47 + 4 | 48 + 4 | 49 + 5 | 50 + 5 | 51 + 5 | 52 + 5 | 53 + 5 | 54 + 5 | 55 + 5 | 56 + 5 | 57 + 5 | 58 + 5 | 59 + 6 | 60 + 6 | 61 + 6 | 62 + 6 | 63 + 6 | 64 + 6 | 65 + 6 | 66 +(66 rows) + +delete from t; +-- Small groups of only 1 tuple each tested around each mode transition point. +insert into t(a, b) select i, i from generate_series(1, 1000) n(i); +analyze t; +explain (costs off) select * from (select * from t order by a) s order by a, b limit 31; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b + -> Limit + -> Sort + Sort Key: a, b + -> Seq Scan on t + Optimizer: Pivotal Optimizer (GPORCA) +(8 rows) + +select * from (select * from t order by a) s order by a, b limit 31; + a | b +----+---- + 1 | 1 + 2 | 2 + 3 | 3 + 4 | 4 + 5 | 5 + 6 | 6 + 7 | 7 + 8 | 8 + 9 | 9 + 10 | 10 + 11 | 11 + 12 | 12 + 13 | 13 + 14 | 14 + 15 | 15 + 16 | 16 + 17 | 17 + 18 | 18 + 19 | 19 + 20 | 20 + 21 | 21 + 22 | 22 + 23 | 23 + 24 | 24 + 25 | 25 + 26 | 26 + 27 | 27 + 28 | 28 + 29 | 29 + 30 | 30 + 31 | 31 +(31 rows) + +explain (costs off) select * from (select * from t order by a) s order by a, b limit 32; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b + -> Limit + -> Sort + Sort Key: a, b + -> Seq Scan on t + Optimizer: Pivotal Optimizer (GPORCA) +(8 rows) + +select * from (select * from t order by a) s order by a, b limit 32; + a | b +----+---- + 1 | 1 + 2 | 2 + 3 | 3 + 4 | 4 + 5 | 5 + 6 | 6 + 7 | 7 + 8 | 8 + 9 | 9 + 10 | 10 + 11 | 11 + 12 | 12 + 13 | 13 + 14 | 14 + 15 | 15 + 16 | 16 + 17 | 17 + 18 | 18 + 19 | 19 + 20 | 20 + 21 | 21 + 22 | 22 + 23 | 23 + 24 | 24 + 25 | 25 + 26 | 26 + 27 | 27 + 28 | 28 + 29 | 29 + 30 | 30 + 31 | 31 + 32 | 32 +(32 rows) + +explain (costs off) select * from (select * from t order by a) s order by a, b limit 33; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b + -> Limit + -> Sort + Sort Key: a, b + -> Seq Scan on t + Optimizer: Pivotal Optimizer (GPORCA) +(8 rows) + +select * from (select * from t order by a) s order by a, b limit 33; + a | b +----+---- + 1 | 1 + 2 | 2 + 3 | 3 + 4 | 4 + 5 | 5 + 6 | 6 + 7 | 7 + 8 | 8 + 9 | 9 + 10 | 10 + 11 | 11 + 12 | 12 + 13 | 13 + 14 | 14 + 15 | 15 + 16 | 16 + 17 | 17 + 18 | 18 + 19 | 19 + 20 | 20 + 21 | 21 + 22 | 22 + 23 | 23 + 24 | 24 + 25 | 25 + 26 | 26 + 27 | 27 + 28 | 28 + 29 | 29 + 30 | 30 + 31 | 31 + 32 | 32 + 33 | 33 +(33 rows) + +explain (costs off) select * from (select * from t order by a) s order by a, b limit 65; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b + -> Limit + -> Sort + Sort Key: a, b + -> Seq Scan on t + Optimizer: Pivotal Optimizer (GPORCA) +(8 rows) + +select * from (select * from t order by a) s order by a, b limit 65; + a | b +----+---- + 1 | 1 + 2 | 2 + 3 | 3 + 4 | 4 + 5 | 5 + 6 | 6 + 7 | 7 + 8 | 8 + 9 | 9 + 10 | 10 + 11 | 11 + 12 | 12 + 13 | 13 + 14 | 14 + 15 | 15 + 16 | 16 + 17 | 17 + 18 | 18 + 19 | 19 + 20 | 20 + 21 | 21 + 22 | 22 + 23 | 23 + 24 | 24 + 25 | 25 + 26 | 26 + 27 | 27 + 28 | 28 + 29 | 29 + 30 | 30 + 31 | 31 + 32 | 32 + 33 | 33 + 34 | 34 + 35 | 35 + 36 | 36 + 37 | 37 + 38 | 38 + 39 | 39 + 40 | 40 + 41 | 41 + 42 | 42 + 43 | 43 + 44 | 44 + 45 | 45 + 46 | 46 + 47 | 47 + 48 | 48 + 49 | 49 + 50 | 50 + 51 | 51 + 52 | 52 + 53 | 53 + 54 | 54 + 55 | 55 + 56 | 56 + 57 | 57 + 58 | 58 + 59 | 59 + 60 | 60 + 61 | 61 + 62 | 62 + 63 | 63 + 64 | 64 + 65 | 65 +(65 rows) + +explain (costs off) select * from (select * from t order by a) s order by a, b limit 66; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b + -> Limit + -> Sort + Sort Key: a, b + -> Seq Scan on t + Optimizer: Pivotal Optimizer (GPORCA) +(8 rows) + +select * from (select * from t order by a) s order by a, b limit 66; + a | b +----+---- + 1 | 1 + 2 | 2 + 3 | 3 + 4 | 4 + 5 | 5 + 6 | 6 + 7 | 7 + 8 | 8 + 9 | 9 + 10 | 10 + 11 | 11 + 12 | 12 + 13 | 13 + 14 | 14 + 15 | 15 + 16 | 16 + 17 | 17 + 18 | 18 + 19 | 19 + 20 | 20 + 21 | 21 + 22 | 22 + 23 | 23 + 24 | 24 + 25 | 25 + 26 | 26 + 27 | 27 + 28 | 28 + 29 | 29 + 30 | 30 + 31 | 31 + 32 | 32 + 33 | 33 + 34 | 34 + 35 | 35 + 36 | 36 + 37 | 37 + 38 | 38 + 39 | 39 + 40 | 40 + 41 | 41 + 42 | 42 + 43 | 43 + 44 | 44 + 45 | 45 + 46 | 46 + 47 | 47 + 48 | 48 + 49 | 49 + 50 | 50 + 51 | 51 + 52 | 52 + 53 | 53 + 54 | 54 + 55 | 55 + 56 | 56 + 57 | 57 + 58 | 58 + 59 | 59 + 60 | 60 + 61 | 61 + 62 | 62 + 63 | 63 + 64 | 64 + 65 | 65 + 66 | 66 +(66 rows) + +delete from t; +drop table t; +-- Incremental sort vs. parallel queries +set min_parallel_table_scan_size = '1kB'; +set min_parallel_index_scan_size = '1kB'; +set parallel_setup_cost = 0; +set parallel_tuple_cost = 0; +set max_parallel_workers_per_gather = 2; +create table t (a int, b int, c int); +insert into t select mod(i,30),mod(i,30),i from generate_series(1,30000) s(i); +create index on t (a); +analyze t; +set enable_incremental_sort = off; +explain (costs off) select a,b,sum(c) from t group by 1,2 order by 1,2,3 limit 1; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b, (sum(c)) + -> Limit + -> Sort + Sort Key: a, b, (sum(c)) + -> HashAggregate + Group Key: a, b + -> Seq Scan on t + Optimizer: Pivotal Optimizer (GPORCA) +(10 rows) + +set enable_incremental_sort = on; +explain (costs off) select a,b,sum(c) from t group by 1,2 order by 1,2,3 limit 1; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: a, b, (sum(c)) + -> Limit + -> Sort + Sort Key: a, b, (sum(c)) + -> HashAggregate + Group Key: a, b + -> Seq Scan on t + Optimizer: Pivotal Optimizer (GPORCA) +(10 rows) + +-- Incremental sort vs. set operations with varno 0 +set enable_hashagg to off; +explain (costs off) select * from t union select * from t order by 1,3; + QUERY PLAN +------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a, t.c + -> GroupAggregate + Group Key: t.a, t.b, t.c + -> Sort + Sort Key: t.a, t.c, t.b + -> Append + -> Seq Scan on t + -> Seq Scan on t t_1 + Optimizer: Pivotal Optimizer (GPORCA) +(10 rows) + +drop table t; +-- Incremental sort shall not be an input for redistribute motion +create table t1 (a int, b int) distributed randomly; +create index ON t1 (a); +-- 10 times +insert into t1 select a, a from generate_series(1, 10000) cross join generate_series(1, 10) as gs(a); +analyze t1; +set enable_hashagg = off; +explain select a, b, count(*) as res from t1 group by a, b; + QUERY PLAN +-------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..440.47 rows=57 width=16) + -> Finalize HashAggregate (cost=0.00..440.47 rows=19 width=16) + Group Key: a, b + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..440.46 rows=19 width=16) + Hash Key: a, b + -> Streaming Partial HashAggregate (cost=0.00..440.46 rows=19 width=16) + Group Key: a, b + -> Seq Scan on t1 (cost=0.00..431.70 rows=33334 width=8) + Optimizer: Pivotal Optimizer (GPORCA) +(9 rows) + +drop table t1; +-- Rescan over incremental sort +create table t (a int, b int, c int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +insert into t select mod(i,30),mod(i,30),i from generate_series(1,30000) s(i); +create index on t (a); +analyze t; +explain (costs off) with cte as ( + select * from t order by a +) +select a, b from t where t.a = (select b from cte where cte.b >= t.b order by a, b limit 1) order by a limit 1; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Sort + Sort Key: t.a + -> Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: t.a + -> Limit + -> Sort + Sort Key: t.a + -> Seq Scan on t + Filter: (a = (SubPlan 1)) + SubPlan 1 + -> Result + -> Limit + -> Sort + Sort Key: t_1.a, t_1.b + -> Result + Filter: (t_1.b >= t.b) + -> Materialize + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Seq Scan on t t_1 + Optimizer: Pivotal Optimizer (GPORCA) +(21 rows) + +with cte as ( + select * from t order by a +) +select a, b from t where t.a = (select b from cte where cte.b >= t.b order by a, b limit 1) order by a limit 1; + a | b +---+--- + 0 | 0 +(1 row) + +drop table t; +-- Sort pushdown can't go below where expressions are part of the rel target. +-- In particular this is interesting for volatile expressions which have to +-- go above joins since otherwise we'll incorrectly use expression evaluations +-- across multiple rows. +set enable_hashagg=off; +set enable_seqscan=off; +set enable_incremental_sort = off; +set parallel_tuple_cost=0; +set parallel_setup_cost=0; +set min_parallel_table_scan_size = 0; +set min_parallel_index_scan_size = 0; +-- Parallel sort below join. +explain (costs off) select distinct sub.unique1, stringu1 +from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub; + QUERY PLAN +----------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: tenk1.unique1, tenk1.stringu1 + -> Unique + Group Key: tenk1.unique1, tenk1.stringu1 + -> Sort + Sort Key: tenk1.unique1, tenk1.stringu1 + -> Nested Loop + -> Index Scan using tenk1_unique1 on tenk1 + -> Function Scan on generate_series + Optimizer: Postgres query optimizer +(10 rows) + +explain (costs off) select sub.unique1, stringu1 +from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub +order by 1, 2; + QUERY PLAN +----------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: tenk1.unique1, tenk1.stringu1 + -> Sort + Sort Key: tenk1.unique1, tenk1.stringu1 + -> Nested Loop + -> Index Scan using tenk1_unique1 on tenk1 + -> Function Scan on generate_series + Optimizer: Postgres query optimizer +(8 rows) + +-- Parallel sort but with expression that can be safely generated at the base rel. +explain (costs off) select distinct sub.unique1, md5(stringu1) +from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub; + QUERY PLAN +---------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: tenk1.unique1, (md5((tenk1.stringu1)::text)) + -> Unique + Group Key: tenk1.unique1, (md5((tenk1.stringu1)::text)) + -> Sort + Sort Key: tenk1.unique1, (md5((tenk1.stringu1)::text)) COLLATE "C" + -> Nested Loop + -> Index Scan using tenk1_unique1 on tenk1 + -> Function Scan on generate_series + Optimizer: Postgres query optimizer +(10 rows) + +explain (costs off) select sub.unique1, md5(stringu1) +from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub +order by 1, 2; + QUERY PLAN +---------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: tenk1.unique1, (md5((tenk1.stringu1)::text)) + -> Sort + Sort Key: tenk1.unique1, (md5((tenk1.stringu1)::text)) COLLATE "C" + -> Nested Loop + -> Index Scan using tenk1_unique1 on tenk1 + -> Function Scan on generate_series + Optimizer: Postgres query optimizer +(8 rows) + +-- Parallel sort but with expression not available until the upper rel. +explain (costs off) select distinct sub.unique1, stringu1 || random()::text +from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub; + QUERY PLAN +--------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: tenk1.unique1, (((tenk1.stringu1)::text || (random())::text)) + -> Unique + Group Key: tenk1.unique1, (((tenk1.stringu1)::text || (random())::text)) + -> Sort + Sort Key: tenk1.unique1, (((tenk1.stringu1)::text || (random())::text)) COLLATE "C" + -> Nested Loop + -> Index Scan using tenk1_unique1 on tenk1 + -> Function Scan on generate_series + Optimizer: Postgres query optimizer +(10 rows) + +explain (costs off) select sub.unique1, stringu1 || random()::text +from tenk1, lateral (select tenk1.unique1 from generate_series(1, 1000)) as sub +order by 1, 2; + QUERY PLAN +--------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: tenk1.unique1, (((tenk1.stringu1)::text || (random())::text)) + -> Sort + Sort Key: tenk1.unique1, (((tenk1.stringu1)::text || (random())::text)) COLLATE "C" + -> Nested Loop + -> Index Scan using tenk1_unique1 on tenk1 + -> Function Scan on generate_series + Optimizer: Postgres query optimizer +(8 rows) + diff --git a/src/test/regress/expected/olap_window_seq.out b/src/test/regress/expected/olap_window_seq.out index 3ef84b8b4ce0..76763cc10a7e 100755 --- a/src/test/regress/expected/olap_window_seq.out +++ b/src/test/regress/expected/olap_window_seq.out @@ -8144,34 +8144,35 @@ EXPLAIN SELECT count(*) over (PARTITION BY a ORDER BY b, c, d) as count1, count(*) over (PARTITION BY a ORDER BY c, b) as count2, count(*) over (PARTITION BY a ORDER BY c, b, d) as count3 FROM foo; - QUERY PLAN ----------------------------------------------------------------------------------------------------------- - Gather Motion 3:1 (slice1; segments: 3) (cost=4.06..4.68 rows=10 width=16) - -> WindowAgg (cost=4.06..4.68 rows=4 width=16) + QUERY PLAN +---------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=1995.51..7105.97 rows=71100 width=64) + -> WindowAgg (cost=1995.51..6157.97 rows=23700 width=64) Partition By: a Order By: b - -> WindowAgg (cost=4.06..4.51 rows=4 width=16) + -> WindowAgg (cost=1995.51..5743.22 rows=23700 width=56) Partition By: a Order By: b, c - -> WindowAgg (cost=4.06..4.31 rows=4 width=16) + -> WindowAgg (cost=1995.51..5269.22 rows=23700 width=48) Partition By: a Order By: b, c, d - -> Sort (cost=4.06..4.08 rows=4 width=16) + -> Incremental Sort (cost=1995.51..4735.97 rows=23700 width=40) Sort Key: a, b, c, d - -> WindowAgg (cost=3.27..3.89 rows=4 width=16) + Presorted Key: a + -> WindowAgg (cost=1993.11..3474.36 rows=23700 width=40) Partition By: a Order By: c - -> WindowAgg (cost=3.27..3.72 rows=4 width=16) + -> WindowAgg (cost=1993.11..3059.61 rows=23700 width=32) Partition By: a Order By: c, b - -> WindowAgg (cost=3.27..3.52 rows=4 width=16) + -> WindowAgg (cost=1993.11..2585.61 rows=23700 width=24) Partition By: a Order By: c, b, d - -> Sort (cost=3.27..3.29 rows=4 width=16) + -> Sort (cost=1993.11..2052.36 rows=23700 width=16) Sort Key: a, c, b, d - -> Seq Scan on foo (cost=0.00..3.10 rows=4 width=16) + -> Seq Scan on foo (cost=0.00..271.00 rows=23700 width=16) Optimizer: Postgres query optimizer -(25 rows) +(26 rows) drop table foo; -- test predicate push down in subqueries for quals containing windowref nodes diff --git a/src/test/regress/expected/partition_pruning.out b/src/test/regress/expected/partition_pruning.out index 8e5b09b7b39a..4b485393c25d 100644 --- a/src/test/regress/expected/partition_pruning.out +++ b/src/test/regress/expected/partition_pruning.out @@ -424,13 +424,14 @@ EXPLAIN SELECT * FROM pt_lt_tab WHERE col2 < 10 ORDER BY col2,col3 LIMIT 5; Limit (cost=364.26..364.33 rows=5 width=12) -> Gather Motion 3:1 (slice1; segments: 3) (cost=364.26..364.34 rows=6 width=12) Merge Key: pt_lt_tab.col2, pt_lt_tab.col3 - -> Limit (cost=364.26..364.26 rows=2 width=12) - -> Sort (cost=364.26..364.26 rows=3 width=12) + -> Limit (cost=202.46..364.37 rows=2 width=12) + -> Incremental Sort (cost=121.51..364.37 rows=3 width=12) Sort Key: pt_lt_tab.col2, pt_lt_tab.col3 + Presorted Key: pt_lt_tab.col2 -> Index Scan using pt_lt_tab_1_prt_part1_col2_idx on pt_lt_tab_1_prt_part1 pt_lt_tab (cost=0.14..364.23 rows=3 width=12) Index Cond: (col2 < '10'::numeric) Optimizer: Postgres query optimizer -(9 rows) +(10 rows) SELECT * FROM pt_lt_tab WHERE col2 > 50 ORDER BY col2,col3 LIMIT 5; col1 | col2 | col3 | col4 @@ -484,10 +485,12 @@ EXPLAIN SELECT * FROM pt_lt_tab WHERE col2 <> 10 ORDER BY col2,col3 LIMIT 5; Limit (cost=2021.46..2021.53 rows=5 width=12) -> Gather Motion 3:1 (slice1; segments: 3) (cost=2021.46..2021.67 rows=15 width=12) Merge Key: pt_lt_tab.col2, pt_lt_tab.col3 - -> Limit (cost=2021.46..2021.47 rows=5 width=12) - -> Sort (cost=2021.46..2021.49 rows=15 width=12) + -> Limit (cost=324.12..953.01 rows=5 width=12) + -> Incremental Sort (cost=135.46..2022.12 rows=15 width=12) Sort Key: pt_lt_tab.col2, pt_lt_tab.col3 - -> Append (cost=0.14..2021.21 rows=15 width=12) + Presorted Key: pt_lt_tab.col2 + -> Merge Append (cost=0.73..2021.44 rows=15 width=12) + Sort Key: pt_lt_tab.col2 -> Index Scan using pt_lt_tab_1_prt_part1_col2_idx on pt_lt_tab_1_prt_part1 pt_lt_tab_1 (cost=0.14..404.23 rows=3 width=12) Filter: (col2 <> '10'::numeric) -> Index Scan using pt_lt_tab_1_prt_part2_col2_idx on pt_lt_tab_1_prt_part2 pt_lt_tab_2 (cost=0.14..404.23 rows=3 width=12) @@ -499,7 +502,7 @@ EXPLAIN SELECT * FROM pt_lt_tab WHERE col2 <> 10 ORDER BY col2,col3 LIMIT 5; -> Index Scan using pt_lt_tab_1_prt_part5_col2_idx on pt_lt_tab_1_prt_part5 pt_lt_tab_5 (cost=0.14..404.23 rows=3 width=12) Filter: (col2 <> '10'::numeric) Optimizer: Postgres query optimizer -(18 rows) +(20 rows) SELECT * FROM pt_lt_tab WHERE col2 > 10 AND col2 < 50 ORDER BY col2,col3 LIMIT 5; col1 | col2 | col3 | col4 @@ -517,10 +520,12 @@ EXPLAIN SELECT * FROM pt_lt_tab WHERE col2 > 10 AND col2 < 50 ORDER BY col2,col3 Limit (cost=1577.34..1577.41 rows=5 width=12) -> Gather Motion 3:1 (slice1; segments: 3) (cost=1577.34..1577.56 rows=15 width=12) Merge Key: pt_lt_tab.col2, pt_lt_tab.col3 - -> Limit (cost=1577.34..1577.36 rows=5 width=12) - -> Sort (cost=1577.34..1577.37 rows=13 width=12) + -> Limit (cost=267.69..828.72 rows=5 width=12) + -> Incremental Sort (cost=122.12..1577.88 rows=13 width=12) Sort Key: pt_lt_tab.col2, pt_lt_tab.col3 - -> Append (cost=0.14..1577.12 rows=13 width=12) + Presorted Key: pt_lt_tab.col2 + -> Merge Append (cost=0.58..1577.29 rows=13 width=12) + Sort Key: pt_lt_tab.col2 -> Index Scan using pt_lt_tab_1_prt_part2_col2_idx on pt_lt_tab_1_prt_part2 pt_lt_tab_1 (cost=0.14..404.27 rows=3 width=12) Index Cond: ((col2 > '10'::numeric) AND (col2 < '50'::numeric)) -> Index Scan using pt_lt_tab_1_prt_part3_col2_idx on pt_lt_tab_1_prt_part3 pt_lt_tab_2 (cost=0.14..404.27 rows=3 width=12) @@ -530,7 +535,7 @@ EXPLAIN SELECT * FROM pt_lt_tab WHERE col2 > 10 AND col2 < 50 ORDER BY col2,col3 -> Index Scan using pt_lt_tab_1_prt_part5_col2_idx on pt_lt_tab_1_prt_part5 pt_lt_tab_4 (cost=0.14..364.25 rows=3 width=12) Index Cond: ((col2 > '10'::numeric) AND (col2 < '50'::numeric)) Optimizer: Postgres query optimizer -(16 rows) +(18 rows) SELECT * FROM pt_lt_tab WHERE col2 > 10 OR col2 = 50 ORDER BY col2,col3 LIMIT 5; col1 | col2 | col3 | col4 @@ -548,10 +553,12 @@ EXPLAIN SELECT * FROM pt_lt_tab WHERE col2 > 10 OR col2 = 50 ORDER BY col2,col3 Limit (cost=1617.23..1617.30 rows=5 width=12) -> Gather Motion 3:1 (slice1; segments: 3) (cost=1617.23..1617.44 rows=15 width=12) Merge Key: pt_lt_tab.col2, pt_lt_tab.col3 - -> Limit (cost=1617.23..1617.24 rows=5 width=12) - -> Sort (cost=1617.23..1617.26 rows=13 width=12) + -> Limit (cost=271.54..833.03 rows=5 width=12) + -> Incremental Sort (cost=121.96..1617.78 rows=13 width=12) Sort Key: pt_lt_tab.col2, pt_lt_tab.col3 - -> Append (cost=0.14..1617.01 rows=13 width=12) + Presorted Key: pt_lt_tab.col2 + -> Merge Append (cost=0.58..1617.18 rows=13 width=12) + Sort Key: pt_lt_tab.col2 -> Index Scan using pt_lt_tab_1_prt_part2_col2_idx on pt_lt_tab_1_prt_part2 pt_lt_tab_1 (cost=0.14..404.24 rows=3 width=12) Filter: ((col2 > '10'::numeric) OR (col2 = '50'::numeric)) -> Index Scan using pt_lt_tab_1_prt_part3_col2_idx on pt_lt_tab_1_prt_part3 pt_lt_tab_2 (cost=0.14..404.24 rows=3 width=12) @@ -561,7 +568,7 @@ EXPLAIN SELECT * FROM pt_lt_tab WHERE col2 > 10 OR col2 = 50 ORDER BY col2,col3 -> Index Scan using pt_lt_tab_1_prt_part5_col2_idx on pt_lt_tab_1_prt_part5 pt_lt_tab_4 (cost=0.14..404.24 rows=3 width=12) Filter: ((col2 > '10'::numeric) OR (col2 = '50'::numeric)) Optimizer: Postgres query optimizer -(16 rows) +(18 rows) SELECT * FROM pt_lt_tab WHERE col2 between 10 AND 50 ORDER BY col2,col3 LIMIT 5; col1 | col2 | col3 | col4 @@ -579,10 +586,12 @@ EXPLAIN SELECT * FROM pt_lt_tab WHERE col2 between 10 AND 50 ORDER BY col2,col3 Limit (cost=1661.54..1661.61 rows=5 width=12) -> Gather Motion 3:1 (slice1; segments: 3) (cost=1661.54..1661.75 rows=15 width=12) Merge Key: pt_lt_tab.col2, pt_lt_tab.col3 - -> Limit (cost=1661.54..1661.55 rows=5 width=12) - -> Sort (cost=1661.54..1661.57 rows=14 width=12) + -> Limit (cost=271.38..811.56 rows=5 width=12) + -> Incremental Sort (cost=116.85..1662.17 rows=14 width=12) Sort Key: pt_lt_tab.col2, pt_lt_tab.col3 - -> Append (cost=0.14..1661.30 rows=14 width=12) + Presorted Key: pt_lt_tab.col2 + -> Merge Append (cost=0.73..1661.52 rows=14 width=12) + Sort Key: pt_lt_tab.col2 -> Index Scan using pt_lt_tab_1_prt_part1_col2_idx on pt_lt_tab_1_prt_part1 pt_lt_tab_1 (cost=0.14..44.15 rows=1 width=12) Index Cond: ((col2 >= '10'::numeric) AND (col2 <= '50'::numeric)) -> Index Scan using pt_lt_tab_1_prt_part2_col2_idx on pt_lt_tab_1_prt_part2 pt_lt_tab_2 (cost=0.14..404.27 rows=3 width=12) @@ -594,7 +603,7 @@ EXPLAIN SELECT * FROM pt_lt_tab WHERE col2 between 10 AND 50 ORDER BY col2,col3 -> Index Scan using pt_lt_tab_1_prt_part5_col2_idx on pt_lt_tab_1_prt_part5 pt_lt_tab_5 (cost=0.14..404.27 rows=3 width=12) Index Cond: ((col2 >= '10'::numeric) AND (col2 <= '50'::numeric)) Optimizer: Postgres query optimizer -(18 rows) +(20 rows) DROP INDEX idx1; -- @description multiple column b-tree index @@ -724,10 +733,12 @@ EXPLAIN SELECT * FROM pt_lt_tab WHERE col2 > 10 AND col1 = 10 ORDER BY col2,col3 Limit (cost=176.68..176.75 rows=5 width=12) -> Gather Motion 1:1 (slice1; segments: 1) (cost=176.68..176.81 rows=9 width=12) Merge Key: pt_lt_tab.col2, pt_lt_tab.col3 - -> Limit (cost=176.68..176.69 rows=3 width=12) - -> Sort (cost=176.68..176.69 rows=4 width=12) + -> Limit (cost=77.69..176.90 rows=3 width=12) + -> Incremental Sort (cost=44.62..176.90 rows=4 width=12) Sort Key: pt_lt_tab.col2, pt_lt_tab.col3 - -> Append (cost=0.14..176.64 rows=4 width=12) + Presorted Key: pt_lt_tab.col2 + -> Merge Append (cost=0.58..176.72 rows=4 width=12) + Sort Key: pt_lt_tab.col2 -> Index Scan using pt_lt_tab_1_prt_part2_col1_col2_idx on pt_lt_tab_1_prt_part2 pt_lt_tab_1 (cost=0.14..44.15 rows=1 width=12) Index Cond: ((col1 = 10) AND (col2 > '10'::numeric)) -> Index Scan using pt_lt_tab_1_prt_part3_col1_col2_idx on pt_lt_tab_1_prt_part3 pt_lt_tab_2 (cost=0.14..44.15 rows=1 width=12) @@ -737,7 +748,7 @@ EXPLAIN SELECT * FROM pt_lt_tab WHERE col2 > 10 AND col1 = 10 ORDER BY col2,col3 -> Index Scan using pt_lt_tab_1_prt_part5_col1_col2_idx on pt_lt_tab_1_prt_part5 pt_lt_tab_4 (cost=0.14..44.15 rows=1 width=12) Index Cond: ((col1 = 10) AND (col2 > '10'::numeric)) Optimizer: Postgres query optimizer -(16 rows) +(18 rows) SELECT * FROM pt_lt_tab WHERE col2 > 10.00 OR col1 = 50 ORDER BY col2,col3 LIMIT 5; col1 | col2 | col3 | col4 @@ -851,10 +862,12 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col1 < 10 ORDER BY col2,col3 LIMIT 5; Limit (cost=2425.52..2425.60 rows=5 width=12) -> Gather Motion 3:1 (slice1; segments: 3) (cost=2425.52..2425.74 rows=15 width=12) Merge Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Limit (cost=2425.52..2425.54 rows=5 width=12) - -> Sort (cost=2425.52..2425.54 rows=8 width=12) + -> Limit (cost=569.23..1895.45 rows=5 width=12) + -> Incremental Sort (cost=303.98..2425.94 rows=8 width=12) Sort Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Append (cost=0.14..2425.40 rows=8 width=12) + Presorted Key: pt_lt_tab_df.col2 + -> Merge Append (cost=0.89..2425.58 rows=8 width=12) + Sort Key: pt_lt_tab_df.col2 -> Index Scan using pt_lt_tab_df_1_prt_part1_col2_col1_key on pt_lt_tab_df_1_prt_part1 pt_lt_tab_df_1 (cost=0.14..404.24 rows=3 width=12) Index Cond: (col1 < 10) -> Index Scan using pt_lt_tab_df_1_prt_part2_col2_col1_key on pt_lt_tab_df_1_prt_part2 pt_lt_tab_df_2 (cost=0.14..404.22 rows=1 width=12) @@ -868,7 +881,7 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col1 < 10 ORDER BY col2,col3 LIMIT 5; -> Index Scan using pt_lt_tab_df_1_prt_def_col2_col1_key on pt_lt_tab_df_1_prt_def pt_lt_tab_df_6 (cost=0.14..404.24 rows=1 width=12) Index Cond: (col1 < 10) Optimizer: Postgres query optimizer -(20 rows) +(22 rows) SELECT * FROM pt_lt_tab_df WHERE col1 > 50 ORDER BY col2,col3 LIMIT 5; col1 | col2 | col3 | col4 @@ -886,10 +899,12 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col1 > 50 ORDER BY col2,col3 LIMIT 5; Limit (cost=2425.54..2425.61 rows=5 width=12) -> Gather Motion 3:1 (slice1; segments: 3) (cost=2425.54..2425.75 rows=15 width=12) Merge Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Limit (cost=2425.54..2425.55 rows=5 width=12) - -> Sort (cost=2425.53..2425.56 rows=8 width=12) + -> Limit (cost=548.18..1829.14 rows=5 width=12) + -> Incremental Sort (cost=291.99..2425.97 rows=8 width=12) Sort Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Append (cost=0.14..2425.41 rows=8 width=12) + Presorted Key: pt_lt_tab_df.col2 + -> Merge Append (cost=0.89..2425.59 rows=8 width=12) + Sort Key: pt_lt_tab_df.col2 -> Index Scan using pt_lt_tab_df_1_prt_part1_col2_col1_key on pt_lt_tab_df_1_prt_part1 pt_lt_tab_df_1 (cost=0.14..404.22 rows=1 width=12) Index Cond: (col1 > 50) -> Index Scan using pt_lt_tab_df_1_prt_part2_col2_col1_key on pt_lt_tab_df_1_prt_part2 pt_lt_tab_df_2 (cost=0.14..404.22 rows=1 width=12) @@ -903,7 +918,7 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col1 > 50 ORDER BY col2,col3 LIMIT 5; -> Index Scan using pt_lt_tab_df_1_prt_def_col2_col1_key on pt_lt_tab_df_1_prt_def pt_lt_tab_df_6 (cost=0.14..404.27 rows=3 width=12) Index Cond: (col1 > 50) Optimizer: Postgres query optimizer -(20 rows) +(22 rows) SELECT * FROM pt_lt_tab_df WHERE col2 = 25 ORDER BY col2,col3 LIMIT 5; col1 | col2 | col3 | col4 @@ -941,10 +956,12 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 <> 10 ORDER BY col2,col3 LIMIT 5; Limit (cost=2425.78..2425.85 rows=5 width=12) -> Gather Motion 3:1 (slice1; segments: 3) (cost=2425.78..2425.99 rows=15 width=12) Merge Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Limit (cost=2425.78..2425.79 rows=5 width=12) - -> Sort (cost=2425.78..2425.82 rows=18 width=12) + -> Limit (cost=364.71..1001.10 rows=5 width=12) + -> Incremental Sort (cost=135.61..2426.60 rows=18 width=12) Sort Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Append (cost=0.14..2425.48 rows=18 width=12) + Presorted Key: pt_lt_tab_df.col2 + -> Merge Append (cost=0.89..2425.79 rows=18 width=12) + Sort Key: pt_lt_tab_df.col2 -> Index Scan using pt_lt_tab_df_1_prt_part1_col2_col1_key on pt_lt_tab_df_1_prt_part1 pt_lt_tab_df_1 (cost=0.14..404.23 rows=3 width=12) Filter: (col2 <> '10'::numeric) -> Index Scan using pt_lt_tab_df_1_prt_part2_col2_col1_key on pt_lt_tab_df_1_prt_part2 pt_lt_tab_df_2 (cost=0.14..404.23 rows=3 width=12) @@ -958,7 +975,7 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 <> 10 ORDER BY col2,col3 LIMIT 5; -> Index Scan using pt_lt_tab_df_1_prt_def_col2_col1_key on pt_lt_tab_df_1_prt_def pt_lt_tab_df_6 (cost=0.14..404.25 rows=3 width=12) Filter: (col2 <> '10'::numeric) Optimizer: Postgres query optimizer -(20 rows) +(22 rows) SELECT * FROM pt_lt_tab_df WHERE col2 > 10 AND col1 = 10 ORDER BY col2,col3 LIMIT 5; col1 | col2 | col3 | col4 @@ -971,10 +988,12 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 > 10 AND col1 = 10 ORDER BY col2,c Limit (cost=1929.31..1929.38 rows=5 width=12) -> Gather Motion 1:1 (slice1; segments: 1) (cost=1929.31..1929.48 rows=12 width=12) Merge Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Limit (cost=1929.31..1929.32 rows=4 width=12) - -> Sort (cost=1929.31..1929.32 rows=5 width=12) + -> Limit (cost=695.09..1929.59 rows=4 width=12) + -> Incremental Sort (cost=386.47..1929.59 rows=5 width=12) Sort Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Append (cost=0.14..1929.25 rows=5 width=12) + Presorted Key: pt_lt_tab_df.col2 + -> Merge Append (cost=0.73..1929.37 rows=5 width=12) + Sort Key: pt_lt_tab_df.col2 -> Index Scan using pt_lt_tab_df_1_prt_part2_col2_col1_key on pt_lt_tab_df_1_prt_part2 pt_lt_tab_df_1 (cost=0.14..404.25 rows=1 width=12) Index Cond: ((col2 > '10'::numeric) AND (col1 = 10)) -> Index Scan using pt_lt_tab_df_1_prt_part3_col2_col1_key on pt_lt_tab_df_1_prt_part3 pt_lt_tab_df_2 (cost=0.14..404.25 rows=1 width=12) @@ -986,7 +1005,7 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 > 10 AND col1 = 10 ORDER BY col2,c -> Index Scan using pt_lt_tab_df_1_prt_def_col2_col1_key on pt_lt_tab_df_1_prt_def pt_lt_tab_df_5 (cost=0.14..312.25 rows=1 width=12) Index Cond: ((col2 > '10'::numeric) AND (col1 = 10)) Optimizer: Postgres query optimizer -(18 rows) +(20 rows) SELECT * FROM pt_lt_tab_df WHERE col2 > 10.00 OR col1 = 50 ORDER BY col2,col3 LIMIT 5; col1 | col2 | col3 | col4 @@ -1004,10 +1023,12 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 > 10.00 OR col1 = 50 ORDER BY col2 Limit (cost=2425.83..2425.90 rows=5 width=12) -> Gather Motion 3:1 (slice1; segments: 3) (cost=2425.83..2426.04 rows=15 width=12) Merge Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Limit (cost=2425.83..2425.84 rows=5 width=12) - -> Sort (cost=2425.82..2425.87 rows=18 width=12) + -> Limit (cost=366.59..1012.20 rows=5 width=12) + -> Incremental Sort (cost=137.69..2426.63 rows=18 width=12) Sort Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Append (cost=0.14..2425.53 rows=18 width=12) + Presorted Key: pt_lt_tab_df.col2 + -> Merge Append (cost=0.89..2425.84 rows=18 width=12) + Sort Key: pt_lt_tab_df.col2 -> Index Scan using pt_lt_tab_df_1_prt_part1_col2_col1_key on pt_lt_tab_df_1_prt_part1 pt_lt_tab_df_1 (cost=0.14..404.24 rows=1 width=12) Filter: ((col2 > 10.00) OR (col1 = 50)) -> Index Scan using pt_lt_tab_df_1_prt_part2_col2_col1_key on pt_lt_tab_df_1_prt_part2 pt_lt_tab_df_2 (cost=0.14..404.24 rows=3 width=12) @@ -1021,7 +1042,7 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 > 10.00 OR col1 = 50 ORDER BY col2 -> Index Scan using pt_lt_tab_df_1_prt_def_col2_col1_key on pt_lt_tab_df_1_prt_def pt_lt_tab_df_6 (cost=0.14..404.26 rows=3 width=12) Filter: ((col2 > 10.00) OR (col1 = 50)) Optimizer: Postgres query optimizer -(20 rows) +(22 rows) SELECT * FROM pt_lt_tab_df WHERE col2 between 10 AND 50 ORDER BY col2,col3 LIMIT 5; col1 | col2 | col3 | col4 @@ -1039,10 +1060,12 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 between 10 AND 50 ORDER BY col2,co Limit (cost=1697.72..1697.79 rows=5 width=12) -> Gather Motion 3:1 (slice1; segments: 3) (cost=1697.72..1697.93 rows=15 width=12) Merge Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Limit (cost=1697.72..1697.73 rows=5 width=12) - -> Sort (cost=1697.71..1697.75 rows=15 width=12) + -> Limit (cost=270.44..788.83 rows=5 width=12) + -> Incremental Sort (cost=111.78..1698.42 rows=15 width=12) Sort Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Append (cost=0.14..1697.46 rows=15 width=12) + Presorted Key: pt_lt_tab_df.col2 + -> Merge Append (cost=0.89..1697.73 rows=15 width=12) + Sort Key: pt_lt_tab_df.col2 -> Index Scan using pt_lt_tab_df_1_prt_part1_col2_col1_key on pt_lt_tab_df_1_prt_part1 pt_lt_tab_df_1 (cost=0.14..44.15 rows=1 width=12) Index Cond: ((col2 >= '10'::numeric) AND (col2 <= '50'::numeric)) -> Index Scan using pt_lt_tab_df_1_prt_part2_col2_col1_key on pt_lt_tab_df_1_prt_part2 pt_lt_tab_df_2 (cost=0.14..404.27 rows=3 width=12) @@ -1056,7 +1079,7 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 between 10 AND 50 ORDER BY col2,co -> Index Scan using pt_lt_tab_df_1_prt_def_col2_col1_key on pt_lt_tab_df_1_prt_def pt_lt_tab_df_6 (cost=0.14..36.15 rows=1 width=12) Index Cond: ((col2 >= '10'::numeric) AND (col2 <= '50'::numeric)) Optimizer: Postgres query optimizer -(20 rows) +(22 rows) ALTER TABLE pt_lt_tab_df DROP CONSTRAINT col2_col1_unique; -- @description Heterogeneous index, index on partition key, b-tree index on all partitions @@ -1081,10 +1104,12 @@ EXPLAIN SELECT * FROM pt_lt_tab WHERE col2 between 1 AND 50 ORDER BY col2,col3 L Limit (cost=2021.70..2021.78 rows=5 width=12) -> Gather Motion 3:1 (slice1; segments: 3) (cost=2021.70..2021.92 rows=15 width=12) Merge Key: pt_lt_tab.col2, pt_lt_tab.col3 - -> Limit (cost=2021.70..2021.72 rows=5 width=12) - -> Sort (cost=2021.70..2021.74 rows=17 width=12) + -> Limit (cost=312.29..883.60 rows=5 width=12) + -> Incremental Sort (cost=122.27..2022.42 rows=17 width=12) Sort Key: pt_lt_tab.col2, pt_lt_tab.col3 - -> Append (cost=0.14..2021.42 rows=17 width=12) + Presorted Key: pt_lt_tab.col2 + -> Merge Append (cost=0.73..2021.68 rows=17 width=12) + Sort Key: pt_lt_tab.col2 -> Index Scan using idx1 on pt_lt_tab_1_prt_part1 pt_lt_tab_1 (cost=0.14..404.27 rows=3 width=12) Index Cond: ((col2 >= '1'::numeric) AND (col2 <= '50'::numeric)) -> Index Scan using idx2 on pt_lt_tab_1_prt_part2 pt_lt_tab_2 (cost=0.14..404.27 rows=3 width=12) @@ -1096,7 +1121,7 @@ EXPLAIN SELECT * FROM pt_lt_tab WHERE col2 between 1 AND 50 ORDER BY col2,col3 L -> Index Scan using idx5 on pt_lt_tab_1_prt_part5 pt_lt_tab_5 (cost=0.14..404.27 rows=3 width=12) Index Cond: ((col2 >= '1'::numeric) AND (col2 <= '50'::numeric)) Optimizer: Postgres query optimizer -(18 rows) +(20 rows) SELECT * FROM pt_lt_tab WHERE col2 > 5 ORDER BY col2,col3 LIMIT 5; col1 | col2 | col3 | col4 @@ -1114,10 +1139,12 @@ EXPLAIN SELECT * FROM pt_lt_tab WHERE col2 > 5 ORDER BY col2,col3 LIMIT 5; Limit (cost=1861.50..1861.57 rows=5 width=12) -> Gather Motion 3:1 (slice1; segments: 3) (cost=1861.50..1861.72 rows=15 width=12) Merge Key: pt_lt_tab.col2, pt_lt_tab.col3 - -> Limit (cost=1861.50..1861.52 rows=5 width=12) - -> Sort (cost=1861.50..1861.54 rows=15 width=12) + -> Limit (cost=297.28..870.36 rows=5 width=12) + -> Incremental Sort (cost=123.40..1862.16 rows=15 width=12) Sort Key: pt_lt_tab.col2, pt_lt_tab.col3 - -> Append (cost=0.14..1861.25 rows=15 width=12) + Presorted Key: pt_lt_tab.col2 + -> Merge Append (cost=0.73..1861.48 rows=15 width=12) + Sort Key: pt_lt_tab.col2 -> Index Scan using idx1 on pt_lt_tab_1_prt_part1 pt_lt_tab_1 (cost=0.14..244.20 rows=2 width=12) Index Cond: (col2 > '5'::numeric) -> Index Scan using idx2 on pt_lt_tab_1_prt_part2 pt_lt_tab_2 (cost=0.14..404.24 rows=3 width=12) @@ -1129,7 +1156,7 @@ EXPLAIN SELECT * FROM pt_lt_tab WHERE col2 > 5 ORDER BY col2,col3 LIMIT 5; -> Index Scan using idx5 on pt_lt_tab_1_prt_part5 pt_lt_tab_5 (cost=0.14..404.24 rows=3 width=12) Index Cond: (col2 > '5'::numeric) Optimizer: Postgres query optimizer -(18 rows) +(20 rows) SELECT * FROM pt_lt_tab WHERE col2 = 5 ORDER BY col2,col3 LIMIT 5; col1 | col2 | col3 | col4 @@ -1294,10 +1321,12 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 between 1 AND 100 ORDER BY col2,co Limit (cost=2334.05..2334.12 rows=5 width=12) -> Gather Motion 3:1 (slice1; segments: 3) (cost=2334.05..2334.26 rows=15 width=12) Merge Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Limit (cost=2334.05..2334.06 rows=5 width=12) - -> Sort (cost=2334.04..2334.09 rows=20 width=12) + -> Limit (cost=339.53..895.04 rows=5 width=12) + -> Incremental Sort (cost=117.82..2334.94 rows=20 width=12) Sort Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Append (cost=0.14..2333.71 rows=20 width=12) + Presorted Key: pt_lt_tab_df.col2 + -> Merge Append (cost=0.89..2334.04 rows=20 width=12) + Sort Key: pt_lt_tab_df.col2 -> Index Scan using idx1 on pt_lt_tab_df_1_prt_part1 pt_lt_tab_df_1 (cost=0.14..404.27 rows=3 width=12) Index Cond: ((col2 >= '1'::numeric) AND (col2 <= '100'::numeric)) -> Index Scan using idx2 on pt_lt_tab_df_1_prt_part2 pt_lt_tab_df_2 (cost=0.14..404.27 rows=3 width=12) @@ -1311,7 +1340,7 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 between 1 AND 100 ORDER BY col2,co -> Index Scan using idx6 on pt_lt_tab_df_1_prt_def pt_lt_tab_df_6 (cost=0.14..312.27 rows=3 width=12) Index Cond: ((col2 >= '1'::numeric) AND (col2 <= '100'::numeric)) Optimizer: Postgres query optimizer -(20 rows) +(22 rows) SELECT * FROM pt_lt_tab_df WHERE col2 > 50 ORDER BY col2,col3 LIMIT 5; col1 | col2 | col3 | col4 @@ -1329,13 +1358,14 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 > 50 ORDER BY col2,col3 LIMIT 5; Limit (cost=312.27..312.35 rows=5 width=12) -> Gather Motion 3:1 (slice1; segments: 3) (cost=312.27..312.37 rows=7 width=12) Merge Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Limit (cost=312.27..312.28 rows=2 width=12) - -> Sort (cost=312.27..312.28 rows=3 width=12) + -> Limit (cost=159.51..312.39 rows=2 width=12) + -> Incremental Sort (cost=93.88..312.39 rows=3 width=12) Sort Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 + Presorted Key: pt_lt_tab_df.col2 -> Index Scan using idx6 on pt_lt_tab_df_1_prt_def pt_lt_tab_df (cost=0.14..312.24 rows=3 width=12) Index Cond: (col2 > '50'::numeric) Optimizer: Postgres query optimizer -(9 rows) +(10 rows) SELECT * FROM pt_lt_tab_df WHERE col2 = 50 ORDER BY col2,col3 LIMIT 5; col1 | col2 | col3 | col4 @@ -1373,10 +1403,12 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 <> 10 ORDER BY col2,col3 LIMIT 5; Limit (cost=2425.78..2425.85 rows=5 width=12) -> Gather Motion 3:1 (slice1; segments: 3) (cost=2425.78..2425.99 rows=15 width=12) Merge Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Limit (cost=2425.78..2425.79 rows=5 width=12) - -> Sort (cost=2425.78..2425.82 rows=18 width=12) + -> Limit (cost=364.71..1001.10 rows=5 width=12) + -> Incremental Sort (cost=135.61..2426.60 rows=18 width=12) Sort Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Append (cost=0.14..2425.48 rows=18 width=12) + Presorted Key: pt_lt_tab_df.col2 + -> Merge Append (cost=0.89..2425.79 rows=18 width=12) + Sort Key: pt_lt_tab_df.col2 -> Index Scan using idx1 on pt_lt_tab_df_1_prt_part1 pt_lt_tab_df_1 (cost=0.14..404.23 rows=3 width=12) Filter: (col2 <> '10'::numeric) -> Index Scan using idx2 on pt_lt_tab_df_1_prt_part2 pt_lt_tab_df_2 (cost=0.14..404.23 rows=3 width=12) @@ -1390,7 +1422,7 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 <> 10 ORDER BY col2,col3 LIMIT 5; -> Index Scan using idx6 on pt_lt_tab_df_1_prt_def pt_lt_tab_df_6 (cost=0.14..404.25 rows=3 width=12) Filter: (col2 <> '10'::numeric) Optimizer: Postgres query optimizer -(20 rows) +(22 rows) SELECT * FROM pt_lt_tab_df WHERE col2 between 1 AND 100 ORDER BY col2,col3 LIMIT 5; col1 | col2 | col3 | col4 @@ -1408,10 +1440,12 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 between 1 AND 100 ORDER BY col2,co Limit (cost=2334.05..2334.12 rows=5 width=12) -> Gather Motion 3:1 (slice1; segments: 3) (cost=2334.05..2334.26 rows=15 width=12) Merge Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Limit (cost=2334.05..2334.06 rows=5 width=12) - -> Sort (cost=2334.04..2334.09 rows=20 width=12) + -> Limit (cost=339.53..895.04 rows=5 width=12) + -> Incremental Sort (cost=117.82..2334.94 rows=20 width=12) Sort Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Append (cost=0.14..2333.71 rows=20 width=12) + Presorted Key: pt_lt_tab_df.col2 + -> Merge Append (cost=0.89..2334.04 rows=20 width=12) + Sort Key: pt_lt_tab_df.col2 -> Index Scan using idx1 on pt_lt_tab_df_1_prt_part1 pt_lt_tab_df_1 (cost=0.14..404.27 rows=3 width=12) Index Cond: ((col2 >= '1'::numeric) AND (col2 <= '100'::numeric)) -> Index Scan using idx2 on pt_lt_tab_df_1_prt_part2 pt_lt_tab_df_2 (cost=0.14..404.27 rows=3 width=12) @@ -1425,7 +1459,7 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 between 1 AND 100 ORDER BY col2,co -> Index Scan using idx6 on pt_lt_tab_df_1_prt_def pt_lt_tab_df_6 (cost=0.14..312.27 rows=3 width=12) Index Cond: ((col2 >= '1'::numeric) AND (col2 <= '100'::numeric)) Optimizer: Postgres query optimizer -(20 rows) +(22 rows) SELECT * FROM pt_lt_tab_df WHERE col2 < 50 AND col1 > 10 ORDER BY col2,col3 LIMIT 5; col1 | col2 | col3 | col4 @@ -1443,10 +1477,12 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 < 50 AND col1 > 10 ORDER BY col2,c Limit (cost=2017.73..2017.80 rows=5 width=12) -> Gather Motion 3:1 (slice1; segments: 3) (cost=2017.73..2017.94 rows=15 width=12) Merge Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Limit (cost=2017.73..2017.74 rows=5 width=12) - -> Sort (cost=2017.73..2017.76 rows=15 width=12) + -> Limit (cost=323.81..952.21 rows=5 width=12) + -> Incremental Sort (cost=135.52..2018.42 rows=15 width=12) Sort Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Append (cost=0.14..2017.48 rows=15 width=12) + Presorted Key: pt_lt_tab_df.col2 + -> Merge Append (cost=0.89..2017.75 rows=15 width=12) + Sort Key: pt_lt_tab_df.col2 -> Index Scan using idx1 on pt_lt_tab_df_1_prt_part1 pt_lt_tab_df_1 (cost=0.14..404.25 rows=1 width=12) Index Cond: (col2 < '50'::numeric) Filter: (col1 > 10) @@ -1466,7 +1502,7 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 < 50 AND col1 > 10 ORDER BY col2,c Index Cond: (col2 < '50'::numeric) Filter: (col1 > 10) Optimizer: Postgres query optimizer -(26 rows) +(28 rows) DROP INDEX idx1; DROP INDEX idx2; @@ -1585,10 +1621,12 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 > 10 AND col1 between 1 AND 100 OR Limit (cost=1929.81..1929.88 rows=5 width=12) -> Gather Motion 3:1 (slice1; segments: 3) (cost=1929.81..1930.03 rows=15 width=12) Merge Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Limit (cost=1929.81..1929.83 rows=5 width=12) - -> Sort (cost=1929.81..1929.85 rows=16 width=12) + -> Limit (cost=300.81..859.29 rows=5 width=12) + -> Incremental Sort (cost=119.73..1930.52 rows=16 width=12) Sort Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Append (cost=0.14..1929.54 rows=16 width=12) + Presorted Key: pt_lt_tab_df.col2 + -> Merge Append (cost=0.73..1929.79 rows=16 width=12) + Sort Key: pt_lt_tab_df.col2 -> Index Scan using idx2 on pt_lt_tab_df_1_prt_part2 pt_lt_tab_df_1 (cost=0.14..404.29 rows=3 width=12) Index Cond: ((col2 > '10'::numeric) AND (col1 >= 1) AND (col1 <= 100)) -> Index Scan using idx3 on pt_lt_tab_df_1_prt_part3 pt_lt_tab_df_2 (cost=0.14..404.29 rows=3 width=12) @@ -1600,7 +1638,7 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 > 10 AND col1 between 1 AND 100 OR -> Index Scan using idx6 on pt_lt_tab_df_1_prt_def pt_lt_tab_df_5 (cost=0.14..312.29 rows=3 width=12) Index Cond: ((col2 > '10'::numeric) AND (col1 >= 1) AND (col1 <= 100)) Optimizer: Postgres query optimizer -(18 rows) +(20 rows) SELECT * FROM pt_lt_tab_df WHERE col1 = 10 ORDER BY col2,col3 LIMIT 5; col1 | col2 | col3 | col4 @@ -1614,10 +1652,12 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col1 = 10 ORDER BY col2,col3 LIMIT 5; Limit (cost=2425.45..2425.52 rows=5 width=12) -> Gather Motion 1:1 (slice1; segments: 1) (cost=2425.45..2425.67 rows=15 width=12) Merge Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Limit (cost=2425.45..2425.47 rows=5 width=12) - -> Sort (cost=2425.45..2425.47 rows=6 width=12) + -> Limit (cost=741.80..2425.80 rows=5 width=12) + -> Incremental Sort (cost=405.00..2425.80 rows=6 width=12) Sort Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Append (cost=0.14..2425.37 rows=6 width=12) + Presorted Key: pt_lt_tab_df.col2 + -> Merge Append (cost=0.89..2425.53 rows=6 width=12) + Sort Key: pt_lt_tab_df.col2 -> Index Scan using idx1 on pt_lt_tab_df_1_prt_part1 pt_lt_tab_df_1 (cost=0.14..404.22 rows=1 width=12) Index Cond: (col1 = 10) -> Index Scan using idx2 on pt_lt_tab_df_1_prt_part2 pt_lt_tab_df_2 (cost=0.14..404.22 rows=1 width=12) @@ -1631,7 +1671,7 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col1 = 10 ORDER BY col2,col3 LIMIT 5; -> Index Scan using idx6 on pt_lt_tab_df_1_prt_def pt_lt_tab_df_6 (cost=0.14..404.24 rows=1 width=12) Index Cond: (col1 = 10) Optimizer: Postgres query optimizer -(20 rows) +(22 rows) DROP INDEX idx1; DROP INDEX idx2; @@ -1686,22 +1726,30 @@ EXPLAIN SELECT * FROM pt_lt_tab_df WHERE col2 > 15 ORDER BY col2,col3 LIMIT 5; Limit (cost=30000000719.94..30000000720.01 rows=5 width=12) -> Gather Motion 3:1 (slice1; segments: 3) (cost=30000000719.94..30000000720.16 rows=15 width=12) Merge Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Limit (cost=30000000719.94..30000000719.96 rows=5 width=12) - -> Sort (cost=30000000719.94..30000000719.98 rows=15 width=12) + -> Limit (cost=30000000117.75..30000000338.56 rows=5 width=12) + -> Incremental Sort (cost=30000000050.76..30000000720.69 rows=15 width=12) Sort Key: pt_lt_tab_df.col2, pt_lt_tab_df.col3 - -> Append (cost=10000000000.00..30000000719.69 rows=15 width=12) - -> Seq Scan on pt_lt_tab_df_1_prt_part2 pt_lt_tab_df_1 (cost=10000000000.00..10000000001.04 rows=2 width=12) - Filter: (col2 > '15'::numeric) - -> Seq Scan on pt_lt_tab_df_1_prt_part3 pt_lt_tab_df_2 (cost=10000000000.00..10000000001.04 rows=3 width=12) - Filter: (col2 > '15'::numeric) - -> Seq Scan on pt_lt_tab_df_1_prt_part4 pt_lt_tab_df_3 (cost=10000000000.00..10000000001.04 rows=3 width=12) - Filter: (col2 > '15'::numeric) + Presorted Key: pt_lt_tab_df.col2 + -> Merge Append (cost=30000000003.52..30000000720.01 rows=15 width=12) + Sort Key: pt_lt_tab_df.col2 + -> Sort (cost=10000000001.05..10000000001.06 rows=2 width=12) + Sort Key: pt_lt_tab_df_1.col2 + -> Seq Scan on pt_lt_tab_df_1_prt_part2 pt_lt_tab_df_1 (cost=10000000000.00..10000000001.04 rows=2 width=12) + Filter: (col2 > '15'::numeric) + -> Sort (cost=10000000001.07..10000000001.08 rows=3 width=12) + Sort Key: pt_lt_tab_df_2.col2 + -> Seq Scan on pt_lt_tab_df_1_prt_part3 pt_lt_tab_df_2 (cost=10000000000.00..10000000001.04 rows=3 width=12) + Filter: (col2 > '15'::numeric) + -> Sort (cost=10000000001.07..10000000001.08 rows=3 width=12) + Sort Key: pt_lt_tab_df_3.col2 + -> Seq Scan on pt_lt_tab_df_1_prt_part4 pt_lt_tab_df_3 (cost=10000000000.00..10000000001.04 rows=3 width=12) + Filter: (col2 > '15'::numeric) -> Index Scan using idx5 on pt_lt_tab_df_1_prt_part5 pt_lt_tab_df_4 (cost=0.14..404.24 rows=3 width=12) Index Cond: (col2 > '15'::numeric) -> Index Scan using idx6 on pt_lt_tab_df_1_prt_def pt_lt_tab_df_5 (cost=0.14..312.24 rows=3 width=12) Index Cond: (col2 > '15'::numeric) Optimizer: Postgres query optimizer -(18 rows) +(26 rows) DROP INDEX idx1; DROP INDEX idx5; diff --git a/src/test/regress/expected/rangefuncs_cdb.out b/src/test/regress/expected/rangefuncs_cdb.out index 760ba2e3e7f2..a2550781fef6 100644 --- a/src/test/regress/expected/rangefuncs_cdb.out +++ b/src/test/regress/expected/rangefuncs_cdb.out @@ -6,7 +6,7 @@ SELECT name, setting FROM pg_settings WHERE name LIKE 'enable%'; enable_groupagg | on enable_hashagg | on enable_hashjoin | on - enable_incremental_sort | off + enable_incremental_sort | on enable_indexonlyscan | on enable_indexscan | on enable_material | on diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out index a787e969a14c..cf419e18d619 100644 --- a/src/test/regress/expected/sysviews.out +++ b/src/test/regress/expected/sysviews.out @@ -93,7 +93,7 @@ select name, setting from pg_settings where name like 'enable%'; enable_groupagg | on enable_hashagg | on enable_hashjoin | on - enable_incremental_sort | off + enable_incremental_sort | on enable_indexonlyscan | on enable_indexscan | on enable_material | on diff --git a/src/test/regress/expected/window.out b/src/test/regress/expected/window.out index b49ee54713b7..f4b42e8a0ec4 100644 --- a/src/test/regress/expected/window.out +++ b/src/test/regress/expected/window.out @@ -3301,8 +3301,9 @@ WHERE first_emp = 1 OR last_emp = 1; -> WindowAgg Partition By: empsalary.depname Order By: empsalary.enroll_date - -> Sort + -> Incremental Sort Sort Key: empsalary.depname, empsalary.enroll_date + Presorted Key: empsalary.depname -> WindowAgg Partition By: empsalary.depname Order By: empsalary.enroll_date @@ -3310,7 +3311,7 @@ WHERE first_emp = 1 OR last_emp = 1; Sort Key: empsalary.depname, empsalary.enroll_date DESC -> Seq Scan on empsalary Optimizer: Postgres query optimizer -(16 rows) +(17 rows) SELECT * FROM (SELECT depname, diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index e037dae2207f..f31bce18abcf 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -105,9 +105,7 @@ test: brin gin gist spgist privileges init_privs security_label collate matview # ---------- # Another group of parallel tests # ---------- -test: create_table_like alter_generic alter_operator misc async dbsize misc_functions sysviews tsrf tid tidscan collate.icu.utf8 -# GPDB_13_MERGE_FIXME: enable incremental sort -# incremental_sort +test: create_table_like alter_generic alter_operator misc async dbsize misc_functions sysviews tsrf tidscan collate.icu.utf8 incremental_sort # rules cannot run concurrently with any test that creates # a view or rule in the public schema diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule index 0c173c05407a..e4350fe7b1bb 100644 --- a/src/test/regress/serial_schedule +++ b/src/test/regress/serial_schedule @@ -94,8 +94,7 @@ test: select_distinct_on #test: select_implicit test: select_having test: subselect -# GPDB_13_MERGE_FIXME: enable incremental sort -# test: incremental_sort +test: incremental_sort test: union test: case test: join diff --git a/src/test/regress/sql/incremental_sort.sql b/src/test/regress/sql/incremental_sort.sql index 3ee11c394b91..7d61e98a0d88 100644 --- a/src/test/regress/sql/incremental_sort.sql +++ b/src/test/regress/sql/incremental_sort.sql @@ -9,12 +9,12 @@ explain (costs off) select * from (select * from tenk1 order by four) t order by four, ten limit 1; --- When work_mem is not enough to sort the entire table, incremental sort +-- When working memory is not enough to sort the entire table, incremental sort -- may be faster if individual groups still fit into work_mem. -set work_mem to '2MB'; +set planner_work_mem=280; explain (costs off) select * from (select * from tenk1 order by four) t order by four, ten; -reset work_mem; +reset planner_work_mem; create table t(a integer, b integer); @@ -158,7 +158,7 @@ set local enable_mergejoin = off; set local enable_material = off; set local enable_sort = off; explain (costs off) select * from t left join (select * from (select * from t order by a) v order by a, b) s on s.a = t.a where t.a in (1, 2); -select * from t left join (select * from (select * from t order by a) v order by a, b) s on s.a = t.a where t.a in (1, 2); +select * from t left join (select * from (select * from t order by a) v order by a, b) s on s.a = t.a where t.a in (1, 2); -- order none rollback; -- Test EXPLAIN ANALYZE with both fullsort and presorted groups. select explain_analyze_without_memory('select * from (select * from t order by a) s order by a, b limit 70'); @@ -206,7 +206,7 @@ set parallel_tuple_cost = 0; set max_parallel_workers_per_gather = 2; create table t (a int, b int, c int); -insert into t select mod(i,10),mod(i,10),i from generate_series(1,10000) s(i); +insert into t select mod(i,30),mod(i,30),i from generate_series(1,30000) s(i); create index on t (a); analyze t; @@ -222,6 +222,34 @@ explain (costs off) select * from t union select * from t order by 1,3; drop table t; +-- Incremental sort shall not be an input for redistribute motion +create table t1 (a int, b int) distributed randomly; +create index ON t1 (a); +-- 10 times +insert into t1 select a, a from generate_series(1, 10000) cross join generate_series(1, 10) as gs(a); +analyze t1; +set enable_hashagg = off; +explain select a, b, count(*) as res from t1 group by a, b; + +drop table t1; + +-- Rescan over incremental sort +create table t (a int, b int, c int); +insert into t select mod(i,30),mod(i,30),i from generate_series(1,30000) s(i); +create index on t (a); +analyze t; +explain (costs off) with cte as ( + select * from t order by a +) +select a, b from t where t.a = (select b from cte where cte.b >= t.b order by a, b limit 1) order by a limit 1; + +with cte as ( + select * from t order by a +) +select a, b from t where t.a = (select b from cte where cte.b >= t.b order by a, b limit 1) order by a limit 1; + +drop table t; + -- Sort pushdown can't go below where expressions are part of the rel target. -- In particular this is interesting for volatile expressions which have to -- go above joins since otherwise we'll incorrectly use expression evaluations