diff --git a/src/backend/executor/execHHashagg.c b/src/backend/executor/execHHashagg.c index a568aadbdb77c2ad72b29b240a658222f0b0e3b0..0953540b8659ff10739be9b2784f43de67f55f1a 100644 --- a/src/backend/executor/execHHashagg.c +++ b/src/backend/executor/execHHashagg.c @@ -31,7 +31,6 @@ #include "utils/elog.h" #include "cdb/memquota.h" #include "utils/workfile_mgr.h" -#include "utils/resource_manager.h" #include "access/hash.h" @@ -89,24 +88,8 @@ typedef enum InputRecordType Assert((hashtable)->mem_for_metadata > 0); \ Assert((hashtable)->mem_for_metadata > (hashtable)->nbuckets * OVERHEAD_PER_BUCKET); \ if ((hashtable)->mem_for_metadata >= (hashtable)->max_mem) \ - { \ - if (IsResGroupEnabled()) \ - { \ - elog(HHA_MSG_LVL, \ - "HashAgg: no enough operator memory for spilling: " \ - "operator memory is %.0f bytes, " \ - "current meta data is %.0f bytes; " \ - "the overuse is allowed in resource group mode", \ - (hashtable)->max_mem, \ - (hashtable)->mem_for_metadata); \ - } \ - else \ - { \ - ereport(ERROR, \ - (errcode(ERRCODE_INTERNAL_ERROR), \ - errmsg(ERRMSG_GP_INSUFFICIENT_STATEMENT_MEMORY))); \ - } \ - } \ + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), \ + errmsg(ERRMSG_GP_INSUFFICIENT_STATEMENT_MEMORY)));\ } while (0) #define GET_TOTAL_USED_SIZE(hashtable) \ @@ -572,7 +555,6 @@ calcHashAggTableSizes(double memquota, /* Memory quota in bytes. */ HashAggTableSizes *out_hats) { double entrysize, nbuckets, nentries; - double orig_memquota = memquota; /* Assume we don't need to spill */ bool expectSpill = false; @@ -586,9 +568,6 @@ calcHashAggTableSizes(double memquota, /* Memory quota in bytes. */ elog(HHA_MSG_LVL, "HashAgg: ngroups = %g, memquota = %g, entrysize = %g", ngroups, memquota, entrysize); - if (out_hats) - out_hats->memquota = memquota; - /* * When all groups can not fit in the memory, we compute * the number of batches to store spilled groups. Currently, we always @@ -600,15 +579,6 @@ calcHashAggTableSizes(double memquota, /* Memory quota in bytes. */ batchfile_mem = BATCHFILE_METADATA * (1 + nbatches); expectSpill = true; - /* In resource group the memory quota could be dynamically enlarged */ - if (IsResGroupEnabled() && memquota < batchfile_mem) - { - if (out_hats) - out_hats->memquota += batchfile_mem - memquota; - - memquota = batchfile_mem; - } - /* * If the memory quota is smaller than the overhead for batch files, * return false. Note that we will always keep at most (nbatches + 1) @@ -634,15 +604,6 @@ calcHashAggTableSizes(double memquota, /* Memory quota in bytes. */ nentries = Max(nentries, gp_hashagg_groups_per_bucket); entries_mem = nentries * entrywidth; - /* In resource group the memory quota could be dynamically enlarged */ - if (IsResGroupEnabled() && memquota < entries_mem) - { - if (out_hats) - out_hats->memquota += 1 + entries_mem - memquota; - - memquota = 1 + entries_mem; - } - /* * If the memory quota is smaller than the minimum number of entries * required, return false @@ -682,15 +643,6 @@ calcHashAggTableSizes(double memquota, /* Memory quota in bytes. */ nbuckets = Max(nbuckets, gp_hashagg_default_nbatches); buckets_mem = nbuckets * OVERHEAD_PER_BUCKET; - /* In resource group the memory quota could be dynamically enlarged */ - if (IsResGroupEnabled() && memquota < buckets_mem) - { - if (out_hats) - out_hats->memquota += buckets_mem - memquota; - - memquota = buckets_mem; - } - /* Reserve memory for the entries + hash table */ memquota -= buckets_mem; @@ -726,13 +678,6 @@ calcHashAggTableSizes(double memquota, /* Memory quota in bytes. */ out_hats->spill = expectSpill; out_hats->workmem_initial = (unsigned)(batchfile_mem); out_hats->workmem_per_entry = (unsigned) entrysize; - - if (IsResGroupEnabled() && out_hats->memquota > orig_memquota) - { - elog(HHA_MSG_LVL, - "HashAgg: auto enlarge operator memory from %.0f to %.0f in resource group mode", - out_hats->memquota, orig_memquota); - } } elog(HHA_MSG_LVL, "HashAgg: nbuckets = %d, nentries = %d, nbatches = %d", @@ -860,7 +805,7 @@ create_agg_hash_table(AggState *aggstate) MemoryContextSwitchTo(oldcxt); - hashtable->max_mem = hashtable->hats.memquota; + hashtable->max_mem = 1024.0 * operatorMemKB; hashtable->mem_for_metadata = sizeof(HashAggTable) + hashtable->nbuckets * OVERHEAD_PER_BUCKET + sizeof(GroupKeysAndAggs); @@ -2000,26 +1945,8 @@ reCalcNumberBatches(HashAggTable *hashtable, SpillFile *spill_file) if (hashtable->mem_for_metadata + nbatches * BATCHFILE_METADATA > hashtable->max_mem) - { - if (IsResGroupEnabled()) - { - elog(HHA_MSG_LVL, - "HashAgg: no enough operator memory for spilling: " - "operator memory is %.0f bytes, " - "current meta data is %.0f bytes, " - "need %lu bytes for %u more batches; " - "the overuse is allowed in resource group mode", - hashtable->max_mem, - hashtable->mem_for_metadata, - nbatches * BATCHFILE_METADATA, - nbatches); - } - else - { - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), - ERRMSG_GP_INSUFFICIENT_STATEMENT_MEMORY)); - } - } + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + ERRMSG_GP_INSUFFICIENT_STATEMENT_MEMORY)); hashtable->hats.nbatches = nbatches; } diff --git a/src/include/executor/execHHashagg.h b/src/include/executor/execHHashagg.h index a40fb8b1494e27c1cdadaf59392979b2ebe8518c..2ce05fcd579359828bc5191c94087586b0d0d678 100644 --- a/src/include/executor/execHHashagg.h +++ b/src/include/executor/execHHashagg.h @@ -126,7 +126,6 @@ typedef struct HashAggTableSizes unsigned workmem_initial; /* Estimated work_mem bytes at #entries=0 */ unsigned workmem_per_entry; /* Additional work_mem bytes per entry */ bool spill; /* Do we expect to spill ? */ - double memquota; /* Minimal required memquota */ } HashAggTableSizes; /* diff --git a/src/test/isolation2/expected/resgroup/resgroup_agghash_memory.out b/src/test/isolation2/expected/resgroup/resgroup_agghash_memory.out deleted file mode 100644 index 8aaac4b7f1e383ae8ed95f082a459121429f212e..0000000000000000000000000000000000000000 --- a/src/test/isolation2/expected/resgroup/resgroup_agghash_memory.out +++ /dev/null @@ -1,78 +0,0 @@ --- When an agghash operator begins to spill it needs 16KB memory as meta data --- for each batch file, when there are many batch files there might not be --- enough operator memory for all of them, in resource queue mode the query --- will fail with an error like below: --- --- ERROR: insufficient memory reserved for statement --- --- In resource group we allow this overuse as the shared memory is designed to --- serve this kind of overuse. - ---start_ignore -DROP TABLE IF EXISTS t1_agghash_mem_test; -DROP ROLE r1_agghash_mem_test; -DROP RESOURCE GROUP rg1_agghash_mem_test; ---end_ignore - -SET optimizer TO off; -SET - -CREATE TABLE t1_agghash_mem_test (c1 int, c2 text) DISTRIBUTED BY (c2); -CREATE -INSERT INTO t1_agghash_mem_test SELECT i, i::text FROM generate_series(1,100000) i; -INSERT 100000 - --- we must ensure spill to be small enough but still > 0. --- - rg1's memory quota is 682 * 1% = 6; --- - per-xact quota is 6/3=2; --- - spill memory is 2 * 60% = 1; -CREATE RESOURCE GROUP rg1_agghash_mem_test WITH (cpu_rate_limit=10, memory_limit=1, memory_shared_quota=0, concurrency=3, memory_spill_ratio=60); -CREATE - -CREATE ROLE r1_agghash_mem_test RESOURCE GROUP rg1_agghash_mem_test; -CREATE -GRANT ALL ON t1_agghash_mem_test TO r1_agghash_mem_test; -GRANT - -SET gp_resgroup_memory_policy TO none; -SET -SET ROLE TO r1_agghash_mem_test; -SET -WITH a AS ( SELECT DISTINCT c2 FROM t1_agghash_mem_test INTERSECT SELECT DISTINCT c2 FROM t1_agghash_mem_test ) SELECT count(*) FROM a; - count --------- - 100000 -(1 row) -RESET role; -RESET - -SET gp_resgroup_memory_policy TO auto; -SET -SET ROLE TO r1_agghash_mem_test; -SET -WITH a AS ( SELECT DISTINCT c2 FROM t1_agghash_mem_test INTERSECT SELECT DISTINCT c2 FROM t1_agghash_mem_test ) SELECT count(*) FROM a; - count --------- - 100000 -(1 row) -RESET role; -RESET - -SET gp_resgroup_memory_policy TO eager_free; -SET -SET ROLE TO r1_agghash_mem_test; -SET -WITH a AS ( SELECT DISTINCT c2 FROM t1_agghash_mem_test INTERSECT SELECT DISTINCT c2 FROM t1_agghash_mem_test ) SELECT count(*) FROM a; - count --------- - 100000 -(1 row) -RESET role; -RESET - -DROP TABLE IF EXISTS t1_agghash_mem_test; -DROP -DROP ROLE r1_agghash_mem_test; -DROP -DROP RESOURCE GROUP rg1_agghash_mem_test; -DROP diff --git a/src/test/isolation2/expected/resgroup/resgroup_operator_memory.out b/src/test/isolation2/expected/resgroup/resgroup_operator_memory.out index cf308c3fd9918a66e230dce3e4ba7bd0efd2f9a1..7be4d18e378fb0d8d63e0b53b697e9ec2ebc5bf4 100644 --- a/src/test/isolation2/expected/resgroup/resgroup_operator_memory.out +++ b/src/test/isolation2/expected/resgroup/resgroup_operator_memory.out @@ -23,7 +23,11 @@ CREATE -- we have to keep the columns provided by them in the target list, instead of -- composing a long SELECT c1,c2,... list we use SELECT * here, but we should -- not output the groupid as it changes each time. -CREATE OR REPLACE VIEW many_ops AS SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 ; +-- +-- hashagg may not work with a small operator memory, so we use UNION ALL +-- instead of UNION to prevent putting a hashagg on top of the append node, +-- and we use a always-false WHERE condition to prevent too much output. +CREATE OR REPLACE VIEW many_ops AS SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 ; CREATE -- we must ensure spill to be small enough but still > 0. @@ -42,12 +46,6 @@ GRANT -- memory reserved, however in resource group mode we assign at least 100KB to -- each operator, no matter it is memory intensive or not. As long as there is -- enough shared memory the query should be executed successfully. --- --- some operators like HashAgg require more memory to run, the memory quota is --- also dynamically increased to meet their minimal requirements. --- --- note: when there is no enough operator memory there should be a warning, --- however warnings are not displayed in isolation2 tests. -- -- positive: there is enough global shared memory @@ -58,10 +56,9 @@ SET SET ROLE TO r1_opmem_test; SET SELECT * FROM many_ops; - groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset ----------+---------------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- - 6437 | default_group | 20 | 30 | 30 | 80 | 10 | vmtracker | -1 -(1 row) + groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset +---------+-----------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- +(0 rows) RESET role; RESET @@ -70,10 +67,9 @@ SET SET ROLE TO r1_opmem_test; SET SELECT * FROM many_ops; - groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset ----------+---------------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- - 6437 | default_group | 20 | 30 | 30 | 80 | 10 | vmtracker | -1 -(1 row) + groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset +---------+-----------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- +(0 rows) RESET role; RESET @@ -82,10 +78,9 @@ SET SET ROLE TO r1_opmem_test; SET SELECT * FROM many_ops; - groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset ----------+---------------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- - 6437 | default_group | 20 | 30 | 30 | 80 | 10 | vmtracker | -1 -(1 row) + groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset +---------+-----------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- +(0 rows) RESET role; RESET @@ -148,10 +143,9 @@ SET SET ROLE TO r1_opmem_test; SET SELECT * FROM many_ops; - groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset ----------+---------------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- - 6437 | default_group | 20 | 30 | 30 | 80 | 10 | vmtracker | -1 -(1 row) + groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset +---------+-----------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- +(0 rows) RESET role; RESET @@ -160,10 +154,9 @@ SET SET ROLE TO r1_opmem_test; SET SELECT * FROM many_ops; - groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset ----------+---------------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- - 6437 | default_group | 20 | 30 | 30 | 80 | 10 | vmtracker | -1 -(1 row) + groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset +---------+-----------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- +(0 rows) RESET role; RESET @@ -172,10 +165,9 @@ SET SET ROLE TO r1_opmem_test; SET SELECT * FROM many_ops; - groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset ----------+---------------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- - 6437 | default_group | 20 | 30 | 30 | 80 | 10 | vmtracker | -1 -(1 row) + groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset +---------+-----------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- +(0 rows) RESET role; RESET @@ -199,10 +191,9 @@ SET SET ROLE TO r1_opmem_test; SET SELECT * FROM many_ops; - groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset ----------+---------------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- - 6437 | default_group | 20 | 30 | 30 | 80 | 10 | vmtracker | -1 -(1 row) + groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset +---------+-----------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- +(0 rows) RESET role; RESET @@ -211,10 +202,9 @@ SET SET ROLE TO r1_opmem_test; SET SELECT * FROM many_ops; - groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset ----------+---------------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- - 6437 | default_group | 20 | 30 | 30 | 80 | 10 | vmtracker | -1 -(1 row) + groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset +---------+-----------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- +(0 rows) RESET role; RESET @@ -223,10 +213,9 @@ SET SET ROLE TO r1_opmem_test; SET SELECT * FROM many_ops; - groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset ----------+---------------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- - 6437 | default_group | 20 | 30 | 30 | 80 | 10 | vmtracker | -1 -(1 row) + groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset +---------+-----------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- +(0 rows) RESET role; RESET @@ -242,10 +231,9 @@ SET SET ROLE TO r1_opmem_test; SET SELECT * FROM many_ops; - groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset ----------+---------------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- - 6437 | default_group | 20 | 30 | 30 | 80 | 10 | vmtracker | -1 -(1 row) + groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset +---------+-----------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- +(0 rows) SELECT f1_opmem_test(); f1_opmem_test --------------- @@ -259,10 +247,9 @@ SET SET ROLE TO r1_opmem_test; SET SELECT * FROM many_ops; - groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset ----------+---------------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- - 6437 | default_group | 20 | 30 | 30 | 80 | 10 | vmtracker | -1 -(1 row) + groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset +---------+-----------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- +(0 rows) SELECT f1_opmem_test(); f1_opmem_test --------------- @@ -276,10 +263,9 @@ SET SET ROLE TO r1_opmem_test; SET SELECT * FROM many_ops; - groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset ----------+---------------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- - 6437 | default_group | 20 | 30 | 30 | 80 | 10 | vmtracker | -1 -(1 row) + groupid | groupname | concurrency | cpu_rate_limit | memory_limit | memory_shared_quota | memory_spill_ratio | memory_auditor | cpuset +---------+-----------+-------------+----------------+--------------+---------------------+--------------------+----------------+-------- +(0 rows) SELECT f1_opmem_test(); f1_opmem_test --------------- diff --git a/src/test/isolation2/isolation2_resgroup_schedule b/src/test/isolation2/isolation2_resgroup_schedule index 8ca1b1e6e0899aba04647d8d295d6dd6bc4d06be..1487958fb867f548fa4167c7a900eac2ecb6730d 100644 --- a/src/test/isolation2/isolation2_resgroup_schedule +++ b/src/test/isolation2/isolation2_resgroup_schedule @@ -34,7 +34,6 @@ test: resgroup/resgroup_cancel_terminate_concurrency # regression tests test: resgroup/resgroup_recreate test: resgroup/resgroup_operator_memory -test: resgroup/resgroup_agghash_memory # parallel tests #test: resgroup/restore_default_resgroup diff --git a/src/test/isolation2/sql/resgroup/resgroup_agghash_memory.sql b/src/test/isolation2/sql/resgroup/resgroup_agghash_memory.sql deleted file mode 100644 index 2b7181818be4a960224b52ed41c883b5b3c6634d..0000000000000000000000000000000000000000 --- a/src/test/isolation2/sql/resgroup/resgroup_agghash_memory.sql +++ /dev/null @@ -1,59 +0,0 @@ --- When an agghash operator begins to spill it needs 16KB memory as meta data --- for each batch file, when there are many batch files there might not be --- enough operator memory for all of them, in resource queue mode the query --- will fail with an error like below: --- --- ERROR: insufficient memory reserved for statement --- --- In resource group we allow this overuse as the shared memory is designed to --- serve this kind of overuse. - ---start_ignore -DROP TABLE IF EXISTS t1_agghash_mem_test; -DROP ROLE r1_agghash_mem_test; -DROP RESOURCE GROUP rg1_agghash_mem_test; ---end_ignore - -SET optimizer TO off; - -CREATE TABLE t1_agghash_mem_test (c1 int, c2 text) DISTRIBUTED BY (c2); -INSERT INTO t1_agghash_mem_test SELECT i, i::text FROM generate_series(1,100000) i; - --- we must ensure spill to be small enough but still > 0. --- - rg1's memory quota is 682 * 1% = 6; --- - per-xact quota is 6/3=2; --- - spill memory is 2 * 60% = 1; -CREATE RESOURCE GROUP rg1_agghash_mem_test - WITH (cpu_rate_limit=10, memory_limit=1, memory_shared_quota=0, - concurrency=3, memory_spill_ratio=60); - -CREATE ROLE r1_agghash_mem_test RESOURCE GROUP rg1_agghash_mem_test; -GRANT ALL ON t1_agghash_mem_test TO r1_agghash_mem_test; - -SET gp_resgroup_memory_policy TO none; -SET ROLE TO r1_agghash_mem_test; -WITH a AS ( - SELECT DISTINCT c2 FROM t1_agghash_mem_test INTERSECT - SELECT DISTINCT c2 FROM t1_agghash_mem_test -) SELECT count(*) FROM a; -RESET role; - -SET gp_resgroup_memory_policy TO auto; -SET ROLE TO r1_agghash_mem_test; -WITH a AS ( - SELECT DISTINCT c2 FROM t1_agghash_mem_test INTERSECT - SELECT DISTINCT c2 FROM t1_agghash_mem_test -) SELECT count(*) FROM a; -RESET role; - -SET gp_resgroup_memory_policy TO eager_free; -SET ROLE TO r1_agghash_mem_test; -WITH a AS ( - SELECT DISTINCT c2 FROM t1_agghash_mem_test INTERSECT - SELECT DISTINCT c2 FROM t1_agghash_mem_test -) SELECT count(*) FROM a; -RESET role; - -DROP TABLE IF EXISTS t1_agghash_mem_test; -DROP ROLE r1_agghash_mem_test; -DROP RESOURCE GROUP rg1_agghash_mem_test; diff --git a/src/test/isolation2/sql/resgroup/resgroup_operator_memory.sql b/src/test/isolation2/sql/resgroup/resgroup_operator_memory.sql index 574303a0cf3d68867da7bab2dacfcccc76a7cf5f..10cfad537ff17eabce3a5288b9816d09ed5a1831 100644 --- a/src/test/isolation2/sql/resgroup/resgroup_operator_memory.sql +++ b/src/test/isolation2/sql/resgroup/resgroup_operator_memory.sql @@ -23,27 +23,31 @@ $$ LANGUAGE plpythonu; -- we have to keep the columns provided by them in the target list, instead of -- composing a long SELECT c1,c2,... list we use SELECT * here, but we should -- not output the groupid as it changes each time. +-- +-- hashagg may not work with a small operator memory, so we use UNION ALL +-- instead of UNION to prevent putting a hashagg on top of the append node, +-- and we use a always-false WHERE condition to prevent too much output. CREATE OR REPLACE VIEW many_ops AS - SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 -UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 -UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 -UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 -UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 -UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 -UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 -UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 -UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 -UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 -UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 -UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 -UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 -UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 -UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 -UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 -UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 -UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 -UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 -UNION SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=6437 + SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 +UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 +UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 +UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 +UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 +UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 +UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 +UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 +UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 +UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 +UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 +UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 +UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 +UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 +UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 +UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 +UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 +UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 +UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 +UNION ALL SELECT * FROM gp_toolkit.gp_resgroup_config WHERE groupid=0 ; -- we must ensure spill to be small enough but still > 0. @@ -61,12 +65,6 @@ GRANT ALL ON many_ops TO r1_opmem_test; -- memory reserved, however in resource group mode we assign at least 100KB to -- each operator, no matter it is memory intensive or not. As long as there is -- enough shared memory the query should be executed successfully. --- --- some operators like HashAgg require more memory to run, the memory quota is --- also dynamically increased to meet their minimal requirements. --- --- note: when there is no enough operator memory there should be a warning, --- however warnings are not displayed in isolation2 tests. -- -- positive: there is enough global shared memory