diff --git a/CHANGELOG.md b/CHANGELOG.md index a79c34ba..235bf7e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -134,6 +134,8 @@ All notable changes to this project will be documented in this file. It uses the serializer previously for scalar values (now simplified), preventing PostgreSQL `IN` lists from producing invalid or mismatched `FixedString` comparisons. Thanks to @jxom for the PR (#333)! +* Fixed `array_agg()` pushdown dropping `NULL` elements because ClickHouse's + `groupArray()` skips them ([#340]). * Fixed `array_position()` pushdown to return `NULL`, rather than zero, when ClickHouse does not find an element. Thanks to Minh Vu for the PR ([#334])! * Fixed `array_length()` pushdown to preserve empty-array and requested @@ -211,6 +213,8 @@ All notable changes to this project will be documented in this file. It uses the "ClickHouse/pg_clickhouse#335 Reject JSON paths containing NULL elements" [#336]: https://github.com/ClickHouse/pg_clickhouse/pull/336 "ClickHouse/pg_clickhouse#336 Preserve array_length semantics" + [#340]: https://github.com/ClickHouse/pg_clickhouse/pull/340 + "ClickHouse/pg_clickhouse#340 Preserve NULLs in array_agg pushdown" ## [v0.3.2] — 2026-06-16 diff --git a/src/custom_types.c b/src/custom_types.c index ea6670dc..2a911b69 100644 --- a/src/custom_types.c +++ b/src/custom_types.c @@ -536,7 +536,8 @@ lookup_builtin_func(Oid funcid, builtin_func_def* def) { def->ch_name = "quantilesExactLow"; return true; case F_ARRAY_AGG_ANYNONARRAY: - def->ch_name = "groupArray"; + def->cf_type = CF_ARRAY_AGG; + def->ch_name = "\1"; return true; case F_MD5_BYTEA: case F_MD5_TEXT: diff --git a/src/deparse.c b/src/deparse.c index 8fadef8f..9c8b41d3 100644 --- a/src/deparse.c +++ b/src/deparse.c @@ -324,6 +324,8 @@ deparseRangeTblRef( static void deparseAggref(Aggref* node, deparse_expr_cxt* context); static void +deparseArrayAggref(Aggref* node, deparse_expr_cxt* context); +static void deparseWindowFunc(WindowFunc* node, deparse_expr_cxt* context); static void appendGroupByClause(List* tlist, deparse_expr_cxt* context); @@ -1235,7 +1237,9 @@ foreign_expr_walker(Node* node, foreign_glob_cxt* glob_cxt, ExprTruthCtx ctx) { } /* groupConcat has no ORDER BY; block ordered string_agg */ - if (agg->aggfnoid == F_STRING_AGG_TEXT_TEXT && agg->aggorder != NIL) { + if ((agg->aggfnoid == F_STRING_AGG_TEXT_TEXT || + agg->aggfnoid == F_ARRAY_AGG_ANYNONARRAY) && + agg->aggorder != NIL) { return false; } @@ -5855,6 +5859,62 @@ deparsePartialStatArray(Aggref* node, AggPartialKind kind, deparse_expr_cxt* con pfree(arg); } +static void +deparseArrayAggref(Aggref* node, deparse_expr_cxt* context) { + StringInfo buf = context->buf; + foreign_glob_cxt glob_cxt; + TargetEntry* arg = NULL; + bool nullable; + ListCell* lc; + + foreach (lc, node->args) { + TargetEntry* tle = lfirst_node(TargetEntry, lc); + + if (!tle->resjunk) { + arg = tle; + break; + } + } + + Assert(arg != NULL); + + memset(&glob_cxt, 0, sizeof(glob_cxt)); + glob_cxt.root = context->root; + glob_cxt.foreignrel = context->foreignrel; + glob_cxt.relids = context->scanrel->relids; + nullable = !expr_never_null((Expr*)arg->expr, &glob_cxt); + + if (nullable) { + appendStringInfoString(buf, "arrayMap(x -> x.1, "); + } + appendStringInfoString(buf, "groupArray"); + if (node->aggfilter) { + appendStringInfoString(buf, "If"); + } + appendStringInfoString(buf, "("); + if (node->aggdistinct != NIL) { + appendStringInfoString(buf, "DISTINCT "); + } + if (nullable) { + appendStringInfoString(buf, "tuple("); + } + deparseExpr((Expr*)arg->expr, context); + if (nullable) { + appendStringInfoChar(buf, ')'); + } + + if (node->aggfilter) { + appendStringInfoString(buf, ",(("); + deparseExpr((Expr*)node->aggfilter, context); + appendStringInfoString(buf, ") > 0)"); + } + + appendStringInfoChar(buf, ')'); + if (nullable) { + appendStringInfoChar(buf, ')'); + } +} + /* * Deparse an Aggref node. */ @@ -5891,6 +5951,12 @@ deparseAggref(Aggref* node, deparse_expr_cxt* context) { cdef = context->func; context->func = appendFunctionName(node->aggfnoid, context); + if (context->func && context->func->cf_type == CF_ARRAY_AGG) { + deparseArrayAggref(node, context); + context->func = cdef; + return; + } + /* 'If' part */ if (context->func && context->func->cf_type == CF_SIGN_COUNT && !node->aggstar) { sign_count_filter = true; diff --git a/src/include/fdw.h b/src/include/fdw.h index e5e4669a..51c6eed2 100644 --- a/src/include/fdw.h +++ b/src/include/fdw.h @@ -406,6 +406,8 @@ typedef enum { * length(arr)-n) */ CF_ARRAY_SORT_DESC, /* array_sort(arr,desc) → * arrayReverseSort/arraySort */ + CF_ARRAY_AGG, /* array_agg → groupArray; tuple-wrap nullable + * inputs because groupArray skips NULLs */ CF_ARRAY_FILL, /* array_fill → arrayWithConstant, * swap+extract */ CF_ARRAY_CONTAINS, /* @> → hasAll(left, right) */ diff --git a/test/expected/aggregates.out b/test/expected/aggregates.out index 085f9508..ae1c5695 100644 --- a/test/expected/aggregates.out +++ b/test/expected/aggregates.out @@ -27,6 +27,16 @@ clickhouse_raw_query ---------------------- +(1 row) + + clickhouse_raw_query +---------------------- + +(1 row) + + clickhouse_raw_query +---------------------- + (1 row) Foreign table "agg_bin.agg_numbers" @@ -50,6 +60,13 @@ FDW options: (database 'agg_test', table_name 'agg_numbers', engine 'MergeTree') Server: agg_bin_svr FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') + Foreign table "agg_bin.null_agg" + Column | Type | Collation | Nullable | Default | FDW options +--------+---------+-----------+----------+---------+------------- + v | integer | | | | +Server: agg_bin_svr +FDW options: (database 'agg_test', table_name 'null_agg', engine 'TinyLog') + Foreign table "agg_http.agg_numbers" Column | Type | Collation | Nullable | Default | FDW options --------+----------+-----------+----------+---------+------------- @@ -71,6 +88,13 @@ FDW options: (database 'agg_test', table_name 'agg_numbers', engine 'MergeTree') Server: agg_http_svr FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') + Foreign table "agg_http.null_agg" + Column | Type | Collation | Nullable | Default | FDW options +--------+---------+-----------+----------+---------+------------- + v | integer | | | | +Server: agg_http_svr +FDW options: (database 'agg_test', table_name 'null_agg', engine 'TinyLog') + -- AVG(UInt64) QUERY PLAN ------------------------------------------------- @@ -601,6 +625,112 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') {5.10,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} (1 row) + QUERY PLAN +-------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------------- + {1,NULL,2,1,NULL} +(1 row) + + QUERY PLAN +-------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------------- + {1,NULL,2,1,NULL} +(1 row) + + QUERY PLAN +-------------------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v) FILTER (WHERE (v > 1))) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArrayIf(tuple(v),(((v > 1)) > 0))) FROM agg_test.null_agg +(4 rows) + + array_agg +----------- + {2} +(1 row) + + QUERY PLAN +-------------------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v) FILTER (WHERE (v > 1))) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArrayIf(tuple(v),(((v > 1)) > 0))) FROM agg_test.null_agg +(4 rows) + + array_agg +----------- + {2} +(1 row) + + QUERY PLAN +----------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(DISTINCT v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(DISTINCT tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------ + {1,NULL,2} +(1 row) + + QUERY PLAN +----------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(DISTINCT v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(DISTINCT tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------ + {1,NULL,2} +(1 row) + + QUERY PLAN +------------------------------------------------------------------------- + Aggregate + Output: array_agg(v ORDER BY v) + -> Foreign Scan on agg_bin.null_agg + Output: v + Remote SQL: SELECT v FROM agg_test.null_agg ORDER BY v ASC NULLS LAST +(5 rows) + + array_agg +------------------- + {1,1,2,NULL,NULL} +(1 row) + + QUERY PLAN +------------------------------------------------------------------------- + Aggregate + Output: array_agg(v ORDER BY v) + -> Foreign Scan on agg_http.null_agg + Output: v + Remote SQL: SELECT v FROM agg_test.null_agg ORDER BY v ASC NULLS LAST +(5 rows) + + array_agg +------------------- + {1,1,2,NULL,NULL} +(1 row) + -- min(UInt64) QUERY PLAN ------------------------------------------------- @@ -1834,9 +1964,11 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') 100 | 100 (4 rows) -NOTICE: drop cascades to 2 other objects +NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to foreign table agg_bin.agg_numbers drop cascades to foreign table agg_bin.hits -NOTICE: drop cascades to 2 other objects +drop cascades to foreign table agg_bin.null_agg +NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to foreign table agg_http.agg_numbers drop cascades to foreign table agg_http.hits +drop cascades to foreign table agg_http.null_agg diff --git a/test/expected/aggregates_1.out b/test/expected/aggregates_1.out index f83aa63b..64f5bc3d 100644 --- a/test/expected/aggregates_1.out +++ b/test/expected/aggregates_1.out @@ -27,6 +27,16 @@ clickhouse_raw_query ---------------------- +(1 row) + + clickhouse_raw_query +---------------------- + +(1 row) + + clickhouse_raw_query +---------------------- + (1 row) Foreign table "agg_bin.agg_numbers" @@ -50,6 +60,13 @@ FDW options: (database 'agg_test', table_name 'agg_numbers', engine 'MergeTree') Server: agg_bin_svr FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') + Foreign table "agg_bin.null_agg" + Column | Type | Collation | Nullable | Default | FDW options +--------+---------+-----------+----------+---------+------------- + v | integer | | | | +Server: agg_bin_svr +FDW options: (database 'agg_test', table_name 'null_agg', engine 'TinyLog') + Foreign table "agg_http.agg_numbers" Column | Type | Collation | Nullable | Default | FDW options --------+----------+-----------+----------+---------+------------- @@ -71,6 +88,13 @@ FDW options: (database 'agg_test', table_name 'agg_numbers', engine 'MergeTree') Server: agg_http_svr FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') + Foreign table "agg_http.null_agg" + Column | Type | Collation | Nullable | Default | FDW options +--------+---------+-----------+----------+---------+------------- + v | integer | | | | +Server: agg_http_svr +FDW options: (database 'agg_test', table_name 'null_agg', engine 'TinyLog') + -- AVG(UInt64) QUERY PLAN ------------------------------------------------- @@ -601,6 +625,112 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') {5.10,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} (1 row) + QUERY PLAN +-------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------------- + {1,NULL,2,1,NULL} +(1 row) + + QUERY PLAN +-------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------------- + {1,NULL,2,1,NULL} +(1 row) + + QUERY PLAN +-------------------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v) FILTER (WHERE (v > 1))) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArrayIf(tuple(v),(((v > 1)) > 0))) FROM agg_test.null_agg +(4 rows) + + array_agg +----------- + {2} +(1 row) + + QUERY PLAN +-------------------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v) FILTER (WHERE (v > 1))) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArrayIf(tuple(v),(((v > 1)) > 0))) FROM agg_test.null_agg +(4 rows) + + array_agg +----------- + {2} +(1 row) + + QUERY PLAN +----------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(DISTINCT v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(DISTINCT tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------ + {1,NULL,2} +(1 row) + + QUERY PLAN +----------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(DISTINCT v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(DISTINCT tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------ + {1,NULL,2} +(1 row) + + QUERY PLAN +----------------------------------------------- + Aggregate + Output: array_agg(v ORDER BY v) + -> Foreign Scan on agg_bin.null_agg + Output: v + Remote SQL: SELECT v FROM agg_test.null_agg +(5 rows) + + array_agg +------------------- + {1,1,2,NULL,NULL} +(1 row) + + QUERY PLAN +----------------------------------------------- + Aggregate + Output: array_agg(v ORDER BY v) + -> Foreign Scan on agg_http.null_agg + Output: v + Remote SQL: SELECT v FROM agg_test.null_agg +(5 rows) + + array_agg +------------------- + {1,1,2,NULL,NULL} +(1 row) + -- min(UInt64) QUERY PLAN ------------------------------------------------- @@ -1800,9 +1930,11 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') 17150.25 | 22867 | 22867 (1 row) -NOTICE: drop cascades to 2 other objects +NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to foreign table agg_bin.agg_numbers drop cascades to foreign table agg_bin.hits -NOTICE: drop cascades to 2 other objects +drop cascades to foreign table agg_bin.null_agg +NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to foreign table agg_http.agg_numbers drop cascades to foreign table agg_http.hits +drop cascades to foreign table agg_http.null_agg diff --git a/test/expected/aggregates_2.out b/test/expected/aggregates_2.out index 12b77545..4edfe4d2 100644 --- a/test/expected/aggregates_2.out +++ b/test/expected/aggregates_2.out @@ -27,6 +27,16 @@ clickhouse_raw_query ---------------------- +(1 row) + + clickhouse_raw_query +---------------------- + +(1 row) + + clickhouse_raw_query +---------------------- + (1 row) Foreign table "agg_bin.agg_numbers" @@ -50,6 +60,13 @@ FDW options: (database 'agg_test', table_name 'agg_numbers', engine 'MergeTree') Server: agg_bin_svr FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') + Foreign table "agg_bin.null_agg" + Column | Type | Collation | Nullable | Default | FDW options +--------+---------+-----------+----------+---------+------------- + v | integer | | | | +Server: agg_bin_svr +FDW options: (database 'agg_test', table_name 'null_agg', engine 'TinyLog') + Foreign table "agg_http.agg_numbers" Column | Type | Collation | Nullable | Default | FDW options --------+----------+-----------+----------+---------+------------- @@ -71,6 +88,13 @@ FDW options: (database 'agg_test', table_name 'agg_numbers', engine 'MergeTree') Server: agg_http_svr FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') + Foreign table "agg_http.null_agg" + Column | Type | Collation | Nullable | Default | FDW options +--------+---------+-----------+----------+---------+------------- + v | integer | | | | +Server: agg_http_svr +FDW options: (database 'agg_test', table_name 'null_agg', engine 'TinyLog') + -- AVG(UInt64) QUERY PLAN ------------------------------------------------- @@ -601,6 +625,112 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') {5.10,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} (1 row) + QUERY PLAN +-------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------------- + {1,NULL,2,1,NULL} +(1 row) + + QUERY PLAN +-------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------------- + {1,NULL,2,1,NULL} +(1 row) + + QUERY PLAN +-------------------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v) FILTER (WHERE (v > 1))) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArrayIf(tuple(v),(((v > 1)) > 0))) FROM agg_test.null_agg +(4 rows) + + array_agg +----------- + {2} +(1 row) + + QUERY PLAN +-------------------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v) FILTER (WHERE (v > 1))) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArrayIf(tuple(v),(((v > 1)) > 0))) FROM agg_test.null_agg +(4 rows) + + array_agg +----------- + {2} +(1 row) + + QUERY PLAN +----------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(DISTINCT v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(DISTINCT tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------ + {1,NULL,2} +(1 row) + + QUERY PLAN +----------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(DISTINCT v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(DISTINCT tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------ + {1,NULL,2} +(1 row) + + QUERY PLAN +----------------------------------------------- + Aggregate + Output: array_agg(v ORDER BY v) + -> Foreign Scan on agg_bin.null_agg + Output: v + Remote SQL: SELECT v FROM agg_test.null_agg +(5 rows) + + array_agg +------------------- + {1,1,2,NULL,NULL} +(1 row) + + QUERY PLAN +----------------------------------------------- + Aggregate + Output: array_agg(v ORDER BY v) + -> Foreign Scan on agg_http.null_agg + Output: v + Remote SQL: SELECT v FROM agg_test.null_agg +(5 rows) + + array_agg +------------------- + {1,1,2,NULL,NULL} +(1 row) + -- min(UInt64) QUERY PLAN ------------------------------------------------- @@ -1786,9 +1916,11 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') 17150.25 | 22867 | 22867 (1 row) -NOTICE: drop cascades to 2 other objects +NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to foreign table agg_bin.agg_numbers drop cascades to foreign table agg_bin.hits -NOTICE: drop cascades to 2 other objects +drop cascades to foreign table agg_bin.null_agg +NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to foreign table agg_http.agg_numbers drop cascades to foreign table agg_http.hits +drop cascades to foreign table agg_http.null_agg diff --git a/test/expected/aggregates_3.out b/test/expected/aggregates_3.out index a4e67d78..1332090b 100644 --- a/test/expected/aggregates_3.out +++ b/test/expected/aggregates_3.out @@ -27,6 +27,16 @@ clickhouse_raw_query ---------------------- +(1 row) + + clickhouse_raw_query +---------------------- + +(1 row) + + clickhouse_raw_query +---------------------- + (1 row) Foreign table "agg_bin.agg_numbers" @@ -50,6 +60,13 @@ FDW options: (database 'agg_test', table_name 'agg_numbers', engine 'MergeTree') Server: agg_bin_svr FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') + Foreign table "agg_bin.null_agg" + Column | Type | Collation | Nullable | Default | FDW options +--------+---------+-----------+----------+---------+------------- + v | integer | | | | +Server: agg_bin_svr +FDW options: (database 'agg_test', table_name 'null_agg', engine 'TinyLog') + Foreign table "agg_http.agg_numbers" Column | Type | Collation | Nullable | Default | FDW options --------+----------+-----------+----------+---------+------------- @@ -71,6 +88,13 @@ FDW options: (database 'agg_test', table_name 'agg_numbers', engine 'MergeTree') Server: agg_http_svr FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') + Foreign table "agg_http.null_agg" + Column | Type | Collation | Nullable | Default | FDW options +--------+---------+-----------+----------+---------+------------- + v | integer | | | | +Server: agg_http_svr +FDW options: (database 'agg_test', table_name 'null_agg', engine 'TinyLog') + -- AVG(UInt64) QUERY PLAN ------------------------------------------------- @@ -601,6 +625,112 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') {5.10,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} (1 row) + QUERY PLAN +-------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------------- + {1,NULL,2,1,NULL} +(1 row) + + QUERY PLAN +-------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------------- + {1,NULL,2,1,NULL} +(1 row) + + QUERY PLAN +-------------------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v) FILTER (WHERE (v > 1))) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArrayIf(tuple(v),(((v > 1)) > 0))) FROM agg_test.null_agg +(4 rows) + + array_agg +----------- + {2} +(1 row) + + QUERY PLAN +-------------------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v) FILTER (WHERE (v > 1))) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArrayIf(tuple(v),(((v > 1)) > 0))) FROM agg_test.null_agg +(4 rows) + + array_agg +----------- + {2} +(1 row) + + QUERY PLAN +----------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(DISTINCT v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(DISTINCT tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------ + {1,NULL,2} +(1 row) + + QUERY PLAN +----------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(DISTINCT v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(DISTINCT tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------ + {1,NULL,2} +(1 row) + + QUERY PLAN +------------------------------------------------------------------------- + Aggregate + Output: array_agg(v ORDER BY v) + -> Foreign Scan on agg_bin.null_agg + Output: v + Remote SQL: SELECT v FROM agg_test.null_agg ORDER BY v ASC NULLS LAST +(5 rows) + + array_agg +------------------- + {1,1,2,NULL,NULL} +(1 row) + + QUERY PLAN +------------------------------------------------------------------------- + Aggregate + Output: array_agg(v ORDER BY v) + -> Foreign Scan on agg_http.null_agg + Output: v + Remote SQL: SELECT v FROM agg_test.null_agg ORDER BY v ASC NULLS LAST +(5 rows) + + array_agg +------------------- + {1,1,2,NULL,NULL} +(1 row) + -- min(UInt64) QUERY PLAN ------------------------------------------------- @@ -1824,9 +1954,11 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h 100 | 100 (4 rows) -NOTICE: drop cascades to 2 other objects +NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to foreign table agg_bin.agg_numbers drop cascades to foreign table agg_bin.hits -NOTICE: drop cascades to 2 other objects +drop cascades to foreign table agg_bin.null_agg +NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to foreign table agg_http.agg_numbers drop cascades to foreign table agg_http.hits +drop cascades to foreign table agg_http.null_agg diff --git a/test/expected/aggregates_4.out b/test/expected/aggregates_4.out index 90b22c49..4ae54c36 100644 --- a/test/expected/aggregates_4.out +++ b/test/expected/aggregates_4.out @@ -27,6 +27,16 @@ clickhouse_raw_query ---------------------- +(1 row) + + clickhouse_raw_query +---------------------- + +(1 row) + + clickhouse_raw_query +---------------------- + (1 row) Foreign table "agg_bin.agg_numbers" @@ -50,6 +60,13 @@ FDW options: (database 'agg_test', table_name 'agg_numbers', engine 'MergeTree') Server: agg_bin_svr FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') + Foreign table "agg_bin.null_agg" + Column | Type | Collation | Nullable | Default | FDW options +--------+---------+-----------+----------+---------+------------- + v | integer | | | | +Server: agg_bin_svr +FDW options: (database 'agg_test', table_name 'null_agg', engine 'TinyLog') + Foreign table "agg_http.agg_numbers" Column | Type | Collation | Nullable | Default | FDW options --------+----------+-----------+----------+---------+------------- @@ -71,6 +88,13 @@ FDW options: (database 'agg_test', table_name 'agg_numbers', engine 'MergeTree') Server: agg_http_svr FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') + Foreign table "agg_http.null_agg" + Column | Type | Collation | Nullable | Default | FDW options +--------+---------+-----------+----------+---------+------------- + v | integer | | | | +Server: agg_http_svr +FDW options: (database 'agg_test', table_name 'null_agg', engine 'TinyLog') + -- AVG(UInt64) QUERY PLAN ------------------------------------------------- @@ -601,6 +625,112 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') {5.10,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} (1 row) + QUERY PLAN +-------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------------- + {1,NULL,2,1,NULL} +(1 row) + + QUERY PLAN +-------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------------- + {1,NULL,2,1,NULL} +(1 row) + + QUERY PLAN +-------------------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v) FILTER (WHERE (v > 1))) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArrayIf(tuple(v),(((v > 1)) > 0))) FROM agg_test.null_agg +(4 rows) + + array_agg +----------- + {2} +(1 row) + + QUERY PLAN +-------------------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v) FILTER (WHERE (v > 1))) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArrayIf(tuple(v),(((v > 1)) > 0))) FROM agg_test.null_agg +(4 rows) + + array_agg +----------- + {2} +(1 row) + + QUERY PLAN +----------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(DISTINCT v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(DISTINCT tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------ + {1,NULL,2} +(1 row) + + QUERY PLAN +----------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(DISTINCT v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(DISTINCT tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------ + {1,NULL,2} +(1 row) + + QUERY PLAN +------------------------------------------------------------------------- + Aggregate + Output: array_agg(v ORDER BY v) + -> Foreign Scan on agg_bin.null_agg + Output: v + Remote SQL: SELECT v FROM agg_test.null_agg ORDER BY v ASC NULLS LAST +(5 rows) + + array_agg +------------------- + {1,1,2,NULL,NULL} +(1 row) + + QUERY PLAN +------------------------------------------------------------------------- + Aggregate + Output: array_agg(v ORDER BY v) + -> Foreign Scan on agg_http.null_agg + Output: v + Remote SQL: SELECT v FROM agg_test.null_agg ORDER BY v ASC NULLS LAST +(5 rows) + + array_agg +------------------- + {1,1,2,NULL,NULL} +(1 row) + -- min(UInt64) QUERY PLAN ------------------------------------------------- @@ -1814,9 +1944,11 @@ CONTEXT: HTTP status code: 500 100 | 100 (4 rows) -NOTICE: drop cascades to 2 other objects +NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to foreign table agg_bin.agg_numbers drop cascades to foreign table agg_bin.hits -NOTICE: drop cascades to 2 other objects +drop cascades to foreign table agg_bin.null_agg +NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to foreign table agg_http.agg_numbers drop cascades to foreign table agg_http.hits +drop cascades to foreign table agg_http.null_agg diff --git a/test/expected/aggregates_5.out b/test/expected/aggregates_5.out index fb3aeffc..532194fa 100644 --- a/test/expected/aggregates_5.out +++ b/test/expected/aggregates_5.out @@ -27,6 +27,16 @@ clickhouse_raw_query ---------------------- +(1 row) + + clickhouse_raw_query +---------------------- + +(1 row) + + clickhouse_raw_query +---------------------- + (1 row) Foreign table "agg_bin.agg_numbers" @@ -50,6 +60,13 @@ FDW options: (database 'agg_test', table_name 'agg_numbers', engine 'MergeTree') Server: agg_bin_svr FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') + Foreign table "agg_bin.null_agg" + Column | Type | Collation | Nullable | Default | FDW options +--------+---------+-----------+----------+---------+------------- + v | integer | | | | +Server: agg_bin_svr +FDW options: (database 'agg_test', table_name 'null_agg', engine 'TinyLog') + Foreign table "agg_http.agg_numbers" Column | Type | Collation | Nullable | Default | FDW options --------+----------+-----------+----------+---------+------------- @@ -71,6 +88,13 @@ FDW options: (database 'agg_test', table_name 'agg_numbers', engine 'MergeTree') Server: agg_http_svr FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') + Foreign table "agg_http.null_agg" + Column | Type | Collation | Nullable | Default | FDW options +--------+---------+-----------+----------+---------+------------- + v | integer | | | | +Server: agg_http_svr +FDW options: (database 'agg_test', table_name 'null_agg', engine 'TinyLog') + -- AVG(UInt64) QUERY PLAN ------------------------------------------------- @@ -601,6 +625,112 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') {5.10,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} (1 row) + QUERY PLAN +-------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------------- + {1,NULL,2,1,NULL} +(1 row) + + QUERY PLAN +-------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------------- + {1,NULL,2,1,NULL} +(1 row) + + QUERY PLAN +-------------------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v) FILTER (WHERE (v > 1))) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArrayIf(tuple(v),(((v > 1)) > 0))) FROM agg_test.null_agg +(4 rows) + + array_agg +----------- + {2} +(1 row) + + QUERY PLAN +-------------------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(v) FILTER (WHERE (v > 1))) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArrayIf(tuple(v),(((v > 1)) > 0))) FROM agg_test.null_agg +(4 rows) + + array_agg +----------- + {2} +(1 row) + + QUERY PLAN +----------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(DISTINCT v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(DISTINCT tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------ + {1,NULL,2} +(1 row) + + QUERY PLAN +----------------------------------------------------------------------------------------------- + Foreign Scan + Output: (array_agg(DISTINCT v)) + Relations: Aggregate on (null_agg) + Remote SQL: SELECT arrayMap(x -> x.1, groupArray(DISTINCT tuple(v))) FROM agg_test.null_agg +(4 rows) + + array_agg +------------ + {1,NULL,2} +(1 row) + + QUERY PLAN +------------------------------------------------------------------------- + Aggregate + Output: array_agg(v ORDER BY v) + -> Foreign Scan on agg_bin.null_agg + Output: v + Remote SQL: SELECT v FROM agg_test.null_agg ORDER BY v ASC NULLS LAST +(5 rows) + + array_agg +------------------- + {1,1,2,NULL,NULL} +(1 row) + + QUERY PLAN +------------------------------------------------------------------------- + Aggregate + Output: array_agg(v ORDER BY v) + -> Foreign Scan on agg_http.null_agg + Output: v + Remote SQL: SELECT v FROM agg_test.null_agg ORDER BY v ASC NULLS LAST +(5 rows) + + array_agg +------------------- + {1,1,2,NULL,NULL} +(1 row) + -- min(UInt64) QUERY PLAN ------------------------------------------------- @@ -1805,9 +1935,11 @@ CONTEXT: HTTP status code: 500 100 | 100 (4 rows) -NOTICE: drop cascades to 2 other objects +NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to foreign table agg_bin.agg_numbers drop cascades to foreign table agg_bin.hits -NOTICE: drop cascades to 2 other objects +drop cascades to foreign table agg_bin.null_agg +NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to foreign table agg_http.agg_numbers drop cascades to foreign table agg_http.hits +drop cascades to foreign table agg_http.null_agg diff --git a/test/sql/aggregates.sql b/test/sql/aggregates.sql index d66caca2..87176033 100644 --- a/test/sql/aggregates.sql +++ b/test/sql/aggregates.sql @@ -119,6 +119,16 @@ SELECT clickhouse_raw_query($$ , (42, 324.78) $$); +SELECT clickhouse_raw_query($$ + CREATE TABLE agg_test.null_agg ( + v Nullable(Int32) + ) ENGINE = TinyLog; +$$); + +SELECT clickhouse_raw_query($$ + INSERT INTO agg_test.null_agg VALUES (1), (NULL), (2), (1), (NULL) +$$); + CREATE SCHEMA agg_bin; CREATE SCHEMA agg_http; IMPORT FOREIGN SCHEMA "agg_test" FROM SERVER agg_bin_svr INTO agg_bin; @@ -240,6 +250,33 @@ SELECT array_agg(cost) FROM agg_bin.hits WHERE id < :id_limit; EXPLAIN (VERBOSE, COSTS OFF) SELECT array_agg(cost) FROM agg_http.hits WHERE cost < :id_limit; SELECT array_agg(cost) FROM agg_http.hits WHERE id < :id_limit; +EXPLAIN (VERBOSE, COSTS OFF) SELECT array_agg(v) FROM agg_bin.null_agg; +SELECT array_agg(v) FROM agg_bin.null_agg; +EXPLAIN (VERBOSE, COSTS OFF) SELECT array_agg(v) FROM agg_http.null_agg; +SELECT array_agg(v) FROM agg_http.null_agg; + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT array_agg(v) FILTER (WHERE v > 1) FROM agg_bin.null_agg; +SELECT array_agg(v) FILTER (WHERE v > 1) FROM agg_bin.null_agg; +EXPLAIN (VERBOSE, COSTS OFF) +SELECT array_agg(v) FILTER (WHERE v > 1) FROM agg_http.null_agg; +SELECT array_agg(v) FILTER (WHERE v > 1) FROM agg_http.null_agg; + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT array_agg(DISTINCT v) FROM agg_bin.null_agg; +SELECT array_agg(DISTINCT v) FROM agg_bin.null_agg; +EXPLAIN (VERBOSE, COSTS OFF) +SELECT array_agg(DISTINCT v) FROM agg_http.null_agg; +SELECT array_agg(DISTINCT v) FROM agg_http.null_agg; + +-- array_agg() with ORDER BY stays local because groupArray() is unordered. +EXPLAIN (VERBOSE, COSTS OFF) +SELECT array_agg(v ORDER BY v) FROM agg_bin.null_agg; +SELECT array_agg(v ORDER BY v) FROM agg_bin.null_agg; +EXPLAIN (VERBOSE, COSTS OFF) +SELECT array_agg(v ORDER BY v) FROM agg_http.null_agg; +SELECT array_agg(v ORDER BY v) FROM agg_http.null_agg; + -- min() \echo -- min(UInt64) EXPLAIN (VERBOSE, COSTS OFF) SELECT min(id) FROM agg_bin.hits;