From c322da86f2e87cf40f1205d64f8bf52114e69b37 Mon Sep 17 00:00:00 2001 From: liangj777 <751234313@qq.com> Date: Mon, 20 Jul 2026 17:48:08 +0800 Subject: [PATCH 1/2] [fix](fe) disable search topn with extra predicates --- .../PushDownScoreTopNIntoOlapScan.java | 19 +- .../test_search_score_topn_predicates.out | 38 ++++ .../test_search_score_topn_predicates.groovy | 191 ++++++++++++++++++ 3 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 regression-test/data/inverted_index_p0/test_search_score_topn_predicates.out create mode 100644 regression-test/suites/inverted_index_p0/test_search_score_topn_predicates.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownScoreTopNIntoOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownScoreTopNIntoOlapScan.java index 68d206a45018de..47ce149aae8e55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownScoreTopNIntoOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownScoreTopNIntoOlapScan.java @@ -199,12 +199,16 @@ private Plan pushDown( return null; } + long scoreLimit = topN.getLimit() + topN.getOffset(); + long pushedScoreLimit = shouldDisableSearchTopN(filter.getConjuncts(), extractedScorePredicate) + ? 0L : scoreLimit; + // All conditions met, perform the push down. // This is the core action: push score() as a virtual column and also push the // topN info. Plan newScan = scan.appendVirtualColumnsAndTopN(ImmutableList.of(scoreAlias), ImmutableList.of(), Optional.empty(), - topN.getOrderKeys(), Optional.of(topN.getLimit() + topN.getOffset()), + topN.getOrderKeys(), Optional.of(pushedScoreLimit), scoreRangeInfo); // Rebuild the plan tree above the new scan. @@ -243,6 +247,19 @@ private Plan pushDown( return topN.withChildren(newProject); } + private boolean shouldDisableSearchTopN(Set conjuncts, Expression extractedScorePredicate) { + List nonScoreConjuncts = conjuncts.stream() + .filter(conjunct -> extractedScorePredicate == null || !conjunct.equals(extractedScorePredicate)) + .collect(ImmutableList.toImmutableList()); + + boolean hasSearchPredicate = nonScoreConjuncts.stream() + .anyMatch(conjunct -> !conjunct.collect(e -> e instanceof SearchExpression).isEmpty()); + if (!hasSearchPredicate) { + return false; + } + return nonScoreConjuncts.size() > 1 || !(nonScoreConjuncts.get(0) instanceof SearchExpression); + } + /** * Extract score range info from a single score predicate. * Only supports min_score semantics (similar to Elasticsearch): diff --git a/regression-test/data/inverted_index_p0/test_search_score_topn_predicates.out b/regression-test/data/inverted_index_p0/test_search_score_topn_predicates.out new file mode 100644 index 00000000000000..645855dd631301 --- /dev/null +++ b/regression-test/data/inverted_index_p0/test_search_score_topn_predicates.out @@ -0,0 +1,38 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !single_search -- +1 + +-- !search_with_equal_predicate -- +3 + +-- !search_with_plain_equal_predicate -- +3 + +-- !search_with_equal_limit_two -- +3 +4 + +-- !search_with_equal_offset -- +4 + +-- !search_with_match_predicate -- +3 + +-- !search_with_range_predicate -- +3 + +-- !multiple_search_predicates -- +4 + +-- !search_with_score_range_only -- +1 +3 + +-- !search_with_score_range_and_other_predicate -- +3 + +-- !nested_search_with_other_predicate -- +7 + +-- !not_search_with_other_search -- +6 diff --git a/regression-test/suites/inverted_index_p0/test_search_score_topn_predicates.groovy b/regression-test/suites/inverted_index_p0/test_search_score_topn_predicates.groovy new file mode 100644 index 00000000000000..7076e81ea5801d --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_search_score_topn_predicates.groovy @@ -0,0 +1,191 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_search_score_topn_predicates", "p0") { + sql "DROP TABLE IF EXISTS test_search_score_topn_predicates" + + sql """ + CREATE TABLE test_search_score_topn_predicates ( + id INT, + status VARCHAR(20), + plain_status VARCHAR(20), + category VARCHAR(20), + title TEXT, + body TEXT, + INDEX idx_status (status) USING INVERTED, + INDEX idx_category (category) USING INVERTED, + INDEX idx_title (title) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "true"), + INDEX idx_body (body) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "true") + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true" + ) + """ + + sql """ + INSERT INTO test_search_score_topn_predicates VALUES + (1, 'drop', 'drop', 'plain', 'apple apple apple apple apple apple apple apple apple apple apple apple', 'alpha'), + (2, 'keep', 'keep', 'plain', 'apple', 'alpha'), + (3, 'keep', 'keep', 'plain', 'apple apple apple apple apple', 'alpha'), + (4, 'keep', 'keep', 'plain', 'apple apple apple', 'beta beta beta'), + (5, 'drop', 'drop', 'plain', 'banana', 'beta beta beta beta beta beta beta beta beta beta beta beta'), + (6, 'keep', 'keep', 'plain', 'pear', 'beta beta beta beta beta'), + (7, 'keep', 'keep', 'special', 'cherry cherry cherry cherry', 'gamma'), + (8, 'drop', 'drop', 'special', 'cherry cherry cherry cherry cherry cherry cherry cherry cherry cherry', 'gamma') + """ + + sql "sync" + sql "set enable_nereids_planner = true" + sql "set enable_fallback_to_original_planner = false" + sql "set enable_segment_limit_pushdown = true" + sql "set enable_inverted_index_query_cache = false" + + qt_single_search """ + SELECT id FROM ( + SELECT id, score() AS s + FROM test_search_score_topn_predicates + WHERE search('title:apple') + ORDER BY s DESC + LIMIT 1 + ) t + ORDER BY id + """ + + qt_search_with_equal_predicate """ + SELECT id FROM ( + SELECT id, score() AS s + FROM test_search_score_topn_predicates + WHERE search('title:apple') AND status = 'keep' + ORDER BY s DESC + LIMIT 1 + ) t + ORDER BY id + """ + + qt_search_with_plain_equal_predicate """ + SELECT id FROM ( + SELECT id, score() AS s + FROM test_search_score_topn_predicates + WHERE search('title:apple') AND plain_status = 'keep' + ORDER BY s DESC + LIMIT 1 + ) t + ORDER BY id + """ + + qt_search_with_equal_limit_two """ + SELECT id FROM ( + SELECT id, score() AS s + FROM test_search_score_topn_predicates + WHERE search('title:apple') AND status = 'keep' + ORDER BY s DESC + LIMIT 2 + ) t + ORDER BY id + """ + + qt_search_with_equal_offset """ + SELECT id FROM ( + SELECT id, score() AS s + FROM test_search_score_topn_predicates + WHERE search('title:apple') AND status = 'keep' + ORDER BY s DESC + LIMIT 1 OFFSET 1 + ) t + ORDER BY id + """ + + qt_search_with_match_predicate """ + SELECT id FROM ( + SELECT id, score() AS s + FROM test_search_score_topn_predicates + WHERE search('title:apple') AND status MATCH 'keep' + ORDER BY s DESC + LIMIT 1 + ) t + ORDER BY id + """ + + qt_search_with_range_predicate """ + SELECT id FROM ( + SELECT id, score() AS s + FROM test_search_score_topn_predicates + WHERE search('title:apple') AND id > 1 + ORDER BY s DESC + LIMIT 1 + ) t + ORDER BY id + """ + + qt_multiple_search_predicates """ + SELECT id FROM ( + SELECT id, score() AS s + FROM test_search_score_topn_predicates + WHERE search('title:apple') AND search('body:beta') + ORDER BY s DESC + LIMIT 1 + ) t + ORDER BY id + """ + + qt_search_with_score_range_only """ + SELECT id FROM ( + SELECT id, score() AS s + FROM test_search_score_topn_predicates + WHERE search('title:apple') AND score() > 0 + ORDER BY s DESC + LIMIT 2 + ) t + ORDER BY id + """ + + qt_search_with_score_range_and_other_predicate """ + SELECT id FROM ( + SELECT id, score() AS s + FROM test_search_score_topn_predicates + WHERE search('title:apple') AND score() > 0 AND status = 'keep' + ORDER BY s DESC + LIMIT 1 + ) t + ORDER BY id + """ + + qt_nested_search_with_other_predicate """ + SELECT id FROM ( + SELECT id, score() AS s + FROM test_search_score_topn_predicates + WHERE (search('title:cherry') OR category MATCH 'special') AND status = 'keep' + ORDER BY s DESC + LIMIT 1 + ) t + ORDER BY id + """ + + qt_not_search_with_other_search """ + SELECT id FROM ( + SELECT id, score() AS s + FROM test_search_score_topn_predicates + WHERE NOT search('title:apple') AND search('body:beta') AND status = 'keep' + ORDER BY s DESC + LIMIT 1 + ) t + ORDER BY id + """ +} From a96afba8eea0ae305b0404caf34d1bb6ee3601f6 Mon Sep 17 00:00:00 2001 From: liangj777 <751234313@qq.com> Date: Tue, 18 Aug 2026 10:47:08 +0800 Subject: [PATCH 2/2] [fix](fe)check overflow and add a test case for it --- .../rewrite/PushDownScoreTopNIntoOlapScan.java | 5 +++-- .../test_search_score_topn_predicates.groovy | 13 +++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownScoreTopNIntoOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownScoreTopNIntoOlapScan.java index 47ce149aae8e55..104f2b0f706188 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownScoreTopNIntoOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownScoreTopNIntoOlapScan.java @@ -194,9 +194,10 @@ private Plan pushDown( } // When limit + offset overflows the long range, the pushed scan limit would wrap to a - // negative value; skip the push-down and let the TopN above the scan apply limit/offset. + // negative value. Fail with the same error as ordinary TopN instead of leaving score() + // unmaterialized and reporting an unrelated score() usage error. if (Utils.addOverflows(topN.getLimit(), topN.getOffset())) { - return null; + throw new AnalysisException("limit + offset overflows the long range"); } long scoreLimit = topN.getLimit() + topN.getOffset(); diff --git a/regression-test/suites/inverted_index_p0/test_search_score_topn_predicates.groovy b/regression-test/suites/inverted_index_p0/test_search_score_topn_predicates.groovy index 7076e81ea5801d..df780835751a14 100644 --- a/regression-test/suites/inverted_index_p0/test_search_score_topn_predicates.groovy +++ b/regression-test/suites/inverted_index_p0/test_search_score_topn_predicates.groovy @@ -188,4 +188,17 @@ suite("test_search_score_topn_predicates", "p0") { ) t ORDER BY id """ + + // limit + offset overflows the long range. score() must report the standard TopN + // overflow error instead of skipping score pushdown and reporting a score() usage error. + test { + sql """ + SELECT id, score() AS s + FROM test_search_score_topn_predicates + WHERE search('title:apple') + ORDER BY s DESC + LIMIT 9223372036854775807 OFFSET 9223372036854775807 + """ + exception "limit + offset overflows the long range" + } }