diff --git a/Cargo.lock b/Cargo.lock index 1e820b0c63d..5bee7197f24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4835,6 +4835,7 @@ dependencies = [ "rand_distr", "rangemap", "rayon", + "regex-syntax", "roaring", "rstest", "serde", diff --git a/Cargo.toml b/Cargo.toml index 68a9002872b..657d9fde72c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,7 +72,7 @@ lance-linalg = { version = "=8.0.0-beta.14", path = "./rust/lance-linalg" } lance-namespace = { version = "=8.0.0-beta.14", path = "./rust/lance-namespace" } lance-namespace-impls = { version = "=8.0.0-beta.14", path = "./rust/lance-namespace-impls" } lance-namespace-datafusion = { version = "=7.0.0-beta.9", path = "./rust/lance-namespace-datafusion" } -lance-namespace-reqwest-client = "0.8.4" +lance-namespace-reqwest-client = "0.8.6" lance-select = { version = "=8.0.0-beta.14", path = "./rust/lance-select" } lance-tokenizer = { version = "=8.0.0-beta.14", path = "./rust/lance-tokenizer" } lance-table = { version = "=8.0.0-beta.14", path = "./rust/lance-table" } @@ -180,6 +180,7 @@ rand_distr = { version = "0.5.1" } rand_xoshiro = "0.7.0" rangemap = { version = "1.0" } rayon = "1.10" +regex-syntax = "0.8.10" roaring = "0.11.4" rstest = "0.26.1" serde = { version = "^1" } diff --git a/docs/src/format/index/scalar/ngram.md b/docs/src/format/index/scalar/ngram.md index bdf78474d50..d437363d264 100644 --- a/docs/src/format/index/scalar/ngram.md +++ b/docs/src/format/index/scalar/ngram.md @@ -29,4 +29,10 @@ The N-gram index provides inexact results for the following query types: | Query Type | Description | Operation | Result Type | |----------------|--------------------------|-------------------------------------------------------|-------------| -| **contains** | Substring search in text | Finds all trigrams in query, intersects posting lists | AtMost | \ No newline at end of file +| **contains** | Substring search in text | Finds all trigrams in query, intersects posting lists | AtMost | +| **regexp_like** / **regexp_match** | Regular-expression match | Derives a necessary trigram condition from the pattern (AND of intersections, OR of unions), then rechecks the true regex | AtMost | +| **LIKE** (infix) | Wildcard match such as `%foo%bar%` | Uses the literal segments of the pattern as a trigram condition, then rechecks the LIKE | AtMost | + +Patterns from which no trigram can be derived - for example `a.b`, `.*`, +case-insensitive matches, or literal runs shorter than three characters - fall +back to rechecking every row. This is always correct, just not accelerated. diff --git a/java/lance-jni/src/merge_insert.rs b/java/lance-jni/src/merge_insert.rs index df4d63bd2f6..4d332ca2c09 100644 --- a/java/lance-jni/src/merge_insert.rs +++ b/java/lance-jni/src/merge_insert.rs @@ -12,7 +12,8 @@ use jni::objects::{JObject, JString, JValueGen}; use jni::sys::jlong; use lance::dataset::scanner::ExprFilter; use lance::dataset::{ - MergeInsertBuilder, MergeStats, WhenMatched, WhenNotMatched, WhenNotMatchedBySource, + MergeInsertBuilder, MergeStats, SourceDedupeBehavior, WhenMatched, WhenNotMatched, + WhenNotMatchedBySource, }; use lance_core::datatypes::Schema; use lance_index::mem_wal::MergedGeneration; @@ -52,6 +53,7 @@ fn inner_merge_insert<'local>( let retry_timeout_ms = extract_retry_timeout_ms(env, &jparam)?; let skip_auto_cleanup = extract_skip_auto_cleanup(env, &jparam)?; let use_index = extract_use_index(env, &jparam)?; + let source_dedupe_behavior = extract_source_dedupe_behavior(env, &jparam)?; let marked_generations = extract_marked_generations(env, &jparam)?; let (new_ds, merge_stats) = unsafe { @@ -71,6 +73,7 @@ fn inner_merge_insert<'local>( .retry_timeout(Duration::from_millis(retry_timeout_ms as u64)) .skip_auto_cleanup(skip_auto_cleanup) .use_index(use_index) + .source_dedupe_behavior(source_dedupe_behavior) .mark_generations_as_merged(marked_generations) .try_build()?; @@ -241,6 +244,29 @@ fn extract_use_index<'local>(env: &mut JNIEnv<'local>, jparam: &JObject) -> Resu Ok(use_index) } +fn extract_source_dedupe_behavior<'local>( + env: &mut JNIEnv<'local>, + jparam: &JObject, +) -> Result { + let behavior: JString = env + .call_method( + jparam, + "sourceDedupeBehaviorValue", + "()Ljava/lang/String;", + &[], + )? + .l()? + .into(); + let behavior = behavior.extract(env)?; + match behavior.as_str() { + "Fail" => Ok(SourceDedupeBehavior::Fail), + "FirstSeen" => Ok(SourceDedupeBehavior::FirstSeen), + _ => Err(Error::input_error(format!( + "Illegal source_dedupe_behavior: {behavior}", + ))), + } +} + fn extract_marked_generations<'local>( env: &mut JNIEnv<'local>, jparam: &JObject, diff --git a/java/src/main/java/org/lance/merge/MergeInsertParams.java b/java/src/main/java/org/lance/merge/MergeInsertParams.java index 2ae27b67cba..ab67e022604 100644 --- a/java/src/main/java/org/lance/merge/MergeInsertParams.java +++ b/java/src/main/java/org/lance/merge/MergeInsertParams.java @@ -39,6 +39,7 @@ public class MergeInsertParams { private long retryTimeoutMs = 30 * 1000; private boolean skipAutoCleanup = false; private boolean useIndex = true; + private SourceDedupeBehavior sourceDedupeBehavior = SourceDedupeBehavior.Fail; private List markedGenerations = Collections.emptyList(); public MergeInsertParams(List on) { @@ -244,6 +245,25 @@ public MergeInsertParams withUseIndex(boolean useIndex) { return this; } + /** + * Control how duplicate source rows that match the same target row are handled. + * + *

Default is {@link SourceDedupeBehavior#Fail}, which errors if the source contains duplicate + * join keys. Use {@link SourceDedupeBehavior#FirstSeen} to keep the first encountered row and + * skip subsequent duplicates. + * + *

If the source contains duplicates and {@code FirstSeen} behavior doesn't match your needs, + * sort the source data before passing it to the merge insert operation. + * + * @param sourceDedupeBehavior The behavior to apply when duplicate source rows are found + * @return This MergeInsertParams instance + */ + public MergeInsertParams withSourceDedupeBehavior(SourceDedupeBehavior sourceDedupeBehavior) { + Preconditions.checkNotNull(sourceDedupeBehavior); + this.sourceDedupeBehavior = sourceDedupeBehavior; + return this; + } + /** * Mark MemWAL generations as merged into the base table. * @@ -319,6 +339,14 @@ public boolean useIndex() { return useIndex; } + public SourceDedupeBehavior sourceDedupeBehavior() { + return sourceDedupeBehavior; + } + + public String sourceDedupeBehaviorValue() { + return sourceDedupeBehavior.name(); + } + @Override public String toString() { return MoreObjects.toStringHelper(this) @@ -337,6 +365,7 @@ public String toString() { .add("retryTimeoutMs", retryTimeoutMs) .add("skipAutoCleanup", skipAutoCleanup) .add("useIndex", useIndex) + .add("sourceDedupeBehavior", sourceDedupeBehavior) .toString(); } @@ -396,4 +425,18 @@ public enum WhenNotMatchedBySource { */ DeleteIf, } + + /** + * Describes how to handle duplicate source rows that match the same target row. + * + *

If the source contains duplicates and {@code FirstSeen} behavior doesn't match your needs, + * sort the source data before passing it to the merge insert operation. + */ + public enum SourceDedupeBehavior { + /** Fail the operation if duplicates are found (default). */ + Fail, + + /** Keep the first seen value and skip subsequent duplicates. */ + FirstSeen, + } } diff --git a/java/src/test/java/org/lance/MergeInsertTest.java b/java/src/test/java/org/lance/MergeInsertTest.java index b738ef8852d..944a9060675 100644 --- a/java/src/test/java/org/lance/MergeInsertTest.java +++ b/java/src/test/java/org/lance/MergeInsertTest.java @@ -257,6 +257,30 @@ private VectorSchemaRoot buildSource(Schema schema, RootAllocator allocator) { return root; } + /** + * Build a source whose join key {@code id=0} appears twice ("First 0", then "Second 0"), so the + * source-dedupe behavior is exercised. Remaining ids (3, 4) are unique matches. + */ + private VectorSchemaRoot buildDuplicateKeySource(Schema schema, RootAllocator allocator) { + List sourceIds = Arrays.asList(0, 0, 3, 4); + List sourceNames = Arrays.asList("First 0", "Second 0", "Source 3", "Source 4"); + + VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator); + root.allocateNew(); + + IntVector idVector = (IntVector) root.getVector("id"); + VarCharVector nameVector = (VarCharVector) root.getVector("name"); + + for (int i = 0; i < sourceIds.size(); i++) { + idVector.setSafe(i, sourceIds.get(i)); + nameVector.setSafe(i, sourceNames.get(i).getBytes(StandardCharsets.UTF_8)); + } + + root.setRowCount(sourceIds.size()); + + return root; + } + private ArrowArrayStream convertToStream(VectorSchemaRoot root, RootAllocator allocator) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); @@ -275,6 +299,55 @@ private ArrowArrayStream convertToStream(VectorSchemaRoot root, RootAllocator al return stream; } + @Test + public void testSourceDedupeFirstSeenKeepsFirst() throws Exception { + // Source has two rows for id=0 ("First 0" then "Second 0"). FirstSeen keeps + // the first encountered row and skips the duplicate. + + try (VectorSchemaRoot source = buildDuplicateKeySource(testDataset.getSchema(), allocator)) { + try (ArrowArrayStream sourceStream = convertToStream(source, allocator)) { + MergeInsertResult result = + dataset.mergeInsert( + new MergeInsertParams(Collections.singletonList("id")) + .withMatchedUpdateAll() + .withNotMatched(MergeInsertParams.WhenNotMatched.InsertAll) + .withSourceDedupeBehavior(MergeInsertParams.SourceDedupeBehavior.FirstSeen), + sourceStream); + + Assertions.assertEquals( + "{0=First 0, 1=Person 1, 2=Person 2, 3=Source 3, 4=Source 4}", + readAll(result.dataset()).toString(), + "FirstSeen should keep the first duplicate source row (id=0) and update unique matches"); + } + } + } + + @Test + public void testSourceDedupeFailWithDuplicates() throws Exception { + // Default behavior (Fail) must error when the source contains duplicate join keys. + + try (VectorSchemaRoot source = buildDuplicateKeySource(testDataset.getSchema(), allocator)) { + try (ArrowArrayStream sourceStream = convertToStream(source, allocator)) { + String originalDataset = readAll(dataset).toString(); + + Assertions.assertThrows( + Exception.class, + () -> + dataset.mergeInsert( + new MergeInsertParams(Collections.singletonList("id")) + .withMatchedUpdateAll() + .withNotMatched(MergeInsertParams.WhenNotMatched.InsertAll) + .withSourceDedupeBehavior(MergeInsertParams.SourceDedupeBehavior.Fail), + sourceStream)); + + Assertions.assertEquals( + originalDataset, + readAll(dataset).toString(), + "Dataset should remain unchanged after a failed mergeInsert"); + } + } + } + @Test public void testMergeInsertWithoutIndex() throws Exception { // Verify that merge insert with useIndex=false still completes and diff --git a/python/Cargo.lock b/python/Cargo.lock index 5dfc1ba47e0..37035be3085 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -4431,6 +4431,7 @@ dependencies = [ "rand_distr", "rangemap", "rayon", + "regex-syntax", "roaring", "serde", "serde_json", diff --git a/python/python/lance/lance/optimize.pyi b/python/python/lance/lance/optimize.pyi index 9a26d23c003..c4b6b6546e6 100644 --- a/python/python/lance/lance/optimize.pyi +++ b/python/python/lance/lance/optimize.pyi @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List +from typing import List, Optional from lance import LanceDataset from lance.fragment import FragmentMetadata @@ -51,5 +51,7 @@ class Compaction: def plan(dataset: "LanceDataset", options: CompactionOptions) -> CompactionPlan: ... @staticmethod def commit( - dataset: "LanceDataset", rewrites: List[RewriteResult] + dataset: "LanceDataset", + rewrites: List[RewriteResult], + options: Optional[CompactionOptions] = None, ) -> CompactionMetrics: ... diff --git a/python/python/lance/optimize.py b/python/python/lance/optimize.py index 8b98308d442..3ac7547960b 100644 --- a/python/python/lance/optimize.py +++ b/python/python/lance/optimize.py @@ -57,6 +57,14 @@ class CompactionOptions(TypedDict): The batch size to use when scanning input fragments. You may want to reduce this if you are running out of memory during compaction. + The default will use the same default from ``scanner``. + """ + io_buffer_size: Optional[int] + """ + The number of bytes to allow to queue up in the I/O buffer when scanning + input fragments. Increasing this can avoid a deadlock that occurs when a + single batch of data is larger than the I/O buffer size. + The default will use the same default from ``scanner``. """ compaction_mode: Optional[ diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index cb34ae3cafc..fc879c9cbaa 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -1460,6 +1460,38 @@ def test_read_blobs_resolves_nested_field_path(dataset_with_nested_blobs): assert [data for _, data in results] == [b"foo", b"baz"] +def test_write_nested_blob_v2_and_take_by_field_path(tmp_path): + packed = b"x" * (70 * 1024) + blob_field = lance.blob_field("blob") + info_fields = [pa.field("name", pa.string()), blob_field] + info_type = pa.struct(info_fields) + info_array = pa.StructArray.from_arrays( + [pa.array(["a", "b", "c"]), lance.blob_array([b"foo", packed, None])], + fields=info_fields, + ) + table = pa.table( + [info_array], + schema=pa.schema([pa.field("info", info_type)]), + ) + + dataset = lance.write_dataset( + table, + tmp_path / "nested_blob_v2", + data_storage_version="2.2", + ) + + desc = dataset.to_table(columns=["info.blob"]).column("info.blob").chunk(0) + assert desc.field("kind").to_pylist()[:2] == [0, 1] + + blobs = dataset.take_blobs("info.blob", indices=[0, 1]) + with blobs[0] as f: + assert f.read() == b"foo" + with blobs[1] as f: + assert f.read() == packed + + assert dataset.take_blobs("info.blob", indices=[2]) == [] + + def test_to_pandas_returns_blob_files_for_projected_nested_fields( dataset_with_nested_blobs, ): diff --git a/python/python/tests/test_optimize.py b/python/python/tests/test_optimize.py index ccd889db116..049ce2cc3a5 100644 --- a/python/python/tests/test_optimize.py +++ b/python/python/tests/test_optimize.py @@ -324,6 +324,47 @@ def test_defer_index_remap(tmp_path: Path): assert any(idx.name == "__lance_frag_reuse" for idx in indices) +@pytest.mark.parametrize("use_commit_options", [True, False]) +def test_defer_index_remap_via_commit_options(tmp_path: Path, use_commit_options: bool): + """Compaction.commit respects defer_index_remap passed in options. + + When options={"defer_index_remap": True} is supplied to Compaction.commit + the __lance_frag_reuse system index must appear in describe_indices(). + When the option is omitted (default) no such system index is written. + """ + base_dir = tmp_path / f"dataset_commit_opts_{use_commit_options}" + data = pa.table({"i": range(6_000), "val": range(6_000)}) + dataset = lance.write_dataset(data, base_dir, max_rows_per_file=1_000) + dataset.create_scalar_index("i", "BTREE") + dataset.delete("i < 500") + + plan = Compaction.plan( + dataset, + options=dict(target_rows_per_fragment=2_000, num_threads=1), + ) + rewrites = [task.execute(dataset) for task in plan.tasks] + + if use_commit_options: + Compaction.commit(dataset, rewrites, options={"defer_index_remap": True}) + else: + Compaction.commit(dataset, rewrites) + + dataset = lance.dataset(base_dir) + indices = dataset.describe_indices() + has_frag_reuse = any(idx.name == "__lance_frag_reuse" for idx in indices) + + if use_commit_options: + assert has_frag_reuse, ( + "expected __lance_frag_reuse system index when defer_index_remap=True " + "is passed to Compaction.commit" + ) + else: + assert not has_frag_reuse, ( + "did not expect __lance_frag_reuse system index when options is omitted " + "from Compaction.commit" + ) + + @pytest.mark.filterwarnings("ignore::DeprecationWarning") def test_describe_indices_matches_list_indices_for_frag_reuse(tmp_path: Path): """describe_indices() and list_indices() must agree on the index_type diff --git a/python/python/tests/test_vector_index.py b/python/python/tests/test_vector_index.py index 292b8079706..4e3addfedb8 100644 --- a/python/python/tests/test_vector_index.py +++ b/python/python/tests/test_vector_index.py @@ -1772,6 +1772,8 @@ def test_index_cast_centroids(tmp_path): values = pa.array([x for arr in centroids for x in arr], pa.float32()) centroids = pa.FixedSizeListArray.from_arrays(values, 128) + # Cast invalidates the attached index; drop it first per the new contract. + dataset.drop_index(index_name) dataset.alter_columns(dict(path="vector", data_type=pa.list_(pa.float16(), 128))) # centroids are f32, but the column is now f16 diff --git a/python/src/dataset/optimize.rs b/python/src/dataset/optimize.rs index 321d7157b86..4bb29246f45 100644 --- a/python/src/dataset/optimize.rs +++ b/python/src/dataset/optimize.rs @@ -58,6 +58,9 @@ fn parse_compaction_options( "batch_size" => { opts.batch_size = value.extract()?; } + "io_buffer_size" => { + opts.io_buffer_size = value.extract()?; + } "compaction_mode" => { let mode_str: Option = value.extract()?; if let Some(mode_str) = mode_str { @@ -551,26 +554,34 @@ impl PyCompaction { /// new version once committed. /// rewrites : List[RewriteResult] /// The results of the compaction tasks to include in the commit. + /// options : dict, optional + /// Compaction options to apply at commit time. + /// When absent or ``None``, defaults to ``CompactionOptions::default()``. /// /// Returns /// ------- /// CompactionMetrics #[staticmethod] + #[pyo3(signature = (dataset, rewrites, options = None))] pub fn commit( dataset: Bound, rewrites: Vec, + options: Option>, ) -> PyResult { let dataset_ref = unwrap_dataset(dataset)?; let dataset = dataset_ref.borrow().clone(); + let config = dataset.ds.manifest.config.clone(); + let opts = match options { + Some(ref dict) => parse_compaction_options(dict, &config)?, + None => CompactionOptions::default(), + }; let rewrites: Vec = rewrites.into_iter().map(|r| r.0).collect(); let mut new_ds = dataset.ds.as_ref().clone(); - // TODO: pass compaction option from plan and execute time - let options: CompactionOptions = CompactionOptions::default(); let fut = commit_compaction( &mut new_ds, rewrites, Arc::new(DatasetIndexRemapperOptions::default()), - &options, + &opts, ); let metrics = rt() .block_on(None, fut)? diff --git a/rust/lance-core/src/datatypes/field.rs b/rust/lance-core/src/datatypes/field.rs index 4c2665a3640..9f06d421949 100644 --- a/rust/lance-core/src/datatypes/field.rs +++ b/rust/lance-core/src/datatypes/field.rs @@ -575,6 +575,18 @@ impl Field { } } + /// Convert blob v2 fields in this field tree to their descriptor view. + pub fn unload_blobs_recursive(&mut self) { + if self.is_blob_v2() { + self.unloaded_mut(); + return; + } + + for child in &mut self.children { + child.unload_blobs_recursive(); + } + } + pub fn project(&self, path_components: &[&str]) -> Result { let mut f = Self { name: self.name.clone(), @@ -1864,6 +1876,54 @@ mod tests { assert_eq!(field.logical_type, BLOB_V2_DESC_LANCE_FIELD.logical_type); } + #[test] + fn unload_blobs_recursive_only_unloads_blob_v2() { + let legacy_metadata = HashMap::from([(BLOB_META_KEY.to_string(), "true".to_string())]); + let blob_v2_metadata = + HashMap::from([(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string())]); + + let mut field: Field = ArrowField::new( + "parent", + DataType::Struct(Fields::from(vec![ + ArrowField::new("legacy_blob", DataType::LargeBinary, true) + .with_metadata(legacy_metadata), + ArrowField::new( + "blob_v2", + DataType::Struct( + vec![ + ArrowField::new("data", DataType::LargeBinary, true), + ArrowField::new("uri", DataType::Utf8, true), + ] + .into(), + ), + true, + ) + .with_metadata(blob_v2_metadata), + ])), + true, + ) + .try_into() + .unwrap(); + + field.unload_blobs_recursive(); + + let legacy_blob = field + .children + .iter() + .find(|f| f.name == "legacy_blob") + .unwrap(); + assert_eq!( + legacy_blob.logical_type, + LogicalType::try_from(&DataType::LargeBinary).unwrap() + ); + assert_eq!(legacy_blob.children.len(), 0); + assert!(legacy_blob.metadata.contains_key(BLOB_META_KEY)); + + let blob_v2 = field.children.iter().find(|f| f.name == "blob_v2").unwrap(); + assert_eq!(blob_v2.logical_type, BLOB_V2_DESC_LANCE_FIELD.logical_type); + assert_eq!(blob_v2.children.len(), 5); + } + #[test] fn project_by_field_accepts_blob_descriptor_projection() { let metadata = HashMap::from([(BLOB_META_KEY.to_string(), "true".to_string())]); diff --git a/rust/lance-file/src/writer.rs b/rust/lance-file/src/writer.rs index 14a4c82bde6..12bd50df6fe 100644 --- a/rust/lance-file/src/writer.rs +++ b/rust/lance-file/src/writer.rs @@ -633,14 +633,11 @@ impl FileWriter { async fn write_global_buffers(&mut self) -> Result> { let schema = self.schema.as_mut().ok_or(Error::invalid_input("No schema provided on writer open and no data provided. Schema is unknown and file cannot be created"))?; schema.metadata = std::mem::take(&mut self.schema_metadata); - // Use descriptor layout for blob v2 in the footer to avoid exposing logical child fields. - // - // TODO(xuanwo): this doesn't work on nested struct, need better solution like fields_per_order_mut? - schema.fields.iter_mut().for_each(|f| { - if f.is_blob_v2() { - f.unloaded_mut(); - } - }); + // Use descriptor layout for blob v2 fields in the footer to avoid exposing logical child fields. + schema + .fields + .iter_mut() + .for_each(|f| f.unload_blobs_recursive()); let file_descriptor = Self::make_file_descriptor(schema, self.rows_written)?; let file_descriptor_bytes = file_descriptor.encode_to_vec(); diff --git a/rust/lance-index/Cargo.toml b/rust/lance-index/Cargo.toml index e3947b57856..85de43c0f9b 100644 --- a/rust/lance-index/Cargo.toml +++ b/rust/lance-index/Cargo.toml @@ -56,6 +56,7 @@ object_store.workspace = true prost.workspace = true prost-types.workspace = true rand.workspace = true +regex-syntax.workspace = true roaring.workspace = true rayon.workspace = true serde_json.workspace = true diff --git a/rust/lance-index/benches/rq.rs b/rust/lance-index/benches/rq.rs index e29ce9c4695..72e0c49820d 100644 --- a/rust/lance-index/benches/rq.rs +++ b/rust/lance-index/benches/rq.rs @@ -512,9 +512,138 @@ fn ex_bulk_paths(c: &mut Criterion) { } } +/// Top-k accumulation through the gated raw-query multi-bit path: binary +/// FastScan, the per-row lower-bound pruning scan, and the exact rerank of +/// the surviving rows. Error factors are present so the gating is enabled. +fn heap_topk(c: &mut Criterion) { + use arrow_array::{ArrayRef, FixedSizeListArray, Float32Array, UInt8Array, UInt64Array}; + use lance_arrow::FixedSizeListArrayExt; + use lance_index::vector::ApproxMode; + use lance_index::vector::bq::transform::{ + ERROR_FACTORS_COLUMN, EX_ADD_FACTORS_COLUMN, EX_SCALE_FACTORS_COLUMN, + }; + use lance_index::vector::storage::DistanceCalculatorOptions; + use std::collections::BinaryHeap; + use std::sync::Arc; + + const TOPK_DIM: usize = 1536; + const TOPK_ROWS: usize = 4096; + const TOPK_K: usize = 10; + const NUM_BITS: u8 = 5; + let ex_bits = NUM_BITS - 1; + + let mut rng = SmallRng::seed_from_u64(99); + let rq = RabitQuantizer::new_with_rotation::( + NUM_BITS, + TOPK_DIM as i32, + RQRotationType::Fast, + ); + let metadata = rq.metadata(None); + + let code_len = TOPK_DIM / 8; + let binary_codes = (0..TOPK_ROWS * code_len) + .map(|_| rng.random()) + .collect::>(); + let ex_code_len = blocked_ex_code_bytes(TOPK_DIM, ex_bits); + let ex_codes = (0..TOPK_ROWS * ex_code_len) + .map(|_| rng.random()) + .collect::>(); + // Factor magnitudes chosen so the lower bounds spread mostly with the add + // factors; once the heap is full the threshold prunes the vast majority + // of rows, like a production multi-partition scan. + let mut rand_factors = |low: f32, high: f32| { + Arc::new(Float32Array::from( + (0..TOPK_ROWS) + .map(|_| rng.random_range(low..high)) + .collect::>(), + )) as ArrayRef + }; + let batch = arrow_array::RecordBatch::try_from_iter(vec![ + ( + ROW_ID, + Arc::new(UInt64Array::from_iter_values(0..TOPK_ROWS as u64)) as ArrayRef, + ), + ( + RABIT_CODE_COLUMN, + Arc::new( + FixedSizeListArray::try_new_from_values( + UInt8Array::from(binary_codes), + code_len as i32, + ) + .unwrap(), + ) as ArrayRef, + ), + (ADD_FACTORS_COLUMN, rand_factors(0.0, 1.0)), + (SCALE_FACTORS_COLUMN, rand_factors(0.0005, 0.0015)), + (ERROR_FACTORS_COLUMN, rand_factors(0.0, 0.01)), + ( + RABIT_BLOCKED_EX_CODE_COLUMN, + Arc::new( + FixedSizeListArray::try_new_from_values( + UInt8Array::from(ex_codes), + ex_code_len as i32, + ) + .unwrap(), + ) as ArrayRef, + ), + (EX_ADD_FACTORS_COLUMN, rand_factors(0.0, 1.0)), + (EX_SCALE_FACTORS_COLUMN, rand_factors(0.00003, 0.0001)), + ]) + .unwrap(); + let storage = + RabitQuantizationStorage::try_from_batch(batch, &metadata, DistanceType::L2, None).unwrap(); + let query: ArrayRef = Arc::new(Float32Array::from( + (0..TOPK_DIM) + .map(|_| rng.random_range(-1.0f32..1.0)) + .collect::>(), + )); + + for (label, approx_mode) in [ + ("normal", ApproxMode::Normal), + ("accurate", ApproxMode::Accurate), + ] { + let mut f32_scratch = Vec::new(); + let calc = storage.dist_calculator_with_scratch( + query.clone(), + 1.0, + None, + &mut f32_scratch, + DistanceCalculatorOptions { approx_mode }, + ); + let mut heap = BinaryHeap::with_capacity(TOPK_K + 1); + let mut dists = Vec::new(); + let mut u16_scratch = Vec::new(); + let mut u8_scratch = Vec::new(); + let mut u32_scratch = Vec::new(); + c.bench_function( + format!( + "RQ heap topk ({label}): num_bits={NUM_BITS}, DIM={TOPK_DIM}, rows={TOPK_ROWS}, k={TOPK_K}" + ) + .as_str(), + |b| { + b.iter(|| { + heap.clear(); + calc.accumulate_topk_with_scratch( + TOPK_K, + None, + None, + |id| id as u64, + &mut heap, + &mut dists, + &mut u16_scratch, + &mut u8_scratch, + &mut u32_scratch, + ); + black_box(heap.len()) + }) + }, + ); + } +} + criterion_group!( name=benches; config = Criterion::default().measurement_time(Duration::from_secs(10)); - targets = construct_dist_table, compute_distances, ex_dot_kernels, ex_code_storage_load, ex_bulk_paths); + targets = construct_dist_table, compute_distances, ex_dot_kernels, ex_code_storage_load, ex_bulk_paths, heap_topk); criterion_main!(benches); diff --git a/rust/lance-index/src/scalar.rs b/rust/lance-index/src/scalar.rs index daec92339f8..a287d277a81 100644 --- a/rust/lance-index/src/scalar.rs +++ b/rust/lance-index/src/scalar.rs @@ -8,6 +8,7 @@ use arrow_array::{BooleanArray, ListArray, RecordBatch, UInt64Array}; use arrow_schema::{Field, Schema}; use async_trait::async_trait; use bytes::Bytes; +use datafusion::functions::regex::regexplike::RegexpLikeFunc; use datafusion::functions::string::contains::ContainsFunc; use datafusion::functions_nested::array_has; use datafusion::physical_plan::SendableRecordBatchStream; @@ -288,6 +289,22 @@ pub trait IndexStore: std::fmt::Debug + Send + Sync + DeepSizeOf { /// This is often useful when remapping or updating async fn copy_index_file(&self, name: &str, dest_store: &dyn IndexStore) -> Result; + /// Copy an index file from this store to a new name in another store, leaving the source intact + async fn copy_index_file_to( + &self, + name: &str, + new_name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + if name == new_name { + self.copy_index_file(name, dest_store).await + } else { + Err(Error::not_supported(format!( + "copying index file {name} to {new_name} is not supported by this index store" + ))) + } + } + /// Rename an index file async fn rename_index_file(&self, name: &str, new_name: &str) -> Result; @@ -633,9 +650,15 @@ impl AnyQuery for LabelListQuery { pub enum TextQuery { /// Retrieve all row ids where the text contains the given string StringContains(String), - // TODO: In the future we should be able to do string-insensitive contains - // as well as partial matches (e.g. LIKE 'foo%') and potentially even - // some regular expressions + /// Retrieve all row ids whose text matches the given regular expression. + /// + /// The pattern is a full regular expression (as accepted by `regexp_like`). + /// The index returns a candidate superset that the scan rechecks, so any + /// pattern is sound; patterns with no usable trigram structure simply fall + /// back to rechecking every row. + Regex(String), + // TODO: In the future we should be able to do case-insensitive contains + // as well as partial matches (e.g. LIKE 'foo%'). } impl AnyQuery for TextQuery { @@ -656,6 +679,17 @@ impl AnyQuery for TextQuery { Expr::Literal(ScalarValue::Utf8(Some(substr.clone())), None), ], }), + // `regexp_like` returns Boolean directly, so the reconstructed + // expression can be used as-is for the recheck filter (no IsNotNull + // wrapper, unlike `regexp_match`). It is the semantic equivalent of + // the original predicate for the "does it match" question. + Self::Regex(pattern) => Expr::ScalarFunction(ScalarFunction { + func: Arc::new(RegexpLikeFunc::new().into()), + args: vec![ + Expr::Column(Column::new_unqualified(col)), + Expr::Literal(ScalarValue::Utf8(Some(pattern.clone())), None), + ], + }), } } diff --git a/rust/lance-index/src/scalar/expression.rs b/rust/lance-index/src/scalar/expression.rs index 38a29e9c43c..ea7fbabc813 100644 --- a/rust/lance-index/src/scalar/expression.rs +++ b/rust/lance-index/src/scalar/expression.rs @@ -781,20 +781,28 @@ impl ScalarQueryParser for LabelListQueryParser { } } -/// A parser for indices that handle string contains queries +/// A parser for indices that handle string `contains` queries, and -- when +/// `supports_regex` is set -- `regexp_like` / `regexp_match` queries. #[derive(Debug, Clone)] pub struct TextQueryParser { index_name: String, index_type: String, needs_recheck: bool, + supports_regex: bool, } impl TextQueryParser { - pub fn new(index_name: String, index_type: String, needs_recheck: bool) -> Self { + pub fn new( + index_name: String, + index_type: String, + needs_recheck: bool, + supports_regex: bool, + ) -> Self { Self { index_name, index_type, needs_recheck, + supports_regex, } } } @@ -837,31 +845,156 @@ impl ScalarQueryParser for TextQueryParser { func: &ScalarUDF, args: &[Expr], ) -> Option { - if args.len() != 2 { + // The first argument is the indexed column; the second is the substring + // / pattern. `contains` takes exactly two arguments; the regex functions + // optionally take a third flags argument. + if args.len() < 2 { return None; } - let scalar = maybe_scalar(&args[1], data_type)?; - match scalar { - ScalarValue::Utf8(Some(scalar_str)) | ScalarValue::LargeUtf8(Some(scalar_str)) => { - if func.name() == "contains" { - let query = TextQuery::StringContains(scalar_str); - Some(IndexedExpression::index_query_with_recheck( - column.to_string(), - self.index_name.clone(), - self.index_type.clone(), - Arc::new(query), - self.needs_recheck, - )) - } else { + // A non-string pattern cannot be handled. + let (ScalarValue::Utf8(Some(pattern)) | ScalarValue::LargeUtf8(Some(pattern))) = + maybe_scalar(&args[1], data_type)? + else { + return None; + }; + + let query = match func.name() { + "contains" if args.len() == 2 => TextQuery::StringContains(pattern), + "regexp_like" | "regexp_match" if self.supports_regex => { + let pattern = match args.get(2) { + Some(flags_expr) => apply_regex_flags(&pattern, flags_expr)?, + None => pattern, + }; + // If the pattern yields no usable trigram (e.g. `a.b`), leave it + // to a full scan instead of routing it to the index, which could + // only answer with an unsupported "recheck everything" result. + if !crate::scalar::ngram::regex_can_use_index(&pattern) { + return None; + } + TextQuery::Regex(pattern) + } + _ => return None, + }; + + Some(IndexedExpression::index_query_with_recheck( + column.to_string(), + self.index_name.clone(), + self.index_type.clone(), + Arc::new(query), + self.needs_recheck, + )) + } + + fn visit_like( + &self, + column: &str, + like: &Like, + pattern: &ScalarValue, + ) -> Option { + // Infix LIKE is accelerated only by the ngram index (via its regex + // machinery). A plain-literal `regexp_like(col, 'foo')` is rewritten to + // `col LIKE '%foo%'` before it reaches the index, so this is the path + // that accelerates those. ILIKE is skipped because its case folding does + // not match the index's normalization. + if !self.supports_regex || like.case_insensitive { + return None; + } + let pattern_str = match pattern { + ScalarValue::Utf8(Some(s)) | ScalarValue::LargeUtf8(Some(s)) => s.as_str(), + _ => return None, + }; + // Translate the LIKE pattern into a loose regex used only for candidate + // generation; the original LIKE stays as the recheck filter, so the + // regex only needs to be a sound superset. + let regex = like_to_regex(pattern_str, like.escape_char)?; + if !crate::scalar::ngram::regex_can_use_index(®ex) { + return None; + } + Some(IndexedExpression { + scalar_query: Some(ScalarIndexExpr::Query(ScalarIndexSearch { + column: column.to_string(), + index_name: self.index_name.clone(), + index_type: self.index_type.clone(), + query: Arc::new(TextQuery::Regex(regex)), + needs_recheck: self.needs_recheck, + fragment_bitmap: None, + })), + refine_expr: Some(Expr::Like(like.clone())), + }) + } +} + +/// Translate a LIKE pattern into a regular expression used purely for ngram +/// candidate generation: `%` becomes `.*`, `_` becomes `.`, and literal +/// characters are regex-escaped. Returns `None` when no literal run is long +/// enough to yield a trigram (the index could not help, so a full scan is left +/// to handle it). +fn like_to_regex(pattern: &str, escape: Option) -> Option { + let mut regex = String::new(); + let mut run = 0usize; + let mut longest_run = 0usize; + let mut chars = pattern.chars(); + while let Some(c) = chars.next() { + let literal = if Some(c) == escape { + // The next character is escaped, i.e. a literal. + chars.next() + } else { + match c { + '%' => { + regex.push_str(".*"); + run = 0; None } + '_' => { + regex.push('.'); + run = 0; + None + } + other => Some(other), } - _ => { - // If the scalar is not a string, we cannot handle it - None + }; + if let Some(lit) = literal { + if regex_syntax::is_meta_character(lit) { + regex.push('\\'); + } + regex.push(lit); + // Only runs of alphanumeric characters can produce a trigram. + if lit.is_alphanumeric() { + run += 1; + longest_run = longest_run.max(run); + } else { + run = 0; } } } + (longest_run >= 3).then_some(regex) +} + +/// Fold the supported `regexp_like` / `regexp_match` flags into an inline prefix +/// on the pattern (e.g. flags `"i"` -> `"(?i)pattern"`). Returns `None` for a +/// non-literal flags argument or an unrecognized flag, so the caller leaves the +/// predicate to a full recheck rather than risk changing its semantics. +fn apply_regex_flags(pattern: &str, flags_expr: &Expr) -> Option { + let (Expr::Literal(ScalarValue::Utf8(Some(flags)), _) + | Expr::Literal(ScalarValue::LargeUtf8(Some(flags)), _)) = flags_expr + else { + return None; + }; + let mut inline = String::new(); + for flag in flags.chars() { + // Only flags expressible as an inline `(?...)` group in the regex crate + // (which the recheck uses) are safe to fold. + if ['i', 's', 'm', 'x'].contains(&flag) { + inline.push(flag); + } else { + return None; + } + } + if inline.is_empty() { + Some(pattern.to_string()) + } else { + Some(format!("(?{inline}){pattern}")) + } } /// A parser for indices that handle queries with the contains_tokens function @@ -1813,7 +1946,18 @@ fn visit_node( Expr::IsFalse(expr) => Ok(visit_is_bool(expr.as_ref(), index_info, false)), Expr::IsTrue(expr) => Ok(visit_is_bool(expr.as_ref(), index_info, true)), Expr::IsNull(expr) => Ok(visit_is_null(expr.as_ref(), index_info, false)), - Expr::IsNotNull(expr) => Ok(visit_is_null(expr.as_ref(), index_info, true)), + Expr::IsNotNull(expr) => { + // `regexp_match(col, pat)` returns a list and is coerced to + // `IsNotNull(regexp_match(...))` before it reaches here. Unwrap that + // so the regex acceleration applies; everything else is a genuine + // IS NOT NULL check. + if let Expr::ScalarFunction(scalar_fn) = expr.as_ref() + && scalar_fn.func.name() == "regexp_match" + { + return Ok(visit_scalar_fn(scalar_fn, index_info)); + } + Ok(visit_is_null(expr.as_ref(), index_info, true)) + } Expr::Not(expr) => visit_not(expr.as_ref(), index_info, depth), Expr::BinaryExpr(binary_expr) => visit_binary_expr(binary_expr, index_info, depth), Expr::ScalarFunction(scalar_fn) => Ok(visit_scalar_fn(scalar_fn, index_info)), @@ -2690,6 +2834,59 @@ mod tests { assert!(matches!(negated.upper, NullableRowAddrMask::BlockList(_))); } + #[test] + fn test_like_to_regex() { + // `%` -> `.*`, `_` -> `.`, with a literal run of at least three chars. + assert_eq!(like_to_regex("%foo%", None).as_deref(), Some(".*foo.*")); + assert_eq!(like_to_regex("foo%bar", None).as_deref(), Some("foo.*bar")); + assert_eq!(like_to_regex("foo_bar", None).as_deref(), Some("foo.bar")); + assert_eq!(like_to_regex("foobar", None).as_deref(), Some("foobar")); + + // Regex metacharacters in the literal portion are escaped. + assert_eq!( + like_to_regex("%a.bcd%", None).as_deref(), + Some(".*a\\.bcd.*") + ); + + // No literal run of three alphanumeric characters -> no index help. + assert_eq!(like_to_regex("%ab%", None), None); + assert_eq!(like_to_regex("%a%b%c%", None), None); + assert_eq!(like_to_regex("%", None), None); + + // The escape character makes the following character a literal. + assert_eq!( + like_to_regex(r"%foo\%bar%", Some('\\')).as_deref(), + Some(".*foo%bar.*") + ); + } + + #[test] + fn test_apply_regex_flags() { + fn flags(s: &str) -> Expr { + Expr::Literal(ScalarValue::Utf8(Some(s.to_string())), None) + } + + // Empty flags leave the pattern untouched (no inline group emitted). + assert_eq!(apply_regex_flags("foo", &flags("")).as_deref(), Some("foo")); + // Supported flags are folded into an inline `(?...)` prefix. + assert_eq!( + apply_regex_flags("foo", &flags("i")).as_deref(), + Some("(?i)foo") + ); + assert_eq!( + apply_regex_flags("foo", &flags("is")).as_deref(), + Some("(?is)foo") + ); + // An unrecognized flag bails out so the caller leaves the predicate to a + // full recheck rather than risk changing its semantics. + assert_eq!(apply_regex_flags("foo", &flags("g")), None); + // A non-string (hence non-literal-flags) argument cannot be folded. + assert_eq!( + apply_regex_flags("foo", &Expr::Literal(ScalarValue::Int32(Some(1)), None)), + None + ); + } + #[test] fn test_extract_like_leading_prefix() { // Simple prefix patterns (no recheck needed) diff --git a/rust/lance-index/src/scalar/fmindex.rs b/rust/lance-index/src/scalar/fmindex.rs index 9677f7471ea..aed1136535a 100644 --- a/rust/lance-index/src/scalar/fmindex.rs +++ b/rust/lance-index/src/scalar/fmindex.rs @@ -1352,6 +1352,12 @@ impl ScalarIndex for FMIndexScalarIndex { Default::default(), ))) } + // Regex queries are routed only to the ngram index (the FM-index's + // query parser advertises `supports_regex = false`), so this is + // unreachable in practice; reject it explicitly rather than silently. + TextQuery::Regex(_) => Err(Error::invalid_input( + "FMIndex does not support regular expression queries", + )), } } fn can_remap(&self) -> bool { @@ -1645,6 +1651,9 @@ impl ScalarIndexPlugin for FMIndexPlugin { Some(Box::new(TextQueryParser::new( index_name, self.name().to_string(), + // needs_recheck: the FM-index returns exact substring matches. + false, + // supports_regex: regex acceleration is only implemented for ngram. false, ))) } diff --git a/rust/lance-index/src/scalar/inverted/builder.rs b/rust/lance-index/src/scalar/inverted/builder.rs index 24b1eb50203..93932f35332 100644 --- a/rust/lance-index/src/scalar/inverted/builder.rs +++ b/rust/lance-index/src/scalar/inverted/builder.rs @@ -407,11 +407,21 @@ impl InvertedIndexBuilder { ) -> Result> { let partition_id = self.next_partition_id() | self.fragment_mask.unwrap_or(0); builder.set_id(partition_id); - let files = builder.write(dest_store).await?; + let files = builder + .write_to(dest_store, self.partition_write_target()) + .await?; self.new_partitions.push(partition_id); Ok(files) } + fn partition_write_target(&self) -> PartitionWriteTarget { + if self.fragment_mask.is_some() { + PartitionWriteTarget::Staged + } else { + PartitionWriteTarget::Final + } + } + fn next_partition_id(&self) -> u64 { self.partitions .iter() @@ -523,7 +533,11 @@ impl InvertedIndexBuilder { if let Some(builder) = merged_tail_partitions { self.new_partitions.push(builder.id()); let mut builder = builder; - files.extend(builder.write(dest_store.as_ref()).await?); + files.extend( + builder + .write_to(dest_store.as_ref(), self.partition_write_target()) + .await?, + ); } log::info!("wait workers indexing elapsed: {:?}", start.elapsed()); Result::Ok(files) @@ -550,12 +564,16 @@ impl InvertedIndexBuilder { .await?; let mut builder = part.into_builder().await?; builder.remap(mapping).await?; - files.extend(builder.write(dest_store).await?); + files.extend( + builder + .write_to(dest_store, self.partition_write_target()) + .await?, + ); } if self.fragment_mask.is_none() { files.push(self.write_metadata(dest_store, &self.partitions).await?); } else { - // in distributed mode, the part_temp_metadata is written by the worker + // in distributed mode, the staged partition metadata is written by the worker for &partition_id in &self.partitions { files.push(self.write_part_metadata(dest_store, partition_id).await?); } @@ -709,26 +727,35 @@ impl InvertedIndexBuilder { .await?; let mut copied = 0; let mut files = Vec::new(); + let target = self.partition_write_target(); for part in self.partitions.iter() { files.push( self.src_store .as_ref() .expect("existing partitions require a source store") - .copy_index_file(&token_file_path(*part), dest_store) + .copy_index_file_to( + &token_file_path(*part), + &target.token_path(*part), + dest_store, + ) .await?, ); files.push( self.src_store .as_ref() .expect("existing partitions require a source store") - .copy_index_file(&posting_file_path(*part), dest_store) + .copy_index_file_to( + &posting_file_path(*part), + &target.posting_path(*part), + dest_store, + ) .await?, ); files.push( self.src_store .as_ref() .expect("existing partitions require a source store") - .copy_index_file(&doc_file_path(*part), dest_store) + .copy_index_file_to(&doc_file_path(*part), &target.doc_path(*part), dest_store) .await?, ); copied += 1; @@ -986,11 +1013,22 @@ impl InnerBuilder { } pub async fn write(&mut self, store: &dyn IndexStore) -> Result> { + self.write_to(store, PartitionWriteTarget::Final).await + } + + async fn write_to( + &mut self, + store: &dyn IndexStore, + target: PartitionWriteTarget, + ) -> Result> { let docs = Arc::new(std::mem::take(&mut self.docs)); let files = vec![ - self.write_posting_lists(store, docs.clone()).await?, - self.write_tokens(store).await?, - self.write_docs(store, docs).await?, + self.write_posting_lists(store, docs.clone(), &target.posting_path(self.id)) + .await?, + self.write_tokens(store, &target.token_path(self.id)) + .await?, + self.write_docs(store, docs, &target.doc_path(self.id)) + .await?, ]; Ok(files) } @@ -1000,11 +1038,12 @@ impl InnerBuilder { &mut self, store: &dyn IndexStore, docs: Arc, + path: &str, ) -> Result { let id = self.id; let mut writer = store .new_index_file( - &posting_file_path(self.id), + path, inverted_list_schema_for_version(self.with_position, self.format_version), ) .await?; @@ -1090,29 +1129,57 @@ impl InnerBuilder { } #[instrument(level = "debug", skip_all)] - async fn write_tokens(&mut self, store: &dyn IndexStore) -> Result { + async fn write_tokens(&mut self, store: &dyn IndexStore, path: &str) -> Result { log::info!("writing tokens of partition {}", self.id); let tokens = std::mem::take(&mut self.tokens); let batch = tokens.to_batch(self.token_set_format)?; - let mut writer = store - .new_index_file(&token_file_path(self.id), batch.schema()) - .await?; + let mut writer = store.new_index_file(path, batch.schema()).await?; writer.write_record_batch(batch).await?; writer.finish().await } #[instrument(level = "debug", skip_all)] - async fn write_docs(&mut self, store: &dyn IndexStore, docs: Arc) -> Result { + async fn write_docs( + &mut self, + store: &dyn IndexStore, + docs: Arc, + path: &str, + ) -> Result { log::info!("writing docs of partition {}", self.id); let batch = docs.to_batch()?; - let mut writer = store - .new_index_file(&doc_file_path(self.id), batch.schema()) - .await?; + let mut writer = store.new_index_file(path, batch.schema()).await?; writer.write_record_batch(batch).await?; writer.finish().await } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PartitionWriteTarget { + Final, + Staged, +} + +impl PartitionWriteTarget { + fn file_path(self, partition_id: u64, suffix: &str) -> String { + match self { + Self::Final => partition_file_path(partition_id, suffix), + Self::Staged => staged_partition_file_path(partition_id, suffix), + } + } + + fn token_path(self, partition_id: u64) -> String { + self.file_path(partition_id, TOKENS_FILE) + } + + fn posting_path(self, partition_id: u64) -> String { + self.file_path(partition_id, INVERT_LIST_FILE) + } + + fn doc_path(self, partition_id: u64) -> String { + self.file_path(partition_id, DOCS_FILE) + } +} + struct IndexWorker { tokenizer: Box, dest_store: Arc, @@ -1430,8 +1497,13 @@ impl IndexWorker { ); let written_partition_id = builder.id(); let mut builder = builder; + let target = if self.fragment_mask.is_some() { + PartitionWriteTarget::Staged + } else { + PartitionWriteTarget::Final + }; let files = builder - .write(self.dest_store.as_ref()) + .write_to(self.dest_store.as_ref(), target) .await .map_err(|err| { Error::execution(format!( @@ -1782,14 +1854,23 @@ pub(crate) fn doc_file_path(partition_id: u64) -> String { } pub(crate) fn part_metadata_file_path(partition_id: u64) -> String { - format!("part_{}_{}", partition_id, METADATA_FILE) + staged_partition_file_path(partition_id, METADATA_FILE) } const PARTITION_FILE_SUFFIXES: [&str; 3] = [TOKENS_FILE, INVERT_LIST_FILE, DOCS_FILE]; -// Each remapped file is renamed twice: first to a temp path (phase 1), then to -// its final path (phase 2). Keep in sync with the two rename loops below in -// `merge_metadata_files`. -const PARTITION_FILE_RENAME_PHASES: u64 = 2; +const STAGED_PARTITION_DIR: &str = "staging"; + +fn partition_file_path(partition_id: u64, suffix: &str) -> String { + format!("part_{}_{}", partition_id, suffix) +} + +fn staged_partition_file_path(partition_id: u64, suffix: &str) -> String { + format!( + "{}/{}", + STAGED_PARTITION_DIR, + partition_file_path(partition_id, suffix) + ) +} pub async fn merge_index_files( object_store: &ObjectStore, @@ -1797,33 +1878,65 @@ pub async fn merge_index_files( store: Arc, progress: Arc, ) -> Result<()> { - // List all partition metadata files in the index directory - let part_metadata_files = list_metadata_files(object_store, index_dir).await?; + let metadata_path = index_dir.clone().join(METADATA_FILE); + if object_store.exists(&metadata_path).await? { + return Ok(()); + } + + // List all staged partition metadata files in the index directory + let index_files = list_index_files(object_store, index_dir).await?; + let part_metadata_files = metadata_files(&index_files); + if part_metadata_files.is_empty() { + return Err(Error::invalid_input_source( + format!( + "No partition metadata files found in index directory: {}", + index_dir + ) + .into(), + )); + } // Call merge_metadata_files function for inverted index merge_metadata_files(store, &part_metadata_files, progress).await } -/// List and filter metadata files from the index directory -/// Returns partition metadata files -async fn list_metadata_files(object_store: &ObjectStore, index_dir: &Path) -> Result> { - // List all partition metadata files in the index directory - let mut part_metadata_files = Vec::new(); - let mut list_stream = object_store.list(Some(index_dir.clone())); +async fn list_index_files(object_store: &ObjectStore, index_dir: &Path) -> Result> { + let mut index_files = Vec::new(); + let mut list_stream = object_store.read_dir_all(index_dir, None); while let Some(item) = list_stream.next().await { match item { Ok(meta) => { - let file_name = meta.location.filename().unwrap_or_default(); - // Filter files matching the pattern part_*_metadata.lance - if file_name.starts_with("part_") && file_name.ends_with("_metadata.lance") { - part_metadata_files.push(file_name.to_string()); - } + let location = meta.location.as_ref().trim_start_matches('/'); + let index_dir = index_dir.as_ref().trim_start_matches('/'); + let relative_path = location + .strip_prefix(index_dir) + .map(|s| s.trim_start_matches('/').to_string()) + .unwrap_or_else(|| meta.location.filename().unwrap_or("").to_string()); + index_files.push(relative_path); } Err(err) => return Err(err), } } + Ok(index_files) +} + +fn metadata_files(index_files: &[String]) -> Vec { + index_files + .iter() + .filter(|file_name| { + file_name.starts_with(&format!("{}/part_", STAGED_PARTITION_DIR)) + && file_name.ends_with("_metadata.lance") + }) + .cloned() + .collect() +} + +#[cfg(test)] +async fn list_metadata_files(object_store: &ObjectStore, index_dir: &Path) -> Result> { + let index_files = list_index_files(object_store, index_dir).await?; + let part_metadata_files = metadata_files(&index_files); if part_metadata_files.is_empty() { return Err(Error::invalid_input_source( format!( @@ -1914,89 +2027,35 @@ async fn merge_metadata_files( progress.stage_complete("read_partition_metadata").await?; // Create ID mapping: sorted original IDs -> 0,1,2... - let mut sorted_ids = all_partitions.clone(); + let mut sorted_ids = all_partitions; sorted_ids.sort(); sorted_ids.dedup(); - let id_mapping: HashMap = sorted_ids + let id_mapping: Vec<(u64, u64)> = sorted_ids .iter() .enumerate() .map(|(new_id, &old_id)| (old_id, new_id as u64)) .collect(); - // Safe rename partition files using temporary files to avoid overwrite - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs(); - - let changed_partition_count = id_mapping - .iter() - .filter(|(old_id, new_id)| old_id != new_id) - .count() as u64; - let total_renames = changed_partition_count - * PARTITION_FILE_SUFFIXES.len() as u64 - * PARTITION_FILE_RENAME_PHASES; + let total_copies = id_mapping.len() as u64 * PARTITION_FILE_SUFFIXES.len() as u64; progress - .stage_start("remap_partition_files", Some(total_renames), "files") + .stage_start("remap_partition_files", Some(total_copies), "files") .await?; - // Phase 1: Move files to temporary locations - let mut temp_files: Vec<(String, String, String)> = Vec::new(); // (temp_path, old_path, final_path) - let mut renamed_files = 0u64; + let mut copied_files = 0u64; - for (&old_id, &new_id) in &id_mapping { - if old_id != new_id { - for suffix in PARTITION_FILE_SUFFIXES { - let old_path = format!("part_{}_{}", old_id, suffix); - let new_path = format!("part_{}_{}", new_id, suffix); - let temp_path = format!("temp_{}_{}", timestamp, old_path); - - // Move to temporary location first to avoid overwrite - if let Err(e) = store.rename_index_file(&old_path, &temp_path).await { - // Rollback phase 1: restore files from temp locations - for (temp_name, old_name, _) in temp_files.iter().rev() { - let _ = store.rename_index_file(temp_name, old_name).await; - } - return Err(Error::index(format!( - "Failed to move {} to temp {}: {}", - old_path, temp_path, e - ))); - } - temp_files.push((temp_path, old_path, new_path)); - renamed_files += 1; - progress - .stage_progress("remap_partition_files", renamed_files) - .await?; - } - } - } - - // Phase 2: Move from temporary to final locations - let mut completed_renames: Vec<(String, String)> = Vec::new(); // (final_path, temp_path) - - for (temp_path, _old_path, final_path) in &temp_files { - if let Err(e) = store.rename_index_file(temp_path, final_path).await { - // Rollback phase 2: restore completed renames and remaining temps - for (final_name, temp_name) in completed_renames.iter().rev() { - let _ = store.rename_index_file(final_name, temp_name).await; - } - // Restore remaining temp files to original locations - for (temp_name, orig_name, _) in temp_files.iter() { - if !completed_renames.iter().any(|(_, t)| t == temp_name) { - let _ = store.rename_index_file(temp_name, orig_name).await; - } - } - return Err(Error::index(format!( - "Failed to rename {} to {}: {}", - temp_path, final_path, e - ))); + for &(old_id, new_id) in &id_mapping { + for suffix in PARTITION_FILE_SUFFIXES { + let staged_path = staged_partition_file_path(old_id, suffix); + let final_path = partition_file_path(new_id, suffix); + store + .copy_index_file_to(&staged_path, &final_path, store.as_ref()) + .await?; + copied_files += 1; + progress + .stage_progress("remap_partition_files", copied_files) + .await?; } - completed_renames.push((final_path.clone(), temp_path.clone())); - renamed_files += 1; - progress - .stage_progress("remap_partition_files", renamed_files) - .await?; } progress.stage_complete("remap_partition_files").await?; @@ -2023,10 +2082,15 @@ async fn merge_metadata_files( progress.stage_progress("write_merged_metadata", 1).await?; progress.stage_complete("write_merged_metadata").await?; - // Cleanup partition metadata files + // Cleanup staged partition metadata files for file_name in part_metadata_files { - if file_name.starts_with("part_") && file_name.ends_with("_metadata.lance") { - let _ = store.delete_index_file(file_name).await; + let _ = store.delete_index_file(file_name).await; + } + for &(old_id, _) in &id_mapping { + for suffix in PARTITION_FILE_SUFFIXES { + let _ = store + .delete_index_file(&staged_partition_file_path(old_id, suffix)) + .await; } } @@ -2246,6 +2310,234 @@ mod tests { } } + #[derive(Debug, Clone)] + struct NoRenameStore { + inner: Arc, + final_delete_count: Option>, + } + + impl NoRenameStore { + fn new(inner: Arc) -> Self { + Self { + inner, + final_delete_count: None, + } + } + + fn with_final_delete_tracking(inner: Arc) -> Self { + Self { + inner, + final_delete_count: Some(Arc::new(AtomicUsize::new(0))), + } + } + + fn final_delete_count(&self) -> usize { + self.final_delete_count + .as_ref() + .map(|count| count.load(Ordering::SeqCst)) + .unwrap_or_default() + } + + fn unwrap_dest_store(dest_store: &dyn IndexStore) -> &dyn IndexStore { + dest_store + .as_any() + .downcast_ref::() + .map(|store| store.inner.as_ref()) + .unwrap_or(dest_store) + } + } + + impl DeepSizeOf for NoRenameStore { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.inner.deep_size_of_children(context) + } + } + + #[async_trait] + impl IndexStore for NoRenameStore { + fn as_any(&self) -> &dyn Any { + self + } + + fn clone_arc(&self) -> Arc { + Arc::new(self.clone()) + } + + fn io_parallelism(&self) -> usize { + self.inner.io_parallelism() + } + + async fn new_index_file( + &self, + name: &str, + schema: Arc, + ) -> Result> { + self.inner.new_index_file(name, schema).await + } + + async fn open_index_file(&self, name: &str) -> Result> { + self.inner.open_index_file(name).await + } + + async fn copy_index_file( + &self, + name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + self.inner + .copy_index_file(name, Self::unwrap_dest_store(dest_store)) + .await + } + + async fn copy_index_file_to( + &self, + name: &str, + new_name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + self.inner + .copy_index_file_to(name, new_name, Self::unwrap_dest_store(dest_store)) + .await + } + + async fn rename_index_file(&self, name: &str, new_name: &str) -> Result { + Err(Error::internal(format!( + "merge_index_files should not rename partition file {name} to {new_name}" + ))) + } + + async fn delete_index_file(&self, name: &str) -> Result<()> { + if name.starts_with("part_") + && let Some(count) = &self.final_delete_count + { + count.fetch_add(1, Ordering::SeqCst); + } + self.inner.delete_index_file(name).await + } + + async fn list_files_with_sizes(&self) -> Result> { + self.inner.list_files_with_sizes().await + } + } + + #[derive(Debug)] + struct FailMetadataStore { + inner: Arc, + } + + impl FailMetadataStore { + fn new(inner: Arc) -> Self { + Self { inner } + } + + fn unwrap_dest_store(dest_store: &dyn IndexStore) -> &dyn IndexStore { + dest_store + .as_any() + .downcast_ref::() + .map(|store| store.inner.as_ref()) + .unwrap_or(dest_store) + } + } + + impl DeepSizeOf for FailMetadataStore { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.inner.deep_size_of_children(context) + } + } + + #[async_trait] + impl IndexStore for FailMetadataStore { + fn as_any(&self) -> &dyn Any { + self + } + + fn clone_arc(&self) -> Arc { + Arc::new(Self { + inner: self.inner.clone(), + }) + } + + fn io_parallelism(&self) -> usize { + self.inner.io_parallelism() + } + + async fn new_index_file( + &self, + name: &str, + schema: Arc, + ) -> Result> { + let writer = self.inner.new_index_file(name, schema).await?; + if name == METADATA_FILE { + Ok(Box::new(FailFinishWriter { inner: writer })) + } else { + Ok(writer) + } + } + + async fn open_index_file(&self, name: &str) -> Result> { + self.inner.open_index_file(name).await + } + + async fn copy_index_file( + &self, + name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + self.inner + .copy_index_file(name, Self::unwrap_dest_store(dest_store)) + .await + } + + async fn copy_index_file_to( + &self, + name: &str, + new_name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + self.inner + .copy_index_file_to(name, new_name, Self::unwrap_dest_store(dest_store)) + .await + } + + async fn rename_index_file(&self, name: &str, new_name: &str) -> Result { + self.inner.rename_index_file(name, new_name).await + } + + async fn delete_index_file(&self, name: &str) -> Result<()> { + self.inner.delete_index_file(name).await + } + + async fn list_files_with_sizes(&self) -> Result> { + self.inner.list_files_with_sizes().await + } + } + + struct FailFinishWriter { + inner: Box, + } + + #[async_trait] + impl IndexWriter for FailFinishWriter { + async fn write_record_batch(&mut self, batch: RecordBatch) -> Result { + self.inner.write_record_batch(batch).await + } + + async fn add_global_buffer(&mut self, data: Bytes) -> Result { + self.inner.add_global_buffer(data).await + } + + async fn finish(&mut self) -> Result { + Err(Error::internal("injected metadata write failure")) + } + + async fn finish_with_metadata( + &mut self, + _metadata: HashMap, + ) -> Result { + Err(Error::internal("injected metadata write failure")) + } + } + #[derive(Debug)] struct CountingWriter { path: String, @@ -2412,12 +2704,446 @@ mod tests { let store = CountingStore::new(); let docs = Arc::new(std::mem::take(&mut builder.docs)); - builder.write_posting_lists(&store, docs).await?; + builder + .write_posting_lists(&store, docs, &posting_file_path(0)) + .await?; assert_eq!(store.write_count(), 1); Ok(()) } + async fn write_partition_file_marker( + store: &dyn IndexStore, + path: &str, + partition_id: u64, + ) -> Result<()> { + let schema = Arc::new(Schema::new(vec![Field::new( + "partition_id", + DataType::UInt64, + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(UInt64Array::from(vec![partition_id]))], + )?; + let mut writer = store.new_index_file(path, schema).await?; + writer.write_record_batch(batch).await?; + writer.finish().await?; + Ok(()) + } + + async fn write_partition_files( + store: &dyn IndexStore, + partition_id: u64, + target: PartitionWriteTarget, + ) -> Result<()> { + write_partition_file_marker(store, &target.token_path(partition_id), partition_id).await?; + write_partition_file_marker(store, &target.posting_path(partition_id), partition_id) + .await?; + write_partition_file_marker(store, &target.doc_path(partition_id), partition_id).await?; + Ok(()) + } + + async fn read_partition_file_marker(store: &dyn IndexStore, path: &str) -> Result { + let reader = store.open_index_file(path).await?; + let batch = reader.read_range(0..1, None).await?; + let partition_ids = batch.column(0).as_primitive::(); + Ok(partition_ids.value(0)) + } + + async fn assert_partition_file_markers( + store: &dyn IndexStore, + partition_id: u64, + expected_marker: u64, + ) -> Result<()> { + assert_eq!( + read_partition_file_marker(store, &token_file_path(partition_id)).await?, + expected_marker + ); + assert_eq!( + read_partition_file_marker(store, &posting_file_path(partition_id)).await?, + expected_marker + ); + assert_eq!( + read_partition_file_marker(store, &doc_file_path(partition_id)).await?, + expected_marker + ); + Ok(()) + } + + #[tokio::test] + async fn test_merge_index_files_remaps_staged_partitions_without_rename() -> Result<()> { + let index_dir = TempDir::default(); + let object_store = Arc::new(ObjectStore::local()); + let base_store: Arc = Arc::new(LanceIndexStore::new( + object_store.clone(), + index_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + let store = Arc::new(NoRenameStore::new(base_store.clone())); + let partitions = vec![5_u64, 1_u64, (17_u64 << 32) | 2]; + let metadata_builder = InvertedIndexBuilder::from_existing_index( + InvertedIndexParams::default(), + None, + Vec::new(), + TokenSetFormat::default(), + None, + RoaringBitmap::new(), + ); + + for partition_id in &partitions { + write_partition_files( + base_store.as_ref(), + *partition_id, + PartitionWriteTarget::Staged, + ) + .await?; + metadata_builder + .write_part_metadata(base_store.as_ref(), *partition_id) + .await?; + } + + merge_index_files( + object_store.as_ref(), + &index_dir.obj_path(), + store, + noop_progress(), + ) + .await?; + + let metadata_reader = base_store.open_index_file(METADATA_FILE).await?; + let metadata = &metadata_reader.schema().metadata; + let written_partitions: Vec = serde_json::from_str( + metadata + .get("partitions") + .expect("partitions missing from metadata"), + )?; + let mut expected_partitions = partitions.clone(); + expected_partitions.sort_unstable(); + expected_partitions.dedup(); + let remapped_partitions = (0..expected_partitions.len() as u64).collect::>(); + assert_eq!(written_partitions, remapped_partitions); + + for (new_id, old_id) in expected_partitions.iter().enumerate() { + assert_partition_file_markers(base_store.as_ref(), new_id as u64, *old_id).await?; + assert!( + base_store + .open_index_file(&part_metadata_file_path(*old_id)) + .await + .is_err(), + "partition metadata should be cleaned up after final metadata is written" + ); + for suffix in PARTITION_FILE_SUFFIXES { + assert!( + base_store + .open_index_file(&staged_partition_file_path(*old_id, suffix)) + .await + .is_err(), + "staged partition files should be cleaned up after final metadata is written" + ); + } + } + + Ok(()) + } + + #[tokio::test] + async fn test_merge_index_files_rewrites_partial_final_files_from_staging() -> Result<()> { + let index_dir = TempDir::default(); + let object_store = Arc::new(ObjectStore::local()); + let base_store: Arc = Arc::new(LanceIndexStore::new( + object_store.clone(), + index_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + let store = Arc::new(NoRenameStore::with_final_delete_tracking( + base_store.clone(), + )); + let partitions = vec![1_u64, 5_u64]; + let metadata_builder = InvertedIndexBuilder::from_existing_index( + InvertedIndexParams::default(), + None, + Vec::new(), + TokenSetFormat::default(), + None, + RoaringBitmap::new(), + ); + + for partition_id in &partitions { + write_partition_files( + base_store.as_ref(), + *partition_id, + PartitionWriteTarget::Staged, + ) + .await?; + metadata_builder + .write_part_metadata(base_store.as_ref(), *partition_id) + .await?; + } + + for suffix in PARTITION_FILE_SUFFIXES { + write_partition_file_marker(base_store.as_ref(), &partition_file_path(1, suffix), 999) + .await?; + } + + merge_index_files( + object_store.as_ref(), + &index_dir.obj_path(), + store.clone(), + noop_progress(), + ) + .await?; + + assert_partition_file_markers(base_store.as_ref(), 0, 1).await?; + assert_partition_file_markers(base_store.as_ref(), 1, 5).await?; + assert_eq!( + store.final_delete_count(), + 0, + "merge should overwrite final partition files without deleting them first" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_distributed_from_existing_copies_existing_partitions_to_staging_and_finalizes() + -> Result<()> { + let object_store = Arc::new(ObjectStore::local()); + let source_dir = TempDir::default(); + let dest_dir = TempDir::default(); + let source_store: Arc = Arc::new(LanceIndexStore::new( + object_store.clone(), + source_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + let dest_store: Arc = Arc::new(LanceIndexStore::new( + object_store.clone(), + dest_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + let merge_store = Arc::new(NoRenameStore::new(dest_store.clone())); + let fragment_mask = 7_u64 << 32; + let partitions = vec![fragment_mask | 5, fragment_mask | 1]; + + for partition_id in &partitions { + write_partition_files( + source_store.as_ref(), + *partition_id, + PartitionWriteTarget::Final, + ) + .await?; + } + + let builder = InvertedIndexBuilder::from_existing_index( + InvertedIndexParams::default(), + Some(source_store.clone()), + partitions.clone(), + TokenSetFormat::default(), + Some(fragment_mask), + RoaringBitmap::new(), + ); + builder.write(dest_store.as_ref()).await?; + + for partition_id in &partitions { + assert_partition_file_markers(source_store.as_ref(), *partition_id, *partition_id) + .await?; + for suffix in PARTITION_FILE_SUFFIXES { + let staged_path = staged_partition_file_path(*partition_id, suffix); + assert_eq!( + read_partition_file_marker(dest_store.as_ref(), &staged_path).await?, + *partition_id + ); + assert!( + dest_store + .open_index_file(&partition_file_path(*partition_id, suffix)) + .await + .is_err(), + "distributed existing partition should be staged instead of copied to root" + ); + } + dest_store + .open_index_file(&part_metadata_file_path(*partition_id)) + .await?; + } + + merge_index_files( + object_store.as_ref(), + &dest_dir.obj_path(), + merge_store, + noop_progress(), + ) + .await?; + + let mut expected_partitions = partitions.clone(); + expected_partitions.sort_unstable(); + for (new_id, old_id) in expected_partitions.iter().enumerate() { + assert_partition_file_markers(dest_store.as_ref(), new_id as u64, *old_id).await?; + for suffix in PARTITION_FILE_SUFFIXES { + assert!( + dest_store + .open_index_file(&staged_partition_file_path(*old_id, suffix)) + .await + .is_err(), + "staged partition files should be cleaned after final metadata is written" + ); + } + } + + Ok(()) + } + + #[tokio::test] + async fn test_merge_index_files_keeps_staging_when_final_metadata_write_fails() -> Result<()> { + let index_dir = TempDir::default(); + let object_store = Arc::new(ObjectStore::local()); + let base_store: Arc = Arc::new(LanceIndexStore::new( + object_store.clone(), + index_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + let failing_store = Arc::new(FailMetadataStore::new(base_store.clone())); + let partitions = vec![1_u64, 5_u64]; + let metadata_builder = InvertedIndexBuilder::from_existing_index( + InvertedIndexParams::default(), + None, + Vec::new(), + TokenSetFormat::default(), + None, + RoaringBitmap::new(), + ); + + for partition_id in &partitions { + write_partition_files( + base_store.as_ref(), + *partition_id, + PartitionWriteTarget::Staged, + ) + .await?; + metadata_builder + .write_part_metadata(base_store.as_ref(), *partition_id) + .await?; + } + + let err = merge_index_files( + object_store.as_ref(), + &index_dir.obj_path(), + failing_store, + noop_progress(), + ) + .await + .unwrap_err(); + assert!( + err.to_string().contains("metadata write failure"), + "expected injected metadata failure, got: {err}" + ); + + for partition_id in &partitions { + base_store + .open_index_file(&part_metadata_file_path(*partition_id)) + .await?; + for suffix in PARTITION_FILE_SUFFIXES { + let staged_path = staged_partition_file_path(*partition_id, suffix); + assert_eq!( + read_partition_file_marker(base_store.as_ref(), &staged_path).await?, + *partition_id + ); + } + } + + Ok(()) + } + + #[tokio::test] + async fn test_distributed_build_writes_partition_data_to_staging() -> Result<()> { + let index_dir = TempDir::default(); + let object_store = ObjectStore::local(); + let store = Arc::new(LanceIndexStore::new( + object_store.into(), + index_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + + let fragment_mask = 7_u64 << 32; + let batch = make_doc_batch("hello world", fragment_mask); + let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)])); + let stream = Box::pin(stream); + let mut builder = InvertedIndexBuilder::new_with_fragment_mask( + InvertedIndexParams::default(), + Some(fragment_mask), + ); + builder.update(stream, store.as_ref(), None).await?; + + let part_metadata_files = + list_metadata_files(&ObjectStore::local(), &index_dir.obj_path()).await?; + assert_eq!(part_metadata_files.len(), 1); + assert!( + part_metadata_files[0].starts_with("staging/part_"), + "partition metadata should be written to staging" + ); + let reader = store.open_index_file(&part_metadata_files[0]).await?; + let partition_ids: Vec = serde_json::from_str( + reader + .schema() + .metadata + .get("partitions") + .expect("partitions missing from metadata"), + )?; + assert_eq!(partition_ids.len(), 1); + let partition_id = partition_ids[0]; + + store + .open_index_file(&staged_partition_file_path(partition_id, TOKENS_FILE)) + .await?; + assert!( + store + .open_index_file(&partition_file_path(partition_id, METADATA_FILE)) + .await + .is_err(), + "distributed build-only metadata should not be written to root partition metadata paths" + ); + assert!( + store + .open_index_file(&token_file_path(partition_id)) + .await + .is_err(), + "distributed build-only data should not be written to final partition paths" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_merge_index_files_is_noop_when_metadata_exists() -> Result<()> { + let index_dir = TempDir::default(); + let object_store = Arc::new(ObjectStore::local()); + let store: Arc = Arc::new(LanceIndexStore::new( + object_store.clone(), + index_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + let metadata_builder = InvertedIndexBuilder::from_existing_index( + InvertedIndexParams::default(), + None, + vec![42], + TokenSetFormat::default(), + None, + RoaringBitmap::new(), + ); + metadata_builder + .write_metadata(store.as_ref(), &[42]) + .await?; + + merge_index_files( + object_store.as_ref(), + &index_dir.obj_path(), + store, + noop_progress(), + ) + .await?; + + Ok(()) + } + #[tokio::test] async fn test_build_only_path_writes_partitions_as_is() -> Result<()> { let src_dir = TempDir::default(); @@ -2856,7 +3582,6 @@ mod tests { } }) .collect::>(); - let read_start = tags .iter() .position(|e| e == "start:read_partition_metadata") @@ -2894,8 +3619,8 @@ mod tests { ); assert_eq!( remap_progress.last().copied().unwrap_or_default(), - 12, - "expected remap_partition_files progress to cover both rename phases" + 6, + "expected remap_partition_files progress to cover staged-to-final copies" ); assert!( tags.iter().any(|e| e == "progress:write_merged_metadata"), diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs index 56547c6510b..8e662f5db6f 100644 --- a/rust/lance-index/src/scalar/inverted/index.rs +++ b/rust/lance-index/src/scalar/inverted/index.rs @@ -2947,10 +2947,9 @@ impl DeepSizeOf for CompressedPositionStorage { #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct SharedPositionStream { codec: PositionStreamCodec, - block_offsets: Vec, - // Stored as `Bytes` so that the cache deserialization path can hand - // ownership of an IPC-decoded slice in without copying. Cloning the - // stream is then an `Arc` bump rather than an O(N) buffer copy. + block_offsets: Arc<[u32]>, + // Stored with shared ownership so cache hits can clone position streams + // without copying either offsets or bytes. bytes: bytes::Bytes, } @@ -2958,7 +2957,7 @@ impl SharedPositionStream { pub fn new(codec: PositionStreamCodec, block_offsets: Vec, bytes: bytes::Bytes) -> Self { Self { codec, - block_offsets, + block_offsets: Arc::from(block_offsets.into_boxed_slice()), bytes, } } @@ -2991,11 +2990,11 @@ impl SharedPositionStream { } pub fn block_offsets(&self) -> &[u32] { - &self.block_offsets + self.block_offsets.as_ref() } pub fn size(&self) -> usize { - self.block_offsets.capacity() * std::mem::size_of::() + self.bytes.len() + self.block_offsets.len() * std::mem::size_of::() + self.bytes.len() } } @@ -5475,6 +5474,21 @@ mod tests { ); } + #[test] + fn test_shared_position_stream_clone_shares_block_offsets() { + let stream = SharedPositionStream::new( + PositionStreamCodec::PackedDelta, + vec![0_u32, 4, 11], + bytes::Bytes::from_static(b"shared position bytes"), + ); + let original_offsets = stream.block_offsets().as_ptr(); + + let cloned = stream.clone(); + + assert_eq!(cloned.block_offsets(), stream.block_offsets()); + assert_eq!(cloned.block_offsets().as_ptr(), original_offsets); + } + #[test] fn test_posting_builder_roundtrip_shared_positions() { let entries = vec![ @@ -6446,6 +6460,16 @@ mod tests { ) -> Result { self.inner.copy_index_file(name, dest_store).await } + async fn copy_index_file_to( + &self, + name: &str, + new_name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + self.inner + .copy_index_file_to(name, new_name, dest_store) + .await + } async fn rename_index_file( &self, name: &str, diff --git a/rust/lance-index/src/scalar/lance_format.rs b/rust/lance-index/src/scalar/lance_format.rs index 562945b8f0d..2f82deb8403 100644 --- a/rust/lance-index/src/scalar/lance_format.rs +++ b/rust/lance-index/src/scalar/lance_format.rs @@ -99,6 +99,24 @@ impl LanceIndexStore { self.file_sizes = file_sizes; self } + + fn index_file_path(&self, name: &str) -> Result { + let relative_path = Path::parse(name).map_err(|err| { + Error::invalid_input(format!("invalid index file path {name:?}: {err}")) + })?; + if self.index_dir.is_root() { + return Ok(relative_path); + } + if relative_path.is_root() { + return Ok(self.index_dir.clone()); + } + Path::parse(format!( + "{}/{}", + self.index_dir.as_ref(), + relative_path.as_ref() + )) + .map_err(|err| Error::invalid_input(format!("invalid index file path {name:?}: {err}"))) + } } #[async_trait] @@ -397,7 +415,7 @@ impl IndexStore for LanceIndexStore { name: &str, schema: Arc, ) -> Result> { - let path = self.index_dir.clone().join(name); + let path = self.index_file_path(name)?; let schema = schema.as_ref().try_into()?; let writer = self.object_store.create(&path).await?; let writer = current_writer::FileWriter::try_new( @@ -415,7 +433,7 @@ impl IndexStore for LanceIndexStore { } async fn open_index_file(&self, name: &str) -> Result> { - let path = self.index_dir.clone().join(name); + let path = self.index_file_path(name)?; // Use cached file size if available, otherwise unknown (requires HEAD call) let cached_size = self .file_sizes @@ -436,7 +454,7 @@ impl IndexStore for LanceIndexStore { Err(e) => { // If the error is a version conflict we can try to read the file with v1 reader if let Error::VersionConflict { .. } = e { - let path = self.index_dir.clone().join(name); + let path = self.index_file_path(name)?; let file_reader = PreviousFileReader::try_new_self_described( &self.object_store, &path, @@ -452,7 +470,16 @@ impl IndexStore for LanceIndexStore { } async fn copy_index_file(&self, name: &str, dest_store: &dyn IndexStore) -> Result { - let path = self.index_dir.clone().join(name); + self.copy_index_file_to(name, name, dest_store).await + } + + async fn copy_index_file_to( + &self, + name: &str, + new_name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + let path = self.index_file_path(name)?; let other_store = dest_store.as_any().downcast_ref::(); match other_store { @@ -460,21 +487,21 @@ impl IndexStore for LanceIndexStore { // If both this store and the destination are lance stores we can use object_store's copy // This does blindly assume that both stores are using the same underlying object_store // but there is no easy way to verify this and it happens to always be true at the moment - let dest_path = dest_store.index_dir.clone().join(name); + let dest_path = dest_store.index_file_path(new_name)?; self.object_store.copy(&path, &dest_path).await?; let size_bytes = match self.file_sizes.get(name) { Some(size_bytes) => *size_bytes, None => self.object_store.size(&path).await?, }; Ok(IndexFile { - path: name.to_string(), + path: new_name.to_string(), size_bytes, }) } _ => { let reader = self.open_index_file(name).await?; let mut writer = dest_store - .new_index_file(name, Arc::new(reader.schema().into())) + .new_index_file(new_name, Arc::new(reader.schema().into())) .await?; for offset in (0..reader.num_rows()).step_by(4096) { @@ -488,8 +515,8 @@ impl IndexStore for LanceIndexStore { } async fn rename_index_file(&self, name: &str, new_name: &str) -> Result { - let path = self.index_dir.clone().join(name); - let new_path = self.index_dir.clone().join(new_name); + let path = self.index_file_path(name)?; + let new_path = self.index_file_path(new_name)?; self.object_store.copy(&path, &new_path).await?; self.object_store.delete(&path).await?; let size_bytes = match self.file_sizes.get(name) { @@ -503,7 +530,7 @@ impl IndexStore for LanceIndexStore { } async fn delete_index_file(&self, name: &str) -> Result<()> { - let path = self.index_dir.clone().join(name); + let path = self.index_file_path(name)?; self.object_store.delete(&path).await } diff --git a/rust/lance-index/src/scalar/ngram.rs b/rust/lance-index/src/scalar/ngram.rs index 72ef8d53a92..b452ef78c85 100644 --- a/rust/lance-index/src/scalar/ngram.rs +++ b/rust/lance-index/src/scalar/ngram.rs @@ -5,7 +5,10 @@ use std::any::Any; use std::collections::BTreeMap; use std::iter::once; use std::time::Instant; -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; use super::lance_format::LanceIndexStore; use super::{ @@ -49,6 +52,9 @@ use roaring::{RoaringBitmap, RoaringTreemap}; use serde::Serialize; use tracing::instrument; +mod ngram_regex; +pub(crate) use ngram_regex::regex_can_use_index; + const TOKENS_COL: &str = "tokens"; const POSTING_LIST_COL: &str = "posting_list"; const POSTINGS_FILENAME: &str = "ngram_postings.lance"; @@ -476,6 +482,45 @@ impl ScalarIndex for NGramIndex { let row_ids = NGramPostingList::intersect(list_refs); Ok(SearchResult::at_most(RowAddrTreeMap::from(row_ids))) } + TextQuery::Regex(pattern) => { + let trigram_query = ngram_regex::regex_to_trigram_query(pattern); + match &trigram_query { + // No usable trigram structure (e.g. `a.b`, `.*`): the index + // cannot prune, so every row must be rechecked. + ngram_regex::TrigramQuery::All => { + Ok(SearchResult::at_least(RowAddrTreeMap::new())) + } + // The pattern is provably unsatisfiable. + ngram_regex::TrigramQuery::None => { + Ok(SearchResult::exact(RowAddrTreeMap::new())) + } + _ => { + let mut tokens = HashSet::new(); + ngram_regex::collect_tokens(&trigram_query, &mut tokens); + // Fetch the posting list for every trigram the condition + // references; a token absent from the index contributes + // an empty list, which `eval_trigram_query` handles. + let present = tokens.into_iter().filter_map(|token| { + self.tokens.get(&token).map(|offset| (token, *offset)) + }); + let lists = futures::stream::iter(present.map(|(token, offset)| { + self.list_reader + .ngram_list(offset, metrics) + .map(move |result| result.map(|list| (token, list))) + })) + .buffer_unordered(self.io_parallelism) + .try_collect::)>>() + .await?; + metrics.record_comparisons(lists.len()); + let bitmaps: HashMap = lists + .into_iter() + .map(|(token, list)| (token, list.bitmap.clone())) + .collect(); + let row_ids = ngram_regex::eval_trigram_query(&trigram_query, &bitmaps); + Ok(SearchResult::at_most(RowAddrTreeMap::from(row_ids))) + } + } + } } } @@ -1279,6 +1324,9 @@ impl ScalarIndexPlugin for NGramIndexPlugin { Some(Box::new(TextQueryParser::new( index_name, self.name().to_string(), + // needs_recheck: ngram results are an inexact candidate superset. + true, + // supports_regex: the ngram index can answer regex queries. true, ))) } @@ -1538,6 +1586,107 @@ mod tests { assert_eq!(expected, res); } + #[test_log::test(tokio::test)] + async fn test_ngram_regex_search() { + // Same corpus as test_basic_ngram_index. + let data = StringArray::from_iter_values([ + "cat", // 0 + "dog", // 1 + "cat dog", // 2 + "dog cat", // 3 + "elephant", // 4 + "mouse", // 5 + "rhino", // 6 + "giraffe", // 7 + "rhinos nose", // 8 + ]); + let row_ids = UInt64Array::from_iter_values((0..data.len()).map(|i| i as u64)); + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, DataType::Utf8, false), + Field::new(ROW_ID, DataType::UInt64, false), + ])); + let data = + RecordBatch::try_new(schema.clone(), vec![Arc::new(data), Arc::new(row_ids)]).unwrap(); + let data = Box::pin(RecordBatchStreamAdapter::new( + schema, + stream::once(std::future::ready(Ok(data))), + )); + + let builder = NGramIndexBuilder::try_new(NGramIndexBuilderOptions::default()).unwrap(); + let (index, _tmpdir) = do_train(builder, data).await; + + async fn search(index: &NGramIndex, pattern: &str) -> SearchResult { + index + .search( + &TextQuery::Regex(pattern.to_string()), + &NoOpMetricsCollector, + ) + .await + .unwrap() + } + + // A plain literal yields the same candidates as contains("cat"). + assert_eq!( + search(&index, "cat").await, + SearchResult::at_most(RowAddrTreeMap::from_iter([0, 2, 3])) + ); + + // Alternation -> union of each branch's rows. + assert_eq!( + search(&index, "(cat|dog)").await, + SearchResult::at_most(RowAddrTreeMap::from_iter([0, 1, 2, 3])) + ); + + // AND across `.*`: must contain both the `rhino` and `nose` trigrams, so + // row 6 ("rhino") is correctly excluded and only row 8 survives. + assert_eq!( + search(&index, "rhino.*nose").await, + SearchResult::at_most(RowAddrTreeMap::from_iter([8])) + ); + + // No derivable trigram -> recheck everything. + assert_eq!( + search(&index, "a.b").await, + SearchResult::at_least(RowAddrTreeMap::new()) + ); + + // A trigram that is absent from the index -> empty candidate set. + assert_eq!( + search(&index, "zzz").await, + SearchResult::at_most(RowAddrTreeMap::new()) + ); + } + + #[test_log::test(tokio::test)] + async fn test_ngram_regex_search_nulls() { + // Rows: cat(0), dog(1), NULL(2), NULL(3), cat dog(4). + let data = simple_data_with_nulls(); + let builder = NGramIndexBuilder::try_new(NGramIndexBuilderOptions::default()).unwrap(); + let (index, _tmpdir) = do_train(builder, data).await; + + // The NULL rows (2, 3) must never appear in the candidate set. + let res = index + .search(&TextQuery::Regex("cat".to_string()), &NoOpMetricsCollector) + .await + .unwrap(); + assert_eq!( + res, + SearchResult::at_most(RowAddrTreeMap::from_iter([0, 4])) + ); + + let res = index + .search( + &TextQuery::Regex("(cat|dog)".to_string()), + &NoOpMetricsCollector, + ) + .await + .unwrap(); + assert_eq!( + res, + SearchResult::at_most(RowAddrTreeMap::from_iter([0, 1, 4])) + ); + } + fn test_data_schema() -> Arc { Arc::new(Schema::new(vec![ Field::new(VALUE_COLUMN_NAME, DataType::Utf8, true), diff --git a/rust/lance-index/src/scalar/ngram/ngram_regex.rs b/rust/lance-index/src/scalar/ngram/ngram_regex.rs new file mode 100644 index 00000000000..ee67c479a71 --- /dev/null +++ b/rust/lance-index/src/scalar/ngram/ngram_regex.rs @@ -0,0 +1,673 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Deriving a trigram pre-filter from a regular expression. +//! +//! This is the query-side counterpart of the ngram index that lets us +//! accelerate `regexp_like` / `regexp_match` predicates the same way the index +//! already accelerates `contains`. The idea (the same one Postgres `pg_trgm` +//! and Russ Cox's Google Code Search use) is to derive, from the regex, a +//! boolean condition over trigram presence that is *necessary* for any string +//! to match, evaluate it against the inverted index, and let the scan recheck +//! the true regex on the surviving rows. +//! +//! The derived condition is a [`TrigramQuery`] -- an AND/OR tree of trigram +//! tokens. `AND` maps onto posting-list intersection and `OR` onto union, which +//! is exactly the set algebra the ngram index is built for. +//! +//! # Soundness +//! +//! The single invariant that matters is that the condition must never require a +//! trigram that a matching string could lack -- otherwise we would drop real +//! matches (a false negative, far worse than a false positive, which the recheck +//! removes). Everything here is therefore a conservative *over*-approximation: +//! when in doubt we emit [`TrigramQuery::All`] ("no constraint, recheck +//! everything"). Concretely: +//! +//! * Every trigram requirement is produced by [`trigrams_of_string`], which runs +//! the *same* tokenizer the index was built with, so a string shorter than a +//! trigram (or with no alphanumeric run) contributes no requirement. +//! * Character classes and case-insensitive folds are treated as a single +//! unknown character (`All`), because the index's normalization does not agree +//! with Unicode case folding (e.g. `(?i)c` also matches `ℂ`, which the index +//! does not fold to `c`). Literal runs -- the common case -- are fully used. +//! * When the exact / prefix / suffix string sets grow past a bound we first fold +//! their trigrams into the running condition and only then drop the strings, so +//! collapsing precision never removes a necessary trigram. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use regex_syntax::hir::{Class, Hir, HirKind}; +use roaring::RoaringTreemap; + +use super::{NGRAM_N, NGRAM_TOKENIZER, ngram_to_token, tokenize_visitor}; + +/// Maximum number of strings kept in an `exact` / `prefix` / `suffix` set before +/// it is folded into the trigram condition and dropped. +const MAX_SET_SIZE: usize = 16; +/// Maximum length (in characters) of a string kept in a set. Longer strings are +/// trimmed to a sound shorter affix. +const MAX_STRING_LEN: usize = 32; + +/// A boolean condition over trigram presence that is *necessary* for a regex to +/// match. `All` means "no constraint" and `None` means "unsatisfiable"; by +/// construction these only ever appear at the root of the tree. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum TrigramQuery { + /// No constraint: every row is a candidate (the scan must recheck all rows). + All, + /// Unsatisfiable: no row can match. + None, + /// The given trigram token must be present. + Trigram(u32), + /// Every child condition must hold (posting-list intersection). + And(Vec), + /// At least one child condition must hold (posting-list union). + Or(Vec), +} + +impl TrigramQuery { + /// Build an `AND` of conditions, applying identity (`All`), absorbing + /// (`None`), flattening, sorting and de-duplication so the result is + /// canonical and free of nested `All`/`None`. + fn and(items: Vec) -> Self { + let mut flat = Vec::with_capacity(items.len()); + for item in items { + match item { + Self::All => {} // identity + Self::None => return Self::None, // absorbing + Self::And(children) => flat.extend(children), // flatten + other => flat.push(other), + } + } + flat.sort(); + flat.dedup(); + match flat.len() { + 0 => Self::All, + 1 => flat.pop().unwrap(), + _ => Self::And(flat), + } + } + + /// Build an `OR` of conditions, applying absorbing (`All`), identity + /// (`None`), flattening, sorting and de-duplication. + fn or(items: Vec) -> Self { + let mut flat = Vec::with_capacity(items.len()); + for item in items { + match item { + Self::All => return Self::All, // absorbing + Self::None => {} // identity + Self::Or(children) => flat.extend(children), // flatten + other => flat.push(other), + } + } + flat.sort(); + flat.dedup(); + match flat.len() { + 0 => Self::None, + 1 => flat.pop().unwrap(), + _ => Self::Or(flat), + } + } +} + +/// Information about the set of strings a sub-expression can match, used to +/// build a necessary trigram condition bottom-up. For every string `s` the +/// sub-expression matches: `s` is in `exact` (when it is `Some`), `s` starts +/// with some member of `prefix` and ends with some member of `suffix`, and `s` +/// satisfies `match_q`. +struct RegexInfo { + /// Whether the sub-expression can match the empty string. + emptyable: bool, + /// The complete set of strings the sub-expression matches, or `None` if that + /// set is unbounded / unknown. + exact: Option>, + /// Strings that every match must start with (empty = unknown). + prefix: BTreeSet, + /// Strings that every match must end with (empty = unknown). + suffix: BTreeSet, + /// A necessary trigram condition for the sub-expression. + match_q: TrigramQuery, +} + +impl RegexInfo { + /// The empty string (also used for zero-width anchors): matches only `""`. + fn empty_string() -> Self { + let empty = BTreeSet::from([String::new()]); + Self { + emptyable: true, + exact: Some(empty.clone()), + prefix: empty.clone(), + suffix: empty, + match_q: TrigramQuery::All, + } + } + + /// A fixed literal string. + fn literal(s: &str) -> Self { + let set = BTreeSet::from([s.to_string()]); + Self { + emptyable: s.is_empty(), + exact: Some(set.clone()), + prefix: set.clone(), + suffix: set, + match_q: trigrams_of_string(s), + } + } + + /// A single unknown character (a character class we cannot pin down). + fn any_char() -> Self { + Self { + emptyable: false, + exact: None, + prefix: BTreeSet::new(), + suffix: BTreeSet::new(), + match_q: TrigramQuery::All, + } + } + + /// Enforce the size/length bounds, folding any information about to be + /// discarded into `match_q` first so that precision loss never drops a + /// necessary trigram. Idempotent. + fn bound(&mut self) { + let oversized_exact = self.exact.as_ref().is_some_and(|exact| { + exact.len() > MAX_SET_SIZE || exact.iter().any(|s| s.chars().count() > MAX_STRING_LEN) + }); + if oversized_exact { + let exact = self.exact.take().expect("checked above"); + self.fold_into_match(&exact); + } + + self.prefix = self + .prefix + .iter() + .map(|s| leading(s, MAX_STRING_LEN)) + .collect(); + if self.prefix.len() > MAX_SET_SIZE { + let prefix = std::mem::take(&mut self.prefix); + self.fold_into_match(&prefix); + } + + self.suffix = self + .suffix + .iter() + .map(|s| trailing(s, MAX_STRING_LEN)) + .collect(); + if self.suffix.len() > MAX_SET_SIZE { + let suffix = std::mem::take(&mut self.suffix); + self.fold_into_match(&suffix); + } + } + + /// AND the trigrams of `set` (a complete set of possible affixes/strings) + /// into `match_q`. Sound because the set is exhaustive for its role. + fn fold_into_match(&mut self, set: &BTreeSet) { + let folded = trigrams_of_set(set.iter()); + let current = std::mem::replace(&mut self.match_q, TrigramQuery::All); + self.match_q = TrigramQuery::and(vec![current, folded]); + } +} + +/// AND together the trigrams of `s`. Reuses the index's own tokenizer so the +/// tokens are normalized (lowercase, ASCII-folded, alphanumeric-bounded) +/// exactly as they were stored. Returns `All` if `s` yields no trigram (too +/// short, or no run of three alphanumeric characters). +fn trigrams_of_string(s: &str) -> TrigramQuery { + let mut tokens = Vec::new(); + tokenize_visitor(&NGRAM_TOKENIZER, s, |ngram| { + tokens.push(TrigramQuery::Trigram(ngram_to_token(ngram, NGRAM_N))); + }); + TrigramQuery::and(tokens) +} + +/// OR together the trigram conditions of each string in `set`. An empty set +/// means "unknown" and yields `All` (no constraint); if any member yields `All` +/// the whole OR is `All`. +fn trigrams_of_set<'a>(set: impl IntoIterator) -> TrigramQuery { + let queries: Vec<_> = set.into_iter().map(|s| trigrams_of_string(s)).collect(); + if queries.is_empty() { + return TrigramQuery::All; + } + TrigramQuery::or(queries) +} + +/// Concatenate every string in `a` with every string in `b`. +fn cross_concat(a: &BTreeSet, b: &BTreeSet) -> BTreeSet { + let mut out = BTreeSet::new(); + for x in a { + for y in b { + out.insert(format!("{x}{y}")); + } + } + out +} + +/// The first `n` characters of `s` (a sound shorter prefix). +fn leading(s: &str, n: usize) -> String { + s.chars().take(n).collect() +} + +/// The last `n` characters of `s` (a sound shorter suffix). +fn trailing(s: &str, n: usize) -> String { + let count = s.chars().count(); + s.chars().skip(count.saturating_sub(n)).collect() +} + +/// If `class` matches exactly one scalar value, return that character. +fn singleton_char(class: &Class) -> Option { + match class { + Class::Unicode(u) => { + let ranges = u.ranges(); + match ranges { + [r] if r.start() == r.end() => Some(r.start()), + _ => None, + } + } + Class::Bytes(b) => { + let ranges = b.ranges(); + match ranges { + [r] if r.start() == r.end() && r.start() < 0x80 => Some(r.start() as char), + _ => None, + } + } + } +} + +/// Compute the [`RegexInfo`] for `hir` bottom-up. +fn analyze(hir: &Hir) -> RegexInfo { + let mut info = match hir.kind() { + // Zero-width: the empty match. Anchors (^, $, \b) carry no trigram. + HirKind::Empty | HirKind::Look(_) => RegexInfo::empty_string(), + HirKind::Literal(lit) => match std::str::from_utf8(&lit.0) { + Ok(s) => RegexInfo::literal(s), + // A literal that is not valid UTF-8 cannot be reasoned about here. + Err(_) => RegexInfo::any_char(), + }, + HirKind::Class(class) => match singleton_char(class) { + Some(ch) => RegexInfo::literal(ch.encode_utf8(&mut [0u8; 4])), + None => RegexInfo::any_char(), + }, + HirKind::Repetition(rep) => { + let inner = analyze(&rep.sub); + let at_least_one = rep.min >= 1; + RegexInfo { + emptyable: !at_least_one || inner.emptyable, + // We do not unroll bounded repetitions, so the matched set is + // unbounded as far as we are concerned. + exact: None, + prefix: if at_least_one { + inner.prefix.clone() + } else { + BTreeSet::new() + }, + suffix: if at_least_one { + inner.suffix.clone() + } else { + BTreeSet::new() + }, + // Only a required occurrence (min >= 1) contributes; the single + // inner match is necessary, never multiplied. + match_q: if at_least_one { + inner.match_q + } else { + TrigramQuery::All + }, + } + } + HirKind::Capture(cap) => analyze(&cap.sub), + HirKind::Concat(subs) => analyze_concat(subs), + HirKind::Alternation(subs) => analyze_alternation(subs), + }; + info.bound(); + info +} + +fn analyze_concat(subs: &[Hir]) -> RegexInfo { + let mut acc = RegexInfo::empty_string(); + for sub in subs { + acc = concat_info(acc, analyze(sub)); + } + acc +} + +/// Combine two adjacent sub-expressions. This is the subtle part: it recovers +/// trigrams that straddle the junction via the cross product of `acc.suffix` and +/// `next.prefix`. +fn concat_info(acc: RegexInfo, next: RegexInfo) -> RegexInfo { + let emptyable = acc.emptyable && next.emptyable; + + // Trigrams spanning the junction (computed from the pre-merge affixes). + let boundary = if acc.suffix.is_empty() || next.prefix.is_empty() { + TrigramQuery::All + } else { + trigrams_of_set(cross_concat(&acc.suffix, &next.prefix).iter()) + }; + + // exact = acc.exact x next.exact, only while both are finite and small. + let exact = match (&acc.exact, &next.exact) { + (Some(a), Some(b)) if a.len().saturating_mul(b.len()) <= MAX_SET_SIZE => { + Some(cross_concat(a, b)) + } + _ => None, + }; + + // A match starts with acc's full string (when known) then next's prefix, + // otherwise with acc's own prefix. + let prefix = match &acc.exact { + Some(a) if !next.prefix.is_empty() => cross_concat(a, &next.prefix), + Some(a) => a.clone(), + None => acc.prefix.clone(), + }; + + // Mirror image for the suffix (driven by the right side). + let suffix = match &next.exact { + Some(b) if !acc.suffix.is_empty() => cross_concat(&acc.suffix, b), + Some(b) => b.clone(), + None => next.suffix.clone(), + }; + + let match_q = TrigramQuery::and(vec![acc.match_q, next.match_q, boundary]); + + let mut info = RegexInfo { + emptyable, + exact, + prefix, + suffix, + match_q, + }; + info.bound(); + info +} + +fn analyze_alternation(subs: &[Hir]) -> RegexInfo { + let infos: Vec = subs.iter().map(analyze).collect(); + + let emptyable = infos.iter().any(|i| i.emptyable); + + let exact = if infos.iter().all(|i| i.exact.is_some()) { + Some( + infos + .iter() + .flat_map(|i| i.exact.as_ref().unwrap().iter().cloned()) + .collect(), + ) + } else { + None + }; + + // A common prefix exists only if every branch contributes one. + let prefix = if infos.iter().all(|i| !i.prefix.is_empty()) { + infos + .iter() + .flat_map(|i| i.prefix.iter().cloned()) + .collect() + } else { + BTreeSet::new() + }; + let suffix = if infos.iter().all(|i| !i.suffix.is_empty()) { + infos + .iter() + .flat_map(|i| i.suffix.iter().cloned()) + .collect() + } else { + BTreeSet::new() + }; + + let match_q = TrigramQuery::or(infos.into_iter().map(|i| i.match_q).collect()); + + RegexInfo { + emptyable, + exact, + prefix, + suffix, + match_q, + } +} + +/// Derive a necessary trigram condition from a regular expression pattern. +/// +/// Returns [`TrigramQuery::All`] when no useful condition can be derived (an +/// unparsable pattern, or one with no trigram-able literal structure such as +/// `a.b` or `.*`); callers must treat that as "recheck everything". +pub fn regex_to_trigram_query(pattern: &str) -> TrigramQuery { + // An unparsable pattern cannot be accelerated; rechecking is still safe. + let Ok(hir) = regex_syntax::parse(pattern) else { + return TrigramQuery::All; + }; + let info = analyze(&hir); + + let mut conditions = vec![info.match_q]; + if let Some(exact) = &info.exact { + if exact.is_empty() { + // The expression matches nothing. + return TrigramQuery::None; + } + conditions.push(trigrams_of_set(exact.iter())); + } + conditions.push(trigrams_of_set(info.prefix.iter())); + conditions.push(trigrams_of_set(info.suffix.iter())); + TrigramQuery::and(conditions) +} + +/// Whether a regular expression yields any trigram condition the index can use +/// to prune candidates. When it does not (e.g. `a.b`, `.*`, or a case-insensitive +/// pattern), callers should leave the predicate to a full scan rather than route +/// it to the index, which would otherwise have to ask the scan to recheck every +/// row -- a path the index result type (`AtLeast`) does not support. +pub fn regex_can_use_index(pattern: &str) -> bool { + regex_to_trigram_query(pattern) != TrigramQuery::All +} + +/// Collect the distinct trigram tokens referenced anywhere in the tree. +pub fn collect_tokens(query: &TrigramQuery, out: &mut HashSet) { + match query { + TrigramQuery::Trigram(token) => { + out.insert(*token); + } + TrigramQuery::And(items) | TrigramQuery::Or(items) => { + for item in items { + collect_tokens(item, out); + } + } + TrigramQuery::All | TrigramQuery::None => {} + } +} + +/// Evaluate the tree against a map of `trigram token -> posting list`. A token +/// missing from the map contributes an empty set (sound: a required trigram that +/// is absent everywhere yields no rows; an absent OR branch contributes +/// nothing). `All` / `None` are handled by the caller before evaluation. +pub fn eval_trigram_query( + query: &TrigramQuery, + bitmaps: &HashMap, +) -> RoaringTreemap { + match query { + TrigramQuery::Trigram(token) => bitmaps.get(token).cloned().unwrap_or_default(), + TrigramQuery::And(items) => { + let mut iter = items.iter(); + let mut acc = match iter.next() { + Some(first) => eval_trigram_query(first, bitmaps), + None => return RoaringTreemap::new(), + }; + for item in iter { + if acc.is_empty() { + break; + } + acc &= &eval_trigram_query(item, bitmaps); + } + acc + } + TrigramQuery::Or(items) => { + let mut acc = RoaringTreemap::new(); + for item in items { + acc |= &eval_trigram_query(item, bitmaps); + } + acc + } + TrigramQuery::All | TrigramQuery::None => RoaringTreemap::new(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A single trigram condition, hashed the same way the index hashes it. + fn tri(trigram: &str) -> TrigramQuery { + TrigramQuery::Trigram(ngram_to_token(trigram, NGRAM_N)) + } + + fn q(pattern: &str) -> TrigramQuery { + regex_to_trigram_query(pattern) + } + + #[test] + fn test_single_literal_trigram() { + assert_eq!(q("foo"), tri("foo")); + } + + #[test] + fn test_multi_trigram_literal() { + assert_eq!( + q("foobar"), + TrigramQuery::and(vec![tri("foo"), tri("oob"), tri("oba"), tri("bar")]) + ); + } + + #[test] + fn test_wildcard_splits_into_and() { + // `.*` breaks the literal run; both sides are required. + assert_eq!( + q("foo.*bar"), + TrigramQuery::and(vec![tri("foo"), tri("bar")]) + ); + } + + #[test] + fn test_alternation_is_or() { + assert_eq!( + q("(cat|dog)"), + TrigramQuery::or(vec![tri("cat"), tri("dog")]) + ); + } + + #[test] + fn test_anchors_are_transparent() { + assert_eq!( + q("^rhino"), + TrigramQuery::and(vec![tri("rhi"), tri("hin"), tri("ino")]) + ); + assert_eq!(q("nose$"), TrigramQuery::and(vec![tri("nos"), tri("ose")])); + } + + #[test] + fn test_boundary_trigram_recovered_across_groups() { + // A capturing group is not merged into the adjacent literals, so this + // exercises the suffix x prefix cross product that recovers the `foo` + // trigram straddling the `(o)` group boundary in "foobar". + assert_eq!( + q("fo(o)bar"), // spellchecker:disable-line + TrigramQuery::and(vec![tri("foo"), tri("oob"), tri("oba"), tri("bar")]) + ); + } + + #[test] + fn test_no_trigram_yields_all() { + // No run of three literal characters anywhere. + assert_eq!(q("a.b"), TrigramQuery::All); + assert_eq!(q(".*"), TrigramQuery::All); + // Every alternation branch is shorter than a trigram, so we must not + // require either two-character branch as a (non-existent) trigram. + assert_eq!(q("fo|ba"), TrigramQuery::All); // spellchecker:disable-line + } + + #[test] + fn test_case_insensitive_not_accelerated() { + // Unicode case folding (e.g. `(?i)c` also matches U+2102) does not agree + // with the index's normalization, so case-insensitive patterns are left + // unaccelerated (correct via recheck) rather than risk a false negative. + assert_eq!(q("(?i)Cat"), TrigramQuery::All); + } + + #[test] + fn test_unparsable_pattern_yields_all() { + assert_eq!(q("("), TrigramQuery::All); + } + + #[test] + fn test_large_alternation_stays_bounded() { + // More than MAX_SET_SIZE branches: must still produce a sound OR without + // panicking or exploding. + let pattern = (0..40) + .map(|i| format!("aa{i:02}zz")) + .collect::>() + .join("|"); + let result = q(&pattern); + // Each branch shares the trigram `aa0`/`aa1`/... and `zz`-ish endings; + // the important property is that it is a sound non-empty condition. + assert_ne!(result, TrigramQuery::None); + } + + #[test] + fn test_plus_requires_inner() { + // `(abc)+` must contain at least one `abc`. + assert_eq!(q("(abc)+"), tri("abc")); + } + + #[test] + fn test_optional_group_is_not_required() { + // `(foo)?bar` -> foo optional, bar required. + assert_eq!(q("(foo)?bar"), tri("bar")); + } + + #[test] + fn test_eval_and_or_with_missing_tokens() { + let foo = ngram_to_token("foo", NGRAM_N); + let bar = ngram_to_token("bar", NGRAM_N); + let mut bitmaps = HashMap::new(); + bitmaps.insert(foo, RoaringTreemap::from_iter([1u64, 2, 3])); + bitmaps.insert(bar, RoaringTreemap::from_iter([2u64, 3, 4])); + // `baz` is absent from the index. + + // AND intersects. + let and = TrigramQuery::and(vec![tri("foo"), tri("bar")]); + assert_eq!( + eval_trigram_query(&and, &bitmaps), + RoaringTreemap::from_iter([2u64, 3]) + ); + + // OR unions. + let or = TrigramQuery::or(vec![tri("foo"), tri("bar")]); + assert_eq!( + eval_trigram_query(&or, &bitmaps), + RoaringTreemap::from_iter([1u64, 2, 3, 4]) + ); + + // A missing token is empty: it zeroes an AND but is harmless in an OR. + let and_missing = TrigramQuery::and(vec![tri("foo"), tri("baz")]); + assert!(eval_trigram_query(&and_missing, &bitmaps).is_empty()); + let or_missing = TrigramQuery::or(vec![tri("foo"), tri("baz")]); + assert_eq!( + eval_trigram_query(&or_missing, &bitmaps), + RoaringTreemap::from_iter([1u64, 2, 3]) + ); + } + + #[test] + fn test_collect_tokens() { + let query = TrigramQuery::and(vec![ + tri("foo"), + TrigramQuery::or(vec![tri("bar"), tri("baz")]), + ]); + let mut tokens = HashSet::new(); + collect_tokens(&query, &mut tokens); + assert_eq!( + tokens, + HashSet::from([ + ngram_to_token("foo", NGRAM_N), + ngram_to_token("bar", NGRAM_N), + ngram_to_token("baz", NGRAM_N), + ]) + ); + } +} diff --git a/rust/lance-index/src/vector/bq.rs b/rust/lance-index/src/vector/bq.rs index ad013683214..7a47fa88d54 100644 --- a/rust/lance-index/src/vector/bq.rs +++ b/rust/lance-index/src/vector/bq.rs @@ -20,6 +20,7 @@ use crate::vector::quantizer::QuantizerBuildParams; pub mod builder; pub(crate) mod dist_table_quant; pub mod ex_dot; +pub mod prune; pub mod rotation; pub mod storage; pub mod transform; diff --git a/rust/lance-index/src/vector/bq/prune.rs b/rust/lance-index/src/vector/bq/prune.rs new file mode 100644 index 00000000000..e67ab6642b8 --- /dev/null +++ b/rust/lance-index/src/vector/bq/prune.rs @@ -0,0 +1,527 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! SIMD kernels for the RaBitQ top-k lower-bound pruning scan. +//! +//! Multi-bit IVF_RQ search gates the exact ex-code rerank with a per-row +//! distance lower bound: after the binary FastScan fills the per-row binary +//! inner products, every row of the partition is classified against the query +//! upper bound and the current top-k heap threshold, and only the survivors +//! (typically well under 1%) are reranked. The classification is the per-row +//! formula of `RabitDistCalculator::raw_query_lower_bound`: +//! +//! ```text +//! lower_bound = (binary_ip - 0.5 * sum_q) * scale_factor +//! + add_factor + query_factor +//! - error_factor * query_error +//! ``` +//! +//! These kernels evaluate the formula and both comparisons for +//! [`PRUNE_LANES`] rows at a time, returning bit masks instead of values so +//! the caller can skip whole groups (the overwhelmingly common case) and run +//! the existing scalar rerank only for the surviving lanes. +//! +//! Correctness contract: +//! +//! - The lower bound is computed with exactly the operation order of the +//! scalar helper — multiplies and adds, never FMA. A fused multiply-add +//! rounds differently, which could prune a row the scalar code would have +//! kept; with bit-identical lower bounds the masks reproduce the scalar +//! `>=` decisions exactly, keeping heap contents and prune-stats counters +//! unchanged. +//! - Comparisons use ordered-quiet GE predicates (`_CMP_GE_OQ`), matching +//! scalar `>=`: a NaN lower bound is never pruned and falls through to the +//! exact rerank. +//! - The heap threshold may be a stale snapshot (it only ever tightens); the +//! caller re-checks surviving lanes against live values, so a stale +//! threshold can only over-select survivors, never wrongly prune. + +use std::sync::LazyLock; + +/// Rows classified per kernel invocation. +pub const PRUNE_LANES: usize = 16; + +/// Per-query constants of the lower-bound formula, mirroring +/// `RabitDistCalculator::raw_query_lower_bound` term by term. +#[derive(Debug, Clone, Copy)] +pub struct LowerBoundTerms { + /// `0.5 * sum_q`, subtracted from the binary inner product. + pub half_sum_q: f32, + pub query_factor: f32, + pub query_error: f32, +} + +/// Classify [`PRUNE_LANES`] rows against the pruning bounds. +/// +/// Arguments are the per-row binary inner products, scale factors, add +/// factors, and error factors, followed by the formula constants, the query +/// upper bound, and the heap threshold (`None` while the heap is not full, +/// which disables the heap mask). +/// +/// Returns `(pruned_upper_bound, pruned_heap)` masks: bit `i` of +/// `pruned_upper_bound` is set when `lower_bound[i] >= upper_bound`, and bit +/// `i` of `pruned_heap` is set when the row is not already pruned by the +/// upper bound and `lower_bound[i] >= heap_threshold`. Surviving rows are the +/// zero bits of the OR of both masks. +pub type PruneMaskFn = fn( + &[f32; PRUNE_LANES], + &[f32; PRUNE_LANES], + &[f32; PRUNE_LANES], + &[f32; PRUNE_LANES], + LowerBoundTerms, + f32, + Option, +) -> (u16, u16); + +/// Resolve the prune-mask kernel for the running CPU once; the result can be +/// cached by the caller for per-partition use. +pub fn prune_mask_kernel() -> PruneMaskFn { + static KERNEL: LazyLock = LazyLock::new(select_prune_mask_kernel); + *KERNEL +} + +fn select_prune_mask_kernel() -> PruneMaskFn { + #[cfg(target_arch = "x86_64")] + { + if std::arch::is_x86_feature_detected!("avx512f") { + return x86::prune_masks_avx512_dispatch; + } + if std::arch::is_x86_feature_detected!("avx2") { + return x86::prune_masks_avx2_dispatch; + } + } + // On aarch64 the plain 16-wide loop auto-vectorizes to NEON (part of the + // baseline), so no dedicated kernel is needed. + prune_masks_portable +} + +/// Portable implementation; also the reference for the SIMD kernels. +fn prune_masks_portable( + dists: &[f32; PRUNE_LANES], + scale_factors: &[f32; PRUNE_LANES], + add_factors: &[f32; PRUNE_LANES], + error_factors: &[f32; PRUNE_LANES], + terms: LowerBoundTerms, + upper_bound: f32, + heap_threshold: Option, +) -> (u16, u16) { + let mut lower_bounds = [0.0f32; PRUNE_LANES]; + for lane in 0..PRUNE_LANES { + lower_bounds[lane] = ((dists[lane] - terms.half_sum_q) * scale_factors[lane] + + add_factors[lane] + + terms.query_factor) + - error_factors[lane] * terms.query_error; + } + let mut pruned_upper_bound = 0u16; + for (lane, lower_bound) in lower_bounds.iter().enumerate() { + pruned_upper_bound |= u16::from(*lower_bound >= upper_bound) << lane; + } + let mut pruned_heap = 0u16; + if let Some(threshold) = heap_threshold { + for (lane, lower_bound) in lower_bounds.iter().enumerate() { + pruned_heap |= u16::from(*lower_bound >= threshold) << lane; + } + pruned_heap &= !pruned_upper_bound; + } + (pruned_upper_bound, pruned_heap) +} + +#[cfg(target_arch = "x86_64")] +mod x86 { + use super::{LowerBoundTerms, PRUNE_LANES}; + use std::arch::x86_64::*; + + /// Lower bounds for 8 lanes with the scalar operation order (no FMA). + #[inline] + #[target_feature(enable = "avx")] + fn lower_bounds_avx( + dists: __m256, + scale_factors: __m256, + add_factors: __m256, + error_factors: __m256, + half_sum_q: __m256, + query_factor: __m256, + query_error: __m256, + ) -> __m256 { + let binary_distance = _mm256_add_ps( + _mm256_add_ps( + _mm256_mul_ps(_mm256_sub_ps(dists, half_sum_q), scale_factors), + add_factors, + ), + query_factor, + ); + _mm256_sub_ps(binary_distance, _mm256_mul_ps(error_factors, query_error)) + } + + #[inline] + #[target_feature(enable = "avx")] + fn ge_mask_avx(lower_bounds_lo: __m256, lower_bounds_hi: __m256, bound: f32) -> u16 { + let bound = _mm256_set1_ps(bound); + let lo = _mm256_movemask_ps(_mm256_cmp_ps::<_CMP_GE_OQ>(lower_bounds_lo, bound)); + let hi = _mm256_movemask_ps(_mm256_cmp_ps::<_CMP_GE_OQ>(lower_bounds_hi, bound)); + (lo | (hi << 8)) as u16 + } + + #[target_feature(enable = "avx2")] + unsafe fn prune_masks_avx2( + dists: &[f32; PRUNE_LANES], + scale_factors: &[f32; PRUNE_LANES], + add_factors: &[f32; PRUNE_LANES], + error_factors: &[f32; PRUNE_LANES], + terms: LowerBoundTerms, + upper_bound: f32, + heap_threshold: Option, + ) -> (u16, u16) { + let half_sum_q = _mm256_set1_ps(terms.half_sum_q); + let query_factor = _mm256_set1_ps(terms.query_factor); + let query_error = _mm256_set1_ps(terms.query_error); + // SAFETY: the array references guarantee 16 readable floats each. + let lower_bounds_lo = unsafe { + lower_bounds_avx( + _mm256_loadu_ps(dists.as_ptr()), + _mm256_loadu_ps(scale_factors.as_ptr()), + _mm256_loadu_ps(add_factors.as_ptr()), + _mm256_loadu_ps(error_factors.as_ptr()), + half_sum_q, + query_factor, + query_error, + ) + }; + let lower_bounds_hi = unsafe { + lower_bounds_avx( + _mm256_loadu_ps(dists.as_ptr().add(8)), + _mm256_loadu_ps(scale_factors.as_ptr().add(8)), + _mm256_loadu_ps(add_factors.as_ptr().add(8)), + _mm256_loadu_ps(error_factors.as_ptr().add(8)), + half_sum_q, + query_factor, + query_error, + ) + }; + let pruned_upper_bound = ge_mask_avx(lower_bounds_lo, lower_bounds_hi, upper_bound); + let pruned_heap = match heap_threshold { + Some(threshold) => { + ge_mask_avx(lower_bounds_lo, lower_bounds_hi, threshold) & !pruned_upper_bound + } + None => 0, + }; + (pruned_upper_bound, pruned_heap) + } + + pub(super) fn prune_masks_avx2_dispatch( + dists: &[f32; PRUNE_LANES], + scale_factors: &[f32; PRUNE_LANES], + add_factors: &[f32; PRUNE_LANES], + error_factors: &[f32; PRUNE_LANES], + terms: LowerBoundTerms, + upper_bound: f32, + heap_threshold: Option, + ) -> (u16, u16) { + // SAFETY: only selected when AVX2 was detected. + unsafe { + prune_masks_avx2( + dists, + scale_factors, + add_factors, + error_factors, + terms, + upper_bound, + heap_threshold, + ) + } + } + + #[target_feature(enable = "avx512f")] + unsafe fn prune_masks_avx512( + dists: &[f32; PRUNE_LANES], + scale_factors: &[f32; PRUNE_LANES], + add_factors: &[f32; PRUNE_LANES], + error_factors: &[f32; PRUNE_LANES], + terms: LowerBoundTerms, + upper_bound: f32, + heap_threshold: Option, + ) -> (u16, u16) { + // SAFETY: the array references guarantee 16 readable floats each. + let (dists, scale_factors, add_factors, error_factors) = unsafe { + ( + _mm512_loadu_ps(dists.as_ptr()), + _mm512_loadu_ps(scale_factors.as_ptr()), + _mm512_loadu_ps(add_factors.as_ptr()), + _mm512_loadu_ps(error_factors.as_ptr()), + ) + }; + let binary_distance = _mm512_add_ps( + _mm512_add_ps( + _mm512_mul_ps( + _mm512_sub_ps(dists, _mm512_set1_ps(terms.half_sum_q)), + scale_factors, + ), + add_factors, + ), + _mm512_set1_ps(terms.query_factor), + ); + let lower_bounds = _mm512_sub_ps( + binary_distance, + _mm512_mul_ps(error_factors, _mm512_set1_ps(terms.query_error)), + ); + let pruned_upper_bound = + _mm512_cmp_ps_mask::<_CMP_GE_OQ>(lower_bounds, _mm512_set1_ps(upper_bound)); + let pruned_heap = match heap_threshold { + Some(threshold) => { + _mm512_cmp_ps_mask::<_CMP_GE_OQ>(lower_bounds, _mm512_set1_ps(threshold)) + & !pruned_upper_bound + } + None => 0, + }; + (pruned_upper_bound, pruned_heap) + } + + pub(super) fn prune_masks_avx512_dispatch( + dists: &[f32; PRUNE_LANES], + scale_factors: &[f32; PRUNE_LANES], + add_factors: &[f32; PRUNE_LANES], + error_factors: &[f32; PRUNE_LANES], + terms: LowerBoundTerms, + upper_bound: f32, + heap_threshold: Option, + ) -> (u16, u16) { + // SAFETY: only selected when AVX-512F was detected. + unsafe { + prune_masks_avx512( + dists, + scale_factors, + add_factors, + error_factors, + terms, + upper_bound, + heap_threshold, + ) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rand::rngs::SmallRng; + use rand::{Rng, SeedableRng}; + + fn available_kernels() -> Vec<(&'static str, PruneMaskFn)> { + // `mut` is only exercised on x86_64 where extra kernels may be pushed. + #[allow(unused_mut)] + let mut kernels = vec![ + ("portable", prune_masks_portable as PruneMaskFn), + ("dispatched", prune_mask_kernel()), + ]; + #[cfg(target_arch = "x86_64")] + { + if std::arch::is_x86_feature_detected!("avx2") { + kernels.push(("avx2", x86::prune_masks_avx2_dispatch)); + } + if std::arch::is_x86_feature_detected!("avx512f") { + kernels.push(("avx512", x86::prune_masks_avx512_dispatch)); + } + } + kernels + } + + /// Per-lane reference mirroring `raw_query_lower_bound` and the scalar + /// pruning checks of the top-k scan. + fn reference_masks( + dists: &[f32; PRUNE_LANES], + scale_factors: &[f32; PRUNE_LANES], + add_factors: &[f32; PRUNE_LANES], + error_factors: &[f32; PRUNE_LANES], + terms: LowerBoundTerms, + upper_bound: f32, + heap_threshold: Option, + ) -> (u16, u16) { + let mut pruned_upper_bound = 0u16; + let mut pruned_heap = 0u16; + for lane in 0..PRUNE_LANES { + let lower_bound = (dists[lane] - terms.half_sum_q) * scale_factors[lane] + + add_factors[lane] + + terms.query_factor + - error_factors[lane] * terms.query_error; + if lower_bound >= upper_bound { + pruned_upper_bound |= 1 << lane; + } else if heap_threshold.is_some_and(|threshold| lower_bound >= threshold) { + pruned_heap |= 1 << lane; + } + } + (pruned_upper_bound, pruned_heap) + } + + #[allow(clippy::too_many_arguments)] + fn assert_kernels_match_reference( + dists: &[f32; PRUNE_LANES], + scale_factors: &[f32; PRUNE_LANES], + add_factors: &[f32; PRUNE_LANES], + error_factors: &[f32; PRUNE_LANES], + terms: LowerBoundTerms, + upper_bound: f32, + heap_threshold: Option, + case: &str, + ) { + let expected = reference_masks( + dists, + scale_factors, + add_factors, + error_factors, + terms, + upper_bound, + heap_threshold, + ); + for (name, kernel) in available_kernels() { + let actual = kernel( + dists, + scale_factors, + add_factors, + error_factors, + terms, + upper_bound, + heap_threshold, + ); + assert_eq!( + actual, expected, + "kernel={name} case={case}: masks {actual:04x?} != {expected:04x?}" + ); + } + } + + #[test] + fn test_prune_masks_match_reference_on_random_inputs() { + let mut rng = SmallRng::seed_from_u64(42); + for round in 0..200 { + let mut dists = [0.0f32; PRUNE_LANES]; + let mut scale_factors = [0.0f32; PRUNE_LANES]; + let mut add_factors = [0.0f32; PRUNE_LANES]; + let mut error_factors = [0.0f32; PRUNE_LANES]; + for lane in 0..PRUNE_LANES { + dists[lane] = rng.random_range(-100.0f32..100.0); + scale_factors[lane] = rng.random_range(-2.0f32..2.0); + add_factors[lane] = rng.random_range(-10.0f32..10.0); + error_factors[lane] = rng.random_range(0.0f32..5.0); + } + let terms = LowerBoundTerms { + half_sum_q: rng.random_range(-50.0f32..50.0), + query_factor: rng.random_range(-10.0f32..10.0), + query_error: rng.random_range(0.0f32..2.0), + }; + let upper_bound = rng.random_range(-50.0f32..50.0); + let heap_threshold = if round % 3 == 0 { + None + } else { + Some(rng.random_range(-50.0f32..50.0)) + }; + assert_kernels_match_reference( + &dists, + &scale_factors, + &add_factors, + &error_factors, + terms, + upper_bound, + heap_threshold, + &format!("random round {round}"), + ); + } + } + + #[test] + fn test_prune_masks_exact_boundaries() { + // With scale=1, err=0, half_sum_q=0, query_factor=0 the lower bound + // is the input itself, so bounds can be placed exactly on lanes. + let dists: [f32; PRUNE_LANES] = std::array::from_fn(|lane| lane as f32); + let scale_factors = [1.0f32; PRUNE_LANES]; + let add_factors = [0.0f32; PRUNE_LANES]; + let error_factors = [0.0f32; PRUNE_LANES]; + let terms = LowerBoundTerms { + half_sum_q: 0.0, + query_factor: 0.0, + query_error: 1.0, + }; + // Equality must prune (scalar uses `>=`): lanes 3.. hit the upper + // bound, lanes 1..3 hit only the heap threshold. + let (pruned_upper_bound, pruned_heap) = prune_masks_portable( + &dists, + &scale_factors, + &add_factors, + &error_factors, + terms, + 3.0, + Some(1.0), + ); + assert_eq!(pruned_upper_bound, 0xfff8); + assert_eq!(pruned_heap, 0x0006); + assert_kernels_match_reference( + &dists, + &scale_factors, + &add_factors, + &error_factors, + terms, + 3.0, + Some(1.0), + "exact boundaries", + ); + // No heap threshold: only the upper-bound mask is set. + assert_kernels_match_reference( + &dists, + &scale_factors, + &add_factors, + &error_factors, + terms, + 3.0, + None, + "no heap threshold", + ); + } + + #[test] + fn test_prune_masks_nan_and_infinity_semantics() { + let mut dists = [0.0f32; PRUNE_LANES]; + dists[0] = f32::NAN; + dists[1] = f32::INFINITY; + dists[2] = f32::NEG_INFINITY; + dists[3] = 1.0; + let mut scale_factors = [1.0f32; PRUNE_LANES]; + scale_factors[4] = f32::NAN; + let add_factors = [0.0f32; PRUNE_LANES]; + let mut error_factors = [0.0f32; PRUNE_LANES]; + error_factors[5] = f32::INFINITY; + let terms = LowerBoundTerms { + half_sum_q: 0.0, + query_factor: 0.0, + query_error: 1.0, + }; + for (upper_bound, heap_threshold) in [ + (0.5, Some(0.0)), + (f32::INFINITY, Some(f32::NEG_INFINITY)), + (f32::NAN, Some(f32::NAN)), + (0.5, None), + ] { + assert_kernels_match_reference( + &dists, + &scale_factors, + &add_factors, + &error_factors, + terms, + upper_bound, + heap_threshold, + &format!("special values ub={upper_bound} thr={heap_threshold:?}"), + ); + } + // NaN lower bounds (lane 0 via a NaN binary inner product, lane 4 via + // a NaN scale factor) must never be pruned by either mask. + let (pruned_upper_bound, pruned_heap) = prune_masks_portable( + &dists, + &scale_factors, + &add_factors, + &error_factors, + terms, + 0.5, + Some(0.0), + ); + assert_eq!(pruned_upper_bound & 0b1_0001, 0); + assert_eq!(pruned_heap & 0b1_0001, 0); + } +} diff --git a/rust/lance-index/src/vector/bq/storage.rs b/rust/lance-index/src/vector/bq/storage.rs index ef88500a7b7..9c355d26960 100644 --- a/rust/lance-index/src/vector/bq/storage.rs +++ b/rust/lance-index/src/vector/bq/storage.rs @@ -17,7 +17,7 @@ use arrow_array::{ use arrow_schema::{DataType, Field, SchemaRef}; use async_trait::async_trait; use bytes::{Bytes, BytesMut}; -use itertools::Itertools; +use itertools::{Itertools, izip}; use lance_arrow::{ArrowFloatType, FixedSizeListArrayExt, FloatArray, RecordBatchExt}; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, ROW_ID, Result}; @@ -48,6 +48,7 @@ use crate::vector::bq::ex_dot::{ EX_DOT_BLOCK_DIMS, ExDotFn, blocked_ex_code_bytes, ex_dot_kernel, pad_query_into, padded_query_len, repack_sequential_row, sequential_matches_blocked, }; +use crate::vector::bq::prune::{LowerBoundTerms, PRUNE_LANES, prune_mask_kernel}; use crate::vector::bq::rotation::{apply_fast_rotation, apply_fast_rotation_in_place}; use crate::vector::bq::transform::{ ADD_FACTORS_COLUMN, ERROR_FACTORS_COLUMN, EX_ADD_FACTORS_COLUMN, EX_SCALE_FACTORS_COLUMN, @@ -136,16 +137,28 @@ fn emit_rabit_prune_stats(message: &str) { ); } -fn record_rabit_prune_stats( +/// Per-scan tallies of the raw-query lower-bound gating, reported through +/// `record_rabit_prune_stats`. +#[derive(Default)] +struct RabitPruneCounters { candidates: usize, pruned_upper_bound: usize, pruned_heap: usize, exact: usize, exact_rejected: usize, -) { +} + +fn record_rabit_prune_stats(counters: &RabitPruneCounters) { if !rabit_prune_stats_enabled() { return; } + let RabitPruneCounters { + candidates, + pruned_upper_bound, + pruned_heap, + exact, + exact_rejected, + } = *counters; let stats = RABIT_PRUNE_STATS.get_or_init(RabitPruneStats::default); let calls = stats.calls.fetch_add(1, Ordering::Relaxed) + 1; @@ -798,6 +811,20 @@ struct RabitDistCalculatorParts<'a> { approx_mode: ApproxMode, } +/// Loop-invariant inputs of the raw-query multi-bit top-k scans: the row +/// count, the resolved ex-code state for exact reranking, and the query +/// bounds. +struct RawQueryTopkContext<'a> { + n: usize, + k: usize, + ex_bits: u8, + ex_codes: &'a [u8], + ex_add_factors: &'a [f32], + ex_scale_factors: &'a [f32], + query_lower_bound: f32, + query_upper_bound: f32, +} + /// Pick the query slice the ex-dot kernels consume: the rotated query itself /// when the dim is block-aligned, otherwise a zero-padded copy. fn kernel_query<'a>(rotated_query: &'a [f32], padded: &'a [f32]) -> &'a [f32] { @@ -1212,26 +1239,27 @@ impl<'a> RabitDistCalculator<'a> { full_dot * ex_scale_factors[id] + ex_add_factors[id] + self.query_factor } + /// Compute the binary inner products into `dists` and resolve the inputs + /// shared by the raw-query multi-bit top-k scans. Returns `None` when the + /// partition has no rows. #[allow(clippy::too_many_arguments)] - fn accumulate_raw_query_multi_bit_topk_with_scratch( + fn raw_query_multi_bit_topk_context( &self, k: usize, lower_bound: Option, upper_bound: Option, - row_ids: impl Iterator, - res: &mut BinaryHeap>, dists: &mut Vec, quantized_dists: &mut Vec, quantized_dists_table: &mut Vec, hacc_quantized_dists: &mut Vec, - ) { + ) -> Option> { let code_len = rabit_binary_code_bytes(self.dim); let n = self.codes.len() / code_len; if n == 0 { dists.clear(); quantized_dists.clear(); hacc_quantized_dists.clear(); - return; + return None; } self.binary_distances_with_scratch( @@ -1243,74 +1271,233 @@ impl<'a> RabitDistCalculator<'a> { hacc_quantized_dists, ); - let ex_bits = self.num_bits - 1; - let ex_codes = self - .ex_codes - .expect("raw-query multi-bit RQ requires ex codes"); - let ex_add_factors = self - .ex_add_factors - .expect("raw-query multi-bit RQ requires ex add factors"); - let ex_scale_factors = self - .ex_scale_factors - .expect("raw-query multi-bit RQ requires ex scale factors"); - let query_lower_bound = lower_bound.unwrap_or(f32::MIN); - let query_upper_bound = upper_bound.unwrap_or(f32::MAX); + Some(RawQueryTopkContext { + n, + k, + ex_bits: self.num_bits - 1, + ex_codes: self + .ex_codes + .expect("raw-query multi-bit RQ requires ex codes"), + ex_add_factors: self + .ex_add_factors + .expect("raw-query multi-bit RQ requires ex add factors"), + ex_scale_factors: self + .ex_scale_factors + .expect("raw-query multi-bit RQ requires ex scale factors"), + query_lower_bound: lower_bound.unwrap_or(f32::MIN), + query_upper_bound: upper_bound.unwrap_or(f32::MAX), + }) + } + + /// Process one candidate row given its lower bound: the bound checks, + /// the exact rerank, and the heap update shared by the sparse scan and + /// the dense scan's surviving lanes and tail. + #[inline] + #[allow(clippy::too_many_arguments)] + fn accumulate_raw_query_multi_bit_row( + &self, + ctx: &RawQueryTopkContext<'_>, + id: usize, + row_id: u64, + binary_ip: f32, + raw_lower_bound: f32, + res: &mut BinaryHeap>, + max_dist: &mut Option, + counters: &mut RabitPruneCounters, + ) { + if raw_lower_bound >= ctx.query_upper_bound { + counters.pruned_upper_bound += 1; + return; + } + if res.len() >= ctx.k && max_dist.is_some_and(|max_dist| raw_lower_bound >= max_dist.0) { + counters.pruned_heap += 1; + return; + } + + counters.exact += 1; + let dist = self.raw_query_multi_bit_exact_distance( + id, + binary_ip, + ctx.ex_bits, + ctx.ex_codes, + ctx.ex_add_factors, + ctx.ex_scale_factors, + ); + if dist < ctx.query_lower_bound || dist >= ctx.query_upper_bound { + counters.exact_rejected += 1; + return; + } + let dist = OrderedFloat(dist); + if res.len() < ctx.k { + res.push(OrderedNode::new(row_id, dist)); + if res.len() == ctx.k { + *max_dist = res.peek().map(|node| node.dist); + } + } else if max_dist.is_some_and(|max_dist| max_dist > dist) { + res.pop(); + res.push(OrderedNode::new(row_id, dist)); + *max_dist = res.peek().map(|node| node.dist); + } + } + + #[allow(clippy::too_many_arguments)] + fn accumulate_raw_query_multi_bit_topk_with_scratch( + &self, + k: usize, + lower_bound: Option, + upper_bound: Option, + row_ids: impl Iterator, + res: &mut BinaryHeap>, + dists: &mut Vec, + quantized_dists: &mut Vec, + quantized_dists_table: &mut Vec, + hacc_quantized_dists: &mut Vec, + ) { + let Some(ctx) = self.raw_query_multi_bit_topk_context( + k, + lower_bound, + upper_bound, + dists, + quantized_dists, + quantized_dists_table, + hacc_quantized_dists, + ) else { + return; + }; let mut max_dist = res.peek().map(|node| node.dist); - let mut candidates = 0; - let mut pruned_upper_bound = 0; - let mut pruned_heap = 0; - let mut exact = 0; - let mut exact_rejected = 0; + let mut counters = RabitPruneCounters::default(); for (id, row_id) in row_ids { let Some(binary_ip) = dists.get(id).copied() else { continue; }; - candidates += 1; + counters.candidates += 1; let Some(raw_lower_bound) = self.raw_query_lower_bound(id, binary_ip) else { continue; }; - if raw_lower_bound >= query_upper_bound { - pruned_upper_bound += 1; - continue; - } - if res.len() >= k && max_dist.is_some_and(|max_dist| raw_lower_bound >= max_dist.0) { - pruned_heap += 1; - continue; + self.accumulate_raw_query_multi_bit_row( + &ctx, + id, + row_id, + binary_ip, + raw_lower_bound, + res, + &mut max_dist, + &mut counters, + ); + } + record_rabit_prune_stats(&counters); + } + + /// Top-k scan over all rows `0..n` in order: classify [`PRUNE_LANES`] + /// rows at a time with the SIMD lower-bound kernel and run the scalar + /// rerank only for the surviving lanes. + #[allow(clippy::too_many_arguments)] + fn accumulate_raw_query_multi_bit_topk_dense_with_scratch( + &self, + k: usize, + lower_bound: Option, + upper_bound: Option, + row_id: impl Fn(u32) -> u64, + res: &mut BinaryHeap>, + dists: &mut Vec, + quantized_dists: &mut Vec, + quantized_dists_table: &mut Vec, + hacc_quantized_dists: &mut Vec, + ) { + let Some(ctx) = self.raw_query_multi_bit_topk_context( + k, + lower_bound, + upper_bound, + dists, + quantized_dists, + quantized_dists_table, + hacc_quantized_dists, + ) else { + return; + }; + let dists = dists.as_slice(); + debug_assert_eq!(dists.len(), ctx.n); + let scale_factors = &self.scale_factors[..ctx.n]; + let add_factors = &self.add_factors[..ctx.n]; + let error_factors = &self + .error_factors + .expect("raw-query lower-bound gating requires error factors")[..ctx.n]; + // Same expression as `raw_query_lower_bound` with `error_factors` + // already resolved; the masks below match it bit for bit. + let lower_bound_of = |id: usize, binary_ip: f32| { + self.raw_query_binary_distance(id, binary_ip) - error_factors[id] * self.query_error + }; + let terms = LowerBoundTerms { + half_sum_q: 0.5 * self.sum_q, + query_factor: self.query_factor, + query_error: self.query_error, + }; + let prune_masks = prune_mask_kernel(); + let mut max_dist = res.peek().map(|node| node.dist); + let mut counters = RabitPruneCounters::default(); + + let (dist_groups, dist_tail) = dists.as_chunks::(); + let (scale_groups, _) = scale_factors.as_chunks::(); + let (add_groups, _) = add_factors.as_chunks::(); + let (error_groups, _) = error_factors.as_chunks::(); + for (group, (dist16, scale16, add16, error16)) in + izip!(dist_groups, scale_groups, add_groups, error_groups).enumerate() + { + counters.candidates += PRUNE_LANES; + // The heap threshold only ever tightens, so this group-start + // snapshot can only over-select survivors (which the per-row + // processing below re-checks against live values), never prune a + // row the scalar scan would have kept. + let heap_threshold = (res.len() >= ctx.k) + .then(|| max_dist.map(|max_dist| max_dist.0)) + .flatten(); + let (pruned_upper_bound, pruned_heap) = prune_masks( + dist16, + scale16, + add16, + error16, + terms, + ctx.query_upper_bound, + heap_threshold, + ); + counters.pruned_upper_bound += pruned_upper_bound.count_ones() as usize; + counters.pruned_heap += pruned_heap.count_ones() as usize; + let mut survivors = !(pruned_upper_bound | pruned_heap); + while survivors != 0 { + let lane = survivors.trailing_zeros() as usize; + survivors &= survivors - 1; + let id = group * PRUNE_LANES + lane; + let binary_ip = dists[id]; + self.accumulate_raw_query_multi_bit_row( + &ctx, + id, + row_id(id as u32), + binary_ip, + lower_bound_of(id, binary_ip), + res, + &mut max_dist, + &mut counters, + ); } + } - exact += 1; - let dist = self.raw_query_multi_bit_exact_distance( + let tail_start = ctx.n - dist_tail.len(); + for (offset, binary_ip) in dist_tail.iter().copied().enumerate() { + let id = tail_start + offset; + counters.candidates += 1; + self.accumulate_raw_query_multi_bit_row( + &ctx, id, + row_id(id as u32), binary_ip, - ex_bits, - ex_codes, - ex_add_factors, - ex_scale_factors, + lower_bound_of(id, binary_ip), + res, + &mut max_dist, + &mut counters, ); - if dist < query_lower_bound || dist >= query_upper_bound { - exact_rejected += 1; - continue; - } - let dist = OrderedFloat(dist); - if res.len() < k { - res.push(OrderedNode::new(row_id, dist)); - if res.len() == k { - max_dist = res.peek().map(|node| node.dist); - } - } else if max_dist.is_some_and(|max_dist| max_dist > dist) { - res.pop(); - res.push(OrderedNode::new(row_id, dist)); - max_dist = res.peek().map(|node| node.dist); - } } - record_rabit_prune_stats( - candidates, - pruned_upper_bound, - pruned_heap, - exact, - exact_rejected, - ); + record_rabit_prune_stats(&counters); } fn raw_query_lower_bound_gating_disabled_reason(&self) -> Option<&'static str> { @@ -1717,13 +1904,11 @@ impl DistCalculator for RabitDistCalculator<'_> { return; } - let code_len = rabit_binary_code_bytes(self.dim); - let n = self.codes.len() / code_len; - self.accumulate_raw_query_multi_bit_topk_with_scratch( + self.accumulate_raw_query_multi_bit_topk_dense_with_scratch( k, lower_bound, upper_bound, - (0..n).map(|id| (id, row_id(id as u32))), + row_id, res, dists, quantized_dists, @@ -2526,6 +2711,8 @@ mod tests { use arrow_array::{ArrayRef, Float32Array, Float64Array, UInt64Array}; use lance_core::ROW_ID; use lance_linalg::distance::DistanceType; + use rand::rngs::SmallRng; + use rand::{Rng, SeedableRng}; use rstest::rstest; use crate::vector::bq::{RQRotationType, builder::RabitQuantizer}; @@ -3741,6 +3928,200 @@ mod tests { } } + /// Inputs crafted so the top-k scan outcomes are fully determined by the + /// factor columns: with zero scale factors, a zero query factor, and a + /// query error of one, the lower bound is + /// `add_factors[id] - error_factors[id]`, and with zero ex scale factors + /// the exact distance is `ex_add_factors[id]`, regardless of the random + /// codes and query. + struct CraftedTopkData { + codes: Vec, + ex_codes: Vec, + dist_table: Vec, + ex_query: Vec, + scale_factors: Vec, + add_factors: Vec, + error_factors: Vec, + ex_scale_factors: Vec, + ex_add_factors: Vec, + } + + const CRAFTED_TOPK_DIM: usize = 64; + const CRAFTED_TOPK_NUM_BITS: u8 = 5; + + impl CraftedTopkData { + fn new( + exact_dists: &[f32], + lower_bound_margins: &[f32], + error_factors: Vec, + rng: &mut SmallRng, + ) -> Self { + let n = exact_dists.len(); + let code_len = rabit_binary_code_bytes(CRAFTED_TOPK_DIM); + let ex_code_len = blocked_ex_code_bytes(CRAFTED_TOPK_DIM, CRAFTED_TOPK_NUM_BITS - 1); + let add_factors = izip!(exact_dists, lower_bound_margins, &error_factors) + .map(|(dist, margin, error)| dist - margin + error) + .collect(); + Self { + codes: (0..n * code_len).map(|_| rng.random()).collect(), + ex_codes: (0..n * ex_code_len).map(|_| rng.random()).collect(), + dist_table: (0..CRAFTED_TOPK_DIM * 4) + .map(|_| rng.random_range(-1.0f32..1.0)) + .collect(), + ex_query: (0..CRAFTED_TOPK_DIM) + .map(|_| rng.random_range(-1.0f32..1.0)) + .collect(), + scale_factors: vec![0.0; n], + add_factors, + error_factors, + ex_scale_factors: vec![0.0; n], + ex_add_factors: exact_dists.to_vec(), + } + } + + fn calculator(&self, approx_mode: ApproxMode) -> RabitDistCalculator<'_> { + RabitDistCalculator::new( + CRAFTED_TOPK_DIM, + CRAFTED_TOPK_NUM_BITS, + RabitQueryEstimator::RawQuery, + Cow::Borrowed(self.dist_table.as_slice()), + Cow::Borrowed(self.ex_query.as_slice()), + 0.7, + &self.codes, + Some(&self.ex_codes), + blocked_ex_code_bytes(CRAFTED_TOPK_DIM, CRAFTED_TOPK_NUM_BITS - 1), + &self.add_factors, + &self.scale_factors, + Some(&self.error_factors), + Some(&self.ex_add_factors), + Some(&self.ex_scale_factors), + None, + 0.0, + 1.0, + approx_mode, + ) + } + } + + fn canonical_heap_rows(heap: BinaryHeap>) -> Vec<(u32, u64)> { + let mut rows = heap + .into_iter() + .map(|node| (node.dist.0.to_bits(), node.id)) + .collect::>(); + rows.sort_unstable(); + rows + } + + /// The dense (SIMD-pruned) scan must reproduce the sparse scalar scan + /// exactly: identical heap contents including row ids, and the k smallest + /// in-bounds exact distances overall. + #[rstest] + fn test_raw_query_multi_bit_topk_dense_matches_sparse( + #[values(ApproxMode::Normal, ApproxMode::Accurate)] approx_mode: ApproxMode, + #[values("descending", "ascending", "random", "duplicates", "duplicate_ties")] + ordering: &str, + ) { + for n in [1usize, 15, 16, 17, 100, 4109] { + let mut rng = SmallRng::seed_from_u64(n as u64 * 31 + ordering.len() as u64); + let exact_dists: Vec = match ordering { + // Improving rows force constant heap updates. + "descending" => (0..n).map(|id| (n - id) as f32).collect(), + // Worsening rows force mass pruning, the common regime. + "ascending" => (0..n).map(|id| id as f32).collect(), + "random" => (0..n).map(|_| rng.random_range(0.0..n as f32)).collect(), + "duplicates" => (0..n).map(|id| (id % 7) as f32).collect(), + // Lower bound equals the distance, so heap-threshold and + // upper-bound comparisons hit exact `>=` ties. + "duplicate_ties" => (0..n).map(|id| (id % 5) as f32).collect(), + _ => unreachable!(), + }; + let (margins, error_factors) = if ordering == "duplicate_ties" { + (vec![0.0; n], vec![0.0; n]) + } else if ordering == "random" { + ( + (0..n).map(|_| rng.random_range(0.0f32..2.0)).collect(), + (0..n).map(|_| rng.random_range(0.0f32..1.0)).collect(), + ) + } else { + ( + vec![1.0; n], + (0..n).map(|_| rng.random_range(0.0f32..1.0)).collect(), + ) + }; + let data = CraftedTopkData::new(&exact_dists, &margins, error_factors, &mut rng); + let calc = data.calculator(approx_mode); + assert!( + calc.raw_query_lower_bound_gating_disabled_reason() + .is_none() + ); + + let max_dist = exact_dists.iter().fold(0.0f32, |acc, dist| acc.max(*dist)); + for k in [1usize, 10, n + 7] { + for bounds in [(None, None), (Some(max_dist * 0.25), Some(max_dist * 0.7))] { + let (lower_bound, upper_bound) = bounds; + let mut dense_heap = BinaryHeap::new(); + let mut sparse_heap = BinaryHeap::new(); + let mut dists = Vec::new(); + let mut u16_scratch = Vec::new(); + let mut u8_scratch = Vec::new(); + let mut u32_scratch = Vec::new(); + // Two passes sharing the heap, as IVF partition probing + // does: the second pass starts with a full, tight heap. + for pass in 0..2u64 { + let offset = pass * n as u64; + calc.accumulate_topk_with_scratch( + k, + lower_bound, + upper_bound, + |id| id as u64 + offset, + &mut dense_heap, + &mut dists, + &mut u16_scratch, + &mut u8_scratch, + &mut u32_scratch, + ); + calc.accumulate_filtered_topk_with_scratch( + k, + lower_bound, + upper_bound, + (0..n as u32).map(|id| (id, id as u64 + offset)), + |_| true, + &mut sparse_heap, + &mut dists, + &mut u16_scratch, + &mut u8_scratch, + &mut u32_scratch, + ); + } + let dense = canonical_heap_rows(dense_heap); + let sparse = canonical_heap_rows(sparse_heap); + assert_eq!( + dense, sparse, + "ordering={ordering} n={n} k={k} bounds={bounds:?} mode={approx_mode:?}" + ); + + // The distance multiset must be the k smallest in-bounds + // distances over both passes. Row ids are not compared: + // evictions among tied maxima depend on heap layout. + let query_lower_bound = lower_bound.unwrap_or(f32::MIN); + let query_upper_bound = upper_bound.unwrap_or(f32::MAX); + let mut expected = (0..2 * n) + .map(|row| exact_dists[row % n]) + .filter(|dist| *dist >= query_lower_bound && *dist < query_upper_bound) + .map(|dist| dist.to_bits()) + .collect::>(); + expected.sort_unstable(); + expected.truncate(k); + let actual = dense.iter().map(|(dist, _)| *dist).collect::>(); + assert_eq!( + actual, expected, + "ordering={ordering} n={n} k={k} bounds={bounds:?} mode={approx_mode:?}" + ); + } + } + } + } + #[test] fn test_raw_query_one_bit_distance_uses_binary_factors_without_ex_columns() { let code_dim = 8usize; diff --git a/rust/lance/Cargo.toml b/rust/lance/Cargo.toml index 74e6faf5c07..440c3fb301a 100644 --- a/rust/lance/Cargo.toml +++ b/rust/lance/Cargo.toml @@ -175,6 +175,10 @@ required-features = ["cli"] name = "scalar_index" harness = false +[[bench]] +name = "regex_ngram" +harness = false + [[bench]] name = "merge_insert" harness = false diff --git a/rust/lance/benches/regex_ngram.rs b/rust/lance/benches/regex_ngram.rs new file mode 100644 index 00000000000..76f597ad9cb --- /dev/null +++ b/rust/lance/benches/regex_ngram.rs @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Benchmark: regex predicate scans over an ngram-indexed string column. +//! +//! Each query is a `regexp_match(doc, '...')` filter against a dataset that has +//! an NGram index on `doc`. The query set spans a selective AND pattern, an +//! alternation, a plain literal (rewritten to an infix LIKE before it reaches +//! the index), and a deliberately non-accelerable pattern (`a.b`, which yields +//! no trigram) that serves as a regression guard. +//! +//! On `main` none of these use the index (regex falls through to a full scan + +//! recheck); with the ngram-regex acceleration the index prunes candidates for +//! the first three while `a.b` stays a full scan. Capture a baseline on `main` +//! with `--save-baseline before_7130`, then compare after the change with +//! `--baseline before_7130`. + +use std::hint::black_box; +use std::sync::Arc; +use std::time::Duration; + +use arrow::array::AsArray; +use arrow_array::{RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use criterion::{Criterion, criterion_group, criterion_main}; +use futures::TryStreamExt; +use lance::Dataset; +use lance::index::DatasetIndexExt; +use lance_core::utils::tempfile::TempStrDir; +use lance_datagen::{RowCount, array}; +use lance_index::IndexType; +use lance_index::scalar::ScalarIndexParams; +#[cfg(target_os = "linux")] +use lance_testing::pprof::{Output, PProfProfiler}; + +const TOTAL: usize = 200_000; + +/// Build the `doc` column: random sentences with rare markers injected into a +/// small fraction of rows so the regex queries have controlled selectivity. +/// The markers (`zqxwvu`, `needlexyz`, `qwerasdf`) are unlikely to appear in +/// the generated English-word sentences. +fn build_docs() -> StringArray { + let mut sentence_gen = array::random_sentence(1, 30, false); + let base = sentence_gen + .generate_default(RowCount::from(TOTAL as u64)) + .unwrap(); + let base = base.as_string::(); + let docs = (0..TOTAL).map(|i| { + let sentence = base.value(i); + if i % 200 == 0 { + // ~0.5% of rows match `zqxwvu.*needlexyz` and `zqxwvu`. + format!("{sentence} zqxwvu needlexyz") + } else if i % 211 == 0 { + // A second marker for the alternation query. + format!("{sentence} qwerasdf") + } else { + sentence.to_string() + } + }); + StringArray::from_iter_values(docs) +} + +async fn build_dataset(tempdir: &TempStrDir) -> Arc { + let schema = Arc::new(Schema::new(vec![Field::new("doc", DataType::Utf8, false)])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(build_docs())]).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + + let mut dataset = Dataset::write(reader, tempdir.as_str(), None) + .await + .unwrap(); + dataset + .create_index( + &["doc"], + IndexType::NGram, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + Arc::new(dataset) +} + +async fn scan_filter(dataset: &Dataset, filter: &str) -> usize { + let mut scanner = dataset.scan(); + scanner.filter(filter).unwrap(); + let stream = scanner.try_into_stream().await.unwrap(); + let batches: Vec = stream.try_collect().await.unwrap(); + batches.iter().map(|b| b.num_rows()).sum() +} + +fn bench_regex_ngram(c: &mut Criterion) { + let rt = tokio::runtime::Runtime::new().unwrap(); + let tempdir = TempStrDir::default(); + let dataset = rt.block_on(build_dataset(&tempdir)); + + let queries = [ + ("selective_and", "regexp_match(doc, 'zqxwvu.*needlexyz')"), + ( + "alternation", + "regexp_match(doc, '(zqxwvu|qwerasdf|needlexyz)')", + ), + ("plain_literal", "regexp_match(doc, 'zqxwvu')"), + ("non_accelerable_a_dot_b", "regexp_match(doc, 'a.b')"), + ]; + + let mut group = c.benchmark_group("regex_ngram"); + group + .sample_size(10) + .measurement_time(Duration::from_secs(15)); + for (name, filter) in queries { + group.bench_function(name, |b| { + b.iter(|| black_box(rt.block_on(scan_filter(&dataset, filter)))); + }); + } + group.finish(); +} + +#[cfg(target_os = "linux")] +criterion_group!( + name = benches; + config = Criterion::default() + .significance_level(0.1) + .sample_size(10) + .with_profiler(PProfProfiler::new(100, Output::Flamegraph(None))); + targets = bench_regex_ngram); + +#[cfg(not(target_os = "linux"))] +criterion_group!( + name = benches; + config = Criterion::default().significance_level(0.1).sample_size(10); + targets = bench_regex_ngram); + +criterion_main!(benches); diff --git a/rust/lance/src/dataset.rs b/rust/lance/src/dataset.rs index 3e0d77704da..2894ef4f8da 100644 --- a/rust/lance/src/dataset.rs +++ b/rust/lance/src/dataset.rs @@ -129,8 +129,8 @@ pub use schema_evolution::{ pub use take::TakeBuilder; use uuid::Uuid; pub use write::merge_insert::{ - MergeInsertBuilder, MergeInsertJob, MergeStats, UncommittedMergeInsert, WhenMatched, - WhenNotMatched, WhenNotMatchedBySource, + MergeInsertBuilder, MergeInsertJob, MergeStats, SourceDedupeBehavior, UncommittedMergeInsert, + WhenMatched, WhenNotMatched, WhenNotMatchedBySource, }; use crate::dataset::index::LanceIndexStoreExt; diff --git a/rust/lance/src/dataset/blob.rs b/rust/lance/src/dataset/blob.rs index 56bcc97cb32..8cdde543e4e 100644 --- a/rust/lance/src/dataset/blob.rs +++ b/rust/lance/src/dataset/blob.rs @@ -12,15 +12,17 @@ use std::{ use arrow::array::AsArray; use arrow::datatypes::{UInt8Type, UInt32Type, UInt64Type}; -use arrow_array::Array; use arrow_array::RecordBatch; use arrow_array::builder::{LargeBinaryBuilder, PrimitiveBuilder, StringBuilder}; -use arrow_schema::DataType as ArrowDataType; +use arrow_array::{Array, ArrayRef}; +use arrow_schema::{DataType as ArrowDataType, Field as ArrowField}; use bytes::Bytes; +use futures::future::BoxFuture; use futures::stream::BoxStream; use futures::{FutureExt, StreamExt, TryStreamExt, stream}; use lance_arrow::{ BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, BLOB_INLINE_SIZE_THRESHOLD_META_KEY, FieldExt, + r#struct::StructArrayExt, }; use lance_io::object_store::{ObjectStore, ObjectStoreParams, ObjectStoreRegistry}; use lance_io::scheduler::{FileScheduler, ScanScheduler, SchedulerConfig}; @@ -259,10 +261,7 @@ pub struct BlobPreprocessor { data_file_key: String, local_counter: u32, pack_writer: PackWriter, - blob_v2_cols: Vec, - inline_thresholds: Vec, - dedicated_thresholds: Vec, - writer_metadata: Vec>, + field_processors: Vec, external_base_resolver: Option>, allow_external_blob_outside_bases: bool, external_blob_mode: ExternalBlobMode, @@ -287,6 +286,64 @@ enum BlobWriteSource<'a> { External(&'a ExternalBlobSource), } +#[derive(Clone, Debug)] +struct BlobPreprocessField { + kind: BlobPreprocessFieldKind, +} + +#[derive(Clone, Debug)] +enum BlobPreprocessFieldKind { + BlobV2 { + inline_threshold: usize, + dedicated_threshold: usize, + writer_metadata: HashMap, + }, + Struct { + children: Vec, + }, + Passthrough, +} + +impl BlobPreprocessField { + fn new(field: &ArrowField) -> Result { + if field.is_blob_v2() { + return Ok(Self { + kind: BlobPreprocessFieldKind::BlobV2 { + inline_threshold: blob_inline_threshold_from_metadata( + field.metadata(), + field.name(), + )?, + dedicated_threshold: blob_dedicated_threshold_from_metadata( + field.metadata(), + field.name(), + )?, + writer_metadata: field.metadata().clone(), + }, + }); + } + + if let ArrowDataType::Struct(children) = field.data_type() { + let children = children + .iter() + .map(|child| Self::new(child.as_ref())) + .collect::>>()?; + if children.iter().any(|child| child.requires_preprocessing()) { + return Ok(Self { + kind: BlobPreprocessFieldKind::Struct { children }, + }); + } + } + + Ok(Self { + kind: BlobPreprocessFieldKind::Passthrough, + }) + } + + fn requires_preprocessing(&self) -> bool { + !matches!(self.kind, BlobPreprocessFieldKind::Passthrough) + } +} + impl ExternalBlobSource { /// Return the logical payload size after applying any external slice. fn size(&self) -> u64 { @@ -378,37 +435,11 @@ impl BlobPreprocessor { pack_writer.max_pack_size = max_bytes; } let arrow_schema = arrow_schema::Schema::from(schema); - let fields = arrow_schema.fields(); - let blob_v2_cols = fields + let field_processors = arrow_schema + .fields() .iter() - .map(|field| field.is_blob_v2()) - .collect::>(); - let inline_thresholds = fields - .iter() - .zip(blob_v2_cols.iter()) - .map(|(field, is_blob_v2)| { - if *is_blob_v2 { - blob_inline_threshold_from_metadata(field.metadata(), field.name()) - } else { - Ok(INLINE_MAX) - } - }) + .map(|field| BlobPreprocessField::new(field.as_ref())) .collect::>>()?; - let dedicated_thresholds = fields - .iter() - .zip(blob_v2_cols.iter()) - .map(|(field, is_blob_v2)| { - if *is_blob_v2 { - blob_dedicated_threshold_from_metadata(field.metadata(), field.name()) - } else { - Ok(DEDICATED_THRESHOLD) - } - }) - .collect::>>()?; - let writer_metadata = fields - .iter() - .map(|field| field.metadata().clone()) - .collect(); Ok(Self { object_store, data_dir, @@ -416,10 +447,7 @@ impl BlobPreprocessor { // Start at 1 to avoid a potential all-zero blob_id value. local_counter: 1, pack_writer, - blob_v2_cols, - inline_thresholds, - dedicated_thresholds, - writer_metadata, + field_processors, external_base_resolver, allow_external_blob_outside_bases, external_blob_mode, @@ -520,7 +548,7 @@ impl BlobPreprocessor { } pub(crate) async fn preprocess_batch(&mut self, batch: &RecordBatch) -> Result { - let expected_columns = self.blob_v2_cols.len(); + let expected_columns = self.field_processors.len(); if batch.num_columns() != expected_columns { return Err(Error::invalid_input(format!( "Unexpected number of columns: expected {}, got {}", @@ -531,246 +559,340 @@ impl BlobPreprocessor { let batch_schema = batch.schema(); let batch_fields = batch_schema.fields(); + let field_processors = self.field_processors.clone(); let mut new_columns = Vec::with_capacity(batch.num_columns()); let mut new_fields = Vec::with_capacity(batch.num_columns()); - for idx in 0..batch.num_columns() { - let array = batch.column(idx); - let field = &batch_fields[idx]; - if !self.blob_v2_cols[idx] { - new_columns.push(array.clone()); - new_fields.push(field.clone()); + for ((processor, array), field) in field_processors + .iter() + .zip(batch.columns().iter()) + .zip(batch_fields.iter()) + { + let (new_column, new_field) = self + .preprocess_field(processor, array.clone(), field) + .await?; + new_columns.push(new_column); + new_fields.push(new_field); + } + + let new_schema = Arc::new(arrow_schema::Schema::new_with_metadata( + new_fields + .iter() + .map(|f| f.as_ref().clone()) + .collect::>(), + batch_schema.metadata().clone(), + )); + + RecordBatch::try_new(new_schema, new_columns) + .map_err(|e| Error::invalid_input(e.to_string())) + } + + fn preprocess_field<'a>( + &'a mut self, + processor: &'a BlobPreprocessField, + array: ArrayRef, + field: &'a Arc, + ) -> BoxFuture<'a, Result<(ArrayRef, Arc)>> { + async move { + match &processor.kind { + BlobPreprocessFieldKind::Passthrough => Ok((array, field.clone())), + BlobPreprocessFieldKind::BlobV2 { + inline_threshold, + dedicated_threshold, + writer_metadata, + } => { + self.preprocess_blob_array( + array, + field.as_ref(), + *inline_threshold, + *dedicated_threshold, + writer_metadata, + ) + .await + } + BlobPreprocessFieldKind::Struct { children } => { + self.preprocess_struct_array(array, field.as_ref(), children) + .await + } + } + } + .boxed() + } + + async fn preprocess_struct_array( + &mut self, + array: ArrayRef, + field: &ArrowField, + children: &[BlobPreprocessField], + ) -> Result<(ArrayRef, Arc)> { + let struct_arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::invalid_input("Struct field was not a struct array"))?; + if struct_arr.num_columns() != children.len() { + return Err(Error::invalid_input(format!( + "Struct field '{}' expected {} children, got {}", + field.name(), + children.len(), + struct_arr.num_columns() + ))); + } + + let struct_arr = struct_arr.normalize_slicing()?; + let parent_nulls = struct_arr.nulls().cloned(); + let pushed_down = struct_arr.pushdown_nulls()?; + let child_fields = pushed_down.fields().clone(); + let child_columns = pushed_down.columns().to_vec(); + + let mut new_columns = Vec::with_capacity(children.len()); + let mut new_fields = Vec::with_capacity(children.len()); + for ((child_processor, child_array), child_field) in children + .iter() + .zip(child_columns.into_iter()) + .zip(child_fields.iter()) + { + let (new_column, new_field) = self + .preprocess_field(child_processor, child_array, child_field) + .await?; + new_columns.push(new_column); + new_fields.push(new_field); + } + + let struct_array = + StructArray::try_new(new_fields.clone().into(), new_columns, parent_nulls)?; + let field = Arc::new( + ArrowField::new( + field.name(), + ArrowDataType::Struct(new_fields.into()), + field.is_nullable(), + ) + .with_metadata(field.metadata().clone()), + ); + Ok((Arc::new(struct_array), field)) + } + + async fn preprocess_blob_array( + &mut self, + array: ArrayRef, + field: &ArrowField, + inline_threshold: usize, + dedicated_threshold: usize, + writer_metadata: &HashMap, + ) -> Result<(ArrayRef, Arc)> { + let struct_arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::invalid_input("Blob column was not a struct array"))?; + + let data_col = struct_arr + .column_by_name("data") + .ok_or_else(|| Error::invalid_input("Blob struct missing `data` field"))? + .as_binary::(); + let uri_col = struct_arr + .column_by_name("uri") + .ok_or_else(|| Error::invalid_input("Blob struct missing `uri` field"))? + .as_string::(); + let position_col = struct_arr + .column_by_name("position") + .map(|col| col.as_primitive::()); + let size_col = struct_arr + .column_by_name("size") + .map(|col| col.as_primitive::()); + + let mut data_builder = LargeBinaryBuilder::with_capacity(struct_arr.len(), 0); + let mut uri_builder = StringBuilder::with_capacity(struct_arr.len(), 0); + let mut blob_id_builder = + PrimitiveBuilder::::with_capacity(struct_arr.len()); + let mut blob_size_builder = + PrimitiveBuilder::::with_capacity(struct_arr.len()); + let mut kind_builder = PrimitiveBuilder::::with_capacity(struct_arr.len()); + let mut position_builder = + PrimitiveBuilder::::with_capacity(struct_arr.len()); + + let struct_nulls = struct_arr.nulls(); + + for i in 0..struct_arr.len() { + if struct_arr.is_null(i) { + data_builder.append_null(); + uri_builder.append_null(); + blob_id_builder.append_null(); + blob_size_builder.append_null(); + kind_builder.append_null(); + position_builder.append_null(); continue; } - let struct_arr = array - .as_any() - .downcast_ref::() - .ok_or_else(|| Error::invalid_input("Blob column was not a struct array"))?; - - let data_col = struct_arr - .column_by_name("data") - .ok_or_else(|| Error::invalid_input("Blob struct missing `data` field"))? - .as_binary::(); - let uri_col = struct_arr - .column_by_name("uri") - .ok_or_else(|| Error::invalid_input("Blob struct missing `uri` field"))? - .as_string::(); - let position_col = struct_arr - .column_by_name("position") - .map(|col| col.as_primitive::()); - let size_col = struct_arr - .column_by_name("size") - .map(|col| col.as_primitive::()); - - let mut data_builder = LargeBinaryBuilder::with_capacity(struct_arr.len(), 0); - let mut uri_builder = StringBuilder::with_capacity(struct_arr.len(), 0); - let mut blob_id_builder = - PrimitiveBuilder::::with_capacity(struct_arr.len()); - let mut blob_size_builder = - PrimitiveBuilder::::with_capacity(struct_arr.len()); - let mut kind_builder = PrimitiveBuilder::::with_capacity(struct_arr.len()); - let mut position_builder = - PrimitiveBuilder::::with_capacity(struct_arr.len()); - - let struct_nulls = struct_arr.nulls(); - - for i in 0..struct_arr.len() { - if struct_arr.is_null(i) { - data_builder.append_null(); - uri_builder.append_null(); - blob_id_builder.append_null(); - blob_size_builder.append_null(); - kind_builder.append_null(); - position_builder.append_null(); - continue; - } + let has_data = !data_col.is_null(i); + let has_uri = !uri_col.is_null(i); + let has_position = position_col + .as_ref() + .map(|col| !col.is_null(i)) + .unwrap_or(false); + let has_size = size_col + .as_ref() + .map(|col| !col.is_null(i)) + .unwrap_or(false); + let data_len = if has_data { data_col.value(i).len() } else { 0 }; - let has_data = !data_col.is_null(i); - let has_uri = !uri_col.is_null(i); - let has_position = position_col - .as_ref() - .map(|col| !col.is_null(i)) - .unwrap_or(false); - let has_size = size_col - .as_ref() - .map(|col| !col.is_null(i)) - .unwrap_or(false); - let data_len = if has_data { data_col.value(i).len() } else { 0 }; - - let dedicated_threshold = self.dedicated_thresholds[idx]; - let inline_threshold = self.inline_thresholds[idx]; - if has_data && data_len > dedicated_threshold { - let blob_id = self.next_blob_id(); - self.write_dedicated(blob_id, BlobWriteSource::Bytes(data_col.value(i))) - .await?; - - kind_builder.append_value(BlobKind::Dedicated as u8); - data_builder.append_null(); - uri_builder.append_null(); - blob_id_builder.append_value(blob_id); - blob_size_builder.append_value(data_len as u64); - position_builder.append_null(); - continue; - } + if has_data && data_len > dedicated_threshold { + let blob_id = self.next_blob_id(); + self.write_dedicated(blob_id, BlobWriteSource::Bytes(data_col.value(i))) + .await?; - if has_data && data_len > inline_threshold { - let (pack_blob_id, position) = self - .write_packed(BlobWriteSource::Bytes(data_col.value(i))) - .await?; + kind_builder.append_value(BlobKind::Dedicated as u8); + data_builder.append_null(); + uri_builder.append_null(); + blob_id_builder.append_value(blob_id); + blob_size_builder.append_value(data_len as u64); + position_builder.append_null(); + continue; + } - kind_builder.append_value(BlobKind::Packed as u8); - data_builder.append_null(); - uri_builder.append_null(); - blob_id_builder.append_value(pack_blob_id); - blob_size_builder.append_value(data_len as u64); - position_builder.append_value(position); - continue; - } + if has_data && data_len > inline_threshold { + let (pack_blob_id, position) = self + .write_packed(BlobWriteSource::Bytes(data_col.value(i))) + .await?; - if has_uri { - let uri_val = uri_col.value(i); - if self.external_blob_mode == ExternalBlobMode::Ingest { - let position = if has_position { - Some( - position_col - .as_ref() - .expect("position column must exist") - .value(i), - ) - } else { - None - }; - let size = if has_size { - Some(size_col.as_ref().expect("size column must exist").value(i)) - } else { - None - }; - let source = self.open_external_source(uri_val, position, size).await?; - let data_len = source.size(); - - if data_len > dedicated_threshold as u64 { - let blob_id = self.next_blob_id(); - self.write_dedicated(blob_id, BlobWriteSource::External(&source)) - .await?; - - kind_builder.append_value(BlobKind::Dedicated as u8); - data_builder.append_null(); - uri_builder.append_null(); - blob_id_builder.append_value(blob_id); - blob_size_builder.append_value(data_len); - position_builder.append_null(); - continue; - } + kind_builder.append_value(BlobKind::Packed as u8); + data_builder.append_null(); + uri_builder.append_null(); + blob_id_builder.append_value(pack_blob_id); + blob_size_builder.append_value(data_len as u64); + position_builder.append_value(position); + continue; + } - if data_len > inline_threshold as u64 { - let (pack_blob_id, position) = self - .write_packed(BlobWriteSource::External(&source)) - .await?; - - kind_builder.append_value(BlobKind::Packed as u8); - data_builder.append_null(); - uri_builder.append_null(); - blob_id_builder.append_value(pack_blob_id); - blob_size_builder.append_value(data_len); - position_builder.append_value(position); - continue; - } + if has_uri { + let uri_val = uri_col.value(i); + if self.external_blob_mode == ExternalBlobMode::Ingest { + let position = if has_position { + Some( + position_col + .as_ref() + .expect("position column must exist") + .value(i), + ) + } else { + None + }; + let size = if has_size { + Some(size_col.as_ref().expect("size column must exist").value(i)) + } else { + None + }; + let source = self.open_external_source(uri_val, position, size).await?; + let data_len = source.size(); - let data = source.read_all().await?; + if data_len > dedicated_threshold as u64 { + let blob_id = self.next_blob_id(); + self.write_dedicated(blob_id, BlobWriteSource::External(&source)) + .await?; - kind_builder.append_value(BlobKind::Inline as u8); - data_builder.append_value(data.as_ref()); + kind_builder.append_value(BlobKind::Dedicated as u8); + data_builder.append_null(); uri_builder.append_null(); - blob_id_builder.append_null(); - blob_size_builder.append_null(); + blob_id_builder.append_value(blob_id); + blob_size_builder.append_value(data_len); position_builder.append_null(); continue; } - let (external_base_id, external_uri_or_path) = - self.resolve_external_reference(uri_val).await?; - kind_builder.append_value(BlobKind::External as u8); - data_builder.append_null(); - uri_builder.append_value(external_uri_or_path); - blob_id_builder.append_value(external_base_id); - if has_position && has_size { - let position = position_col - .as_ref() - .expect("position column must exist") - .value(i); - let size = size_col.as_ref().expect("size column must exist").value(i); - blob_size_builder.append_value(size); + if data_len > inline_threshold as u64 { + let (pack_blob_id, position) = self + .write_packed(BlobWriteSource::External(&source)) + .await?; + + kind_builder.append_value(BlobKind::Packed as u8); + data_builder.append_null(); + uri_builder.append_null(); + blob_id_builder.append_value(pack_blob_id); + blob_size_builder.append_value(data_len); position_builder.append_value(position); - } else { - blob_size_builder.append_null(); - position_builder.append_null(); + continue; } - continue; - } - if has_data { + let data = source.read_all().await?; + kind_builder.append_value(BlobKind::Inline as u8); - let value = data_col.value(i); - data_builder.append_value(value); + data_builder.append_value(data.as_ref()); uri_builder.append_null(); blob_id_builder.append_null(); blob_size_builder.append_null(); position_builder.append_null(); + continue; + } + + let (external_base_id, external_uri_or_path) = + self.resolve_external_reference(uri_val).await?; + kind_builder.append_value(BlobKind::External as u8); + data_builder.append_null(); + uri_builder.append_value(external_uri_or_path); + blob_id_builder.append_value(external_base_id); + if has_position && has_size { + let position = position_col + .as_ref() + .expect("position column must exist") + .value(i); + let size = size_col.as_ref().expect("size column must exist").value(i); + blob_size_builder.append_value(size); + position_builder.append_value(position); } else { - data_builder.append_null(); - uri_builder.append_null(); - blob_id_builder.append_null(); blob_size_builder.append_null(); - kind_builder.append_null(); position_builder.append_null(); } + continue; } - let child_fields = vec![ - arrow_schema::Field::new("kind", ArrowDataType::UInt8, true), - arrow_schema::Field::new("data", ArrowDataType::LargeBinary, true), - arrow_schema::Field::new("uri", ArrowDataType::Utf8, true), - arrow_schema::Field::new("blob_id", ArrowDataType::UInt32, true), - arrow_schema::Field::new("blob_size", ArrowDataType::UInt64, true), - arrow_schema::Field::new("position", ArrowDataType::UInt64, true), - ]; - - let struct_array = arrow_array::StructArray::try_new( - child_fields.clone().into(), - vec![ - Arc::new(kind_builder.finish()), - Arc::new(data_builder.finish()), - Arc::new(uri_builder.finish()), - Arc::new(blob_id_builder.finish()), - Arc::new(blob_size_builder.finish()), - Arc::new(position_builder.finish()), - ], - struct_nulls.cloned(), - )?; - - new_columns.push(Arc::new(struct_array)); - new_fields.push(Arc::new( - arrow_schema::Field::new( - field.name(), - ArrowDataType::Struct(child_fields.into()), - field.is_nullable(), - ) - .with_metadata(self.writer_metadata[idx].clone()), - )); + if has_data { + kind_builder.append_value(BlobKind::Inline as u8); + let value = data_col.value(i); + data_builder.append_value(value); + uri_builder.append_null(); + blob_id_builder.append_null(); + blob_size_builder.append_null(); + position_builder.append_null(); + } else { + data_builder.append_null(); + uri_builder.append_null(); + blob_id_builder.append_null(); + blob_size_builder.append_null(); + kind_builder.append_null(); + position_builder.append_null(); + } } - let new_schema = Arc::new(arrow_schema::Schema::new_with_metadata( - new_fields - .iter() - .map(|f| f.as_ref().clone()) - .collect::>(), - batch_schema.metadata().clone(), - )); + let child_fields = vec![ + ArrowField::new("kind", ArrowDataType::UInt8, true), + ArrowField::new("data", ArrowDataType::LargeBinary, true), + ArrowField::new("uri", ArrowDataType::Utf8, true), + ArrowField::new("blob_id", ArrowDataType::UInt32, true), + ArrowField::new("blob_size", ArrowDataType::UInt64, true), + ArrowField::new("position", ArrowDataType::UInt64, true), + ]; - RecordBatch::try_new(new_schema, new_columns) - .map_err(|e| Error::invalid_input(e.to_string())) + let struct_array = StructArray::try_new( + child_fields.clone().into(), + vec![ + Arc::new(kind_builder.finish()), + Arc::new(data_builder.finish()), + Arc::new(uri_builder.finish()), + Arc::new(blob_id_builder.finish()), + Arc::new(blob_size_builder.finish()), + Arc::new(position_builder.finish()), + ], + struct_nulls.cloned(), + )?; + + let field = Arc::new( + ArrowField::new( + field.name(), + ArrowDataType::Struct(child_fields.into()), + field.is_nullable(), + ) + .with_metadata(writer_metadata.clone()), + ); + Ok((Arc::new(struct_array), field)) } pub(crate) async fn finish(&mut self) -> Result<()> { @@ -2171,7 +2293,7 @@ mod tests { }; use arrow_array::RecordBatch; use arrow_array::{ - ArrayRef, RecordBatchIterator, StringArray, StructArray, UInt32Array, UInt64Array, + Array, ArrayRef, RecordBatchIterator, StringArray, StructArray, UInt32Array, UInt64Array, }; use arrow_schema::{DataType, Field, Schema}; use async_trait::async_trait; @@ -2227,6 +2349,32 @@ mod tests { expected: Vec, } + fn nested_blob_v2_batch(blob_array: ArrayRef) -> (Arc, RecordBatch) { + let blob_field = blob_field("blob", true); + let info_fields = vec![Field::new("name", DataType::Utf8, false), blob_field]; + let info_array: ArrayRef = Arc::new( + StructArray::try_new( + info_fields.clone().into(), + vec![ + Arc::new(StringArray::from_iter_values( + (0..blob_array.len()).map(|idx| format!("name-{idx}")), + )) as ArrayRef, + blob_array, + ], + None, + ) + .unwrap(), + ); + + let schema = Arc::new(Schema::new(vec![Field::new( + "info", + DataType::Struct(info_fields.into()), + true, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![info_array]).unwrap(); + (schema, batch) + } + #[cfg(feature = "azure")] fn azure_store_params(account_name: &str) -> ObjectStoreParams { ObjectStoreParams { @@ -3114,6 +3262,114 @@ mod tests { assert_eq!(second.as_ref(), b"world"); } + #[tokio::test] + async fn test_write_and_take_nested_blob_v2() { + let test_dir = TempStrDir::default(); + let packed_payload = vec![0x4A; super::INLINE_MAX + 1024]; + + let mut blob_builder = BlobArrayBuilder::new(3); + blob_builder.push_bytes(b"hello").unwrap(); + blob_builder.push_bytes(&packed_payload).unwrap(); + blob_builder.push_null().unwrap(); + let blob_array: ArrayRef = blob_builder.finish().unwrap(); + + let (schema, batch) = nested_blob_v2_batch(blob_array); + let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); + + let dataset = Arc::new( + Dataset::write( + reader, + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + let info_batch = dataset + .scan() + .project(&["info"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let blob_desc = info_batch + .column(0) + .as_struct() + .column_by_name("blob") + .unwrap() + .as_struct(); + assert_eq!( + blob_desc + .column_by_name("kind") + .unwrap() + .as_primitive::() + .value(0), + BlobKind::Inline as u8 + ); + assert_eq!( + blob_desc + .column_by_name("kind") + .unwrap() + .as_primitive::() + .value(1), + BlobKind::Packed as u8 + ); + + let blobs = dataset + .take_blobs_by_indices(&[0, 1], "info.blob") + .await + .unwrap(); + assert_eq!(blobs.len(), 2); + assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"hello"); + assert_eq!( + blobs[1].read().await.unwrap().as_ref(), + packed_payload.as_slice() + ); + + let null_blobs = dataset + .take_blobs_by_indices(&[2], "info.blob") + .await + .unwrap(); + assert!(null_blobs.is_empty()); + } + + #[tokio::test] + async fn test_nested_blob_v2_requires_v2_2() { + let test_dir = TempStrDir::default(); + + let mut blob_builder = BlobArrayBuilder::new(1); + blob_builder.push_bytes(b"hello").unwrap(); + let blob_array: ArrayRef = blob_builder.finish().unwrap(); + + let (schema, batch) = nested_blob_v2_batch(blob_array); + let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); + + let result = Dataset::write( + reader, + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_1), + ..Default::default() + }), + ) + .await; + + assert!( + result.is_err(), + "Nested blob v2 should be rejected for file version 2.1" + ); + assert!( + result + .unwrap_err() + .to_string() + .contains("Blob v2 requires file version >= 2.2") + ); + } + #[tokio::test] async fn test_blob_file_read_empty_range_returns_empty_bytes() { let store = reject_empty_range_store(); diff --git a/rust/lance/src/dataset/mem_wal/scanner/block_list.rs b/rust/lance/src/dataset/mem_wal/scanner/block_list.rs index 684fde48da1..8a293c3f988 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/block_list.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/block_list.rs @@ -47,9 +47,12 @@ pub async fn compute_source_block_lists( flushed_cache: Option<&Arc>, ) -> Result { // Hash each non-base source's membership, grouped by shard (generations are - // per-shard, so supersession is within-shard only). + // per-shard, so supersession is within-shard only). Flushed-generation PK + // scans (cold-cache S3 reads) run concurrently; order is irrelevant — the + // per-shard lists are sorted by generation below. let mut by_shard: ShardGenSets = HashMap::new(); let mut has_base = false; + let mut flushed_loads = Vec::new(); for source in sources { match source { LsmDataSource::BaseTable { .. } => has_base = true, @@ -72,14 +75,20 @@ pub async fn compute_source_block_lists( .. } => { // Cached by immutable path so repeated searches skip the PK scan. - let hashes = flushed_pk_hashes(path, pk_columns, session, flushed_cache).await?; - by_shard - .entry(*shard_id) - .or_default() - .push((*generation, hashes)); + flushed_loads.push(async move { + flushed_pk_hashes(path, pk_columns, session, flushed_cache) + .await + .map(|hashes| (*shard_id, *generation, hashes)) + }); } } } + for (shard_id, generation, hashes) in futures::future::try_join_all(flushed_loads).await? { + by_shard + .entry(shard_id) + .or_default() + .push((generation, hashes)); + } let mut blocked: SourceBlockLists = HashMap::new(); // Base (shardless, oldest) is superseded by every non-base generation. @@ -115,22 +124,25 @@ pub async fn fresh_tier_block_list( session: Option<&Arc>, flushed_cache: Option<&Arc>, ) -> Result>>> { - let mut sets = Vec::new(); - for source in sources { - let set = match source { - LsmDataSource::BaseTable { .. } => continue, - LsmDataSource::ActiveMemTable { batch_store, .. } => { - Arc::new(pk_hashes_from_batch_store(batch_store, pk_columns)?) - } + // Flushed PK scans run concurrently (cold-cache S3 reads); ordered + // try_join_all keeps the source order of the returned sets. + let sets = futures::future::try_join_all(sources.iter().map(|source| async move { + Ok::<_, lance_core::Error>(match source { + LsmDataSource::BaseTable { .. } => None, + LsmDataSource::ActiveMemTable { batch_store, .. } => Some(Arc::new( + pk_hashes_from_batch_store(batch_store, pk_columns)?, + )), LsmDataSource::FlushedMemTable { path, .. } => { - flushed_pk_hashes(path, pk_columns, session, flushed_cache).await? + Some(flushed_pk_hashes(path, pk_columns, session, flushed_cache).await?) } - }; - if !set.is_empty() { - sets.push(set); - } - } - Ok(sets) + }) + })) + .await?; + Ok(sets + .into_iter() + .flatten() + .filter(|set| !set.is_empty()) + .collect()) } /// Hash the PK membership of an in-memory memtable (active or frozen) from its diff --git a/rust/lance/src/dataset/mem_wal/scanner/builder.rs b/rust/lance/src/dataset/mem_wal/scanner/builder.rs index ade4164d485..508605c4642 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/builder.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/builder.rs @@ -125,6 +125,9 @@ pub struct LsmScanner { /// Cache of opened flushed-generation datasets. When set, repeated /// queries against the same generation skip the manifest read entirely. flushed_cache: Option>, + /// Over-fetch multiple for block-listed sources in search plans + /// (see [`super::LsmFtsSearchPlanner::with_overfetch_factor`]). + overfetch_factor: Option, } impl LsmScanner { @@ -160,6 +163,7 @@ impl LsmScanner { pk_columns, session, flushed_cache: None, + overfetch_factor: None, } } @@ -198,6 +202,7 @@ impl LsmScanner { pk_columns, session: None, flushed_cache: None, + overfetch_factor: None, } } @@ -253,6 +258,14 @@ impl LsmScanner { self } + /// Set the over-fetch multiple block-listed sources use in search plans + /// so they still yield `k` live rows after cross-generation dedup. + /// Threaded into [`super::LsmFtsSearchPlanner`]; clamped to `>= 1.0`. + pub fn with_overfetch_factor(mut self, factor: f64) -> Self { + self.overfetch_factor = Some(factor); + self + } + /// Project specific columns. /// /// If not called, all columns from the base schema are included. @@ -370,6 +383,9 @@ impl LsmScanner { if let Some(cache) = &self.flushed_cache { planner = planner.with_flushed_cache(cache.clone()); } + if let Some(factor) = self.overfetch_factor { + planner = planner.with_overfetch_factor(factor); + } planner .plan_scan( @@ -405,6 +421,9 @@ impl LsmScanner { if let Some(cache) = &self.flushed_cache { planner = planner.with_flushed_cache(cache.clone()); } + if let Some(factor) = self.overfetch_factor { + planner = planner.with_overfetch_factor(factor); + } planner .plan_search(column, query, k, self.projection.as_deref()) .await diff --git a/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs b/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs index e3ef44d8b1a..626b0effe3c 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs @@ -167,23 +167,39 @@ impl LsmFtsSearchPlanner { .await?; let overfetch = self.overfetch_factor.max(1.0); - let mut per_source_plans: Vec> = Vec::with_capacity(sources.len()); - for source in &sources { - let is_active = matches!(source, LsmDataSource::ActiveMemTable { .. }); - let blocked = block_lists.get(&(source.shard_id(), source.generation())); - // Over-fetch a blocked source so the post-filter still yields k live - // rows. The active arm returns all matches (no builder limit), so its - // within-source dedup needs no over-fetch hint. - let fetch_k = if blocked.is_some() { - ((k as f64) * overfetch).ceil() as usize - } else { - k - }; - - let plan = self - .build_source_plan(source, column, &query, fetch_k, projection, is_active) - .await?; + // Stage the per-source over-fetch decisions, then build every source + // plan concurrently — the builds are independent and a sequential loop + // was the dominant serial planning cost at multiple generations. + let arm_inputs: Vec<_> = sources + .iter() + .map(|source| { + let is_active = matches!(source, LsmDataSource::ActiveMemTable { .. }); + let blocked = block_lists.get(&(source.shard_id(), source.generation())); + // Over-fetch a blocked source so the post-filter still yields k live + // rows. The active arm returns all matches (no builder limit), so its + // within-source dedup needs no over-fetch hint. + let fetch_k = if blocked.is_some() { + ((k as f64) * overfetch).ceil() as usize + } else { + k + }; + (source, is_active, blocked, fetch_k) + }) + .collect(); + let built = + futures::future::try_join_all(arm_inputs.iter().map( + |(source, is_active, _, fetch_k)| { + Box::pin(self.build_source_plan( + source, column, &query, *fetch_k, projection, *is_active, + )) + }, + )) + .await?; + let mut per_source_plans: Vec> = Vec::with_capacity(sources.len()); + for ((_, is_active, blocked, _), plan) in arm_inputs.iter().zip(built) { + let is_active = *is_active; + let blocked = *blocked; // Dedup, mirroring LsmVectorSearchPlanner: // * active: collapse duplicate-PK appends to the newest insert // (larger _rowid = inserted later). The FTS index is append-only, @@ -219,8 +235,11 @@ impl LsmFtsSearchPlanner { per_source_plans.into_iter().next().unwrap() } else { #[allow(deprecated)] - let union: Arc = Arc::new(UnionExec::new(per_source_plans)); - union + // The downstream `SortPreservingMergeExec` already spawns one driver + // task per input partition (one per union arm) via `spawn_buffered`, + // so each arm's per-arm CPU (posting decode, BM25) runs on its own + // task without an extra repartition. + Arc::new(UnionExec::new(per_source_plans)) }; let score_idx = merged.schema().index_of(SCORE_COLUMN).map_err(|_| { diff --git a/rust/lance/src/dataset/mem_wal/scanner/planner.rs b/rust/lance/src/dataset/mem_wal/scanner/planner.rs index f3f15e2e680..eca0255be1c 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/planner.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/planner.rs @@ -34,6 +34,10 @@ pub struct LsmScanPlanner { session: Option>, /// Cache of opened flushed-generation datasets. flushed_cache: Option>, + /// Over-fetch multiple for the per-source limit pushdown: block-listed + /// sources scan `(offset + limit) * factor` rows so cross-gen dedup drops + /// still leave enough live rows. Clamped to `>= 1.0`. + overfetch_factor: f64, } impl LsmScanPlanner { @@ -49,6 +53,7 @@ impl LsmScanPlanner { base_schema, session: None, flushed_cache: None, + overfetch_factor: 1.0, } } @@ -66,6 +71,13 @@ impl LsmScanPlanner { self } + /// Set the over-fetch multiple for the per-source limit pushdown + /// (see the field docs). Clamped to `>= 1.0` at use. + pub fn with_overfetch_factor(mut self, factor: f64) -> Self { + self.overfetch_factor = factor; + self + } + /// Create scan plan with deduplication. /// /// # Arguments @@ -130,23 +142,59 @@ impl LsmScanPlanner { // cross-gen block-list, not from output ordering. let sources: Vec<_> = sources.into_iter().rev().collect(); + // Per-source limit pushdown: an unordered LIMIT needs only + // `offset + limit` live rows from EACH source to fill the global + // limit after dedup (any-N semantics), so cap every on-disk source + // instead of scanning whole generations and trimming above the + // union. Block-listed sources over-fetch by `overfetch_factor` so + // cross-gen dedup drops still leave `n_needed` live rows; the + // PkHashFilter warns when that was not enough. The active memtable + // is in-memory and within-gen append duplicates are resolved by its + // own dedup, so it is never capped here. + let n_needed = limit.map(|l| l.saturating_add(offset.unwrap_or(0))); + let overfetch = self.overfetch_factor.max(1.0); + let mut source_plans = Vec::new(); for source in sources { let is_base = matches!(source, LsmDataSource::BaseTable { .. }); - let scan = self.build_source_scan(&source, projection, filter).await?; + let is_active = matches!(source, LsmDataSource::ActiveMemTable { .. }); + let blocked = block_lists + .get(&(source.shard_id(), source.generation())) + .cloned(); + let fetch = match (n_needed, is_active) { + (Some(n), false) => Some(if blocked.is_some() { + ((n as f64) * overfetch).ceil() as usize + } else { + n + }), + _ => None, + }; + let scan = self + .build_source_scan(&source, projection, filter, fetch) + .await?; // Drop cross-generation stale rows (PKs superseded by a newer gen). - // `k = 0`: there is no top-k, so the under-fetch warning never fires. - let scan = match block_lists.get(&(source.shard_id(), source.generation())) { + // With a limit, `k = n_needed` arms the under-fetch warning; with + // no limit `k = 0` keeps it silent. + let scan = match blocked { Some(set) => Arc::new(PkHashFilterExec::new( scan, self.pk_columns.clone(), - set.clone(), - 0, + set, + n_needed.unwrap_or(0), )) as Arc, None => scan, }; + // Post-block-list cap: each source contributes at most `n_needed` + // live rows toward the global limit. + let scan: Arc = match n_needed { + Some(n) if !is_active => Arc::new( + datafusion::physical_plan::limit::LocalLimitExec::new(scan, n), + ), + _ => scan, + }; + // When `_rowaddr` is surfaced, NULL it for non-base arms: only base // values are meaningful (e.g. for `take_rows`); per-source addresses // collide with base IDs. @@ -229,6 +277,7 @@ impl LsmScanPlanner { source: &LsmDataSource, projection: Option<&[String]>, filter: Option<&Expr>, + fetch: Option, ) -> Result> { match source { LsmDataSource::BaseTable { dataset } => { @@ -247,6 +296,11 @@ impl LsmScanPlanner { if let Some(expr) = filter { scanner.filter_expr(expr.clone()); } + // Per-source limit pushdown (post-filter rows): bounds the + // physical scan instead of trimming above the union. + if let Some(fetch) = fetch { + scanner.limit(Some(fetch as i64), None)?; + } scanner.create_plan().await } @@ -264,6 +318,12 @@ impl LsmScanPlanner { if let Some(expr) = filter { scanner.filter_expr(expr.clone()); } + // Per-source limit pushdown: flushed generations are + // within-gen live (dedup-on-flush deletion vectors), so any + // `fetch` post-filter rows are valid contributions. + if let Some(fetch) = fetch { + scanner.limit(Some(fetch as i64), None)?; + } scanner.create_plan().await } diff --git a/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs b/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs index b6b1f952b25..878063321aa 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs @@ -233,30 +233,47 @@ impl LsmVectorSearchPlanner { // `block_lists` is non-empty exactly when a newer generation exists. let refine_base = refine_base_table || !block_lists.is_empty(); + // Stage per-source over-fetch decisions, then build every KNN plan + // concurrently — the builds are independent and a sequential loop was + // the dominant serial planning cost at multiple generations. + let arm_inputs: Vec<_> = sources + .iter() + .map(|source| { + let generation = source.generation(); + let is_base = matches!(source, LsmDataSource::BaseTable { .. }); + let is_active = matches!(source, LsmDataSource::ActiveMemTable { .. }); + // Over-fetch when the post-source filter can drop candidates: a + // blocked source loses superseded rows; the active source's + // within-source dedup collapses duplicate-PK HNSW nodes. Block + // lookup is per shard — generations are per-shard. + let blocked = block_lists.get(&(source.shard_id(), generation)); + let fetch_k = if blocked.is_some() || is_active { + ((k as f64) * overfetch_factor).ceil() as usize + } else { + k + }; + (source, is_base, is_active, blocked, fetch_k) + }) + .collect(); + let built = futures::future::try_join_all(arm_inputs.iter().map( + |(source, is_base, _, _, fetch_k)| { + Box::pin(self.build_knn_plan( + source, + query_vector, + *fetch_k, + nprobes, + projection, + *is_base && refine_base, + )) + }, + )) + .await?; + let mut knn_plans = Vec::new(); - for source in &sources { - let generation = source.generation(); - let is_base = matches!(source, LsmDataSource::BaseTable { .. }); - let is_active = matches!(source, LsmDataSource::ActiveMemTable { .. }); - // Over-fetch when the post-source filter can drop candidates: a - // blocked source loses superseded rows; the active source's - // within-source dedup collapses duplicate-PK HNSW nodes. Block - // lookup is per shard — generations are per-shard. - let blocked = block_lists.get(&(source.shard_id(), generation)); - let fetch_k = if blocked.is_some() || is_active { - ((k as f64) * overfetch_factor).ceil() as usize - } else { - k - }; - let knn = Box::pin(self.build_knn_plan( - source, - query_vector, - fetch_k, - nprobes, - projection, - is_base && refine_base, - )) - .await?; + for ((_, is_base, is_active, blocked, _), knn) in arm_inputs.iter().zip(built) { + let is_base = *is_base; + let is_active = *is_active; + let blocked = *blocked; // Make each source independently newest-per-PK before the union: // * active: the append-only HNSW returns one node per inserted // version, so collapse duplicate PKs to the newest insert @@ -301,6 +318,10 @@ impl LsmVectorSearchPlanner { // No cross-source dedup needed (see struct doc): SortExec(per partition) // + SortPreservingMerge does the p-way distance-ordered top-k merge. #[allow(deprecated)] + // The downstream `SortPreservingMergeExec` already spawns one driver + // task per input partition (one per union arm) via `spawn_buffered`, so + // each arm's per-arm CPU (HNSW search, distance refine) runs on its own + // task without an extra repartition. let merged: Arc = Arc::new(UnionExec::new(knn_plans)); let distance_idx = merged.schema().index_of(DISTANCE_COLUMN).map_err(|_| { diff --git a/rust/lance/src/dataset/optimize.rs b/rust/lance/src/dataset/optimize.rs index d591e42cc73..56cf74c1a62 100644 --- a/rust/lance/src/dataset/optimize.rs +++ b/rust/lance/src/dataset/optimize.rs @@ -191,6 +191,13 @@ pub struct CompactionOptions { /// specified then the default (see /// [`crate::dataset::Scanner::batch_size`]) will be used. pub batch_size: Option, + /// The number of bytes to allow to queue up in the I/O buffer when scanning + /// the input fragments. If not specified then the default (see + /// [`crate::dataset::Scanner::io_buffer_size`]) will be used. + /// + /// Increasing this can avoid a deadlock that occurs when a single batch of + /// data is larger than the I/O buffer size. + pub io_buffer_size: Option, /// Whether to defer remapping indices during compaction. If true, indices will /// not be remapped during this compaction operation. Instead, the fragment reuse index /// is updated and will be used to perform remapping later. @@ -237,6 +244,7 @@ impl Default for CompactionOptions { num_threads: None, max_bytes_per_file: None, batch_size: None, + io_buffer_size: None, defer_index_remap: false, compaction_mode: None, enable_binary_copy: false, @@ -264,6 +272,7 @@ impl CompactionOptions { /// - `lance.compaction.materialize_deletions_threshold` /// - `lance.compaction.defer_index_remap` /// - `lance.compaction.batch_size` + /// - `lance.compaction.io_buffer_size` /// - `lance.compaction.compaction_mode` /// - `lance.compaction.binary_copy_read_batch_bytes` /// - `lance.compaction.max_source_fragments` @@ -347,6 +356,14 @@ impl CompactionOptions { )) })?); } + "io_buffer_size" => { + self.io_buffer_size = Some(value.parse().map_err(|_| { + Error::invalid_input(format!( + "Invalid value for {}: '{}' (expected a non-negative integer)", + key, value + )) + })?); + } "compaction_mode" => { self.compaction_mode = Some(CompactionMode::try_from(value.as_str())?); } @@ -1194,6 +1211,8 @@ async fn transform_blob_v2_batch( /// and preserve insertion order. /// - `batch_size`: Optional batch size; if provided, set it on the scanner to control /// read batching. +/// - `io_buffer_size`: Optional I/O buffer size in bytes; if provided, set it on the +/// scanner to control how much data is queued during reads. /// - `with_frags`: Whether to scan only the specified old fragments and force /// in-order reading. /// - `capture_row_ids`: When index remapping is needed, include and capture the @@ -1209,6 +1228,7 @@ async fn prepare_reader( dataset: &Dataset, fragments: &[Fragment], batch_size: Option, + io_buffer_size: Option, with_frags: bool, capture_row_ids: bool, ) -> Result<( @@ -1234,6 +1254,9 @@ async fn prepare_reader( if let Some(bs) = batch_size { scanner.batch_size(bs); } + if let Some(io_buffer_size) = io_buffer_size { + scanner.io_buffer_size(io_buffer_size); + } if with_frags { scanner .with_fragments(fragments.to_vec()) @@ -1515,6 +1538,7 @@ async fn rewrite_files( dataset.as_ref(), &fragments, options.batch_size, + options.io_buffer_size, true, needs_remapping, ) @@ -2636,6 +2660,57 @@ mod tests { assert_eq!(scanned_data, data); } + #[rstest] + #[tokio::test] + async fn test_compact_with_io_buffer_size( + #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)] + data_storage_version: LanceFileVersion, + ) { + // Compaction should succeed and produce correct results when an + // explicit io_buffer_size is provided via CompactionOptions. + let test_dir = TempStrDir::default(); + let test_uri = &test_dir; + + let data = sample_data(); + + // Create a table with 2 small fragments so there is something to compact. + let reader = RecordBatchIterator::new(vec![Ok(data.clone())], data.schema()); + let write_params = WriteParams { + max_rows_per_file: 5_000, + max_rows_per_group: 1_000, + data_storage_version: Some(data_storage_version), + ..Default::default() + }; + let mut dataset = Dataset::write(reader, test_uri, Some(write_params)) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 2); + + let options = CompactionOptions { + // A generous buffer so the read does not deadlock on large batches. + io_buffer_size: Some(256 * 1024 * 1024), + ..Default::default() + }; + let plan = plan_compaction(&dataset, &options).await.unwrap(); + assert_eq!(plan.tasks().len(), 1); + + let metrics = compact_files(&mut dataset, options, None).await.unwrap(); + assert_eq!(metrics.fragments_removed, 2); + assert_eq!(metrics.fragments_added, 1); + + // All rows are preserved after compaction. + let scanner = dataset.scan(); + let batches = scanner + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let scanned_data = concat_batches(&batches[0].schema(), &batches).unwrap(); + assert_eq!(scanned_data.num_rows(), data.num_rows()); + } + #[rstest] #[tokio::test] async fn test_compact_deletions( @@ -4683,6 +4758,10 @@ mod tests { "lance.compaction.batch_size".to_string(), "4096".to_string(), ), + ( + "lance.compaction.io_buffer_size".to_string(), + "1073741824".to_string(), + ), ( "lance.compaction.compaction_mode".to_string(), "try_binary_copy".to_string(), @@ -4701,6 +4780,7 @@ mod tests { assert!((opts.materialize_deletions_threshold - 0.25).abs() < f32::EPSILON); assert!(opts.defer_index_remap); assert_eq!(opts.batch_size, Some(4096)); + assert_eq!(opts.io_buffer_size, Some(1_073_741_824)); assert_eq!(opts.compaction_mode, Some(CompactionMode::TryBinaryCopy)); assert_eq!(opts.binary_copy_read_batch_bytes, Some(8_388_608)); } diff --git a/rust/lance/src/dataset/scanner.rs b/rust/lance/src/dataset/scanner.rs index 1112721bb33..2d75104d26e 100644 --- a/rust/lance/src/dataset/scanner.rs +++ b/rust/lance/src/dataset/scanner.rs @@ -8412,6 +8412,198 @@ mod test { .unwrap(); } + #[tokio::test] + async fn test_ngram_regex_index_scan() { + use arrow::array::AsArray; + + // A small, fixed corpus written across multiple fragments so the ngram + // index spans fragment boundaries. + let values = [ + "rhino", // 0 + "rhinos nose", // 1 + "cat", // 2 + "dog", // 3 + "cat dog", // 4 + "elephant", // 5 + "catalog", // 6 + "scatter", // 7 + "rhino horn", // 8 + "mouse", // 9 + "category", // 10 + "dogma", // 11 + ]; + let array = StringArray::from_iter_values(values); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "s", + DataType::Utf8, + false, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(array)]).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let write_params = WriteParams { + max_rows_per_file: 4, // 12 rows -> 3 fragments + ..Default::default() + }; + let mut dataset = Dataset::write(reader, "memory://test_ngram_regex", Some(write_params)) + .await + .unwrap(); + dataset + .create_index( + &["s"], + IndexType::NGram, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + assert!( + dataset.get_fragments().len() > 1, + "expected a multi-fragment dataset" + ); + + // Scan with `filter` and return the matched `s` values, sorted. + async fn matched(dataset: &Dataset, filter: &str) -> Vec { + let mut scan = dataset.scan(); + scan.filter(filter).unwrap(); + let batches = scan + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let mut out = Vec::new(); + for batch in batches { + let col = batch.column_by_name("s").unwrap().as_string::(); + out.extend(col.iter().flatten().map(|s| s.to_string())); + } + out.sort(); + out + } + + // `regexp_like`: a plain literal substring. + assert_eq!( + matched(&dataset, "regexp_like(s, 'rhino')").await, + ["rhino", "rhino horn", "rhinos nose"] + ); + // `regexp_match` (coerced to `IsNotNull(regexp_match(...))`) accelerates too. + assert_eq!( + matched(&dataset, "regexp_match(s, 'rhino')").await, + ["rhino", "rhino horn", "rhinos nose"] + ); + // Anchored: recheck must drop trigram false positives -- the `cat` + // trigram also occurs in cat dog / catalog / scatter / category. + assert_eq!(matched(&dataset, "regexp_like(s, 'cat$')").await, ["cat"]); + // AND across `.*`: row 8 ("rhino horn") shares the rhino trigrams but + // lacks the nose trigrams, so only "rhinos nose" survives. + assert_eq!( + matched(&dataset, "regexp_like(s, 'rhino.*nose')").await, + ["rhinos nose"] + ); + // Alternation. + assert_eq!( + matched(&dataset, "regexp_like(s, '(catalog|elephant)')").await, + ["catalog", "elephant"] + ); + // A non-accelerable pattern (no trigram derivable) still returns correct + // results via a full recheck. + assert_eq!(matched(&dataset, "regexp_like(s, 'o.m')").await, ["dogma"]); + // A case-insensitive flag is not accelerated (the index normalization + // disagrees with Unicode case folding) but must still return correct + // results via a full recheck -- here matching despite the upper-case + // pattern. This exercises the three-argument `regexp_like` flags path. + assert_eq!( + matched(&dataset, "regexp_like(s, 'RHINO', 'i')").await, + ["rhino", "rhino horn", "rhinos nose"] + ); + + // Infix LIKE is accelerated through the same machinery (a plain-literal + // `regexp_like` is rewritten to LIKE before it reaches the index). + assert_eq!( + matched(&dataset, "s LIKE '%rhino%'").await, + ["rhino", "rhino horn", "rhinos nose"] + ); + // Prefix LIKE: recheck drops "scatter", which contains the `cat` trigram + // but does not start with "cat". + assert_eq!( + matched(&dataset, "s LIKE 'cat%'").await, + ["cat", "cat dog", "catalog", "category"] + ); + + // The ngram index is actually engaged for every accelerated form. + for filter in [ + "regexp_like(s, 'rhino')", + "regexp_match(s, 'rhino')", + "s LIKE '%rhino%'", + ] { + let mut scan = dataset.scan(); + scan.filter(filter).unwrap(); + let plan = scan.create_plan().await.unwrap(); + let plan_str = format!( + "{}", + datafusion::physical_plan::displayable(plan.as_ref()).indent(true) + ); + assert!( + plan_str.contains("ScalarIndexQuery") && plan_str.contains("NGram"), + "expected ngram index usage for `{filter}`, got plan:\n{plan_str}" + ); + } + } + + #[tokio::test] + async fn test_ngram_regex_non_accelerable_recheck() { + // `a.b` yields no trigram, so the index returns "recheck everything". + // This must still produce ALL correct matches across fragments, not an + // empty set (a regression test for the AtLeast recheck path, which a + // single-match case would not catch). + let unit = ["acb", "dog", "axb", "cat", "qqq", "rhino"]; + let values: Vec<&str> = unit.iter().copied().cycle().take(60).collect(); + let array = StringArray::from_iter_values(values); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "text", + DataType::Utf8, + false, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(array)]).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let write_params = WriteParams { + max_rows_per_file: 20, // 60 rows -> 3 fragments + ..Default::default() + }; + let mut dataset = + Dataset::write(reader, "memory://test_ngram_regex_ne", Some(write_params)) + .await + .unwrap(); + dataset + .create_index( + &["text"], + IndexType::NGram, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + async fn count(dataset: &Dataset, filter: &str) -> usize { + let mut scan = dataset.scan(); + scan.filter(filter).unwrap(); + let batches = scan + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + batches.iter().map(|b| b.num_rows()).sum() + } + + // "acb" and "axb" each appear 10 times in the 60 rows -> 20 matches. + assert_eq!(count(&dataset, "regexp_match(text, 'a.b')").await, 20); + assert_eq!(count(&dataset, "regexp_like(text, 'a.b')").await, 20); + } + #[tokio::test] async fn test_like_prefix_with_btree_index() { // Create dataset with string data that has various prefixes diff --git a/rust/lance/src/dataset/schema_evolution.rs b/rust/lance/src/dataset/schema_evolution.rs index a2c8f05a89f..5ef35a33ab7 100644 --- a/rust/lance/src/dataset/schema_evolution.rs +++ b/rust/lance/src/dataset/schema_evolution.rs @@ -12,6 +12,7 @@ use super::{ transaction::{Operation, Transaction}, write::cleanup_data_fragments, }; +use crate::index::DatasetIndexExt; use crate::{Error, Result, io::exec::Planner}; use arrow::compute::CastOptions; use arrow::compute::can_cast_types; @@ -615,6 +616,17 @@ async fn add_columns_from_stream( debug_assert_eq!(batch.num_columns(), 1); let mut rows_remaining = batch.num_rows(); + // The updater yields an empty batch when every row in a read batch + // has been deleted (e.g. a whole batch falls within the deletion + // vector). There is nothing to pull from the stream in that case, so + // feed an empty batch back to keep the updater in sync and continue. + if rows_remaining == 0 { + updater + .update(RecordBatch::new_empty(stream.schema())) + .await?; + continue; + } + let mut batches = Vec::new(); while rows_remaining > 0 { @@ -736,6 +748,41 @@ pub(super) async fn alter_columns( new_schema.validate()?; + // If any column being cast has an attached index, fail fast. Cast operations + // rewrite the underlying column data and silently invalidate any index on the + // affected column(s). The current behavior is to drop such indices without + // warning, which has caused production incidents where vector search silently + // regressed to brute-force scan. We require users to explicitly drop the + // index before altering the column type, so the action is never silent. + if !cast_fields.is_empty() { + let indices = dataset.load_indices().await?; + let affected: Vec<&lance_table::format::IndexMetadata> = indices + .iter() + .filter(|idx| { + cast_fields + .iter() + .any(|(old, _)| idx.fields.contains(&old.id)) + }) + .collect(); + if !affected.is_empty() { + let affected_cols: Vec = cast_fields + .iter() + .filter(|(old, _)| affected.iter().any(|i| i.fields.contains(&old.id))) + .map(|(old, _)| old.name.clone()) + .collect(); + let affected_idx_names: Vec = affected.iter().map(|i| i.name.clone()).collect(); + return Err(Error::invalid_input(format!( + "Cannot cast column(s) [{}] to a new type: they have {} index(es) \ + attached: [{}]. Cast rewrites column data and invalidates any index \ + on the affected column(s). Drop the index(es) with drop_index() \ + before altering, then recreate them after the cast completes.", + affected_cols.join(", "), + affected.len(), + affected_idx_names.join(", "), + ))); + } + } + // If we aren't casting a column, we don't need to touch the fragments. let transaction = if cast_fields.is_empty() { Transaction::new( @@ -1037,6 +1084,69 @@ mod test { Ok(()) } + #[tokio::test] + async fn test_add_columns_with_fully_deleted_batch() -> Result<()> { + // Regression test: when an entire read batch has been deleted, the + // updater yields a 0-row batch. The inner loop then never runs and + // `batches` stays empty, so `concat_batches(&batches[0]..)` used to + // panic with "index out of bounds: the len is 0 but the index is 0". + // + // A single fragment holds 105 rows; deleting the trailing 5 rows means + // that, when read with batch_size=50, the third batch [100..105) is + // fully filtered out and produces an empty batch. + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "i", + DataType::Int32, + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..105))], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + + let test_dir = TempStrDir::default(); + let test_uri = &test_dir; + let mut dataset = Dataset::write( + reader, + test_uri, + Some(WriteParams { + max_rows_per_file: 200, // keep all rows in a single fragment + ..Default::default() + }), + ) + .await?; + + // Delete the entire trailing batch [100..105). + dataset.delete("i >= 100").await?; + assert_eq!(dataset.count_rows(None).await?, 100); + + let new_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "j", + DataType::Int32, + false, + )])); + let new_batch = RecordBatch::try_new( + new_schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..100))], + )?; + let reader = RecordBatchIterator::new(vec![Ok(new_batch)], new_schema.clone()); + + // Read with batch_size=50 so the deleted trailing rows form a full empty batch. + dataset + .add_columns(NewColumnTransform::Reader(Box::new(reader)), None, Some(50)) + .await?; + + let data = dataset.scan().try_into_batch().await?; + assert_eq!(data.num_rows(), 100); + assert_eq!( + data.column_by_name("j").unwrap().as_ref(), + &Int32Array::from_iter_values(0..100) + ); + + Ok(()) + } + #[rstest] #[tokio::test] async fn test_add_columns_cleans_up_blob_v2_data_on_stream_error( @@ -2574,7 +2684,6 @@ mod test { ) -> Result<()> { // Create a table with 2 scalar columns, 1 vector column - use crate::index::DatasetIndexExt; use arrow::datatypes::{Int32Type, Int64Type}; use arrow_array::{Float16Array, Float32Array, Int64Array, ListArray}; use half::f16; @@ -2675,7 +2784,10 @@ mod test { assert_eq!(f.files.len(), 2); }); - // Cast scalar column with index, should not keep index (TODO: keep it) + // Cast scalar column with index. The index must be dropped first; cast + // is now a fail-fast operation when an index is attached, see + // test_alter_columns_cast_fails_with_attached_index for that path. + dataset.drop_index("i_idx").await?; dataset .alter_columns(&[ColumnAlteration::new("i".into()).cast_to(DataType::Int64)]) .await?; @@ -2696,7 +2808,8 @@ mod test { ]); assert_eq!(&ArrowSchema::from(dataset.schema()), &expected_schema); - // We currently lose the index when casting a column + // The scalar index on `i` is gone (we dropped it); the vector index on + // `vec` is still present. let indices = dataset.load_indices().await?; assert_eq!(indices.len(), 1); @@ -2705,7 +2818,8 @@ mod test { assert_eq!(f.files.len(), 3); }); - // Cast vector column, should not keep index (TODO: keep it) + // Cast vector column. Drop its index first (same reason as above). + dataset.drop_index("vec_idx").await?; dataset .alter_columns(&[ ColumnAlteration::new("vec".into()).cast_to(DataType::FixedSizeList( @@ -2773,6 +2887,120 @@ mod test { Ok(()) } + /// Cast on a column with an attached index must fail fast rather than + /// silently dropping the index. This guards against the historical behavior + /// where cast would rewrite column data and the index would vanish without + /// any error or warning, causing vector search to silently regress to a + /// brute-force scan. + #[rstest] + #[tokio::test] + async fn test_alter_columns_cast_fails_with_attached_index( + #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)] + data_storage_version: LanceFileVersion, + ) -> Result<()> { + use lance_arrow::FixedSizeListArrayExt; + use lance_index::IndexType; + use lance_linalg::distance::MetricType; + use lance_testing::datagen::generate_random_array; + + use crate::index::vector::VectorIndexParams; + + // Build a small dataset with one indexed vector column. + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "vec", + DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float32, true)), + 64, + ), + false, + )])); + let nrows = 256; + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new( + ::try_new_from_values( + generate_random_array(64 * nrows as usize), + 64, + ) + .unwrap(), + )], + )?; + + let test_dir = TempStrDir::default(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), + &test_dir, + Some(WriteParams { + data_storage_version: Some(data_storage_version), + ..Default::default() + }), + ) + .await?; + + // Build an IVF_PQ index on the vector column. + let params = VectorIndexParams::ivf_pq(4, 8, 8, MetricType::L2, 50); + dataset + .create_index(&["vec"], IndexType::Vector, None, ¶ms, false) + .await?; + + let indices_before = dataset.load_indices().await?; + assert_eq!(indices_before.len(), 1, "precondition: index exists"); + let index_name = indices_before[0].name.clone(); + + // Attempting to cast the indexed column must fail with a clear message + // that names the offending index(es). + let result = dataset + .alter_columns(&[ + ColumnAlteration::new("vec".into()).cast_to(DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float16, true)), + 64, + )), + ]) + .await; + let err = result.expect_err("cast on indexed column should fail"); + let msg = err.to_string(); + assert!( + msg.contains("vec") && msg.contains(&index_name), + "error should mention column and index name, got: {msg}" + ); + assert!( + msg.contains("drop_index"), + "error should suggest the remediation, got: {msg}" + ); + + // The dataset must be unchanged: schema is still float32, index still present. + assert_eq!( + dataset.schema().field("vec").unwrap().data_type(), + DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float32, true)), + 64, + ), + ); + let indices_after = dataset.load_indices().await?; + assert_eq!(indices_after.len(), 1, "index should still exist"); + assert_eq!(indices_after[0].name, index_name); + + // Sanity check: after dropping the index, the same cast should succeed. + dataset.drop_index(&index_name).await?; + dataset + .alter_columns(&[ + ColumnAlteration::new("vec".into()).cast_to(DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float16, true)), + 64, + )), + ]) + .await?; + assert_eq!( + dataset.schema().field("vec").unwrap().data_type(), + DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float16, true)), + 64, + ), + ); + + Ok(()) + } + #[rstest] #[tokio::test] async fn test_drop_columns( diff --git a/rust/lance/src/dataset/write.rs b/rust/lance/src/dataset/write.rs index 37a3d584271..ff0a119158c 100644 --- a/rust/lance/src/dataset/write.rs +++ b/rust/lance/src/dataset/write.rs @@ -583,7 +583,7 @@ pub async fn do_write_fragments( }; let external_base_resolver = if storage_version >= LanceFileVersion::V2_2 - && schema.fields.iter().any(|field| field.is_blob_v2()) + && schema.fields_pre_order().any(|field| field.is_blob_v2()) { Some(Arc::new( build_external_base_resolver(dataset, ¶ms).await?, @@ -1061,7 +1061,8 @@ pub async fn write_fragments_internal( (converted_schema, params.storage_version_or_default()) }; - if storage_version < LanceFileVersion::V2_2 && schema.fields.iter().any(|f| f.is_blob_v2()) { + if storage_version < LanceFileVersion::V2_2 && schema.fields_pre_order().any(|f| f.is_blob_v2()) + { return Err(Error::invalid_input(format!( "Blob v2 requires file version >= 2.2 (got {:?})", storage_version @@ -1069,13 +1070,10 @@ pub async fn write_fragments_internal( } if storage_version >= LanceFileVersion::V2_2 - && schema - .fields - .iter() - .any(|f| f.metadata.contains_key(BLOB_META_KEY)) + && let Some(blob_field_path) = legacy_blob_field_path(&schema) { return Err(Error::invalid_input(format!( - "Legacy blob columns (field metadata key {BLOB_META_KEY:?}) are not supported for file version >= 2.2. Use the blob v2 extension type (ARROW:extension:name = \"lance.blob.v2\") and the new blob APIs (e.g. lance::blob::blob_field / lance::blob::BlobArrayBuilder)." + "Legacy blob columns (field metadata key {BLOB_META_KEY:?}) are not supported for file version >= 2.2. Found legacy blob field: {blob_field_path}. Use the blob v2 extension type (ARROW:extension:name = \"lance.blob.v2\") and the new blob APIs (e.g. lance::blob::blob_field / lance::blob::BlobArrayBuilder)." ))); } @@ -1094,6 +1092,17 @@ pub async fn write_fragments_internal( Ok((fragments, schema)) } +fn legacy_blob_field_path(schema: &Schema) -> Option { + schema + .fields_pre_order() + .find(|field| field.metadata.contains_key(BLOB_META_KEY)) + .map(|field| { + schema + .field_path(field.id) + .unwrap_or_else(|_| field.name.clone()) + }) +} + #[async_trait::async_trait] pub trait GenericWriter: Send { /// Write the given batches to the file @@ -1234,7 +1243,7 @@ pub(super) async fn open_update_writer( // flow through WriteParams. Rebuild the external base resolver here so blob // v2 reference columns can resolve dataset-registered external URIs. let external_base_resolver = if storage_version >= LanceFileVersion::V2_2 - && schema.fields.iter().any(|f| f.is_blob_v2()) + && schema.fields_pre_order().any(|f| f.is_blob_v2()) { Some(Arc::new( build_external_base_resolver(Some(dataset), &WriteParams::default()).await?, diff --git a/rust/lance/src/dataset/write/insert.rs b/rust/lance/src/dataset/write/insert.rs index 20209ed7f30..bfd702c9c3b 100644 --- a/rust/lance/src/dataset/write/insert.rs +++ b/rust/lance/src/dataset/write/insert.rs @@ -442,7 +442,7 @@ struct WriteContext<'a> { mod test { use std::collections::HashMap; - use arrow_array::{BinaryArray, Int32Array, RecordBatchReader, StructArray}; + use arrow_array::{ArrayRef, BinaryArray, Int32Array, RecordBatchReader, StructArray}; use arrow_schema::{ArrowError, DataType, Field, Schema}; use lance_arrow::BLOB_META_KEY; @@ -559,6 +559,41 @@ mod test { } } + #[tokio::test] + async fn create_v2_2_dataset_rejects_nested_legacy_blob_schema() { + let image_field = Field::new("image_bytes", DataType::Binary, true).with_metadata( + HashMap::from([(BLOB_META_KEY.to_string(), "true".to_string())]), + ); + let schema = Arc::new(Schema::new(vec![Field::new( + "summary_image_nested", + DataType::Struct(vec![image_field.clone()].into()), + true, + )])); + let image_values: ArrayRef = Arc::new(BinaryArray::from(vec![Some(b"abc".as_slice())])); + let nested_values = StructArray::from(vec![(Arc::new(image_field), image_values)]); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(nested_values)]).unwrap(); + + let dataset = InsertBuilder::new("memory://forced-nested-blob-v2") + .with_params(&WriteParams { + mode: WriteMode::Create, + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }) + .execute_stream(RecordBatchIterator::new(vec![Ok(batch)], schema.clone())) + .await; + + let err = dataset.unwrap_err(); + match err { + Error::InvalidInput { source, .. } => { + let message = source.to_string(); + assert!(message.contains("Legacy blob columns")); + assert!(message.contains("summary_image_nested.image_bytes")); + assert!(message.contains("lance.blob.v2")); + } + other => panic!("unexpected error: {other:?}"), + } + } + mod external_error { use super::*; use std::fmt; diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs index 78e5c429527..69acc69b6da 100644 --- a/rust/lance/src/index.rs +++ b/rust/lance/src/index.rs @@ -57,7 +57,7 @@ use lance_io::utils::{ read_version, }; use lance_table::format::{Fragment, SelfDescribingFileReader}; -use lance_table::format::{IndexMetadata, list_index_files_with_sizes}; +use lance_table::format::{IndexFile, IndexMetadata, list_index_files_with_sizes}; use lance_table::io::manifest::read_manifest_indexes; use roaring::RoaringBitmap; use scalar::index_matches_criteria; @@ -166,7 +166,8 @@ pub(crate) async fn build_index_metadata_from_segments( let mut new_indices = Vec::with_capacity(segments.len()); for segment in segments { let (uuid, fragment_bitmap, index_details, index_version) = segment.into_parts(); - if index_details.type_url.ends_with("InvertedIndexDetails") { + let is_inverted_index = index_details.type_url.ends_with("InvertedIndexDetails"); + if is_inverted_index { let metadata = IndexMetadata { uuid, name: index_name.to_string(), @@ -183,7 +184,10 @@ pub(crate) async fn build_index_metadata_from_segments( .await?; } let index_dir = dataset.indices_dir().clone().join(uuid.to_string()); - let files = list_index_files_with_sizes(&dataset.object_store, &index_dir).await?; + let mut files = list_index_files_with_sizes(&dataset.object_store, &index_dir).await?; + if is_inverted_index { + retain_committed_inverted_files(&mut files); + } new_indices.push(IndexMetadata { uuid, name: index_name.to_string(), @@ -201,6 +205,10 @@ pub(crate) async fn build_index_metadata_from_segments( Ok(new_indices) } +fn retain_committed_inverted_files(files: &mut Vec) { + files.retain(|file| !file.path.starts_with("staging/")); +} + fn validate_segment_index_details(index_name: &str, segments: &[IndexMetadata]) -> Result<()> { let mut type_url = None::<&str>; for segment in segments { diff --git a/rust/lance/src/index/create.rs b/rust/lance/src/index/create.rs index 2b6992e4849..bbb055463dc 100644 --- a/rust/lance/src/index/create.rs +++ b/rust/lance/src/index/create.rs @@ -1993,6 +1993,23 @@ mod tests { let segments = input_segments.clone(); assert_eq!(segments.len(), input_segments.len()); + crate::index::scalar::inverted::finalize_segment_files_if_needed( + &dataset, + &input_segments[0], + ) + .await + .unwrap(); + let stale_staging_path = dataset + .indices_dir() + .join(input_segments[0].uuid.to_string()) + .join("staging") + .join("orphan.lance"); + dataset + .object_store + .put(&stale_staging_path, b"stale") + .await + .unwrap(); + dataset .commit_existing_index_segments("text_idx", "text", segments) .await @@ -2016,6 +2033,19 @@ mod tests { let indices = dataset.load_indices_by_name("text_idx").await.unwrap(); assert_eq!(indices.len(), input_segments.len()); + let finalized_segment = indices + .iter() + .find(|index| index.uuid == input_segments[0].uuid) + .expect("finalized segment should be committed"); + assert!( + finalized_segment + .files + .as_ref() + .expect("committed segment should track files") + .iter() + .all(|file| !file.path.starts_with("staging/")), + "stale staging files must not be committed in IndexMetadata.files" + ); } #[tokio::test] diff --git a/rust/lance/src/io/exec/take.rs b/rust/lance/src/io/exec/take.rs index 977a9c88dce..c3642cdb043 100644 --- a/rust/lance/src/io/exec/take.rs +++ b/rust/lance/src/io/exec/take.rs @@ -4,6 +4,7 @@ use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Mutex}; +use std::task::Poll; use arrow::array::AsArray; use arrow::compute::{TakeOptions, concat_batches}; @@ -27,6 +28,7 @@ use lance_arrow::RecordBatchExt; use lance_core::datatypes::{Field, OnMissing, Projection}; use lance_core::error::{DataFusionResult, LanceOptionExt}; use lance_core::utils::address::RowAddress; +use lance_core::utils::futures::FinallyStreamExt; use lance_core::utils::tokio::get_num_compute_intensive_cpus; use lance_core::{ROW_ADDR, ROW_ID}; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; @@ -353,10 +355,6 @@ impl TakeStream { (None, None) => {} } - self.metrics - .baseline_metrics - .record_output(new_data.num_rows()); - self.metrics.batches_processed.add(1); Ok(batch.merge_with_schema(&new_data, self.output_schema.as_ref())?) } @@ -364,8 +362,10 @@ impl TakeStream { self: Arc, input: S, ) -> impl Stream> { - let scan_scheduler = self.scan_scheduler.clone(); - let metrics = self.metrics.clone(); + let result_scan_scheduler = self.scan_scheduler.clone(); + let final_scan_scheduler = self.scan_scheduler.clone(); + let result_metrics = self.metrics.clone(); + let final_metrics = self.metrics.clone(); let batches = input .enumerate() .map(move |(batch_index, batch)| { @@ -378,8 +378,24 @@ impl TakeStream { }) .boxed(); batches - .inspect_ok(move |_| metrics.io_metrics.record(&scan_scheduler)) .try_buffered(get_num_compute_intensive_cpus()) + .map(move |result| { + if result.is_ok() { + result_metrics.batches_processed.add(1); + } + result_metrics.io_metrics.record(&result_scan_scheduler); + match result_metrics + .baseline_metrics + .record_poll(Poll::Ready(Some(result))) + { + Poll::Ready(Some(result)) => result, + _ => unreachable!("record_poll returned a different poll state"), + } + }) + .finally(move || { + final_metrics.baseline_metrics.done(); + final_metrics.io_metrics.record(&final_scan_scheduler); + }) } } @@ -839,6 +855,80 @@ mod tests { } } + #[tokio::test(flavor = "current_thread")] + async fn test_take_records_output_and_io_metrics() { + use datafusion::physical_plan::metrics::MetricValue; + use lance_datafusion::utils::{BYTES_READ_METRIC, IOPS_METRIC, REQUESTS_METRIC}; + let TestFixture { + dataset, + _tmp_dir_guard, + } = test_fixture().await; + + let row_addrs = UInt64Array::from(vec![0_u64, 1, 2, 3, 4]); + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + ROW_ADDR, + DataType::UInt64, + true, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(row_addrs)]).unwrap(); + let stream = futures::stream::iter(vec![Ok(batch)]); + let stream = Box::pin(RecordBatchStreamAdapter::new(schema, stream)); + let input = Arc::new(OneShotExec::new(stream)); + + let projection = dataset + .empty_projection() + .union_column("s", OnMissing::Error) + .unwrap(); + + let take_exec = TakeExec::try_new(dataset, input, projection) + .unwrap() + .unwrap(); + + let stream = take_exec + .execute(0, Arc::new(TaskContext::default())) + .unwrap(); + let batches: Vec = stream.try_collect().await.unwrap(); + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 5); + + let metrics = take_exec.metrics().unwrap(); + + let output_batches: usize = metrics + .iter() + .filter_map(|m| match m.value() { + MetricValue::OutputBatches(count) => Some(count.value()), + _ => None, + }) + .sum(); + + let output_bytes: usize = metrics + .iter() + .filter_map(|m| match m.value() { + MetricValue::OutputBytes(count) => Some(count.value()), + _ => None, + }) + .sum(); + + let gauge = |name: &str| -> usize { + metrics + .iter_gauges() + .find_map(|(metric_name, gauge)| { + (metric_name.as_ref() == name).then(|| gauge.value()) + }) + .unwrap_or(0) + }; + + let bytes_read = gauge(BYTES_READ_METRIC); + let iops = gauge(IOPS_METRIC); + let requests = gauge(REQUESTS_METRIC); + + assert_eq!(metrics.output_rows(), Some(5)); + assert_eq!(metrics.find_count("batches_processed").unwrap().value(), 1); + assert!( + output_batches > 0 && output_bytes > 0 && bytes_read > 0 && iops > 0 && requests > 0, + "expected positive TakeExec metrics, got output_batches={output_batches}, output_bytes={output_bytes}, bytes_read={bytes_read}, iops={iops}, requests={requests}" + ); + } + #[tokio::test] async fn test_take_order() { let TestFixture {