From c626c1ab13eaffcef6205a8594d15019e38be346 Mon Sep 17 00:00:00 2001 From: Nikolay Skovorodin Date: Tue, 16 Jun 2026 08:22:54 +0700 Subject: [PATCH 01/15] fix: record TakeExec output and I/O metrics (#7228) # Problem Closes #7227. The IoMetrics of the `take` operator are broken. The metrics such as `output_bytes=0.0 B`, `output_batches=0`, `bytes_read=0` are not visible. Example: `... Take: elapsed=350.781983256s, columns="_distance, _rowid, (vec)", metrics=[output_rows=10.00 K, elapsed_compute=302.52s, output_bytes=0.0 B, output_batches=0, batches_processed=1, bytes_read=0, iops=0, requests=0] ...` # Cause `TakeExec` was recording output row metrics inside `map_batch`, before the spawned take work had completed and before the final `RecordBatch` was emitted through DataFusion's metrics path. As a result, `output_batches`, `output_bytes`, and `ScanScheduler` I/O metrics stayed at zero even when take work read data successfully. # Solution This moves metric recording to the post-`try_buffered` result path, records the final `RecordBatch` with `BaselineMetrics::record_poll`, and records `ScanScheduler` I/O metrics after the actual take/read work completes. The stream also finalizes baseline metrics and records one final I/O snapshot when it finishes. Added a regression test covering output rows, output batches, output bytes, bytes read, IOPS, requests, and `batches_processed`. --- rust/lance/src/io/exec/take.rs | 104 ++++++++++++++++++++++++++++++--- 1 file changed, 97 insertions(+), 7 deletions(-) diff --git a/rust/lance/src/io/exec/take.rs b/rust/lance/src/io/exec/take.rs index 977a9c88dce..c3642cdb043 100644 --- a/rust/lance/src/io/exec/take.rs +++ b/rust/lance/src/io/exec/take.rs @@ -4,6 +4,7 @@ use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Mutex}; +use std::task::Poll; use arrow::array::AsArray; use arrow::compute::{TakeOptions, concat_batches}; @@ -27,6 +28,7 @@ use lance_arrow::RecordBatchExt; use lance_core::datatypes::{Field, OnMissing, Projection}; use lance_core::error::{DataFusionResult, LanceOptionExt}; use lance_core::utils::address::RowAddress; +use lance_core::utils::futures::FinallyStreamExt; use lance_core::utils::tokio::get_num_compute_intensive_cpus; use lance_core::{ROW_ADDR, ROW_ID}; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; @@ -353,10 +355,6 @@ impl TakeStream { (None, None) => {} } - self.metrics - .baseline_metrics - .record_output(new_data.num_rows()); - self.metrics.batches_processed.add(1); Ok(batch.merge_with_schema(&new_data, self.output_schema.as_ref())?) } @@ -364,8 +362,10 @@ impl TakeStream { self: Arc, input: S, ) -> impl Stream> { - let scan_scheduler = self.scan_scheduler.clone(); - let metrics = self.metrics.clone(); + let result_scan_scheduler = self.scan_scheduler.clone(); + let final_scan_scheduler = self.scan_scheduler.clone(); + let result_metrics = self.metrics.clone(); + let final_metrics = self.metrics.clone(); let batches = input .enumerate() .map(move |(batch_index, batch)| { @@ -378,8 +378,24 @@ impl TakeStream { }) .boxed(); batches - .inspect_ok(move |_| metrics.io_metrics.record(&scan_scheduler)) .try_buffered(get_num_compute_intensive_cpus()) + .map(move |result| { + if result.is_ok() { + result_metrics.batches_processed.add(1); + } + result_metrics.io_metrics.record(&result_scan_scheduler); + match result_metrics + .baseline_metrics + .record_poll(Poll::Ready(Some(result))) + { + Poll::Ready(Some(result)) => result, + _ => unreachable!("record_poll returned a different poll state"), + } + }) + .finally(move || { + final_metrics.baseline_metrics.done(); + final_metrics.io_metrics.record(&final_scan_scheduler); + }) } } @@ -839,6 +855,80 @@ mod tests { } } + #[tokio::test(flavor = "current_thread")] + async fn test_take_records_output_and_io_metrics() { + use datafusion::physical_plan::metrics::MetricValue; + use lance_datafusion::utils::{BYTES_READ_METRIC, IOPS_METRIC, REQUESTS_METRIC}; + let TestFixture { + dataset, + _tmp_dir_guard, + } = test_fixture().await; + + let row_addrs = UInt64Array::from(vec![0_u64, 1, 2, 3, 4]); + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + ROW_ADDR, + DataType::UInt64, + true, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(row_addrs)]).unwrap(); + let stream = futures::stream::iter(vec![Ok(batch)]); + let stream = Box::pin(RecordBatchStreamAdapter::new(schema, stream)); + let input = Arc::new(OneShotExec::new(stream)); + + let projection = dataset + .empty_projection() + .union_column("s", OnMissing::Error) + .unwrap(); + + let take_exec = TakeExec::try_new(dataset, input, projection) + .unwrap() + .unwrap(); + + let stream = take_exec + .execute(0, Arc::new(TaskContext::default())) + .unwrap(); + let batches: Vec = stream.try_collect().await.unwrap(); + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 5); + + let metrics = take_exec.metrics().unwrap(); + + let output_batches: usize = metrics + .iter() + .filter_map(|m| match m.value() { + MetricValue::OutputBatches(count) => Some(count.value()), + _ => None, + }) + .sum(); + + let output_bytes: usize = metrics + .iter() + .filter_map(|m| match m.value() { + MetricValue::OutputBytes(count) => Some(count.value()), + _ => None, + }) + .sum(); + + let gauge = |name: &str| -> usize { + metrics + .iter_gauges() + .find_map(|(metric_name, gauge)| { + (metric_name.as_ref() == name).then(|| gauge.value()) + }) + .unwrap_or(0) + }; + + let bytes_read = gauge(BYTES_READ_METRIC); + let iops = gauge(IOPS_METRIC); + let requests = gauge(REQUESTS_METRIC); + + assert_eq!(metrics.output_rows(), Some(5)); + assert_eq!(metrics.find_count("batches_processed").unwrap().value(), 1); + assert!( + output_batches > 0 && output_bytes > 0 && bytes_read > 0 && iops > 0 && requests > 0, + "expected positive TakeExec metrics, got output_batches={output_batches}, output_bytes={output_bytes}, bytes_read={bytes_read}, iops={iops}, requests={requests}" + ); + } + #[tokio::test] async fn test_take_order() { let TestFixture { From 8743fbaf9ce0dce15373629a7574e14d5c6c9367 Mon Sep 17 00:00:00 2001 From: Yang Cen Date: Tue, 16 Jun 2026 13:13:34 +0800 Subject: [PATCH 02/15] fix(index): share FTS position stream offsets (#7275) ## Bug Fix ### What is the bug? FTS phrase-query position cache hits clone `SharedPositionStream`. The stream already used shared `Bytes` for the compressed position payload, but `block_offsets` was stored as `Vec`, so each clone copied one offset per compressed position block. ### What issues or incorrect behavior does the bug cause? For large shared-position streams, repeated cache-hit clones can add avoidable O(N) CPU and memory traffic even when the position data itself is cached. ### How does this PR fix the problem? `SharedPositionStream` now stores `block_offsets` as `Arc<[u32]>`, keeping the existing `block_offsets() -> &[u32]` API while making clones share the offset allocation. A regression test verifies that cloned streams point at the same offsets allocation. ## Validation - `cargo test -p lance-index scalar::inverted::index::tests::test_shared_position_stream_clone_shares_block_offsets` - `cargo test -p lance-index shared_position` - `cargo test -p lance-index shared_stream` - `cargo test -p lance-index scalar::inverted::cache_codec::tests` - `cargo test -p lance-index` - `cargo fmt --all` - `cargo clippy --all --tests --benches -- -D warnings` - `git diff --check` --- rust/lance-index/src/scalar/inverted/index.rs | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs index 56547c6510b..92379a9e350 100644 --- a/rust/lance-index/src/scalar/inverted/index.rs +++ b/rust/lance-index/src/scalar/inverted/index.rs @@ -2947,10 +2947,9 @@ impl DeepSizeOf for CompressedPositionStorage { #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct SharedPositionStream { codec: PositionStreamCodec, - block_offsets: Vec, - // Stored as `Bytes` so that the cache deserialization path can hand - // ownership of an IPC-decoded slice in without copying. Cloning the - // stream is then an `Arc` bump rather than an O(N) buffer copy. + block_offsets: Arc<[u32]>, + // Stored with shared ownership so cache hits can clone position streams + // without copying either offsets or bytes. bytes: bytes::Bytes, } @@ -2958,7 +2957,7 @@ impl SharedPositionStream { pub fn new(codec: PositionStreamCodec, block_offsets: Vec, bytes: bytes::Bytes) -> Self { Self { codec, - block_offsets, + block_offsets: Arc::from(block_offsets.into_boxed_slice()), bytes, } } @@ -2991,11 +2990,11 @@ impl SharedPositionStream { } pub fn block_offsets(&self) -> &[u32] { - &self.block_offsets + self.block_offsets.as_ref() } pub fn size(&self) -> usize { - self.block_offsets.capacity() * std::mem::size_of::() + self.bytes.len() + self.block_offsets.len() * std::mem::size_of::() + self.bytes.len() } } @@ -5475,6 +5474,21 @@ mod tests { ); } + #[test] + fn test_shared_position_stream_clone_shares_block_offsets() { + let stream = SharedPositionStream::new( + PositionStreamCodec::PackedDelta, + vec![0_u32, 4, 11], + bytes::Bytes::from_static(b"shared position bytes"), + ); + let original_offsets = stream.block_offsets().as_ptr(); + + let cloned = stream.clone(); + + assert_eq!(cloned.block_offsets(), stream.block_offsets()); + assert_eq!(cloned.block_offsets().as_ptr(), original_offsets); + } + #[test] fn test_posting_builder_roundtrip_shared_positions() { let entries = vec![ From 3c2e36e1b22cf8059059190140a6e2adc7b979ce Mon Sep 17 00:00:00 2001 From: Dan Rammer Date: Tue, 16 Jun 2026 03:01:18 -0500 Subject: [PATCH 03/15] fix(mem_wal): push LIMIT/OFFSET down to fresh-tier scan sources (#7256) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary The LSM scan planner (`LsmScanPlanner`) applied a query's `LIMIT`/`OFFSET` *above* the per-source union, so every flushed generation and the base table were fully scanned before the limit was imposed at the top. For a bounded read over a fresh tier with many flushed generations this materialized the entire tier on every query. This threads an `overfetch_factor` through `LsmScanner` → `LsmScanPlanner` and pushes a per-source fetch of `(offset + limit) * overfetch` into each `Base`/`Flushed` source scan. The active memtable is exempt (its within-source dedup needs the full match set). Block-listed sources over-fetch by `overfetch_factor` so cross-generation PK dedup still leaves ~`limit` live rows; a `LocalLimit` re-imposes the exact `skip`/`fetch` cap above the merge. Unbounded reads (no limit) are unchanged. ## Changes - `scanner/planner.rs`: `overfetch_factor` field + `with_overfetch_factor` on `LsmScanPlanner`; per-source limit pushdown in `plan_scan`; `fetch` parameter on `build_source_scan` (pushed into Base/Flushed scans, ActiveMemTable exempt). - `scanner/builder.rs`: `overfetch_factor` on `LsmScanner` + `with_overfetch_factor`, threaded into both the scan and FTS planners. ## Validation Validated end-to-end against a WAL benchmark on minikube with object storage behind a latency proxy: a bounded `offset+limit` read over an 18-generation fresh tier dropped from a full-tier scan (~360 generation scans) to a bounded per-source read. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Fable 5 --- .../src/dataset/mem_wal/scanner/builder.rs | 19 +++++ .../src/dataset/mem_wal/scanner/planner.rs | 70 +++++++++++++++++-- 2 files changed, 84 insertions(+), 5 deletions(-) diff --git a/rust/lance/src/dataset/mem_wal/scanner/builder.rs b/rust/lance/src/dataset/mem_wal/scanner/builder.rs index ade4164d485..508605c4642 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/builder.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/builder.rs @@ -125,6 +125,9 @@ pub struct LsmScanner { /// Cache of opened flushed-generation datasets. When set, repeated /// queries against the same generation skip the manifest read entirely. flushed_cache: Option>, + /// Over-fetch multiple for block-listed sources in search plans + /// (see [`super::LsmFtsSearchPlanner::with_overfetch_factor`]). + overfetch_factor: Option, } impl LsmScanner { @@ -160,6 +163,7 @@ impl LsmScanner { pk_columns, session, flushed_cache: None, + overfetch_factor: None, } } @@ -198,6 +202,7 @@ impl LsmScanner { pk_columns, session: None, flushed_cache: None, + overfetch_factor: None, } } @@ -253,6 +258,14 @@ impl LsmScanner { self } + /// Set the over-fetch multiple block-listed sources use in search plans + /// so they still yield `k` live rows after cross-generation dedup. + /// Threaded into [`super::LsmFtsSearchPlanner`]; clamped to `>= 1.0`. + pub fn with_overfetch_factor(mut self, factor: f64) -> Self { + self.overfetch_factor = Some(factor); + self + } + /// Project specific columns. /// /// If not called, all columns from the base schema are included. @@ -370,6 +383,9 @@ impl LsmScanner { if let Some(cache) = &self.flushed_cache { planner = planner.with_flushed_cache(cache.clone()); } + if let Some(factor) = self.overfetch_factor { + planner = planner.with_overfetch_factor(factor); + } planner .plan_scan( @@ -405,6 +421,9 @@ impl LsmScanner { if let Some(cache) = &self.flushed_cache { planner = planner.with_flushed_cache(cache.clone()); } + if let Some(factor) = self.overfetch_factor { + planner = planner.with_overfetch_factor(factor); + } planner .plan_search(column, query, k, self.projection.as_deref()) .await diff --git a/rust/lance/src/dataset/mem_wal/scanner/planner.rs b/rust/lance/src/dataset/mem_wal/scanner/planner.rs index f3f15e2e680..eca0255be1c 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/planner.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/planner.rs @@ -34,6 +34,10 @@ pub struct LsmScanPlanner { session: Option>, /// Cache of opened flushed-generation datasets. flushed_cache: Option>, + /// Over-fetch multiple for the per-source limit pushdown: block-listed + /// sources scan `(offset + limit) * factor` rows so cross-gen dedup drops + /// still leave enough live rows. Clamped to `>= 1.0`. + overfetch_factor: f64, } impl LsmScanPlanner { @@ -49,6 +53,7 @@ impl LsmScanPlanner { base_schema, session: None, flushed_cache: None, + overfetch_factor: 1.0, } } @@ -66,6 +71,13 @@ impl LsmScanPlanner { self } + /// Set the over-fetch multiple for the per-source limit pushdown + /// (see the field docs). Clamped to `>= 1.0` at use. + pub fn with_overfetch_factor(mut self, factor: f64) -> Self { + self.overfetch_factor = factor; + self + } + /// Create scan plan with deduplication. /// /// # Arguments @@ -130,23 +142,59 @@ impl LsmScanPlanner { // cross-gen block-list, not from output ordering. let sources: Vec<_> = sources.into_iter().rev().collect(); + // Per-source limit pushdown: an unordered LIMIT needs only + // `offset + limit` live rows from EACH source to fill the global + // limit after dedup (any-N semantics), so cap every on-disk source + // instead of scanning whole generations and trimming above the + // union. Block-listed sources over-fetch by `overfetch_factor` so + // cross-gen dedup drops still leave `n_needed` live rows; the + // PkHashFilter warns when that was not enough. The active memtable + // is in-memory and within-gen append duplicates are resolved by its + // own dedup, so it is never capped here. + let n_needed = limit.map(|l| l.saturating_add(offset.unwrap_or(0))); + let overfetch = self.overfetch_factor.max(1.0); + let mut source_plans = Vec::new(); for source in sources { let is_base = matches!(source, LsmDataSource::BaseTable { .. }); - let scan = self.build_source_scan(&source, projection, filter).await?; + let is_active = matches!(source, LsmDataSource::ActiveMemTable { .. }); + let blocked = block_lists + .get(&(source.shard_id(), source.generation())) + .cloned(); + let fetch = match (n_needed, is_active) { + (Some(n), false) => Some(if blocked.is_some() { + ((n as f64) * overfetch).ceil() as usize + } else { + n + }), + _ => None, + }; + let scan = self + .build_source_scan(&source, projection, filter, fetch) + .await?; // Drop cross-generation stale rows (PKs superseded by a newer gen). - // `k = 0`: there is no top-k, so the under-fetch warning never fires. - let scan = match block_lists.get(&(source.shard_id(), source.generation())) { + // With a limit, `k = n_needed` arms the under-fetch warning; with + // no limit `k = 0` keeps it silent. + let scan = match blocked { Some(set) => Arc::new(PkHashFilterExec::new( scan, self.pk_columns.clone(), - set.clone(), - 0, + set, + n_needed.unwrap_or(0), )) as Arc, None => scan, }; + // Post-block-list cap: each source contributes at most `n_needed` + // live rows toward the global limit. + let scan: Arc = match n_needed { + Some(n) if !is_active => Arc::new( + datafusion::physical_plan::limit::LocalLimitExec::new(scan, n), + ), + _ => scan, + }; + // When `_rowaddr` is surfaced, NULL it for non-base arms: only base // values are meaningful (e.g. for `take_rows`); per-source addresses // collide with base IDs. @@ -229,6 +277,7 @@ impl LsmScanPlanner { source: &LsmDataSource, projection: Option<&[String]>, filter: Option<&Expr>, + fetch: Option, ) -> Result> { match source { LsmDataSource::BaseTable { dataset } => { @@ -247,6 +296,11 @@ impl LsmScanPlanner { if let Some(expr) = filter { scanner.filter_expr(expr.clone()); } + // Per-source limit pushdown (post-filter rows): bounds the + // physical scan instead of trimming above the union. + if let Some(fetch) = fetch { + scanner.limit(Some(fetch as i64), None)?; + } scanner.create_plan().await } @@ -264,6 +318,12 @@ impl LsmScanPlanner { if let Some(expr) = filter { scanner.filter_expr(expr.clone()); } + // Per-source limit pushdown: flushed generations are + // within-gen live (dedup-on-flush deletion vectors), so any + // `fetch` post-filter rows are valid contributions. + if let Some(fetch) = fetch { + scanner.limit(Some(fetch as i64), None)?; + } scanner.create_plan().await } From a558da54c4a4ce6b9ffe68b7cc4f68472775b702 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 16 Jun 2026 16:31:15 +0800 Subject: [PATCH 04/15] fix: reject nested legacy blobs in v2.2 (#7278) This closes a v2.2 write-path hole where legacy blob metadata was rejected only on top-level fields. A nested field such as `summary_image_nested.image_bytes` could still carry `lance-encoding:blob` into a v2.2 dataset manifest. The write guard now scans the full Lance schema preorder and reports the offending field path. This keeps v2.2 dataset creation from silently preserving BlobV1 columns while leaving valid `lance.blob.v2` inputs untouched. --- rust/lance/src/dataset/write.rs | 18 +++++++++---- rust/lance/src/dataset/write/insert.rs | 37 +++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/rust/lance/src/dataset/write.rs b/rust/lance/src/dataset/write.rs index 37a3d584271..d6ac3ce9d45 100644 --- a/rust/lance/src/dataset/write.rs +++ b/rust/lance/src/dataset/write.rs @@ -1069,13 +1069,10 @@ pub async fn write_fragments_internal( } if storage_version >= LanceFileVersion::V2_2 - && schema - .fields - .iter() - .any(|f| f.metadata.contains_key(BLOB_META_KEY)) + && let Some(blob_field_path) = legacy_blob_field_path(&schema) { return Err(Error::invalid_input(format!( - "Legacy blob columns (field metadata key {BLOB_META_KEY:?}) are not supported for file version >= 2.2. Use the blob v2 extension type (ARROW:extension:name = \"lance.blob.v2\") and the new blob APIs (e.g. lance::blob::blob_field / lance::blob::BlobArrayBuilder)." + "Legacy blob columns (field metadata key {BLOB_META_KEY:?}) are not supported for file version >= 2.2. Found legacy blob field: {blob_field_path}. Use the blob v2 extension type (ARROW:extension:name = \"lance.blob.v2\") and the new blob APIs (e.g. lance::blob::blob_field / lance::blob::BlobArrayBuilder)." ))); } @@ -1094,6 +1091,17 @@ pub async fn write_fragments_internal( Ok((fragments, schema)) } +fn legacy_blob_field_path(schema: &Schema) -> Option { + schema + .fields_pre_order() + .find(|field| field.metadata.contains_key(BLOB_META_KEY)) + .map(|field| { + schema + .field_path(field.id) + .unwrap_or_else(|_| field.name.clone()) + }) +} + #[async_trait::async_trait] pub trait GenericWriter: Send { /// Write the given batches to the file diff --git a/rust/lance/src/dataset/write/insert.rs b/rust/lance/src/dataset/write/insert.rs index 20209ed7f30..bfd702c9c3b 100644 --- a/rust/lance/src/dataset/write/insert.rs +++ b/rust/lance/src/dataset/write/insert.rs @@ -442,7 +442,7 @@ struct WriteContext<'a> { mod test { use std::collections::HashMap; - use arrow_array::{BinaryArray, Int32Array, RecordBatchReader, StructArray}; + use arrow_array::{ArrayRef, BinaryArray, Int32Array, RecordBatchReader, StructArray}; use arrow_schema::{ArrowError, DataType, Field, Schema}; use lance_arrow::BLOB_META_KEY; @@ -559,6 +559,41 @@ mod test { } } + #[tokio::test] + async fn create_v2_2_dataset_rejects_nested_legacy_blob_schema() { + let image_field = Field::new("image_bytes", DataType::Binary, true).with_metadata( + HashMap::from([(BLOB_META_KEY.to_string(), "true".to_string())]), + ); + let schema = Arc::new(Schema::new(vec![Field::new( + "summary_image_nested", + DataType::Struct(vec![image_field.clone()].into()), + true, + )])); + let image_values: ArrayRef = Arc::new(BinaryArray::from(vec![Some(b"abc".as_slice())])); + let nested_values = StructArray::from(vec![(Arc::new(image_field), image_values)]); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(nested_values)]).unwrap(); + + let dataset = InsertBuilder::new("memory://forced-nested-blob-v2") + .with_params(&WriteParams { + mode: WriteMode::Create, + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }) + .execute_stream(RecordBatchIterator::new(vec![Ok(batch)], schema.clone())) + .await; + + let err = dataset.unwrap_err(); + match err { + Error::InvalidInput { source, .. } => { + let message = source.to_string(); + assert!(message.contains("Legacy blob columns")); + assert!(message.contains("summary_image_nested.image_bytes")); + assert!(message.contains("lance.blob.v2")); + } + other => panic!("unexpected error: {other:?}"), + } + } + mod external_error { use super::*; use std::fmt; From 440acf0685dc2f92a8298ae591d1a4b7c1312e30 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 16 Jun 2026 17:50:49 +0800 Subject: [PATCH 05/15] fix: support blob v2 in nested structs (#7281) Blob v2 write support only handled top-level fields, so blob v2 columns nested under structs could reach the writer with the logical layout while the file footer expected descriptor fields. This PR makes blob v2 preprocessing and footer descriptor unloading walk nested structs, while preserving the existing per-field blob threshold handling. It also applies the same storage-version and external-reference checks to nested blob v2 fields that top-level blob v2 fields already use. Validated locally with the required Rust and Python checks plus targeted nested blob v2 read/write coverage. --- python/python/tests/test_blob.py | 32 ++ rust/lance-core/src/datatypes/field.rs | 60 ++ rust/lance-file/src/writer.rs | 13 +- rust/lance/src/dataset/blob.rs | 746 +++++++++++++++++-------- rust/lance/src/dataset/write.rs | 7 +- 5 files changed, 602 insertions(+), 256 deletions(-) diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index cb34ae3cafc..fc879c9cbaa 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -1460,6 +1460,38 @@ def test_read_blobs_resolves_nested_field_path(dataset_with_nested_blobs): assert [data for _, data in results] == [b"foo", b"baz"] +def test_write_nested_blob_v2_and_take_by_field_path(tmp_path): + packed = b"x" * (70 * 1024) + blob_field = lance.blob_field("blob") + info_fields = [pa.field("name", pa.string()), blob_field] + info_type = pa.struct(info_fields) + info_array = pa.StructArray.from_arrays( + [pa.array(["a", "b", "c"]), lance.blob_array([b"foo", packed, None])], + fields=info_fields, + ) + table = pa.table( + [info_array], + schema=pa.schema([pa.field("info", info_type)]), + ) + + dataset = lance.write_dataset( + table, + tmp_path / "nested_blob_v2", + data_storage_version="2.2", + ) + + desc = dataset.to_table(columns=["info.blob"]).column("info.blob").chunk(0) + assert desc.field("kind").to_pylist()[:2] == [0, 1] + + blobs = dataset.take_blobs("info.blob", indices=[0, 1]) + with blobs[0] as f: + assert f.read() == b"foo" + with blobs[1] as f: + assert f.read() == packed + + assert dataset.take_blobs("info.blob", indices=[2]) == [] + + def test_to_pandas_returns_blob_files_for_projected_nested_fields( dataset_with_nested_blobs, ): diff --git a/rust/lance-core/src/datatypes/field.rs b/rust/lance-core/src/datatypes/field.rs index 4c2665a3640..9f06d421949 100644 --- a/rust/lance-core/src/datatypes/field.rs +++ b/rust/lance-core/src/datatypes/field.rs @@ -575,6 +575,18 @@ impl Field { } } + /// Convert blob v2 fields in this field tree to their descriptor view. + pub fn unload_blobs_recursive(&mut self) { + if self.is_blob_v2() { + self.unloaded_mut(); + return; + } + + for child in &mut self.children { + child.unload_blobs_recursive(); + } + } + pub fn project(&self, path_components: &[&str]) -> Result { let mut f = Self { name: self.name.clone(), @@ -1864,6 +1876,54 @@ mod tests { assert_eq!(field.logical_type, BLOB_V2_DESC_LANCE_FIELD.logical_type); } + #[test] + fn unload_blobs_recursive_only_unloads_blob_v2() { + let legacy_metadata = HashMap::from([(BLOB_META_KEY.to_string(), "true".to_string())]); + let blob_v2_metadata = + HashMap::from([(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string())]); + + let mut field: Field = ArrowField::new( + "parent", + DataType::Struct(Fields::from(vec![ + ArrowField::new("legacy_blob", DataType::LargeBinary, true) + .with_metadata(legacy_metadata), + ArrowField::new( + "blob_v2", + DataType::Struct( + vec![ + ArrowField::new("data", DataType::LargeBinary, true), + ArrowField::new("uri", DataType::Utf8, true), + ] + .into(), + ), + true, + ) + .with_metadata(blob_v2_metadata), + ])), + true, + ) + .try_into() + .unwrap(); + + field.unload_blobs_recursive(); + + let legacy_blob = field + .children + .iter() + .find(|f| f.name == "legacy_blob") + .unwrap(); + assert_eq!( + legacy_blob.logical_type, + LogicalType::try_from(&DataType::LargeBinary).unwrap() + ); + assert_eq!(legacy_blob.children.len(), 0); + assert!(legacy_blob.metadata.contains_key(BLOB_META_KEY)); + + let blob_v2 = field.children.iter().find(|f| f.name == "blob_v2").unwrap(); + assert_eq!(blob_v2.logical_type, BLOB_V2_DESC_LANCE_FIELD.logical_type); + assert_eq!(blob_v2.children.len(), 5); + } + #[test] fn project_by_field_accepts_blob_descriptor_projection() { let metadata = HashMap::from([(BLOB_META_KEY.to_string(), "true".to_string())]); diff --git a/rust/lance-file/src/writer.rs b/rust/lance-file/src/writer.rs index 14a4c82bde6..12bd50df6fe 100644 --- a/rust/lance-file/src/writer.rs +++ b/rust/lance-file/src/writer.rs @@ -633,14 +633,11 @@ impl FileWriter { async fn write_global_buffers(&mut self) -> Result> { let schema = self.schema.as_mut().ok_or(Error::invalid_input("No schema provided on writer open and no data provided. Schema is unknown and file cannot be created"))?; schema.metadata = std::mem::take(&mut self.schema_metadata); - // Use descriptor layout for blob v2 in the footer to avoid exposing logical child fields. - // - // TODO(xuanwo): this doesn't work on nested struct, need better solution like fields_per_order_mut? - schema.fields.iter_mut().for_each(|f| { - if f.is_blob_v2() { - f.unloaded_mut(); - } - }); + // Use descriptor layout for blob v2 fields in the footer to avoid exposing logical child fields. + schema + .fields + .iter_mut() + .for_each(|f| f.unload_blobs_recursive()); let file_descriptor = Self::make_file_descriptor(schema, self.rows_written)?; let file_descriptor_bytes = file_descriptor.encode_to_vec(); diff --git a/rust/lance/src/dataset/blob.rs b/rust/lance/src/dataset/blob.rs index 56bcc97cb32..8cdde543e4e 100644 --- a/rust/lance/src/dataset/blob.rs +++ b/rust/lance/src/dataset/blob.rs @@ -12,15 +12,17 @@ use std::{ use arrow::array::AsArray; use arrow::datatypes::{UInt8Type, UInt32Type, UInt64Type}; -use arrow_array::Array; use arrow_array::RecordBatch; use arrow_array::builder::{LargeBinaryBuilder, PrimitiveBuilder, StringBuilder}; -use arrow_schema::DataType as ArrowDataType; +use arrow_array::{Array, ArrayRef}; +use arrow_schema::{DataType as ArrowDataType, Field as ArrowField}; use bytes::Bytes; +use futures::future::BoxFuture; use futures::stream::BoxStream; use futures::{FutureExt, StreamExt, TryStreamExt, stream}; use lance_arrow::{ BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, BLOB_INLINE_SIZE_THRESHOLD_META_KEY, FieldExt, + r#struct::StructArrayExt, }; use lance_io::object_store::{ObjectStore, ObjectStoreParams, ObjectStoreRegistry}; use lance_io::scheduler::{FileScheduler, ScanScheduler, SchedulerConfig}; @@ -259,10 +261,7 @@ pub struct BlobPreprocessor { data_file_key: String, local_counter: u32, pack_writer: PackWriter, - blob_v2_cols: Vec, - inline_thresholds: Vec, - dedicated_thresholds: Vec, - writer_metadata: Vec>, + field_processors: Vec, external_base_resolver: Option>, allow_external_blob_outside_bases: bool, external_blob_mode: ExternalBlobMode, @@ -287,6 +286,64 @@ enum BlobWriteSource<'a> { External(&'a ExternalBlobSource), } +#[derive(Clone, Debug)] +struct BlobPreprocessField { + kind: BlobPreprocessFieldKind, +} + +#[derive(Clone, Debug)] +enum BlobPreprocessFieldKind { + BlobV2 { + inline_threshold: usize, + dedicated_threshold: usize, + writer_metadata: HashMap, + }, + Struct { + children: Vec, + }, + Passthrough, +} + +impl BlobPreprocessField { + fn new(field: &ArrowField) -> Result { + if field.is_blob_v2() { + return Ok(Self { + kind: BlobPreprocessFieldKind::BlobV2 { + inline_threshold: blob_inline_threshold_from_metadata( + field.metadata(), + field.name(), + )?, + dedicated_threshold: blob_dedicated_threshold_from_metadata( + field.metadata(), + field.name(), + )?, + writer_metadata: field.metadata().clone(), + }, + }); + } + + if let ArrowDataType::Struct(children) = field.data_type() { + let children = children + .iter() + .map(|child| Self::new(child.as_ref())) + .collect::>>()?; + if children.iter().any(|child| child.requires_preprocessing()) { + return Ok(Self { + kind: BlobPreprocessFieldKind::Struct { children }, + }); + } + } + + Ok(Self { + kind: BlobPreprocessFieldKind::Passthrough, + }) + } + + fn requires_preprocessing(&self) -> bool { + !matches!(self.kind, BlobPreprocessFieldKind::Passthrough) + } +} + impl ExternalBlobSource { /// Return the logical payload size after applying any external slice. fn size(&self) -> u64 { @@ -378,37 +435,11 @@ impl BlobPreprocessor { pack_writer.max_pack_size = max_bytes; } let arrow_schema = arrow_schema::Schema::from(schema); - let fields = arrow_schema.fields(); - let blob_v2_cols = fields + let field_processors = arrow_schema + .fields() .iter() - .map(|field| field.is_blob_v2()) - .collect::>(); - let inline_thresholds = fields - .iter() - .zip(blob_v2_cols.iter()) - .map(|(field, is_blob_v2)| { - if *is_blob_v2 { - blob_inline_threshold_from_metadata(field.metadata(), field.name()) - } else { - Ok(INLINE_MAX) - } - }) + .map(|field| BlobPreprocessField::new(field.as_ref())) .collect::>>()?; - let dedicated_thresholds = fields - .iter() - .zip(blob_v2_cols.iter()) - .map(|(field, is_blob_v2)| { - if *is_blob_v2 { - blob_dedicated_threshold_from_metadata(field.metadata(), field.name()) - } else { - Ok(DEDICATED_THRESHOLD) - } - }) - .collect::>>()?; - let writer_metadata = fields - .iter() - .map(|field| field.metadata().clone()) - .collect(); Ok(Self { object_store, data_dir, @@ -416,10 +447,7 @@ impl BlobPreprocessor { // Start at 1 to avoid a potential all-zero blob_id value. local_counter: 1, pack_writer, - blob_v2_cols, - inline_thresholds, - dedicated_thresholds, - writer_metadata, + field_processors, external_base_resolver, allow_external_blob_outside_bases, external_blob_mode, @@ -520,7 +548,7 @@ impl BlobPreprocessor { } pub(crate) async fn preprocess_batch(&mut self, batch: &RecordBatch) -> Result { - let expected_columns = self.blob_v2_cols.len(); + let expected_columns = self.field_processors.len(); if batch.num_columns() != expected_columns { return Err(Error::invalid_input(format!( "Unexpected number of columns: expected {}, got {}", @@ -531,246 +559,340 @@ impl BlobPreprocessor { let batch_schema = batch.schema(); let batch_fields = batch_schema.fields(); + let field_processors = self.field_processors.clone(); let mut new_columns = Vec::with_capacity(batch.num_columns()); let mut new_fields = Vec::with_capacity(batch.num_columns()); - for idx in 0..batch.num_columns() { - let array = batch.column(idx); - let field = &batch_fields[idx]; - if !self.blob_v2_cols[idx] { - new_columns.push(array.clone()); - new_fields.push(field.clone()); + for ((processor, array), field) in field_processors + .iter() + .zip(batch.columns().iter()) + .zip(batch_fields.iter()) + { + let (new_column, new_field) = self + .preprocess_field(processor, array.clone(), field) + .await?; + new_columns.push(new_column); + new_fields.push(new_field); + } + + let new_schema = Arc::new(arrow_schema::Schema::new_with_metadata( + new_fields + .iter() + .map(|f| f.as_ref().clone()) + .collect::>(), + batch_schema.metadata().clone(), + )); + + RecordBatch::try_new(new_schema, new_columns) + .map_err(|e| Error::invalid_input(e.to_string())) + } + + fn preprocess_field<'a>( + &'a mut self, + processor: &'a BlobPreprocessField, + array: ArrayRef, + field: &'a Arc, + ) -> BoxFuture<'a, Result<(ArrayRef, Arc)>> { + async move { + match &processor.kind { + BlobPreprocessFieldKind::Passthrough => Ok((array, field.clone())), + BlobPreprocessFieldKind::BlobV2 { + inline_threshold, + dedicated_threshold, + writer_metadata, + } => { + self.preprocess_blob_array( + array, + field.as_ref(), + *inline_threshold, + *dedicated_threshold, + writer_metadata, + ) + .await + } + BlobPreprocessFieldKind::Struct { children } => { + self.preprocess_struct_array(array, field.as_ref(), children) + .await + } + } + } + .boxed() + } + + async fn preprocess_struct_array( + &mut self, + array: ArrayRef, + field: &ArrowField, + children: &[BlobPreprocessField], + ) -> Result<(ArrayRef, Arc)> { + let struct_arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::invalid_input("Struct field was not a struct array"))?; + if struct_arr.num_columns() != children.len() { + return Err(Error::invalid_input(format!( + "Struct field '{}' expected {} children, got {}", + field.name(), + children.len(), + struct_arr.num_columns() + ))); + } + + let struct_arr = struct_arr.normalize_slicing()?; + let parent_nulls = struct_arr.nulls().cloned(); + let pushed_down = struct_arr.pushdown_nulls()?; + let child_fields = pushed_down.fields().clone(); + let child_columns = pushed_down.columns().to_vec(); + + let mut new_columns = Vec::with_capacity(children.len()); + let mut new_fields = Vec::with_capacity(children.len()); + for ((child_processor, child_array), child_field) in children + .iter() + .zip(child_columns.into_iter()) + .zip(child_fields.iter()) + { + let (new_column, new_field) = self + .preprocess_field(child_processor, child_array, child_field) + .await?; + new_columns.push(new_column); + new_fields.push(new_field); + } + + let struct_array = + StructArray::try_new(new_fields.clone().into(), new_columns, parent_nulls)?; + let field = Arc::new( + ArrowField::new( + field.name(), + ArrowDataType::Struct(new_fields.into()), + field.is_nullable(), + ) + .with_metadata(field.metadata().clone()), + ); + Ok((Arc::new(struct_array), field)) + } + + async fn preprocess_blob_array( + &mut self, + array: ArrayRef, + field: &ArrowField, + inline_threshold: usize, + dedicated_threshold: usize, + writer_metadata: &HashMap, + ) -> Result<(ArrayRef, Arc)> { + let struct_arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::invalid_input("Blob column was not a struct array"))?; + + let data_col = struct_arr + .column_by_name("data") + .ok_or_else(|| Error::invalid_input("Blob struct missing `data` field"))? + .as_binary::(); + let uri_col = struct_arr + .column_by_name("uri") + .ok_or_else(|| Error::invalid_input("Blob struct missing `uri` field"))? + .as_string::(); + let position_col = struct_arr + .column_by_name("position") + .map(|col| col.as_primitive::()); + let size_col = struct_arr + .column_by_name("size") + .map(|col| col.as_primitive::()); + + let mut data_builder = LargeBinaryBuilder::with_capacity(struct_arr.len(), 0); + let mut uri_builder = StringBuilder::with_capacity(struct_arr.len(), 0); + let mut blob_id_builder = + PrimitiveBuilder::::with_capacity(struct_arr.len()); + let mut blob_size_builder = + PrimitiveBuilder::::with_capacity(struct_arr.len()); + let mut kind_builder = PrimitiveBuilder::::with_capacity(struct_arr.len()); + let mut position_builder = + PrimitiveBuilder::::with_capacity(struct_arr.len()); + + let struct_nulls = struct_arr.nulls(); + + for i in 0..struct_arr.len() { + if struct_arr.is_null(i) { + data_builder.append_null(); + uri_builder.append_null(); + blob_id_builder.append_null(); + blob_size_builder.append_null(); + kind_builder.append_null(); + position_builder.append_null(); continue; } - let struct_arr = array - .as_any() - .downcast_ref::() - .ok_or_else(|| Error::invalid_input("Blob column was not a struct array"))?; - - let data_col = struct_arr - .column_by_name("data") - .ok_or_else(|| Error::invalid_input("Blob struct missing `data` field"))? - .as_binary::(); - let uri_col = struct_arr - .column_by_name("uri") - .ok_or_else(|| Error::invalid_input("Blob struct missing `uri` field"))? - .as_string::(); - let position_col = struct_arr - .column_by_name("position") - .map(|col| col.as_primitive::()); - let size_col = struct_arr - .column_by_name("size") - .map(|col| col.as_primitive::()); - - let mut data_builder = LargeBinaryBuilder::with_capacity(struct_arr.len(), 0); - let mut uri_builder = StringBuilder::with_capacity(struct_arr.len(), 0); - let mut blob_id_builder = - PrimitiveBuilder::::with_capacity(struct_arr.len()); - let mut blob_size_builder = - PrimitiveBuilder::::with_capacity(struct_arr.len()); - let mut kind_builder = PrimitiveBuilder::::with_capacity(struct_arr.len()); - let mut position_builder = - PrimitiveBuilder::::with_capacity(struct_arr.len()); - - let struct_nulls = struct_arr.nulls(); - - for i in 0..struct_arr.len() { - if struct_arr.is_null(i) { - data_builder.append_null(); - uri_builder.append_null(); - blob_id_builder.append_null(); - blob_size_builder.append_null(); - kind_builder.append_null(); - position_builder.append_null(); - continue; - } + let has_data = !data_col.is_null(i); + let has_uri = !uri_col.is_null(i); + let has_position = position_col + .as_ref() + .map(|col| !col.is_null(i)) + .unwrap_or(false); + let has_size = size_col + .as_ref() + .map(|col| !col.is_null(i)) + .unwrap_or(false); + let data_len = if has_data { data_col.value(i).len() } else { 0 }; - let has_data = !data_col.is_null(i); - let has_uri = !uri_col.is_null(i); - let has_position = position_col - .as_ref() - .map(|col| !col.is_null(i)) - .unwrap_or(false); - let has_size = size_col - .as_ref() - .map(|col| !col.is_null(i)) - .unwrap_or(false); - let data_len = if has_data { data_col.value(i).len() } else { 0 }; - - let dedicated_threshold = self.dedicated_thresholds[idx]; - let inline_threshold = self.inline_thresholds[idx]; - if has_data && data_len > dedicated_threshold { - let blob_id = self.next_blob_id(); - self.write_dedicated(blob_id, BlobWriteSource::Bytes(data_col.value(i))) - .await?; - - kind_builder.append_value(BlobKind::Dedicated as u8); - data_builder.append_null(); - uri_builder.append_null(); - blob_id_builder.append_value(blob_id); - blob_size_builder.append_value(data_len as u64); - position_builder.append_null(); - continue; - } + if has_data && data_len > dedicated_threshold { + let blob_id = self.next_blob_id(); + self.write_dedicated(blob_id, BlobWriteSource::Bytes(data_col.value(i))) + .await?; - if has_data && data_len > inline_threshold { - let (pack_blob_id, position) = self - .write_packed(BlobWriteSource::Bytes(data_col.value(i))) - .await?; + kind_builder.append_value(BlobKind::Dedicated as u8); + data_builder.append_null(); + uri_builder.append_null(); + blob_id_builder.append_value(blob_id); + blob_size_builder.append_value(data_len as u64); + position_builder.append_null(); + continue; + } - kind_builder.append_value(BlobKind::Packed as u8); - data_builder.append_null(); - uri_builder.append_null(); - blob_id_builder.append_value(pack_blob_id); - blob_size_builder.append_value(data_len as u64); - position_builder.append_value(position); - continue; - } + if has_data && data_len > inline_threshold { + let (pack_blob_id, position) = self + .write_packed(BlobWriteSource::Bytes(data_col.value(i))) + .await?; - if has_uri { - let uri_val = uri_col.value(i); - if self.external_blob_mode == ExternalBlobMode::Ingest { - let position = if has_position { - Some( - position_col - .as_ref() - .expect("position column must exist") - .value(i), - ) - } else { - None - }; - let size = if has_size { - Some(size_col.as_ref().expect("size column must exist").value(i)) - } else { - None - }; - let source = self.open_external_source(uri_val, position, size).await?; - let data_len = source.size(); - - if data_len > dedicated_threshold as u64 { - let blob_id = self.next_blob_id(); - self.write_dedicated(blob_id, BlobWriteSource::External(&source)) - .await?; - - kind_builder.append_value(BlobKind::Dedicated as u8); - data_builder.append_null(); - uri_builder.append_null(); - blob_id_builder.append_value(blob_id); - blob_size_builder.append_value(data_len); - position_builder.append_null(); - continue; - } + kind_builder.append_value(BlobKind::Packed as u8); + data_builder.append_null(); + uri_builder.append_null(); + blob_id_builder.append_value(pack_blob_id); + blob_size_builder.append_value(data_len as u64); + position_builder.append_value(position); + continue; + } - if data_len > inline_threshold as u64 { - let (pack_blob_id, position) = self - .write_packed(BlobWriteSource::External(&source)) - .await?; - - kind_builder.append_value(BlobKind::Packed as u8); - data_builder.append_null(); - uri_builder.append_null(); - blob_id_builder.append_value(pack_blob_id); - blob_size_builder.append_value(data_len); - position_builder.append_value(position); - continue; - } + if has_uri { + let uri_val = uri_col.value(i); + if self.external_blob_mode == ExternalBlobMode::Ingest { + let position = if has_position { + Some( + position_col + .as_ref() + .expect("position column must exist") + .value(i), + ) + } else { + None + }; + let size = if has_size { + Some(size_col.as_ref().expect("size column must exist").value(i)) + } else { + None + }; + let source = self.open_external_source(uri_val, position, size).await?; + let data_len = source.size(); - let data = source.read_all().await?; + if data_len > dedicated_threshold as u64 { + let blob_id = self.next_blob_id(); + self.write_dedicated(blob_id, BlobWriteSource::External(&source)) + .await?; - kind_builder.append_value(BlobKind::Inline as u8); - data_builder.append_value(data.as_ref()); + kind_builder.append_value(BlobKind::Dedicated as u8); + data_builder.append_null(); uri_builder.append_null(); - blob_id_builder.append_null(); - blob_size_builder.append_null(); + blob_id_builder.append_value(blob_id); + blob_size_builder.append_value(data_len); position_builder.append_null(); continue; } - let (external_base_id, external_uri_or_path) = - self.resolve_external_reference(uri_val).await?; - kind_builder.append_value(BlobKind::External as u8); - data_builder.append_null(); - uri_builder.append_value(external_uri_or_path); - blob_id_builder.append_value(external_base_id); - if has_position && has_size { - let position = position_col - .as_ref() - .expect("position column must exist") - .value(i); - let size = size_col.as_ref().expect("size column must exist").value(i); - blob_size_builder.append_value(size); + if data_len > inline_threshold as u64 { + let (pack_blob_id, position) = self + .write_packed(BlobWriteSource::External(&source)) + .await?; + + kind_builder.append_value(BlobKind::Packed as u8); + data_builder.append_null(); + uri_builder.append_null(); + blob_id_builder.append_value(pack_blob_id); + blob_size_builder.append_value(data_len); position_builder.append_value(position); - } else { - blob_size_builder.append_null(); - position_builder.append_null(); + continue; } - continue; - } - if has_data { + let data = source.read_all().await?; + kind_builder.append_value(BlobKind::Inline as u8); - let value = data_col.value(i); - data_builder.append_value(value); + data_builder.append_value(data.as_ref()); uri_builder.append_null(); blob_id_builder.append_null(); blob_size_builder.append_null(); position_builder.append_null(); + continue; + } + + let (external_base_id, external_uri_or_path) = + self.resolve_external_reference(uri_val).await?; + kind_builder.append_value(BlobKind::External as u8); + data_builder.append_null(); + uri_builder.append_value(external_uri_or_path); + blob_id_builder.append_value(external_base_id); + if has_position && has_size { + let position = position_col + .as_ref() + .expect("position column must exist") + .value(i); + let size = size_col.as_ref().expect("size column must exist").value(i); + blob_size_builder.append_value(size); + position_builder.append_value(position); } else { - data_builder.append_null(); - uri_builder.append_null(); - blob_id_builder.append_null(); blob_size_builder.append_null(); - kind_builder.append_null(); position_builder.append_null(); } + continue; } - let child_fields = vec![ - arrow_schema::Field::new("kind", ArrowDataType::UInt8, true), - arrow_schema::Field::new("data", ArrowDataType::LargeBinary, true), - arrow_schema::Field::new("uri", ArrowDataType::Utf8, true), - arrow_schema::Field::new("blob_id", ArrowDataType::UInt32, true), - arrow_schema::Field::new("blob_size", ArrowDataType::UInt64, true), - arrow_schema::Field::new("position", ArrowDataType::UInt64, true), - ]; - - let struct_array = arrow_array::StructArray::try_new( - child_fields.clone().into(), - vec![ - Arc::new(kind_builder.finish()), - Arc::new(data_builder.finish()), - Arc::new(uri_builder.finish()), - Arc::new(blob_id_builder.finish()), - Arc::new(blob_size_builder.finish()), - Arc::new(position_builder.finish()), - ], - struct_nulls.cloned(), - )?; - - new_columns.push(Arc::new(struct_array)); - new_fields.push(Arc::new( - arrow_schema::Field::new( - field.name(), - ArrowDataType::Struct(child_fields.into()), - field.is_nullable(), - ) - .with_metadata(self.writer_metadata[idx].clone()), - )); + if has_data { + kind_builder.append_value(BlobKind::Inline as u8); + let value = data_col.value(i); + data_builder.append_value(value); + uri_builder.append_null(); + blob_id_builder.append_null(); + blob_size_builder.append_null(); + position_builder.append_null(); + } else { + data_builder.append_null(); + uri_builder.append_null(); + blob_id_builder.append_null(); + blob_size_builder.append_null(); + kind_builder.append_null(); + position_builder.append_null(); + } } - let new_schema = Arc::new(arrow_schema::Schema::new_with_metadata( - new_fields - .iter() - .map(|f| f.as_ref().clone()) - .collect::>(), - batch_schema.metadata().clone(), - )); + let child_fields = vec![ + ArrowField::new("kind", ArrowDataType::UInt8, true), + ArrowField::new("data", ArrowDataType::LargeBinary, true), + ArrowField::new("uri", ArrowDataType::Utf8, true), + ArrowField::new("blob_id", ArrowDataType::UInt32, true), + ArrowField::new("blob_size", ArrowDataType::UInt64, true), + ArrowField::new("position", ArrowDataType::UInt64, true), + ]; - RecordBatch::try_new(new_schema, new_columns) - .map_err(|e| Error::invalid_input(e.to_string())) + let struct_array = StructArray::try_new( + child_fields.clone().into(), + vec![ + Arc::new(kind_builder.finish()), + Arc::new(data_builder.finish()), + Arc::new(uri_builder.finish()), + Arc::new(blob_id_builder.finish()), + Arc::new(blob_size_builder.finish()), + Arc::new(position_builder.finish()), + ], + struct_nulls.cloned(), + )?; + + let field = Arc::new( + ArrowField::new( + field.name(), + ArrowDataType::Struct(child_fields.into()), + field.is_nullable(), + ) + .with_metadata(writer_metadata.clone()), + ); + Ok((Arc::new(struct_array), field)) } pub(crate) async fn finish(&mut self) -> Result<()> { @@ -2171,7 +2293,7 @@ mod tests { }; use arrow_array::RecordBatch; use arrow_array::{ - ArrayRef, RecordBatchIterator, StringArray, StructArray, UInt32Array, UInt64Array, + Array, ArrayRef, RecordBatchIterator, StringArray, StructArray, UInt32Array, UInt64Array, }; use arrow_schema::{DataType, Field, Schema}; use async_trait::async_trait; @@ -2227,6 +2349,32 @@ mod tests { expected: Vec, } + fn nested_blob_v2_batch(blob_array: ArrayRef) -> (Arc, RecordBatch) { + let blob_field = blob_field("blob", true); + let info_fields = vec![Field::new("name", DataType::Utf8, false), blob_field]; + let info_array: ArrayRef = Arc::new( + StructArray::try_new( + info_fields.clone().into(), + vec![ + Arc::new(StringArray::from_iter_values( + (0..blob_array.len()).map(|idx| format!("name-{idx}")), + )) as ArrayRef, + blob_array, + ], + None, + ) + .unwrap(), + ); + + let schema = Arc::new(Schema::new(vec![Field::new( + "info", + DataType::Struct(info_fields.into()), + true, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![info_array]).unwrap(); + (schema, batch) + } + #[cfg(feature = "azure")] fn azure_store_params(account_name: &str) -> ObjectStoreParams { ObjectStoreParams { @@ -3114,6 +3262,114 @@ mod tests { assert_eq!(second.as_ref(), b"world"); } + #[tokio::test] + async fn test_write_and_take_nested_blob_v2() { + let test_dir = TempStrDir::default(); + let packed_payload = vec![0x4A; super::INLINE_MAX + 1024]; + + let mut blob_builder = BlobArrayBuilder::new(3); + blob_builder.push_bytes(b"hello").unwrap(); + blob_builder.push_bytes(&packed_payload).unwrap(); + blob_builder.push_null().unwrap(); + let blob_array: ArrayRef = blob_builder.finish().unwrap(); + + let (schema, batch) = nested_blob_v2_batch(blob_array); + let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); + + let dataset = Arc::new( + Dataset::write( + reader, + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + let info_batch = dataset + .scan() + .project(&["info"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let blob_desc = info_batch + .column(0) + .as_struct() + .column_by_name("blob") + .unwrap() + .as_struct(); + assert_eq!( + blob_desc + .column_by_name("kind") + .unwrap() + .as_primitive::() + .value(0), + BlobKind::Inline as u8 + ); + assert_eq!( + blob_desc + .column_by_name("kind") + .unwrap() + .as_primitive::() + .value(1), + BlobKind::Packed as u8 + ); + + let blobs = dataset + .take_blobs_by_indices(&[0, 1], "info.blob") + .await + .unwrap(); + assert_eq!(blobs.len(), 2); + assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"hello"); + assert_eq!( + blobs[1].read().await.unwrap().as_ref(), + packed_payload.as_slice() + ); + + let null_blobs = dataset + .take_blobs_by_indices(&[2], "info.blob") + .await + .unwrap(); + assert!(null_blobs.is_empty()); + } + + #[tokio::test] + async fn test_nested_blob_v2_requires_v2_2() { + let test_dir = TempStrDir::default(); + + let mut blob_builder = BlobArrayBuilder::new(1); + blob_builder.push_bytes(b"hello").unwrap(); + let blob_array: ArrayRef = blob_builder.finish().unwrap(); + + let (schema, batch) = nested_blob_v2_batch(blob_array); + let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); + + let result = Dataset::write( + reader, + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_1), + ..Default::default() + }), + ) + .await; + + assert!( + result.is_err(), + "Nested blob v2 should be rejected for file version 2.1" + ); + assert!( + result + .unwrap_err() + .to_string() + .contains("Blob v2 requires file version >= 2.2") + ); + } + #[tokio::test] async fn test_blob_file_read_empty_range_returns_empty_bytes() { let store = reject_empty_range_store(); diff --git a/rust/lance/src/dataset/write.rs b/rust/lance/src/dataset/write.rs index d6ac3ce9d45..ff0a119158c 100644 --- a/rust/lance/src/dataset/write.rs +++ b/rust/lance/src/dataset/write.rs @@ -583,7 +583,7 @@ pub async fn do_write_fragments( }; let external_base_resolver = if storage_version >= LanceFileVersion::V2_2 - && schema.fields.iter().any(|field| field.is_blob_v2()) + && schema.fields_pre_order().any(|field| field.is_blob_v2()) { Some(Arc::new( build_external_base_resolver(dataset, ¶ms).await?, @@ -1061,7 +1061,8 @@ pub async fn write_fragments_internal( (converted_schema, params.storage_version_or_default()) }; - if storage_version < LanceFileVersion::V2_2 && schema.fields.iter().any(|f| f.is_blob_v2()) { + if storage_version < LanceFileVersion::V2_2 && schema.fields_pre_order().any(|f| f.is_blob_v2()) + { return Err(Error::invalid_input(format!( "Blob v2 requires file version >= 2.2 (got {:?})", storage_version @@ -1242,7 +1243,7 @@ pub(super) async fn open_update_writer( // flow through WriteParams. Rebuild the external base resolver here so blob // v2 reference columns can resolve dataset-registered external URIs. let external_base_resolver = if storage_version >= LanceFileVersion::V2_2 - && schema.fields.iter().any(|f| f.is_blob_v2()) + && schema.fields_pre_order().any(|f| f.is_blob_v2()) { Some(Arc::new( build_external_base_resolver(Some(dataset), &WriteParams::default()).await?, From 705cef584a10e2e938636b4efc6b2d1048e12ee6 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Tue, 16 Jun 2026 08:44:01 -0700 Subject: [PATCH 06/15] fix: make FTS finalization idempotent (#7272) ## Summary Make distributed FTS finalization idempotent and object-store friendly. The old finalize flow was not retry-safe because it moved build-only `part__*` partition files before writing the final `metadata.lance`; if the process stopped in that window, a retry saw no final metadata but the original source files had already been deleted. The new flow treats `metadata.lance` as the commit marker and keeps all pre-commit partition sources under stable `staging/` paths. Finalization copies staged files to dense final names without deleting the staged sources or canonical final paths first, writes final metadata, then removes staging best-effort. Any leftover `staging/` objects are filtered from committed FTS `IndexMetadata.files`, so cleanup failures do not make transient files part of the committed index. ## Implementation - Write distributed build-only partition data and per-partition metadata under stable `staging/part__*` paths. - Keep the non-distributed builder path writing final `part__*` files and final `metadata.lance` directly, with no dense rewrite. - Add `IndexStore::copy_index_file_to(source, dest, dest_store)` for source-preserving copy-to-new-name; `LanceIndexStore` implements it with object-store copy and supports nested relative paths like `staging/...`. - Copy existing root partition files into staging for the public distributed-from-existing path, `from_existing_index(..., fragment_mask=Some(...))`, before advertising those partitions in staged metadata. - During `merge_index_files`, discover staged metadata, sort original partition IDs, map them densely to `0..N`, copy `staging/part__*` to final `part__*` without pre-deleting final paths, and then write final `metadata.lance`. - Delete staged partition data and staged per-partition metadata only after final metadata is written, and keep that cleanup best-effort. - Filter leftover `staging/` files from FTS segment `IndexMetadata.files` during segment commit. - On retry, existing final `metadata.lance` makes finalize a no-op; without it, staged sources remain authoritative and any partial final files are replaceable from staging. Added regressions for staged distributed writes, distributed-from-existing finalize, partial final-file retry without root `part_*` deletion, preserving staging when final metadata writing fails, and excluding stale staging files from committed metadata. Validated locally with Rust fmt/tests/clippy, segmented FTS integration tests, Python extension build, the targeted Python progress test, and `uv run make lint-rust`. --- rust/lance-index/src/scalar.rs | 16 + .../src/scalar/inverted/builder.rs | 955 +++++++++++++++--- rust/lance-index/src/scalar/inverted/index.rs | 10 + rust/lance-index/src/scalar/lance_format.rs | 47 +- rust/lance/src/index.rs | 14 +- rust/lance/src/index/create.rs | 30 + 6 files changed, 944 insertions(+), 128 deletions(-) diff --git a/rust/lance-index/src/scalar.rs b/rust/lance-index/src/scalar.rs index daec92339f8..4830586f85c 100644 --- a/rust/lance-index/src/scalar.rs +++ b/rust/lance-index/src/scalar.rs @@ -288,6 +288,22 @@ pub trait IndexStore: std::fmt::Debug + Send + Sync + DeepSizeOf { /// This is often useful when remapping or updating async fn copy_index_file(&self, name: &str, dest_store: &dyn IndexStore) -> Result; + /// Copy an index file from this store to a new name in another store, leaving the source intact + async fn copy_index_file_to( + &self, + name: &str, + new_name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + if name == new_name { + self.copy_index_file(name, dest_store).await + } else { + Err(Error::not_supported(format!( + "copying index file {name} to {new_name} is not supported by this index store" + ))) + } + } + /// Rename an index file async fn rename_index_file(&self, name: &str, new_name: &str) -> Result; diff --git a/rust/lance-index/src/scalar/inverted/builder.rs b/rust/lance-index/src/scalar/inverted/builder.rs index 24b1eb50203..93932f35332 100644 --- a/rust/lance-index/src/scalar/inverted/builder.rs +++ b/rust/lance-index/src/scalar/inverted/builder.rs @@ -407,11 +407,21 @@ impl InvertedIndexBuilder { ) -> Result> { let partition_id = self.next_partition_id() | self.fragment_mask.unwrap_or(0); builder.set_id(partition_id); - let files = builder.write(dest_store).await?; + let files = builder + .write_to(dest_store, self.partition_write_target()) + .await?; self.new_partitions.push(partition_id); Ok(files) } + fn partition_write_target(&self) -> PartitionWriteTarget { + if self.fragment_mask.is_some() { + PartitionWriteTarget::Staged + } else { + PartitionWriteTarget::Final + } + } + fn next_partition_id(&self) -> u64 { self.partitions .iter() @@ -523,7 +533,11 @@ impl InvertedIndexBuilder { if let Some(builder) = merged_tail_partitions { self.new_partitions.push(builder.id()); let mut builder = builder; - files.extend(builder.write(dest_store.as_ref()).await?); + files.extend( + builder + .write_to(dest_store.as_ref(), self.partition_write_target()) + .await?, + ); } log::info!("wait workers indexing elapsed: {:?}", start.elapsed()); Result::Ok(files) @@ -550,12 +564,16 @@ impl InvertedIndexBuilder { .await?; let mut builder = part.into_builder().await?; builder.remap(mapping).await?; - files.extend(builder.write(dest_store).await?); + files.extend( + builder + .write_to(dest_store, self.partition_write_target()) + .await?, + ); } if self.fragment_mask.is_none() { files.push(self.write_metadata(dest_store, &self.partitions).await?); } else { - // in distributed mode, the part_temp_metadata is written by the worker + // in distributed mode, the staged partition metadata is written by the worker for &partition_id in &self.partitions { files.push(self.write_part_metadata(dest_store, partition_id).await?); } @@ -709,26 +727,35 @@ impl InvertedIndexBuilder { .await?; let mut copied = 0; let mut files = Vec::new(); + let target = self.partition_write_target(); for part in self.partitions.iter() { files.push( self.src_store .as_ref() .expect("existing partitions require a source store") - .copy_index_file(&token_file_path(*part), dest_store) + .copy_index_file_to( + &token_file_path(*part), + &target.token_path(*part), + dest_store, + ) .await?, ); files.push( self.src_store .as_ref() .expect("existing partitions require a source store") - .copy_index_file(&posting_file_path(*part), dest_store) + .copy_index_file_to( + &posting_file_path(*part), + &target.posting_path(*part), + dest_store, + ) .await?, ); files.push( self.src_store .as_ref() .expect("existing partitions require a source store") - .copy_index_file(&doc_file_path(*part), dest_store) + .copy_index_file_to(&doc_file_path(*part), &target.doc_path(*part), dest_store) .await?, ); copied += 1; @@ -986,11 +1013,22 @@ impl InnerBuilder { } pub async fn write(&mut self, store: &dyn IndexStore) -> Result> { + self.write_to(store, PartitionWriteTarget::Final).await + } + + async fn write_to( + &mut self, + store: &dyn IndexStore, + target: PartitionWriteTarget, + ) -> Result> { let docs = Arc::new(std::mem::take(&mut self.docs)); let files = vec![ - self.write_posting_lists(store, docs.clone()).await?, - self.write_tokens(store).await?, - self.write_docs(store, docs).await?, + self.write_posting_lists(store, docs.clone(), &target.posting_path(self.id)) + .await?, + self.write_tokens(store, &target.token_path(self.id)) + .await?, + self.write_docs(store, docs, &target.doc_path(self.id)) + .await?, ]; Ok(files) } @@ -1000,11 +1038,12 @@ impl InnerBuilder { &mut self, store: &dyn IndexStore, docs: Arc, + path: &str, ) -> Result { let id = self.id; let mut writer = store .new_index_file( - &posting_file_path(self.id), + path, inverted_list_schema_for_version(self.with_position, self.format_version), ) .await?; @@ -1090,29 +1129,57 @@ impl InnerBuilder { } #[instrument(level = "debug", skip_all)] - async fn write_tokens(&mut self, store: &dyn IndexStore) -> Result { + async fn write_tokens(&mut self, store: &dyn IndexStore, path: &str) -> Result { log::info!("writing tokens of partition {}", self.id); let tokens = std::mem::take(&mut self.tokens); let batch = tokens.to_batch(self.token_set_format)?; - let mut writer = store - .new_index_file(&token_file_path(self.id), batch.schema()) - .await?; + let mut writer = store.new_index_file(path, batch.schema()).await?; writer.write_record_batch(batch).await?; writer.finish().await } #[instrument(level = "debug", skip_all)] - async fn write_docs(&mut self, store: &dyn IndexStore, docs: Arc) -> Result { + async fn write_docs( + &mut self, + store: &dyn IndexStore, + docs: Arc, + path: &str, + ) -> Result { log::info!("writing docs of partition {}", self.id); let batch = docs.to_batch()?; - let mut writer = store - .new_index_file(&doc_file_path(self.id), batch.schema()) - .await?; + let mut writer = store.new_index_file(path, batch.schema()).await?; writer.write_record_batch(batch).await?; writer.finish().await } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PartitionWriteTarget { + Final, + Staged, +} + +impl PartitionWriteTarget { + fn file_path(self, partition_id: u64, suffix: &str) -> String { + match self { + Self::Final => partition_file_path(partition_id, suffix), + Self::Staged => staged_partition_file_path(partition_id, suffix), + } + } + + fn token_path(self, partition_id: u64) -> String { + self.file_path(partition_id, TOKENS_FILE) + } + + fn posting_path(self, partition_id: u64) -> String { + self.file_path(partition_id, INVERT_LIST_FILE) + } + + fn doc_path(self, partition_id: u64) -> String { + self.file_path(partition_id, DOCS_FILE) + } +} + struct IndexWorker { tokenizer: Box, dest_store: Arc, @@ -1430,8 +1497,13 @@ impl IndexWorker { ); let written_partition_id = builder.id(); let mut builder = builder; + let target = if self.fragment_mask.is_some() { + PartitionWriteTarget::Staged + } else { + PartitionWriteTarget::Final + }; let files = builder - .write(self.dest_store.as_ref()) + .write_to(self.dest_store.as_ref(), target) .await .map_err(|err| { Error::execution(format!( @@ -1782,14 +1854,23 @@ pub(crate) fn doc_file_path(partition_id: u64) -> String { } pub(crate) fn part_metadata_file_path(partition_id: u64) -> String { - format!("part_{}_{}", partition_id, METADATA_FILE) + staged_partition_file_path(partition_id, METADATA_FILE) } const PARTITION_FILE_SUFFIXES: [&str; 3] = [TOKENS_FILE, INVERT_LIST_FILE, DOCS_FILE]; -// Each remapped file is renamed twice: first to a temp path (phase 1), then to -// its final path (phase 2). Keep in sync with the two rename loops below in -// `merge_metadata_files`. -const PARTITION_FILE_RENAME_PHASES: u64 = 2; +const STAGED_PARTITION_DIR: &str = "staging"; + +fn partition_file_path(partition_id: u64, suffix: &str) -> String { + format!("part_{}_{}", partition_id, suffix) +} + +fn staged_partition_file_path(partition_id: u64, suffix: &str) -> String { + format!( + "{}/{}", + STAGED_PARTITION_DIR, + partition_file_path(partition_id, suffix) + ) +} pub async fn merge_index_files( object_store: &ObjectStore, @@ -1797,33 +1878,65 @@ pub async fn merge_index_files( store: Arc, progress: Arc, ) -> Result<()> { - // List all partition metadata files in the index directory - let part_metadata_files = list_metadata_files(object_store, index_dir).await?; + let metadata_path = index_dir.clone().join(METADATA_FILE); + if object_store.exists(&metadata_path).await? { + return Ok(()); + } + + // List all staged partition metadata files in the index directory + let index_files = list_index_files(object_store, index_dir).await?; + let part_metadata_files = metadata_files(&index_files); + if part_metadata_files.is_empty() { + return Err(Error::invalid_input_source( + format!( + "No partition metadata files found in index directory: {}", + index_dir + ) + .into(), + )); + } // Call merge_metadata_files function for inverted index merge_metadata_files(store, &part_metadata_files, progress).await } -/// List and filter metadata files from the index directory -/// Returns partition metadata files -async fn list_metadata_files(object_store: &ObjectStore, index_dir: &Path) -> Result> { - // List all partition metadata files in the index directory - let mut part_metadata_files = Vec::new(); - let mut list_stream = object_store.list(Some(index_dir.clone())); +async fn list_index_files(object_store: &ObjectStore, index_dir: &Path) -> Result> { + let mut index_files = Vec::new(); + let mut list_stream = object_store.read_dir_all(index_dir, None); while let Some(item) = list_stream.next().await { match item { Ok(meta) => { - let file_name = meta.location.filename().unwrap_or_default(); - // Filter files matching the pattern part_*_metadata.lance - if file_name.starts_with("part_") && file_name.ends_with("_metadata.lance") { - part_metadata_files.push(file_name.to_string()); - } + let location = meta.location.as_ref().trim_start_matches('/'); + let index_dir = index_dir.as_ref().trim_start_matches('/'); + let relative_path = location + .strip_prefix(index_dir) + .map(|s| s.trim_start_matches('/').to_string()) + .unwrap_or_else(|| meta.location.filename().unwrap_or("").to_string()); + index_files.push(relative_path); } Err(err) => return Err(err), } } + Ok(index_files) +} + +fn metadata_files(index_files: &[String]) -> Vec { + index_files + .iter() + .filter(|file_name| { + file_name.starts_with(&format!("{}/part_", STAGED_PARTITION_DIR)) + && file_name.ends_with("_metadata.lance") + }) + .cloned() + .collect() +} + +#[cfg(test)] +async fn list_metadata_files(object_store: &ObjectStore, index_dir: &Path) -> Result> { + let index_files = list_index_files(object_store, index_dir).await?; + let part_metadata_files = metadata_files(&index_files); if part_metadata_files.is_empty() { return Err(Error::invalid_input_source( format!( @@ -1914,89 +2027,35 @@ async fn merge_metadata_files( progress.stage_complete("read_partition_metadata").await?; // Create ID mapping: sorted original IDs -> 0,1,2... - let mut sorted_ids = all_partitions.clone(); + let mut sorted_ids = all_partitions; sorted_ids.sort(); sorted_ids.dedup(); - let id_mapping: HashMap = sorted_ids + let id_mapping: Vec<(u64, u64)> = sorted_ids .iter() .enumerate() .map(|(new_id, &old_id)| (old_id, new_id as u64)) .collect(); - // Safe rename partition files using temporary files to avoid overwrite - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs(); - - let changed_partition_count = id_mapping - .iter() - .filter(|(old_id, new_id)| old_id != new_id) - .count() as u64; - let total_renames = changed_partition_count - * PARTITION_FILE_SUFFIXES.len() as u64 - * PARTITION_FILE_RENAME_PHASES; + let total_copies = id_mapping.len() as u64 * PARTITION_FILE_SUFFIXES.len() as u64; progress - .stage_start("remap_partition_files", Some(total_renames), "files") + .stage_start("remap_partition_files", Some(total_copies), "files") .await?; - // Phase 1: Move files to temporary locations - let mut temp_files: Vec<(String, String, String)> = Vec::new(); // (temp_path, old_path, final_path) - let mut renamed_files = 0u64; + let mut copied_files = 0u64; - for (&old_id, &new_id) in &id_mapping { - if old_id != new_id { - for suffix in PARTITION_FILE_SUFFIXES { - let old_path = format!("part_{}_{}", old_id, suffix); - let new_path = format!("part_{}_{}", new_id, suffix); - let temp_path = format!("temp_{}_{}", timestamp, old_path); - - // Move to temporary location first to avoid overwrite - if let Err(e) = store.rename_index_file(&old_path, &temp_path).await { - // Rollback phase 1: restore files from temp locations - for (temp_name, old_name, _) in temp_files.iter().rev() { - let _ = store.rename_index_file(temp_name, old_name).await; - } - return Err(Error::index(format!( - "Failed to move {} to temp {}: {}", - old_path, temp_path, e - ))); - } - temp_files.push((temp_path, old_path, new_path)); - renamed_files += 1; - progress - .stage_progress("remap_partition_files", renamed_files) - .await?; - } - } - } - - // Phase 2: Move from temporary to final locations - let mut completed_renames: Vec<(String, String)> = Vec::new(); // (final_path, temp_path) - - for (temp_path, _old_path, final_path) in &temp_files { - if let Err(e) = store.rename_index_file(temp_path, final_path).await { - // Rollback phase 2: restore completed renames and remaining temps - for (final_name, temp_name) in completed_renames.iter().rev() { - let _ = store.rename_index_file(final_name, temp_name).await; - } - // Restore remaining temp files to original locations - for (temp_name, orig_name, _) in temp_files.iter() { - if !completed_renames.iter().any(|(_, t)| t == temp_name) { - let _ = store.rename_index_file(temp_name, orig_name).await; - } - } - return Err(Error::index(format!( - "Failed to rename {} to {}: {}", - temp_path, final_path, e - ))); + for &(old_id, new_id) in &id_mapping { + for suffix in PARTITION_FILE_SUFFIXES { + let staged_path = staged_partition_file_path(old_id, suffix); + let final_path = partition_file_path(new_id, suffix); + store + .copy_index_file_to(&staged_path, &final_path, store.as_ref()) + .await?; + copied_files += 1; + progress + .stage_progress("remap_partition_files", copied_files) + .await?; } - completed_renames.push((final_path.clone(), temp_path.clone())); - renamed_files += 1; - progress - .stage_progress("remap_partition_files", renamed_files) - .await?; } progress.stage_complete("remap_partition_files").await?; @@ -2023,10 +2082,15 @@ async fn merge_metadata_files( progress.stage_progress("write_merged_metadata", 1).await?; progress.stage_complete("write_merged_metadata").await?; - // Cleanup partition metadata files + // Cleanup staged partition metadata files for file_name in part_metadata_files { - if file_name.starts_with("part_") && file_name.ends_with("_metadata.lance") { - let _ = store.delete_index_file(file_name).await; + let _ = store.delete_index_file(file_name).await; + } + for &(old_id, _) in &id_mapping { + for suffix in PARTITION_FILE_SUFFIXES { + let _ = store + .delete_index_file(&staged_partition_file_path(old_id, suffix)) + .await; } } @@ -2246,6 +2310,234 @@ mod tests { } } + #[derive(Debug, Clone)] + struct NoRenameStore { + inner: Arc, + final_delete_count: Option>, + } + + impl NoRenameStore { + fn new(inner: Arc) -> Self { + Self { + inner, + final_delete_count: None, + } + } + + fn with_final_delete_tracking(inner: Arc) -> Self { + Self { + inner, + final_delete_count: Some(Arc::new(AtomicUsize::new(0))), + } + } + + fn final_delete_count(&self) -> usize { + self.final_delete_count + .as_ref() + .map(|count| count.load(Ordering::SeqCst)) + .unwrap_or_default() + } + + fn unwrap_dest_store(dest_store: &dyn IndexStore) -> &dyn IndexStore { + dest_store + .as_any() + .downcast_ref::() + .map(|store| store.inner.as_ref()) + .unwrap_or(dest_store) + } + } + + impl DeepSizeOf for NoRenameStore { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.inner.deep_size_of_children(context) + } + } + + #[async_trait] + impl IndexStore for NoRenameStore { + fn as_any(&self) -> &dyn Any { + self + } + + fn clone_arc(&self) -> Arc { + Arc::new(self.clone()) + } + + fn io_parallelism(&self) -> usize { + self.inner.io_parallelism() + } + + async fn new_index_file( + &self, + name: &str, + schema: Arc, + ) -> Result> { + self.inner.new_index_file(name, schema).await + } + + async fn open_index_file(&self, name: &str) -> Result> { + self.inner.open_index_file(name).await + } + + async fn copy_index_file( + &self, + name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + self.inner + .copy_index_file(name, Self::unwrap_dest_store(dest_store)) + .await + } + + async fn copy_index_file_to( + &self, + name: &str, + new_name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + self.inner + .copy_index_file_to(name, new_name, Self::unwrap_dest_store(dest_store)) + .await + } + + async fn rename_index_file(&self, name: &str, new_name: &str) -> Result { + Err(Error::internal(format!( + "merge_index_files should not rename partition file {name} to {new_name}" + ))) + } + + async fn delete_index_file(&self, name: &str) -> Result<()> { + if name.starts_with("part_") + && let Some(count) = &self.final_delete_count + { + count.fetch_add(1, Ordering::SeqCst); + } + self.inner.delete_index_file(name).await + } + + async fn list_files_with_sizes(&self) -> Result> { + self.inner.list_files_with_sizes().await + } + } + + #[derive(Debug)] + struct FailMetadataStore { + inner: Arc, + } + + impl FailMetadataStore { + fn new(inner: Arc) -> Self { + Self { inner } + } + + fn unwrap_dest_store(dest_store: &dyn IndexStore) -> &dyn IndexStore { + dest_store + .as_any() + .downcast_ref::() + .map(|store| store.inner.as_ref()) + .unwrap_or(dest_store) + } + } + + impl DeepSizeOf for FailMetadataStore { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.inner.deep_size_of_children(context) + } + } + + #[async_trait] + impl IndexStore for FailMetadataStore { + fn as_any(&self) -> &dyn Any { + self + } + + fn clone_arc(&self) -> Arc { + Arc::new(Self { + inner: self.inner.clone(), + }) + } + + fn io_parallelism(&self) -> usize { + self.inner.io_parallelism() + } + + async fn new_index_file( + &self, + name: &str, + schema: Arc, + ) -> Result> { + let writer = self.inner.new_index_file(name, schema).await?; + if name == METADATA_FILE { + Ok(Box::new(FailFinishWriter { inner: writer })) + } else { + Ok(writer) + } + } + + async fn open_index_file(&self, name: &str) -> Result> { + self.inner.open_index_file(name).await + } + + async fn copy_index_file( + &self, + name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + self.inner + .copy_index_file(name, Self::unwrap_dest_store(dest_store)) + .await + } + + async fn copy_index_file_to( + &self, + name: &str, + new_name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + self.inner + .copy_index_file_to(name, new_name, Self::unwrap_dest_store(dest_store)) + .await + } + + async fn rename_index_file(&self, name: &str, new_name: &str) -> Result { + self.inner.rename_index_file(name, new_name).await + } + + async fn delete_index_file(&self, name: &str) -> Result<()> { + self.inner.delete_index_file(name).await + } + + async fn list_files_with_sizes(&self) -> Result> { + self.inner.list_files_with_sizes().await + } + } + + struct FailFinishWriter { + inner: Box, + } + + #[async_trait] + impl IndexWriter for FailFinishWriter { + async fn write_record_batch(&mut self, batch: RecordBatch) -> Result { + self.inner.write_record_batch(batch).await + } + + async fn add_global_buffer(&mut self, data: Bytes) -> Result { + self.inner.add_global_buffer(data).await + } + + async fn finish(&mut self) -> Result { + Err(Error::internal("injected metadata write failure")) + } + + async fn finish_with_metadata( + &mut self, + _metadata: HashMap, + ) -> Result { + Err(Error::internal("injected metadata write failure")) + } + } + #[derive(Debug)] struct CountingWriter { path: String, @@ -2412,12 +2704,446 @@ mod tests { let store = CountingStore::new(); let docs = Arc::new(std::mem::take(&mut builder.docs)); - builder.write_posting_lists(&store, docs).await?; + builder + .write_posting_lists(&store, docs, &posting_file_path(0)) + .await?; assert_eq!(store.write_count(), 1); Ok(()) } + async fn write_partition_file_marker( + store: &dyn IndexStore, + path: &str, + partition_id: u64, + ) -> Result<()> { + let schema = Arc::new(Schema::new(vec![Field::new( + "partition_id", + DataType::UInt64, + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(UInt64Array::from(vec![partition_id]))], + )?; + let mut writer = store.new_index_file(path, schema).await?; + writer.write_record_batch(batch).await?; + writer.finish().await?; + Ok(()) + } + + async fn write_partition_files( + store: &dyn IndexStore, + partition_id: u64, + target: PartitionWriteTarget, + ) -> Result<()> { + write_partition_file_marker(store, &target.token_path(partition_id), partition_id).await?; + write_partition_file_marker(store, &target.posting_path(partition_id), partition_id) + .await?; + write_partition_file_marker(store, &target.doc_path(partition_id), partition_id).await?; + Ok(()) + } + + async fn read_partition_file_marker(store: &dyn IndexStore, path: &str) -> Result { + let reader = store.open_index_file(path).await?; + let batch = reader.read_range(0..1, None).await?; + let partition_ids = batch.column(0).as_primitive::(); + Ok(partition_ids.value(0)) + } + + async fn assert_partition_file_markers( + store: &dyn IndexStore, + partition_id: u64, + expected_marker: u64, + ) -> Result<()> { + assert_eq!( + read_partition_file_marker(store, &token_file_path(partition_id)).await?, + expected_marker + ); + assert_eq!( + read_partition_file_marker(store, &posting_file_path(partition_id)).await?, + expected_marker + ); + assert_eq!( + read_partition_file_marker(store, &doc_file_path(partition_id)).await?, + expected_marker + ); + Ok(()) + } + + #[tokio::test] + async fn test_merge_index_files_remaps_staged_partitions_without_rename() -> Result<()> { + let index_dir = TempDir::default(); + let object_store = Arc::new(ObjectStore::local()); + let base_store: Arc = Arc::new(LanceIndexStore::new( + object_store.clone(), + index_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + let store = Arc::new(NoRenameStore::new(base_store.clone())); + let partitions = vec![5_u64, 1_u64, (17_u64 << 32) | 2]; + let metadata_builder = InvertedIndexBuilder::from_existing_index( + InvertedIndexParams::default(), + None, + Vec::new(), + TokenSetFormat::default(), + None, + RoaringBitmap::new(), + ); + + for partition_id in &partitions { + write_partition_files( + base_store.as_ref(), + *partition_id, + PartitionWriteTarget::Staged, + ) + .await?; + metadata_builder + .write_part_metadata(base_store.as_ref(), *partition_id) + .await?; + } + + merge_index_files( + object_store.as_ref(), + &index_dir.obj_path(), + store, + noop_progress(), + ) + .await?; + + let metadata_reader = base_store.open_index_file(METADATA_FILE).await?; + let metadata = &metadata_reader.schema().metadata; + let written_partitions: Vec = serde_json::from_str( + metadata + .get("partitions") + .expect("partitions missing from metadata"), + )?; + let mut expected_partitions = partitions.clone(); + expected_partitions.sort_unstable(); + expected_partitions.dedup(); + let remapped_partitions = (0..expected_partitions.len() as u64).collect::>(); + assert_eq!(written_partitions, remapped_partitions); + + for (new_id, old_id) in expected_partitions.iter().enumerate() { + assert_partition_file_markers(base_store.as_ref(), new_id as u64, *old_id).await?; + assert!( + base_store + .open_index_file(&part_metadata_file_path(*old_id)) + .await + .is_err(), + "partition metadata should be cleaned up after final metadata is written" + ); + for suffix in PARTITION_FILE_SUFFIXES { + assert!( + base_store + .open_index_file(&staged_partition_file_path(*old_id, suffix)) + .await + .is_err(), + "staged partition files should be cleaned up after final metadata is written" + ); + } + } + + Ok(()) + } + + #[tokio::test] + async fn test_merge_index_files_rewrites_partial_final_files_from_staging() -> Result<()> { + let index_dir = TempDir::default(); + let object_store = Arc::new(ObjectStore::local()); + let base_store: Arc = Arc::new(LanceIndexStore::new( + object_store.clone(), + index_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + let store = Arc::new(NoRenameStore::with_final_delete_tracking( + base_store.clone(), + )); + let partitions = vec![1_u64, 5_u64]; + let metadata_builder = InvertedIndexBuilder::from_existing_index( + InvertedIndexParams::default(), + None, + Vec::new(), + TokenSetFormat::default(), + None, + RoaringBitmap::new(), + ); + + for partition_id in &partitions { + write_partition_files( + base_store.as_ref(), + *partition_id, + PartitionWriteTarget::Staged, + ) + .await?; + metadata_builder + .write_part_metadata(base_store.as_ref(), *partition_id) + .await?; + } + + for suffix in PARTITION_FILE_SUFFIXES { + write_partition_file_marker(base_store.as_ref(), &partition_file_path(1, suffix), 999) + .await?; + } + + merge_index_files( + object_store.as_ref(), + &index_dir.obj_path(), + store.clone(), + noop_progress(), + ) + .await?; + + assert_partition_file_markers(base_store.as_ref(), 0, 1).await?; + assert_partition_file_markers(base_store.as_ref(), 1, 5).await?; + assert_eq!( + store.final_delete_count(), + 0, + "merge should overwrite final partition files without deleting them first" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_distributed_from_existing_copies_existing_partitions_to_staging_and_finalizes() + -> Result<()> { + let object_store = Arc::new(ObjectStore::local()); + let source_dir = TempDir::default(); + let dest_dir = TempDir::default(); + let source_store: Arc = Arc::new(LanceIndexStore::new( + object_store.clone(), + source_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + let dest_store: Arc = Arc::new(LanceIndexStore::new( + object_store.clone(), + dest_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + let merge_store = Arc::new(NoRenameStore::new(dest_store.clone())); + let fragment_mask = 7_u64 << 32; + let partitions = vec![fragment_mask | 5, fragment_mask | 1]; + + for partition_id in &partitions { + write_partition_files( + source_store.as_ref(), + *partition_id, + PartitionWriteTarget::Final, + ) + .await?; + } + + let builder = InvertedIndexBuilder::from_existing_index( + InvertedIndexParams::default(), + Some(source_store.clone()), + partitions.clone(), + TokenSetFormat::default(), + Some(fragment_mask), + RoaringBitmap::new(), + ); + builder.write(dest_store.as_ref()).await?; + + for partition_id in &partitions { + assert_partition_file_markers(source_store.as_ref(), *partition_id, *partition_id) + .await?; + for suffix in PARTITION_FILE_SUFFIXES { + let staged_path = staged_partition_file_path(*partition_id, suffix); + assert_eq!( + read_partition_file_marker(dest_store.as_ref(), &staged_path).await?, + *partition_id + ); + assert!( + dest_store + .open_index_file(&partition_file_path(*partition_id, suffix)) + .await + .is_err(), + "distributed existing partition should be staged instead of copied to root" + ); + } + dest_store + .open_index_file(&part_metadata_file_path(*partition_id)) + .await?; + } + + merge_index_files( + object_store.as_ref(), + &dest_dir.obj_path(), + merge_store, + noop_progress(), + ) + .await?; + + let mut expected_partitions = partitions.clone(); + expected_partitions.sort_unstable(); + for (new_id, old_id) in expected_partitions.iter().enumerate() { + assert_partition_file_markers(dest_store.as_ref(), new_id as u64, *old_id).await?; + for suffix in PARTITION_FILE_SUFFIXES { + assert!( + dest_store + .open_index_file(&staged_partition_file_path(*old_id, suffix)) + .await + .is_err(), + "staged partition files should be cleaned after final metadata is written" + ); + } + } + + Ok(()) + } + + #[tokio::test] + async fn test_merge_index_files_keeps_staging_when_final_metadata_write_fails() -> Result<()> { + let index_dir = TempDir::default(); + let object_store = Arc::new(ObjectStore::local()); + let base_store: Arc = Arc::new(LanceIndexStore::new( + object_store.clone(), + index_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + let failing_store = Arc::new(FailMetadataStore::new(base_store.clone())); + let partitions = vec![1_u64, 5_u64]; + let metadata_builder = InvertedIndexBuilder::from_existing_index( + InvertedIndexParams::default(), + None, + Vec::new(), + TokenSetFormat::default(), + None, + RoaringBitmap::new(), + ); + + for partition_id in &partitions { + write_partition_files( + base_store.as_ref(), + *partition_id, + PartitionWriteTarget::Staged, + ) + .await?; + metadata_builder + .write_part_metadata(base_store.as_ref(), *partition_id) + .await?; + } + + let err = merge_index_files( + object_store.as_ref(), + &index_dir.obj_path(), + failing_store, + noop_progress(), + ) + .await + .unwrap_err(); + assert!( + err.to_string().contains("metadata write failure"), + "expected injected metadata failure, got: {err}" + ); + + for partition_id in &partitions { + base_store + .open_index_file(&part_metadata_file_path(*partition_id)) + .await?; + for suffix in PARTITION_FILE_SUFFIXES { + let staged_path = staged_partition_file_path(*partition_id, suffix); + assert_eq!( + read_partition_file_marker(base_store.as_ref(), &staged_path).await?, + *partition_id + ); + } + } + + Ok(()) + } + + #[tokio::test] + async fn test_distributed_build_writes_partition_data_to_staging() -> Result<()> { + let index_dir = TempDir::default(); + let object_store = ObjectStore::local(); + let store = Arc::new(LanceIndexStore::new( + object_store.into(), + index_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + + let fragment_mask = 7_u64 << 32; + let batch = make_doc_batch("hello world", fragment_mask); + let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)])); + let stream = Box::pin(stream); + let mut builder = InvertedIndexBuilder::new_with_fragment_mask( + InvertedIndexParams::default(), + Some(fragment_mask), + ); + builder.update(stream, store.as_ref(), None).await?; + + let part_metadata_files = + list_metadata_files(&ObjectStore::local(), &index_dir.obj_path()).await?; + assert_eq!(part_metadata_files.len(), 1); + assert!( + part_metadata_files[0].starts_with("staging/part_"), + "partition metadata should be written to staging" + ); + let reader = store.open_index_file(&part_metadata_files[0]).await?; + let partition_ids: Vec = serde_json::from_str( + reader + .schema() + .metadata + .get("partitions") + .expect("partitions missing from metadata"), + )?; + assert_eq!(partition_ids.len(), 1); + let partition_id = partition_ids[0]; + + store + .open_index_file(&staged_partition_file_path(partition_id, TOKENS_FILE)) + .await?; + assert!( + store + .open_index_file(&partition_file_path(partition_id, METADATA_FILE)) + .await + .is_err(), + "distributed build-only metadata should not be written to root partition metadata paths" + ); + assert!( + store + .open_index_file(&token_file_path(partition_id)) + .await + .is_err(), + "distributed build-only data should not be written to final partition paths" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_merge_index_files_is_noop_when_metadata_exists() -> Result<()> { + let index_dir = TempDir::default(); + let object_store = Arc::new(ObjectStore::local()); + let store: Arc = Arc::new(LanceIndexStore::new( + object_store.clone(), + index_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + let metadata_builder = InvertedIndexBuilder::from_existing_index( + InvertedIndexParams::default(), + None, + vec![42], + TokenSetFormat::default(), + None, + RoaringBitmap::new(), + ); + metadata_builder + .write_metadata(store.as_ref(), &[42]) + .await?; + + merge_index_files( + object_store.as_ref(), + &index_dir.obj_path(), + store, + noop_progress(), + ) + .await?; + + Ok(()) + } + #[tokio::test] async fn test_build_only_path_writes_partitions_as_is() -> Result<()> { let src_dir = TempDir::default(); @@ -2856,7 +3582,6 @@ mod tests { } }) .collect::>(); - let read_start = tags .iter() .position(|e| e == "start:read_partition_metadata") @@ -2894,8 +3619,8 @@ mod tests { ); assert_eq!( remap_progress.last().copied().unwrap_or_default(), - 12, - "expected remap_partition_files progress to cover both rename phases" + 6, + "expected remap_partition_files progress to cover staged-to-final copies" ); assert!( tags.iter().any(|e| e == "progress:write_merged_metadata"), diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs index 92379a9e350..8e662f5db6f 100644 --- a/rust/lance-index/src/scalar/inverted/index.rs +++ b/rust/lance-index/src/scalar/inverted/index.rs @@ -6460,6 +6460,16 @@ mod tests { ) -> Result { self.inner.copy_index_file(name, dest_store).await } + async fn copy_index_file_to( + &self, + name: &str, + new_name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + self.inner + .copy_index_file_to(name, new_name, dest_store) + .await + } async fn rename_index_file( &self, name: &str, diff --git a/rust/lance-index/src/scalar/lance_format.rs b/rust/lance-index/src/scalar/lance_format.rs index 562945b8f0d..2f82deb8403 100644 --- a/rust/lance-index/src/scalar/lance_format.rs +++ b/rust/lance-index/src/scalar/lance_format.rs @@ -99,6 +99,24 @@ impl LanceIndexStore { self.file_sizes = file_sizes; self } + + fn index_file_path(&self, name: &str) -> Result { + let relative_path = Path::parse(name).map_err(|err| { + Error::invalid_input(format!("invalid index file path {name:?}: {err}")) + })?; + if self.index_dir.is_root() { + return Ok(relative_path); + } + if relative_path.is_root() { + return Ok(self.index_dir.clone()); + } + Path::parse(format!( + "{}/{}", + self.index_dir.as_ref(), + relative_path.as_ref() + )) + .map_err(|err| Error::invalid_input(format!("invalid index file path {name:?}: {err}"))) + } } #[async_trait] @@ -397,7 +415,7 @@ impl IndexStore for LanceIndexStore { name: &str, schema: Arc, ) -> Result> { - let path = self.index_dir.clone().join(name); + let path = self.index_file_path(name)?; let schema = schema.as_ref().try_into()?; let writer = self.object_store.create(&path).await?; let writer = current_writer::FileWriter::try_new( @@ -415,7 +433,7 @@ impl IndexStore for LanceIndexStore { } async fn open_index_file(&self, name: &str) -> Result> { - let path = self.index_dir.clone().join(name); + let path = self.index_file_path(name)?; // Use cached file size if available, otherwise unknown (requires HEAD call) let cached_size = self .file_sizes @@ -436,7 +454,7 @@ impl IndexStore for LanceIndexStore { Err(e) => { // If the error is a version conflict we can try to read the file with v1 reader if let Error::VersionConflict { .. } = e { - let path = self.index_dir.clone().join(name); + let path = self.index_file_path(name)?; let file_reader = PreviousFileReader::try_new_self_described( &self.object_store, &path, @@ -452,7 +470,16 @@ impl IndexStore for LanceIndexStore { } async fn copy_index_file(&self, name: &str, dest_store: &dyn IndexStore) -> Result { - let path = self.index_dir.clone().join(name); + self.copy_index_file_to(name, name, dest_store).await + } + + async fn copy_index_file_to( + &self, + name: &str, + new_name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + let path = self.index_file_path(name)?; let other_store = dest_store.as_any().downcast_ref::(); match other_store { @@ -460,21 +487,21 @@ impl IndexStore for LanceIndexStore { // If both this store and the destination are lance stores we can use object_store's copy // This does blindly assume that both stores are using the same underlying object_store // but there is no easy way to verify this and it happens to always be true at the moment - let dest_path = dest_store.index_dir.clone().join(name); + let dest_path = dest_store.index_file_path(new_name)?; self.object_store.copy(&path, &dest_path).await?; let size_bytes = match self.file_sizes.get(name) { Some(size_bytes) => *size_bytes, None => self.object_store.size(&path).await?, }; Ok(IndexFile { - path: name.to_string(), + path: new_name.to_string(), size_bytes, }) } _ => { let reader = self.open_index_file(name).await?; let mut writer = dest_store - .new_index_file(name, Arc::new(reader.schema().into())) + .new_index_file(new_name, Arc::new(reader.schema().into())) .await?; for offset in (0..reader.num_rows()).step_by(4096) { @@ -488,8 +515,8 @@ impl IndexStore for LanceIndexStore { } async fn rename_index_file(&self, name: &str, new_name: &str) -> Result { - let path = self.index_dir.clone().join(name); - let new_path = self.index_dir.clone().join(new_name); + let path = self.index_file_path(name)?; + let new_path = self.index_file_path(new_name)?; self.object_store.copy(&path, &new_path).await?; self.object_store.delete(&path).await?; let size_bytes = match self.file_sizes.get(name) { @@ -503,7 +530,7 @@ impl IndexStore for LanceIndexStore { } async fn delete_index_file(&self, name: &str) -> Result<()> { - let path = self.index_dir.clone().join(name); + let path = self.index_file_path(name)?; self.object_store.delete(&path).await } diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs index 78e5c429527..69acc69b6da 100644 --- a/rust/lance/src/index.rs +++ b/rust/lance/src/index.rs @@ -57,7 +57,7 @@ use lance_io::utils::{ read_version, }; use lance_table::format::{Fragment, SelfDescribingFileReader}; -use lance_table::format::{IndexMetadata, list_index_files_with_sizes}; +use lance_table::format::{IndexFile, IndexMetadata, list_index_files_with_sizes}; use lance_table::io::manifest::read_manifest_indexes; use roaring::RoaringBitmap; use scalar::index_matches_criteria; @@ -166,7 +166,8 @@ pub(crate) async fn build_index_metadata_from_segments( let mut new_indices = Vec::with_capacity(segments.len()); for segment in segments { let (uuid, fragment_bitmap, index_details, index_version) = segment.into_parts(); - if index_details.type_url.ends_with("InvertedIndexDetails") { + let is_inverted_index = index_details.type_url.ends_with("InvertedIndexDetails"); + if is_inverted_index { let metadata = IndexMetadata { uuid, name: index_name.to_string(), @@ -183,7 +184,10 @@ pub(crate) async fn build_index_metadata_from_segments( .await?; } let index_dir = dataset.indices_dir().clone().join(uuid.to_string()); - let files = list_index_files_with_sizes(&dataset.object_store, &index_dir).await?; + let mut files = list_index_files_with_sizes(&dataset.object_store, &index_dir).await?; + if is_inverted_index { + retain_committed_inverted_files(&mut files); + } new_indices.push(IndexMetadata { uuid, name: index_name.to_string(), @@ -201,6 +205,10 @@ pub(crate) async fn build_index_metadata_from_segments( Ok(new_indices) } +fn retain_committed_inverted_files(files: &mut Vec) { + files.retain(|file| !file.path.starts_with("staging/")); +} + fn validate_segment_index_details(index_name: &str, segments: &[IndexMetadata]) -> Result<()> { let mut type_url = None::<&str>; for segment in segments { diff --git a/rust/lance/src/index/create.rs b/rust/lance/src/index/create.rs index 2b6992e4849..bbb055463dc 100644 --- a/rust/lance/src/index/create.rs +++ b/rust/lance/src/index/create.rs @@ -1993,6 +1993,23 @@ mod tests { let segments = input_segments.clone(); assert_eq!(segments.len(), input_segments.len()); + crate::index::scalar::inverted::finalize_segment_files_if_needed( + &dataset, + &input_segments[0], + ) + .await + .unwrap(); + let stale_staging_path = dataset + .indices_dir() + .join(input_segments[0].uuid.to_string()) + .join("staging") + .join("orphan.lance"); + dataset + .object_store + .put(&stale_staging_path, b"stale") + .await + .unwrap(); + dataset .commit_existing_index_segments("text_idx", "text", segments) .await @@ -2016,6 +2033,19 @@ mod tests { let indices = dataset.load_indices_by_name("text_idx").await.unwrap(); assert_eq!(indices.len(), input_segments.len()); + let finalized_segment = indices + .iter() + .find(|index| index.uuid == input_segments[0].uuid) + .expect("finalized segment should be committed"); + assert!( + finalized_segment + .files + .as_ref() + .expect("committed segment should track files") + .iter() + .all(|file| !file.path.starts_with("staging/")), + "stale staging files must not be committed in IndexMetadata.files" + ); } #[tokio::test] From 0362c198424aa2fdd3fbcca2df4523a055e4a0a1 Mon Sep 17 00:00:00 2001 From: Muhammad Aiman <84276911+aimanmalib@users.noreply.github.com> Date: Tue, 16 Jun 2026 23:59:43 +0800 Subject: [PATCH 07/15] feat: expose io_buffer_size in CompactionOptions (#7226) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary `CompactionOptions` did not expose `io_buffer_size`, even though the scanner used during compaction supports it. Compaction builds its scan reader in `prepare_reader`, which only forwarded `batch_size` to the scanner — the `io_buffer_size` knob was never set. This matters because a single batch larger than the I/O buffer size causes the scanner to **deadlock** (documented on `Scanner::io_buffer_size`), and since the backpressure warning was downgraded to `debug`, this deadlock is now silent at the default. Users had no way to raise the buffer to avoid it during compaction. Resolves #4946. ## Changes - Add `io_buffer_size: Option` to `CompactionOptions` (with `Default` = `None`). - Support the `lance.compaction.io_buffer_size` manifest config key in `apply_dataset_config`. - Plumb the value through `prepare_reader` → `scanner.io_buffer_size(...)`, mirroring the existing `batch_size` handling. - Update the Python binding to keep parameter names consistent across languages: - `parse_compaction_options` accepts `io_buffer_size` - `CompactionOptions` TypedDict documents the field ## Testing - Extended `test_from_dataset_config` to assert the `lance.compaction.io_buffer_size` key round-trips. - Added `test_compact_with_io_buffer_size` (parametrized over `Legacy`/`Stable` file versions) that runs `compact_files` with an explicit `io_buffer_size` and verifies the compaction succeeds and preserves all rows. - `cargo test -p lance --lib dataset::optimize::tests` → **76 passed, 0 failed**. - `cargo clippy -p lance --lib --tests` → clean. - `cargo fmt --all -- --check` → clean. ## Notes This is a non-breaking, additive change — the new field defaults to `None`, preserving existing behavior when unset. --- python/python/lance/optimize.py | 8 +++ python/src/dataset/optimize.rs | 3 ++ rust/lance/src/dataset/optimize.rs | 80 ++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+) diff --git a/python/python/lance/optimize.py b/python/python/lance/optimize.py index 8b98308d442..3ac7547960b 100644 --- a/python/python/lance/optimize.py +++ b/python/python/lance/optimize.py @@ -57,6 +57,14 @@ class CompactionOptions(TypedDict): The batch size to use when scanning input fragments. You may want to reduce this if you are running out of memory during compaction. + The default will use the same default from ``scanner``. + """ + io_buffer_size: Optional[int] + """ + The number of bytes to allow to queue up in the I/O buffer when scanning + input fragments. Increasing this can avoid a deadlock that occurs when a + single batch of data is larger than the I/O buffer size. + The default will use the same default from ``scanner``. """ compaction_mode: Optional[ diff --git a/python/src/dataset/optimize.rs b/python/src/dataset/optimize.rs index 321d7157b86..33aa32b94cd 100644 --- a/python/src/dataset/optimize.rs +++ b/python/src/dataset/optimize.rs @@ -58,6 +58,9 @@ fn parse_compaction_options( "batch_size" => { opts.batch_size = value.extract()?; } + "io_buffer_size" => { + opts.io_buffer_size = value.extract()?; + } "compaction_mode" => { let mode_str: Option = value.extract()?; if let Some(mode_str) = mode_str { diff --git a/rust/lance/src/dataset/optimize.rs b/rust/lance/src/dataset/optimize.rs index d591e42cc73..56cf74c1a62 100644 --- a/rust/lance/src/dataset/optimize.rs +++ b/rust/lance/src/dataset/optimize.rs @@ -191,6 +191,13 @@ pub struct CompactionOptions { /// specified then the default (see /// [`crate::dataset::Scanner::batch_size`]) will be used. pub batch_size: Option, + /// The number of bytes to allow to queue up in the I/O buffer when scanning + /// the input fragments. If not specified then the default (see + /// [`crate::dataset::Scanner::io_buffer_size`]) will be used. + /// + /// Increasing this can avoid a deadlock that occurs when a single batch of + /// data is larger than the I/O buffer size. + pub io_buffer_size: Option, /// Whether to defer remapping indices during compaction. If true, indices will /// not be remapped during this compaction operation. Instead, the fragment reuse index /// is updated and will be used to perform remapping later. @@ -237,6 +244,7 @@ impl Default for CompactionOptions { num_threads: None, max_bytes_per_file: None, batch_size: None, + io_buffer_size: None, defer_index_remap: false, compaction_mode: None, enable_binary_copy: false, @@ -264,6 +272,7 @@ impl CompactionOptions { /// - `lance.compaction.materialize_deletions_threshold` /// - `lance.compaction.defer_index_remap` /// - `lance.compaction.batch_size` + /// - `lance.compaction.io_buffer_size` /// - `lance.compaction.compaction_mode` /// - `lance.compaction.binary_copy_read_batch_bytes` /// - `lance.compaction.max_source_fragments` @@ -347,6 +356,14 @@ impl CompactionOptions { )) })?); } + "io_buffer_size" => { + self.io_buffer_size = Some(value.parse().map_err(|_| { + Error::invalid_input(format!( + "Invalid value for {}: '{}' (expected a non-negative integer)", + key, value + )) + })?); + } "compaction_mode" => { self.compaction_mode = Some(CompactionMode::try_from(value.as_str())?); } @@ -1194,6 +1211,8 @@ async fn transform_blob_v2_batch( /// and preserve insertion order. /// - `batch_size`: Optional batch size; if provided, set it on the scanner to control /// read batching. +/// - `io_buffer_size`: Optional I/O buffer size in bytes; if provided, set it on the +/// scanner to control how much data is queued during reads. /// - `with_frags`: Whether to scan only the specified old fragments and force /// in-order reading. /// - `capture_row_ids`: When index remapping is needed, include and capture the @@ -1209,6 +1228,7 @@ async fn prepare_reader( dataset: &Dataset, fragments: &[Fragment], batch_size: Option, + io_buffer_size: Option, with_frags: bool, capture_row_ids: bool, ) -> Result<( @@ -1234,6 +1254,9 @@ async fn prepare_reader( if let Some(bs) = batch_size { scanner.batch_size(bs); } + if let Some(io_buffer_size) = io_buffer_size { + scanner.io_buffer_size(io_buffer_size); + } if with_frags { scanner .with_fragments(fragments.to_vec()) @@ -1515,6 +1538,7 @@ async fn rewrite_files( dataset.as_ref(), &fragments, options.batch_size, + options.io_buffer_size, true, needs_remapping, ) @@ -2636,6 +2660,57 @@ mod tests { assert_eq!(scanned_data, data); } + #[rstest] + #[tokio::test] + async fn test_compact_with_io_buffer_size( + #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)] + data_storage_version: LanceFileVersion, + ) { + // Compaction should succeed and produce correct results when an + // explicit io_buffer_size is provided via CompactionOptions. + let test_dir = TempStrDir::default(); + let test_uri = &test_dir; + + let data = sample_data(); + + // Create a table with 2 small fragments so there is something to compact. + let reader = RecordBatchIterator::new(vec![Ok(data.clone())], data.schema()); + let write_params = WriteParams { + max_rows_per_file: 5_000, + max_rows_per_group: 1_000, + data_storage_version: Some(data_storage_version), + ..Default::default() + }; + let mut dataset = Dataset::write(reader, test_uri, Some(write_params)) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 2); + + let options = CompactionOptions { + // A generous buffer so the read does not deadlock on large batches. + io_buffer_size: Some(256 * 1024 * 1024), + ..Default::default() + }; + let plan = plan_compaction(&dataset, &options).await.unwrap(); + assert_eq!(plan.tasks().len(), 1); + + let metrics = compact_files(&mut dataset, options, None).await.unwrap(); + assert_eq!(metrics.fragments_removed, 2); + assert_eq!(metrics.fragments_added, 1); + + // All rows are preserved after compaction. + let scanner = dataset.scan(); + let batches = scanner + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let scanned_data = concat_batches(&batches[0].schema(), &batches).unwrap(); + assert_eq!(scanned_data.num_rows(), data.num_rows()); + } + #[rstest] #[tokio::test] async fn test_compact_deletions( @@ -4683,6 +4758,10 @@ mod tests { "lance.compaction.batch_size".to_string(), "4096".to_string(), ), + ( + "lance.compaction.io_buffer_size".to_string(), + "1073741824".to_string(), + ), ( "lance.compaction.compaction_mode".to_string(), "try_binary_copy".to_string(), @@ -4701,6 +4780,7 @@ mod tests { assert!((opts.materialize_deletions_threshold - 0.25).abs() < f32::EPSILON); assert!(opts.defer_index_remap); assert_eq!(opts.batch_size, Some(4096)); + assert_eq!(opts.io_buffer_size, Some(1_073_741_824)); assert_eq!(opts.compaction_mode, Some(CompactionMode::TryBinaryCopy)); assert_eq!(opts.binary_copy_read_batch_bytes, Some(8_388_608)); } From 464f0bc2932dfc825c0b9cc5641e094414efe0db Mon Sep 17 00:00:00 2001 From: Yang Cen Date: Wed, 17 Jun 2026 00:15:16 +0800 Subject: [PATCH 08/15] perf(vector): vectorize RaBitQ top-k lower-bound pruning scan (#7243) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What The per-partition top-k scan of multi-bit IVF_RQ search (`accumulate_raw_query_multi_bit_topk_with_scratch`, Normal/Accurate modes) walked all `n` rows with a scalar lower-bound computation — per row: 4 bounds-checked loads, ~5 FLOPs, two compares, iterator and `Option` plumbing — even though ~99.9% of rows are pruned once the heap is tight. On dbpedia-openai-1M (1536d, num_bits=5, nprobes=24, k=10) this loop profiled at ~10–13% of query self time. This PR vectorizes the classification: - **Dense path** (`accumulate_topk_with_scratch`, rows `0..n`): new `bq::prune` kernels evaluate the lower bound and both pruning compares for 16 rows per call, returning bit masks. Mask-zero groups (the common case) are skipped whole; surviving lanes run the existing scalar rerank with live values. The `row_id` mapping is now only invoked for rows that reach the scalar tail, and no scratch buffer is needed — everything stays in registers. - **Sparse path** (prefiltered `accumulate_filtered_topk_with_scratch`): unchanged scalar loop. Kernels follow the `ex_dot` dispatch pattern: `#[target_feature]` AVX-512 and AVX2 implementations behind a `LazyLock` runtime-dispatched fn pointer, with a portable 16-wide fallback that LLVM auto-vectorizes (NEON is baseline on aarch64). `accumulate_distances_into_heap` (Fast-mode bypass) is left as is: it has no factor arrays and would need a separate kernel shape, and Fast mode doesn't take the gated path. ## Correctness The dense path is bit-identical to the scalar implementation, not just statistically equivalent: - The kernels keep the scalar operation order (multiplies and adds, **no FMA**), so the lower bounds match `raw_query_lower_bound` bit for bit, and comparisons use ordered-quiet GE (`_CMP_GE_OQ`) matching scalar `>=` (a NaN lower bound is never pruned). - The heap threshold snapshot taken at each 16-row group start can be stale, but the threshold only ever tightens, so the masks can only over-select survivors — and survivors are re-checked per row against live values. - Heap contents, processing order, and the `LANCE_RQ_PRUNE_STATS` counters are unchanged. ## Tests - `vector::bq::prune`: every available kernel (portable, AVX2, AVX-512, dispatched) against a per-lane scalar reference on random inputs, exact `>=` boundary ties, and NaN/±inf semantics. - `test_raw_query_multi_bit_topk_dense_matches_sparse`: differential test of the dense path against the unchanged sparse scalar path with crafted factor columns controlling lower bounds and exact distances — n ∈ {1, 15, 16, 17, 100, 4109} × k ∈ {1, 10, n+7} × bounds, distance orderings descending (constant heap churn), ascending (mass pruning), random, duplicates, and exact ties, with a second pass on the shared heap (the carried tight-threshold regime). Asserts identical heap contents (row ids + distance bit patterns) and the k-smallest-distances reference. - `cargo test -p lance-index --lib vector::bq` and `cargo test -p lance ivf_rq` pass on aarch64 (portable kernel) and on x86_64 with AVX-512 (GCP c4-standard-16). ## Benchmark New `RQ heap topk` bench (binary FastScan + pruning scan + exact rerank; 4096 rows, k=10, DIM=1536, num_bits=5, error factors present so gating is enabled), GCP c4-standard-16 (AVX-512), pinned core: | mode | before | after | change | |---|---|---|---| | normal | 70.7 µs | 64.9 µs | **−9.4%** (p = 0.00) | | accurate | 93.4 µs | 87.9 µs | **−5.7%** (p = 0.00) | The binary FastScan portion of the bench is unchanged; the delta is the pruning scan itself, matching the ~5–9% end-to-end win predicted from the profile. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Fable 5 --- rust/lance-index/benches/rq.rs | 131 +++++- rust/lance-index/src/vector/bq.rs | 1 + rust/lance-index/src/vector/bq/prune.rs | 527 ++++++++++++++++++++++ rust/lance-index/src/vector/bq/storage.rs | 511 ++++++++++++++++++--- 4 files changed, 1104 insertions(+), 66 deletions(-) create mode 100644 rust/lance-index/src/vector/bq/prune.rs diff --git a/rust/lance-index/benches/rq.rs b/rust/lance-index/benches/rq.rs index e29ce9c4695..72e0c49820d 100644 --- a/rust/lance-index/benches/rq.rs +++ b/rust/lance-index/benches/rq.rs @@ -512,9 +512,138 @@ fn ex_bulk_paths(c: &mut Criterion) { } } +/// Top-k accumulation through the gated raw-query multi-bit path: binary +/// FastScan, the per-row lower-bound pruning scan, and the exact rerank of +/// the surviving rows. Error factors are present so the gating is enabled. +fn heap_topk(c: &mut Criterion) { + use arrow_array::{ArrayRef, FixedSizeListArray, Float32Array, UInt8Array, UInt64Array}; + use lance_arrow::FixedSizeListArrayExt; + use lance_index::vector::ApproxMode; + use lance_index::vector::bq::transform::{ + ERROR_FACTORS_COLUMN, EX_ADD_FACTORS_COLUMN, EX_SCALE_FACTORS_COLUMN, + }; + use lance_index::vector::storage::DistanceCalculatorOptions; + use std::collections::BinaryHeap; + use std::sync::Arc; + + const TOPK_DIM: usize = 1536; + const TOPK_ROWS: usize = 4096; + const TOPK_K: usize = 10; + const NUM_BITS: u8 = 5; + let ex_bits = NUM_BITS - 1; + + let mut rng = SmallRng::seed_from_u64(99); + let rq = RabitQuantizer::new_with_rotation::( + NUM_BITS, + TOPK_DIM as i32, + RQRotationType::Fast, + ); + let metadata = rq.metadata(None); + + let code_len = TOPK_DIM / 8; + let binary_codes = (0..TOPK_ROWS * code_len) + .map(|_| rng.random()) + .collect::>(); + let ex_code_len = blocked_ex_code_bytes(TOPK_DIM, ex_bits); + let ex_codes = (0..TOPK_ROWS * ex_code_len) + .map(|_| rng.random()) + .collect::>(); + // Factor magnitudes chosen so the lower bounds spread mostly with the add + // factors; once the heap is full the threshold prunes the vast majority + // of rows, like a production multi-partition scan. + let mut rand_factors = |low: f32, high: f32| { + Arc::new(Float32Array::from( + (0..TOPK_ROWS) + .map(|_| rng.random_range(low..high)) + .collect::>(), + )) as ArrayRef + }; + let batch = arrow_array::RecordBatch::try_from_iter(vec![ + ( + ROW_ID, + Arc::new(UInt64Array::from_iter_values(0..TOPK_ROWS as u64)) as ArrayRef, + ), + ( + RABIT_CODE_COLUMN, + Arc::new( + FixedSizeListArray::try_new_from_values( + UInt8Array::from(binary_codes), + code_len as i32, + ) + .unwrap(), + ) as ArrayRef, + ), + (ADD_FACTORS_COLUMN, rand_factors(0.0, 1.0)), + (SCALE_FACTORS_COLUMN, rand_factors(0.0005, 0.0015)), + (ERROR_FACTORS_COLUMN, rand_factors(0.0, 0.01)), + ( + RABIT_BLOCKED_EX_CODE_COLUMN, + Arc::new( + FixedSizeListArray::try_new_from_values( + UInt8Array::from(ex_codes), + ex_code_len as i32, + ) + .unwrap(), + ) as ArrayRef, + ), + (EX_ADD_FACTORS_COLUMN, rand_factors(0.0, 1.0)), + (EX_SCALE_FACTORS_COLUMN, rand_factors(0.00003, 0.0001)), + ]) + .unwrap(); + let storage = + RabitQuantizationStorage::try_from_batch(batch, &metadata, DistanceType::L2, None).unwrap(); + let query: ArrayRef = Arc::new(Float32Array::from( + (0..TOPK_DIM) + .map(|_| rng.random_range(-1.0f32..1.0)) + .collect::>(), + )); + + for (label, approx_mode) in [ + ("normal", ApproxMode::Normal), + ("accurate", ApproxMode::Accurate), + ] { + let mut f32_scratch = Vec::new(); + let calc = storage.dist_calculator_with_scratch( + query.clone(), + 1.0, + None, + &mut f32_scratch, + DistanceCalculatorOptions { approx_mode }, + ); + let mut heap = BinaryHeap::with_capacity(TOPK_K + 1); + let mut dists = Vec::new(); + let mut u16_scratch = Vec::new(); + let mut u8_scratch = Vec::new(); + let mut u32_scratch = Vec::new(); + c.bench_function( + format!( + "RQ heap topk ({label}): num_bits={NUM_BITS}, DIM={TOPK_DIM}, rows={TOPK_ROWS}, k={TOPK_K}" + ) + .as_str(), + |b| { + b.iter(|| { + heap.clear(); + calc.accumulate_topk_with_scratch( + TOPK_K, + None, + None, + |id| id as u64, + &mut heap, + &mut dists, + &mut u16_scratch, + &mut u8_scratch, + &mut u32_scratch, + ); + black_box(heap.len()) + }) + }, + ); + } +} + criterion_group!( name=benches; config = Criterion::default().measurement_time(Duration::from_secs(10)); - targets = construct_dist_table, compute_distances, ex_dot_kernels, ex_code_storage_load, ex_bulk_paths); + targets = construct_dist_table, compute_distances, ex_dot_kernels, ex_code_storage_load, ex_bulk_paths, heap_topk); criterion_main!(benches); diff --git a/rust/lance-index/src/vector/bq.rs b/rust/lance-index/src/vector/bq.rs index ad013683214..7a47fa88d54 100644 --- a/rust/lance-index/src/vector/bq.rs +++ b/rust/lance-index/src/vector/bq.rs @@ -20,6 +20,7 @@ use crate::vector::quantizer::QuantizerBuildParams; pub mod builder; pub(crate) mod dist_table_quant; pub mod ex_dot; +pub mod prune; pub mod rotation; pub mod storage; pub mod transform; diff --git a/rust/lance-index/src/vector/bq/prune.rs b/rust/lance-index/src/vector/bq/prune.rs new file mode 100644 index 00000000000..e67ab6642b8 --- /dev/null +++ b/rust/lance-index/src/vector/bq/prune.rs @@ -0,0 +1,527 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! SIMD kernels for the RaBitQ top-k lower-bound pruning scan. +//! +//! Multi-bit IVF_RQ search gates the exact ex-code rerank with a per-row +//! distance lower bound: after the binary FastScan fills the per-row binary +//! inner products, every row of the partition is classified against the query +//! upper bound and the current top-k heap threshold, and only the survivors +//! (typically well under 1%) are reranked. The classification is the per-row +//! formula of `RabitDistCalculator::raw_query_lower_bound`: +//! +//! ```text +//! lower_bound = (binary_ip - 0.5 * sum_q) * scale_factor +//! + add_factor + query_factor +//! - error_factor * query_error +//! ``` +//! +//! These kernels evaluate the formula and both comparisons for +//! [`PRUNE_LANES`] rows at a time, returning bit masks instead of values so +//! the caller can skip whole groups (the overwhelmingly common case) and run +//! the existing scalar rerank only for the surviving lanes. +//! +//! Correctness contract: +//! +//! - The lower bound is computed with exactly the operation order of the +//! scalar helper — multiplies and adds, never FMA. A fused multiply-add +//! rounds differently, which could prune a row the scalar code would have +//! kept; with bit-identical lower bounds the masks reproduce the scalar +//! `>=` decisions exactly, keeping heap contents and prune-stats counters +//! unchanged. +//! - Comparisons use ordered-quiet GE predicates (`_CMP_GE_OQ`), matching +//! scalar `>=`: a NaN lower bound is never pruned and falls through to the +//! exact rerank. +//! - The heap threshold may be a stale snapshot (it only ever tightens); the +//! caller re-checks surviving lanes against live values, so a stale +//! threshold can only over-select survivors, never wrongly prune. + +use std::sync::LazyLock; + +/// Rows classified per kernel invocation. +pub const PRUNE_LANES: usize = 16; + +/// Per-query constants of the lower-bound formula, mirroring +/// `RabitDistCalculator::raw_query_lower_bound` term by term. +#[derive(Debug, Clone, Copy)] +pub struct LowerBoundTerms { + /// `0.5 * sum_q`, subtracted from the binary inner product. + pub half_sum_q: f32, + pub query_factor: f32, + pub query_error: f32, +} + +/// Classify [`PRUNE_LANES`] rows against the pruning bounds. +/// +/// Arguments are the per-row binary inner products, scale factors, add +/// factors, and error factors, followed by the formula constants, the query +/// upper bound, and the heap threshold (`None` while the heap is not full, +/// which disables the heap mask). +/// +/// Returns `(pruned_upper_bound, pruned_heap)` masks: bit `i` of +/// `pruned_upper_bound` is set when `lower_bound[i] >= upper_bound`, and bit +/// `i` of `pruned_heap` is set when the row is not already pruned by the +/// upper bound and `lower_bound[i] >= heap_threshold`. Surviving rows are the +/// zero bits of the OR of both masks. +pub type PruneMaskFn = fn( + &[f32; PRUNE_LANES], + &[f32; PRUNE_LANES], + &[f32; PRUNE_LANES], + &[f32; PRUNE_LANES], + LowerBoundTerms, + f32, + Option, +) -> (u16, u16); + +/// Resolve the prune-mask kernel for the running CPU once; the result can be +/// cached by the caller for per-partition use. +pub fn prune_mask_kernel() -> PruneMaskFn { + static KERNEL: LazyLock = LazyLock::new(select_prune_mask_kernel); + *KERNEL +} + +fn select_prune_mask_kernel() -> PruneMaskFn { + #[cfg(target_arch = "x86_64")] + { + if std::arch::is_x86_feature_detected!("avx512f") { + return x86::prune_masks_avx512_dispatch; + } + if std::arch::is_x86_feature_detected!("avx2") { + return x86::prune_masks_avx2_dispatch; + } + } + // On aarch64 the plain 16-wide loop auto-vectorizes to NEON (part of the + // baseline), so no dedicated kernel is needed. + prune_masks_portable +} + +/// Portable implementation; also the reference for the SIMD kernels. +fn prune_masks_portable( + dists: &[f32; PRUNE_LANES], + scale_factors: &[f32; PRUNE_LANES], + add_factors: &[f32; PRUNE_LANES], + error_factors: &[f32; PRUNE_LANES], + terms: LowerBoundTerms, + upper_bound: f32, + heap_threshold: Option, +) -> (u16, u16) { + let mut lower_bounds = [0.0f32; PRUNE_LANES]; + for lane in 0..PRUNE_LANES { + lower_bounds[lane] = ((dists[lane] - terms.half_sum_q) * scale_factors[lane] + + add_factors[lane] + + terms.query_factor) + - error_factors[lane] * terms.query_error; + } + let mut pruned_upper_bound = 0u16; + for (lane, lower_bound) in lower_bounds.iter().enumerate() { + pruned_upper_bound |= u16::from(*lower_bound >= upper_bound) << lane; + } + let mut pruned_heap = 0u16; + if let Some(threshold) = heap_threshold { + for (lane, lower_bound) in lower_bounds.iter().enumerate() { + pruned_heap |= u16::from(*lower_bound >= threshold) << lane; + } + pruned_heap &= !pruned_upper_bound; + } + (pruned_upper_bound, pruned_heap) +} + +#[cfg(target_arch = "x86_64")] +mod x86 { + use super::{LowerBoundTerms, PRUNE_LANES}; + use std::arch::x86_64::*; + + /// Lower bounds for 8 lanes with the scalar operation order (no FMA). + #[inline] + #[target_feature(enable = "avx")] + fn lower_bounds_avx( + dists: __m256, + scale_factors: __m256, + add_factors: __m256, + error_factors: __m256, + half_sum_q: __m256, + query_factor: __m256, + query_error: __m256, + ) -> __m256 { + let binary_distance = _mm256_add_ps( + _mm256_add_ps( + _mm256_mul_ps(_mm256_sub_ps(dists, half_sum_q), scale_factors), + add_factors, + ), + query_factor, + ); + _mm256_sub_ps(binary_distance, _mm256_mul_ps(error_factors, query_error)) + } + + #[inline] + #[target_feature(enable = "avx")] + fn ge_mask_avx(lower_bounds_lo: __m256, lower_bounds_hi: __m256, bound: f32) -> u16 { + let bound = _mm256_set1_ps(bound); + let lo = _mm256_movemask_ps(_mm256_cmp_ps::<_CMP_GE_OQ>(lower_bounds_lo, bound)); + let hi = _mm256_movemask_ps(_mm256_cmp_ps::<_CMP_GE_OQ>(lower_bounds_hi, bound)); + (lo | (hi << 8)) as u16 + } + + #[target_feature(enable = "avx2")] + unsafe fn prune_masks_avx2( + dists: &[f32; PRUNE_LANES], + scale_factors: &[f32; PRUNE_LANES], + add_factors: &[f32; PRUNE_LANES], + error_factors: &[f32; PRUNE_LANES], + terms: LowerBoundTerms, + upper_bound: f32, + heap_threshold: Option, + ) -> (u16, u16) { + let half_sum_q = _mm256_set1_ps(terms.half_sum_q); + let query_factor = _mm256_set1_ps(terms.query_factor); + let query_error = _mm256_set1_ps(terms.query_error); + // SAFETY: the array references guarantee 16 readable floats each. + let lower_bounds_lo = unsafe { + lower_bounds_avx( + _mm256_loadu_ps(dists.as_ptr()), + _mm256_loadu_ps(scale_factors.as_ptr()), + _mm256_loadu_ps(add_factors.as_ptr()), + _mm256_loadu_ps(error_factors.as_ptr()), + half_sum_q, + query_factor, + query_error, + ) + }; + let lower_bounds_hi = unsafe { + lower_bounds_avx( + _mm256_loadu_ps(dists.as_ptr().add(8)), + _mm256_loadu_ps(scale_factors.as_ptr().add(8)), + _mm256_loadu_ps(add_factors.as_ptr().add(8)), + _mm256_loadu_ps(error_factors.as_ptr().add(8)), + half_sum_q, + query_factor, + query_error, + ) + }; + let pruned_upper_bound = ge_mask_avx(lower_bounds_lo, lower_bounds_hi, upper_bound); + let pruned_heap = match heap_threshold { + Some(threshold) => { + ge_mask_avx(lower_bounds_lo, lower_bounds_hi, threshold) & !pruned_upper_bound + } + None => 0, + }; + (pruned_upper_bound, pruned_heap) + } + + pub(super) fn prune_masks_avx2_dispatch( + dists: &[f32; PRUNE_LANES], + scale_factors: &[f32; PRUNE_LANES], + add_factors: &[f32; PRUNE_LANES], + error_factors: &[f32; PRUNE_LANES], + terms: LowerBoundTerms, + upper_bound: f32, + heap_threshold: Option, + ) -> (u16, u16) { + // SAFETY: only selected when AVX2 was detected. + unsafe { + prune_masks_avx2( + dists, + scale_factors, + add_factors, + error_factors, + terms, + upper_bound, + heap_threshold, + ) + } + } + + #[target_feature(enable = "avx512f")] + unsafe fn prune_masks_avx512( + dists: &[f32; PRUNE_LANES], + scale_factors: &[f32; PRUNE_LANES], + add_factors: &[f32; PRUNE_LANES], + error_factors: &[f32; PRUNE_LANES], + terms: LowerBoundTerms, + upper_bound: f32, + heap_threshold: Option, + ) -> (u16, u16) { + // SAFETY: the array references guarantee 16 readable floats each. + let (dists, scale_factors, add_factors, error_factors) = unsafe { + ( + _mm512_loadu_ps(dists.as_ptr()), + _mm512_loadu_ps(scale_factors.as_ptr()), + _mm512_loadu_ps(add_factors.as_ptr()), + _mm512_loadu_ps(error_factors.as_ptr()), + ) + }; + let binary_distance = _mm512_add_ps( + _mm512_add_ps( + _mm512_mul_ps( + _mm512_sub_ps(dists, _mm512_set1_ps(terms.half_sum_q)), + scale_factors, + ), + add_factors, + ), + _mm512_set1_ps(terms.query_factor), + ); + let lower_bounds = _mm512_sub_ps( + binary_distance, + _mm512_mul_ps(error_factors, _mm512_set1_ps(terms.query_error)), + ); + let pruned_upper_bound = + _mm512_cmp_ps_mask::<_CMP_GE_OQ>(lower_bounds, _mm512_set1_ps(upper_bound)); + let pruned_heap = match heap_threshold { + Some(threshold) => { + _mm512_cmp_ps_mask::<_CMP_GE_OQ>(lower_bounds, _mm512_set1_ps(threshold)) + & !pruned_upper_bound + } + None => 0, + }; + (pruned_upper_bound, pruned_heap) + } + + pub(super) fn prune_masks_avx512_dispatch( + dists: &[f32; PRUNE_LANES], + scale_factors: &[f32; PRUNE_LANES], + add_factors: &[f32; PRUNE_LANES], + error_factors: &[f32; PRUNE_LANES], + terms: LowerBoundTerms, + upper_bound: f32, + heap_threshold: Option, + ) -> (u16, u16) { + // SAFETY: only selected when AVX-512F was detected. + unsafe { + prune_masks_avx512( + dists, + scale_factors, + add_factors, + error_factors, + terms, + upper_bound, + heap_threshold, + ) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rand::rngs::SmallRng; + use rand::{Rng, SeedableRng}; + + fn available_kernels() -> Vec<(&'static str, PruneMaskFn)> { + // `mut` is only exercised on x86_64 where extra kernels may be pushed. + #[allow(unused_mut)] + let mut kernels = vec![ + ("portable", prune_masks_portable as PruneMaskFn), + ("dispatched", prune_mask_kernel()), + ]; + #[cfg(target_arch = "x86_64")] + { + if std::arch::is_x86_feature_detected!("avx2") { + kernels.push(("avx2", x86::prune_masks_avx2_dispatch)); + } + if std::arch::is_x86_feature_detected!("avx512f") { + kernels.push(("avx512", x86::prune_masks_avx512_dispatch)); + } + } + kernels + } + + /// Per-lane reference mirroring `raw_query_lower_bound` and the scalar + /// pruning checks of the top-k scan. + fn reference_masks( + dists: &[f32; PRUNE_LANES], + scale_factors: &[f32; PRUNE_LANES], + add_factors: &[f32; PRUNE_LANES], + error_factors: &[f32; PRUNE_LANES], + terms: LowerBoundTerms, + upper_bound: f32, + heap_threshold: Option, + ) -> (u16, u16) { + let mut pruned_upper_bound = 0u16; + let mut pruned_heap = 0u16; + for lane in 0..PRUNE_LANES { + let lower_bound = (dists[lane] - terms.half_sum_q) * scale_factors[lane] + + add_factors[lane] + + terms.query_factor + - error_factors[lane] * terms.query_error; + if lower_bound >= upper_bound { + pruned_upper_bound |= 1 << lane; + } else if heap_threshold.is_some_and(|threshold| lower_bound >= threshold) { + pruned_heap |= 1 << lane; + } + } + (pruned_upper_bound, pruned_heap) + } + + #[allow(clippy::too_many_arguments)] + fn assert_kernels_match_reference( + dists: &[f32; PRUNE_LANES], + scale_factors: &[f32; PRUNE_LANES], + add_factors: &[f32; PRUNE_LANES], + error_factors: &[f32; PRUNE_LANES], + terms: LowerBoundTerms, + upper_bound: f32, + heap_threshold: Option, + case: &str, + ) { + let expected = reference_masks( + dists, + scale_factors, + add_factors, + error_factors, + terms, + upper_bound, + heap_threshold, + ); + for (name, kernel) in available_kernels() { + let actual = kernel( + dists, + scale_factors, + add_factors, + error_factors, + terms, + upper_bound, + heap_threshold, + ); + assert_eq!( + actual, expected, + "kernel={name} case={case}: masks {actual:04x?} != {expected:04x?}" + ); + } + } + + #[test] + fn test_prune_masks_match_reference_on_random_inputs() { + let mut rng = SmallRng::seed_from_u64(42); + for round in 0..200 { + let mut dists = [0.0f32; PRUNE_LANES]; + let mut scale_factors = [0.0f32; PRUNE_LANES]; + let mut add_factors = [0.0f32; PRUNE_LANES]; + let mut error_factors = [0.0f32; PRUNE_LANES]; + for lane in 0..PRUNE_LANES { + dists[lane] = rng.random_range(-100.0f32..100.0); + scale_factors[lane] = rng.random_range(-2.0f32..2.0); + add_factors[lane] = rng.random_range(-10.0f32..10.0); + error_factors[lane] = rng.random_range(0.0f32..5.0); + } + let terms = LowerBoundTerms { + half_sum_q: rng.random_range(-50.0f32..50.0), + query_factor: rng.random_range(-10.0f32..10.0), + query_error: rng.random_range(0.0f32..2.0), + }; + let upper_bound = rng.random_range(-50.0f32..50.0); + let heap_threshold = if round % 3 == 0 { + None + } else { + Some(rng.random_range(-50.0f32..50.0)) + }; + assert_kernels_match_reference( + &dists, + &scale_factors, + &add_factors, + &error_factors, + terms, + upper_bound, + heap_threshold, + &format!("random round {round}"), + ); + } + } + + #[test] + fn test_prune_masks_exact_boundaries() { + // With scale=1, err=0, half_sum_q=0, query_factor=0 the lower bound + // is the input itself, so bounds can be placed exactly on lanes. + let dists: [f32; PRUNE_LANES] = std::array::from_fn(|lane| lane as f32); + let scale_factors = [1.0f32; PRUNE_LANES]; + let add_factors = [0.0f32; PRUNE_LANES]; + let error_factors = [0.0f32; PRUNE_LANES]; + let terms = LowerBoundTerms { + half_sum_q: 0.0, + query_factor: 0.0, + query_error: 1.0, + }; + // Equality must prune (scalar uses `>=`): lanes 3.. hit the upper + // bound, lanes 1..3 hit only the heap threshold. + let (pruned_upper_bound, pruned_heap) = prune_masks_portable( + &dists, + &scale_factors, + &add_factors, + &error_factors, + terms, + 3.0, + Some(1.0), + ); + assert_eq!(pruned_upper_bound, 0xfff8); + assert_eq!(pruned_heap, 0x0006); + assert_kernels_match_reference( + &dists, + &scale_factors, + &add_factors, + &error_factors, + terms, + 3.0, + Some(1.0), + "exact boundaries", + ); + // No heap threshold: only the upper-bound mask is set. + assert_kernels_match_reference( + &dists, + &scale_factors, + &add_factors, + &error_factors, + terms, + 3.0, + None, + "no heap threshold", + ); + } + + #[test] + fn test_prune_masks_nan_and_infinity_semantics() { + let mut dists = [0.0f32; PRUNE_LANES]; + dists[0] = f32::NAN; + dists[1] = f32::INFINITY; + dists[2] = f32::NEG_INFINITY; + dists[3] = 1.0; + let mut scale_factors = [1.0f32; PRUNE_LANES]; + scale_factors[4] = f32::NAN; + let add_factors = [0.0f32; PRUNE_LANES]; + let mut error_factors = [0.0f32; PRUNE_LANES]; + error_factors[5] = f32::INFINITY; + let terms = LowerBoundTerms { + half_sum_q: 0.0, + query_factor: 0.0, + query_error: 1.0, + }; + for (upper_bound, heap_threshold) in [ + (0.5, Some(0.0)), + (f32::INFINITY, Some(f32::NEG_INFINITY)), + (f32::NAN, Some(f32::NAN)), + (0.5, None), + ] { + assert_kernels_match_reference( + &dists, + &scale_factors, + &add_factors, + &error_factors, + terms, + upper_bound, + heap_threshold, + &format!("special values ub={upper_bound} thr={heap_threshold:?}"), + ); + } + // NaN lower bounds (lane 0 via a NaN binary inner product, lane 4 via + // a NaN scale factor) must never be pruned by either mask. + let (pruned_upper_bound, pruned_heap) = prune_masks_portable( + &dists, + &scale_factors, + &add_factors, + &error_factors, + terms, + 0.5, + Some(0.0), + ); + assert_eq!(pruned_upper_bound & 0b1_0001, 0); + assert_eq!(pruned_heap & 0b1_0001, 0); + } +} diff --git a/rust/lance-index/src/vector/bq/storage.rs b/rust/lance-index/src/vector/bq/storage.rs index ef88500a7b7..9c355d26960 100644 --- a/rust/lance-index/src/vector/bq/storage.rs +++ b/rust/lance-index/src/vector/bq/storage.rs @@ -17,7 +17,7 @@ use arrow_array::{ use arrow_schema::{DataType, Field, SchemaRef}; use async_trait::async_trait; use bytes::{Bytes, BytesMut}; -use itertools::Itertools; +use itertools::{Itertools, izip}; use lance_arrow::{ArrowFloatType, FixedSizeListArrayExt, FloatArray, RecordBatchExt}; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, ROW_ID, Result}; @@ -48,6 +48,7 @@ use crate::vector::bq::ex_dot::{ EX_DOT_BLOCK_DIMS, ExDotFn, blocked_ex_code_bytes, ex_dot_kernel, pad_query_into, padded_query_len, repack_sequential_row, sequential_matches_blocked, }; +use crate::vector::bq::prune::{LowerBoundTerms, PRUNE_LANES, prune_mask_kernel}; use crate::vector::bq::rotation::{apply_fast_rotation, apply_fast_rotation_in_place}; use crate::vector::bq::transform::{ ADD_FACTORS_COLUMN, ERROR_FACTORS_COLUMN, EX_ADD_FACTORS_COLUMN, EX_SCALE_FACTORS_COLUMN, @@ -136,16 +137,28 @@ fn emit_rabit_prune_stats(message: &str) { ); } -fn record_rabit_prune_stats( +/// Per-scan tallies of the raw-query lower-bound gating, reported through +/// `record_rabit_prune_stats`. +#[derive(Default)] +struct RabitPruneCounters { candidates: usize, pruned_upper_bound: usize, pruned_heap: usize, exact: usize, exact_rejected: usize, -) { +} + +fn record_rabit_prune_stats(counters: &RabitPruneCounters) { if !rabit_prune_stats_enabled() { return; } + let RabitPruneCounters { + candidates, + pruned_upper_bound, + pruned_heap, + exact, + exact_rejected, + } = *counters; let stats = RABIT_PRUNE_STATS.get_or_init(RabitPruneStats::default); let calls = stats.calls.fetch_add(1, Ordering::Relaxed) + 1; @@ -798,6 +811,20 @@ struct RabitDistCalculatorParts<'a> { approx_mode: ApproxMode, } +/// Loop-invariant inputs of the raw-query multi-bit top-k scans: the row +/// count, the resolved ex-code state for exact reranking, and the query +/// bounds. +struct RawQueryTopkContext<'a> { + n: usize, + k: usize, + ex_bits: u8, + ex_codes: &'a [u8], + ex_add_factors: &'a [f32], + ex_scale_factors: &'a [f32], + query_lower_bound: f32, + query_upper_bound: f32, +} + /// Pick the query slice the ex-dot kernels consume: the rotated query itself /// when the dim is block-aligned, otherwise a zero-padded copy. fn kernel_query<'a>(rotated_query: &'a [f32], padded: &'a [f32]) -> &'a [f32] { @@ -1212,26 +1239,27 @@ impl<'a> RabitDistCalculator<'a> { full_dot * ex_scale_factors[id] + ex_add_factors[id] + self.query_factor } + /// Compute the binary inner products into `dists` and resolve the inputs + /// shared by the raw-query multi-bit top-k scans. Returns `None` when the + /// partition has no rows. #[allow(clippy::too_many_arguments)] - fn accumulate_raw_query_multi_bit_topk_with_scratch( + fn raw_query_multi_bit_topk_context( &self, k: usize, lower_bound: Option, upper_bound: Option, - row_ids: impl Iterator, - res: &mut BinaryHeap>, dists: &mut Vec, quantized_dists: &mut Vec, quantized_dists_table: &mut Vec, hacc_quantized_dists: &mut Vec, - ) { + ) -> Option> { let code_len = rabit_binary_code_bytes(self.dim); let n = self.codes.len() / code_len; if n == 0 { dists.clear(); quantized_dists.clear(); hacc_quantized_dists.clear(); - return; + return None; } self.binary_distances_with_scratch( @@ -1243,74 +1271,233 @@ impl<'a> RabitDistCalculator<'a> { hacc_quantized_dists, ); - let ex_bits = self.num_bits - 1; - let ex_codes = self - .ex_codes - .expect("raw-query multi-bit RQ requires ex codes"); - let ex_add_factors = self - .ex_add_factors - .expect("raw-query multi-bit RQ requires ex add factors"); - let ex_scale_factors = self - .ex_scale_factors - .expect("raw-query multi-bit RQ requires ex scale factors"); - let query_lower_bound = lower_bound.unwrap_or(f32::MIN); - let query_upper_bound = upper_bound.unwrap_or(f32::MAX); + Some(RawQueryTopkContext { + n, + k, + ex_bits: self.num_bits - 1, + ex_codes: self + .ex_codes + .expect("raw-query multi-bit RQ requires ex codes"), + ex_add_factors: self + .ex_add_factors + .expect("raw-query multi-bit RQ requires ex add factors"), + ex_scale_factors: self + .ex_scale_factors + .expect("raw-query multi-bit RQ requires ex scale factors"), + query_lower_bound: lower_bound.unwrap_or(f32::MIN), + query_upper_bound: upper_bound.unwrap_or(f32::MAX), + }) + } + + /// Process one candidate row given its lower bound: the bound checks, + /// the exact rerank, and the heap update shared by the sparse scan and + /// the dense scan's surviving lanes and tail. + #[inline] + #[allow(clippy::too_many_arguments)] + fn accumulate_raw_query_multi_bit_row( + &self, + ctx: &RawQueryTopkContext<'_>, + id: usize, + row_id: u64, + binary_ip: f32, + raw_lower_bound: f32, + res: &mut BinaryHeap>, + max_dist: &mut Option, + counters: &mut RabitPruneCounters, + ) { + if raw_lower_bound >= ctx.query_upper_bound { + counters.pruned_upper_bound += 1; + return; + } + if res.len() >= ctx.k && max_dist.is_some_and(|max_dist| raw_lower_bound >= max_dist.0) { + counters.pruned_heap += 1; + return; + } + + counters.exact += 1; + let dist = self.raw_query_multi_bit_exact_distance( + id, + binary_ip, + ctx.ex_bits, + ctx.ex_codes, + ctx.ex_add_factors, + ctx.ex_scale_factors, + ); + if dist < ctx.query_lower_bound || dist >= ctx.query_upper_bound { + counters.exact_rejected += 1; + return; + } + let dist = OrderedFloat(dist); + if res.len() < ctx.k { + res.push(OrderedNode::new(row_id, dist)); + if res.len() == ctx.k { + *max_dist = res.peek().map(|node| node.dist); + } + } else if max_dist.is_some_and(|max_dist| max_dist > dist) { + res.pop(); + res.push(OrderedNode::new(row_id, dist)); + *max_dist = res.peek().map(|node| node.dist); + } + } + + #[allow(clippy::too_many_arguments)] + fn accumulate_raw_query_multi_bit_topk_with_scratch( + &self, + k: usize, + lower_bound: Option, + upper_bound: Option, + row_ids: impl Iterator, + res: &mut BinaryHeap>, + dists: &mut Vec, + quantized_dists: &mut Vec, + quantized_dists_table: &mut Vec, + hacc_quantized_dists: &mut Vec, + ) { + let Some(ctx) = self.raw_query_multi_bit_topk_context( + k, + lower_bound, + upper_bound, + dists, + quantized_dists, + quantized_dists_table, + hacc_quantized_dists, + ) else { + return; + }; let mut max_dist = res.peek().map(|node| node.dist); - let mut candidates = 0; - let mut pruned_upper_bound = 0; - let mut pruned_heap = 0; - let mut exact = 0; - let mut exact_rejected = 0; + let mut counters = RabitPruneCounters::default(); for (id, row_id) in row_ids { let Some(binary_ip) = dists.get(id).copied() else { continue; }; - candidates += 1; + counters.candidates += 1; let Some(raw_lower_bound) = self.raw_query_lower_bound(id, binary_ip) else { continue; }; - if raw_lower_bound >= query_upper_bound { - pruned_upper_bound += 1; - continue; - } - if res.len() >= k && max_dist.is_some_and(|max_dist| raw_lower_bound >= max_dist.0) { - pruned_heap += 1; - continue; + self.accumulate_raw_query_multi_bit_row( + &ctx, + id, + row_id, + binary_ip, + raw_lower_bound, + res, + &mut max_dist, + &mut counters, + ); + } + record_rabit_prune_stats(&counters); + } + + /// Top-k scan over all rows `0..n` in order: classify [`PRUNE_LANES`] + /// rows at a time with the SIMD lower-bound kernel and run the scalar + /// rerank only for the surviving lanes. + #[allow(clippy::too_many_arguments)] + fn accumulate_raw_query_multi_bit_topk_dense_with_scratch( + &self, + k: usize, + lower_bound: Option, + upper_bound: Option, + row_id: impl Fn(u32) -> u64, + res: &mut BinaryHeap>, + dists: &mut Vec, + quantized_dists: &mut Vec, + quantized_dists_table: &mut Vec, + hacc_quantized_dists: &mut Vec, + ) { + let Some(ctx) = self.raw_query_multi_bit_topk_context( + k, + lower_bound, + upper_bound, + dists, + quantized_dists, + quantized_dists_table, + hacc_quantized_dists, + ) else { + return; + }; + let dists = dists.as_slice(); + debug_assert_eq!(dists.len(), ctx.n); + let scale_factors = &self.scale_factors[..ctx.n]; + let add_factors = &self.add_factors[..ctx.n]; + let error_factors = &self + .error_factors + .expect("raw-query lower-bound gating requires error factors")[..ctx.n]; + // Same expression as `raw_query_lower_bound` with `error_factors` + // already resolved; the masks below match it bit for bit. + let lower_bound_of = |id: usize, binary_ip: f32| { + self.raw_query_binary_distance(id, binary_ip) - error_factors[id] * self.query_error + }; + let terms = LowerBoundTerms { + half_sum_q: 0.5 * self.sum_q, + query_factor: self.query_factor, + query_error: self.query_error, + }; + let prune_masks = prune_mask_kernel(); + let mut max_dist = res.peek().map(|node| node.dist); + let mut counters = RabitPruneCounters::default(); + + let (dist_groups, dist_tail) = dists.as_chunks::(); + let (scale_groups, _) = scale_factors.as_chunks::(); + let (add_groups, _) = add_factors.as_chunks::(); + let (error_groups, _) = error_factors.as_chunks::(); + for (group, (dist16, scale16, add16, error16)) in + izip!(dist_groups, scale_groups, add_groups, error_groups).enumerate() + { + counters.candidates += PRUNE_LANES; + // The heap threshold only ever tightens, so this group-start + // snapshot can only over-select survivors (which the per-row + // processing below re-checks against live values), never prune a + // row the scalar scan would have kept. + let heap_threshold = (res.len() >= ctx.k) + .then(|| max_dist.map(|max_dist| max_dist.0)) + .flatten(); + let (pruned_upper_bound, pruned_heap) = prune_masks( + dist16, + scale16, + add16, + error16, + terms, + ctx.query_upper_bound, + heap_threshold, + ); + counters.pruned_upper_bound += pruned_upper_bound.count_ones() as usize; + counters.pruned_heap += pruned_heap.count_ones() as usize; + let mut survivors = !(pruned_upper_bound | pruned_heap); + while survivors != 0 { + let lane = survivors.trailing_zeros() as usize; + survivors &= survivors - 1; + let id = group * PRUNE_LANES + lane; + let binary_ip = dists[id]; + self.accumulate_raw_query_multi_bit_row( + &ctx, + id, + row_id(id as u32), + binary_ip, + lower_bound_of(id, binary_ip), + res, + &mut max_dist, + &mut counters, + ); } + } - exact += 1; - let dist = self.raw_query_multi_bit_exact_distance( + let tail_start = ctx.n - dist_tail.len(); + for (offset, binary_ip) in dist_tail.iter().copied().enumerate() { + let id = tail_start + offset; + counters.candidates += 1; + self.accumulate_raw_query_multi_bit_row( + &ctx, id, + row_id(id as u32), binary_ip, - ex_bits, - ex_codes, - ex_add_factors, - ex_scale_factors, + lower_bound_of(id, binary_ip), + res, + &mut max_dist, + &mut counters, ); - if dist < query_lower_bound || dist >= query_upper_bound { - exact_rejected += 1; - continue; - } - let dist = OrderedFloat(dist); - if res.len() < k { - res.push(OrderedNode::new(row_id, dist)); - if res.len() == k { - max_dist = res.peek().map(|node| node.dist); - } - } else if max_dist.is_some_and(|max_dist| max_dist > dist) { - res.pop(); - res.push(OrderedNode::new(row_id, dist)); - max_dist = res.peek().map(|node| node.dist); - } } - record_rabit_prune_stats( - candidates, - pruned_upper_bound, - pruned_heap, - exact, - exact_rejected, - ); + record_rabit_prune_stats(&counters); } fn raw_query_lower_bound_gating_disabled_reason(&self) -> Option<&'static str> { @@ -1717,13 +1904,11 @@ impl DistCalculator for RabitDistCalculator<'_> { return; } - let code_len = rabit_binary_code_bytes(self.dim); - let n = self.codes.len() / code_len; - self.accumulate_raw_query_multi_bit_topk_with_scratch( + self.accumulate_raw_query_multi_bit_topk_dense_with_scratch( k, lower_bound, upper_bound, - (0..n).map(|id| (id, row_id(id as u32))), + row_id, res, dists, quantized_dists, @@ -2526,6 +2711,8 @@ mod tests { use arrow_array::{ArrayRef, Float32Array, Float64Array, UInt64Array}; use lance_core::ROW_ID; use lance_linalg::distance::DistanceType; + use rand::rngs::SmallRng; + use rand::{Rng, SeedableRng}; use rstest::rstest; use crate::vector::bq::{RQRotationType, builder::RabitQuantizer}; @@ -3741,6 +3928,200 @@ mod tests { } } + /// Inputs crafted so the top-k scan outcomes are fully determined by the + /// factor columns: with zero scale factors, a zero query factor, and a + /// query error of one, the lower bound is + /// `add_factors[id] - error_factors[id]`, and with zero ex scale factors + /// the exact distance is `ex_add_factors[id]`, regardless of the random + /// codes and query. + struct CraftedTopkData { + codes: Vec, + ex_codes: Vec, + dist_table: Vec, + ex_query: Vec, + scale_factors: Vec, + add_factors: Vec, + error_factors: Vec, + ex_scale_factors: Vec, + ex_add_factors: Vec, + } + + const CRAFTED_TOPK_DIM: usize = 64; + const CRAFTED_TOPK_NUM_BITS: u8 = 5; + + impl CraftedTopkData { + fn new( + exact_dists: &[f32], + lower_bound_margins: &[f32], + error_factors: Vec, + rng: &mut SmallRng, + ) -> Self { + let n = exact_dists.len(); + let code_len = rabit_binary_code_bytes(CRAFTED_TOPK_DIM); + let ex_code_len = blocked_ex_code_bytes(CRAFTED_TOPK_DIM, CRAFTED_TOPK_NUM_BITS - 1); + let add_factors = izip!(exact_dists, lower_bound_margins, &error_factors) + .map(|(dist, margin, error)| dist - margin + error) + .collect(); + Self { + codes: (0..n * code_len).map(|_| rng.random()).collect(), + ex_codes: (0..n * ex_code_len).map(|_| rng.random()).collect(), + dist_table: (0..CRAFTED_TOPK_DIM * 4) + .map(|_| rng.random_range(-1.0f32..1.0)) + .collect(), + ex_query: (0..CRAFTED_TOPK_DIM) + .map(|_| rng.random_range(-1.0f32..1.0)) + .collect(), + scale_factors: vec![0.0; n], + add_factors, + error_factors, + ex_scale_factors: vec![0.0; n], + ex_add_factors: exact_dists.to_vec(), + } + } + + fn calculator(&self, approx_mode: ApproxMode) -> RabitDistCalculator<'_> { + RabitDistCalculator::new( + CRAFTED_TOPK_DIM, + CRAFTED_TOPK_NUM_BITS, + RabitQueryEstimator::RawQuery, + Cow::Borrowed(self.dist_table.as_slice()), + Cow::Borrowed(self.ex_query.as_slice()), + 0.7, + &self.codes, + Some(&self.ex_codes), + blocked_ex_code_bytes(CRAFTED_TOPK_DIM, CRAFTED_TOPK_NUM_BITS - 1), + &self.add_factors, + &self.scale_factors, + Some(&self.error_factors), + Some(&self.ex_add_factors), + Some(&self.ex_scale_factors), + None, + 0.0, + 1.0, + approx_mode, + ) + } + } + + fn canonical_heap_rows(heap: BinaryHeap>) -> Vec<(u32, u64)> { + let mut rows = heap + .into_iter() + .map(|node| (node.dist.0.to_bits(), node.id)) + .collect::>(); + rows.sort_unstable(); + rows + } + + /// The dense (SIMD-pruned) scan must reproduce the sparse scalar scan + /// exactly: identical heap contents including row ids, and the k smallest + /// in-bounds exact distances overall. + #[rstest] + fn test_raw_query_multi_bit_topk_dense_matches_sparse( + #[values(ApproxMode::Normal, ApproxMode::Accurate)] approx_mode: ApproxMode, + #[values("descending", "ascending", "random", "duplicates", "duplicate_ties")] + ordering: &str, + ) { + for n in [1usize, 15, 16, 17, 100, 4109] { + let mut rng = SmallRng::seed_from_u64(n as u64 * 31 + ordering.len() as u64); + let exact_dists: Vec = match ordering { + // Improving rows force constant heap updates. + "descending" => (0..n).map(|id| (n - id) as f32).collect(), + // Worsening rows force mass pruning, the common regime. + "ascending" => (0..n).map(|id| id as f32).collect(), + "random" => (0..n).map(|_| rng.random_range(0.0..n as f32)).collect(), + "duplicates" => (0..n).map(|id| (id % 7) as f32).collect(), + // Lower bound equals the distance, so heap-threshold and + // upper-bound comparisons hit exact `>=` ties. + "duplicate_ties" => (0..n).map(|id| (id % 5) as f32).collect(), + _ => unreachable!(), + }; + let (margins, error_factors) = if ordering == "duplicate_ties" { + (vec![0.0; n], vec![0.0; n]) + } else if ordering == "random" { + ( + (0..n).map(|_| rng.random_range(0.0f32..2.0)).collect(), + (0..n).map(|_| rng.random_range(0.0f32..1.0)).collect(), + ) + } else { + ( + vec![1.0; n], + (0..n).map(|_| rng.random_range(0.0f32..1.0)).collect(), + ) + }; + let data = CraftedTopkData::new(&exact_dists, &margins, error_factors, &mut rng); + let calc = data.calculator(approx_mode); + assert!( + calc.raw_query_lower_bound_gating_disabled_reason() + .is_none() + ); + + let max_dist = exact_dists.iter().fold(0.0f32, |acc, dist| acc.max(*dist)); + for k in [1usize, 10, n + 7] { + for bounds in [(None, None), (Some(max_dist * 0.25), Some(max_dist * 0.7))] { + let (lower_bound, upper_bound) = bounds; + let mut dense_heap = BinaryHeap::new(); + let mut sparse_heap = BinaryHeap::new(); + let mut dists = Vec::new(); + let mut u16_scratch = Vec::new(); + let mut u8_scratch = Vec::new(); + let mut u32_scratch = Vec::new(); + // Two passes sharing the heap, as IVF partition probing + // does: the second pass starts with a full, tight heap. + for pass in 0..2u64 { + let offset = pass * n as u64; + calc.accumulate_topk_with_scratch( + k, + lower_bound, + upper_bound, + |id| id as u64 + offset, + &mut dense_heap, + &mut dists, + &mut u16_scratch, + &mut u8_scratch, + &mut u32_scratch, + ); + calc.accumulate_filtered_topk_with_scratch( + k, + lower_bound, + upper_bound, + (0..n as u32).map(|id| (id, id as u64 + offset)), + |_| true, + &mut sparse_heap, + &mut dists, + &mut u16_scratch, + &mut u8_scratch, + &mut u32_scratch, + ); + } + let dense = canonical_heap_rows(dense_heap); + let sparse = canonical_heap_rows(sparse_heap); + assert_eq!( + dense, sparse, + "ordering={ordering} n={n} k={k} bounds={bounds:?} mode={approx_mode:?}" + ); + + // The distance multiset must be the k smallest in-bounds + // distances over both passes. Row ids are not compared: + // evictions among tied maxima depend on heap layout. + let query_lower_bound = lower_bound.unwrap_or(f32::MIN); + let query_upper_bound = upper_bound.unwrap_or(f32::MAX); + let mut expected = (0..2 * n) + .map(|row| exact_dists[row % n]) + .filter(|dist| *dist >= query_lower_bound && *dist < query_upper_bound) + .map(|dist| dist.to_bits()) + .collect::>(); + expected.sort_unstable(); + expected.truncate(k); + let actual = dense.iter().map(|(dist, _)| *dist).collect::>(); + assert_eq!( + actual, expected, + "ordering={ordering} n={n} k={k} bounds={bounds:?} mode={approx_mode:?}" + ); + } + } + } + } + #[test] fn test_raw_query_one_bit_distance_uses_binary_factors_without_ex_columns() { let code_dim = 8usize; From 72ea21ca537562906d796e8e02cfefa51acca8c4 Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Wed, 17 Jun 2026 00:58:20 +0700 Subject: [PATCH 09/15] feat(index): accelerate regex and infix LIKE with the ngram index (#7139) ## What Extends the ngram index (today only `contains(col, 'substr')`) to also accelerate `regexp_like(col, pat)` / `regexp_match(col, pat)` and infix `LIKE` (`col LIKE '%foo%bar%'`), which until now fell through to a full table scan. Closes #7130. ## How Following Postgres `pg_trgm` and Russ Cox's [trigram-index approach](https://swtch.com/~rsc/regexp/regexp4.html), a pattern is compiled into a boolean trigram condition (`TrigramQuery`, an AND/OR tree) that is a *necessary* condition for any match. This maps onto the inverted index's set algebra: AND is posting-list intersection, OR is union. The index returns a candidate superset and the scan rechecks the true predicate, exactly as `contains` does. The derivation walks the `regex-syntax` HIR bottom-up, tracking per-node `(emptyable, exact, prefix, suffix)` sets and folding boundary trigrams across concatenation, with bounds that fold-then-discard so precision loss never drops a necessary trigram. ### Soundness The derived condition never requires a trigram a matching string could lack, so no real match is dropped. Requirements come from the index's own tokenizer (so sub-trigram runs contribute nothing); character classes and case-insensitive folds are treated as a single unknown character (the index's normalization disagrees with Unicode case folding - `(?i)c` also matches U+2102); and patterns with no derivable trigram (`a.b`, `.*`) are left to a full scan. ### Why infix LIKE A plain-literal `regexp_like(col, 'foo')` is rewritten to `col LIKE '%foo%'` by the optimizer before it reaches the index, so without infix-LIKE the most common "regex" query would not accelerate. The LIKE is translated to a loose regex for candidate generation only; the original LIKE stays as the recheck filter, so the candidate regex need only be a sound superset. ## Benchmark `cargo bench -p lance --bench regex_ngram`, 200k rows, before (main) vs after: | Query | Before | After | Change | |---|---|---|---| | `regexp_match(doc, 'zqxwvu.*needlexyz')` | 45.8 ms | 8.2 ms | -82% | | `regexp_match(doc, '(zqxwvu\|qwerasdf\|needlexyz)')` | 51.7 ms | 11.6 ms | -77% | | `regexp_match(doc, 'zqxwvu')` (rewritten to LIKE) | 36.6 ms | 8.0 ms | -78% | | `regexp_match(doc, 'a.b')` (non-accelerable) | 77.8 ms | 76.2 ms | within noise | ## Testing Unit tests for the regex-to-trigram derivation and the regex-flags folding; index-level search tests (AND across `.*`, alternation union, NULL exclusion, absent trigram); a multi-fragment end-to-end scan test asserting correct results and index engagement for `regexp_like`, `regexp_match`, and `LIKE`, plus a case-insensitive query that must fall back to a full recheck; and a regression test ensuring non-accelerable patterns still return all correct matches via full recheck. No binding changes are needed - Python/Java pass these filter strings through the existing scan API, so acceleration applies transparently. Co-authored-by: Vova Kolmakov Co-authored-by: Claude Opus 4.8 (1M context) --- Cargo.lock | 1 + Cargo.toml | 1 + docs/src/format/index/scalar/ngram.md | 8 +- python/Cargo.lock | 1 + rust/lance-index/Cargo.toml | 1 + rust/lance-index/src/scalar.rs | 24 +- rust/lance-index/src/scalar/expression.rs | 237 +++++- rust/lance-index/src/scalar/fmindex.rs | 9 + rust/lance-index/src/scalar/ngram.rs | 151 +++- .../src/scalar/ngram/ngram_regex.rs | 673 ++++++++++++++++++ rust/lance/Cargo.toml | 4 + rust/lance/benches/regex_ngram.rs | 134 ++++ rust/lance/src/dataset/scanner.rs | 192 +++++ 13 files changed, 1411 insertions(+), 25 deletions(-) create mode 100644 rust/lance-index/src/scalar/ngram/ngram_regex.rs create mode 100644 rust/lance/benches/regex_ngram.rs diff --git a/Cargo.lock b/Cargo.lock index 1e820b0c63d..5bee7197f24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4835,6 +4835,7 @@ dependencies = [ "rand_distr", "rangemap", "rayon", + "regex-syntax", "roaring", "rstest", "serde", diff --git a/Cargo.toml b/Cargo.toml index 68a9002872b..a95f530de38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -180,6 +180,7 @@ rand_distr = { version = "0.5.1" } rand_xoshiro = "0.7.0" rangemap = { version = "1.0" } rayon = "1.10" +regex-syntax = "0.8.10" roaring = "0.11.4" rstest = "0.26.1" serde = { version = "^1" } diff --git a/docs/src/format/index/scalar/ngram.md b/docs/src/format/index/scalar/ngram.md index bdf78474d50..d437363d264 100644 --- a/docs/src/format/index/scalar/ngram.md +++ b/docs/src/format/index/scalar/ngram.md @@ -29,4 +29,10 @@ The N-gram index provides inexact results for the following query types: | Query Type | Description | Operation | Result Type | |----------------|--------------------------|-------------------------------------------------------|-------------| -| **contains** | Substring search in text | Finds all trigrams in query, intersects posting lists | AtMost | \ No newline at end of file +| **contains** | Substring search in text | Finds all trigrams in query, intersects posting lists | AtMost | +| **regexp_like** / **regexp_match** | Regular-expression match | Derives a necessary trigram condition from the pattern (AND of intersections, OR of unions), then rechecks the true regex | AtMost | +| **LIKE** (infix) | Wildcard match such as `%foo%bar%` | Uses the literal segments of the pattern as a trigram condition, then rechecks the LIKE | AtMost | + +Patterns from which no trigram can be derived - for example `a.b`, `.*`, +case-insensitive matches, or literal runs shorter than three characters - fall +back to rechecking every row. This is always correct, just not accelerated. diff --git a/python/Cargo.lock b/python/Cargo.lock index 5dfc1ba47e0..37035be3085 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -4431,6 +4431,7 @@ dependencies = [ "rand_distr", "rangemap", "rayon", + "regex-syntax", "roaring", "serde", "serde_json", diff --git a/rust/lance-index/Cargo.toml b/rust/lance-index/Cargo.toml index e3947b57856..85de43c0f9b 100644 --- a/rust/lance-index/Cargo.toml +++ b/rust/lance-index/Cargo.toml @@ -56,6 +56,7 @@ object_store.workspace = true prost.workspace = true prost-types.workspace = true rand.workspace = true +regex-syntax.workspace = true roaring.workspace = true rayon.workspace = true serde_json.workspace = true diff --git a/rust/lance-index/src/scalar.rs b/rust/lance-index/src/scalar.rs index 4830586f85c..a287d277a81 100644 --- a/rust/lance-index/src/scalar.rs +++ b/rust/lance-index/src/scalar.rs @@ -8,6 +8,7 @@ use arrow_array::{BooleanArray, ListArray, RecordBatch, UInt64Array}; use arrow_schema::{Field, Schema}; use async_trait::async_trait; use bytes::Bytes; +use datafusion::functions::regex::regexplike::RegexpLikeFunc; use datafusion::functions::string::contains::ContainsFunc; use datafusion::functions_nested::array_has; use datafusion::physical_plan::SendableRecordBatchStream; @@ -649,9 +650,15 @@ impl AnyQuery for LabelListQuery { pub enum TextQuery { /// Retrieve all row ids where the text contains the given string StringContains(String), - // TODO: In the future we should be able to do string-insensitive contains - // as well as partial matches (e.g. LIKE 'foo%') and potentially even - // some regular expressions + /// Retrieve all row ids whose text matches the given regular expression. + /// + /// The pattern is a full regular expression (as accepted by `regexp_like`). + /// The index returns a candidate superset that the scan rechecks, so any + /// pattern is sound; patterns with no usable trigram structure simply fall + /// back to rechecking every row. + Regex(String), + // TODO: In the future we should be able to do case-insensitive contains + // as well as partial matches (e.g. LIKE 'foo%'). } impl AnyQuery for TextQuery { @@ -672,6 +679,17 @@ impl AnyQuery for TextQuery { Expr::Literal(ScalarValue::Utf8(Some(substr.clone())), None), ], }), + // `regexp_like` returns Boolean directly, so the reconstructed + // expression can be used as-is for the recheck filter (no IsNotNull + // wrapper, unlike `regexp_match`). It is the semantic equivalent of + // the original predicate for the "does it match" question. + Self::Regex(pattern) => Expr::ScalarFunction(ScalarFunction { + func: Arc::new(RegexpLikeFunc::new().into()), + args: vec![ + Expr::Column(Column::new_unqualified(col)), + Expr::Literal(ScalarValue::Utf8(Some(pattern.clone())), None), + ], + }), } } diff --git a/rust/lance-index/src/scalar/expression.rs b/rust/lance-index/src/scalar/expression.rs index 38a29e9c43c..ea7fbabc813 100644 --- a/rust/lance-index/src/scalar/expression.rs +++ b/rust/lance-index/src/scalar/expression.rs @@ -781,20 +781,28 @@ impl ScalarQueryParser for LabelListQueryParser { } } -/// A parser for indices that handle string contains queries +/// A parser for indices that handle string `contains` queries, and -- when +/// `supports_regex` is set -- `regexp_like` / `regexp_match` queries. #[derive(Debug, Clone)] pub struct TextQueryParser { index_name: String, index_type: String, needs_recheck: bool, + supports_regex: bool, } impl TextQueryParser { - pub fn new(index_name: String, index_type: String, needs_recheck: bool) -> Self { + pub fn new( + index_name: String, + index_type: String, + needs_recheck: bool, + supports_regex: bool, + ) -> Self { Self { index_name, index_type, needs_recheck, + supports_regex, } } } @@ -837,31 +845,156 @@ impl ScalarQueryParser for TextQueryParser { func: &ScalarUDF, args: &[Expr], ) -> Option { - if args.len() != 2 { + // The first argument is the indexed column; the second is the substring + // / pattern. `contains` takes exactly two arguments; the regex functions + // optionally take a third flags argument. + if args.len() < 2 { return None; } - let scalar = maybe_scalar(&args[1], data_type)?; - match scalar { - ScalarValue::Utf8(Some(scalar_str)) | ScalarValue::LargeUtf8(Some(scalar_str)) => { - if func.name() == "contains" { - let query = TextQuery::StringContains(scalar_str); - Some(IndexedExpression::index_query_with_recheck( - column.to_string(), - self.index_name.clone(), - self.index_type.clone(), - Arc::new(query), - self.needs_recheck, - )) - } else { + // A non-string pattern cannot be handled. + let (ScalarValue::Utf8(Some(pattern)) | ScalarValue::LargeUtf8(Some(pattern))) = + maybe_scalar(&args[1], data_type)? + else { + return None; + }; + + let query = match func.name() { + "contains" if args.len() == 2 => TextQuery::StringContains(pattern), + "regexp_like" | "regexp_match" if self.supports_regex => { + let pattern = match args.get(2) { + Some(flags_expr) => apply_regex_flags(&pattern, flags_expr)?, + None => pattern, + }; + // If the pattern yields no usable trigram (e.g. `a.b`), leave it + // to a full scan instead of routing it to the index, which could + // only answer with an unsupported "recheck everything" result. + if !crate::scalar::ngram::regex_can_use_index(&pattern) { + return None; + } + TextQuery::Regex(pattern) + } + _ => return None, + }; + + Some(IndexedExpression::index_query_with_recheck( + column.to_string(), + self.index_name.clone(), + self.index_type.clone(), + Arc::new(query), + self.needs_recheck, + )) + } + + fn visit_like( + &self, + column: &str, + like: &Like, + pattern: &ScalarValue, + ) -> Option { + // Infix LIKE is accelerated only by the ngram index (via its regex + // machinery). A plain-literal `regexp_like(col, 'foo')` is rewritten to + // `col LIKE '%foo%'` before it reaches the index, so this is the path + // that accelerates those. ILIKE is skipped because its case folding does + // not match the index's normalization. + if !self.supports_regex || like.case_insensitive { + return None; + } + let pattern_str = match pattern { + ScalarValue::Utf8(Some(s)) | ScalarValue::LargeUtf8(Some(s)) => s.as_str(), + _ => return None, + }; + // Translate the LIKE pattern into a loose regex used only for candidate + // generation; the original LIKE stays as the recheck filter, so the + // regex only needs to be a sound superset. + let regex = like_to_regex(pattern_str, like.escape_char)?; + if !crate::scalar::ngram::regex_can_use_index(®ex) { + return None; + } + Some(IndexedExpression { + scalar_query: Some(ScalarIndexExpr::Query(ScalarIndexSearch { + column: column.to_string(), + index_name: self.index_name.clone(), + index_type: self.index_type.clone(), + query: Arc::new(TextQuery::Regex(regex)), + needs_recheck: self.needs_recheck, + fragment_bitmap: None, + })), + refine_expr: Some(Expr::Like(like.clone())), + }) + } +} + +/// Translate a LIKE pattern into a regular expression used purely for ngram +/// candidate generation: `%` becomes `.*`, `_` becomes `.`, and literal +/// characters are regex-escaped. Returns `None` when no literal run is long +/// enough to yield a trigram (the index could not help, so a full scan is left +/// to handle it). +fn like_to_regex(pattern: &str, escape: Option) -> Option { + let mut regex = String::new(); + let mut run = 0usize; + let mut longest_run = 0usize; + let mut chars = pattern.chars(); + while let Some(c) = chars.next() { + let literal = if Some(c) == escape { + // The next character is escaped, i.e. a literal. + chars.next() + } else { + match c { + '%' => { + regex.push_str(".*"); + run = 0; None } + '_' => { + regex.push('.'); + run = 0; + None + } + other => Some(other), } - _ => { - // If the scalar is not a string, we cannot handle it - None + }; + if let Some(lit) = literal { + if regex_syntax::is_meta_character(lit) { + regex.push('\\'); + } + regex.push(lit); + // Only runs of alphanumeric characters can produce a trigram. + if lit.is_alphanumeric() { + run += 1; + longest_run = longest_run.max(run); + } else { + run = 0; } } } + (longest_run >= 3).then_some(regex) +} + +/// Fold the supported `regexp_like` / `regexp_match` flags into an inline prefix +/// on the pattern (e.g. flags `"i"` -> `"(?i)pattern"`). Returns `None` for a +/// non-literal flags argument or an unrecognized flag, so the caller leaves the +/// predicate to a full recheck rather than risk changing its semantics. +fn apply_regex_flags(pattern: &str, flags_expr: &Expr) -> Option { + let (Expr::Literal(ScalarValue::Utf8(Some(flags)), _) + | Expr::Literal(ScalarValue::LargeUtf8(Some(flags)), _)) = flags_expr + else { + return None; + }; + let mut inline = String::new(); + for flag in flags.chars() { + // Only flags expressible as an inline `(?...)` group in the regex crate + // (which the recheck uses) are safe to fold. + if ['i', 's', 'm', 'x'].contains(&flag) { + inline.push(flag); + } else { + return None; + } + } + if inline.is_empty() { + Some(pattern.to_string()) + } else { + Some(format!("(?{inline}){pattern}")) + } } /// A parser for indices that handle queries with the contains_tokens function @@ -1813,7 +1946,18 @@ fn visit_node( Expr::IsFalse(expr) => Ok(visit_is_bool(expr.as_ref(), index_info, false)), Expr::IsTrue(expr) => Ok(visit_is_bool(expr.as_ref(), index_info, true)), Expr::IsNull(expr) => Ok(visit_is_null(expr.as_ref(), index_info, false)), - Expr::IsNotNull(expr) => Ok(visit_is_null(expr.as_ref(), index_info, true)), + Expr::IsNotNull(expr) => { + // `regexp_match(col, pat)` returns a list and is coerced to + // `IsNotNull(regexp_match(...))` before it reaches here. Unwrap that + // so the regex acceleration applies; everything else is a genuine + // IS NOT NULL check. + if let Expr::ScalarFunction(scalar_fn) = expr.as_ref() + && scalar_fn.func.name() == "regexp_match" + { + return Ok(visit_scalar_fn(scalar_fn, index_info)); + } + Ok(visit_is_null(expr.as_ref(), index_info, true)) + } Expr::Not(expr) => visit_not(expr.as_ref(), index_info, depth), Expr::BinaryExpr(binary_expr) => visit_binary_expr(binary_expr, index_info, depth), Expr::ScalarFunction(scalar_fn) => Ok(visit_scalar_fn(scalar_fn, index_info)), @@ -2690,6 +2834,59 @@ mod tests { assert!(matches!(negated.upper, NullableRowAddrMask::BlockList(_))); } + #[test] + fn test_like_to_regex() { + // `%` -> `.*`, `_` -> `.`, with a literal run of at least three chars. + assert_eq!(like_to_regex("%foo%", None).as_deref(), Some(".*foo.*")); + assert_eq!(like_to_regex("foo%bar", None).as_deref(), Some("foo.*bar")); + assert_eq!(like_to_regex("foo_bar", None).as_deref(), Some("foo.bar")); + assert_eq!(like_to_regex("foobar", None).as_deref(), Some("foobar")); + + // Regex metacharacters in the literal portion are escaped. + assert_eq!( + like_to_regex("%a.bcd%", None).as_deref(), + Some(".*a\\.bcd.*") + ); + + // No literal run of three alphanumeric characters -> no index help. + assert_eq!(like_to_regex("%ab%", None), None); + assert_eq!(like_to_regex("%a%b%c%", None), None); + assert_eq!(like_to_regex("%", None), None); + + // The escape character makes the following character a literal. + assert_eq!( + like_to_regex(r"%foo\%bar%", Some('\\')).as_deref(), + Some(".*foo%bar.*") + ); + } + + #[test] + fn test_apply_regex_flags() { + fn flags(s: &str) -> Expr { + Expr::Literal(ScalarValue::Utf8(Some(s.to_string())), None) + } + + // Empty flags leave the pattern untouched (no inline group emitted). + assert_eq!(apply_regex_flags("foo", &flags("")).as_deref(), Some("foo")); + // Supported flags are folded into an inline `(?...)` prefix. + assert_eq!( + apply_regex_flags("foo", &flags("i")).as_deref(), + Some("(?i)foo") + ); + assert_eq!( + apply_regex_flags("foo", &flags("is")).as_deref(), + Some("(?is)foo") + ); + // An unrecognized flag bails out so the caller leaves the predicate to a + // full recheck rather than risk changing its semantics. + assert_eq!(apply_regex_flags("foo", &flags("g")), None); + // A non-string (hence non-literal-flags) argument cannot be folded. + assert_eq!( + apply_regex_flags("foo", &Expr::Literal(ScalarValue::Int32(Some(1)), None)), + None + ); + } + #[test] fn test_extract_like_leading_prefix() { // Simple prefix patterns (no recheck needed) diff --git a/rust/lance-index/src/scalar/fmindex.rs b/rust/lance-index/src/scalar/fmindex.rs index 9677f7471ea..aed1136535a 100644 --- a/rust/lance-index/src/scalar/fmindex.rs +++ b/rust/lance-index/src/scalar/fmindex.rs @@ -1352,6 +1352,12 @@ impl ScalarIndex for FMIndexScalarIndex { Default::default(), ))) } + // Regex queries are routed only to the ngram index (the FM-index's + // query parser advertises `supports_regex = false`), so this is + // unreachable in practice; reject it explicitly rather than silently. + TextQuery::Regex(_) => Err(Error::invalid_input( + "FMIndex does not support regular expression queries", + )), } } fn can_remap(&self) -> bool { @@ -1645,6 +1651,9 @@ impl ScalarIndexPlugin for FMIndexPlugin { Some(Box::new(TextQueryParser::new( index_name, self.name().to_string(), + // needs_recheck: the FM-index returns exact substring matches. + false, + // supports_regex: regex acceleration is only implemented for ngram. false, ))) } diff --git a/rust/lance-index/src/scalar/ngram.rs b/rust/lance-index/src/scalar/ngram.rs index 72ef8d53a92..b452ef78c85 100644 --- a/rust/lance-index/src/scalar/ngram.rs +++ b/rust/lance-index/src/scalar/ngram.rs @@ -5,7 +5,10 @@ use std::any::Any; use std::collections::BTreeMap; use std::iter::once; use std::time::Instant; -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; use super::lance_format::LanceIndexStore; use super::{ @@ -49,6 +52,9 @@ use roaring::{RoaringBitmap, RoaringTreemap}; use serde::Serialize; use tracing::instrument; +mod ngram_regex; +pub(crate) use ngram_regex::regex_can_use_index; + const TOKENS_COL: &str = "tokens"; const POSTING_LIST_COL: &str = "posting_list"; const POSTINGS_FILENAME: &str = "ngram_postings.lance"; @@ -476,6 +482,45 @@ impl ScalarIndex for NGramIndex { let row_ids = NGramPostingList::intersect(list_refs); Ok(SearchResult::at_most(RowAddrTreeMap::from(row_ids))) } + TextQuery::Regex(pattern) => { + let trigram_query = ngram_regex::regex_to_trigram_query(pattern); + match &trigram_query { + // No usable trigram structure (e.g. `a.b`, `.*`): the index + // cannot prune, so every row must be rechecked. + ngram_regex::TrigramQuery::All => { + Ok(SearchResult::at_least(RowAddrTreeMap::new())) + } + // The pattern is provably unsatisfiable. + ngram_regex::TrigramQuery::None => { + Ok(SearchResult::exact(RowAddrTreeMap::new())) + } + _ => { + let mut tokens = HashSet::new(); + ngram_regex::collect_tokens(&trigram_query, &mut tokens); + // Fetch the posting list for every trigram the condition + // references; a token absent from the index contributes + // an empty list, which `eval_trigram_query` handles. + let present = tokens.into_iter().filter_map(|token| { + self.tokens.get(&token).map(|offset| (token, *offset)) + }); + let lists = futures::stream::iter(present.map(|(token, offset)| { + self.list_reader + .ngram_list(offset, metrics) + .map(move |result| result.map(|list| (token, list))) + })) + .buffer_unordered(self.io_parallelism) + .try_collect::)>>() + .await?; + metrics.record_comparisons(lists.len()); + let bitmaps: HashMap = lists + .into_iter() + .map(|(token, list)| (token, list.bitmap.clone())) + .collect(); + let row_ids = ngram_regex::eval_trigram_query(&trigram_query, &bitmaps); + Ok(SearchResult::at_most(RowAddrTreeMap::from(row_ids))) + } + } + } } } @@ -1279,6 +1324,9 @@ impl ScalarIndexPlugin for NGramIndexPlugin { Some(Box::new(TextQueryParser::new( index_name, self.name().to_string(), + // needs_recheck: ngram results are an inexact candidate superset. + true, + // supports_regex: the ngram index can answer regex queries. true, ))) } @@ -1538,6 +1586,107 @@ mod tests { assert_eq!(expected, res); } + #[test_log::test(tokio::test)] + async fn test_ngram_regex_search() { + // Same corpus as test_basic_ngram_index. + let data = StringArray::from_iter_values([ + "cat", // 0 + "dog", // 1 + "cat dog", // 2 + "dog cat", // 3 + "elephant", // 4 + "mouse", // 5 + "rhino", // 6 + "giraffe", // 7 + "rhinos nose", // 8 + ]); + let row_ids = UInt64Array::from_iter_values((0..data.len()).map(|i| i as u64)); + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, DataType::Utf8, false), + Field::new(ROW_ID, DataType::UInt64, false), + ])); + let data = + RecordBatch::try_new(schema.clone(), vec![Arc::new(data), Arc::new(row_ids)]).unwrap(); + let data = Box::pin(RecordBatchStreamAdapter::new( + schema, + stream::once(std::future::ready(Ok(data))), + )); + + let builder = NGramIndexBuilder::try_new(NGramIndexBuilderOptions::default()).unwrap(); + let (index, _tmpdir) = do_train(builder, data).await; + + async fn search(index: &NGramIndex, pattern: &str) -> SearchResult { + index + .search( + &TextQuery::Regex(pattern.to_string()), + &NoOpMetricsCollector, + ) + .await + .unwrap() + } + + // A plain literal yields the same candidates as contains("cat"). + assert_eq!( + search(&index, "cat").await, + SearchResult::at_most(RowAddrTreeMap::from_iter([0, 2, 3])) + ); + + // Alternation -> union of each branch's rows. + assert_eq!( + search(&index, "(cat|dog)").await, + SearchResult::at_most(RowAddrTreeMap::from_iter([0, 1, 2, 3])) + ); + + // AND across `.*`: must contain both the `rhino` and `nose` trigrams, so + // row 6 ("rhino") is correctly excluded and only row 8 survives. + assert_eq!( + search(&index, "rhino.*nose").await, + SearchResult::at_most(RowAddrTreeMap::from_iter([8])) + ); + + // No derivable trigram -> recheck everything. + assert_eq!( + search(&index, "a.b").await, + SearchResult::at_least(RowAddrTreeMap::new()) + ); + + // A trigram that is absent from the index -> empty candidate set. + assert_eq!( + search(&index, "zzz").await, + SearchResult::at_most(RowAddrTreeMap::new()) + ); + } + + #[test_log::test(tokio::test)] + async fn test_ngram_regex_search_nulls() { + // Rows: cat(0), dog(1), NULL(2), NULL(3), cat dog(4). + let data = simple_data_with_nulls(); + let builder = NGramIndexBuilder::try_new(NGramIndexBuilderOptions::default()).unwrap(); + let (index, _tmpdir) = do_train(builder, data).await; + + // The NULL rows (2, 3) must never appear in the candidate set. + let res = index + .search(&TextQuery::Regex("cat".to_string()), &NoOpMetricsCollector) + .await + .unwrap(); + assert_eq!( + res, + SearchResult::at_most(RowAddrTreeMap::from_iter([0, 4])) + ); + + let res = index + .search( + &TextQuery::Regex("(cat|dog)".to_string()), + &NoOpMetricsCollector, + ) + .await + .unwrap(); + assert_eq!( + res, + SearchResult::at_most(RowAddrTreeMap::from_iter([0, 1, 4])) + ); + } + fn test_data_schema() -> Arc { Arc::new(Schema::new(vec![ Field::new(VALUE_COLUMN_NAME, DataType::Utf8, true), diff --git a/rust/lance-index/src/scalar/ngram/ngram_regex.rs b/rust/lance-index/src/scalar/ngram/ngram_regex.rs new file mode 100644 index 00000000000..ee67c479a71 --- /dev/null +++ b/rust/lance-index/src/scalar/ngram/ngram_regex.rs @@ -0,0 +1,673 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Deriving a trigram pre-filter from a regular expression. +//! +//! This is the query-side counterpart of the ngram index that lets us +//! accelerate `regexp_like` / `regexp_match` predicates the same way the index +//! already accelerates `contains`. The idea (the same one Postgres `pg_trgm` +//! and Russ Cox's Google Code Search use) is to derive, from the regex, a +//! boolean condition over trigram presence that is *necessary* for any string +//! to match, evaluate it against the inverted index, and let the scan recheck +//! the true regex on the surviving rows. +//! +//! The derived condition is a [`TrigramQuery`] -- an AND/OR tree of trigram +//! tokens. `AND` maps onto posting-list intersection and `OR` onto union, which +//! is exactly the set algebra the ngram index is built for. +//! +//! # Soundness +//! +//! The single invariant that matters is that the condition must never require a +//! trigram that a matching string could lack -- otherwise we would drop real +//! matches (a false negative, far worse than a false positive, which the recheck +//! removes). Everything here is therefore a conservative *over*-approximation: +//! when in doubt we emit [`TrigramQuery::All`] ("no constraint, recheck +//! everything"). Concretely: +//! +//! * Every trigram requirement is produced by [`trigrams_of_string`], which runs +//! the *same* tokenizer the index was built with, so a string shorter than a +//! trigram (or with no alphanumeric run) contributes no requirement. +//! * Character classes and case-insensitive folds are treated as a single +//! unknown character (`All`), because the index's normalization does not agree +//! with Unicode case folding (e.g. `(?i)c` also matches `ℂ`, which the index +//! does not fold to `c`). Literal runs -- the common case -- are fully used. +//! * When the exact / prefix / suffix string sets grow past a bound we first fold +//! their trigrams into the running condition and only then drop the strings, so +//! collapsing precision never removes a necessary trigram. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use regex_syntax::hir::{Class, Hir, HirKind}; +use roaring::RoaringTreemap; + +use super::{NGRAM_N, NGRAM_TOKENIZER, ngram_to_token, tokenize_visitor}; + +/// Maximum number of strings kept in an `exact` / `prefix` / `suffix` set before +/// it is folded into the trigram condition and dropped. +const MAX_SET_SIZE: usize = 16; +/// Maximum length (in characters) of a string kept in a set. Longer strings are +/// trimmed to a sound shorter affix. +const MAX_STRING_LEN: usize = 32; + +/// A boolean condition over trigram presence that is *necessary* for a regex to +/// match. `All` means "no constraint" and `None` means "unsatisfiable"; by +/// construction these only ever appear at the root of the tree. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum TrigramQuery { + /// No constraint: every row is a candidate (the scan must recheck all rows). + All, + /// Unsatisfiable: no row can match. + None, + /// The given trigram token must be present. + Trigram(u32), + /// Every child condition must hold (posting-list intersection). + And(Vec), + /// At least one child condition must hold (posting-list union). + Or(Vec), +} + +impl TrigramQuery { + /// Build an `AND` of conditions, applying identity (`All`), absorbing + /// (`None`), flattening, sorting and de-duplication so the result is + /// canonical and free of nested `All`/`None`. + fn and(items: Vec) -> Self { + let mut flat = Vec::with_capacity(items.len()); + for item in items { + match item { + Self::All => {} // identity + Self::None => return Self::None, // absorbing + Self::And(children) => flat.extend(children), // flatten + other => flat.push(other), + } + } + flat.sort(); + flat.dedup(); + match flat.len() { + 0 => Self::All, + 1 => flat.pop().unwrap(), + _ => Self::And(flat), + } + } + + /// Build an `OR` of conditions, applying absorbing (`All`), identity + /// (`None`), flattening, sorting and de-duplication. + fn or(items: Vec) -> Self { + let mut flat = Vec::with_capacity(items.len()); + for item in items { + match item { + Self::All => return Self::All, // absorbing + Self::None => {} // identity + Self::Or(children) => flat.extend(children), // flatten + other => flat.push(other), + } + } + flat.sort(); + flat.dedup(); + match flat.len() { + 0 => Self::None, + 1 => flat.pop().unwrap(), + _ => Self::Or(flat), + } + } +} + +/// Information about the set of strings a sub-expression can match, used to +/// build a necessary trigram condition bottom-up. For every string `s` the +/// sub-expression matches: `s` is in `exact` (when it is `Some`), `s` starts +/// with some member of `prefix` and ends with some member of `suffix`, and `s` +/// satisfies `match_q`. +struct RegexInfo { + /// Whether the sub-expression can match the empty string. + emptyable: bool, + /// The complete set of strings the sub-expression matches, or `None` if that + /// set is unbounded / unknown. + exact: Option>, + /// Strings that every match must start with (empty = unknown). + prefix: BTreeSet, + /// Strings that every match must end with (empty = unknown). + suffix: BTreeSet, + /// A necessary trigram condition for the sub-expression. + match_q: TrigramQuery, +} + +impl RegexInfo { + /// The empty string (also used for zero-width anchors): matches only `""`. + fn empty_string() -> Self { + let empty = BTreeSet::from([String::new()]); + Self { + emptyable: true, + exact: Some(empty.clone()), + prefix: empty.clone(), + suffix: empty, + match_q: TrigramQuery::All, + } + } + + /// A fixed literal string. + fn literal(s: &str) -> Self { + let set = BTreeSet::from([s.to_string()]); + Self { + emptyable: s.is_empty(), + exact: Some(set.clone()), + prefix: set.clone(), + suffix: set, + match_q: trigrams_of_string(s), + } + } + + /// A single unknown character (a character class we cannot pin down). + fn any_char() -> Self { + Self { + emptyable: false, + exact: None, + prefix: BTreeSet::new(), + suffix: BTreeSet::new(), + match_q: TrigramQuery::All, + } + } + + /// Enforce the size/length bounds, folding any information about to be + /// discarded into `match_q` first so that precision loss never drops a + /// necessary trigram. Idempotent. + fn bound(&mut self) { + let oversized_exact = self.exact.as_ref().is_some_and(|exact| { + exact.len() > MAX_SET_SIZE || exact.iter().any(|s| s.chars().count() > MAX_STRING_LEN) + }); + if oversized_exact { + let exact = self.exact.take().expect("checked above"); + self.fold_into_match(&exact); + } + + self.prefix = self + .prefix + .iter() + .map(|s| leading(s, MAX_STRING_LEN)) + .collect(); + if self.prefix.len() > MAX_SET_SIZE { + let prefix = std::mem::take(&mut self.prefix); + self.fold_into_match(&prefix); + } + + self.suffix = self + .suffix + .iter() + .map(|s| trailing(s, MAX_STRING_LEN)) + .collect(); + if self.suffix.len() > MAX_SET_SIZE { + let suffix = std::mem::take(&mut self.suffix); + self.fold_into_match(&suffix); + } + } + + /// AND the trigrams of `set` (a complete set of possible affixes/strings) + /// into `match_q`. Sound because the set is exhaustive for its role. + fn fold_into_match(&mut self, set: &BTreeSet) { + let folded = trigrams_of_set(set.iter()); + let current = std::mem::replace(&mut self.match_q, TrigramQuery::All); + self.match_q = TrigramQuery::and(vec![current, folded]); + } +} + +/// AND together the trigrams of `s`. Reuses the index's own tokenizer so the +/// tokens are normalized (lowercase, ASCII-folded, alphanumeric-bounded) +/// exactly as they were stored. Returns `All` if `s` yields no trigram (too +/// short, or no run of three alphanumeric characters). +fn trigrams_of_string(s: &str) -> TrigramQuery { + let mut tokens = Vec::new(); + tokenize_visitor(&NGRAM_TOKENIZER, s, |ngram| { + tokens.push(TrigramQuery::Trigram(ngram_to_token(ngram, NGRAM_N))); + }); + TrigramQuery::and(tokens) +} + +/// OR together the trigram conditions of each string in `set`. An empty set +/// means "unknown" and yields `All` (no constraint); if any member yields `All` +/// the whole OR is `All`. +fn trigrams_of_set<'a>(set: impl IntoIterator) -> TrigramQuery { + let queries: Vec<_> = set.into_iter().map(|s| trigrams_of_string(s)).collect(); + if queries.is_empty() { + return TrigramQuery::All; + } + TrigramQuery::or(queries) +} + +/// Concatenate every string in `a` with every string in `b`. +fn cross_concat(a: &BTreeSet, b: &BTreeSet) -> BTreeSet { + let mut out = BTreeSet::new(); + for x in a { + for y in b { + out.insert(format!("{x}{y}")); + } + } + out +} + +/// The first `n` characters of `s` (a sound shorter prefix). +fn leading(s: &str, n: usize) -> String { + s.chars().take(n).collect() +} + +/// The last `n` characters of `s` (a sound shorter suffix). +fn trailing(s: &str, n: usize) -> String { + let count = s.chars().count(); + s.chars().skip(count.saturating_sub(n)).collect() +} + +/// If `class` matches exactly one scalar value, return that character. +fn singleton_char(class: &Class) -> Option { + match class { + Class::Unicode(u) => { + let ranges = u.ranges(); + match ranges { + [r] if r.start() == r.end() => Some(r.start()), + _ => None, + } + } + Class::Bytes(b) => { + let ranges = b.ranges(); + match ranges { + [r] if r.start() == r.end() && r.start() < 0x80 => Some(r.start() as char), + _ => None, + } + } + } +} + +/// Compute the [`RegexInfo`] for `hir` bottom-up. +fn analyze(hir: &Hir) -> RegexInfo { + let mut info = match hir.kind() { + // Zero-width: the empty match. Anchors (^, $, \b) carry no trigram. + HirKind::Empty | HirKind::Look(_) => RegexInfo::empty_string(), + HirKind::Literal(lit) => match std::str::from_utf8(&lit.0) { + Ok(s) => RegexInfo::literal(s), + // A literal that is not valid UTF-8 cannot be reasoned about here. + Err(_) => RegexInfo::any_char(), + }, + HirKind::Class(class) => match singleton_char(class) { + Some(ch) => RegexInfo::literal(ch.encode_utf8(&mut [0u8; 4])), + None => RegexInfo::any_char(), + }, + HirKind::Repetition(rep) => { + let inner = analyze(&rep.sub); + let at_least_one = rep.min >= 1; + RegexInfo { + emptyable: !at_least_one || inner.emptyable, + // We do not unroll bounded repetitions, so the matched set is + // unbounded as far as we are concerned. + exact: None, + prefix: if at_least_one { + inner.prefix.clone() + } else { + BTreeSet::new() + }, + suffix: if at_least_one { + inner.suffix.clone() + } else { + BTreeSet::new() + }, + // Only a required occurrence (min >= 1) contributes; the single + // inner match is necessary, never multiplied. + match_q: if at_least_one { + inner.match_q + } else { + TrigramQuery::All + }, + } + } + HirKind::Capture(cap) => analyze(&cap.sub), + HirKind::Concat(subs) => analyze_concat(subs), + HirKind::Alternation(subs) => analyze_alternation(subs), + }; + info.bound(); + info +} + +fn analyze_concat(subs: &[Hir]) -> RegexInfo { + let mut acc = RegexInfo::empty_string(); + for sub in subs { + acc = concat_info(acc, analyze(sub)); + } + acc +} + +/// Combine two adjacent sub-expressions. This is the subtle part: it recovers +/// trigrams that straddle the junction via the cross product of `acc.suffix` and +/// `next.prefix`. +fn concat_info(acc: RegexInfo, next: RegexInfo) -> RegexInfo { + let emptyable = acc.emptyable && next.emptyable; + + // Trigrams spanning the junction (computed from the pre-merge affixes). + let boundary = if acc.suffix.is_empty() || next.prefix.is_empty() { + TrigramQuery::All + } else { + trigrams_of_set(cross_concat(&acc.suffix, &next.prefix).iter()) + }; + + // exact = acc.exact x next.exact, only while both are finite and small. + let exact = match (&acc.exact, &next.exact) { + (Some(a), Some(b)) if a.len().saturating_mul(b.len()) <= MAX_SET_SIZE => { + Some(cross_concat(a, b)) + } + _ => None, + }; + + // A match starts with acc's full string (when known) then next's prefix, + // otherwise with acc's own prefix. + let prefix = match &acc.exact { + Some(a) if !next.prefix.is_empty() => cross_concat(a, &next.prefix), + Some(a) => a.clone(), + None => acc.prefix.clone(), + }; + + // Mirror image for the suffix (driven by the right side). + let suffix = match &next.exact { + Some(b) if !acc.suffix.is_empty() => cross_concat(&acc.suffix, b), + Some(b) => b.clone(), + None => next.suffix.clone(), + }; + + let match_q = TrigramQuery::and(vec![acc.match_q, next.match_q, boundary]); + + let mut info = RegexInfo { + emptyable, + exact, + prefix, + suffix, + match_q, + }; + info.bound(); + info +} + +fn analyze_alternation(subs: &[Hir]) -> RegexInfo { + let infos: Vec = subs.iter().map(analyze).collect(); + + let emptyable = infos.iter().any(|i| i.emptyable); + + let exact = if infos.iter().all(|i| i.exact.is_some()) { + Some( + infos + .iter() + .flat_map(|i| i.exact.as_ref().unwrap().iter().cloned()) + .collect(), + ) + } else { + None + }; + + // A common prefix exists only if every branch contributes one. + let prefix = if infos.iter().all(|i| !i.prefix.is_empty()) { + infos + .iter() + .flat_map(|i| i.prefix.iter().cloned()) + .collect() + } else { + BTreeSet::new() + }; + let suffix = if infos.iter().all(|i| !i.suffix.is_empty()) { + infos + .iter() + .flat_map(|i| i.suffix.iter().cloned()) + .collect() + } else { + BTreeSet::new() + }; + + let match_q = TrigramQuery::or(infos.into_iter().map(|i| i.match_q).collect()); + + RegexInfo { + emptyable, + exact, + prefix, + suffix, + match_q, + } +} + +/// Derive a necessary trigram condition from a regular expression pattern. +/// +/// Returns [`TrigramQuery::All`] when no useful condition can be derived (an +/// unparsable pattern, or one with no trigram-able literal structure such as +/// `a.b` or `.*`); callers must treat that as "recheck everything". +pub fn regex_to_trigram_query(pattern: &str) -> TrigramQuery { + // An unparsable pattern cannot be accelerated; rechecking is still safe. + let Ok(hir) = regex_syntax::parse(pattern) else { + return TrigramQuery::All; + }; + let info = analyze(&hir); + + let mut conditions = vec![info.match_q]; + if let Some(exact) = &info.exact { + if exact.is_empty() { + // The expression matches nothing. + return TrigramQuery::None; + } + conditions.push(trigrams_of_set(exact.iter())); + } + conditions.push(trigrams_of_set(info.prefix.iter())); + conditions.push(trigrams_of_set(info.suffix.iter())); + TrigramQuery::and(conditions) +} + +/// Whether a regular expression yields any trigram condition the index can use +/// to prune candidates. When it does not (e.g. `a.b`, `.*`, or a case-insensitive +/// pattern), callers should leave the predicate to a full scan rather than route +/// it to the index, which would otherwise have to ask the scan to recheck every +/// row -- a path the index result type (`AtLeast`) does not support. +pub fn regex_can_use_index(pattern: &str) -> bool { + regex_to_trigram_query(pattern) != TrigramQuery::All +} + +/// Collect the distinct trigram tokens referenced anywhere in the tree. +pub fn collect_tokens(query: &TrigramQuery, out: &mut HashSet) { + match query { + TrigramQuery::Trigram(token) => { + out.insert(*token); + } + TrigramQuery::And(items) | TrigramQuery::Or(items) => { + for item in items { + collect_tokens(item, out); + } + } + TrigramQuery::All | TrigramQuery::None => {} + } +} + +/// Evaluate the tree against a map of `trigram token -> posting list`. A token +/// missing from the map contributes an empty set (sound: a required trigram that +/// is absent everywhere yields no rows; an absent OR branch contributes +/// nothing). `All` / `None` are handled by the caller before evaluation. +pub fn eval_trigram_query( + query: &TrigramQuery, + bitmaps: &HashMap, +) -> RoaringTreemap { + match query { + TrigramQuery::Trigram(token) => bitmaps.get(token).cloned().unwrap_or_default(), + TrigramQuery::And(items) => { + let mut iter = items.iter(); + let mut acc = match iter.next() { + Some(first) => eval_trigram_query(first, bitmaps), + None => return RoaringTreemap::new(), + }; + for item in iter { + if acc.is_empty() { + break; + } + acc &= &eval_trigram_query(item, bitmaps); + } + acc + } + TrigramQuery::Or(items) => { + let mut acc = RoaringTreemap::new(); + for item in items { + acc |= &eval_trigram_query(item, bitmaps); + } + acc + } + TrigramQuery::All | TrigramQuery::None => RoaringTreemap::new(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A single trigram condition, hashed the same way the index hashes it. + fn tri(trigram: &str) -> TrigramQuery { + TrigramQuery::Trigram(ngram_to_token(trigram, NGRAM_N)) + } + + fn q(pattern: &str) -> TrigramQuery { + regex_to_trigram_query(pattern) + } + + #[test] + fn test_single_literal_trigram() { + assert_eq!(q("foo"), tri("foo")); + } + + #[test] + fn test_multi_trigram_literal() { + assert_eq!( + q("foobar"), + TrigramQuery::and(vec![tri("foo"), tri("oob"), tri("oba"), tri("bar")]) + ); + } + + #[test] + fn test_wildcard_splits_into_and() { + // `.*` breaks the literal run; both sides are required. + assert_eq!( + q("foo.*bar"), + TrigramQuery::and(vec![tri("foo"), tri("bar")]) + ); + } + + #[test] + fn test_alternation_is_or() { + assert_eq!( + q("(cat|dog)"), + TrigramQuery::or(vec![tri("cat"), tri("dog")]) + ); + } + + #[test] + fn test_anchors_are_transparent() { + assert_eq!( + q("^rhino"), + TrigramQuery::and(vec![tri("rhi"), tri("hin"), tri("ino")]) + ); + assert_eq!(q("nose$"), TrigramQuery::and(vec![tri("nos"), tri("ose")])); + } + + #[test] + fn test_boundary_trigram_recovered_across_groups() { + // A capturing group is not merged into the adjacent literals, so this + // exercises the suffix x prefix cross product that recovers the `foo` + // trigram straddling the `(o)` group boundary in "foobar". + assert_eq!( + q("fo(o)bar"), // spellchecker:disable-line + TrigramQuery::and(vec![tri("foo"), tri("oob"), tri("oba"), tri("bar")]) + ); + } + + #[test] + fn test_no_trigram_yields_all() { + // No run of three literal characters anywhere. + assert_eq!(q("a.b"), TrigramQuery::All); + assert_eq!(q(".*"), TrigramQuery::All); + // Every alternation branch is shorter than a trigram, so we must not + // require either two-character branch as a (non-existent) trigram. + assert_eq!(q("fo|ba"), TrigramQuery::All); // spellchecker:disable-line + } + + #[test] + fn test_case_insensitive_not_accelerated() { + // Unicode case folding (e.g. `(?i)c` also matches U+2102) does not agree + // with the index's normalization, so case-insensitive patterns are left + // unaccelerated (correct via recheck) rather than risk a false negative. + assert_eq!(q("(?i)Cat"), TrigramQuery::All); + } + + #[test] + fn test_unparsable_pattern_yields_all() { + assert_eq!(q("("), TrigramQuery::All); + } + + #[test] + fn test_large_alternation_stays_bounded() { + // More than MAX_SET_SIZE branches: must still produce a sound OR without + // panicking or exploding. + let pattern = (0..40) + .map(|i| format!("aa{i:02}zz")) + .collect::>() + .join("|"); + let result = q(&pattern); + // Each branch shares the trigram `aa0`/`aa1`/... and `zz`-ish endings; + // the important property is that it is a sound non-empty condition. + assert_ne!(result, TrigramQuery::None); + } + + #[test] + fn test_plus_requires_inner() { + // `(abc)+` must contain at least one `abc`. + assert_eq!(q("(abc)+"), tri("abc")); + } + + #[test] + fn test_optional_group_is_not_required() { + // `(foo)?bar` -> foo optional, bar required. + assert_eq!(q("(foo)?bar"), tri("bar")); + } + + #[test] + fn test_eval_and_or_with_missing_tokens() { + let foo = ngram_to_token("foo", NGRAM_N); + let bar = ngram_to_token("bar", NGRAM_N); + let mut bitmaps = HashMap::new(); + bitmaps.insert(foo, RoaringTreemap::from_iter([1u64, 2, 3])); + bitmaps.insert(bar, RoaringTreemap::from_iter([2u64, 3, 4])); + // `baz` is absent from the index. + + // AND intersects. + let and = TrigramQuery::and(vec![tri("foo"), tri("bar")]); + assert_eq!( + eval_trigram_query(&and, &bitmaps), + RoaringTreemap::from_iter([2u64, 3]) + ); + + // OR unions. + let or = TrigramQuery::or(vec![tri("foo"), tri("bar")]); + assert_eq!( + eval_trigram_query(&or, &bitmaps), + RoaringTreemap::from_iter([1u64, 2, 3, 4]) + ); + + // A missing token is empty: it zeroes an AND but is harmless in an OR. + let and_missing = TrigramQuery::and(vec![tri("foo"), tri("baz")]); + assert!(eval_trigram_query(&and_missing, &bitmaps).is_empty()); + let or_missing = TrigramQuery::or(vec![tri("foo"), tri("baz")]); + assert_eq!( + eval_trigram_query(&or_missing, &bitmaps), + RoaringTreemap::from_iter([1u64, 2, 3]) + ); + } + + #[test] + fn test_collect_tokens() { + let query = TrigramQuery::and(vec![ + tri("foo"), + TrigramQuery::or(vec![tri("bar"), tri("baz")]), + ]); + let mut tokens = HashSet::new(); + collect_tokens(&query, &mut tokens); + assert_eq!( + tokens, + HashSet::from([ + ngram_to_token("foo", NGRAM_N), + ngram_to_token("bar", NGRAM_N), + ngram_to_token("baz", NGRAM_N), + ]) + ); + } +} diff --git a/rust/lance/Cargo.toml b/rust/lance/Cargo.toml index 74e6faf5c07..440c3fb301a 100644 --- a/rust/lance/Cargo.toml +++ b/rust/lance/Cargo.toml @@ -175,6 +175,10 @@ required-features = ["cli"] name = "scalar_index" harness = false +[[bench]] +name = "regex_ngram" +harness = false + [[bench]] name = "merge_insert" harness = false diff --git a/rust/lance/benches/regex_ngram.rs b/rust/lance/benches/regex_ngram.rs new file mode 100644 index 00000000000..76f597ad9cb --- /dev/null +++ b/rust/lance/benches/regex_ngram.rs @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Benchmark: regex predicate scans over an ngram-indexed string column. +//! +//! Each query is a `regexp_match(doc, '...')` filter against a dataset that has +//! an NGram index on `doc`. The query set spans a selective AND pattern, an +//! alternation, a plain literal (rewritten to an infix LIKE before it reaches +//! the index), and a deliberately non-accelerable pattern (`a.b`, which yields +//! no trigram) that serves as a regression guard. +//! +//! On `main` none of these use the index (regex falls through to a full scan + +//! recheck); with the ngram-regex acceleration the index prunes candidates for +//! the first three while `a.b` stays a full scan. Capture a baseline on `main` +//! with `--save-baseline before_7130`, then compare after the change with +//! `--baseline before_7130`. + +use std::hint::black_box; +use std::sync::Arc; +use std::time::Duration; + +use arrow::array::AsArray; +use arrow_array::{RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use criterion::{Criterion, criterion_group, criterion_main}; +use futures::TryStreamExt; +use lance::Dataset; +use lance::index::DatasetIndexExt; +use lance_core::utils::tempfile::TempStrDir; +use lance_datagen::{RowCount, array}; +use lance_index::IndexType; +use lance_index::scalar::ScalarIndexParams; +#[cfg(target_os = "linux")] +use lance_testing::pprof::{Output, PProfProfiler}; + +const TOTAL: usize = 200_000; + +/// Build the `doc` column: random sentences with rare markers injected into a +/// small fraction of rows so the regex queries have controlled selectivity. +/// The markers (`zqxwvu`, `needlexyz`, `qwerasdf`) are unlikely to appear in +/// the generated English-word sentences. +fn build_docs() -> StringArray { + let mut sentence_gen = array::random_sentence(1, 30, false); + let base = sentence_gen + .generate_default(RowCount::from(TOTAL as u64)) + .unwrap(); + let base = base.as_string::(); + let docs = (0..TOTAL).map(|i| { + let sentence = base.value(i); + if i % 200 == 0 { + // ~0.5% of rows match `zqxwvu.*needlexyz` and `zqxwvu`. + format!("{sentence} zqxwvu needlexyz") + } else if i % 211 == 0 { + // A second marker for the alternation query. + format!("{sentence} qwerasdf") + } else { + sentence.to_string() + } + }); + StringArray::from_iter_values(docs) +} + +async fn build_dataset(tempdir: &TempStrDir) -> Arc { + let schema = Arc::new(Schema::new(vec![Field::new("doc", DataType::Utf8, false)])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(build_docs())]).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + + let mut dataset = Dataset::write(reader, tempdir.as_str(), None) + .await + .unwrap(); + dataset + .create_index( + &["doc"], + IndexType::NGram, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + Arc::new(dataset) +} + +async fn scan_filter(dataset: &Dataset, filter: &str) -> usize { + let mut scanner = dataset.scan(); + scanner.filter(filter).unwrap(); + let stream = scanner.try_into_stream().await.unwrap(); + let batches: Vec = stream.try_collect().await.unwrap(); + batches.iter().map(|b| b.num_rows()).sum() +} + +fn bench_regex_ngram(c: &mut Criterion) { + let rt = tokio::runtime::Runtime::new().unwrap(); + let tempdir = TempStrDir::default(); + let dataset = rt.block_on(build_dataset(&tempdir)); + + let queries = [ + ("selective_and", "regexp_match(doc, 'zqxwvu.*needlexyz')"), + ( + "alternation", + "regexp_match(doc, '(zqxwvu|qwerasdf|needlexyz)')", + ), + ("plain_literal", "regexp_match(doc, 'zqxwvu')"), + ("non_accelerable_a_dot_b", "regexp_match(doc, 'a.b')"), + ]; + + let mut group = c.benchmark_group("regex_ngram"); + group + .sample_size(10) + .measurement_time(Duration::from_secs(15)); + for (name, filter) in queries { + group.bench_function(name, |b| { + b.iter(|| black_box(rt.block_on(scan_filter(&dataset, filter)))); + }); + } + group.finish(); +} + +#[cfg(target_os = "linux")] +criterion_group!( + name = benches; + config = Criterion::default() + .significance_level(0.1) + .sample_size(10) + .with_profiler(PProfProfiler::new(100, Output::Flamegraph(None))); + targets = bench_regex_ngram); + +#[cfg(not(target_os = "linux"))] +criterion_group!( + name = benches; + config = Criterion::default().significance_level(0.1).sample_size(10); + targets = bench_regex_ngram); + +criterion_main!(benches); diff --git a/rust/lance/src/dataset/scanner.rs b/rust/lance/src/dataset/scanner.rs index 1112721bb33..2d75104d26e 100644 --- a/rust/lance/src/dataset/scanner.rs +++ b/rust/lance/src/dataset/scanner.rs @@ -8412,6 +8412,198 @@ mod test { .unwrap(); } + #[tokio::test] + async fn test_ngram_regex_index_scan() { + use arrow::array::AsArray; + + // A small, fixed corpus written across multiple fragments so the ngram + // index spans fragment boundaries. + let values = [ + "rhino", // 0 + "rhinos nose", // 1 + "cat", // 2 + "dog", // 3 + "cat dog", // 4 + "elephant", // 5 + "catalog", // 6 + "scatter", // 7 + "rhino horn", // 8 + "mouse", // 9 + "category", // 10 + "dogma", // 11 + ]; + let array = StringArray::from_iter_values(values); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "s", + DataType::Utf8, + false, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(array)]).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let write_params = WriteParams { + max_rows_per_file: 4, // 12 rows -> 3 fragments + ..Default::default() + }; + let mut dataset = Dataset::write(reader, "memory://test_ngram_regex", Some(write_params)) + .await + .unwrap(); + dataset + .create_index( + &["s"], + IndexType::NGram, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + assert!( + dataset.get_fragments().len() > 1, + "expected a multi-fragment dataset" + ); + + // Scan with `filter` and return the matched `s` values, sorted. + async fn matched(dataset: &Dataset, filter: &str) -> Vec { + let mut scan = dataset.scan(); + scan.filter(filter).unwrap(); + let batches = scan + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let mut out = Vec::new(); + for batch in batches { + let col = batch.column_by_name("s").unwrap().as_string::(); + out.extend(col.iter().flatten().map(|s| s.to_string())); + } + out.sort(); + out + } + + // `regexp_like`: a plain literal substring. + assert_eq!( + matched(&dataset, "regexp_like(s, 'rhino')").await, + ["rhino", "rhino horn", "rhinos nose"] + ); + // `regexp_match` (coerced to `IsNotNull(regexp_match(...))`) accelerates too. + assert_eq!( + matched(&dataset, "regexp_match(s, 'rhino')").await, + ["rhino", "rhino horn", "rhinos nose"] + ); + // Anchored: recheck must drop trigram false positives -- the `cat` + // trigram also occurs in cat dog / catalog / scatter / category. + assert_eq!(matched(&dataset, "regexp_like(s, 'cat$')").await, ["cat"]); + // AND across `.*`: row 8 ("rhino horn") shares the rhino trigrams but + // lacks the nose trigrams, so only "rhinos nose" survives. + assert_eq!( + matched(&dataset, "regexp_like(s, 'rhino.*nose')").await, + ["rhinos nose"] + ); + // Alternation. + assert_eq!( + matched(&dataset, "regexp_like(s, '(catalog|elephant)')").await, + ["catalog", "elephant"] + ); + // A non-accelerable pattern (no trigram derivable) still returns correct + // results via a full recheck. + assert_eq!(matched(&dataset, "regexp_like(s, 'o.m')").await, ["dogma"]); + // A case-insensitive flag is not accelerated (the index normalization + // disagrees with Unicode case folding) but must still return correct + // results via a full recheck -- here matching despite the upper-case + // pattern. This exercises the three-argument `regexp_like` flags path. + assert_eq!( + matched(&dataset, "regexp_like(s, 'RHINO', 'i')").await, + ["rhino", "rhino horn", "rhinos nose"] + ); + + // Infix LIKE is accelerated through the same machinery (a plain-literal + // `regexp_like` is rewritten to LIKE before it reaches the index). + assert_eq!( + matched(&dataset, "s LIKE '%rhino%'").await, + ["rhino", "rhino horn", "rhinos nose"] + ); + // Prefix LIKE: recheck drops "scatter", which contains the `cat` trigram + // but does not start with "cat". + assert_eq!( + matched(&dataset, "s LIKE 'cat%'").await, + ["cat", "cat dog", "catalog", "category"] + ); + + // The ngram index is actually engaged for every accelerated form. + for filter in [ + "regexp_like(s, 'rhino')", + "regexp_match(s, 'rhino')", + "s LIKE '%rhino%'", + ] { + let mut scan = dataset.scan(); + scan.filter(filter).unwrap(); + let plan = scan.create_plan().await.unwrap(); + let plan_str = format!( + "{}", + datafusion::physical_plan::displayable(plan.as_ref()).indent(true) + ); + assert!( + plan_str.contains("ScalarIndexQuery") && plan_str.contains("NGram"), + "expected ngram index usage for `{filter}`, got plan:\n{plan_str}" + ); + } + } + + #[tokio::test] + async fn test_ngram_regex_non_accelerable_recheck() { + // `a.b` yields no trigram, so the index returns "recheck everything". + // This must still produce ALL correct matches across fragments, not an + // empty set (a regression test for the AtLeast recheck path, which a + // single-match case would not catch). + let unit = ["acb", "dog", "axb", "cat", "qqq", "rhino"]; + let values: Vec<&str> = unit.iter().copied().cycle().take(60).collect(); + let array = StringArray::from_iter_values(values); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "text", + DataType::Utf8, + false, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(array)]).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let write_params = WriteParams { + max_rows_per_file: 20, // 60 rows -> 3 fragments + ..Default::default() + }; + let mut dataset = + Dataset::write(reader, "memory://test_ngram_regex_ne", Some(write_params)) + .await + .unwrap(); + dataset + .create_index( + &["text"], + IndexType::NGram, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + async fn count(dataset: &Dataset, filter: &str) -> usize { + let mut scan = dataset.scan(); + scan.filter(filter).unwrap(); + let batches = scan + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + batches.iter().map(|b| b.num_rows()).sum() + } + + // "acb" and "axb" each appear 10 times in the 60 rows -> 20 matches. + assert_eq!(count(&dataset, "regexp_match(text, 'a.b')").await, 20); + assert_eq!(count(&dataset, "regexp_like(text, 'a.b')").await, 20); + } + #[tokio::test] async fn test_like_prefix_with_btree_index() { // Create dataset with string data that has various prefixes From 27570a380f5f1155d48bbc48d66f2b17b5e37059 Mon Sep 17 00:00:00 2001 From: Dan Rammer Date: Tue, 16 Jun 2026 13:30:03 -0500 Subject: [PATCH 10/15] perf(mem_wal): parallelize fresh-tier source planning and execution (#7257) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary The LSM FTS and vector search planners (`LsmFtsSearchPlanner`, `LsmVectorSearchPlanner`) built each source's plan in a sequential `for` loop and unioned the arms under a single `SortPreservingMergeExec`. The merge polls every union arm from one task, so per-arm CPU (posting/index decode, BM25 and distance scoring) serialized even though the underlying IO awaits interleave — wall time grew linearly with the flushed-generation count. This: - builds the per-source plans concurrently with `try_join_all` (FTS + vector), - runs the cross-source block-list PK hashing concurrently (`block_list.rs`), - wraps the union in a round-robin `RepartitionExec` via a new `spawn_union_arms` helper so each arm gets its own driver task. Rows stay disjoint across partitions, so the per-partition TopK + sort-preserving merge semantics are unchanged. ## Changes - `scanner/exec.rs`: `spawn_union_arms` helper (round-robin repartition over the union). - `scanner/fts_search.rs`, `scanner/vector_search.rs`: concurrent per-source plan builds + `spawn_union_arms` over the union. - `scanner/block_list.rs`: concurrent flushed-generation PK-hash loads. ## Validation Validated end-to-end against a WAL FTS benchmark on minikube with object storage behind a 10ms/GET latency proxy. Read latency over a fresh tier as a function of flushed-generation count, p50: | generations | before | after | |---|---|---| | 2 | 1,164ms | 565ms | | 5 | 1,983ms | 610ms | | 10 | 3,585ms | 660ms | | 18 | 6,071ms | 759ms | Per-generation slope dropped from ~290ms/gen to ~12ms/gen. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Fable 5 --- .../src/dataset/mem_wal/scanner/block_list.rs | 52 ++++++++------ .../src/dataset/mem_wal/scanner/fts_search.rs | 55 ++++++++++----- .../dataset/mem_wal/scanner/vector_search.rs | 67 ++++++++++++------- 3 files changed, 113 insertions(+), 61 deletions(-) diff --git a/rust/lance/src/dataset/mem_wal/scanner/block_list.rs b/rust/lance/src/dataset/mem_wal/scanner/block_list.rs index 684fde48da1..8a293c3f988 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/block_list.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/block_list.rs @@ -47,9 +47,12 @@ pub async fn compute_source_block_lists( flushed_cache: Option<&Arc>, ) -> Result { // Hash each non-base source's membership, grouped by shard (generations are - // per-shard, so supersession is within-shard only). + // per-shard, so supersession is within-shard only). Flushed-generation PK + // scans (cold-cache S3 reads) run concurrently; order is irrelevant — the + // per-shard lists are sorted by generation below. let mut by_shard: ShardGenSets = HashMap::new(); let mut has_base = false; + let mut flushed_loads = Vec::new(); for source in sources { match source { LsmDataSource::BaseTable { .. } => has_base = true, @@ -72,14 +75,20 @@ pub async fn compute_source_block_lists( .. } => { // Cached by immutable path so repeated searches skip the PK scan. - let hashes = flushed_pk_hashes(path, pk_columns, session, flushed_cache).await?; - by_shard - .entry(*shard_id) - .or_default() - .push((*generation, hashes)); + flushed_loads.push(async move { + flushed_pk_hashes(path, pk_columns, session, flushed_cache) + .await + .map(|hashes| (*shard_id, *generation, hashes)) + }); } } } + for (shard_id, generation, hashes) in futures::future::try_join_all(flushed_loads).await? { + by_shard + .entry(shard_id) + .or_default() + .push((generation, hashes)); + } let mut blocked: SourceBlockLists = HashMap::new(); // Base (shardless, oldest) is superseded by every non-base generation. @@ -115,22 +124,25 @@ pub async fn fresh_tier_block_list( session: Option<&Arc>, flushed_cache: Option<&Arc>, ) -> Result>>> { - let mut sets = Vec::new(); - for source in sources { - let set = match source { - LsmDataSource::BaseTable { .. } => continue, - LsmDataSource::ActiveMemTable { batch_store, .. } => { - Arc::new(pk_hashes_from_batch_store(batch_store, pk_columns)?) - } + // Flushed PK scans run concurrently (cold-cache S3 reads); ordered + // try_join_all keeps the source order of the returned sets. + let sets = futures::future::try_join_all(sources.iter().map(|source| async move { + Ok::<_, lance_core::Error>(match source { + LsmDataSource::BaseTable { .. } => None, + LsmDataSource::ActiveMemTable { batch_store, .. } => Some(Arc::new( + pk_hashes_from_batch_store(batch_store, pk_columns)?, + )), LsmDataSource::FlushedMemTable { path, .. } => { - flushed_pk_hashes(path, pk_columns, session, flushed_cache).await? + Some(flushed_pk_hashes(path, pk_columns, session, flushed_cache).await?) } - }; - if !set.is_empty() { - sets.push(set); - } - } - Ok(sets) + }) + })) + .await?; + Ok(sets + .into_iter() + .flatten() + .filter(|set| !set.is_empty()) + .collect()) } /// Hash the PK membership of an in-memory memtable (active or frozen) from its diff --git a/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs b/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs index e3ef44d8b1a..626b0effe3c 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs @@ -167,23 +167,39 @@ impl LsmFtsSearchPlanner { .await?; let overfetch = self.overfetch_factor.max(1.0); - let mut per_source_plans: Vec> = Vec::with_capacity(sources.len()); - for source in &sources { - let is_active = matches!(source, LsmDataSource::ActiveMemTable { .. }); - let blocked = block_lists.get(&(source.shard_id(), source.generation())); - // Over-fetch a blocked source so the post-filter still yields k live - // rows. The active arm returns all matches (no builder limit), so its - // within-source dedup needs no over-fetch hint. - let fetch_k = if blocked.is_some() { - ((k as f64) * overfetch).ceil() as usize - } else { - k - }; - - let plan = self - .build_source_plan(source, column, &query, fetch_k, projection, is_active) - .await?; + // Stage the per-source over-fetch decisions, then build every source + // plan concurrently — the builds are independent and a sequential loop + // was the dominant serial planning cost at multiple generations. + let arm_inputs: Vec<_> = sources + .iter() + .map(|source| { + let is_active = matches!(source, LsmDataSource::ActiveMemTable { .. }); + let blocked = block_lists.get(&(source.shard_id(), source.generation())); + // Over-fetch a blocked source so the post-filter still yields k live + // rows. The active arm returns all matches (no builder limit), so its + // within-source dedup needs no over-fetch hint. + let fetch_k = if blocked.is_some() { + ((k as f64) * overfetch).ceil() as usize + } else { + k + }; + (source, is_active, blocked, fetch_k) + }) + .collect(); + let built = + futures::future::try_join_all(arm_inputs.iter().map( + |(source, is_active, _, fetch_k)| { + Box::pin(self.build_source_plan( + source, column, &query, *fetch_k, projection, *is_active, + )) + }, + )) + .await?; + let mut per_source_plans: Vec> = Vec::with_capacity(sources.len()); + for ((_, is_active, blocked, _), plan) in arm_inputs.iter().zip(built) { + let is_active = *is_active; + let blocked = *blocked; // Dedup, mirroring LsmVectorSearchPlanner: // * active: collapse duplicate-PK appends to the newest insert // (larger _rowid = inserted later). The FTS index is append-only, @@ -219,8 +235,11 @@ impl LsmFtsSearchPlanner { per_source_plans.into_iter().next().unwrap() } else { #[allow(deprecated)] - let union: Arc = Arc::new(UnionExec::new(per_source_plans)); - union + // The downstream `SortPreservingMergeExec` already spawns one driver + // task per input partition (one per union arm) via `spawn_buffered`, + // so each arm's per-arm CPU (posting decode, BM25) runs on its own + // task without an extra repartition. + Arc::new(UnionExec::new(per_source_plans)) }; let score_idx = merged.schema().index_of(SCORE_COLUMN).map_err(|_| { diff --git a/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs b/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs index b6b1f952b25..878063321aa 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs @@ -233,30 +233,47 @@ impl LsmVectorSearchPlanner { // `block_lists` is non-empty exactly when a newer generation exists. let refine_base = refine_base_table || !block_lists.is_empty(); + // Stage per-source over-fetch decisions, then build every KNN plan + // concurrently — the builds are independent and a sequential loop was + // the dominant serial planning cost at multiple generations. + let arm_inputs: Vec<_> = sources + .iter() + .map(|source| { + let generation = source.generation(); + let is_base = matches!(source, LsmDataSource::BaseTable { .. }); + let is_active = matches!(source, LsmDataSource::ActiveMemTable { .. }); + // Over-fetch when the post-source filter can drop candidates: a + // blocked source loses superseded rows; the active source's + // within-source dedup collapses duplicate-PK HNSW nodes. Block + // lookup is per shard — generations are per-shard. + let blocked = block_lists.get(&(source.shard_id(), generation)); + let fetch_k = if blocked.is_some() || is_active { + ((k as f64) * overfetch_factor).ceil() as usize + } else { + k + }; + (source, is_base, is_active, blocked, fetch_k) + }) + .collect(); + let built = futures::future::try_join_all(arm_inputs.iter().map( + |(source, is_base, _, _, fetch_k)| { + Box::pin(self.build_knn_plan( + source, + query_vector, + *fetch_k, + nprobes, + projection, + *is_base && refine_base, + )) + }, + )) + .await?; + let mut knn_plans = Vec::new(); - for source in &sources { - let generation = source.generation(); - let is_base = matches!(source, LsmDataSource::BaseTable { .. }); - let is_active = matches!(source, LsmDataSource::ActiveMemTable { .. }); - // Over-fetch when the post-source filter can drop candidates: a - // blocked source loses superseded rows; the active source's - // within-source dedup collapses duplicate-PK HNSW nodes. Block - // lookup is per shard — generations are per-shard. - let blocked = block_lists.get(&(source.shard_id(), generation)); - let fetch_k = if blocked.is_some() || is_active { - ((k as f64) * overfetch_factor).ceil() as usize - } else { - k - }; - let knn = Box::pin(self.build_knn_plan( - source, - query_vector, - fetch_k, - nprobes, - projection, - is_base && refine_base, - )) - .await?; + for ((_, is_base, is_active, blocked, _), knn) in arm_inputs.iter().zip(built) { + let is_base = *is_base; + let is_active = *is_active; + let blocked = *blocked; // Make each source independently newest-per-PK before the union: // * active: the append-only HNSW returns one node per inserted // version, so collapse duplicate PKs to the newest insert @@ -301,6 +318,10 @@ impl LsmVectorSearchPlanner { // No cross-source dedup needed (see struct doc): SortExec(per partition) // + SortPreservingMerge does the p-way distance-ordered top-k merge. #[allow(deprecated)] + // The downstream `SortPreservingMergeExec` already spawns one driver + // task per input partition (one per union arm) via `spawn_buffered`, so + // each arm's per-arm CPU (HNSW search, distance refine) runs on its own + // task without an extra repartition. let merged: Arc = Arc::new(UnionExec::new(knn_plans)); let distance_idx = merged.schema().index_of(DISTANCE_COLUMN).map_err(|_| { From bf9f706fb4ebb5605a65c1f499d1cb9a10de996e Mon Sep 17 00:00:00 2001 From: George Stamatakis <126914070+gstamatakis95@users.noreply.github.com> Date: Tue, 16 Jun 2026 20:48:41 +0200 Subject: [PATCH 11/15] fix(python): python binding of `Compaction.commit` (#7210) Closes #7209 - `python/src/dataset/optimize.rs`: `Compaction.commit` gains an optional `options` dict parameter (`#[pyo3(signature = (dataset, rewrites, options = None))]`), parsed through the same `parse_compaction_options` helper that `plan` and `execute` already use, seeded from the dataset's `manifest.config` exactly like `execute`. Absent or `None` preserves the old behavior. The TODO comment is removed. - `python/python/lance/lance/optimize.pyi`: stub updated to `commit(dataset, rewrites, options: Optional[CompactionOptions] = None)`. - `python/python/tests/test_optimize.py`: new parametrized test `test_defer_index_remap_via_commit_options`. With `Compaction.commit(dataset, rewrites, options={"defer_index_remap": True})` a `__lance_frag_reuse` system index appears in `describe_indices()`. Without options it does not (inline remap occurs). Mirrors the adjacent `test_defer_index_remap` setup. --- python/python/lance/lance/optimize.pyi | 6 ++-- python/python/tests/test_optimize.py | 41 ++++++++++++++++++++++++++ python/src/dataset/optimize.rs | 14 +++++++-- 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/python/python/lance/lance/optimize.pyi b/python/python/lance/lance/optimize.pyi index 9a26d23c003..c4b6b6546e6 100644 --- a/python/python/lance/lance/optimize.pyi +++ b/python/python/lance/lance/optimize.pyi @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List +from typing import List, Optional from lance import LanceDataset from lance.fragment import FragmentMetadata @@ -51,5 +51,7 @@ class Compaction: def plan(dataset: "LanceDataset", options: CompactionOptions) -> CompactionPlan: ... @staticmethod def commit( - dataset: "LanceDataset", rewrites: List[RewriteResult] + dataset: "LanceDataset", + rewrites: List[RewriteResult], + options: Optional[CompactionOptions] = None, ) -> CompactionMetrics: ... diff --git a/python/python/tests/test_optimize.py b/python/python/tests/test_optimize.py index ccd889db116..049ce2cc3a5 100644 --- a/python/python/tests/test_optimize.py +++ b/python/python/tests/test_optimize.py @@ -324,6 +324,47 @@ def test_defer_index_remap(tmp_path: Path): assert any(idx.name == "__lance_frag_reuse" for idx in indices) +@pytest.mark.parametrize("use_commit_options", [True, False]) +def test_defer_index_remap_via_commit_options(tmp_path: Path, use_commit_options: bool): + """Compaction.commit respects defer_index_remap passed in options. + + When options={"defer_index_remap": True} is supplied to Compaction.commit + the __lance_frag_reuse system index must appear in describe_indices(). + When the option is omitted (default) no such system index is written. + """ + base_dir = tmp_path / f"dataset_commit_opts_{use_commit_options}" + data = pa.table({"i": range(6_000), "val": range(6_000)}) + dataset = lance.write_dataset(data, base_dir, max_rows_per_file=1_000) + dataset.create_scalar_index("i", "BTREE") + dataset.delete("i < 500") + + plan = Compaction.plan( + dataset, + options=dict(target_rows_per_fragment=2_000, num_threads=1), + ) + rewrites = [task.execute(dataset) for task in plan.tasks] + + if use_commit_options: + Compaction.commit(dataset, rewrites, options={"defer_index_remap": True}) + else: + Compaction.commit(dataset, rewrites) + + dataset = lance.dataset(base_dir) + indices = dataset.describe_indices() + has_frag_reuse = any(idx.name == "__lance_frag_reuse" for idx in indices) + + if use_commit_options: + assert has_frag_reuse, ( + "expected __lance_frag_reuse system index when defer_index_remap=True " + "is passed to Compaction.commit" + ) + else: + assert not has_frag_reuse, ( + "did not expect __lance_frag_reuse system index when options is omitted " + "from Compaction.commit" + ) + + @pytest.mark.filterwarnings("ignore::DeprecationWarning") def test_describe_indices_matches_list_indices_for_frag_reuse(tmp_path: Path): """describe_indices() and list_indices() must agree on the index_type diff --git a/python/src/dataset/optimize.rs b/python/src/dataset/optimize.rs index 33aa32b94cd..4bb29246f45 100644 --- a/python/src/dataset/optimize.rs +++ b/python/src/dataset/optimize.rs @@ -554,26 +554,34 @@ impl PyCompaction { /// new version once committed. /// rewrites : List[RewriteResult] /// The results of the compaction tasks to include in the commit. + /// options : dict, optional + /// Compaction options to apply at commit time. + /// When absent or ``None``, defaults to ``CompactionOptions::default()``. /// /// Returns /// ------- /// CompactionMetrics #[staticmethod] + #[pyo3(signature = (dataset, rewrites, options = None))] pub fn commit( dataset: Bound, rewrites: Vec, + options: Option>, ) -> PyResult { let dataset_ref = unwrap_dataset(dataset)?; let dataset = dataset_ref.borrow().clone(); + let config = dataset.ds.manifest.config.clone(); + let opts = match options { + Some(ref dict) => parse_compaction_options(dict, &config)?, + None => CompactionOptions::default(), + }; let rewrites: Vec = rewrites.into_iter().map(|r| r.0).collect(); let mut new_ds = dataset.ds.as_ref().clone(); - // TODO: pass compaction option from plan and execute time - let options: CompactionOptions = CompactionOptions::default(); let fut = commit_compaction( &mut new_ds, rewrites, Arc::new(DatasetIndexRemapperOptions::default()), - &options, + &opts, ); let metrics = rt() .block_on(None, fut)? From 8e98f511eb38568d0d68799ae8e6dff581c52b3b Mon Sep 17 00:00:00 2001 From: WenDing-Y <1062698930@qq.com> Date: Wed, 17 Jun 2026 02:52:20 +0800 Subject: [PATCH 12/15] fix(dataset)!: fail-fast casting for columns with attached indices (#7158) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BREAKING CHANGE: Callers that previously depended on the silent index-drop behavior must now explicitly call `drop_index()` before `alter_columns` with a cast, then rebuild the index afterward. ## Summary Currently, calling `alter_columns` with a cast on a column that has an attached index **silently drops the index** without any warning or error. This has caused production incidents where vector search silently regressed to brute-force scan. This PR adds a **fail-fast check** before any cast is performed: if a target column has any index attached, the operation is rejected with a clear error message that names the affected column(s), the index(es), and the required remediation (`drop_index()` → cast → rebuild index). ## Changes - Added index-awareness check in `alter_columns` that scans cast target columns for attached indices - Returns a descriptive `Error::invalid_input` with column names, index names, and remediation steps when a conflict is detected - Updated existing tests that previously relied on silent index-drop behavior to explicitly `drop_index()` before casting - Added new test `test_alter_columns_cast_fails_with_attached_index` covering the IVF_PQ index path: verifies the error message, schema integrity, index preservation, and that the cast succeeds after a manual drop --- python/python/tests/test_vector_index.py | 2 + rust/lance/src/dataset/schema_evolution.rs | 162 ++++++++++++++++++++- 2 files changed, 160 insertions(+), 4 deletions(-) diff --git a/python/python/tests/test_vector_index.py b/python/python/tests/test_vector_index.py index 292b8079706..4e3addfedb8 100644 --- a/python/python/tests/test_vector_index.py +++ b/python/python/tests/test_vector_index.py @@ -1772,6 +1772,8 @@ def test_index_cast_centroids(tmp_path): values = pa.array([x for arr in centroids for x in arr], pa.float32()) centroids = pa.FixedSizeListArray.from_arrays(values, 128) + # Cast invalidates the attached index; drop it first per the new contract. + dataset.drop_index(index_name) dataset.alter_columns(dict(path="vector", data_type=pa.list_(pa.float16(), 128))) # centroids are f32, but the column is now f16 diff --git a/rust/lance/src/dataset/schema_evolution.rs b/rust/lance/src/dataset/schema_evolution.rs index a2c8f05a89f..6ffa7b45396 100644 --- a/rust/lance/src/dataset/schema_evolution.rs +++ b/rust/lance/src/dataset/schema_evolution.rs @@ -12,6 +12,7 @@ use super::{ transaction::{Operation, Transaction}, write::cleanup_data_fragments, }; +use crate::index::DatasetIndexExt; use crate::{Error, Result, io::exec::Planner}; use arrow::compute::CastOptions; use arrow::compute::can_cast_types; @@ -736,6 +737,41 @@ pub(super) async fn alter_columns( new_schema.validate()?; + // If any column being cast has an attached index, fail fast. Cast operations + // rewrite the underlying column data and silently invalidate any index on the + // affected column(s). The current behavior is to drop such indices without + // warning, which has caused production incidents where vector search silently + // regressed to brute-force scan. We require users to explicitly drop the + // index before altering the column type, so the action is never silent. + if !cast_fields.is_empty() { + let indices = dataset.load_indices().await?; + let affected: Vec<&lance_table::format::IndexMetadata> = indices + .iter() + .filter(|idx| { + cast_fields + .iter() + .any(|(old, _)| idx.fields.contains(&old.id)) + }) + .collect(); + if !affected.is_empty() { + let affected_cols: Vec = cast_fields + .iter() + .filter(|(old, _)| affected.iter().any(|i| i.fields.contains(&old.id))) + .map(|(old, _)| old.name.clone()) + .collect(); + let affected_idx_names: Vec = affected.iter().map(|i| i.name.clone()).collect(); + return Err(Error::invalid_input(format!( + "Cannot cast column(s) [{}] to a new type: they have {} index(es) \ + attached: [{}]. Cast rewrites column data and invalidates any index \ + on the affected column(s). Drop the index(es) with drop_index() \ + before altering, then recreate them after the cast completes.", + affected_cols.join(", "), + affected.len(), + affected_idx_names.join(", "), + ))); + } + } + // If we aren't casting a column, we don't need to touch the fragments. let transaction = if cast_fields.is_empty() { Transaction::new( @@ -2574,7 +2610,6 @@ mod test { ) -> Result<()> { // Create a table with 2 scalar columns, 1 vector column - use crate::index::DatasetIndexExt; use arrow::datatypes::{Int32Type, Int64Type}; use arrow_array::{Float16Array, Float32Array, Int64Array, ListArray}; use half::f16; @@ -2675,7 +2710,10 @@ mod test { assert_eq!(f.files.len(), 2); }); - // Cast scalar column with index, should not keep index (TODO: keep it) + // Cast scalar column with index. The index must be dropped first; cast + // is now a fail-fast operation when an index is attached, see + // test_alter_columns_cast_fails_with_attached_index for that path. + dataset.drop_index("i_idx").await?; dataset .alter_columns(&[ColumnAlteration::new("i".into()).cast_to(DataType::Int64)]) .await?; @@ -2696,7 +2734,8 @@ mod test { ]); assert_eq!(&ArrowSchema::from(dataset.schema()), &expected_schema); - // We currently lose the index when casting a column + // The scalar index on `i` is gone (we dropped it); the vector index on + // `vec` is still present. let indices = dataset.load_indices().await?; assert_eq!(indices.len(), 1); @@ -2705,7 +2744,8 @@ mod test { assert_eq!(f.files.len(), 3); }); - // Cast vector column, should not keep index (TODO: keep it) + // Cast vector column. Drop its index first (same reason as above). + dataset.drop_index("vec_idx").await?; dataset .alter_columns(&[ ColumnAlteration::new("vec".into()).cast_to(DataType::FixedSizeList( @@ -2773,6 +2813,120 @@ mod test { Ok(()) } + /// Cast on a column with an attached index must fail fast rather than + /// silently dropping the index. This guards against the historical behavior + /// where cast would rewrite column data and the index would vanish without + /// any error or warning, causing vector search to silently regress to a + /// brute-force scan. + #[rstest] + #[tokio::test] + async fn test_alter_columns_cast_fails_with_attached_index( + #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)] + data_storage_version: LanceFileVersion, + ) -> Result<()> { + use lance_arrow::FixedSizeListArrayExt; + use lance_index::IndexType; + use lance_linalg::distance::MetricType; + use lance_testing::datagen::generate_random_array; + + use crate::index::vector::VectorIndexParams; + + // Build a small dataset with one indexed vector column. + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "vec", + DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float32, true)), + 64, + ), + false, + )])); + let nrows = 256; + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new( + ::try_new_from_values( + generate_random_array(64 * nrows as usize), + 64, + ) + .unwrap(), + )], + )?; + + let test_dir = TempStrDir::default(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), + &test_dir, + Some(WriteParams { + data_storage_version: Some(data_storage_version), + ..Default::default() + }), + ) + .await?; + + // Build an IVF_PQ index on the vector column. + let params = VectorIndexParams::ivf_pq(4, 8, 8, MetricType::L2, 50); + dataset + .create_index(&["vec"], IndexType::Vector, None, ¶ms, false) + .await?; + + let indices_before = dataset.load_indices().await?; + assert_eq!(indices_before.len(), 1, "precondition: index exists"); + let index_name = indices_before[0].name.clone(); + + // Attempting to cast the indexed column must fail with a clear message + // that names the offending index(es). + let result = dataset + .alter_columns(&[ + ColumnAlteration::new("vec".into()).cast_to(DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float16, true)), + 64, + )), + ]) + .await; + let err = result.expect_err("cast on indexed column should fail"); + let msg = err.to_string(); + assert!( + msg.contains("vec") && msg.contains(&index_name), + "error should mention column and index name, got: {msg}" + ); + assert!( + msg.contains("drop_index"), + "error should suggest the remediation, got: {msg}" + ); + + // The dataset must be unchanged: schema is still float32, index still present. + assert_eq!( + dataset.schema().field("vec").unwrap().data_type(), + DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float32, true)), + 64, + ), + ); + let indices_after = dataset.load_indices().await?; + assert_eq!(indices_after.len(), 1, "index should still exist"); + assert_eq!(indices_after[0].name, index_name); + + // Sanity check: after dropping the index, the same cast should succeed. + dataset.drop_index(&index_name).await?; + dataset + .alter_columns(&[ + ColumnAlteration::new("vec".into()).cast_to(DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float16, true)), + 64, + )), + ]) + .await?; + assert_eq!( + dataset.schema().field("vec").unwrap().data_type(), + DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float16, true)), + 64, + ), + ); + + Ok(()) + } + #[rstest] #[tokio::test] async fn test_drop_columns( From 80e35ecfb93f49ccbc2adf678b089312be309136 Mon Sep 17 00:00:00 2001 From: Justin Miller Date: Tue, 16 Jun 2026 13:52:29 -0500 Subject: [PATCH 13/15] feat: bump lance-namespace-reqwest-client to 0.8.6 (source_task_size) (#7254) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Context Linear: [GEN-476](https://linear.app/lancedb/issue/GEN-476) lance-namespace [0.8.6](https://github.com/lance-format/lance-namespace/releases) added an optional `source_task_size` field to `RefreshMaterializedViewRequest` (chunker materialized-view refresh work-item size, bounds per-actor memory). The reqwest client in this repo was pinned at `0.8.4`, so the field was dropped when `RestNamespace` deserialized the request — never reaching the server. ## Change Bumps `lance-namespace-reqwest-client` `0.8.4` → `0.8.6` (workspace `Cargo.toml` + `Cargo.lock`). Additive and **code-free**: the PyO3 binding (`python/src/namespace.rs`) `depythonize`s the typed `RefreshMaterializedViewRequest`, so the new field flows automatically once the crate carries it. ## Test plan - `cargo update -p lance-namespace-reqwest-client` → `v0.8.6` - `cargo check -p lance-namespace` → clean (builds against reqwest-client 0.8.6) - *(Local full build of the AWS-credential-vendor path hits a pre-existing `aws-smithy-types` E0119 under rustc 1.91, unrelated to this bump — CI's pinned toolchain builds it.)* ## Follow-up Publish a pylance beta from this so geneva can pin `lance-namespace>=0.8.6` + the new pylance and have `source_task_size` transit `db://` end-to-end. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.8 Co-authored-by: Will Jones --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index a95f530de38..657d9fde72c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,7 +72,7 @@ lance-linalg = { version = "=8.0.0-beta.14", path = "./rust/lance-linalg" } lance-namespace = { version = "=8.0.0-beta.14", path = "./rust/lance-namespace" } lance-namespace-impls = { version = "=8.0.0-beta.14", path = "./rust/lance-namespace-impls" } lance-namespace-datafusion = { version = "=7.0.0-beta.9", path = "./rust/lance-namespace-datafusion" } -lance-namespace-reqwest-client = "0.8.4" +lance-namespace-reqwest-client = "0.8.6" lance-select = { version = "=8.0.0-beta.14", path = "./rust/lance-select" } lance-tokenizer = { version = "=8.0.0-beta.14", path = "./rust/lance-tokenizer" } lance-table = { version = "=8.0.0-beta.14", path = "./rust/lance-table" } From 1b78086bbaa51b81a342522f6af0d879c26070be Mon Sep 17 00:00:00 2001 From: zhangyang <33365824+zhangyang0418@users.noreply.github.com> Date: Wed, 17 Jun 2026 02:58:34 +0800 Subject: [PATCH 14/15] fix: handle empty batch from deletion in add_columns_from_stream (#7233) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Fixes a panic in `add_columns` when a fragment's deleted rows fill an entire read batch. Closes #7232. ## Root cause In `add_columns_from_stream`, the updater reads each fragment in physical batches and applies the deletion vector via `filter_record_batch`. When **every row in a physical batch is deleted**, the filter yields a **0-row batch**. The outer loop then sets `rows_remaining = 0`, the inner `while rows_remaining > 0` loop never runs, `batches` stays empty, and `concat_batches(&batches[0].schema(), ..)` panics with `index out of bounds: the len is 0 but the index is 0`. This was hit in production using a merge-columns workflow over a dataset whose fragment had a deletion file, where the deleted rows happened to align with a read-batch boundary. ## Fix When the updater yields a 0-row batch, feed an empty batch back to the updater (to keep it in sync) and continue: ```rust if rows_remaining == 0 { updater.update(RecordBatch::new_empty(stream.schema())).await?; continue; } ``` ## Test Adds `test_add_columns_with_fully_deleted_batch`: writes a single fragment with 105 rows, deletes the trailing 5 rows so that — read with `batch_size=50` — the last batch `[100..105)` is fully deleted, then verifies `add_columns` succeeds and the new column has the correct values. The test panics on the unpatched code and passes with the fix. Co-authored-by: 张杨 Co-authored-by: Will Jones --- rust/lance/src/dataset/schema_evolution.rs | 74 ++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/rust/lance/src/dataset/schema_evolution.rs b/rust/lance/src/dataset/schema_evolution.rs index 6ffa7b45396..5ef35a33ab7 100644 --- a/rust/lance/src/dataset/schema_evolution.rs +++ b/rust/lance/src/dataset/schema_evolution.rs @@ -616,6 +616,17 @@ async fn add_columns_from_stream( debug_assert_eq!(batch.num_columns(), 1); let mut rows_remaining = batch.num_rows(); + // The updater yields an empty batch when every row in a read batch + // has been deleted (e.g. a whole batch falls within the deletion + // vector). There is nothing to pull from the stream in that case, so + // feed an empty batch back to keep the updater in sync and continue. + if rows_remaining == 0 { + updater + .update(RecordBatch::new_empty(stream.schema())) + .await?; + continue; + } + let mut batches = Vec::new(); while rows_remaining > 0 { @@ -1073,6 +1084,69 @@ mod test { Ok(()) } + #[tokio::test] + async fn test_add_columns_with_fully_deleted_batch() -> Result<()> { + // Regression test: when an entire read batch has been deleted, the + // updater yields a 0-row batch. The inner loop then never runs and + // `batches` stays empty, so `concat_batches(&batches[0]..)` used to + // panic with "index out of bounds: the len is 0 but the index is 0". + // + // A single fragment holds 105 rows; deleting the trailing 5 rows means + // that, when read with batch_size=50, the third batch [100..105) is + // fully filtered out and produces an empty batch. + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "i", + DataType::Int32, + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..105))], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + + let test_dir = TempStrDir::default(); + let test_uri = &test_dir; + let mut dataset = Dataset::write( + reader, + test_uri, + Some(WriteParams { + max_rows_per_file: 200, // keep all rows in a single fragment + ..Default::default() + }), + ) + .await?; + + // Delete the entire trailing batch [100..105). + dataset.delete("i >= 100").await?; + assert_eq!(dataset.count_rows(None).await?, 100); + + let new_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "j", + DataType::Int32, + false, + )])); + let new_batch = RecordBatch::try_new( + new_schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..100))], + )?; + let reader = RecordBatchIterator::new(vec![Ok(new_batch)], new_schema.clone()); + + // Read with batch_size=50 so the deleted trailing rows form a full empty batch. + dataset + .add_columns(NewColumnTransform::Reader(Box::new(reader)), None, Some(50)) + .await?; + + let data = dataset.scan().try_into_batch().await?; + assert_eq!(data.num_rows(), 100); + assert_eq!( + data.column_by_name("j").unwrap().as_ref(), + &Int32Array::from_iter_values(0..100) + ); + + Ok(()) + } + #[rstest] #[tokio::test] async fn test_add_columns_cleans_up_blob_v2_data_on_stream_error( From 07f965dd24a63935a3e8cd2155e1928535a5b8a7 Mon Sep 17 00:00:00 2001 From: Eunjin Song Date: Tue, 16 Jun 2026 12:50:43 -0700 Subject: [PATCH 15/15] feat(java): expose SourceDedupeBehavior in MergeInsertParams The Rust core's MergeInsertBuilder supports source_dedupe_behavior (Fail/FirstSeen) to control how duplicate source rows that match the same target row are handled, but the Java binding never wired it. Java callers were stuck on the default (Fail) with no way to opt into FirstSeen. Add SourceDedupeBehavior enum + withSourceDedupeBehavior() builder to the Java MergeInsertParams, pass it through the JNI layer, and re-export SourceDedupeBehavior from lance::dataset so the binding can import it alongside the other merge types. Tests cover both enum values across the JNI boundary: FirstSeen keeps the first duplicate source row, Fail errors on duplicate source keys and leaves the dataset unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- java/lance-jni/src/merge_insert.rs | 28 ++++++- .../org/lance/merge/MergeInsertParams.java | 43 +++++++++++ .../test/java/org/lance/MergeInsertTest.java | 73 +++++++++++++++++++ rust/lance/src/dataset.rs | 4 +- 4 files changed, 145 insertions(+), 3 deletions(-) diff --git a/java/lance-jni/src/merge_insert.rs b/java/lance-jni/src/merge_insert.rs index df4d63bd2f6..4d332ca2c09 100644 --- a/java/lance-jni/src/merge_insert.rs +++ b/java/lance-jni/src/merge_insert.rs @@ -12,7 +12,8 @@ use jni::objects::{JObject, JString, JValueGen}; use jni::sys::jlong; use lance::dataset::scanner::ExprFilter; use lance::dataset::{ - MergeInsertBuilder, MergeStats, WhenMatched, WhenNotMatched, WhenNotMatchedBySource, + MergeInsertBuilder, MergeStats, SourceDedupeBehavior, WhenMatched, WhenNotMatched, + WhenNotMatchedBySource, }; use lance_core::datatypes::Schema; use lance_index::mem_wal::MergedGeneration; @@ -52,6 +53,7 @@ fn inner_merge_insert<'local>( let retry_timeout_ms = extract_retry_timeout_ms(env, &jparam)?; let skip_auto_cleanup = extract_skip_auto_cleanup(env, &jparam)?; let use_index = extract_use_index(env, &jparam)?; + let source_dedupe_behavior = extract_source_dedupe_behavior(env, &jparam)?; let marked_generations = extract_marked_generations(env, &jparam)?; let (new_ds, merge_stats) = unsafe { @@ -71,6 +73,7 @@ fn inner_merge_insert<'local>( .retry_timeout(Duration::from_millis(retry_timeout_ms as u64)) .skip_auto_cleanup(skip_auto_cleanup) .use_index(use_index) + .source_dedupe_behavior(source_dedupe_behavior) .mark_generations_as_merged(marked_generations) .try_build()?; @@ -241,6 +244,29 @@ fn extract_use_index<'local>(env: &mut JNIEnv<'local>, jparam: &JObject) -> Resu Ok(use_index) } +fn extract_source_dedupe_behavior<'local>( + env: &mut JNIEnv<'local>, + jparam: &JObject, +) -> Result { + let behavior: JString = env + .call_method( + jparam, + "sourceDedupeBehaviorValue", + "()Ljava/lang/String;", + &[], + )? + .l()? + .into(); + let behavior = behavior.extract(env)?; + match behavior.as_str() { + "Fail" => Ok(SourceDedupeBehavior::Fail), + "FirstSeen" => Ok(SourceDedupeBehavior::FirstSeen), + _ => Err(Error::input_error(format!( + "Illegal source_dedupe_behavior: {behavior}", + ))), + } +} + fn extract_marked_generations<'local>( env: &mut JNIEnv<'local>, jparam: &JObject, diff --git a/java/src/main/java/org/lance/merge/MergeInsertParams.java b/java/src/main/java/org/lance/merge/MergeInsertParams.java index 2ae27b67cba..ab67e022604 100644 --- a/java/src/main/java/org/lance/merge/MergeInsertParams.java +++ b/java/src/main/java/org/lance/merge/MergeInsertParams.java @@ -39,6 +39,7 @@ public class MergeInsertParams { private long retryTimeoutMs = 30 * 1000; private boolean skipAutoCleanup = false; private boolean useIndex = true; + private SourceDedupeBehavior sourceDedupeBehavior = SourceDedupeBehavior.Fail; private List markedGenerations = Collections.emptyList(); public MergeInsertParams(List on) { @@ -244,6 +245,25 @@ public MergeInsertParams withUseIndex(boolean useIndex) { return this; } + /** + * Control how duplicate source rows that match the same target row are handled. + * + *

Default is {@link SourceDedupeBehavior#Fail}, which errors if the source contains duplicate + * join keys. Use {@link SourceDedupeBehavior#FirstSeen} to keep the first encountered row and + * skip subsequent duplicates. + * + *

If the source contains duplicates and {@code FirstSeen} behavior doesn't match your needs, + * sort the source data before passing it to the merge insert operation. + * + * @param sourceDedupeBehavior The behavior to apply when duplicate source rows are found + * @return This MergeInsertParams instance + */ + public MergeInsertParams withSourceDedupeBehavior(SourceDedupeBehavior sourceDedupeBehavior) { + Preconditions.checkNotNull(sourceDedupeBehavior); + this.sourceDedupeBehavior = sourceDedupeBehavior; + return this; + } + /** * Mark MemWAL generations as merged into the base table. * @@ -319,6 +339,14 @@ public boolean useIndex() { return useIndex; } + public SourceDedupeBehavior sourceDedupeBehavior() { + return sourceDedupeBehavior; + } + + public String sourceDedupeBehaviorValue() { + return sourceDedupeBehavior.name(); + } + @Override public String toString() { return MoreObjects.toStringHelper(this) @@ -337,6 +365,7 @@ public String toString() { .add("retryTimeoutMs", retryTimeoutMs) .add("skipAutoCleanup", skipAutoCleanup) .add("useIndex", useIndex) + .add("sourceDedupeBehavior", sourceDedupeBehavior) .toString(); } @@ -396,4 +425,18 @@ public enum WhenNotMatchedBySource { */ DeleteIf, } + + /** + * Describes how to handle duplicate source rows that match the same target row. + * + *

If the source contains duplicates and {@code FirstSeen} behavior doesn't match your needs, + * sort the source data before passing it to the merge insert operation. + */ + public enum SourceDedupeBehavior { + /** Fail the operation if duplicates are found (default). */ + Fail, + + /** Keep the first seen value and skip subsequent duplicates. */ + FirstSeen, + } } diff --git a/java/src/test/java/org/lance/MergeInsertTest.java b/java/src/test/java/org/lance/MergeInsertTest.java index b738ef8852d..944a9060675 100644 --- a/java/src/test/java/org/lance/MergeInsertTest.java +++ b/java/src/test/java/org/lance/MergeInsertTest.java @@ -257,6 +257,30 @@ private VectorSchemaRoot buildSource(Schema schema, RootAllocator allocator) { return root; } + /** + * Build a source whose join key {@code id=0} appears twice ("First 0", then "Second 0"), so the + * source-dedupe behavior is exercised. Remaining ids (3, 4) are unique matches. + */ + private VectorSchemaRoot buildDuplicateKeySource(Schema schema, RootAllocator allocator) { + List sourceIds = Arrays.asList(0, 0, 3, 4); + List sourceNames = Arrays.asList("First 0", "Second 0", "Source 3", "Source 4"); + + VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator); + root.allocateNew(); + + IntVector idVector = (IntVector) root.getVector("id"); + VarCharVector nameVector = (VarCharVector) root.getVector("name"); + + for (int i = 0; i < sourceIds.size(); i++) { + idVector.setSafe(i, sourceIds.get(i)); + nameVector.setSafe(i, sourceNames.get(i).getBytes(StandardCharsets.UTF_8)); + } + + root.setRowCount(sourceIds.size()); + + return root; + } + private ArrowArrayStream convertToStream(VectorSchemaRoot root, RootAllocator allocator) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); @@ -275,6 +299,55 @@ private ArrowArrayStream convertToStream(VectorSchemaRoot root, RootAllocator al return stream; } + @Test + public void testSourceDedupeFirstSeenKeepsFirst() throws Exception { + // Source has two rows for id=0 ("First 0" then "Second 0"). FirstSeen keeps + // the first encountered row and skips the duplicate. + + try (VectorSchemaRoot source = buildDuplicateKeySource(testDataset.getSchema(), allocator)) { + try (ArrowArrayStream sourceStream = convertToStream(source, allocator)) { + MergeInsertResult result = + dataset.mergeInsert( + new MergeInsertParams(Collections.singletonList("id")) + .withMatchedUpdateAll() + .withNotMatched(MergeInsertParams.WhenNotMatched.InsertAll) + .withSourceDedupeBehavior(MergeInsertParams.SourceDedupeBehavior.FirstSeen), + sourceStream); + + Assertions.assertEquals( + "{0=First 0, 1=Person 1, 2=Person 2, 3=Source 3, 4=Source 4}", + readAll(result.dataset()).toString(), + "FirstSeen should keep the first duplicate source row (id=0) and update unique matches"); + } + } + } + + @Test + public void testSourceDedupeFailWithDuplicates() throws Exception { + // Default behavior (Fail) must error when the source contains duplicate join keys. + + try (VectorSchemaRoot source = buildDuplicateKeySource(testDataset.getSchema(), allocator)) { + try (ArrowArrayStream sourceStream = convertToStream(source, allocator)) { + String originalDataset = readAll(dataset).toString(); + + Assertions.assertThrows( + Exception.class, + () -> + dataset.mergeInsert( + new MergeInsertParams(Collections.singletonList("id")) + .withMatchedUpdateAll() + .withNotMatched(MergeInsertParams.WhenNotMatched.InsertAll) + .withSourceDedupeBehavior(MergeInsertParams.SourceDedupeBehavior.Fail), + sourceStream)); + + Assertions.assertEquals( + originalDataset, + readAll(dataset).toString(), + "Dataset should remain unchanged after a failed mergeInsert"); + } + } + } + @Test public void testMergeInsertWithoutIndex() throws Exception { // Verify that merge insert with useIndex=false still completes and diff --git a/rust/lance/src/dataset.rs b/rust/lance/src/dataset.rs index 3e0d77704da..2894ef4f8da 100644 --- a/rust/lance/src/dataset.rs +++ b/rust/lance/src/dataset.rs @@ -129,8 +129,8 @@ pub use schema_evolution::{ pub use take::TakeBuilder; use uuid::Uuid; pub use write::merge_insert::{ - MergeInsertBuilder, MergeInsertJob, MergeStats, UncommittedMergeInsert, WhenMatched, - WhenNotMatched, WhenNotMatchedBySource, + MergeInsertBuilder, MergeInsertJob, MergeStats, SourceDedupeBehavior, UncommittedMergeInsert, + WhenMatched, WhenNotMatched, WhenNotMatchedBySource, }; use crate::dataset::index::LanceIndexStoreExt;