Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ lance-linalg = { version = "=8.0.0-beta.14", path = "./rust/lance-linalg" }
lance-namespace = { version = "=8.0.0-beta.14", path = "./rust/lance-namespace" }
lance-namespace-impls = { version = "=8.0.0-beta.14", path = "./rust/lance-namespace-impls" }
lance-namespace-datafusion = { version = "=7.0.0-beta.9", path = "./rust/lance-namespace-datafusion" }
lance-namespace-reqwest-client = "0.8.4"
lance-namespace-reqwest-client = "0.8.6"
lance-select = { version = "=8.0.0-beta.14", path = "./rust/lance-select" }
lance-tokenizer = { version = "=8.0.0-beta.14", path = "./rust/lance-tokenizer" }
lance-table = { version = "=8.0.0-beta.14", path = "./rust/lance-table" }
Expand Down Expand Up @@ -180,6 +180,7 @@ rand_distr = { version = "0.5.1" }
rand_xoshiro = "0.7.0"
rangemap = { version = "1.0" }
rayon = "1.10"
regex-syntax = "0.8.10"
roaring = "0.11.4"
rstest = "0.26.1"
serde = { version = "^1" }
Expand Down
8 changes: 7 additions & 1 deletion docs/src/format/index/scalar/ngram.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,10 @@ The N-gram index provides inexact results for the following query types:

| Query Type | Description | Operation | Result Type |
|----------------|--------------------------|-------------------------------------------------------|-------------|
| **contains** | Substring search in text | Finds all trigrams in query, intersects posting lists | AtMost |
| **contains** | Substring search in text | Finds all trigrams in query, intersects posting lists | AtMost |
| **regexp_like** / **regexp_match** | Regular-expression match | Derives a necessary trigram condition from the pattern (AND of intersections, OR of unions), then rechecks the true regex | AtMost |
| **LIKE** (infix) | Wildcard match such as `%foo%bar%` | Uses the literal segments of the pattern as a trigram condition, then rechecks the LIKE | AtMost |

Patterns from which no trigram can be derived - for example `a.b`, `.*`,
case-insensitive matches, or literal runs shorter than three characters - fall
back to rechecking every row. This is always correct, just not accelerated.
28 changes: 27 additions & 1 deletion java/lance-jni/src/merge_insert.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ use jni::objects::{JObject, JString, JValueGen};
use jni::sys::jlong;
use lance::dataset::scanner::ExprFilter;
use lance::dataset::{
MergeInsertBuilder, MergeStats, WhenMatched, WhenNotMatched, WhenNotMatchedBySource,
MergeInsertBuilder, MergeStats, SourceDedupeBehavior, WhenMatched, WhenNotMatched,
WhenNotMatchedBySource,
};
use lance_core::datatypes::Schema;
use lance_index::mem_wal::MergedGeneration;
Expand Down Expand Up @@ -52,6 +53,7 @@ fn inner_merge_insert<'local>(
let retry_timeout_ms = extract_retry_timeout_ms(env, &jparam)?;
let skip_auto_cleanup = extract_skip_auto_cleanup(env, &jparam)?;
let use_index = extract_use_index(env, &jparam)?;
let source_dedupe_behavior = extract_source_dedupe_behavior(env, &jparam)?;
let marked_generations = extract_marked_generations(env, &jparam)?;

let (new_ds, merge_stats) = unsafe {
Expand All @@ -71,6 +73,7 @@ fn inner_merge_insert<'local>(
.retry_timeout(Duration::from_millis(retry_timeout_ms as u64))
.skip_auto_cleanup(skip_auto_cleanup)
.use_index(use_index)
.source_dedupe_behavior(source_dedupe_behavior)
.mark_generations_as_merged(marked_generations)
.try_build()?;

Expand Down Expand Up @@ -241,6 +244,29 @@ fn extract_use_index<'local>(env: &mut JNIEnv<'local>, jparam: &JObject) -> Resu
Ok(use_index)
}

fn extract_source_dedupe_behavior<'local>(
env: &mut JNIEnv<'local>,
jparam: &JObject,
) -> Result<SourceDedupeBehavior> {
let behavior: JString = env
.call_method(
jparam,
"sourceDedupeBehaviorValue",
"()Ljava/lang/String;",
&[],
)?
.l()?
.into();
let behavior = behavior.extract(env)?;
match behavior.as_str() {
"Fail" => Ok(SourceDedupeBehavior::Fail),
"FirstSeen" => Ok(SourceDedupeBehavior::FirstSeen),
_ => Err(Error::input_error(format!(
"Illegal source_dedupe_behavior: {behavior}",
))),
}
}

fn extract_marked_generations<'local>(
env: &mut JNIEnv<'local>,
jparam: &JObject,
Expand Down
43 changes: 43 additions & 0 deletions java/src/main/java/org/lance/merge/MergeInsertParams.java
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ public class MergeInsertParams {
private long retryTimeoutMs = 30 * 1000;
private boolean skipAutoCleanup = false;
private boolean useIndex = true;
private SourceDedupeBehavior sourceDedupeBehavior = SourceDedupeBehavior.Fail;
private List<MergedGeneration> markedGenerations = Collections.emptyList();

public MergeInsertParams(List<String> on) {
Expand Down Expand Up @@ -244,6 +245,25 @@ public MergeInsertParams withUseIndex(boolean useIndex) {
return this;
}

/**
* Control how duplicate source rows that match the same target row are handled.
*
* <p>Default is {@link SourceDedupeBehavior#Fail}, which errors if the source contains duplicate
* join keys. Use {@link SourceDedupeBehavior#FirstSeen} to keep the first encountered row and
* skip subsequent duplicates.
*
* <p>If the source contains duplicates and {@code FirstSeen} behavior doesn't match your needs,
* sort the source data before passing it to the merge insert operation.
*
* @param sourceDedupeBehavior The behavior to apply when duplicate source rows are found
* @return This MergeInsertParams instance
*/
public MergeInsertParams withSourceDedupeBehavior(SourceDedupeBehavior sourceDedupeBehavior) {
Preconditions.checkNotNull(sourceDedupeBehavior);
this.sourceDedupeBehavior = sourceDedupeBehavior;
return this;
}

/**
* Mark MemWAL generations as merged into the base table.
*
Expand Down Expand Up @@ -319,6 +339,14 @@ public boolean useIndex() {
return useIndex;
}

public SourceDedupeBehavior sourceDedupeBehavior() {
return sourceDedupeBehavior;
}

public String sourceDedupeBehaviorValue() {
return sourceDedupeBehavior.name();
}

@Override
public String toString() {
return MoreObjects.toStringHelper(this)
Expand All @@ -337,6 +365,7 @@ public String toString() {
.add("retryTimeoutMs", retryTimeoutMs)
.add("skipAutoCleanup", skipAutoCleanup)
.add("useIndex", useIndex)
.add("sourceDedupeBehavior", sourceDedupeBehavior)
.toString();
}

Expand Down Expand Up @@ -396,4 +425,18 @@ public enum WhenNotMatchedBySource {
*/
DeleteIf,
}

/**
* Describes how to handle duplicate source rows that match the same target row.
*
* <p>If the source contains duplicates and {@code FirstSeen} behavior doesn't match your needs,
* sort the source data before passing it to the merge insert operation.
*/
public enum SourceDedupeBehavior {
/** Fail the operation if duplicates are found (default). */
Fail,

/** Keep the first seen value and skip subsequent duplicates. */
FirstSeen,
}
}
73 changes: 73 additions & 0 deletions java/src/test/java/org/lance/MergeInsertTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,30 @@ private VectorSchemaRoot buildSource(Schema schema, RootAllocator allocator) {
return root;
}

/**
* Build a source whose join key {@code id=0} appears twice ("First 0", then "Second 0"), so the
* source-dedupe behavior is exercised. Remaining ids (3, 4) are unique matches.
*/
private VectorSchemaRoot buildDuplicateKeySource(Schema schema, RootAllocator allocator) {
List<Integer> sourceIds = Arrays.asList(0, 0, 3, 4);
List<String> sourceNames = Arrays.asList("First 0", "Second 0", "Source 3", "Source 4");

VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator);
root.allocateNew();

IntVector idVector = (IntVector) root.getVector("id");
VarCharVector nameVector = (VarCharVector) root.getVector("name");

for (int i = 0; i < sourceIds.size(); i++) {
idVector.setSafe(i, sourceIds.get(i));
nameVector.setSafe(i, sourceNames.get(i).getBytes(StandardCharsets.UTF_8));
}

root.setRowCount(sourceIds.size());

return root;
}

private ArrowArrayStream convertToStream(VectorSchemaRoot root, RootAllocator allocator)
throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
Expand All @@ -275,6 +299,55 @@ private ArrowArrayStream convertToStream(VectorSchemaRoot root, RootAllocator al
return stream;
}

@Test
public void testSourceDedupeFirstSeenKeepsFirst() throws Exception {
// Source has two rows for id=0 ("First 0" then "Second 0"). FirstSeen keeps
// the first encountered row and skips the duplicate.

try (VectorSchemaRoot source = buildDuplicateKeySource(testDataset.getSchema(), allocator)) {
try (ArrowArrayStream sourceStream = convertToStream(source, allocator)) {
MergeInsertResult result =
dataset.mergeInsert(
new MergeInsertParams(Collections.singletonList("id"))
.withMatchedUpdateAll()
.withNotMatched(MergeInsertParams.WhenNotMatched.InsertAll)
.withSourceDedupeBehavior(MergeInsertParams.SourceDedupeBehavior.FirstSeen),
sourceStream);

Assertions.assertEquals(
"{0=First 0, 1=Person 1, 2=Person 2, 3=Source 3, 4=Source 4}",
readAll(result.dataset()).toString(),
"FirstSeen should keep the first duplicate source row (id=0) and update unique matches");
}
}
}

@Test
public void testSourceDedupeFailWithDuplicates() throws Exception {
// Default behavior (Fail) must error when the source contains duplicate join keys.

try (VectorSchemaRoot source = buildDuplicateKeySource(testDataset.getSchema(), allocator)) {
try (ArrowArrayStream sourceStream = convertToStream(source, allocator)) {
String originalDataset = readAll(dataset).toString();

Assertions.assertThrows(
Exception.class,
() ->
dataset.mergeInsert(
new MergeInsertParams(Collections.singletonList("id"))
.withMatchedUpdateAll()
.withNotMatched(MergeInsertParams.WhenNotMatched.InsertAll)
.withSourceDedupeBehavior(MergeInsertParams.SourceDedupeBehavior.Fail),
sourceStream));

Assertions.assertEquals(
originalDataset,
readAll(dataset).toString(),
"Dataset should remain unchanged after a failed mergeInsert");
}
}
}

@Test
public void testMergeInsertWithoutIndex() throws Exception {
// Verify that merge insert with useIndex=false still completes and
Expand Down
1 change: 1 addition & 0 deletions python/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 4 additions & 2 deletions python/python/lance/lance/optimize.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import List
from typing import List, Optional

from lance import LanceDataset
from lance.fragment import FragmentMetadata
Expand Down Expand Up @@ -51,5 +51,7 @@ class Compaction:
def plan(dataset: "LanceDataset", options: CompactionOptions) -> CompactionPlan: ...
@staticmethod
def commit(
dataset: "LanceDataset", rewrites: List[RewriteResult]
dataset: "LanceDataset",
rewrites: List[RewriteResult],
options: Optional[CompactionOptions] = None,
) -> CompactionMetrics: ...
8 changes: 8 additions & 0 deletions python/python/lance/optimize.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,14 @@ class CompactionOptions(TypedDict):
The batch size to use when scanning input fragments. You may want
to reduce this if you are running out of memory during compaction.

The default will use the same default from ``scanner``.
"""
io_buffer_size: Optional[int]
"""
The number of bytes to allow to queue up in the I/O buffer when scanning
input fragments. Increasing this can avoid a deadlock that occurs when a
single batch of data is larger than the I/O buffer size.

The default will use the same default from ``scanner``.
"""
compaction_mode: Optional[
Expand Down
32 changes: 32 additions & 0 deletions python/python/tests/test_blob.py
Original file line number Diff line number Diff line change
Expand Up @@ -1460,6 +1460,38 @@ def test_read_blobs_resolves_nested_field_path(dataset_with_nested_blobs):
assert [data for _, data in results] == [b"foo", b"baz"]


def test_write_nested_blob_v2_and_take_by_field_path(tmp_path):
packed = b"x" * (70 * 1024)
blob_field = lance.blob_field("blob")
info_fields = [pa.field("name", pa.string()), blob_field]
info_type = pa.struct(info_fields)
info_array = pa.StructArray.from_arrays(
[pa.array(["a", "b", "c"]), lance.blob_array([b"foo", packed, None])],
fields=info_fields,
)
table = pa.table(
[info_array],
schema=pa.schema([pa.field("info", info_type)]),
)

dataset = lance.write_dataset(
table,
tmp_path / "nested_blob_v2",
data_storage_version="2.2",
)

desc = dataset.to_table(columns=["info.blob"]).column("info.blob").chunk(0)
assert desc.field("kind").to_pylist()[:2] == [0, 1]

blobs = dataset.take_blobs("info.blob", indices=[0, 1])
with blobs[0] as f:
assert f.read() == b"foo"
with blobs[1] as f:
assert f.read() == packed

assert dataset.take_blobs("info.blob", indices=[2]) == []


def test_to_pandas_returns_blob_files_for_projected_nested_fields(
dataset_with_nested_blobs,
):
Expand Down
41 changes: 41 additions & 0 deletions python/python/tests/test_optimize.py
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,47 @@ def test_defer_index_remap(tmp_path: Path):
assert any(idx.name == "__lance_frag_reuse" for idx in indices)


@pytest.mark.parametrize("use_commit_options", [True, False])
def test_defer_index_remap_via_commit_options(tmp_path: Path, use_commit_options: bool):
"""Compaction.commit respects defer_index_remap passed in options.

When options={"defer_index_remap": True} is supplied to Compaction.commit
the __lance_frag_reuse system index must appear in describe_indices().
When the option is omitted (default) no such system index is written.
"""
base_dir = tmp_path / f"dataset_commit_opts_{use_commit_options}"
data = pa.table({"i": range(6_000), "val": range(6_000)})
dataset = lance.write_dataset(data, base_dir, max_rows_per_file=1_000)
dataset.create_scalar_index("i", "BTREE")
dataset.delete("i < 500")

plan = Compaction.plan(
dataset,
options=dict(target_rows_per_fragment=2_000, num_threads=1),
)
rewrites = [task.execute(dataset) for task in plan.tasks]

if use_commit_options:
Compaction.commit(dataset, rewrites, options={"defer_index_remap": True})
else:
Compaction.commit(dataset, rewrites)

dataset = lance.dataset(base_dir)
indices = dataset.describe_indices()
has_frag_reuse = any(idx.name == "__lance_frag_reuse" for idx in indices)

if use_commit_options:
assert has_frag_reuse, (
"expected __lance_frag_reuse system index when defer_index_remap=True "
"is passed to Compaction.commit"
)
else:
assert not has_frag_reuse, (
"did not expect __lance_frag_reuse system index when options is omitted "
"from Compaction.commit"
)


@pytest.mark.filterwarnings("ignore::DeprecationWarning")
def test_describe_indices_matches_list_indices_for_frag_reuse(tmp_path: Path):
"""describe_indices() and list_indices() must agree on the index_type
Expand Down
2 changes: 2 additions & 0 deletions python/python/tests/test_vector_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -1772,6 +1772,8 @@ def test_index_cast_centroids(tmp_path):
values = pa.array([x for arr in centroids for x in arr], pa.float32())
centroids = pa.FixedSizeListArray.from_arrays(values, 128)

# Cast invalidates the attached index; drop it first per the new contract.
dataset.drop_index(index_name)
dataset.alter_columns(dict(path="vector", data_type=pa.list_(pa.float16(), 128)))

# centroids are f32, but the column is now f16
Expand Down
Loading