+ * They used to be sent under snake_case names ({@code rescore_limit}, + * {@code training_limit}, ...) that Weaviate looks up verbatim and therefore + * never read, so every one of them was dropped on write and came back null. + * Asserting on {@code enabled} alone did not catch it. + */ + /** + * A dynamic index keeps its quantizer inside "hnsw"/"flat". + * + *
+ * The client used to write it beside them, where the server never looks, so a
+ * dynamic index created through this client came back unquantized -- and even
+ * one quantized by other means read as {@code quantization() == null}.
+ */
+ @Test
+ public void test_dynamicIndexQuantizationRoundTrip() throws IOException {
+ // Arrange: "dynamic" is only accepted by a server started with
+ // ASYNC_INDEXING=true, which the shared container is not -- asynchronous
+ // indexing would make freshly inserted vectors searchable only eventually,
+ // and the other suites rely on them being searchable at once.
+ var asyncClient = asyncIndexing.getClient();
+ var nsThings = ns("Things");
+
+ var things = asyncClient.collections.create(nsThings,
+ c -> c.vectorConfig(VectorConfig.selfProvided(
+ self -> self
+ .vectorIndex(Dynamic.of(idx -> idx
+ .hnsw(Hnsw.of(hnsw -> hnsw.ef(64)))
+ .flat(Flat.of(flat -> flat.vectorCacheMaxObjects(1000)))
+ .threshold(5000)))
+ .quantization(Quantization.rq(rq -> rq.rescoreLimit(20).bits(8))))));
+
+ // Act
+ var config = things.config.get();
+
+ // Assert: the server kept it, and we can read it back off the dynamic index.
+ Assertions.assertThat(config).get()
+ .extracting(CollectionConfig::vectors)
+ .extracting("default", InstanceOfAssertFactories.type(VectorConfig.class))
+ .satisfies(vector -> {
+ Assertions.assertThat(vector.vectorIndex())
+ .as("index type").isInstanceOf(Dynamic.class);
+ Assertions.assertThat(vector.quantization())
+ .as("quantizer nested under hnsw").isNotNull()
+ .asInstanceOf(InstanceOfAssertFactories.type(RQ.class))
+ .returns(20, RQ::rescoreLimit)
+ .returns(8, RQ::bits);
+ });
+ }
+
+ @Test
+ public void test_quantizerSettingsRoundTrip() throws IOException {
+ // Arrange
+ var nsThings = ns("Things");
+
+ var things = client.collections.create(nsThings,
+ c -> c.vectorConfig(VectorConfig.selfProvided(
+ self -> self.quantization(Quantization.rq(rq -> rq.rescoreLimit(42).bits(8))))));
+
+ // Act
+ var config = things.config.get();
+
+ // Assert
+ Assertions.assertThat(config).get()
+ .extracting(CollectionConfig::vectors)
+ .extracting("default", InstanceOfAssertFactories.type(VectorConfig.class))
+ .extracting(VectorConfig::quantization)
+ .asInstanceOf(InstanceOfAssertFactories.type(RQ.class))
+ .returns(42, RQ::rescoreLimit)
+ .returns(8, RQ::bits);
+ }
+
@Test
public void test_updateGenerative() throws IOException {
// Arrange
diff --git a/src/main/java/io/weaviate/client6/v1/api/collections/VectorConfig.java b/src/main/java/io/weaviate/client6/v1/api/collections/VectorConfig.java
index 4fa6ae6d0..ca7d47c08 100644
--- a/src/main/java/io/weaviate/client6/v1/api/collections/VectorConfig.java
+++ b/src/main/java/io/weaviate/client6/v1/api/collections/VectorConfig.java
@@ -6,6 +6,7 @@
import java.util.function.Function;
import com.google.gson.Gson;
+import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.TypeAdapter;
@@ -50,6 +51,7 @@
import io.weaviate.client6.v1.internal.ObjectBuilder;
import io.weaviate.client6.v1.internal.TaggedUnion;
import io.weaviate.client6.v1.internal.json.JsonEnum;
+import io.weaviate.client6.v1.internal.json.QuantizerJson;
public interface VectorConfig extends TaggedUnion
+ * Shared by the read/write sides of {@code VectorConfig} and by the collection
+ * update request, which all have to agree on where the quantizer keys live.
+ */
+public final class QuantizerJson {
+ /** Prevent public initialization. */
+ private QuantizerJson() {
+ }
+
+ /**
+ * The object inside {@code vectorIndexConfig} which holds the quantizer keys.
+ *
+ *
+ * For {@code hnsw} and {@code flat} that is {@code vectorIndexConfig} itself,
+ * but a {@code dynamic} index is
+ *
+ * A vector config has a single quantization slot, so a dynamic index is read
+ * from {@code hnsw} when it has a quantizer and from {@code flat} otherwise,
+ * and written to {@code hnsw} -- which is also the only sub-index accepting
+ * every quantizer type, {@code flat} being limited to {@code bq}. Giving
+ * {@code hnsw} and {@code flat} different quantizers is not
+ * expressible through this client.
+ *
+ * @param create when true the nested object is created if the payload does not
+ * have one yet; when false an empty object is returned instead,
+ * so the caller simply finds nothing.
+ */
+ public static JsonObject host(JsonObject vectorIndexConfig, JsonElement vectorIndexType, boolean create) {
+ if (!isDynamic(vectorIndexType)) {
+ return vectorIndexConfig;
+ }
+
+ for (var subIndex : new String[] { "hnsw", "flat" }) {
+ var nested = vectorIndexConfig.get(subIndex);
+ if (nested != null && nested.isJsonObject() && (create || hasQuantizer(nested.getAsJsonObject()))) {
+ return nested.getAsJsonObject();
+ }
+ }
+
+ if (create) {
+ var hnsw = new JsonObject();
+ vectorIndexConfig.add("hnsw", hnsw);
+ return hnsw;
+ }
+ return new JsonObject();
+ }
+
+ private static boolean isDynamic(JsonElement vectorIndexType) {
+ return vectorIndexType != null
+ && vectorIndexType.isJsonPrimitive()
+ && VectorIndex.Kind.DYNAMIC.jsonValue().equals(vectorIndexType.getAsString());
+ }
+
+ private static boolean hasQuantizer(JsonObject subIndex) {
+ for (var kind : Quantization.Kind.values()) {
+ if (subIndex.has(kind.jsonValue())) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
diff --git a/src/test/java/io/weaviate/client6/v1/api/collections/DynamicQuantizationTest.java b/src/test/java/io/weaviate/client6/v1/api/collections/DynamicQuantizationTest.java
new file mode 100644
index 000000000..b902e1019
--- /dev/null
+++ b/src/test/java/io/weaviate/client6/v1/api/collections/DynamicQuantizationTest.java
@@ -0,0 +1,112 @@
+package io.weaviate.client6.v1.api.collections;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import io.weaviate.client6.v1.api.collections.vectorindex.Distance;
+import io.weaviate.client6.v1.api.collections.vectorindex.Dynamic;
+import io.weaviate.client6.v1.internal.json.JSON;
+
+/**
+ * Reading a quantizer off a dynamic index.
+ *
+ *
+ * The round-trip cases live in {@code JSONTest}; these cover the shapes the
+ * server can return but this client never writes, since a single
+ * {@code quantization()} slot always writes to {@code hnsw}.
+ */
+public class DynamicQuantizationTest {
+
+ private static String dynamic(String hnswExtra, String flatExtra) {
+ return """
+ {
+ "vectorIndexType": "dynamic",
+ "vectorizer": {"none": {}},
+ "vectorIndexConfig": {
+ "threshold": 10000,
+ "hnsw": {"ef": -1%s},
+ "flat": {"vectorCacheMaxObjects": 1000000%s}
+ }
+ }
+ """.formatted(hnswExtra, flatExtra);
+ }
+
+ /** The reported case: a dynamic index with RQ enabled on hnsw. */
+ @Test
+ public void test_readsQuantizerFromHnsw() {
+ var config = JSON.deserialize(
+ dynamic(", \"rq\": {\"enabled\": true, \"bits\": 8, \"rescoreLimit\": 20}", ""),
+ VectorConfig.class);
+
+ Assertions.assertThat(config.quantization())
+ .isNotNull()
+ .returns(Quantization.Kind.RQ, Quantization::_kind);
+ Assertions.assertThat(config.quantization().asRQ().rescoreLimit()).isEqualTo(20);
+ Assertions.assertThat(config.quantization().asRQ().bits()).isEqualTo(8);
+ }
+
+ /** flat carries its own quantizer, and only bq is valid there. */
+ @Test
+ public void test_readsQuantizerFromFlatWhenHnswHasNone() {
+ var config = JSON.deserialize(
+ dynamic("", ", \"bq\": {\"enabled\": true, \"rescoreLimit\": 5}"),
+ VectorConfig.class);
+
+ Assertions.assertThat(config.quantization())
+ .isNotNull()
+ .returns(Quantization.Kind.BQ, Quantization::_kind);
+ Assertions.assertThat(config.quantization().asBQ().rescoreLimit()).isEqualTo(5);
+ }
+
+ /**
+ * hnsw and flat can carry different quantizers, which one slot cannot
+ * represent. hnsw wins; see QuantizerJson.
+ */
+ @Test
+ public void test_prefersHnswWhenBothCarryOne() {
+ var config = JSON.deserialize(
+ dynamic(", \"rq\": {\"enabled\": true, \"bits\": 8}", ", \"bq\": {\"enabled\": true}"),
+ VectorConfig.class);
+
+ Assertions.assertThat(config.quantization())
+ .returns(Quantization.Kind.RQ, Quantization::_kind);
+ }
+
+ @Test
+ public void test_noQuantizerIsNull() {
+ var config = JSON.deserialize(dynamic("", ""), VectorConfig.class);
+
+ Assertions.assertThat(config.quantization()).isNull();
+ }
+
+ /** A disabled quantizer reads as absent, same as for hnsw and flat indexes. */
+ @Test
+ public void test_disabledQuantizerIsNull() {
+ var config = JSON.deserialize(
+ dynamic(", \"rq\": {\"enabled\": false, \"bits\": 8}", ""),
+ VectorConfig.class);
+
+ Assertions.assertThat(config.quantization()).isNull();
+ }
+
+ /** The dynamic level carries its own distance, which used to be dropped. */
+ @Test
+ public void test_readsDistanceAtDynamicLevel() {
+ var config = JSON.deserialize("""
+ {
+ "vectorIndexType": "dynamic",
+ "vectorizer": {"none": {}},
+ "vectorIndexConfig": {
+ "threshold": 10000,
+ "distance": "l2-squared",
+ "hnsw": {"ef": -1},
+ "flat": {"vectorCacheMaxObjects": 1000000}
+ }
+ }
+ """, VectorConfig.class);
+
+ Assertions.assertThat(config.vectorIndex().asDynamic())
+ .returns(Distance.L2_SQUARED, Dynamic::distance)
+ .returns(10000L, Dynamic::threshold);
+ }
+}
diff --git a/src/test/java/io/weaviate/client6/v1/internal/json/JSONTest.java b/src/test/java/io/weaviate/client6/v1/internal/json/JSONTest.java
index e7c0d6203..d4de18ba2 100644
--- a/src/test/java/io/weaviate/client6/v1/internal/json/JSONTest.java
+++ b/src/test/java/io/weaviate/client6/v1/internal/json/JSONTest.java
@@ -1188,6 +1188,34 @@ public static Object[][] testCases() {
}
""",
},
+ // A dynamic index nests its quantizer inside "hnsw"/"flat" -- the client
+ // used to put it beside them, where the server never looks, and could not
+ // read it back either.
+ {
+ VectorConfig.class,
+ SelfProvidedVectorizer.of(none -> none
+ .vectorIndex(Dynamic.of(idx -> idx
+ .hnsw(Hnsw.of(hnsw -> hnsw.ef(1)))
+ .flat(Flat.of(flat -> flat.vectorCacheMaxObjects(100)))
+ .threshold(5)
+ .distance(Distance.COSINE)))
+ .quantization(Quantization.rq(rq -> rq.rescoreLimit(20).bits(8)))),
+ """
+ {
+ "vectorIndexType": "dynamic",
+ "vectorizer": {"none": {}},
+ "vectorIndexConfig": {
+ "flat": {"vectorCacheMaxObjects": 100},
+ "hnsw": {
+ "ef": 1,
+ "rq": {"enabled": true, "rescoreLimit": 20, "bits": 8}
+ },
+ "threshold": 5,
+ "distance": "cosine"
+ }
+ }
+ """,
+ },
{
VectorConfig.class,
SelfProvidedVectorizer.of(none -> none
@@ -1206,11 +1234,13 @@ public static Object[][] testCases() {
"pq": {
"enabled": true,
"centroids": 8,
- "encoder_distribution": "normal",
- "encoder_type": "tile",
+ "encoder": {
+ "type": "tile",
+ "distribution": "normal"
+ },
"segments": 16,
- "training_limit": 1024,
- "bit_compression": true
+ "trainingLimit": 1024,
+ "bitCompression": true
}
}
}
@@ -1230,8 +1260,8 @@ public static Object[][] testCases() {
"vectorIndexConfig": {
"sq": {
"enabled": true,
- "rescore_limit": 10,
- "training_limit": 1024,
+ "rescoreLimit": 10,
+ "trainingLimit": 1024,
"cache": true
}
}
@@ -1251,7 +1281,7 @@ public static Object[][] testCases() {
"vectorIndexConfig": {
"rq": {
"enabled": true,
- "rescore_limit": 10,
+ "rescoreLimit": 10,
"bits": 8
}
}
@@ -1271,7 +1301,7 @@ public static Object[][] testCases() {
"vectorIndexConfig": {
"bq": {
"enabled": true,
- "rescore_limit": 10,
+ "rescoreLimit": 10,
"cache": true
}
}
{distance, threshold, hnsw: {...}, flat: {...}} and each
+ * sub-index carries its own quantizer, so the keys sit one level deeper.
+ *
+ *