diff --git a/samples/UpdateStreamDocuments.java b/samples/UpdateStreamDocuments.java index a539ce78..2c9dae61 100644 --- a/samples/UpdateStreamDocuments.java +++ b/samples/UpdateStreamDocuments.java @@ -34,6 +34,23 @@ public static void main(String[] args) throws IOException, InterruptedException, DeleteDocument document3 = new DeleteDocument("https://my.document3.uri"); updateStreamService.delete(document3); + PartialUpdateDocument document4 = new PartialUpdateDocument("https://my.document4.uri", PartialUpdateOperator.FIELD_VALUE_REPLACE, "title", "My new title"); + updateStreamService.addPartialUpdate(document4); + + PartialUpdateDocument document5 = new PartialUpdateDocument("https://my.document5.uri", PartialUpdateOperator.DICTIONARY_PUT, "dictionaryAttribute", new HashMap<>() {{ + put("newkey", "newvalue"); + }}); + updateStreamService.addPartialUpdate(document5); + + PartialUpdateDocument document6 = new PartialUpdateDocument("https://my.document6.uri", PartialUpdateOperator.ARRAY_APPEND, "arrayAttribute", new String[]{"newValue"}); + updateStreamService.addPartialUpdate(document6); + + PartialUpdateDocument document7 = new PartialUpdateDocument("https://my.document7.uri", PartialUpdateOperator.ARRAY_REMOVE, "arrayAttribute", new String[]{"oldValue"}); + updateStreamService.addPartialUpdate(document7); + + PartialUpdateDocument document8 = new PartialUpdateDocument("https://my.document8.uri", PartialUpdateOperator.DICIONARY_REMOVE, "dictionaryAttribute", "oldkey"); + updateStreamService.addPartialUpdate(document8); + updateStreamService.close(); } } diff --git a/src/main/java/com/coveo/pushapiclient/DocumentUploadQueue.java b/src/main/java/com/coveo/pushapiclient/DocumentUploadQueue.java index 5e81f55f..83168820 100644 --- a/src/main/java/com/coveo/pushapiclient/DocumentUploadQueue.java +++ b/src/main/java/com/coveo/pushapiclient/DocumentUploadQueue.java @@ -8,11 +8,11 @@ /** Represents a queue for uploading documents using a specified upload strategy */ class DocumentUploadQueue { private static final Logger logger = LogManager.getLogger(DocumentUploadQueue.class); - private final UploadStrategy uploader; - private final int maxQueueSize = 5 * 1024 * 1024; - private ArrayList documentToAddList; - private ArrayList documentToDeleteList; - private int size; + protected final UploadStrategy uploader; + protected final int maxQueueSize = 5 * 1024 * 1024; + protected ArrayList documentToAddList; + protected ArrayList documentToDeleteList; + protected int size; /** * Constructs a new DocumentUploadQueue object with a default maximum queue size limit of 5MB. diff --git a/src/main/java/com/coveo/pushapiclient/PartialUpdateDocument.java b/src/main/java/com/coveo/pushapiclient/PartialUpdateDocument.java new file mode 100644 index 00000000..e60bd5c5 --- /dev/null +++ b/src/main/java/com/coveo/pushapiclient/PartialUpdateDocument.java @@ -0,0 +1,74 @@ +package com.coveo.pushapiclient; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import java.util.Map; + +public class PartialUpdateDocument { + + /** The documentId of the document. */ + public String documentId; + + /** The operator of the document. */ + public PartialUpdateOperator operator; + + /** The field to update. */ + public String field; + + /** The value of the field to be updated. */ + public Object value; + + public JsonObject marshalJsonObject() { + return new Gson().toJsonTree(this).getAsJsonObject(); + } + + /** + * Creates a new PartialUpdateDocument. The type of the value provided is constrained by the + * operator. + * + *
    + *
  • PartialUpdateOperator.ARRAY_APPEND: value must be an array + *
  • PartialUpdateOperator.ARRAY_REMOVE: value must be an array + *
  • PartialUpdateOperator.FIELD_VALUE_REPLACE: value can be any type + *
  • PartialUpdateOperator.DICTIONARY_PUT: value must be a Map + *
  • PartialUpdateOperator.DICTIONARY_REMOVE: value must be a String or an Array + *
+ * + * @param documentId The id of the document. + * @param operator The operator to use. + * @param field The field to update. + * @param value The value to update the field with. + */ + public PartialUpdateDocument( + String documentId, PartialUpdateOperator operator, String field, Object value) { + if (operator == null) throw new IllegalArgumentException("Operator cannot be null"); + if (field == null) throw new IllegalArgumentException("Field cannot be null"); + if (documentId == null) throw new IllegalArgumentException("DocumentId cannot be null"); + + this.documentId = documentId; + this.operator = operator; + this.field = field; + + switch (operator) { + case ARRAYAPPEND: + case ARRAYREMOVE: + if (!value.getClass().isArray()) + throw new IllegalArgumentException("Value must be an array for operator " + operator); + break; + case FIELDVALUEREPLACE: + break; + case DICTIONARYPUT: + if (!(value instanceof Map)) + throw new IllegalArgumentException("Value must be a Map for operator " + operator); + break; + case DICTIONARYREMOVE: + if (!(value instanceof String) && !value.getClass().isArray()) + throw new IllegalArgumentException( + "Value must be a String or an Array for operator " + operator); + break; + default: + throw new IllegalArgumentException("Invalid operator " + operator); + } + this.value = value; + } +} diff --git a/src/main/java/com/coveo/pushapiclient/PartialUpdateOperator.java b/src/main/java/com/coveo/pushapiclient/PartialUpdateOperator.java new file mode 100644 index 00000000..9e2aabd6 --- /dev/null +++ b/src/main/java/com/coveo/pushapiclient/PartialUpdateOperator.java @@ -0,0 +1,29 @@ +package com.coveo.pushapiclient; + +public enum PartialUpdateOperator { + ARRAYAPPEND { + public String toString() { + return "arrayAppend"; + } + }, + ARRAYREMOVE { + public String toString() { + return "arrayRemove"; + } + }, + FIELDVALUEREPLACE { + public String toString() { + return "fieldValueReplace"; + } + }, + DICTIONARYPUT { + public String toString() { + return "dictionaryPut"; + } + }, + DICTIONARYREMOVE { + public String toString() { + return "dictionaryRemove"; + } + } +} diff --git a/src/main/java/com/coveo/pushapiclient/StreamDocumentUploadQueue.java b/src/main/java/com/coveo/pushapiclient/StreamDocumentUploadQueue.java new file mode 100644 index 00000000..7a039f7a --- /dev/null +++ b/src/main/java/com/coveo/pushapiclient/StreamDocumentUploadQueue.java @@ -0,0 +1,79 @@ +package com.coveo.pushapiclient; + +import java.io.IOException; +import java.util.ArrayList; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class StreamDocumentUploadQueue extends DocumentUploadQueue { + + private static final Logger logger = LogManager.getLogger(StreamDocumentUploadQueue.class); + protected ArrayList documentToPartiallyUpdateList; + + public StreamDocumentUploadQueue(UploadStrategy uploader) { + super(uploader); + this.documentToPartiallyUpdateList = new ArrayList<>(); + } + + /** + * Flushes the accumulated documents by applying the upload strategy. + * + * @throws IOException If an I/O error occurs during the upload. + * @throws InterruptedException If the upload process is interrupted. + */ + @Override + public void flush() throws IOException, InterruptedException { + if (this.isEmpty()) { + logger.debug("Empty batch. Skipping upload"); + return; + } + // TODO: LENS-871: support concurrent requests + StreamUpdate stream = this.getStream(); + logger.info("Uploading document Stream"); + this.uploader.apply(stream); + + this.size = 0; + this.documentToAddList.clear(); + this.documentToDeleteList.clear(); + this.documentToPartiallyUpdateList.clear(); + } + + /** + * Adds the {@link PartialUpdateDocument} to the upload queue and flushes the queue if it exceeds + * the maximum content length. See {@link PartialUpdateDocument#flush}. + * + * @param document The document to be deleted from the index. + * @throws IOException If an I/O error occurs during the upload. + * @throws InterruptedException If the upload process is interrupted. + */ + public void add(PartialUpdateDocument document) throws IOException, InterruptedException { + if (document == null) { + return; + } + + final int sizeOfDoc = document.marshalJsonObject().toString().getBytes().length; + if (this.size + sizeOfDoc >= this.maxQueueSize) { + this.flush(); + } + documentToPartiallyUpdateList.add(document); + logger.info("Adding document to batch: " + document.documentId); + this.size += sizeOfDoc; + } + + public StreamUpdate getStream() { + return new StreamUpdate( + new ArrayList<>(this.documentToAddList), + new ArrayList<>(this.documentToDeleteList), + new ArrayList<>(this.documentToPartiallyUpdateList)); + } + + @Override + public BatchUpdate getBatch() { + throw new UnsupportedOperationException("StreamDocumentUploadQueue does not support getBatch"); + } + + @Override + public boolean isEmpty() { + return super.isEmpty() && documentToPartiallyUpdateList.isEmpty(); + } +} diff --git a/src/main/java/com/coveo/pushapiclient/StreamUpdate.java b/src/main/java/com/coveo/pushapiclient/StreamUpdate.java new file mode 100644 index 00000000..8af10533 --- /dev/null +++ b/src/main/java/com/coveo/pushapiclient/StreamUpdate.java @@ -0,0 +1,60 @@ +package com.coveo.pushapiclient; + +import com.google.gson.JsonObject; +import java.util.List; + +public class StreamUpdate extends BatchUpdate { + + private final List partialUpdate; + + public StreamUpdate( + List addOrUpdate, + List delete, + List partialUpdate) { + super(addOrUpdate, delete); + this.partialUpdate = partialUpdate; + } + + @Override + public StreamUpdateRecord marshal() { + return new StreamUpdateRecord( + this.getAddOrUpdate().stream() + .map(DocumentBuilder::marshalJsonObject) + .toArray(JsonObject[]::new), + this.getDelete().stream().map(DeleteDocument::marshalJsonObject).toArray(JsonObject[]::new), + this.partialUpdate.stream() + .map(PartialUpdateDocument::marshalJsonObject) + .toArray(JsonObject[]::new)); + } + + public List getPartialUpdate() { + return partialUpdate; + } + + @Override + public String toString() { + return "StreamUpdate[" + + "addOrUpdate=" + + getAddOrUpdate() + + ", delete=" + + getDelete() + + ", partialUpdate=" + + partialUpdate + + ']'; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + StreamUpdate that = (StreamUpdate) obj; + return getAddOrUpdate().equals(that.getAddOrUpdate()) + && getDelete().equals(that.getDelete()) + && partialUpdate.equals(that.partialUpdate); + } + + @Override + public int hashCode() { + return super.hashCode() + partialUpdate.hashCode(); + } +} diff --git a/src/main/java/com/coveo/pushapiclient/StreamUpdateRecord.java b/src/main/java/com/coveo/pushapiclient/StreamUpdateRecord.java new file mode 100644 index 00000000..7df8934c --- /dev/null +++ b/src/main/java/com/coveo/pushapiclient/StreamUpdateRecord.java @@ -0,0 +1,48 @@ +package com.coveo.pushapiclient; + +import com.google.gson.JsonObject; +import java.util.Arrays; + +public class StreamUpdateRecord extends BatchUpdateRecord { + + private final JsonObject[] partialUpdate; + + public StreamUpdateRecord( + JsonObject[] addOrUpdate, JsonObject[] delete, JsonObject[] partialUpdate) { + super(addOrUpdate, delete); + this.partialUpdate = partialUpdate; + } + + public JsonObject[] getPartialUpdate() { + return partialUpdate; + } + + @Override + public String toString() { + return "StreamUpdateRecord[" + + "addOrUpdate=" + + Arrays.toString(this.getAddOrUpdate()) + + ", delete=" + + Arrays.toString(this.getDelete()) + + ", partialUpdate=" + + Arrays.toString(partialUpdate) + + ']'; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + StreamUpdateRecord that = (StreamUpdateRecord) obj; + return Arrays.equals(this.getAddOrUpdate(), that.getAddOrUpdate()) + && Arrays.equals(this.getDelete(), that.getDelete()) + && Arrays.equals(partialUpdate, that.partialUpdate); + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + Arrays.hashCode(partialUpdate); + return result; + } +} diff --git a/src/main/java/com/coveo/pushapiclient/UpdateStreamService.java b/src/main/java/com/coveo/pushapiclient/UpdateStreamService.java index a9a58946..7e25028e 100644 --- a/src/main/java/com/coveo/pushapiclient/UpdateStreamService.java +++ b/src/main/java/com/coveo/pushapiclient/UpdateStreamService.java @@ -46,7 +46,10 @@ public UpdateStreamService(StreamEnabledSource source, BackoffOptions options) { source.getApiKey(), source.getOrganizationId(), source.getPlatformUrl(), options); this.updateStreamServiceInternal = new UpdateStreamServiceInternal( - source, new DocumentUploadQueue(this.getUploadStrategy()), this.platformClient, logger); + source, + new StreamDocumentUploadQueue(this.getUploadStrategy()), + this.platformClient, + logger); } /** @@ -85,6 +88,46 @@ public void addOrUpdate(DocumentBuilder document) throws IOException, Interrupte fileContainer = updateStreamServiceInternal.addOrUpdate(document); } + /** + * Adds a document containing the specific field, and it's value to be updated. If there is no + * file container open to receive the documents, this function will open a file container before + * uploading the partial update details into it. More details on partial updates can be found in + * the + * Partial item updates section. + * + *

If called several times, the service will automatically batch documents and create new + * stream chunks whenever the data payload exceeds the batch size limit set for the + * Stream API. + * + *

Once there are no more documents to add, it is important to call the {@link + * UpdateStreamService#close} function in order to send any buffered documents and push the file + * container. Otherwise, changes will not be reflected in the index. + * + *

+ * + *

{@code
+   * //...
+   * UpdateStreamService service = new UpdateStreamService(source));
+   * for (PartialUpdateDocument document : fictionalDocumentList) {
+   *     service.addPartialUpdate(document);
+   * }
+   * service.close(document);
+   * }
+ * + *

For more code samples, @see `samples/UpdateStreamDocuments.java` + * + * @param document The partial update document to push to your file container + * @throws InterruptedException If the creation of the file container or adding the document is + * interrupted. + * @throws IOException If the creation of the file container or adding the document fails. + */ + public void addPartialUpdate(PartialUpdateDocument document) + throws IOException, InterruptedException { + fileContainer = updateStreamServiceInternal.addPartialUpdate(document); + } + /** * Adds documents to an open file container be deleted. If there is no file container open to * receive the documents, this function will open a file container before uploading documents into @@ -140,8 +183,9 @@ public HttpResponse close() } private UploadStrategy getUploadStrategy() { - return (batchUpdate) -> { - String batchUpdateJson = new Gson().toJson(batchUpdate.marshal()); + return (streamUpdate) -> { + String batchUpdateJson = new Gson().toJson(streamUpdate.marshal()); + System.out.println(batchUpdateJson); return this.platformClient.uploadContentToFileContainer(fileContainer, batchUpdateJson); }; } diff --git a/src/main/java/com/coveo/pushapiclient/UpdateStreamServiceInternal.java b/src/main/java/com/coveo/pushapiclient/UpdateStreamServiceInternal.java index 53a37896..32f7ddc9 100644 --- a/src/main/java/com/coveo/pushapiclient/UpdateStreamServiceInternal.java +++ b/src/main/java/com/coveo/pushapiclient/UpdateStreamServiceInternal.java @@ -11,12 +11,12 @@ class UpdateStreamServiceInternal { private final Logger logger; private final StreamEnabledSource source; private final PlatformClient platformClient; - private final DocumentUploadQueue queue; + private final StreamDocumentUploadQueue queue; private FileContainer fileContainer; public UpdateStreamServiceInternal( final StreamEnabledSource source, - final DocumentUploadQueue queue, + final StreamDocumentUploadQueue queue, final PlatformClient platformClient, final Logger logger) { this.source = source; @@ -34,6 +34,15 @@ public FileContainer addOrUpdate(DocumentBuilder document) return this.fileContainer; } + public FileContainer addPartialUpdate(PartialUpdateDocument document) + throws IOException, InterruptedException { + if (this.fileContainer == null) { + this.fileContainer = this.createFileContainer(); + } + queue.add(document); + return this.fileContainer; + } + public FileContainer delete(DeleteDocument document) throws IOException, InterruptedException { if (this.fileContainer == null) { this.fileContainer = this.createFileContainer(); diff --git a/src/test/java/com/coveo/pushapiclient/PartialUpdateDocumentTest.java b/src/test/java/com/coveo/pushapiclient/PartialUpdateDocumentTest.java new file mode 100644 index 00000000..4023df58 --- /dev/null +++ b/src/test/java/com/coveo/pushapiclient/PartialUpdateDocumentTest.java @@ -0,0 +1,110 @@ +package com.coveo.pushapiclient; + +import static org.junit.Assert.*; + +import java.util.Map; +import org.junit.Test; + +public class PartialUpdateDocumentTest { + + @Test + public void shouldCreatePartialUpdateDocumentWithArrayAppendOperator() { + String[] value = {"value1", "value2"}; + PartialUpdateDocument document = + new PartialUpdateDocument("doc1", PartialUpdateOperator.ARRAYAPPEND, "field1", value); + assertEquals("doc1", document.documentId); + assertEquals(PartialUpdateOperator.ARRAYAPPEND, document.operator); + assertEquals("field1", document.field); + assertArrayEquals(value, (String[]) document.value); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldThrowExceptionWhenValueIsNotArrayForArrayAppendOperator() { + PartialUpdateDocument document = + new PartialUpdateDocument("doc1", PartialUpdateOperator.ARRAYAPPEND, "field1", "value1"); + } + + @Test + public void shouldCreatePartialUpdateDocumentWithFieldValueReplaceOperator() { + String value = "value1"; + PartialUpdateDocument document = + new PartialUpdateDocument("doc1", PartialUpdateOperator.FIELDVALUEREPLACE, "field1", value); + assertEquals("doc1", document.documentId); + assertEquals(PartialUpdateOperator.FIELDVALUEREPLACE, document.operator); + assertEquals("field1", document.field); + assertEquals(value, document.value); + } + + public void shouldNotThrowExceptionWhenValueIsNull() { + PartialUpdateDocument document = + new PartialUpdateDocument("doc1", PartialUpdateOperator.FIELDVALUEREPLACE, "field1", null); + assertEquals("doc1", document.documentId); + assertEquals(PartialUpdateOperator.FIELDVALUEREPLACE, document.operator); + assertEquals("field1", document.field); + assertNull(document.value); + } + + @Test + public void shouldCreatePartialUpdateDocumentWithDictionaryPutOperator() { + Map value = Map.of("key1", "value1"); + PartialUpdateDocument document = + new PartialUpdateDocument("doc1", PartialUpdateOperator.DICTIONARYPUT, "field1", value); + assertEquals("doc1", document.documentId); + assertEquals(PartialUpdateOperator.DICTIONARYPUT, document.operator); + assertEquals("field1", document.field); + assertEquals(value, document.value); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldThrowExceptionWhenValueIsNotJsonForDictionaryPutOperator() { + PartialUpdateDocument document = + new PartialUpdateDocument("doc1", PartialUpdateOperator.DICTIONARYPUT, "field1", "value1"); + } + + @Test + public void shouldCreatePartialUpdateDocumentWithDictionaryRemoveOperator() { + String value = "value1"; + PartialUpdateDocument document = + new PartialUpdateDocument("doc1", PartialUpdateOperator.DICTIONARYREMOVE, "field1", value); + assertEquals("doc1", document.documentId); + assertEquals(PartialUpdateOperator.DICTIONARYREMOVE, document.operator); + assertEquals("field1", document.field); + assertEquals(value, document.value); + + String[] value2 = {"value1", "value2"}; + PartialUpdateDocument document2 = + new PartialUpdateDocument("doc2", PartialUpdateOperator.DICTIONARYREMOVE, "field2", value2); + assertEquals("doc2", document2.documentId); + assertEquals(PartialUpdateOperator.DICTIONARYREMOVE, document2.operator); + assertEquals("field2", document2.field); + assertEquals(value2, document2.value); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldThrowExceptionWhenValueIsNotStringOrArrayForDictionaryRemoveOperator() { + PartialUpdateDocument document = + new PartialUpdateDocument("doc1", PartialUpdateOperator.DICTIONARYREMOVE, "field1", 123); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldThrowExceptionWhenInvalidOperatorIsUsed() { + PartialUpdateDocument document = new PartialUpdateDocument("doc1", null, "field1", "value1"); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldThrowExceptionWhenDocumentIdIsNull() { + PartialUpdateDocument document = + new PartialUpdateDocument(null, PartialUpdateOperator.ARRAYAPPEND, "field1", "value1"); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldThrowExceptionWhenFieldIsNull() { + PartialUpdateDocument document = + new PartialUpdateDocument("doc1", PartialUpdateOperator.ARRAYAPPEND, null, "value1"); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldThrowExceptionWhenOperatorIsNull() { + PartialUpdateDocument document = new PartialUpdateDocument("doc1", null, "field1", "value1"); + } +} diff --git a/src/test/java/com/coveo/pushapiclient/StreamDocumentUploadQueueTest.java b/src/test/java/com/coveo/pushapiclient/StreamDocumentUploadQueueTest.java new file mode 100644 index 00000000..12cd2f54 --- /dev/null +++ b/src/test/java/com/coveo/pushapiclient/StreamDocumentUploadQueueTest.java @@ -0,0 +1,250 @@ +package com.coveo.pushapiclient; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.io.IOException; +import java.util.ArrayList; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +public class StreamDocumentUploadQueueTest { + + @Mock private UploadStrategy uploadStrategy; + + @InjectMocks private StreamDocumentUploadQueue queue; + + private AutoCloseable closeable; + private DocumentBuilder documentToAdd; + private DeleteDocument documentToDelete; + private PartialUpdateDocument partialUpdateDocument; + + private int oneMegaByte = 1 * 1024 * 1024; + + private String generateStringFromBytes(int numBytes) { + // Check if the number of bytes is valid + if (numBytes <= 0) { + return ""; + } + + // Create a byte array with the specified length + byte[] bytes = new byte[numBytes]; + + // Fill the byte array with a pattern of ASCII characters + byte pattern = 65; // ASCII value for 'A' + for (int i = 0; i < numBytes; i++) { + bytes[i] = pattern; + } + + return new String(bytes); + } + + private DocumentBuilder generateDocumentFromSize(int numBytes) { + return new DocumentBuilder("https://my.document.uri?ref=1", "My bulky document") + .withData(generateStringFromBytes(numBytes)); + } + + private PartialUpdateDocument generatePartialUpdateDocumentFromSize(int numBytes) { + return new PartialUpdateDocument( + "https://my.document.uri?ref=1", + PartialUpdateOperator.FIELDVALUEREPLACE, + "field", + generateStringFromBytes(numBytes)); + } + + @Before + public void setup() { + String twoMegaByteData = generateStringFromBytes(2 * oneMegaByte); + + documentToAdd = + new DocumentBuilder("https://my.document.uri?ref=1", "My new document") + .withData(twoMegaByteData); + + documentToDelete = new DeleteDocument("https://my.document.uri?ref=3"); + + partialUpdateDocument = + new PartialUpdateDocument( + "https://my.document.uri?ref=4", + PartialUpdateOperator.FIELDVALUEREPLACE, + "field", + "value"); + + closeable = MockitoAnnotations.openMocks(this); + } + + @After + public void closeService() throws Exception { + closeable.close(); + } + + @Test + public void testIsEmpty() throws IOException, InterruptedException { + assertTrue(queue.isEmpty()); + } + + @Test + public void testIsNotEmpty() throws IOException, InterruptedException { + queue.add(documentToAdd); + assertFalse(queue.isEmpty()); + } + + @Test + public void testShouldReturnBatch() throws IOException, InterruptedException { + StreamUpdate batchUpdate = + new StreamUpdate( + new ArrayList<>() { + { + add(documentToAdd); + } + }, + new ArrayList<>() { + { + add(documentToDelete); + } + }, + new ArrayList<>() { + { + add(partialUpdateDocument); + } + }); + queue.add(documentToAdd); + queue.add(documentToDelete); + queue.add(partialUpdateDocument); + + assertEquals(batchUpdate, queue.getStream()); + } + + @Test + public void testFlushShouldNotUploadDocumentsWhenRequiredSizeIsNotMet() + throws IOException, InterruptedException { + // Adding 2MB document to the queue => queue has now 3MB of free space + // (5MB - 2MB = 3MB) + queue.add(documentToAdd); + // Adding 2MB document to the queue => queue has now 1MB of free space + // (3MB - 2MB = 1MB) + queue.add(documentToDelete); + + // The maximum queue size has not been reached yet (1MB left of free space). + // Therefore, the accumulated documents will not be automatically flushed. + // Unless the user runs `.flush()` the queue will keep the 4MB of documents + verify(uploadStrategy, times(0)).apply(any(BatchUpdate.class)); + } + + @Test + public void testShouldAutomaticallyFlushAccumulatedDocuments() + throws IOException, InterruptedException { + DocumentBuilder firstBulkyDocument = generateDocumentFromSize(2 * oneMegaByte); + PartialUpdateDocument secondBulkyDocument = + generatePartialUpdateDocumentFromSize(2 * oneMegaByte); + DocumentBuilder thirdBulkyDocument = generateDocumentFromSize(2 * oneMegaByte); + ArrayList emptyList = new ArrayList<>(); + StreamUpdate firstBatch = + new StreamUpdate( + new ArrayList<>() { + { + add(firstBulkyDocument); + } + }, + emptyList, + new ArrayList<>() { + { + add(secondBulkyDocument); + } + }); + + // Adding 3 documents of 2MB to the queue. After adding the first 2 documents, + // the queue size will reach 6MB, which exceeds the maximum queue size + // limit by 1MB. Therefore, the 2 first added documents will automatically be + // uploaded to the source. + queue.add(firstBulkyDocument); + queue.add(secondBulkyDocument); + verify(uploadStrategy, times(0)).apply(any(BatchUpdate.class)); + + // The 3rd document added to the queue will be included in a separate batch, + // which will not be uploaded unless the `flush()` method is called or until the + // queue size limit has been reached + queue.add(thirdBulkyDocument); + + verify(uploadStrategy, times(1)).apply(any(BatchUpdate.class)); + verify(uploadStrategy, times(1)).apply(firstBatch); + } + + @Test + public void testShouldManuallyFlushAccumulatedDocuments() + throws IOException, InterruptedException { + DocumentBuilder firstBulkyDocument = generateDocumentFromSize(2 * oneMegaByte); + PartialUpdateDocument secondBulkyDocument = + generatePartialUpdateDocumentFromSize(2 * oneMegaByte); + DocumentBuilder thirdBulkyDocument = generateDocumentFromSize(2 * oneMegaByte); + ArrayList emptyList = new ArrayList<>(); + ArrayList partialEmptyList = new ArrayList<>(); + StreamUpdate firstBatch = + new StreamUpdate( + new ArrayList<>() { + { + add(firstBulkyDocument); + } + }, + emptyList, + new ArrayList<>() { + { + add(secondBulkyDocument); + } + }); + + StreamUpdate secondBatch = + new StreamUpdate( + new ArrayList<>() { + { + add(thirdBulkyDocument); + } + }, + emptyList, + partialEmptyList); + + // Adding 3 documents of 2MB to the queue. After adding the first 2 documents, + // the queue size will reach 6MB, which exceeds the maximum queue size + // limit. Therefore, the 2 first added documents will automatically be uploaded + // to the source. + queue.add(firstBulkyDocument); + queue.add(secondBulkyDocument); + queue.add(thirdBulkyDocument); + + queue.flush(); + + // Additional flush will have no effect if documents where already flushed + queue.flush(); + + verify(uploadStrategy, times(2)).apply(any(StreamUpdate.class)); + verify(uploadStrategy, times(1)).apply(firstBatch); + verify(uploadStrategy, times(1)).apply(secondBatch); + } + + @Test + public void testAddingEmptyDocument() throws IOException, InterruptedException { + DocumentBuilder nullDocument = null; + + queue.add(nullDocument); + queue.flush(); + + verify(uploadStrategy, times(0)).apply(any(StreamUpdate.class)); + } + + @Rule public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void getBatchShouldThrowUnsupportedOperationException() { + expectedException.expect(UnsupportedOperationException.class); + queue.getBatch(); + } +} diff --git a/src/test/java/com/coveo/pushapiclient/StreamUpdateRecordTest.java b/src/test/java/com/coveo/pushapiclient/StreamUpdateRecordTest.java new file mode 100644 index 00000000..e8478f25 --- /dev/null +++ b/src/test/java/com/coveo/pushapiclient/StreamUpdateRecordTest.java @@ -0,0 +1,69 @@ +package com.coveo.pushapiclient; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import org.junit.Before; +import org.junit.Test; + +public class StreamUpdateRecordTest { + + private StreamUpdateRecord sur1; + private StreamUpdateRecord sur2; + private StreamUpdateRecord sur3; + private StreamUpdateRecord sur4; + + @Before + public void setUp() throws Exception { + Gson gson = new Gson(); + JsonObject json1 = gson.fromJson("{ \"key\": \"value1\" }", JsonObject.class); + JsonObject json2 = gson.fromJson("{ \"key\": \"value1\" }", JsonObject.class); + JsonObject json3 = gson.fromJson("{ \"key3\": \"value3\" }", JsonObject.class); + JsonObject json4 = gson.fromJson("{ \"key4\": \"value4\" }", JsonObject.class); + JsonObject json5 = gson.fromJson("{ \"key4\": \"value4\" }", JsonObject.class); + JsonObject json6 = gson.fromJson("{ \"key4\": \"value4\" }", JsonObject.class); + + JsonObject[] jsonArray1 = new JsonObject[] {json1, json3}; + JsonObject[] jsonArray2 = new JsonObject[] {json2, json3}; + JsonObject[] jsonArray3 = new JsonObject[] {json1, json3}; + JsonObject[] jsonArray4 = new JsonObject[] {json4, json3}; + JsonObject[] jsonArray5 = new JsonObject[] {json5, json6}; + JsonObject[] jsonArray6 = new JsonObject[] {json5, json3}; + + sur1 = new StreamUpdateRecord(jsonArray1, jsonArray2, jsonArray5); + sur2 = new StreamUpdateRecord(jsonArray1, jsonArray2, jsonArray5); + sur3 = new StreamUpdateRecord(jsonArray3, jsonArray4, jsonArray6); + sur4 = sur1; + } + + @Test + public void testToString() { + assertEquals(sur1.toString(), sur1.toString()); + assertEquals(sur1.toString(), sur2.toString()); + assertEquals(sur1.toString(), sur4.toString()); + assertNotEquals(sur1.toString(), sur3.toString()); + } + + @Test + public void testEquals() { + assertTrue(sur1.equals(sur1)); + assertTrue(sur1.equals(sur2)); + assertFalse(sur1.equals(sur3)); + assertTrue(sur1.equals(sur4)); + assertFalse(sur1.equals(null)); + } + + @Test + public void testHashCode() { + assertEquals(sur1.hashCode(), sur1.hashCode()); + assertEquals(sur1.hashCode(), sur2.hashCode()); + assertEquals(sur1.hashCode(), sur4.hashCode()); + assertEquals(sur3.hashCode(), sur3.hashCode()); + assertNotEquals(sur1.hashCode(), sur3.hashCode()); + assertNotEquals(sur2.hashCode(), sur3.hashCode()); + } +} diff --git a/src/test/java/com/coveo/pushapiclient/StreamUpdateTest.java b/src/test/java/com/coveo/pushapiclient/StreamUpdateTest.java new file mode 100644 index 00000000..2e68ed9e --- /dev/null +++ b/src/test/java/com/coveo/pushapiclient/StreamUpdateTest.java @@ -0,0 +1,99 @@ +package com.coveo.pushapiclient; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import org.junit.Before; +import org.junit.Test; + +public class StreamUpdateTest { + + private StreamUpdate stream1; + private StreamUpdate stream2; + private StreamUpdate stream3; + private StreamUpdate stream4; + + @Before + public void setUp() { + DocumentBuilder db1 = new DocumentBuilder("some_uri", "some_title"); + DocumentBuilder db2 = new DocumentBuilder("some_uri", "some_title"); + DocumentBuilder db3 = new DocumentBuilder("some_other_uri", "some_title"); + DocumentBuilder db4 = new DocumentBuilder("some_uri", "some_other_title"); + + List list1 = new ArrayList<>(); + list1.add(db1); + list1.add(db3); + + List list2 = new ArrayList<>(); + list2.add(db1); + list2.add(db4); + + List list3 = new ArrayList<>(); + list3.add(db2); + list3.add(db3); + list3.add(db4); + + DeleteDocument del1 = new DeleteDocument("123"); + DeleteDocument del2 = new DeleteDocument("456"); + DeleteDocument del3 = new DeleteDocument("789"); + + List delList1 = new ArrayList<>(); + delList1.add(del1); + delList1.add(del2); + + List delList2 = new ArrayList<>(); + delList2.add(del2); + delList2.add(del3); + + PartialUpdateDocument partialUpdateDocument1 = + new PartialUpdateDocument("123", PartialUpdateOperator.FIELDVALUEREPLACE, "field", "value"); + PartialUpdateDocument partialUpdateDocument2 = + new PartialUpdateDocument( + "456", PartialUpdateOperator.FIELDVALUEREPLACE, "field2", "value2"); + PartialUpdateDocument partialUpdateDocument3 = + new PartialUpdateDocument( + "789", PartialUpdateOperator.FIELDVALUEREPLACE, "field3", "value3"); + + List partialUpdateDocuments1 = new ArrayList<>(); + partialUpdateDocuments1.add(partialUpdateDocument1); + partialUpdateDocuments1.add(partialUpdateDocument2); + + List partialUpdateDocuments2 = new ArrayList<>(); + partialUpdateDocuments2.add(partialUpdateDocument2); + partialUpdateDocuments2.add(partialUpdateDocument3); + + stream1 = new StreamUpdate(list1, delList1, partialUpdateDocuments1); + stream2 = new StreamUpdate(list1, delList1, partialUpdateDocuments1); + stream3 = new StreamUpdate(list2, delList2, partialUpdateDocuments2); + stream4 = stream1; + } + + @Test + public void testToString() { + assertEquals(stream1.toString(), stream1.toString()); + assertEquals(stream1.toString(), stream2.toString()); + assertEquals(stream1.toString(), stream4.toString()); + assertNotEquals(stream1.toString(), stream3.toString()); + } + + @Test + public void testEquals() { + assertFalse(stream1.equals(null)); + assertTrue(stream1.equals(stream1)); + assertTrue(stream1.equals(stream2)); + assertTrue(stream1.equals(stream4)); + assertFalse(stream1.equals(stream3)); + } + + @Test + public void testHashCode() { + assertEquals(stream1.hashCode(), stream1.hashCode()); + assertEquals(stream1.hashCode(), stream2.hashCode()); + assertEquals(stream1.hashCode(), stream4.hashCode()); + assertNotEquals(stream1.hashCode(), stream3.hashCode()); + } +} diff --git a/src/test/java/com/coveo/pushapiclient/UpdateStreamServiceInternalTest.java b/src/test/java/com/coveo/pushapiclient/UpdateStreamServiceInternalTest.java index 9ee3f3f9..726769b8 100644 --- a/src/test/java/com/coveo/pushapiclient/UpdateStreamServiceInternalTest.java +++ b/src/test/java/com/coveo/pushapiclient/UpdateStreamServiceInternalTest.java @@ -22,7 +22,7 @@ public class UpdateStreamServiceInternalTest { private static final String SOURCE_ID = "my-source-id"; @Mock private StreamEnabledSource source; @Mock private PlatformClient platformClient; - @Mock private DocumentUploadQueue queue; + @Mock private StreamDocumentUploadQueue queue; @Mock private HttpResponse httpResponse; @Mock private Logger logger; @@ -32,6 +32,9 @@ public class UpdateStreamServiceInternalTest { private DocumentBuilder documentB; private DeleteDocument deleteDocumentA; private DeleteDocument deleteDocumentB; + private PartialUpdateDocument partialUpdateDocumentA; + private PartialUpdateDocument partialUpdateDocumentB; + private AutoCloseable closeable; @Before @@ -40,6 +43,18 @@ public void setUp() throws Exception { documentB = new DocumentBuilder("https://my.document.uri?ref=2", "My second document title"); deleteDocumentA = new DeleteDocument("https://my.document.uri?ref=3"); deleteDocumentB = new DeleteDocument("https://my.document.uri?ref=4"); + partialUpdateDocumentA = + new PartialUpdateDocument( + "https://my.document.uri?ref=5", + PartialUpdateOperator.FIELDVALUEREPLACE, + "fieldA", + "valueA"); + partialUpdateDocumentB = + new PartialUpdateDocument( + "https://my.document.uri?ref=6", + PartialUpdateOperator.FIELDVALUEREPLACE, + "fieldB", + "valueB"); closeable = MockitoAnnotations.openMocks(this); @@ -63,15 +78,19 @@ public void addOrUpdateShouldCreateFileContainer() throws IOException, Interrupt } @Test - public void addOrUpdateAndDeleteShouldAddDocumentsToQueue() + public void addOrUpdateAndPartialAndDeleteShouldAddDocumentsToQueue() throws IOException, InterruptedException { service.addOrUpdate(documentA); service.addOrUpdate(documentB); service.delete(deleteDocumentA); + service.addPartialUpdate(partialUpdateDocumentA); + service.addPartialUpdate(partialUpdateDocumentB); verify(queue, times(1)).add(documentA); verify(queue, times(1)).add(documentB); verify(queue, times(1)).add(deleteDocumentA); + verify(queue, times(1)).add(partialUpdateDocumentA); + verify(queue, times(1)).add(partialUpdateDocumentB); } @Test @@ -82,6 +101,14 @@ public void deleteShouldCreateFileContainer() throws IOException, InterruptedExc verify(this.platformClient, times(1)).createFileContainer(); } + @Test + public void partialUpdateShouldCreateFileContainer() throws IOException, InterruptedException { + service.addPartialUpdate(partialUpdateDocumentA); + service.addPartialUpdate(partialUpdateDocumentB); + + verify(this.platformClient, times(1)).createFileContainer(); + } + @Test public void closeShouldPushFileContainerOnAddOrUpdate() throws IOException, InterruptedException, NoOpenFileContainerException {