From f047b68c0aa531e389c408fcd6f897e8401c7a1b Mon Sep 17 00:00:00 2001 From: Olivier Lamothe Date: Mon, 5 Jul 2021 08:47:03 -0400 Subject: [PATCH 1/2] feat: add support for compressed binary data --- .../pushapiclient/CompressedBinaryData.java | 23 +++++++------------ .../coveo/pushapiclient/DocumentBuilder.java | 8 +++---- .../coveo/pushapiclient/PlatformClient.java | 4 ++-- .../java/com/coveo/pushapiclient/Source.java | 3 ++- .../com/coveo/testlocally/TestingLocally.java | 18 +++++++++++++-- 5 files changed, 31 insertions(+), 25 deletions(-) diff --git a/src/main/java/com/coveo/pushapiclient/CompressedBinaryData.java b/src/main/java/com/coveo/pushapiclient/CompressedBinaryData.java index 56810b38..c8352e40 100644 --- a/src/main/java/com/coveo/pushapiclient/CompressedBinaryData.java +++ b/src/main/java/com/coveo/pushapiclient/CompressedBinaryData.java @@ -2,24 +2,17 @@ /** * The original binary item content, compressed using one of the supported compression types (Deflate, GZip, LZMA, Uncompressed, or ZLib), and then Base64 encoded. - * + *

* You can use this parameter when you're pushing a compressed binary item (such as XML/HTML, PDF, Word, or binary) whose size is less than 5 MB. - * + *

* Whenever you're pushing an item whose size is 5 MB or more, use the CompressedBinaryDataFileIdproperty instead. - * + *

* If you're pushing less than 5 MB of textual (non-binary) content, you can use the data property instead. - * + *

* See https://docs.coveo.com/en/73 for more information. + * + * @param data The base64 encoded binary data. Example: `eJxzrUjMLchJBQAK4ALN` + * @param compressionType The compression type that was applied to your document. */ -public class CompressedBinaryData { - /** - * The compression type that was applied to your document. - */ - public CompressionType compressionType; - /** - * The base64 encoded binary data. - * - * Example: `eJxzrUjMLchJBQAK4ALN` - */ - public String data; +public record CompressedBinaryData(String data, CompressionType compressionType) { } \ No newline at end of file diff --git a/src/main/java/com/coveo/pushapiclient/DocumentBuilder.java b/src/main/java/com/coveo/pushapiclient/DocumentBuilder.java index edda89ab..6379123b 100644 --- a/src/main/java/com/coveo/pushapiclient/DocumentBuilder.java +++ b/src/main/java/com/coveo/pushapiclient/DocumentBuilder.java @@ -85,7 +85,6 @@ public DocumentBuilder withPermanentId(String permanentId) { } public DocumentBuilder withCompressedBinaryData(CompressedBinaryData compressedBinaryData) { - this.validateCompressedBinaryData(compressedBinaryData.data); this.document.compressedBinaryData = compressedBinaryData; return this; } @@ -152,6 +151,9 @@ public String marshal() { jsonDocument.add(key, new Gson().toJsonTree(value)); }); jsonDocument.remove("metadata"); + if (this.document.compressedBinaryData != null) { + jsonDocument.addProperty("compressedBinaryData", this.document.compressedBinaryData.data()); + } return jsonDocument.toString(); } @@ -165,10 +167,6 @@ private void setMetadataValue(String key, Object metadataValue) { this.document.metadata.put(key, metadataValue); } - private void validateCompressedBinaryData(String data) { - // TODO - } - private void validateFileExtension(String fileExtension) { // TODO } diff --git a/src/main/java/com/coveo/pushapiclient/PlatformClient.java b/src/main/java/com/coveo/pushapiclient/PlatformClient.java index 3f6535ef..3bdc40a1 100644 --- a/src/main/java/com/coveo/pushapiclient/PlatformClient.java +++ b/src/main/java/com/coveo/pushapiclient/PlatformClient.java @@ -113,9 +113,9 @@ public HttpResponse manageSecurityIdentities(String securityProviderId, return this.httpClient.send(request, HttpResponse.BodyHandlers.ofString()); } - public HttpResponse pushDocument(String sourceId, String documentJSON, String documentId) throws IOException, InterruptedException { + public HttpResponse pushDocument(String sourceId, String documentJSON, String documentId, CompressionType compressionType) throws IOException, InterruptedException { String[] headers = this.getHeaders(this.getAuthorizationHeader(), this.getContentTypeApplicationJSONHeader()); - URI uri = URI.create(this.getBasePushURL() + String.format("/sources/%s/documents?documentId=%s", sourceId, documentId)); + URI uri = URI.create(this.getBasePushURL() + String.format("/sources/%s/documents?documentId=%s&compressionType=%s", sourceId, documentId, compressionType.toString())); HttpRequest request = HttpRequest.newBuilder() .headers(headers) diff --git a/src/main/java/com/coveo/pushapiclient/Source.java b/src/main/java/com/coveo/pushapiclient/Source.java index 84813148..5fa5574b 100644 --- a/src/main/java/com/coveo/pushapiclient/Source.java +++ b/src/main/java/com/coveo/pushapiclient/Source.java @@ -35,6 +35,7 @@ public HttpResponse manageSecurityIdentities(String securityProviderId, } public HttpResponse addOrUpdateDocument(String sourceId, DocumentBuilder docBuilder) throws IOException, InterruptedException { - return this.platformClient.pushDocument(sourceId, docBuilder.marshal(), docBuilder.getDocument().uri); + CompressionType compressionType = docBuilder.getDocument().compressedBinaryData != null ? docBuilder.getDocument().compressedBinaryData.compressionType() : CompressionType.UNCOMPRESSED; + return this.platformClient.pushDocument(sourceId, docBuilder.marshal(), docBuilder.getDocument().uri, compressionType); } } diff --git a/src/main/java/com/coveo/testlocally/TestingLocally.java b/src/main/java/com/coveo/testlocally/TestingLocally.java index afdb857d..329cbb02 100644 --- a/src/main/java/com/coveo/testlocally/TestingLocally.java +++ b/src/main/java/com/coveo/testlocally/TestingLocally.java @@ -6,7 +6,7 @@ import java.io.IOException; import java.net.http.HttpResponse; -import java.util.Date; +import java.util.Base64; import java.util.HashMap; public class TestingLocally { @@ -27,7 +27,10 @@ public static void main(String[] args) { } public static void testPushDocument(String sourceId, Source source) { - DocumentBuilder doc = new DocumentBuilder("https://perdu.com", "the title").withData("this is searchable").withDate(new Date()); + DocumentBuilder doc = new DocumentBuilder("https://perdu.com", "the title") + .withData("this is searchable") + .withDate(123l) + .withModifiedDate(0l); DocumentBuilder docWithMetadata = new DocumentBuilder("https://perdu.com/3", "the title 3").withMetadata(new HashMap<>() {{ put("foo", "bar"); put("my_field_1", "1"); @@ -35,14 +38,25 @@ public static void testPushDocument(String sourceId, Source source) { put("my_field_3", 1234); put("my_field_4", new String[]{"a", "b", "c"}); }}); + String encoded = Base64.getEncoder().encodeToString("this is binary data".getBytes()); + String decoded = new String(Base64.getDecoder().decode(encoded)); + DocumentBuilder docWithBinaryData = new DocumentBuilder("https://perdu.com/binarydata", "the title binary data") + .withCompressedBinaryData(new CompressedBinaryData(encoded, CompressionType.UNCOMPRESSED)).withFileExtension(".txt"); + + System.out.println(doc.marshal()); System.out.println(docWithMetadata.marshal()); + System.out.println(docWithBinaryData.marshal()); try { source.addOrUpdateDocument(sourceId, doc); source.addOrUpdateDocument(sourceId, docWithMetadata); + source.addOrUpdateDocument(sourceId, docWithBinaryData); } catch (IOException | InterruptedException e) { System.out.println(e); } + + + String asdf = "if you just let technology evolve; fault my librarian; believe in the market; "; } public static void testManageIdentities(Source source) { From b6467e6cde1c00b44548b8c44bbc68224828708d Mon Sep 17 00:00:00 2001 From: Olivier Lamothe Date: Mon, 5 Jul 2021 08:48:00 -0400 Subject: [PATCH 2/2] cleanup --- src/main/java/com/coveo/testlocally/TestingLocally.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/coveo/testlocally/TestingLocally.java b/src/main/java/com/coveo/testlocally/TestingLocally.java index 329cbb02..7c07f4a9 100644 --- a/src/main/java/com/coveo/testlocally/TestingLocally.java +++ b/src/main/java/com/coveo/testlocally/TestingLocally.java @@ -55,8 +55,6 @@ public static void testPushDocument(String sourceId, Source source) { System.out.println(e); } - - String asdf = "if you just let technology evolve; fault my librarian; believe in the market; "; } public static void testManageIdentities(Source source) {