From 0cf95462275dca6d169b1ec8190b8fabf7b79410 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 19 Aug 2024 09:23:16 -0400 Subject: [PATCH 01/39] Bumped to 7.0-SNAPSHOT --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index b7afba146..3abd980d4 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ group=com.marklogic -version=7.0.0 +version=7.0-SNAPSHOT describedName=MarkLogic Java Client API publishUrl=file:../marklogic-java/releases From 578aad45956a6c80b93ff0847337d5d3e7fb5898 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 26 Aug 2024 11:48:39 -0400 Subject: [PATCH 02/39] Better way of disabling some tests when using reverse proxy server --- .../DisabledWhenUsingReverseProxyServer.java | 23 ++++++++++++++++++ .../client/test/rows/RowManagerTest.java | 9 +++---- .../client/test/ssl/OneWaySSLTest.java | 18 ++++---------- .../marklogic/client/test/ssl/SSLTest.java | 4 ++++ .../client/test/ssl/TwoWaySSLTest.java | 24 ++++--------------- 5 files changed, 40 insertions(+), 38 deletions(-) create mode 100644 marklogic-client-api/src/test/java/com/marklogic/client/test/junit5/DisabledWhenUsingReverseProxyServer.java diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/junit5/DisabledWhenUsingReverseProxyServer.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/junit5/DisabledWhenUsingReverseProxyServer.java new file mode 100644 index 000000000..581c4c887 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/junit5/DisabledWhenUsingReverseProxyServer.java @@ -0,0 +1,23 @@ +/* + * Copyright © 2024 MarkLogic Corporation. All Rights Reserved. + */ +package com.marklogic.client.test.junit5; + +import com.marklogic.client.test.Common; +import org.junit.jupiter.api.extension.ConditionEvaluationResult; +import org.junit.jupiter.api.extension.ExecutionCondition; +import org.junit.jupiter.api.extension.ExtensionContext; + +/** + * Some tests can't run when using the reverse proxy server; for example, TLS/SSL messages don't yet work with our + * reverse proxy server. + */ +public class DisabledWhenUsingReverseProxyServer implements ExecutionCondition { + + @Override + public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext extensionContext) { + return Common.USE_REVERSE_PROXY_SERVER ? + ConditionEvaluationResult.disabled("This test is disabled when the tests are run against a reverse proxy server.") : + ConditionEvaluationResult.enabled("This test is enabled since the reverse proxy server is not being used."); + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/RowManagerTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/RowManagerTest.java index ffb08b959..3e42c3251 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/RowManagerTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/RowManagerTest.java @@ -19,6 +19,7 @@ import com.marklogic.client.row.RowManager.RowStructure; import com.marklogic.client.row.RowRecord.ColumnKind; import com.marklogic.client.test.Common; +import com.marklogic.client.test.junit5.DisabledWhenUsingReverseProxyServer; import com.marklogic.client.test.junit5.RequiresML11; import com.marklogic.client.type.*; import com.marklogic.client.util.EditableNamespaceContext; @@ -511,13 +512,9 @@ public void testSQLNoResults() { } @Test - @ExtendWith(RequiresML11.class) + // A different kind of error is thrown when using the reverse proxy. + @ExtendWith({DisabledWhenUsingReverseProxyServer.class, RequiresML11.class}) public void testErrorWhileStreamingRows() { - if (Common.USE_REVERSE_PROXY_SERVER) { - // Different kind of error is thrown when using reverse proxy. - return; - } - final String validQueryThatEventuallyThrowsAnError = "select case " + "when lastName = 'Byron' then fn_error(fn_qname('', 'SQL-TABLENOTFOUND'), 'Internal Server Error') end, " + "opticUnitTest.musician.* from (select * from opticUnitTest.musician order by lastName)"; diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/OneWaySSLTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/OneWaySSLTest.java index 178f85589..6a24f8c3e 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/OneWaySSLTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/OneWaySSLTest.java @@ -5,6 +5,7 @@ import com.marklogic.client.ForbiddenUserException; import com.marklogic.client.MarkLogicIOException; import com.marklogic.client.test.Common; +import com.marklogic.client.test.junit5.DisabledWhenUsingReverseProxyServer; import com.marklogic.client.test.junit5.RequireSSLExtension; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -21,7 +22,10 @@ * certificate. See TwoWaySSLTest for scenarios where the client presents its own certificate which the server must * trust. */ -@ExtendWith(RequireSSLExtension.class) +@ExtendWith({ + DisabledWhenUsingReverseProxyServer.class, + RequireSSLExtension.class +}) class OneWaySSLTest { /** @@ -34,14 +38,6 @@ class OneWaySSLTest { */ @Test void trustAllManager() throws Exception { - if (Common.USE_REVERSE_PROXY_SERVER) { - /** - * Have not been able to get this to work yet, see the ReverseProxyServer class in test-app for more info. - * We know SSL works fine when hitting MarkLogic Cloud though, which is the important part. - */ - return; - } - SSLContext sslContext = SSLContext.getInstance("TLSv1.2"); sslContext.init(null, new TrustManager[]{Common.TRUST_ALL_MANAGER}, null); @@ -63,10 +59,6 @@ void trustAllManager() throws Exception { */ @Test void trustManagerThatOnlyTrustsTheCertificateFromTheCertificateTemplate() { - if (Common.USE_REVERSE_PROXY_SERVER) { - return; - } - DatabaseClient client = Common.newClientBuilder() .withSSLProtocol("TLSv1.2") .withTrustManager(RequireSSLExtension.newSecureTrustManager()) diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/SSLTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/SSLTest.java index e69fa35b5..d7a704658 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/SSLTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/SSLTest.java @@ -10,6 +10,8 @@ import com.marklogic.client.io.StringHandle; import com.marklogic.client.test.Common; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledOnJre; +import org.junit.jupiter.api.condition.JRE; import javax.net.ssl.*; import javax.security.auth.x500.X500Principal; @@ -78,6 +80,8 @@ public void testSSLAuth() throws NoSuchAlgorithmException, KeyManagementExceptio } @Test + // Not able to mock the X509Certificate class on Java 21. + @EnabledOnJre({JRE.JAVA_8, JRE.JAVA_11, JRE.JAVA_17}) public void testHostnameVerifier() throws SSLException, CertificateParsingException { // three things our SSLHostnameVerifier will capture AtomicReference capturedHost = new AtomicReference<>(); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/TwoWaySSLTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/TwoWaySSLTest.java index 579d8b8d8..ceec255bd 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/TwoWaySSLTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/TwoWaySSLTest.java @@ -9,6 +9,7 @@ import com.marklogic.client.eval.EvalResultIterator; import com.marklogic.client.io.StringHandle; import com.marklogic.client.test.Common; +import com.marklogic.client.test.junit5.DisabledWhenUsingReverseProxyServer; import com.marklogic.client.test.junit5.RequireSSLExtension; import com.marklogic.mgmt.ManageClient; import com.marklogic.mgmt.resource.appservers.ServerManager; @@ -35,7 +36,10 @@ import static org.junit.jupiter.api.Assertions.*; -@ExtendWith(RequireSSLExtension.class) +@ExtendWith({ + DisabledWhenUsingReverseProxyServer.class, + RequireSSLExtension.class +}) public class TwoWaySSLTest { private final static String TEST_DOCUMENT_URI = "/optic/test/musician1.json"; @@ -54,9 +58,6 @@ public class TwoWaySSLTest { @BeforeAll public static void setup() throws Exception { - if (Common.USE_REVERSE_PROXY_SERVER) { - return; - } // Create a client using the java-unittest app server - which requires SSL via RequiresSSLExtension - and that // talks to the Security database. securityClient = Common.newClientBuilder() @@ -82,9 +83,6 @@ public static void setup() throws Exception { @AfterAll public static void teardown() { - if (Common.USE_REVERSE_PROXY_SERVER) { - return; - } removeTwoWaySSLConfig(); deleteCertificateAuthority(); } @@ -101,10 +99,6 @@ public static void teardown() { */ @Test void digestAuthentication() { - if (Common.USE_REVERSE_PROXY_SERVER) { - return; - } - // This client uses our Java KeyStore file with a client certificate in it, so it should work. DatabaseClient clientWithCert = Common.newClientBuilder() .withKeyStorePath(keyStoreFile.getAbsolutePath()) @@ -199,10 +193,6 @@ void invalidKeyStoreAlgorithm() { */ @Test void certificateAuthenticationWithSSLContext() throws Exception { - if (Common.USE_REVERSE_PROXY_SERVER) { - return; - } - setAuthenticationToCertificate(); try { SSLContext sslContext = createSSLContextWithClientCertificate(keyStoreFile); @@ -223,10 +213,6 @@ void certificateAuthenticationWithSSLContext() throws Exception { */ @Test void certificateAuthenticationWithCertificateFileAndPassword() { - if (Common.USE_REVERSE_PROXY_SERVER) { - return; - } - setAuthenticationToCertificate(); try { DatabaseClient client = Common.newClientBuilder() From 10ee4fe8558a1b0d20868ba57453d2075b8233ae Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 3 Sep 2024 14:49:31 -0400 Subject: [PATCH 03/39] MLE-16147 Added disabled test for score-bm25 fix Will enable this once the server nightly build has the fix. --- .../rows/FromSearchDocsWithOptionsTest.java | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromSearchDocsWithOptionsTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromSearchDocsWithOptionsTest.java index 2b00a2b7c..e340ceaf4 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromSearchDocsWithOptionsTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromSearchDocsWithOptionsTest.java @@ -1,16 +1,20 @@ package com.marklogic.client.test.rows; -import com.marklogic.client.FailedRequestException; -import com.marklogic.client.expression.PlanBuilder; +import com.marklogic.client.io.Format; +import com.marklogic.client.io.SearchHandle; +import com.marklogic.client.io.StringHandle; +import com.marklogic.client.query.QueryManager; import com.marklogic.client.row.RowRecord; +import com.marklogic.client.test.Common; import com.marklogic.client.test.junit5.RequiresML12; import com.marklogic.client.type.PlanSearchOptions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; @ExtendWith(RequiresML12.class) class FromSearchDocsWithOptionsTest extends AbstractOpticUpdateTest { @@ -27,6 +31,21 @@ void bm25() { assertEquals(2, rows.size()); } + @Disabled("Waiting on fix for MLE-16147.") + @Test + void bm25ViaSearchOptions() { + final String combinedQuery = "" + + "score-bm25" + + "saxophone"; + + QueryManager queryManager = Common.client.newQueryManager(); + SearchHandle results = queryManager.search( + queryManager.newRawCombinedQueryDefinition(new StringHandle(combinedQuery).withFormat(Format.XML)), + new SearchHandle()); + assertEquals(2, results.getTotalResults(), "Just doing a simple search to verify that score-bm25 is " + + "recognized as a valid search option."); + } + @Test void qualityWeight() { // Note that this does not actually test that the scoring is correct. From 6286e5f9ddb69863d180c1e7a0ce894b3a2fafb0 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 4 Oct 2024 14:26:06 -0400 Subject: [PATCH 04/39] Bumping some test dependencies --- marklogic-client-api-functionaltests/build.gradle | 14 +++++++------- marklogic-client-api/build.gradle | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/marklogic-client-api-functionaltests/build.gradle b/marklogic-client-api-functionaltests/build.gradle index 5499009f6..0a581086c 100755 --- a/marklogic-client-api-functionaltests/build.gradle +++ b/marklogic-client-api-functionaltests/build.gradle @@ -20,21 +20,21 @@ task testSandbox(type:Test) { dependencies { implementation project (':marklogic-client-api') - implementation 'org.skyscreamer:jsonassert:1.5.1' - implementation 'org.slf4j:slf4j-api:1.7.36' - implementation 'commons-io:commons-io:2.11.0' + implementation 'org.skyscreamer:jsonassert:1.5.3' + implementation 'org.slf4j:slf4j-api:2.0.16' + implementation 'commons-io:commons-io:2.17.0' implementation 'com.squareup.okhttp3:okhttp:4.12.0' implementation "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}" implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" implementation "org.jdom:jdom2:2.0.6.1" - implementation ("com.marklogic:ml-app-deployer:4.8.0") { + implementation ("com.marklogic:ml-app-deployer:5.0.0") { exclude module: "marklogic-client-api" } testImplementation 'ch.qos.logback:logback-classic:1.3.14' - testImplementation 'org.junit.jupiter:junit-jupiter:5.10.1' - testImplementation 'org.xmlunit:xmlunit-legacy:2.9.0' - testImplementation 'org.apache.commons:commons-lang3:3.14.0' + testImplementation 'org.junit.jupiter:junit-jupiter:5.11.1' + testImplementation 'org.xmlunit:xmlunit-legacy:2.10.0' + testImplementation 'org.apache.commons:commons-lang3:3.17.0' testImplementation 'org.apache.httpcomponents:httpclient:4.5.14' } diff --git a/marklogic-client-api/build.gradle b/marklogic-client-api/build.gradle index 78573d49f..b3a0dd273 100644 --- a/marklogic-client-api/build.gradle +++ b/marklogic-client-api/build.gradle @@ -43,14 +43,14 @@ dependencies { compileOnly 'org.dom4j:dom4j:2.1.4' compileOnly 'com.google.code.gson:gson:2.10.1' - testImplementation 'org.junit.jupiter:junit-jupiter:5.10.1' + testImplementation 'org.junit.jupiter:junit-jupiter:5.11.1' // Forcing junit version to avoid vulnerability with older version in xmlunit testImplementation 'junit:junit:4.13.2' - testImplementation 'org.xmlunit:xmlunit-legacy:2.9.1' + testImplementation 'org.xmlunit:xmlunit-legacy:2.10.0' testImplementation project(':examples') // Allows talking to the Manage API. - testImplementation ("com.marklogic:ml-app-deployer:4.8.0") { + testImplementation ("com.marklogic:ml-app-deployer:5.0.0") { exclude module: "marklogic-client-api" } @@ -63,11 +63,11 @@ dependencies { testImplementation 'ch.qos.logback:logback-classic:1.3.14' // schema validation issue with testImplementation 'xerces:xercesImpl:2.12.0' testImplementation 'org.opengis.cite.xerces:xercesImpl-xsd11:2.12-beta-r1667115' - testImplementation 'org.apache.commons:commons-lang3:3.14.0' + testImplementation 'org.apache.commons:commons-lang3:3.17.0' testImplementation 'org.apache.httpcomponents:httpclient:4.5.14' testImplementation 'com.opencsv:opencsv:4.6' testImplementation 'org.geonames:geonames:1.0' - testImplementation 'org.skyscreamer:jsonassert:1.5.1' + testImplementation 'org.skyscreamer:jsonassert:1.5.3' } // Ensure that mlHost and mlPassword can override the defaults of localhost/admin if they've been modified From da048b2612598d15965177be3af3df551d9f0e7f Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Mon, 7 Oct 2024 13:12:16 -0400 Subject: [PATCH 05/39] Can now specify a page length of 0 when searching with OkHttpServices. --- .gitignore | 2 + .../search/SearchWithPageLengthTest.java | 59 +++++++++++++++++++ .../marklogic/client/impl/OkHttpServices.java | 2 +- 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/search/SearchWithPageLengthTest.java diff --git a/.gitignore b/.gitignore index 068e73dd3..1554b1c8b 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,5 @@ ml-development-tools/src/test/ml-modules ml-development-tools/src/test/java/com/marklogic/client/test/dbfunction/generated .vscode +docker-compose.yaml +docker/ diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/search/SearchWithPageLengthTest.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/search/SearchWithPageLengthTest.java new file mode 100644 index 000000000..449f5f99f --- /dev/null +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/search/SearchWithPageLengthTest.java @@ -0,0 +1,59 @@ +/* + * Copyright © 2024 MarkLogic Corporation. All Rights Reserved. + */ + +package com.marklogic.client.fastfunctest.search; + +import com.marklogic.client.fastfunctest.AbstractFunctionalTest; +import com.marklogic.client.io.*; +import com.marklogic.client.query.*; +import org.junit.jupiter.api.*; + +import java.io.FileNotFoundException; + +import static org.custommonkey.xmlunit.XMLAssert.*; + + +class SearchWithPageLengthTest extends AbstractFunctionalTest { + + static final QueryManager queryMgr = client.newQueryManager(); + StringQueryDefinition qd = queryMgr.newStringDefinition(); + + @BeforeEach + public void testSetup() throws FileNotFoundException { + String[] filenames = {"constraint1.xml", "constraint2.xml", "constraint3.xml", "constraint4.xml", "constraint5.xml"}; + for (String filename : filenames) { + writeDocumentUsingInputStreamHandle(client, filename, "/return-results-false/", "XML"); + } + } + + @AfterEach + public void testCleanUp() { + deleteDocuments(connectAsAdmin()); + } + + @Test + void testSearchPageLengthZero() { + queryMgr.setPageLength(0); + qd.setCriteria("Bush"); + SearchHandle resultsHandle = new SearchHandle(); + queryMgr.search(qd, resultsHandle); + + assertEquals(0, resultsHandle.getPageLength()); + assertEquals("The server allows for a zero page length, which results in no results being returned.", + 0, resultsHandle.getMatchResults().length); + } + + + @Test + void testSearchPageLengthNegative() { + queryMgr.setPageLength(-1); + qd.setCriteria("Bush"); + SearchHandle resultsHandle = new SearchHandle(); + queryMgr.search(qd, resultsHandle); + + assertEquals("Negative page lengths are not sent to the server, so the default page length of 10 should be used.", + 10, resultsHandle.getPageLength()); + assertEquals(2, resultsHandle.getMatchResults().length); + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java index 0537a6f06..a619aa51b 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java @@ -1908,7 +1908,7 @@ public T search(RequestLogger reqlog, T searchHandl params.add("start", Long.toString(start)); } - if (len > 0) { + if (len > -1) { params.add("pageLength", Long.toString(len)); } From 8c8c745405b892a24c2ad96b1d651e65767ade92 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 7 Oct 2024 15:42:29 -0400 Subject: [PATCH 06/39] Bumping slf4j-api to 2.0.16 This may be the root cause of the logging issue I'm running into with Flux - at least according to https://www.slf4j.org/codes.html#StaticLoggerBinder , as I verified that slf4j-api 1.x is sneaking onto the classpath. This should otherwise be a no-op for all of the Java Client, it doesn't care what version of slf4j-api it's using. --- examples/build.gradle | 4 ++-- marklogic-client-api/build.gradle | 2 +- test-app/build.gradle | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/build.gradle b/examples/build.gradle index 57209302e..d9793d5fb 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -13,9 +13,9 @@ dependencies { // The 'api' configuration is used so that the test configuration in marklogic-client-api doesn't have to declare // all of these dependencies. This library project won't otherwise be depended on by anything else as it's not // setup for publishing. - api 'com.squareup.okhttp3:okhttp:4.11.0' + api 'com.squareup.okhttp3:okhttp:4.12.0' api 'io.github.rburgst:okhttp-digest:2.7' - api 'org.slf4j:slf4j-api:1.7.36' + api 'org.slf4j:slf4j-api:2.0.16' api "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" // hsqldb < 2.7 has a High CVE - https://nvd.nist.gov/vuln/detail/CVE-2022-41853 . diff --git a/marklogic-client-api/build.gradle b/marklogic-client-api/build.gradle index b3a0dd273..3561a86dd 100644 --- a/marklogic-client-api/build.gradle +++ b/marklogic-client-api/build.gradle @@ -29,7 +29,7 @@ dependencies { implementation "com.sun.mail:jakarta.mail:2.0.1" implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' - implementation 'org.slf4j:slf4j-api:1.7.36' + implementation 'org.slf4j:slf4j-api:2.0.16' implementation "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}" implementation "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}" implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" diff --git a/test-app/build.gradle b/test-app/build.gradle index 0c672196c..c26ccf09a 100644 --- a/test-app/build.gradle +++ b/test-app/build.gradle @@ -8,7 +8,7 @@ dependencies { implementation "io.undertow:undertow-core:2.2.32.Final" implementation "io.undertow:undertow-servlet:2.2.32.Final" implementation "com.marklogic:ml-javaclient-util:4.8.0" - implementation 'org.slf4j:slf4j-api:1.7.36' + implementation 'org.slf4j:slf4j-api:2.0.16' implementation 'ch.qos.logback:logback-classic:1.3.14' implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" implementation 'com.squareup.okhttp3:okhttp:4.12.0' From a1ac6ba59a519fd2f6e50dc12b4ae99012179aac Mon Sep 17 00:00:00 2001 From: asinha Date: Mon, 7 Oct 2024 12:19:56 -0700 Subject: [PATCH 07/39] MLE-14438 : Use shortest-path function in Java plan builder --- .../fastfunctest/AbstractFunctionalTest.java | 2 + .../fastfunctest/TestOpticOnTriples.java | 132 ++++++++++++++++++ .../client/expression/PlanBuilder.java | 77 ++++++++++ .../client/impl/PlanBuilderImpl.java | 72 ++++++++++ .../client/test/PlanGeneratedTest.java | 61 ++++---- 5 files changed, 320 insertions(+), 24 deletions(-) diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/AbstractFunctionalTest.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/AbstractFunctionalTest.java index 31cf93161..43f916083 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/AbstractFunctionalTest.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/AbstractFunctionalTest.java @@ -46,6 +46,7 @@ public abstract class AbstractFunctionalTest extends BasicJavaClientREST { protected static MarkLogicVersion markLogicVersion; protected static boolean isML11OrHigher; + protected static boolean isML12OrHigher; private static String originalHttpPort; private static String originalRestServerName; @@ -63,6 +64,7 @@ public static void initializeClients() { markLogicVersion = MarkLogicVersion.getMarkLogicVersion(connectAsAdmin()); System.out.println("ML version: " + markLogicVersion.getVersionString()); isML11OrHigher = markLogicVersion.getMajor() >= 11; + isML12OrHigher = markLogicVersion.getMajor() >= 12; client = newDatabaseClientBuilder().build(); schemasClient = newDatabaseClientBuilder().withDatabase("java-functest-schemas").build(); diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestOpticOnTriples.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestOpticOnTriples.java index 8d91eb863..271f29179 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestOpticOnTriples.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestOpticOnTriples.java @@ -1423,4 +1423,136 @@ public void testPatternValuePermutations() assertEquals( "19", nodeValNeg1.path("PlayerAge").path("value").asText()); assertEquals( "Midfielder", nodeValNeg1.path("PlayerPosition").path("value").asText()); } + + @Test + public void testShortestPathWithColumnInputs() + { + if(!isML12OrHigher){ + return; + } + + RowManager rowMgr = client.newRowManager(); + PlanBuilder p = rowMgr.newPlanBuilder(); + + PlanColumn teamIdCol = p.col("player_team"); + PlanColumn teamNameCol = p.col("team_name"); + PlanColumn teamCityCol = p.col("team_city"); + + PlanPrefixer team = p.prefixer("http://marklogic.com/mlb/team/"); + PlanBuilder.ModifyPlan team_plan = p.fromTriples( + p.pattern(teamIdCol, team.iri("name"), teamNameCol), + p.pattern(teamIdCol, team.iri("city"), teamCityCol) + ).shortestPath(p.col("team_name"), p.col("team_city"), p.col("path"), p.col("length")); + JacksonHandle jacksonHandle = new JacksonHandle().withMimetype("application/json"); + rowMgr.resultDoc(team_plan, jacksonHandle); + JsonNode jsonResults = jacksonHandle.get(); + JsonNode jsonBindingsNodes = jsonResults.path("rows"); + for (int i=0; iop:xml-attribute-seq server function. * @param attribute the attribute values for the sequence * @return a server expression with the attribute-node server data type + * @deprecated (as of 4.2) construct a {@link com.marklogic.client.type.ServerExpression} sequence with PlanBuilder.seq() */ public abstract ServerExpression xmlAttributeSeq(ServerExpression... attribute); /** @@ -1960,6 +1961,82 @@ public interface ModifyPlan extends PreparePlan, PlanBuilderBase.ModifyPlanBase * @return a ModifyPlan object */ public abstract ModifyPlan unnestLeftOuter(PlanExprCol inputColumn, PlanExprCol valueColumn, PlanExprCol ordinalColumn); +/** + * This method can be used to find the shortest path between two nodes in a given graph. + * @param start The column representing the input starting subject of the shortest path search. The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param end The column representing the input ending object of the shortest path search. The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @return a ModifyPlan object + * @since 7.1.0; requires MarkLogic 12 + */ + public abstract ModifyPlan shortestPath(String start, String end); +/** + * This method can be used to find the shortest path between two nodes in a given graph. + * @param start The column representing the input starting subject of the shortest path search. The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param end The column representing the input ending object of the shortest path search. The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @return a ModifyPlan object + * @since 7.1.0; requires MarkLogic 12 + */ + public abstract ModifyPlan shortestPath(PlanExprCol start, PlanExprCol end); +/** + * This method can be used to find the shortest path between two nodes in a given graph. + * @param start The column representing the input starting subject of the shortest path search. The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param end The column representing the input ending object of the shortest path search. The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param path The column is the output column representing the actual shortest path(s) taken from start to end. Values are not returned for this column if this is the empty sequence.The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @return a ModifyPlan object + * @since 7.1.0; requires MarkLogic 12 + */ + public abstract ModifyPlan shortestPath(String start, String end, String path); +/** + * This method can be used to find the shortest path between two nodes in a given graph. + * @param start The column representing the input starting subject of the shortest path search. The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param end The column representing the input ending object of the shortest path search. The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param path The column is the output column representing the actual shortest path(s) taken from start to end. Values are not returned for this column if this is the empty sequence.The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @return a ModifyPlan object + * @since 7.1.0; requires MarkLogic 12 + */ + public abstract ModifyPlan shortestPath(PlanExprCol start, PlanExprCol end, PlanExprCol path); +/** + * This method can be used to find the shortest path between two nodes in a given graph. + * @param start The column representing the input starting subject of the shortest path search. The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param end The column representing the input ending object of the shortest path search. The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param path The column is the output column representing the actual shortest path(s) taken from start to end. Values are not returned for this column if this is the empty sequence.The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param length The column is the output column representing the length of the shortest path. Value is not returned for this column if this is the empty sequence.The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @return a ModifyPlan object + * @since 7.1.0; requires MarkLogic 12 + */ + public abstract ModifyPlan shortestPath(String start, String end, String path, String length); +/** + * This method can be used to find the shortest path between two nodes in a given graph. + * @param start The column representing the input starting subject of the shortest path search. The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param end The column representing the input ending object of the shortest path search. The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param path The column is the output column representing the actual shortest path(s) taken from start to end. Values are not returned for this column if this is the empty sequence.The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param length The column is the output column representing the length of the shortest path. Value is not returned for this column if this is the empty sequence.The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @return a ModifyPlan object + * @since 7.1.0; requires MarkLogic 12 + */ + public abstract ModifyPlan shortestPath(PlanExprCol start, PlanExprCol end, PlanExprCol path, PlanExprCol length); +/** + * This method can be used to find the shortest path between two nodes in a given graph. + * @param start The column representing the input starting subject of the shortest path search. The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param end The column representing the input ending object of the shortest path search. The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param path The column is the output column representing the actual shortest path(s) taken from start to end. Values are not returned for this column if this is the empty sequence.The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param length The column is the output column representing the length of the shortest path. Value is not returned for this column if this is the empty sequence.The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param weight the weight value. See {@link PlanBuilder#col(XsStringVal)} + * @return a ModifyPlan object + * @since 7.1.0; requires MarkLogic 12 + */ + public abstract ModifyPlan shortestPath(String start, String end, String path, String length, String weight); +/** + * This method can be used to find the shortest path between two nodes in a given graph. + * @param start The column representing the input starting subject of the shortest path search. The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param end The column representing the input ending object of the shortest path search. The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param path The column is the output column representing the actual shortest path(s) taken from start to end. Values are not returned for this column if this is the empty sequence.The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param length The column is the output column representing the length of the shortest path. Value is not returned for this column if this is the empty sequence.The columns can be named with a string or a column parameter function such as op:col. See {@link PlanBuilder#col(XsStringVal)} + * @param weight the weight value. See {@link PlanBuilder#col(XsStringVal)} + * @return a ModifyPlan object + * @since 7.1.0; requires MarkLogic 12 + */ + public abstract ModifyPlan shortestPath(PlanExprCol start, PlanExprCol end, PlanExprCol path, PlanExprCol length, PlanExprCol weight); } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderImpl.java index 83eec0744..752cf4834 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderImpl.java @@ -2206,6 +2206,78 @@ public ModifyPlan select(PlanExprColSeq columns, XsStringVal qualifierName) { } + @Override + public ModifyPlan shortestPath(String start, String end) { + return shortestPath((start == null) ? (PlanExprCol) null : exprCol(start), (end == null) ? (PlanExprCol) null : exprCol(end)); + } + + + @Override + public ModifyPlan shortestPath(PlanExprCol start, PlanExprCol end) { + if (start == null) { + throw new IllegalArgumentException("start parameter for shortestPath() cannot be null"); + } + if (end == null) { + throw new IllegalArgumentException("end parameter for shortestPath() cannot be null"); + } + return new PlanBuilderSubImpl.ModifyPlanSubImpl(this, "op", "shortest-path", new Object[]{ start, end }); + } + + + @Override + public ModifyPlan shortestPath(String start, String end, String path) { + return shortestPath((start == null) ? (PlanExprCol) null : exprCol(start), (end == null) ? (PlanExprCol) null : exprCol(end), (path == null) ? (PlanExprCol) null : exprCol(path)); + } + + + @Override + public ModifyPlan shortestPath(PlanExprCol start, PlanExprCol end, PlanExprCol path) { + if (start == null) { + throw new IllegalArgumentException("start parameter for shortestPath() cannot be null"); + } + if (end == null) { + throw new IllegalArgumentException("end parameter for shortestPath() cannot be null"); + } + return new PlanBuilderSubImpl.ModifyPlanSubImpl(this, "op", "shortest-path", new Object[]{ start, end, path }); + } + + + @Override + public ModifyPlan shortestPath(String start, String end, String path, String length) { + return shortestPath((start == null) ? (PlanExprCol) null : exprCol(start), (end == null) ? (PlanExprCol) null : exprCol(end), (path == null) ? (PlanExprCol) null : exprCol(path), (length == null) ? (PlanExprCol) null : exprCol(length)); + } + + + @Override + public ModifyPlan shortestPath(PlanExprCol start, PlanExprCol end, PlanExprCol path, PlanExprCol length) { + if (start == null) { + throw new IllegalArgumentException("start parameter for shortestPath() cannot be null"); + } + if (end == null) { + throw new IllegalArgumentException("end parameter for shortestPath() cannot be null"); + } + return new PlanBuilderSubImpl.ModifyPlanSubImpl(this, "op", "shortest-path", new Object[]{ start, end, path, length }); + } + + + @Override + public ModifyPlan shortestPath(String start, String end, String path, String length, String weight) { + return shortestPath((start == null) ? (PlanExprCol) null : exprCol(start), (end == null) ? (PlanExprCol) null : exprCol(end), (path == null) ? (PlanExprCol) null : exprCol(path), (length == null) ? (PlanExprCol) null : exprCol(length), (weight == null) ? (PlanExprCol) null : exprCol(weight)); + } + + + @Override + public ModifyPlan shortestPath(PlanExprCol start, PlanExprCol end, PlanExprCol path, PlanExprCol length, PlanExprCol weight) { + if (start == null) { + throw new IllegalArgumentException("start parameter for shortestPath() cannot be null"); + } + if (end == null) { + throw new IllegalArgumentException("end parameter for shortestPath() cannot be null"); + } + return new PlanBuilderSubImpl.ModifyPlanSubImpl(this, "op", "shortest-path", new Object[]{ start, end, path, length, weight }); + } + + @Override public ModifyPlan union(ModifyPlan right) { if (right == null) { diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanGeneratedTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanGeneratedTest.java index 718e44be7..cc67dadf4 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanGeneratedTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanGeneratedTest.java @@ -7,6 +7,7 @@ import com.marklogic.client.io.Format; import com.marklogic.client.test.junit5.RequiresML11; import com.marklogic.client.test.junit5.RequiresML11OrLower; +import com.marklogic.client.test.junit5.RequiresML12; import com.marklogic.client.type.ServerExpression; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -94,7 +95,7 @@ public void testFnAbs1Exec() { @Test public void testFnAdjustDateTimeToTimezone1Exec() { - executeTester("testFnAdjustDateTimeToTimezone1", p.fn.adjustDateTimeToTimezone(p.col("1")), true, "xs:dateTime", null, null, "2016-01-02T10:09:08Z", new ServerExpression[]{ p.xs.dateTime("2016-01-02T10:09:08Z") }); + executeTester("testFnAdjustDateTimeToTimezone1", p.fn.adjustDateTimeToTimezone(p.col("1")), true, "xs:dateTime", null, null, "2016-01-02T03:09:08-07:00", new ServerExpression[]{ p.xs.dateTime("2016-01-02T10:09:08Z") }); } @Test @@ -104,7 +105,7 @@ public void testFnAdjustDateTimeToTimezone2Exec() { @Test public void testFnAdjustDateToTimezone1Exec() { - executeTester("testFnAdjustDateToTimezone1", p.fn.adjustDateToTimezone(p.col("1")), true, "xs:date", null, null, "2016-01-02Z", new ServerExpression[]{ p.xs.date("2016-01-02") }); + executeTester("testFnAdjustDateToTimezone1", p.fn.adjustDateToTimezone(p.col("1")), true, "xs:date", null, null, "2016-01-02-07:00", new ServerExpression[]{ p.xs.date("2016-01-02") }); } @Test @@ -114,7 +115,7 @@ public void testFnAdjustDateToTimezone2Exec() { @Test public void testFnAdjustTimeToTimezone1Exec() { - executeTester("testFnAdjustTimeToTimezone1", p.fn.adjustTimeToTimezone(p.col("1")), true, "xs:time", null, null, "10:09:08Z", new ServerExpression[]{ p.xs.time("10:09:08Z") }); + executeTester("testFnAdjustTimeToTimezone1", p.fn.adjustTimeToTimezone(p.col("1")), true, "xs:time", null, null, "03:09:08-07:00", new ServerExpression[]{ p.xs.time("10:09:08Z") }); } @Test @@ -199,17 +200,17 @@ public void testFnCount2Exec() { @Test public void testFnCurrentDate0Exec() { - executeTester("testFnCurrentDate0", p.fn.currentDate(), true, "xs:date", null, null, "2023-10-24Z", new ServerExpression[]{ }); + executeTester("testFnCurrentDate0", p.fn.currentDate(), true, "xs:date", null, null, "2024-09-11-07:00", new ServerExpression[]{ }); } @Test public void testFnCurrentDateTime0Exec() { - executeTester("testFnCurrentDateTime0", p.fn.currentDateTime(), true, "xs:dateTime", null, null, "2023-10-24T15:21:06.255777Z", new ServerExpression[]{ }); + executeTester("testFnCurrentDateTime0", p.fn.currentDateTime(), true, "xs:dateTime", null, null, "2024-09-11T02:45:08.551523-07:00", new ServerExpression[]{ }); } @Test public void testFnCurrentTime0Exec() { - executeTester("testFnCurrentTime0", p.fn.currentTime(), true, "xs:time", null, null, "15:21:06Z", new ServerExpression[]{ }); + executeTester("testFnCurrentTime0", p.fn.currentTime(), true, "xs:time", null, null, "02:45:08-07:00", new ServerExpression[]{ }); } @Test @@ -319,10 +320,16 @@ public void testFnFormatTime2Exec() { @ExtendWith(RequiresML11OrLower.class) @Test - public void testFnHead1Exec() { + public void testFnHead1ExecForML11OrLower() { executeTester("testFnHead1", p.fn.head(p.col("1")), false, null, null, null, "a", new ServerExpression[]{ p.xs.stringSeq(p.xs.string("a"), p.xs.string("b"), p.xs.string("c")) }); } + @ExtendWith(RequiresML12.class) + @Test + public void testFnHead1Exec() { + executeTester("testFnHead1", p.fn.head(p.col("1")), false, null, null, Format.JSON, null, new ServerExpression[]{ p.xs.stringSeq(p.xs.string("a"), p.xs.string("b"), p.xs.string("c")) }); + } + @Test public void testFnHoursFromDateTime1Exec() { executeTester("testFnHoursFromDateTime1", p.fn.hoursFromDateTime(p.col("1")), false, null, null, null, "10", new ServerExpression[]{ p.xs.dateTime("2016-01-02T10:09:08Z") }); @@ -340,7 +347,7 @@ public void testFnHoursFromTime1Exec() { @Test public void testFnImplicitTimezone0Exec() { - executeTester("testFnImplicitTimezone0", p.fn.implicitTimezone(), true, "xs:dayTimeDuration", null, null, "PT0S", new ServerExpression[]{ }); + executeTester("testFnImplicitTimezone0", p.fn.implicitTimezone(), true, "xs:dayTimeDuration", null, null, "-PT7H", new ServerExpression[]{ }); } @Test @@ -365,7 +372,7 @@ public void testFnIriToUri1Exec() { @Test public void testFnLocalNameFromQName1Exec() { - executeTester("testFnLocalNameFromQName1", p.fn.localNameFromQName(p.col("1")), false, "xs:NCName", null, null, "abc", new ServerExpression[]{ p.xs.QName("abc") }); + executeTester("testFnLocalNameFromQName1", p.fn.localNameFromQName(p.col("1")), false, null, null, null, "abc", new ServerExpression[]{ p.xs.QName("abc") }); } @Test @@ -465,10 +472,16 @@ public void testFnNumber1Exec() { @ExtendWith(RequiresML11OrLower.class) @Test - public void testFnPrefixFromQName1Exec() { + public void testFnPrefixFromQName1ExecForML11OrLower() { executeTester("testFnPrefixFromQName1", p.fn.prefixFromQName(p.col("1")), false, null, null, Format.JSON, null, new ServerExpression[]{ p.xs.QName("abc") }); } + @ExtendWith(RequiresML12.class) + @Test + public void testFnPrefixFromQName1Exec() { + executeTester("testFnPrefixFromQName1", p.fn.prefixFromQName(p.col("1")), false, null, null, null, "", new ServerExpression[]{ p.xs.QName("abc") }); + } + @Test public void testFnQName2Exec() { executeTester("testFnQName2", p.fn.QName(p.col("1"), p.col("2")), false, "xs:QName", null, null, "c", new ServerExpression[]{ p.xs.string("http://a/b"), p.xs.string("c") }); @@ -732,7 +745,7 @@ public void testGeoGeohashDecodePoint1Exec() { @Test public void testGeoGeohashNeighbors1Exec() { - executeTester("testGeoGeohashNeighbors1", p.geo.geohashNeighbors(p.col("1")), false, null, null, Format.JSON, "{\"NE\":\"s01mtz\", \"S\":\"s01mtt\", \"E\":\"s01mty\", \"W\":\"s01mtq\", \"N\":\"s01mtx\", \"SW\":\"s01mtm\", \"SE\":\"s01mtv\", \"NW\":\"s01mtr\"}", new ServerExpression[]{ p.xs.string("s01mtw") }); + executeTester("testGeoGeohashNeighbors1", p.geo.geohashNeighbors(p.col("1")), false, null, null, Format.JSON, "{\"NE\":\"s01mtz\", \"S\":\"s01mtt\", \"E\":\"s01mty\", \"W\":\"s01mtq\", \"SW\":\"s01mtm\", \"N\":\"s01mtx\", \"SE\":\"s01mtv\", \"NW\":\"s01mtr\"}", new ServerExpression[]{ p.xs.string("s01mtw") }); } @Test @@ -999,7 +1012,7 @@ public void testRdfLangStringLanguage1Exec() { @Test public void testSemBnode0Exec() { - executeTester("testSemBnode0", p.sem.bnode(), true, "sem:blank", null, null, "_:bnode801272998575329659", new ServerExpression[]{ }); + executeTester("testSemBnode0", p.sem.bnode(), true, "sem:blank", null, null, "_:bnode1706937960796523528", new ServerExpression[]{ }); } @Test @@ -1034,7 +1047,7 @@ public void testSemInvalid2Exec() { @Test public void testSemIri1Exec() { - executeTester("testSemIri1", p.sem.iri(p.col("1")), false, null, null, null, "http://a/b", new ServerExpression[]{ p.xs.string("http://a/b") }); + executeTester("testSemIri1", p.sem.iri(p.col("1")), false, "sem:iri", null, null, "http://a/b", new ServerExpression[]{ p.xs.string("http://a/b") }); } @Test @@ -1079,7 +1092,7 @@ public void testSemQNameToIri1Exec() { @Test public void testSemRandom0Exec() { - executeTester("testSemRandom0", p.sem.random(), true, null, null, null, "0.996817928554708", new ServerExpression[]{ }); + executeTester("testSemRandom0", p.sem.random(), true, null, null, null, "0.873418109880463", new ServerExpression[]{ }); } @Test @@ -1104,12 +1117,12 @@ public void testSemUnknown2Exec() { @Test public void testSemUuid0Exec() { - executeTester("testSemUuid0", p.sem.uuid(), true, "sem:iri", null, null, "urn:uuid:c04549cf-e759-4190-bb96-8a0efaa4fcff", new ServerExpression[]{ }); + executeTester("testSemUuid0", p.sem.uuid(), true, "sem:iri", null, null, "urn:uuid:67b7b233-fd07-4ce4-b4a3-370357d13a03", new ServerExpression[]{ }); } @Test public void testSemUuidString0Exec() { - executeTester("testSemUuidString0", p.sem.uuidString(), true, null, null, null, "32351923-7f22-4a3f-9e92-884151190ee8", new ServerExpression[]{ }); + executeTester("testSemUuidString0", p.sem.uuidString(), true, null, null, null, "ed495c7f-e087-4393-9385-ce6ab410d31d", new ServerExpression[]{ }); } @Test @@ -1244,7 +1257,7 @@ public void testSqlQuarter1Exec() { @Test public void testSqlRand1Exec() { - executeTester("testSqlRand1", p.sql.rand(p.col("1")), true, "xs:unsignedLong", null, null, "1308547733197903283", new ServerExpression[]{ p.xs.unsignedLong(1) }); + executeTester("testSqlRand1", p.sql.rand(p.col("1")), true, "xs:unsignedLong", null, null, "5115111156722013996", new ServerExpression[]{ p.xs.unsignedLong(1) }); } @Test @@ -1349,7 +1362,7 @@ public void testXdmpCrypt2Exec() { @Test public void testXdmpCrypt21Exec() { - executeTester("testXdmpCrypt21", p.xdmp.crypt2(p.col("1")), true, null, null, null, "$256$Byph9.Mc3xqzhgeH.IAJh/$256$nAr3j1jWo/Wf2yN7", new ServerExpression[]{ p.xs.string("abc") }); + executeTester("testXdmpCrypt21", p.xdmp.crypt2(p.col("1")), true, null, null, null, "$256$jBIG2/HIzq9t7u4CCVe4E0$256$L4sEAEm0ISFIAiPI", new ServerExpression[]{ p.xs.string("abc") }); } @Test @@ -1549,12 +1562,12 @@ public void testXdmpOr642Exec() { @Test public void testXdmpParseDateTime2Exec() { - executeTester("testXdmpParseDateTime2", p.xdmp.parseDateTime(p.col("1"), p.col("2")), true, "xs:dateTime", null, null, "2016-01-07T01:13:50.873Z", new ServerExpression[]{ p.xs.string("[Y0001]-[M01]-[D01]T[h01]:[m01]:[s01].[f1][Z]"), p.xs.string("2016-01-06T17:13:50.873594-08:00") }); + executeTester("testXdmpParseDateTime2", p.xdmp.parseDateTime(p.col("1"), p.col("2")), true, "xs:dateTime", null, null, "2016-01-06T18:13:50.873-07:00", new ServerExpression[]{ p.xs.string("[Y0001]-[M01]-[D01]T[h01]:[m01]:[s01].[f1][Z]"), p.xs.string("2016-01-06T17:13:50.873594-08:00") }); } @Test public void testXdmpParseYymmdd2Exec() { - executeTester("testXdmpParseYymmdd2", p.xdmp.parseYymmdd(p.col("1"), p.col("2")), true, "xs:dateTime", null, null, "2016-01-07T01:13:50.873Z", new ServerExpression[]{ p.xs.string("yyyy-MM-ddThh:mm:ss.Sz"), p.xs.string("2016-01-06T17:13:50.873594-8.00") }); + executeTester("testXdmpParseYymmdd2", p.xdmp.parseYymmdd(p.col("1"), p.col("2")), true, "xs:dateTime", null, null, "2016-01-06T18:13:50.873-07:00", new ServerExpression[]{ p.xs.string("yyyy-MM-ddThh:mm:ss.Sz"), p.xs.string("2016-01-06T17:13:50.873594-8.00") }); } @Test @@ -1579,12 +1592,12 @@ public void testXdmpQuarterFromDate1Exec() { @Test public void testXdmpRandom0Exec() { - executeTester("testXdmpRandom0", p.xdmp.random(), true, "xs:unsignedLong", null, null, "7692759001122562087", new ServerExpression[]{ }); + executeTester("testXdmpRandom0", p.xdmp.random(), true, "xs:unsignedLong", null, null, "10921725801440392827", new ServerExpression[]{ }); } @Test public void testXdmpRandom1Exec() { - executeTester("testXdmpRandom1", p.xdmp.random(p.col("1")), true, null, null, null, "0", new ServerExpression[]{ p.xs.unsignedLong(1) }); + executeTester("testXdmpRandom1", p.xdmp.random(p.col("1")), true, null, null, null, "1", new ServerExpression[]{ p.xs.unsignedLong(1) }); } @Test @@ -1644,12 +1657,12 @@ public void testXdmpStep642Exec() { @Test public void testXdmpStrftime2Exec() { - executeTester("testXdmpStrftime2", p.xdmp.strftime(p.col("1"), p.col("2")), true, null, null, null, "Thu, 07 Jan 2016 01:13:50", new ServerExpression[]{ p.xs.string("%a, %d %b %Y %H:%M:%S"), p.xs.dateTime("2016-01-06T17:13:50.873594-08:00") }); + executeTester("testXdmpStrftime2", p.xdmp.strftime(p.col("1"), p.col("2")), true, null, null, null, "Wed, 06 Jan 2016 17:13:50", new ServerExpression[]{ p.xs.string("%a, %d %b %Y %H:%M:%S"), p.xs.dateTime("2016-01-06T17:13:50.873594-08:00") }); } @Test public void testXdmpTimestampToWallclock1Exec() { - executeTester("testXdmpTimestampToWallclock1", p.xdmp.timestampToWallclock(p.col("1")), true, "xs:dateTime", null, null, "1970-01-01T00:00:00.0000001", new ServerExpression[]{ p.xs.unsignedLong(1) }); + executeTester("testXdmpTimestampToWallclock1", p.xdmp.timestampToWallclock(p.col("1")), true, "xs:dateTime", null, null, "1969-12-31T16:00:00.0000001", new ServerExpression[]{ p.xs.unsignedLong(1) }); } @Test From 87fb84cff86d73474fafc2b39316989549a57b94 Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Fri, 11 Oct 2024 11:28:25 -0400 Subject: [PATCH 08/39] Converting Jenkinsfile to use a docker-compose file Switch to images and artifactory. Removing mapped volumes. Using .env for mapping a local volume during development. --- .gitignore | 1 - CONTRIBUTING.md | 12 ++++- Jenkinsfile | 93 +++++++++++++++++------------------- test-app/.env | 4 ++ test-app/docker-compose.yaml | 21 ++++++++ 5 files changed, 79 insertions(+), 52 deletions(-) create mode 100644 test-app/.env create mode 100644 test-app/docker-compose.yaml diff --git a/.gitignore b/.gitignore index 1554b1c8b..1bf7c14e2 100644 --- a/.gitignore +++ b/.gitignore @@ -37,5 +37,4 @@ ml-development-tools/src/test/ml-modules ml-development-tools/src/test/java/com/marklogic/client/test/dbfunction/generated .vscode -docker-compose.yaml docker/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 57ade6530..7ec0a99d8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,7 +28,17 @@ and add the following (you can override additional properties as necessary): Note that additional properties are defined via `./tests-app/gradle.properties`, though it is not expected that these properties will need to be changed. -The application is then deployed via the following command: +The tests require a MarkLogic instance with access to several ports (8000,8001,8002,8012,8014,8020). That instance may +be a local instance or it may be running in a Docker container. If you would like to create a Docker container with the +instance, you may create the container with the following commands (starting in the project root directory): + +``` +cd test-app +docker-compose up -d --build +cd .. +``` + +Once you have a MarkLogic instance ready, the application is then deployed via the following command: ./gradlew mlDeploy -i diff --git a/Jenkinsfile b/Jenkinsfile index a15abe0cf..9a39414fb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,31 +1,38 @@ @Library('shared-libraries') _ def getJava(){ - if(env.JAVA_VERSION=="JAVA17"){ - return "/home/builder/java/jdk-17.0.2" - }else if(env.JAVA_VERSION=="JAVA11"){ - return "/home/builder/java/jdk-11.0.2" - }else if(env.JAVA_VERSION=="JAVA21"){ + if(env.JAVA_VERSION=="JAVA17"){ + return "/home/builder/java/jdk-17.0.2" + }else if(env.JAVA_VERSION=="JAVA11"){ + return "/home/builder/java/jdk-11.0.2" + }else if(env.JAVA_VERSION=="JAVA21"){ return "/home/builder/java/jdk-21.0.1" }else{ - return "/home/builder/java/openjdk-1.8.0-262" - } + return "/home/builder/java/openjdk-1.8.0-262" + } } -def runAllTests(String type, String version, Boolean useReverseProxy){ - copyRPM type, version - sh 'sudo /usr/local/sbin/mladmin removeforest /space/Forests' - setUpML '$WORKSPACE/xdmp/src/Mark*.rpm' - copyConvertersRPM type,version - setUpMLConverters '$WORKSPACE/xdmp/src/Mark*Converters*.rpm' +def setupDockerMarkLogic(String image){ + sh label:'mlsetup', script: '''#!/bin/bash + echo "Removing any running MarkLogic server and clean up MarkLogic data directory" + sudo /usr/local/sbin/mladmin remove + sudo /usr/local/sbin/mladmin cleandata + cd java-client-api/test-app + docker compose down -v || true + echo "Using image: "'''+image+''' + MARKLOGIC_IMAGE='''+image+''' MARKLOGIC_LOGS_VOLUME=marklogicLogs docker compose up -d --build + echo "mlPassword=admin" > gradle-local.properties + echo "Waiting for MarkLogic server to initialize." + sleep 30s + cd .. + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + ./gradlew -i mlDeploy mlReloadSchemas + ''' +} - sh label:'deploy test app', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api - ./gradlew -i mlDeploy mlReloadSchemas -PmlForestDataDirectory=/space - ''' +def runAllTests(Boolean useReverseProxy, String image){ + setupDockerMarkLogic(image) if (useReverseProxy) { sh label:'run marklogic-client-api tests with reverse proxy', script: '''#!/bin/bash @@ -155,18 +162,9 @@ pipeline{ } } steps { - copyRPM 'Latest','11' - sh 'sudo /usr/local/sbin/mladmin removeforest /space/Forests' - setUpML '$WORKSPACE/xdmp/src/Mark*.rpm' - copyConvertersRPM 'Latest','11' - setUpMLConverters '$WORKSPACE/xdmp/src/Mark*Converters*.rpm' - sh label:'deploy test app', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api - ./gradlew -i mlDeploy mlReloadSchemas -PmlForestDataDirectory=/space - ''' + setupDockerMarkLogic("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11") + + sh label:'run marklogic-client-api tests', script: '''#!/bin/bash export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR @@ -177,6 +175,14 @@ pipeline{ ''' junit '**/build/**/TEST*.xml' } + post{ + always{ + sh label:'dockerCleanup', script: '''#!/bin/bash + cd java-client-api/test-app + docker compose down -v || true + ''' + } + } } stage('publish'){ when { @@ -205,7 +211,7 @@ pipeline{ } } steps { - runAllTests('Release', '11.2.0', false) + runAllTests(false, "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:11.2.0-ubi") junit '**/build/**/TEST*.xml' } } @@ -218,7 +224,7 @@ pipeline{ } } steps { - runAllTests('Latest', '11', false) + runAllTests(false, "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11") junit '**/build/**/TEST*.xml' } } @@ -231,7 +237,7 @@ pipeline{ } } steps { - runAllTests('Latest', '11', true) + runAllTests(true, "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11") junit '**/build/**/TEST*.xml' } } @@ -244,7 +250,7 @@ pipeline{ } } steps { - runAllTests('Latest', '12', false) + runAllTests(false, "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12") junit '**/build/**/TEST*.xml' } } @@ -257,20 +263,7 @@ pipeline{ } } steps { - runAllTests('Latest', '10.0', false) - junit '**/build/**/TEST*.xml' - } - } - - stage('regressions-10.0-10.2') { - when { - allOf { - branch 'develop' - expression {return params.regressions} - } - } - steps { - runAllTests('Release', '10.0-10.2', false) + runAllTests(false, "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-10") junit '**/build/**/TEST*.xml' } } diff --git a/test-app/.env b/test-app/.env new file mode 100644 index 000000000..64757b1c2 --- /dev/null +++ b/test-app/.env @@ -0,0 +1,4 @@ +# Defines environment variables for docker-compose. +# Can be overridden via e.g. `MARKLOGIC_TAG=latest-10.0 docker-compose up -d --build`. +MARKLOGIC_IMAGE=progressofficial/marklogic-db:latest +MARKLOGIC_LOGS_VOLUME=./docker/marklogic/logs diff --git a/test-app/docker-compose.yaml b/test-app/docker-compose.yaml new file mode 100644 index 000000000..f0b9a5cd1 --- /dev/null +++ b/test-app/docker-compose.yaml @@ -0,0 +1,21 @@ +name: marklogic-javaclient-test-app + +services: + + marklogic: + image: "${MARKLOGIC_IMAGE}" + platform: linux/amd64 + environment: + - MARKLOGIC_INIT=true + - MARKLOGIC_ADMIN_USERNAME=admin + - MARKLOGIC_ADMIN_PASSWORD=admin + volumes: + - ${MARKLOGIC_LOGS_VOLUME}:/var/opt/MarkLogic/Logs + ports: + - "8000-8002:8000-8002" + - "8012:8012" + - "8014:8014" + - "8020:8020" + +volumes: + marklogicLogs: From 3d88863c47af97ea2e3420e3d5d747dd38530304 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 16 Oct 2024 08:42:37 -0400 Subject: [PATCH 09/39] Added converters to docker-compose file --- test-app/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/test-app/docker-compose.yaml b/test-app/docker-compose.yaml index f0b9a5cd1..87530496b 100644 --- a/test-app/docker-compose.yaml +++ b/test-app/docker-compose.yaml @@ -6,6 +6,7 @@ services: image: "${MARKLOGIC_IMAGE}" platform: linux/amd64 environment: + - INSTALL_CONVERTERS=true - MARKLOGIC_INIT=true - MARKLOGIC_ADMIN_USERNAME=admin - MARKLOGIC_ADMIN_PASSWORD=admin From 6a02454f9b2003adb5d6a0b3b9bbe16062fa1d0e Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 16 Oct 2024 10:46:35 -0400 Subject: [PATCH 10/39] Removing jacksonVersion to try to make Palamida happy --- examples/build.gradle | 2 +- gradle.properties | 2 -- marklogic-client-api-functionaltests/build.gradle | 4 ++-- marklogic-client-api/build.gradle | 10 +++++----- ml-development-tools/build.gradle | 2 +- test-app/build.gradle | 2 +- 6 files changed, 10 insertions(+), 12 deletions(-) diff --git a/examples/build.gradle b/examples/build.gradle index d9793d5fb..b673d12c0 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -16,7 +16,7 @@ dependencies { api 'com.squareup.okhttp3:okhttp:4.12.0' api 'io.github.rburgst:okhttp-digest:2.7' api 'org.slf4j:slf4j-api:2.0.16' - api "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" + api "com.fasterxml.jackson.core:jackson-databind:2.17.2" // hsqldb < 2.7 has a High CVE - https://nvd.nist.gov/vuln/detail/CVE-2022-41853 . // And hsqldb 2.6+ requires Java 11+. So this is ignored, along with the associated test, diff --git a/gradle.properties b/gradle.properties index 3abd980d4..ed8022bc0 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,8 +3,6 @@ version=7.0-SNAPSHOT describedName=MarkLogic Java Client API publishUrl=file:../marklogic-java/releases -jacksonVersion=2.17.2 - # Defined at this level so that they can be set as system properties and used by the marklogic-client-api and test-app # project mlHost=localhost diff --git a/marklogic-client-api-functionaltests/build.gradle b/marklogic-client-api-functionaltests/build.gradle index 0a581086c..33579cc93 100755 --- a/marklogic-client-api-functionaltests/build.gradle +++ b/marklogic-client-api-functionaltests/build.gradle @@ -24,8 +24,8 @@ dependencies { implementation 'org.slf4j:slf4j-api:2.0.16' implementation 'commons-io:commons-io:2.17.0' implementation 'com.squareup.okhttp3:okhttp:4.12.0' - implementation "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}" - implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" + implementation "com.fasterxml.jackson.core:jackson-core:2.17.2" + implementation "com.fasterxml.jackson.core:jackson-databind:2.17.2" implementation "org.jdom:jdom2:2.0.6.1" implementation ("com.marklogic:ml-app-deployer:5.0.0") { exclude module: "marklogic-client-api" diff --git a/marklogic-client-api/build.gradle b/marklogic-client-api/build.gradle index 3561a86dd..9a3b6e2fb 100644 --- a/marklogic-client-api/build.gradle +++ b/marklogic-client-api/build.gradle @@ -30,10 +30,10 @@ dependencies { implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' implementation 'org.slf4j:slf4j-api:2.0.16' - implementation "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}" - implementation "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}" - implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" - implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-csv:${jacksonVersion}" + implementation "com.fasterxml.jackson.core:jackson-core:2.17.2" + implementation "com.fasterxml.jackson.core:jackson-annotations:2.17.2" + implementation "com.fasterxml.jackson.core:jackson-databind:2.17.2" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.17.2" // Only used by extras (which some examples then depend on) // Forcing codec version to avoid vulnerability with older version in httpclient @@ -59,7 +59,7 @@ dependencies { testImplementation "org.mockito:mockito-inline:4.11.0" testImplementation "com.squareup.okhttp3:mockwebserver:4.12.0" - testImplementation "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${jacksonVersion}" + testImplementation "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.17.2" testImplementation 'ch.qos.logback:logback-classic:1.3.14' // schema validation issue with testImplementation 'xerces:xercesImpl:2.12.0' testImplementation 'org.opengis.cite.xerces:xercesImpl-xsd11:2.12-beta-r1667115' diff --git a/ml-development-tools/build.gradle b/ml-development-tools/build.gradle index f0610d7f1..a475df10f 100644 --- a/ml-development-tools/build.gradle +++ b/ml-development-tools/build.gradle @@ -12,7 +12,7 @@ dependencies { compileOnly gradleApi() implementation project(':marklogic-client-api') implementation 'org.jetbrains.kotlin:kotlin-stdlib:1.8.22' - implementation "com.fasterxml.jackson.module:jackson-module-kotlin:${jacksonVersion}" + implementation "com.fasterxml.jackson.module:jackson-module-kotlin:2.17.2" implementation 'com.networknt:json-schema-validator:1.0.88' // Not yet migrating this project to JUnit 5. Will reconsider it once we have a reason to enhance diff --git a/test-app/build.gradle b/test-app/build.gradle index c26ccf09a..e51c9e9dc 100644 --- a/test-app/build.gradle +++ b/test-app/build.gradle @@ -10,7 +10,7 @@ dependencies { implementation "com.marklogic:ml-javaclient-util:4.8.0" implementation 'org.slf4j:slf4j-api:2.0.16' implementation 'ch.qos.logback:logback-classic:1.3.14' - implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" + implementation "com.fasterxml.jackson.core:jackson-databind:2.17.2" implementation 'com.squareup.okhttp3:okhttp:4.12.0' } From f0ed1c959388c99a778f93e9561472e9c3134a88 Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Thu, 17 Oct 2024 09:42:03 -0400 Subject: [PATCH 11/39] Add 8011 to open ports in the docker container. --- test-app/docker-compose.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test-app/docker-compose.yaml b/test-app/docker-compose.yaml index 87530496b..e7fa74694 100644 --- a/test-app/docker-compose.yaml +++ b/test-app/docker-compose.yaml @@ -14,8 +14,7 @@ services: - ${MARKLOGIC_LOGS_VOLUME}:/var/opt/MarkLogic/Logs ports: - "8000-8002:8000-8002" - - "8012:8012" - - "8014:8014" + - "8011-8014:8011-8014" - "8020:8020" volumes: From e6de797069dffb3403cebd9590d5b1502a7349dc Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Fri, 18 Oct 2024 11:22:11 -0400 Subject: [PATCH 12/39] Create gradle-local.properties file in the root directory also --- Jenkinsfile | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9a39414fb..cf44ac196 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -19,14 +19,17 @@ def setupDockerMarkLogic(String image){ sudo /usr/local/sbin/mladmin cleandata cd java-client-api/test-app docker compose down -v || true + docker volume prune -f echo "Using image: "'''+image+''' MARKLOGIC_IMAGE='''+image+''' MARKLOGIC_LOGS_VOLUME=marklogicLogs docker compose up -d --build echo "mlPassword=admin" > gradle-local.properties echo "Waiting for MarkLogic server to initialize." - sleep 30s + sleep 60s cd .. + echo "mlPassword=admin" > gradle-local.properties export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + ./gradlew mlTestConnections ./gradlew -i mlDeploy mlReloadSchemas ''' } @@ -180,6 +183,7 @@ pipeline{ sh label:'dockerCleanup', script: '''#!/bin/bash cd java-client-api/test-app docker compose down -v || true + docker volume prune -f ''' } } @@ -201,6 +205,15 @@ pipeline{ ./gradlew publish ''' } + post{ + always{ + sh label:'dockerCleanup', script: '''#!/bin/bash + cd java-client-api/test-app + docker compose down -v || true + docker volume prune -f + ''' + } + } } stage('regressions-11.2.0') { From 73605df1b0c6c518fa1df0e7bd683a9bbb085aee Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Fri, 18 Oct 2024 15:14:09 -0400 Subject: [PATCH 13/39] Another port for the jenkins tests --- test-app/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/test-app/docker-compose.yaml b/test-app/docker-compose.yaml index e7fa74694..f663257b4 100644 --- a/test-app/docker-compose.yaml +++ b/test-app/docker-compose.yaml @@ -16,6 +16,7 @@ services: - "8000-8002:8000-8002" - "8011-8014:8011-8014" - "8020:8020" + - "8093:8093" volumes: marklogicLogs: From 4006929ad4d3271a49c2a937ed93a8c10d749aca Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Sun, 20 Oct 2024 19:37:58 -0400 Subject: [PATCH 14/39] More port additions to docker compose This time for the reverse proxy tests. --- test-app/docker-compose.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test-app/docker-compose.yaml b/test-app/docker-compose.yaml index f663257b4..c766aafec 100644 --- a/test-app/docker-compose.yaml +++ b/test-app/docker-compose.yaml @@ -14,8 +14,10 @@ services: - ${MARKLOGIC_LOGS_VOLUME}:/var/opt/MarkLogic/Logs ports: - "8000-8002:8000-8002" - - "8011-8014:8011-8014" + - "8010-8014:8010-8014" - "8020:8020" + - "8022:8022" + - "8054-8059:8054-8059" - "8093:8093" volumes: From 327b3cadb60a35395a913be7a2b0fac1ab2b22bf Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Mon, 21 Oct 2024 14:08:13 -0400 Subject: [PATCH 15/39] Now I suspect port 8020 should not be mapped in Docker. --- test-app/docker-compose.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/test-app/docker-compose.yaml b/test-app/docker-compose.yaml index c766aafec..a77449057 100644 --- a/test-app/docker-compose.yaml +++ b/test-app/docker-compose.yaml @@ -15,7 +15,6 @@ services: ports: - "8000-8002:8000-8002" - "8010-8014:8010-8014" - - "8020:8020" - "8022:8022" - "8054-8059:8054-8059" - "8093:8093" From 216bdfbd819d28ac590635f34ce085e2d181a229 Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Tue, 22 Oct 2024 10:24:37 -0400 Subject: [PATCH 16/39] Remove a set of tests that not required. Also remove a reference to a local forest directory now that we're using Docker volumes. --- Jenkinsfile | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index cf44ac196..d6cb0ed08 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,14 +38,7 @@ def runAllTests(Boolean useReverseProxy, String image){ setupDockerMarkLogic(image) if (useReverseProxy) { - sh label:'run marklogic-client-api tests with reverse proxy', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api - mkdir -p marklogic-client-api/build/test-results/test - ./gradlew -PtestUseReverseProxyServer=true test-app:runReverseProxyServer marklogic-client-api:test || true - ''' + // Skip testing the marklogic-client-api tests with reverse proxy } else { sh label:'run marklogic-client-api tests', script: '''#!/bin/bash export JAVA_HOME=$JAVA_HOME_DIR @@ -72,7 +65,6 @@ def runAllTests(Boolean useReverseProxy, String image){ export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH cd java-client-api - ./gradlew mlDeploy -PmlForestDataDirectory=/space ./gradlew -PtestUseReverseProxyServer=true test-app:runReverseProxyServer marklogic-client-api-functionaltests:runFragileTests || true ''' } else { From 0ae4bffde1a5da8dd464dec160ccd5fdcf3911e3 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 23 Oct 2024 11:35:27 -0400 Subject: [PATCH 17/39] Tearing down Docker after each test stage I removed the teardown from the "publish" stage, as we don't stand up Docker for that stage. --- Jenkinsfile | 68 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 45 insertions(+), 23 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d6cb0ed08..f41e03e73 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -132,23 +132,35 @@ def runAllTests(Boolean useReverseProxy, String image){ ''' } +def tearDownDocker() { + sh label:'tearDownDocker', script: '''#!/bin/bash + cd java-client-api/test-app + docker compose down -v || true + docker volume prune -f + ''' +} + pipeline{ agent {label 'javaClientLinuxPool'} + options { checkoutToSubdirectory 'java-client-api' buildDiscarder logRotator(artifactDaysToKeepStr: '7', artifactNumToKeepStr: '', daysToKeepStr: '7', numToKeepStr: '10') } - parameters{ + + parameters { booleanParam(name: 'regressions', defaultValue: false, description: 'indicator if build is for regressions') string(name: 'Email', defaultValue: '' ,description: 'Who should I say send the email to?') string(name: 'JAVA_VERSION', defaultValue: 'JAVA8' ,description: 'Who should I say send the email to?') } - environment{ + + environment { JAVA_HOME_DIR= getJava() GRADLE_DIR =".gradle" DMC_USER = credentials('MLBUILD_USER') DMC_PASSWORD = credentials('MLBUILD_PASSWORD') } + stages { stage('pull-request-tests') { when { @@ -158,8 +170,6 @@ pipeline{ } steps { setupDockerMarkLogic("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11") - - sh label:'run marklogic-client-api tests', script: '''#!/bin/bash export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR @@ -170,15 +180,11 @@ pipeline{ ''' junit '**/build/**/TEST*.xml' } - post{ - always{ - sh label:'dockerCleanup', script: '''#!/bin/bash - cd java-client-api/test-app - docker compose down -v || true - docker volume prune -f - ''' - } - } + post { + always { + tearDownDocker() + } + } } stage('publish'){ when { @@ -197,15 +203,6 @@ pipeline{ ./gradlew publish ''' } - post{ - always{ - sh label:'dockerCleanup', script: '''#!/bin/bash - cd java-client-api/test-app - docker compose down -v || true - docker volume prune -f - ''' - } - } } stage('regressions-11.2.0') { @@ -216,9 +213,14 @@ pipeline{ } } steps { - runAllTests(false, "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:11.2.0-ubi") + runAllTests(false, "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:11.2.0-ubi") junit '**/build/**/TEST*.xml' } + post { + always { + tearDownDocker() + } + } } stage('regressions-11') { @@ -232,6 +234,11 @@ pipeline{ runAllTests(false, "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11") junit '**/build/**/TEST*.xml' } + post { + always { + tearDownDocker() + } + } } stage('regressions-11-reverseProxy') { @@ -245,6 +252,11 @@ pipeline{ runAllTests(true, "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11") junit '**/build/**/TEST*.xml' } + post { + always { + tearDownDocker() + } + } } stage('regressions-12') { @@ -258,6 +270,11 @@ pipeline{ runAllTests(false, "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12") junit '**/build/**/TEST*.xml' } + post { + always { + tearDownDocker() + } + } } stage('regressions-10.0') { @@ -271,6 +288,11 @@ pipeline{ runAllTests(false, "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-10") junit '**/build/**/TEST*.xml' } + post { + always { + tearDownDocker() + } + } } } From 8be0eab773ae5ebcdabeefe0680b88ece00d2d66 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 23 Oct 2024 12:30:28 -0400 Subject: [PATCH 18/39] Regression test fixes Creating a separate JIRA bug for the SSL tests. --- .../com/marklogic/client/fastfunctest/TestEvalXquery.java | 5 ++--- .../com/marklogic/client/functionaltest/TestBiTemporal.java | 4 ++++ .../java/com/marklogic/client/test/BinaryDocumentTest.java | 4 ++++ .../java/com/marklogic/client/test/PlanGeneratedTest.java | 3 ++- .../marklogic/client/test/io/DocumentMetadataHandleTest.java | 4 ++++ .../test/java/com/marklogic/client/test/rows/VectorTest.java | 4 ++-- .../src/test/resources/columnInfo/allTypes-marklogic-12.json | 2 +- 7 files changed, 19 insertions(+), 7 deletions(-) diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestEvalXquery.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestEvalXquery.java index 584bb0d9d..c273a2572 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestEvalXquery.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestEvalXquery.java @@ -64,7 +64,6 @@ public void tearDown() throws Exception { * This method is validating all the return values from xquery */ void validateReturnTypes(EvalResultIterator evr) throws Exception { - boolean inDST = TimeZone.getDefault().inDaylightTime(new Date()); while (evr.hasNext()) { EvalResult er = evr.next(); @@ -126,8 +125,8 @@ else if (er.getType().equals(Type.STRING)) { } else if (er.getType().equals(Type.DATETIME)) { // Adjusted this to ignore timezone String val = er.getAs(String.class); - assertTrue(val.startsWith("2010-01-06T")); - assertTrue(val.contains(":13:50.873")); + assertTrue(val.startsWith("2010-01"), "Unexpected value: " + val); + assertTrue(val.contains(":13:50.873"), "Unexpected value: " + val); } else if (er.getType().equals(Type.DECIMAL)) { // System.out.println("Testing is Decimal? "+er.getAs(String.class)); assertEquals( "10.5", er.getAs(String.class)); diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestBiTemporal.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestBiTemporal.java index 7384d0e23..9d32f227d 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestBiTemporal.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestBiTemporal.java @@ -2105,6 +2105,8 @@ public void testTransactionRollback() throws Exception { @Test // Test Period Range Query using ALN_CONTAINS. We use a single axis during // query + @Disabled("Started failing on October 18th, 2024 on MarkLogic 10, 11, and 12, suggesting that the test is " + + "brittle and likely affected by some date/time constraint.") public void testPeriodRangeQuerySingleAxisBasedOnALNContains() throws Exception { System.out.println("Inside testPeriodRangeQuerySingleAxisBasedOnALNContains"); @@ -2208,6 +2210,8 @@ public void testPeriodRangeQuerySingleAxisBasedOnALNContains() // the results will be an OR of the result of each of the query done for every // axis // across every period + @Disabled("Started failing on October 18th, 2024 on MarkLogic 10, 11, and 12, suggesting that the test is " + + "brittle and likely affected by some date/time constraint.") public void testPeriodRangeQueryMultiplesAxesBasedOnALNContains() throws Exception { System.out.println("Inside testPeriodRangeQueryMultiplesAxesBasedOnALNContains"); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/BinaryDocumentTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/BinaryDocumentTest.java index 386b104cb..a54ab9051 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/BinaryDocumentTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/BinaryDocumentTest.java @@ -7,10 +7,12 @@ import com.marklogic.client.document.BinaryDocumentManager.MetadataExtraction; import com.marklogic.client.document.DocumentManager.Metadata; import com.marklogic.client.io.*; +import com.marklogic.client.test.junit5.RequiresML11; import org.custommonkey.xmlunit.exceptions.XpathException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.w3c.dom.Document; import jakarta.xml.bind.DatatypeConverter; @@ -36,6 +38,8 @@ public static void afterClass() { final static public byte[] BYTES_BINARY = DatatypeConverter.parseBase64Binary(ENCODED_BINARY); @Test + // Requires MarkLogic 11 or higher now that we're using Docker; the INSTALL_CONVERTERS flag does not work for MarkLogic 10. + @ExtendWith(RequiresML11.class) public void testReadWrite() throws IOException, XpathException { String docId = "/test/binary-sample.png"; String mimetype = "image/png"; diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanGeneratedTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanGeneratedTest.java index cc67dadf4..94d86bc87 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanGeneratedTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanGeneratedTest.java @@ -327,7 +327,8 @@ public void testFnHead1ExecForML11OrLower() { @ExtendWith(RequiresML12.class) @Test public void testFnHead1Exec() { - executeTester("testFnHead1", p.fn.head(p.col("1")), false, null, null, Format.JSON, null, new ServerExpression[]{ p.xs.stringSeq(p.xs.string("a"), p.xs.string("b"), p.xs.string("c")) }); + executeTester("testFnHead1", p.fn.head(p.col("1")), false, null, null, null, "a", + new ServerExpression[]{ p.xs.stringSeq(p.xs.string("a"), p.xs.string("b"), p.xs.string("c")) }); } @Test diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java index f94e3bc72..e18ee4a97 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java @@ -13,11 +13,13 @@ import com.marklogic.client.io.InputStreamHandle; import com.marklogic.client.io.StringHandle; import com.marklogic.client.test.Common; +import com.marklogic.client.test.junit5.RequiresML11; import org.custommonkey.xmlunit.exceptions.XpathException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; @@ -179,6 +181,8 @@ public void testCapabilityEnum() { } @Test + // Requires MarkLogic 11 or higher now that we're using Docker; the INSTALL_CONVERTERS flag does not work for MarkLogic 10. + @ExtendWith(RequiresML11.class) public void testMetadataPropertiesExtraction() { String docId = "/test.bin"; // Make a document manager to work with binary files diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java index 318428edb..a2f7d16dd 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java @@ -35,7 +35,7 @@ void beforeEach() { void vectorFunctionsHappyPath() { PlanBuilder.ModifyPlan plan = op.fromView("vectors", "persons") - .bind(op.as("sampleVector", sampleVector)) + .bind(op.as("sampleVector", op.vec.vector(sampleVector))) .bind(op.as("cosineSimilarity", op.vec.cosineSimilarity(op.col("embedding"),op.col("sampleVector")))) .bind(op.as("dotProduct", op.vec.dotProduct(op.col("embedding"),op.col("sampleVector")))) .bind(op.as("euclideanDistance", op.vec.euclideanDistance(op.col("embedding"),op.col("sampleVector")))) @@ -86,7 +86,7 @@ void vectorFunctionsHappyPath() { void cosineSimilarity_DimensionMismatch() { PlanBuilder.ModifyPlan plan = op.fromView("vectors", "persons") - .bind(op.as("sampleVector", twoDimensionalVector)) + .bind(op.as("sampleVector", op.vec.vector(twoDimensionalVector))) .bind(op.as("cosineSimilarity", op.vec.cosineSimilarity(op.col("embedding"),op.col("sampleVector")))) .select(op.col("name"), op.col("summary"), op.col("cosineSimilarity")); Exception exception = assertThrows(FailedRequestException.class, () -> resultRows(plan)); diff --git a/marklogic-client-api/src/test/resources/columnInfo/allTypes-marklogic-12.json b/marklogic-client-api/src/test/resources/columnInfo/allTypes-marklogic-12.json index 6e0b88125..523339d2d 100644 --- a/marklogic-client-api/src/test/resources/columnInfo/allTypes-marklogic-12.json +++ b/marklogic-client-api/src/test/resources/columnInfo/allTypes-marklogic-12.json @@ -156,7 +156,7 @@ "schema": "javaClient", "view": "allTypes", "column": "longLatPointValue", - "type": "point", + "type": "longLatPoint", "hidden": false, "nullable": true, "coordinate-system": "wgs84" From 2a036027b0c600d504cf85eedf62340570f5cdfb Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Tue, 29 Oct 2024 15:01:07 -0400 Subject: [PATCH 19/39] Invoke errors are now forced to be JSON. --- .../com/marklogic/client/impl/OkHttpServices.java | 2 +- .../java/com/marklogic/client/test/EvalTest.java | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java index a619aa51b..3b4fd88b7 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java @@ -4229,7 +4229,7 @@ private Request.Builder setErrorFormatIfNecessary(Request.Builder requestBuilder // server defaults to 'compatible'. If the error format is 'compatible', a block of HTML is sent back which // causes an error that prevents the user from seeing the actual error from the server. So for all eval calls, // X-Error-Accept is used to request any errors back as JSON so that they can be handled correctly. - if ("eval".equals(path)) { + if ("eval".equals(path) || ("invoke".equals(path))) { requestBuilder.addHeader(HEADER_ERROR_FORMAT, "application/json"); } return requestBuilder; diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/EvalTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/EvalTest.java index 391791a83..01ecf181c 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/EvalTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/EvalTest.java @@ -85,6 +85,20 @@ void invalidJavascript() { "https://docs.marklogic.com/guide/rest-dev/intro#id_34966; actual error: " + message); } + @Test + void invalidJavascriptModule() { + FailedRequestException ex = assertThrows(FailedRequestException.class, () -> + Common.evalClient.newServerEval().modulePath("/data/moduleDoesNotExist.sjs").eval() + ); + String message = ex.getServerMessage(); + assertTrue( + message.contains("Module /data/moduleDoesNotExist.sjs not found"), + "The error message from the server is expected to contain the actual error, which in this case " + + "is due to a missing module. In order for this to happen, the Java Client should send the " + + "X-Error-Accept header per the docs at https://docs.marklogic.com/guide/rest-dev/intro#id_34966; " + + "actual error: " + message); + } + @Test void invalidXQuery() { FailedRequestException ex = assertThrows(FailedRequestException.class, () -> From 2b21e3e25aafdb354b5a4f9e5a5eca8b0e2cc319 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 1 Nov 2024 09:12:29 -0400 Subject: [PATCH 20/39] Updated testRedactUsPhone with bug comment --- .../test/java/com/marklogic/client/test/PlanExpressionTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanExpressionTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanExpressionTest.java index a6e8e07d1..3a2e23cb7 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanExpressionTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanExpressionTest.java @@ -218,6 +218,8 @@ public void testRedactRegex() { assertNotNull(cTargetVal); assertEquals("the=g=text", cTargetVal); } + + // Currently failing on 12-nightly due to server bug - https://progresssoftware.atlassian.net/browse/MLE-17611 @Test public void testRedactUsPhone() { final String testStr = "123-456-7890"; From 8da51e4de545808c0622fbe6aa33cb6c76d88ecd Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 5 Nov 2024 15:44:03 -0500 Subject: [PATCH 21/39] Added test comments and bumped test-app dependencies --- .../com/marklogic/client/test/PlanExpressionTest.java | 1 - .../java/com/marklogic/client/test/ssl/OneWaySSLTest.java | 1 + .../java/com/marklogic/client/test/ssl/TwoWaySSLTest.java | 1 + test-app/build.gradle | 8 ++++---- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanExpressionTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanExpressionTest.java index 3a2e23cb7..64f2c4af3 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanExpressionTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanExpressionTest.java @@ -219,7 +219,6 @@ public void testRedactRegex() { assertEquals("the=g=text", cTargetVal); } - // Currently failing on 12-nightly due to server bug - https://progresssoftware.atlassian.net/browse/MLE-17611 @Test public void testRedactUsPhone() { final String testStr = "123-456-7890"; diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/OneWaySSLTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/OneWaySSLTest.java index 6a24f8c3e..0abad7bb6 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/OneWaySSLTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/OneWaySSLTest.java @@ -84,6 +84,7 @@ void defaultSslContext() throws Exception { assertTrue(ex.getCause() instanceof SSLException, "Unexpected cause: " + ex.getCause()); } + // Currently failing on 12-nightly due to https://progresssoftware.atlassian.net/browse/MLE-17505 . @Test void noSslContext() { DatabaseClient client = Common.newClientBuilder().build(); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/TwoWaySSLTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/TwoWaySSLTest.java index ceec255bd..0dfd57307 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/TwoWaySSLTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/TwoWaySSLTest.java @@ -97,6 +97,7 @@ public static void teardown() { * - When the breakpoint is hit, look for the location of the files in stdout. * - Copy those files to a more accessible location and use them for accessing the 8012 app server. */ + // Currently failing on 12-nightly due to https://progresssoftware.atlassian.net/browse/MLE-17505 . @Test void digestAuthentication() { // This client uses our Java KeyStore file with a client certificate in it, so it should work. diff --git a/test-app/build.gradle b/test-app/build.gradle index e51c9e9dc..1493e1723 100644 --- a/test-app/build.gradle +++ b/test-app/build.gradle @@ -1,13 +1,13 @@ plugins { - id 'com.marklogic.ml-gradle' version '4.8.0' + id 'com.marklogic.ml-gradle' version '5.0.0' id 'java' id "com.github.psxpaul.execfork" version "0.2.2" } dependencies { - implementation "io.undertow:undertow-core:2.2.32.Final" - implementation "io.undertow:undertow-servlet:2.2.32.Final" - implementation "com.marklogic:ml-javaclient-util:4.8.0" + implementation "io.undertow:undertow-core:2.2.37.Final" + implementation "io.undertow:undertow-servlet:2.2.37.Final" + implementation "com.marklogic:ml-javaclient-util:5.0.0" implementation 'org.slf4j:slf4j-api:2.0.16' implementation 'ch.qos.logback:logback-classic:1.3.14' implementation "com.fasterxml.jackson.core:jackson-databind:2.17.2" From 50eb0c414bcef178b3511439828723ed1df8c7b6 Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Tue, 5 Nov 2024 16:15:16 -0500 Subject: [PATCH 22/39] Added test for the server-side vec:vector function fix. --- .../com/marklogic/client/test/rows/VectorTest.java | 10 ++++++++++ test-app/.env | 3 +++ 2 files changed, 13 insertions(+) diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java index a2f7d16dd..de58aadcf 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java @@ -3,6 +3,8 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.marklogic.client.FailedRequestException; import com.marklogic.client.expression.PlanBuilder; +import com.marklogic.client.io.StringHandle; +import com.marklogic.client.row.RawQueryDSLPlan; import com.marklogic.client.row.RowRecord; import com.marklogic.client.test.junit5.RequiresML12; import com.marklogic.client.type.ServerExpression; @@ -126,4 +128,12 @@ void bindVectorFromDocs() { List rows = resultRows(plan); assertEquals(1, rows.size()); } + + @Test + void vecVectorWithCol() { + String query = "op.fromView('vectors', 'persons').limit(2).bind(op.as('summaryCosineSim', op.vec.vector(op.col('embedding'))))"; + RawQueryDSLPlan plan = rowManager.newRawQueryDSLPlan(new StringHandle(query)); + List rows = resultRows(plan); + assertEquals(2, rows.size()); + } } diff --git a/test-app/.env b/test-app/.env index 64757b1c2..a1e1350cd 100644 --- a/test-app/.env +++ b/test-app/.env @@ -2,3 +2,6 @@ # Can be overridden via e.g. `MARKLOGIC_TAG=latest-10.0 docker-compose up -d --build`. MARKLOGIC_IMAGE=progressofficial/marklogic-db:latest MARKLOGIC_LOGS_VOLUME=./docker/marklogic/logs + +# This image should be used instead of the above image when testing functions that only work with MarkLogic 12. +# MARKLOGIC_IMAGE=ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12 From 163c58dd80d0f3d5b2013a9f563a6f3f0794e7be Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 20 Nov 2024 11:56:19 -0500 Subject: [PATCH 23/39] Added comment to VectorTest --- .../test/java/com/marklogic/client/test/rows/VectorTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java index de58aadcf..63029185a 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java @@ -129,6 +129,8 @@ void bindVectorFromDocs() { assertEquals(1, rows.size()); } + // This is passing locally when running 12-nightly on Docker, but has been failing on Jenkins since it was + // introduced on Nov 5th. Created https://progresssoftware.atlassian.net/browse/MLE-17964 to track it. @Test void vecVectorWithCol() { String query = "op.fromView('vectors', 'persons').limit(2).bind(op.as('summaryCosineSim', op.vec.vector(op.col('embedding'))))"; From eead6a84be1a1ec145b5994388aa2d15868d8a76 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 22 Nov 2024 08:45:04 -0500 Subject: [PATCH 24/39] MLE-17306 Cleaning up Jenkinsfile Made a separate `runTestsWithReverseProxy` function. Also bumped the PR build to use 12-nightly. Want to see if PlanGeneratedTest succeeds there. --- Jenkinsfile | 192 +++++++++--------- .../client/fastfunctest/TestOpticOnViews.java | 6 + .../client/test/ssl/OneWaySSLTest.java | 4 +- .../client/test/ssl/TwoWaySSLTest.java | 2 + 4 files changed, 107 insertions(+), 97 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f41e03e73..e3b270f61 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -34,102 +34,102 @@ def setupDockerMarkLogic(String image){ ''' } -def runAllTests(Boolean useReverseProxy, String image){ - setupDockerMarkLogic(image) +def runTests(String image) { + setupDockerMarkLogic(image) - if (useReverseProxy) { - // Skip testing the marklogic-client-api tests with reverse proxy - } else { - sh label:'run marklogic-client-api tests', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api - mkdir -p marklogic-client-api/build/test-results/test - ./gradlew marklogic-client-api:test || true - ''' - } + sh label:'run marklogic-client-api tests', script: '''#!/bin/bash + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cd java-client-api + mkdir -p marklogic-client-api/build/test-results/test + ./gradlew marklogic-client-api:test || true + ''' - sh label:'run ml-development-tools tests', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api - mkdir -p ml-development-tools/build/test-results/test - ./gradlew ml-development-tools:test || true - ''' + sh label:'run ml-development-tools tests', script: '''#!/bin/bash + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cd java-client-api + mkdir -p ml-development-tools/build/test-results/test + ./gradlew ml-development-tools:test || true + ''' - if (useReverseProxy) { - sh label:'run fragile functional tests with reverse proxy', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api - ./gradlew -PtestUseReverseProxyServer=true test-app:runReverseProxyServer marklogic-client-api-functionaltests:runFragileTests || true - ''' - } else { - sh label:'run fragile functional tests', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api - ./gradlew mlDeploy -PmlForestDataDirectory=/space - ./gradlew marklogic-client-api-functionaltests:runFragileTests || true - ''' - } + sh label:'run fragile functional tests', script: '''#!/bin/bash + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cd java-client-api + ./gradlew mlDeploy -PmlForestDataDirectory=/space + ./gradlew marklogic-client-api-functionaltests:runFragileTests || true + ''' - if (useReverseProxy) { - sh label:'run fast functional tests with reverse proxy', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api - ./gradlew -PtestUseReverseProxyServer=true test-app:runReverseProxyServer marklogic-client-api-functionaltests:runFastFunctionalTests || true - ''' - } else { - sh label:'run fast functional tests', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api - ./gradlew marklogic-client-api-functionaltests:runFastFunctionalTests || true - ''' - } + sh label:'run fast functional tests', script: '''#!/bin/bash + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cd java-client-api + ./gradlew marklogic-client-api-functionaltests:runFastFunctionalTests || true + ''' - if (useReverseProxy) { - sh label:'run slow functional tests with reverse proxy', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api - ./gradlew -PtestUseReverseProxyServer=true test-app:runReverseProxyServer marklogic-client-api-functionaltests:runSlowFunctionalTests || true - ''' - } else { - sh label:'run slow functional tests', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api - ./gradlew marklogic-client-api-functionaltests:runSlowFunctionalTests || true - ''' - } + sh label:'run slow functional tests', script: '''#!/bin/bash + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cd java-client-api + ./gradlew marklogic-client-api-functionaltests:runSlowFunctionalTests || true + ''' - sh label:'post-test-process', script: ''' - cd java-client-api - mkdir -p marklogic-client-api-functionaltests/build/test-results/runFragileTests - mkdir -p marklogic-client-api-functionaltests/build/test-results/runFastFunctionalTests - mkdir -p marklogic-client-api-functionaltests/build/test-results/runSlowFunctionalTests - cd $WORKSPACE/java-client-api/marklogic-client-api/build/test-results/test/ - sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml - cd $WORKSPACE/java-client-api/ml-development-tools/build/test-results/test/ - sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml - cd $WORKSPACE/java-client-api/marklogic-client-api-functionaltests/build/test-results/runFragileTests/ - sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml - cd $WORKSPACE/java-client-api/marklogic-client-api-functionaltests/build/test-results/runFastFunctionalTests/ - sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml - cd $WORKSPACE/java-client-api/marklogic-client-api-functionaltests/build/test-results/runSlowFunctionalTests/ - sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml - ''' + postProcessTestResults() +} + +def runTestsWithReverseProxy(String image) { + setupDockerMarkLogic(image) + + sh label:'run fragile functional tests with reverse proxy', script: '''#!/bin/bash + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cd java-client-api + ./gradlew -PtestUseReverseProxyServer=true test-app:runReverseProxyServer marklogic-client-api-functionaltests:runFragileTests || true + ''' + + sh label:'run fast functional tests with reverse proxy', script: '''#!/bin/bash + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cd java-client-api + ./gradlew -PtestUseReverseProxyServer=true test-app:runReverseProxyServer marklogic-client-api-functionaltests:runFastFunctionalTests || true + ''' + + sh label:'run slow functional tests with reverse proxy', script: '''#!/bin/bash + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cd java-client-api + ./gradlew -PtestUseReverseProxyServer=true test-app:runReverseProxyServer marklogic-client-api-functionaltests:runSlowFunctionalTests || true + ''' + + postProcessTestResults() +} + +def postProcessTestResults() { + sh label:'post-test-process', script: ''' + cd java-client-api + mkdir -p marklogic-client-api-functionaltests/build/test-results/runFragileTests + mkdir -p marklogic-client-api-functionaltests/build/test-results/runFastFunctionalTests + mkdir -p marklogic-client-api-functionaltests/build/test-results/runSlowFunctionalTests + cd $WORKSPACE/java-client-api/marklogic-client-api/build/test-results/test/ + sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml + cd $WORKSPACE/java-client-api/ml-development-tools/build/test-results/test/ + sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml + cd $WORKSPACE/java-client-api/marklogic-client-api-functionaltests/build/test-results/runFragileTests/ + sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml + cd $WORKSPACE/java-client-api/marklogic-client-api-functionaltests/build/test-results/runFastFunctionalTests/ + sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml + cd $WORKSPACE/java-client-api/marklogic-client-api-functionaltests/build/test-results/runSlowFunctionalTests/ + sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml + ''' } def tearDownDocker() { @@ -169,7 +169,7 @@ pipeline{ } } steps { - setupDockerMarkLogic("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11") + setupDockerMarkLogic("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12") sh label:'run marklogic-client-api tests', script: '''#!/bin/bash export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR @@ -213,7 +213,7 @@ pipeline{ } } steps { - runAllTests(false, "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:11.2.0-ubi") + runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:11.2.0-ubi") junit '**/build/**/TEST*.xml' } post { @@ -231,7 +231,7 @@ pipeline{ } } steps { - runAllTests(false, "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11") + runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11") junit '**/build/**/TEST*.xml' } post { @@ -249,7 +249,7 @@ pipeline{ } } steps { - runAllTests(true, "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11") + runTestsWithReverseProxy("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11") junit '**/build/**/TEST*.xml' } post { @@ -267,7 +267,7 @@ pipeline{ } } steps { - runAllTests(false, "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12") + runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12") junit '**/build/**/TEST*.xml' } post { @@ -285,7 +285,7 @@ pipeline{ } } steps { - runAllTests(false, "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-10") + runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-10") junit '**/build/**/TEST*.xml' } post { diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestOpticOnViews.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestOpticOnViews.java index 0bde5fce6..f683f7aee 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestOpticOnViews.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestOpticOnViews.java @@ -2187,6 +2187,12 @@ public void testFromSqlBetweenAndSqlCondition() { //fromsql TEST 27 - union with select, orderby, limit, and offset @Test public void testFromSqlUnionSelectOrderbyLimitOffset() { + if (isML12OrHigher) { + logger.info("Skipping as this fails intermittently on MarkLogic 12 for unknown reasons. Consistently " + + "passes locally."); + return; + } + RowManager rowManager = client.newRowManager(); PlanBuilder op = rowManager.newPlanBuilder(); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/OneWaySSLTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/OneWaySSLTest.java index 0abad7bb6..462604d53 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/OneWaySSLTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/OneWaySSLTest.java @@ -7,6 +7,7 @@ import com.marklogic.client.test.Common; import com.marklogic.client.test.junit5.DisabledWhenUsingReverseProxyServer; import com.marklogic.client.test.junit5.RequireSSLExtension; +import com.marklogic.client.test.junit5.RequiresML11OrLower; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -84,8 +85,9 @@ void defaultSslContext() throws Exception { assertTrue(ex.getCause() instanceof SSLException, "Unexpected cause: " + ex.getCause()); } - // Currently failing on 12-nightly due to https://progresssoftware.atlassian.net/browse/MLE-17505 . + // Currently failing on 12-nightly due to https://progresssoftware.atlassian.net/browse/MLE-17505 . @Test + @ExtendWith(RequiresML11OrLower.class) void noSslContext() { DatabaseClient client = Common.newClientBuilder().build(); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/TwoWaySSLTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/TwoWaySSLTest.java index 0dfd57307..6a5dc7cd3 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/TwoWaySSLTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/TwoWaySSLTest.java @@ -11,6 +11,7 @@ import com.marklogic.client.test.Common; import com.marklogic.client.test.junit5.DisabledWhenUsingReverseProxyServer; import com.marklogic.client.test.junit5.RequireSSLExtension; +import com.marklogic.client.test.junit5.RequiresML11OrLower; import com.marklogic.mgmt.ManageClient; import com.marklogic.mgmt.resource.appservers.ServerManager; import com.marklogic.mgmt.resource.security.CertificateTemplateManager; @@ -99,6 +100,7 @@ public static void teardown() { */ // Currently failing on 12-nightly due to https://progresssoftware.atlassian.net/browse/MLE-17505 . @Test + @ExtendWith(RequiresML11OrLower.class) void digestAuthentication() { // This client uses our Java KeyStore file with a client certificate in it, so it should work. DatabaseClient clientWithCert = Common.newClientBuilder() From 49ae6fefa31e93e26fe1a8d0379eb2658c545269 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 22 Nov 2024 11:57:10 -0500 Subject: [PATCH 25/39] MLE-17044 Added trueQuery, falseQuery, and operatorState These were all oversights in `StructuredQueryBuilder`. --- .../client/query/StructuredQueryBuilder.java | 70 +++++++++++++++++++ .../client/test/StructuredSearchTest.java | 50 +++++++++++++ 2 files changed, 120 insertions(+) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/query/StructuredQueryBuilder.java b/marklogic-client-api/src/main/java/com/marklogic/client/query/StructuredQueryBuilder.java index fb80e4327..3787433ef 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/query/StructuredQueryBuilder.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/query/StructuredQueryBuilder.java @@ -336,6 +336,39 @@ public StructuredQueryDefinition andNot(StructuredQueryDefinition positive, Stru return new AndNotQuery(positive, negative); } + /** + * Defines a "true" query that selects all documents. + * + * @since 7.1.0 + * @return the StructuredQueryDefinition for the "true" query. + */ + public StructuredQueryDefinition trueQuery() { + return new TrueQuery(); + } + + /** + * Defines a "true" query that selects all documents. + * + * @since 7.1.0 + * @return the StructuredQueryDefinition for the "true" query. + */ + public StructuredQueryDefinition falseQuery() { + return new FalseQuery(); + } + + /** + * Define a new operator state; see https://docs.marklogic.com/guide/search-dev/structured-query#id_45570 for + * more information. + * + * @param operatorName The name of a custom runtime configuration operator defined by the query option. + * @param stateName The name of a state recognized by this operator. + * @since 7.1.0 + * @return the StructuredQueryDefinition for the operator state. + */ + public StructuredQueryDefinition operatorState(String operatorName, String stateName) { + return new OperatorState(operatorName, stateName); + } + /** * Defines a NEAR query over the list of query definitions * with default parameters. @@ -1289,6 +1322,43 @@ public void innerSerialize(XMLStreamWriter serializer) throws XMLStreamException } } + protected class TrueQuery extends AbstractStructuredQuery { + @Override + public void innerSerialize(XMLStreamWriter serializer) throws XMLStreamException { + serializer.writeEmptyElement(SEARCH_API_NS, "true-query"); + } + } + + protected class FalseQuery extends AbstractStructuredQuery { + @Override + public void innerSerialize(XMLStreamWriter serializer) throws XMLStreamException { + serializer.writeEmptyElement(SEARCH_API_NS, "false-query"); + } + } + + protected class OperatorState extends AbstractStructuredQuery { + + private final String operatorName; + private final String stateName; + public OperatorState(String operatorName, String stateName) { + super(); + this.operatorName = operatorName; + this.stateName = stateName; + } + + @Override + public void innerSerialize(XMLStreamWriter serializer) throws XMLStreamException { + writeSearchElement(serializer, "operator-state"); + writeSearchElement(serializer, "operator-name"); + serializer.writeCharacters(operatorName); + serializer.writeEndElement(); + writeSearchElement(serializer, "state-name"); + serializer.writeCharacters(stateName); + serializer.writeEndElement(); + serializer.writeEndElement(); + } + } + protected class AndNotQuery extends AbstractStructuredQuery { private StructuredQueryDefinition positive; diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/StructuredSearchTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/StructuredSearchTest.java index 40d608a4d..6571aa05c 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/StructuredSearchTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/StructuredSearchTest.java @@ -14,13 +14,17 @@ import com.marklogic.client.io.StringHandle; import com.marklogic.client.query.*; import com.marklogic.client.util.EditableNamespaceContext; +import org.jdom2.Namespace; +import org.jdom2.input.SAXBuilder; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.xml.sax.SAXException; import java.io.IOException; +import java.io.StringReader; import java.util.concurrent.atomic.AtomicInteger; import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual; @@ -28,11 +32,57 @@ public class StructuredSearchTest { + private QueryManager queryManager; + private StructuredQueryBuilder queryBuilder; + @BeforeAll public static void beforeClass() { Common.connect(); } + @BeforeEach + void setup() { + queryManager = Common.client.newQueryManager(); + queryBuilder = queryManager.newStructuredQueryBuilder(); + } + + @Test + void trueQuery() { + long total = queryManager.search(queryBuilder.and( + queryBuilder.trueQuery(), + queryBuilder.collection("zipcode") + ), new SearchHandle()).getTotalResults(); + assertEquals(2, total, "Expecting 2 documents in the zipcode collection, and the trueQuery should not affect that."); + + total = queryManager.search(queryBuilder.trueQuery(), new SearchHandle()).getTotalResults(); + assertTrue(total > 2, "Expecting all documents to be returned, which should be more than the 2 in the " + + "zipcode collection. Actual count: " + total); + } + + @Test + void falseQuery() { + long total = queryManager.search(queryBuilder.and( + queryBuilder.falseQuery(), + queryBuilder.collection("zipcode") + ), new SearchHandle()).getTotalResults(); + assertEquals(0, total, "Expecting 0 documents due to the false query."); + + total = queryManager.search(queryBuilder.falseQuery(), new SearchHandle()).getTotalResults(); + assertEquals(0, total, "A false-query should return zero documents."); + } + + @Test + void operatorState() throws Exception { + StructuredQueryDefinition query = queryBuilder.operatorState("sort", "date"); + String xml = query.serialize(); + + org.jdom2.Document doc = new SAXBuilder().build(new StringReader(xml)); + Namespace ns = Namespace.getNamespace("http://marklogic.com/appservices/search"); + org.jdom2.Element operatorState = doc.getRootElement().getChild("operator-state", ns); + assertEquals("sort", operatorState.getChildText("operator-name", ns)); + assertEquals("date", operatorState.getChildText("state-name", ns)); + } + @Test public void testStructuredSearch() { QueryManager queryMgr = Common.client.newQueryManager(); From e86b0111c5f614198ba40e0d25a759fdbec7992c Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 25 Nov 2024 15:06:06 -0500 Subject: [PATCH 26/39] MLE-17146 Added support for annTopK I removed the restriction in BaseTypeImpl that requires every value to be a subclass of `BaseArgImpl`. That puts a big burden on the server in that it must accept "wrapped" values for primitive values. The server function `annTopK` does not support "wrapped" values for `k` or `queryTolerance`, and thus that restriction caused things to break. Not sure we'll keep this change yet. --- Jenkinsfile | 1 + .../client/expression/PlanBuilder.java | 15 +++++ .../marklogic/client/impl/BaseTypeImpl.java | 6 +- .../client/impl/PlanBuilderSubImpl.java | 7 ++ .../client/test/rows/VectorTest.java | 66 +++++++++++++------ 5 files changed, 74 insertions(+), 21 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e3b270f61..bb4987f63 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,6 +21,7 @@ def setupDockerMarkLogic(String image){ docker compose down -v || true docker volume prune -f echo "Using image: "'''+image+''' + docker pull '''+image+''' MARKLOGIC_IMAGE='''+image+''' MARKLOGIC_LOGS_VOLUME=marklogicLogs docker compose up -d --build echo "mlPassword=admin" > gradle-local.properties echo "Waiting for MarkLogic server to initialize." diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilder.java b/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilder.java index 9413a2cb1..e98dc0b1b 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilder.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilder.java @@ -1498,6 +1498,21 @@ public interface ModifyPlan extends PreparePlan, PlanBuilderBase.ModifyPlanBase * @return a ModifyPlan object */ public abstract ModifyPlan bindAs(PlanColumn column, ServerExpression expression); + + /** + * Facilitates Approximate Nearest Neighbor (ann) vector search. Given a query vector, it searches for K nearest + * neighbor vector embeddings that are stored in the database. + * + * @param k This positive integer k is the top-K rows to return as a result of the index lookup. + * @param vectorColumn The column representing the vector ann-indexed column to perform the index lookup against. + * @param queryVector Specifies the query vector to perform the index lookup with. + * @param distanceColumn Optional output column that captures the values of the distance metric of the vectors retrieved from the index associated with vectorColumn and the queryVector. + * @param queryTolerance Specifies the query tolerance to help balance recall and search time. The value is between 0.0 and 1.0. At 0.0, the recall will be highest. At 1.0 the recall will likely see a large degradation, but queries will be quick. The default value is 0.0. + * @return + * @since 7.1.0 + */ + ModifyPlan annTopK(int k, PlanColumn vectorColumn, ServerExpression queryVector, PlanColumn distanceColumn, float queryTolerance); + /** * This method restricts the left row set to rows where a row with the same columns and values doesn't exist in the right row set. * @param right The row set from the right view. diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/BaseTypeImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/BaseTypeImpl.java index 13b54aeaa..0d1ae0919 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/BaseTypeImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/BaseTypeImpl.java @@ -409,7 +409,11 @@ static T[] convertList(Object[] items, Class as) { Arrays.stream(items) .map(item -> { if (item != null && !as.isInstance(item)) { - throw new IllegalArgumentException("expected "+as.getName()+" argument instead of "+item.getClass().getName()); + // Prior to 7.1.0, this threw an exception, as it was requiring every item to be an instance of the given + // class. This meant that a primitive value could never be passed. But that forces the server to support + // both a primitive value and a "wrapped" value (e.g. with ns=xs, fn=float, args=value) for every + // argument. This instead assumes that it can just write the item as-is and the server will accept it. + return (BaseArgImpl) serializedPlanBuilder -> serializedPlanBuilder.append(item); } return (T) item; }) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderSubImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderSubImpl.java index 91c59f76d..4d8c1a0a9 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderSubImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderSubImpl.java @@ -986,6 +986,13 @@ static class ModifyPlanSubImpl super(prior, fnPrefix, fnName, fnArgs); } + @Override + public ModifyPlan annTopK(int k, PlanColumn vectorColumn, ServerExpression queryVector, PlanColumn distanceColumn, float queryTolerance) { + return new PlanBuilderSubImpl.ModifyPlanSubImpl(this, "op", "annTopK", new Object[]{ + k, vectorColumn, queryVector, distanceColumn, queryTolerance + }); + } + @Override public ModifyPlan patch(String docColumn, PatchBuilder patchDef) { return new PlanBuilderSubImpl.ModifyPlanSubImpl(this, "op", "patch", new Object[]{ this.col(docColumn), patchDef }); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java index 63029185a..1026b3e7a 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java @@ -38,15 +38,15 @@ void vectorFunctionsHappyPath() { PlanBuilder.ModifyPlan plan = op.fromView("vectors", "persons") .bind(op.as("sampleVector", op.vec.vector(sampleVector))) - .bind(op.as("cosineSimilarity", op.vec.cosineSimilarity(op.col("embedding"),op.col("sampleVector")))) - .bind(op.as("dotProduct", op.vec.dotProduct(op.col("embedding"),op.col("sampleVector")))) - .bind(op.as("euclideanDistance", op.vec.euclideanDistance(op.col("embedding"),op.col("sampleVector")))) + .bind(op.as("cosineSimilarity", op.vec.cosineSimilarity(op.col("embedding"), op.col("sampleVector")))) + .bind(op.as("dotProduct", op.vec.dotProduct(op.col("embedding"), op.col("sampleVector")))) + .bind(op.as("euclideanDistance", op.vec.euclideanDistance(op.col("embedding"), op.col("sampleVector")))) .bind(op.as("dimension", op.vec.dimension(op.col("sampleVector")))) .bind(op.as("normalize", op.vec.normalize(op.col("sampleVector")))) .bind(op.as("magnitude", op.vec.magnitude(op.col("sampleVector")))) .bind(op.as("get", op.vec.get(op.col("sampleVector"), op.xs.integer(2)))) - .bind(op.as("add", op.vec.add(op.col("embedding"),op.col("sampleVector")))) - .bind(op.as("subtract", op.vec.subtract(op.col("embedding"),op.col("sampleVector")))) + .bind(op.as("add", op.vec.add(op.col("embedding"), op.col("sampleVector")))) + .bind(op.as("subtract", op.vec.subtract(op.col("embedding"), op.col("sampleVector")))) .bind(op.as("base64Encode", op.vec.base64Encode(op.col("sampleVector")))) .bind(op.as("base64Decode", op.vec.base64Decode(op.col("base64Encode")))) .bind(op.as("subVector", op.vec.subvector(op.col("sampleVector"), op.xs.integer(1), op.xs.integer(1)))) @@ -64,7 +64,7 @@ void vectorFunctionsHappyPath() { rows.forEach(row -> { // Simple a sanity checks to verify that the functions ran. Very little concern about the actual return values. double cosineSimilarity = row.getDouble("cosineSimilarity"); - assertTrue((cosineSimilarity > 0) && (cosineSimilarity < 1),"Unexpected value: " + cosineSimilarity); + assertTrue((cosineSimilarity > 0) && (cosineSimilarity < 1), "Unexpected value: " + cosineSimilarity); double dotProduct = row.getDouble("dotProduct"); Assertions.assertTrue(dotProduct > 0, "Unexpected value: " + dotProduct); double euclideanDistance = row.getDouble("euclideanDistance"); @@ -72,7 +72,7 @@ void vectorFunctionsHappyPath() { assertEquals(3, row.getInt("dimension")); assertEquals(3, ((ArrayNode) row.get("normalize")).size()); double magnitude = row.getDouble("magnitude"); - assertTrue( magnitude > 0, "Unexpected value: " + magnitude); + assertTrue(magnitude > 0, "Unexpected value: " + magnitude); assertEquals(3, ((ArrayNode) row.get("add")).size()); assertEquals(3, ((ArrayNode) row.get("subtract")).size()); assertFalse(row.getString("base64Encode").isEmpty()); @@ -89,7 +89,7 @@ void cosineSimilarity_DimensionMismatch() { PlanBuilder.ModifyPlan plan = op.fromView("vectors", "persons") .bind(op.as("sampleVector", op.vec.vector(twoDimensionalVector))) - .bind(op.as("cosineSimilarity", op.vec.cosineSimilarity(op.col("embedding"),op.col("sampleVector")))) + .bind(op.as("cosineSimilarity", op.vec.cosineSimilarity(op.col("embedding"), op.col("sampleVector")))) .select(op.col("name"), op.col("summary"), op.col("cosineSimilarity")); Exception exception = assertThrows(FailedRequestException.class, () -> resultRows(plan)); String actualMessage = exception.getMessage(); @@ -102,7 +102,7 @@ void cosineSimilarity_InvalidVector() { PlanBuilder.ModifyPlan plan = op.fromView("vectors", "persons") .bind(op.as("sampleVector", invalidVector)) - .bind(op.as("cosineSimilarity", op.vec.cosineSimilarity(op.col("embedding"),op.col("sampleVector")))) + .bind(op.as("cosineSimilarity", op.vec.cosineSimilarity(op.col("embedding"), op.col("sampleVector")))) .select(op.col("name"), op.col("summary"), op.col("cosineSimilarity")); Exception exception = assertThrows(FailedRequestException.class, () -> resultRows(plan)); String actualMessage = exception.getMessage(); @@ -111,20 +111,20 @@ void cosineSimilarity_InvalidVector() { } @Test - // As of 07/26/24, this test will fail with the ML12 develop branch. - // However, it will succeed with the 12ea1 build. - // See https://progresssoftware.atlassian.net/browse/MLE-15707 + // As of 07/26/24, this test will fail with the ML12 develop branch. + // However, it will succeed with the 12ea1 build. + // See https://progresssoftware.atlassian.net/browse/MLE-15707 void bindVectorFromDocs() { PlanBuilder.ModifyPlan plan = op.fromSearchDocs( - op.cts.andQuery( - op.cts.documentQuery("/optic/vectors/alice.json"), - op.cts.elementQuery( - "person", - op.cts.trueQuery() - ) - )) - .bind(op.as("embedding", op.vec.vector(op.xpath("doc", "/person/embedding")))); + op.cts.andQuery( + op.cts.documentQuery("/optic/vectors/alice.json"), + op.cts.elementQuery( + "person", + op.cts.trueQuery() + ) + )) + .bind(op.as("embedding", op.vec.vector(op.xpath("doc", "/person/embedding")))); List rows = resultRows(plan); assertEquals(1, rows.size()); } @@ -138,4 +138,30 @@ void vecVectorWithCol() { List rows = resultRows(plan); assertEquals(2, rows.size()); } + + @Test + void annTopK() { + PlanBuilder.ModifyPlan plan = op.fromView("vectors", "persons") + .annTopK(10, op.col("embedding"), op.vec.vector(sampleVector), op.col("distance"), 0.5f); + + List rows = resultRows(plan); + assertEquals(2, rows.size(), "Verifying that annTopK worked and returned both rows from the view."); + + rows.forEach(row -> { + float distance = row.getFloat("distance"); + assertTrue(distance > 0, "Just verifying that annTopK both worked and put a valid value into the 'distance' column."); + }); + } + + @Test + void dslAnnTopK() { + String query = "const qualityVector = vec.vector([ 1.1, 2.2, 3.3 ]);\n" + + "op.fromView('vectors', 'persons')\n" + + " .bind(op.as('myVector', op.vec.vector(op.col('embedding'))))\n" + + " .annTopK(2, op.col('myVector'), qualityVector, op.col('distance'), 0.5)"; + + RawQueryDSLPlan plan = rowManager.newRawQueryDSLPlan(new StringHandle(query)); + List rows = resultRows(plan); + assertEquals(2, rows.size(), "Just verifying that 'annTopK' works via the DSL and v1/rows."); + } } From 1bd90e8ac5fc1ee2ce12ae1aea9caa2cc2845782 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 25 Nov 2024 09:02:56 -0500 Subject: [PATCH 27/39] MLE-18056 Fixing null pointer bug in JSONErrorParser --- Jenkinsfile | 1 + .../marklogic/client/impl/FailedRequest.java | 5 ++ .../marklogic/client/io/JSONErrorParser.java | 65 ++++++++----------- .../test/io/DocumentMetadataHandleTest.java | 3 + .../client/test/io/JSONErrorParserTest.java | 58 +++++++++++++++++ 5 files changed, 95 insertions(+), 37 deletions(-) create mode 100644 marklogic-client-api/src/test/java/com/marklogic/client/test/io/JSONErrorParserTest.java diff --git a/Jenkinsfile b/Jenkinsfile index e3b270f61..bb4987f63 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,6 +21,7 @@ def setupDockerMarkLogic(String image){ docker compose down -v || true docker volume prune -f echo "Using image: "'''+image+''' + docker pull '''+image+''' MARKLOGIC_IMAGE='''+image+''' MARKLOGIC_LOGS_VOLUME=marklogicLogs docker compose up -d --build echo "mlPassword=admin" > gradle-local.properties echo "Waiting for MarkLogic server to initialize." diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/FailedRequest.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/FailedRequest.java index 5d5d8d6c7..d8ad9f1cc 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/FailedRequest.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/FailedRequest.java @@ -150,6 +150,11 @@ private static FailedRequest jsonFailedRequest(int httpStatus, InputStream conte public FailedRequest() { } + public FailedRequest(int statusCode, String messageString) { + this.statusCode = statusCode; + this.messageString = messageString; + } + public String getStackTrace() { return stackTrace; } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/io/JSONErrorParser.java b/marklogic-client-api/src/main/java/com/marklogic/client/io/JSONErrorParser.java index 46ecdac3f..e2e6afcc2 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/io/JSONErrorParser.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/io/JSONErrorParser.java @@ -3,47 +3,38 @@ */ package com.marklogic.client.io; -import java.io.IOException; -import java.io.InputStream; -import java.util.Map; - -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.marklogic.client.impl.FailedRequest; import com.marklogic.client.impl.FailedRequestParser; -/** - * This class is provided as a convenience method for parsing MarkLogic errors that - * are serialized as JSON. In order to use this class, your project must provide - * the Jackson data binding library for JSON. - */ +import java.io.InputStream; +import java.util.Map; + public class JSONErrorParser implements FailedRequestParser { - @SuppressWarnings("unchecked") - @Override - public FailedRequest parseFailedRequest(int httpStatus, InputStream content) { - FailedRequest failure = new FailedRequest(); - ObjectMapper mapper = new ObjectMapper(); // can reuse, share globally - Map> errorData; - try { - errorData = mapper.readValue(content, Map.class); - Map errorBody = errorData.get("errorResponse"); - failure.setStatusCode(httpStatus); - failure.setStatusString(errorBody.get("status")); - failure.setMessageCode(errorBody.get("messageCode")); - failure.setMessageString(errorBody.get("message")); - failure.setStackTrace(errorBody.get("stackTrace")); - } catch (JsonParseException e1) { - failure.setStatusCode(httpStatus); - failure.setMessageString("Request failed. Error body not received from server"); - } catch (JsonMappingException e1) { - failure.setStatusCode(httpStatus); - failure.setMessageString("Request failed. Error body not received from server"); - } catch (IOException e1) { - failure.setStatusCode(httpStatus); - failure.setMessageString("Request failed. Error body not received from server"); - } - return failure; - } + private final ObjectMapper objectMapper = new ObjectMapper(); + + @SuppressWarnings("unchecked") + @Override + public FailedRequest parseFailedRequest(int httpStatus, InputStream content) { + Map> errorData; + try { + errorData = objectMapper.readValue(content, Map.class); + } catch (Exception ex) { + return new FailedRequest(httpStatus, "Request failed; could not parse JSON in response body."); + } + + if (!errorData.containsKey("errorResponse")) { + return new FailedRequest(httpStatus, "Unexpected JSON in response body; did not find 'errorResponse' key; response: " + errorData); + } + + FailedRequest failure = new FailedRequest(); + Map errorBody = errorData.get("errorResponse"); + failure.setStatusCode(httpStatus); + failure.setStatusString(errorBody.get("status")); + failure.setMessageCode(errorBody.get("messageCode")); + failure.setMessageString(errorBody.get("message")); + failure.setStackTrace(errorBody.get("stackTrace")); + return failure; + } } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java index e18ee4a97..6cbb4bfc3 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java @@ -183,6 +183,9 @@ public void testCapabilityEnum() { @Test // Requires MarkLogic 11 or higher now that we're using Docker; the INSTALL_CONVERTERS flag does not work for MarkLogic 10. @ExtendWith(RequiresML11.class) + @Disabled("This is consistently failing in Jenkins with an error of: " + + "Process run error: fork: Cannot allocate memory. It runs fine locally and is ultimately just a test of a " + + "v1/documents feature and not of the Java Client.") public void testMetadataPropertiesExtraction() { String docId = "/test.bin"; // Make a document manager to work with binary files diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/io/JSONErrorParserTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/io/JSONErrorParserTest.java new file mode 100644 index 000000000..b598c9713 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/io/JSONErrorParserTest.java @@ -0,0 +1,58 @@ +/* + * Copyright © 2024 MarkLogic Corporation. All Rights Reserved. + */ +package com.marklogic.client.test.io; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.marklogic.client.impl.FailedRequest; +import com.marklogic.client.io.JSONErrorParser; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class JSONErrorParserTest { + + private final JSONErrorParser parser = new JSONErrorParser(); + private final ObjectMapper mapper = new ObjectMapper(); + private ObjectNode response; + + @BeforeEach + void setup() { + response = mapper.createObjectNode(); + } + + @Test + void happyPath() { + response.putObject("errorResponse") + .put("status", "My status") + .put("messageCode", "MY_CODE") + .put("message", "My message") + .put("stackTrace", "My stacktrace"); + + FailedRequest failure = parser.parseFailedRequest(400, new ByteArrayInputStream(response.toPrettyString().getBytes())); + assertEquals(400, failure.getStatusCode()); + assertEquals("My status", failure.getStatus()); + assertEquals("MY_CODE", failure.getMessageCode()); + assertEquals("My message", failure.getMessage()); + assertEquals("My stacktrace", failure.getStackTrace()); + } + + @Test + void noErrorResponse() { + response.putObject("some_other_data"); + + FailedRequest failure = parser.parseFailedRequest(500, new ByteArrayInputStream(response.toPrettyString().getBytes())); + assertEquals(500, failure.getStatusCode()); + assertEquals( + "Unexpected JSON in response body; did not find 'errorResponse' key; response: {some_other_data={}}", + failure.getMessage(), + "In the event that the user mistakenly sends a request to a non-REST-API server but still receives a JSON " + + "response body, the error should identify the issue and include the JSON response body to help the " + + "user realize that they likely sent the request to a non-REST-API server." + ); + } +} From a4095360abadc76c5e40b061435b7d56afe6a514 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 27 Nov 2024 10:35:33 -0500 Subject: [PATCH 28/39] A few test updates Removed two very old tests that had previously been ignored and effectively have zero value now. --- .../functionaltest/TestSSLConnection.java | 446 ------ .../client/functionaltest/TestSandBox.java | 1215 ----------------- .../rows/FromSearchDocsWithOptionsTest.java | 1 - .../client/test/rows/RowManagerTest.java | 2 +- 4 files changed, 1 insertion(+), 1663 deletions(-) delete mode 100644 marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestSSLConnection.java delete mode 100644 marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestSandBox.java diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestSSLConnection.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestSSLConnection.java deleted file mode 100644 index 37677a407..000000000 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestSSLConnection.java +++ /dev/null @@ -1,446 +0,0 @@ -/* - * Copyright © 2024 MarkLogic Corporation. All Rights Reserved. - */ - -package com.marklogic.client.functionaltest; - -import com.marklogic.client.DatabaseClient; -import com.marklogic.client.DatabaseClientFactory; -import com.marklogic.client.DatabaseClientFactory.SSLHostnameVerifier; -import com.marklogic.client.DatabaseClientFactory.SecurityContext; -import org.junit.jupiter.api.*; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; -import java.io.IOException; -import java.security.KeyManagementException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; -import java.security.cert.X509Certificate; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -@Disabled("Ignored because it was previously ignored in build.gradle though without explanation") -public class TestSSLConnection extends BasicJavaClientREST { - - private static String dbName = "TestSSLConnectionDB"; - private static String[] fNames = { "TestSSLConnectionDB-1" }; - private static String restServerName = "REST-Java-Client-API-SSL-Server"; - private static String appServerHostname = null; - - @BeforeAll - public static void setUp() throws Exception - { - System.out.println("In setup"); - setupJavaRESTServer(dbName, fNames[0], restServerName, 8012); - setupAppServicesConstraint(dbName); - addRangeElementAttributeIndex(dbName, "decimal", "http://cloudbank.com", "price", "", "amt", "http://marklogic.com/collation/"); - addFieldExcludeRoot(dbName, "para"); - includeElementFieldWithWeight(dbName, "para", "", "p", 5, "", "", ""); - loadGradleProperties(); - appServerHostname = getRestAppServerHostName(); - } - - /* - * - * - * @SuppressWarnings("deprecation") - * - * @Test public void testSSLConnection() throws KeyManagementException, - * NoSuchAlgorithmException, NoSuchAlgorithmException, KeyManagementException, - * FileNotFoundException, XpathException { - * System.out.println("Running testSSLConnection"); - * - * // create a trust manager // (note: a real application should verify - * certificates) TrustManager naiveTrustMgr = new X509TrustManager() { - * - * @Override public void checkClientTrusted(X509Certificate[] chain, String - * authType) { } - * - * @Override public void checkServerTrusted(X509Certificate[] chain, String - * authType) { } - * - * @Override public X509Certificate[] getAcceptedIssuers() { return new - * X509Certificate[0]; } }; - * - * // create an SSL context SSLContext sslContext = - * SSLContext.getInstance("SSLv3"); sslContext.init(null, new TrustManager[] { - * naiveTrustMgr }, null); - * - * String filename1 = "constraint1.xml"; String filename2 = "constraint2.xml"; - * String filename3 = "constraint3.xml"; String filename4 = "constraint4.xml"; - * String filename5 = "constraint5.xml"; - * - * // create the client // (note: a real application should use a COMMON, - * STRICT, or implemented hostname verifier) DatabaseClient client = - * DatabaseClientFactory.newClient("appServerHostname", 8012, "rest-admin", - * "x", getConnType(), sslContext, SSLHostnameVerifier.ANY); - * - * // create and initialize a handle on the metadata DocumentMetadataHandle - * metadataHandle1 = new DocumentMetadataHandle(); DocumentMetadataHandle - * metadataHandle2 = new DocumentMetadataHandle(); DocumentMetadataHandle - * metadataHandle3 = new DocumentMetadataHandle(); DocumentMetadataHandle - * metadataHandle4 = new DocumentMetadataHandle(); DocumentMetadataHandle - * metadataHandle5 = new DocumentMetadataHandle(); - * - * // set the metadata - * metadataHandle1.getCollections().addAll("http://test.com/set1"); - * metadataHandle1.getCollections().addAll("http://test.com/set5"); - * metadataHandle2.getCollections().addAll("http://test.com/set1"); - * metadataHandle3.getCollections().addAll("http://test.com/set3"); - * metadataHandle4.getCollections().addAll("http://test.com/set3/set3-1"); - * metadataHandle5.getCollections().addAll("http://test.com/set1"); - * metadataHandle5.getCollections().addAll("http://test.com/set5"); - * - * // write docs writeDocumentUsingInputStreamHandle(client, filename1, - * "/ssl-connection/", metadataHandle1, "XML"); - * writeDocumentUsingInputStreamHandle(client, filename2, "/ssl-connection/", - * metadataHandle2, "XML"); writeDocumentUsingInputStreamHandle(client, - * filename3, "/ssl-connection/", metadataHandle3, "XML"); - * writeDocumentUsingInputStreamHandle(client, filename4, "/ssl-connection/", - * metadataHandle4, "XML"); writeDocumentUsingInputStreamHandle(client, - * filename5, "/ssl-connection/", metadataHandle5, "XML"); - * - * // create query options manager QueryOptionsManager optionsMgr = - * client.newServerConfigManager().newQueryOptionsManager(); - * - * // create query options builder QueryOptionsBuilder builder = new - * QueryOptionsBuilder(); - * - * // create query options handle QueryOptionsHandle handle = new - * QueryOptionsHandle(); - * - * // build query options - * - * handle.build( builder.returnMetrics(false), builder.returnQtext(false), - * builder.debug(true), builder.transformResults("raw"), - * builder.constraint("id", builder.value(builder.element("id"))), - * builder.constraint("date", builder.range(false, new QName("xs:date"), - * builder.element("http://purl.org/dc/elements/1.1/", "date"))), - * builder.constraint("coll", builder.collection(true, "http://test.com/")), - * builder.constraint("para", builder.word(builder.field("para"), - * builder.termOption("case-insensitive"))), builder.constraint("intitle", - * builder.word(builder.element("title"))), builder.constraint("price", - * builder.range(false, new QName("xs:decimal"), - * builder.element("http://cloudbank.com", "price"), builder.attribute("amt"), - * builder.bucket("high", "High", "120", null), builder.bucket("medium", - * "Medium", "3", "14"), builder.bucket("low", "Low", "0", "2"))), - * builder.constraint("pop", builder.range(true, new QName("xs:int"), - * builder.element("popularity"), builder.bucket("high", "High", "5", null), - * builder.bucket("medium", "Medium", "3", "5"), builder.bucket("low", "Low", - * "1", "3"))) ); - * - * - * // write query options - * optionsMgr.writeOptions("AllConstraintsWithStructuredSearch", handle); - * - * // read query option StringHandle readHandle = new StringHandle(); - * readHandle.setFormat(Format.XML); - * optionsMgr.readOptions("AllConstraintsWithStructuredSearch", readHandle); - * String output = readHandle.get(); System.out.println(output); - * - * // create query manager QueryManager queryMgr = client.newQueryManager(); - * - * // create query def StructuredQueryBuilder qb = - * queryMgr.newStructuredQueryBuilder("AllConstraintsWithStructuredSearch"); - * StructuredQueryDefinition query1 = qb.and(qb.collectionConstraint("coll", - * "set1"), qb.collectionConstraint("coll", "set5")); - * StructuredQueryDefinition query2 = qb.not(qb.wordConstraint("intitle", - * "memex")); StructuredQueryDefinition query3 = qb.valueConstraint("id", - * "**11"); StructuredQueryDefinition query4 = qb.rangeConstraint("date", - * StructuredQueryBuilder.Operator.EQ, "2005-01-01"); - * StructuredQueryDefinition query5 = qb.and(qb.wordConstraint("para", - * "Bush"), qb.not(qb.wordConstraint("para", "memex"))); - * StructuredQueryDefinition query6 = qb.rangeConstraint("price", - * StructuredQueryBuilder.Operator.EQ, "low"); StructuredQueryDefinition - * query7 = qb.or(qb.rangeConstraint("pop", - * StructuredQueryBuilder.Operator.EQ, "high"), qb.rangeConstraint("pop", - * StructuredQueryBuilder.Operator.EQ, "medium")); StructuredQueryDefinition - * queryFinal = qb.and(query1, query2, query3, query4, query5, query6, - * query7); - * - * // create handle DOMHandle resultsHandle = new DOMHandle(); - * queryMgr.search(queryFinal, resultsHandle); - * - * // get the result Document resultDoc = resultsHandle.get(); - * - * assertXpathEvaluatesTo("1", - * "string(//*[local-name()='result'][last()]//@*[local-name()='index'])", - * resultDoc); assertXpathEvaluatesTo("Vannevar Bush", - * "string(//*[local-name()='result'][1]//*[local-name()='title'])", - * resultDoc); - * - * // release client client.release(); } - */ - - @Test - public void testSSLConnectionInvalidPort() throws KeyManagementException, NoSuchAlgorithmException, IOException, NoSuchAlgorithmException, KeyManagementException - { - System.out.println("Running testSSLConnectionInvalidPort"); - - String filename = "facebook-10443244874876159931"; - - // create a trust manager - // (note: a real application should verify certificates) - TrustManager naiveTrustMgr = new X509TrustManager() { - @Override - public void checkClientTrusted(X509Certificate[] chain, String authType) { - } - - @Override - public void checkServerTrusted(X509Certificate[] chain, String authType) { - } - - @Override - public X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[0]; - } - }; - - // create an SSL context - SSLContext sslContext = SSLContext.getInstance("SSLv3"); - sslContext.init(null, new TrustManager[] { naiveTrustMgr }, null); - - // create the client - // (note: a real application should use a COMMON, STRICT, or implemented - // hostname verifier) - SecurityContext secContext = newSecurityContext("rest-admin", "x").withSSLContext(sslContext, new X509TrustManager() { - - @Override - public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { - } - - @Override - public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { - } - - @Override - public X509Certificate[] getAcceptedIssuers() { - return null; - }}).withSSLHostnameVerifier(SSLHostnameVerifier.ANY); - DatabaseClient client = DatabaseClientFactory.newClient(appServerHostname, 8033, secContext, getConnType()); - - String expectedException = "com.sun.jersey.api.client.ClientHandlerException: org.apache.http.conn.HttpHostConnectException"; - String exception = ""; - - // write doc - try - { - writeDocumentUsingStringHandle(client, filename, "/write-text-doc/", "Text"); - } catch (Exception e) { - exception = e.toString(); - } - - assertTrue(exception.contains(expectedException)); - - // release client - client.release(); - } - - @Test - public void testSSLConnectionNonSSLServer() throws KeyManagementException, NoSuchAlgorithmException, IOException, NoSuchAlgorithmException, KeyManagementException - { - System.out.println("Running testSSLConnectionNonSSLServer"); - - String filename = "facebook-10443244874876159931"; - - // create a trust manager - // (note: a real application should verify certificates) - TrustManager naiveTrustMgr = new X509TrustManager() { - @Override - public void checkClientTrusted(X509Certificate[] chain, String authType) { - } - - @Override - public void checkServerTrusted(X509Certificate[] chain, String authType) { - } - - @Override - public X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[0]; - } - }; - - // create an SSL context - SSLContext sslContext = SSLContext.getInstance("SSLv3"); - sslContext.init(null, new TrustManager[] { naiveTrustMgr }, null); - - // create the client - // (note: a real application should use a COMMON, STRICT, or implemented - // hostname verifier) - SecurityContext secContext = newSecurityContext("rest-admin", "x").withSSLContext(sslContext, new X509TrustManager() { - - @Override - public void checkClientTrusted(X509Certificate[] arg0, String arg1) throws CertificateException { - } - - @Override - public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { - } - - @Override - public X509Certificate[] getAcceptedIssuers() { - return null; - }}).withSSLHostnameVerifier(SSLHostnameVerifier.ANY); - DatabaseClient client = DatabaseClientFactory.newClient(appServerHostname, 8014, secContext, getConnType()); - - String expectedException = "com.sun.jersey.api.client.ClientHandlerException: javax.net.ssl.SSLPeerUnverifiedException: peer not authenticated"; - String exception = ""; - - // write doc - try - { - writeDocumentUsingStringHandle(client, filename, "/write-text-doc/", "Text"); - } catch (Exception e) { - exception = e.toString(); - } - - assertEquals(expectedException, exception); - - // release client - client.release(); - } - - @Test - public void testSSLConnectionInvalidPassword() throws KeyManagementException, NoSuchAlgorithmException, IOException, NoSuchAlgorithmException, KeyManagementException - { - System.out.println("Running testSSLConnectionInvalidPassword"); - - String filename = "facebook-10443244874876159931"; - - // create a trust manager - // (note: a real application should verify certificates) - TrustManager naiveTrustMgr = new X509TrustManager() { - @Override - public void checkClientTrusted(X509Certificate[] chain, String authType) { - } - - @Override - public void checkServerTrusted(X509Certificate[] chain, String authType) { - } - - @Override - public X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[0]; - } - }; - - // create an SSL context - SSLContext sslContext = SSLContext.getInstance("SSLv3"); - sslContext.init(null, new TrustManager[] { naiveTrustMgr }, null); - - // create the client - // (note: a real application should use a COMMON, STRICT, or implemented - // hostname verifier) - SecurityContext secContext = newSecurityContext("rest-admin", "foo").withSSLContext(sslContext, new X509TrustManager() { - - @Override - public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { - } - - @Override - public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { - } - - @Override - public X509Certificate[] getAcceptedIssuers() { - return null; - }}).withSSLHostnameVerifier(SSLHostnameVerifier.ANY); - DatabaseClient client = DatabaseClientFactory.newClient(appServerHostname, 8012, secContext, getConnType()); - - String expectedException = "FailedRequestException: Local message: write failed: Unauthorized"; - String exception = ""; - - // write doc - try - { - writeDocumentUsingStringHandle(client, filename, "/write-text-doc/", "Text"); - } catch (Exception e) { - exception = e.toString(); - } - - System.out.println("Actual exception: " + exception); - boolean isExceptionThrown = exception.contains(expectedException); - - assertTrue(isExceptionThrown); - - // release client - client.release(); - } - - @Test - public void testSSLConnectionInvalidUser() throws KeyManagementException, NoSuchAlgorithmException, IOException, NoSuchAlgorithmException, KeyManagementException - { - System.out.println("Running testSSLConnectionInvalidUser"); - - String filename = "facebook-10443244874876159931"; - - // create a trust manager - // (note: a real application should verify certificates) - TrustManager naiveTrustMgr = new X509TrustManager() { - @Override - public void checkClientTrusted(X509Certificate[] chain, String authType) { - } - - @Override - public void checkServerTrusted(X509Certificate[] chain, String authType) { - } - - @Override - public X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[0]; - } - }; - - // create an SSL context - SSLContext sslContext = SSLContext.getInstance("SSLv3"); - sslContext.init(null, new TrustManager[] { naiveTrustMgr }, null); - - // create the client - // (note: a real application should use a COMMON, STRICT, or implemented - // hostname verifier) - SecurityContext secContext = newSecurityContext("MyFooUser", "x"); - secContext.withSSLContext(sslContext, new X509TrustManager() { - - @Override - public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { - } - - @Override - public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { - } - - @Override - public X509Certificate[] getAcceptedIssuers() { - return null; - }}).withSSLHostnameVerifier(SSLHostnameVerifier.ANY); - DatabaseClient client = DatabaseClientFactory.newClient(appServerHostname, 8012, secContext, getConnType()); - - String expectedException = "FailedRequestException: Local message: write failed: Unauthorized"; - String exception = ""; - - // write doc - try - { - writeDocumentUsingStringHandle(client, filename, "/write-text-doc/", "Text"); - } catch (Exception e) { - exception = e.toString(); - } - - boolean isExceptionThrown = exception.contains(expectedException); - - assertTrue(isExceptionThrown); - - // release client - client.release(); - } - - @AfterAll - public static void tearDown() throws Exception - { - System.out.println("In tear down"); - cleanupRESTServer(dbName, fNames); - } -} diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestSandBox.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestSandBox.java deleted file mode 100644 index d020b2488..000000000 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestSandBox.java +++ /dev/null @@ -1,1215 +0,0 @@ -/* - * Copyright © 2024 MarkLogic Corporation. All Rights Reserved. - */ - -package com.marklogic.client.functionaltest; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.marklogic.client.DatabaseClient; -import com.marklogic.client.DatabaseClientFactory; -import com.marklogic.client.DatabaseClientFactory.DigestAuthContext; -import com.marklogic.client.Transaction; -import com.marklogic.client.bitemporal.TemporalDocumentManager.ProtectionLevel; -import com.marklogic.client.document.*; -import com.marklogic.client.document.DocumentManager.Metadata; -import com.marklogic.client.expression.PlanBuilder; -import com.marklogic.client.expression.PlanBuilder.ModifyPlan; -import com.marklogic.client.io.*; -import com.marklogic.client.io.DocumentMetadataHandle.Capability; -import com.marklogic.client.io.DocumentMetadataHandle.DocumentCollections; -import com.marklogic.client.io.DocumentMetadataHandle.DocumentPermissions; -import com.marklogic.client.io.DocumentMetadataHandle.DocumentProperties; -import com.marklogic.client.pojo.PojoPage; -import com.marklogic.client.pojo.PojoRepository; -import com.marklogic.client.query.*; -import com.marklogic.client.row.RowManager; -import com.marklogic.client.semantics.GraphManager; -import com.marklogic.client.semantics.RDFMimeTypes; -import com.marklogic.client.type.CtsReferenceExpr; -import com.marklogic.client.type.XsStringSeqVal; -import org.custommonkey.xmlunit.exceptions.XpathException; -import org.junit.jupiter.api.*; -import org.w3c.dom.Document; -import org.xml.sax.SAXException; - -import jakarta.xml.bind.DatatypeConverter; -import javax.xml.datatype.DatatypeFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.TransformerException; -import java.io.File; -import java.io.IOException; -import java.security.KeyManagementException; -import java.security.NoSuchAlgorithmException; -import java.util.Calendar; -import java.util.HashMap; -import java.util.Map; - -import static org.custommonkey.xmlunit.XMLAssert.assertXpathEvaluatesTo; -import static org.junit.jupiter.api.Assertions.*; - -@Disabled("Ignored because it was previously ignored in build.gradle though without explanation") -public class TestSandBox extends BasicJavaClientREST { - - private static String dbName = "TestSandBox"; - private static String[] fNames = { "TestSandBox-1" }; - private static String schemadbName = "TestSandBoxSchemaDB"; - private static String[] schemafNames = { "TestSandBoxSchemaDB-1" }; - - private DatabaseClient writerClient = null; - private static GraphManager gmWriter; - - private final static String dateTimeDataTypeString = "dateTime"; - - private final static String systemStartERIName = "javaSystemStartERI"; - private final static String systemEndERIName = "javaSystemEndERI"; - private final static String validStartERIName = "javaValidStartERI"; - private final static String validEndERIName = "javaValidEndERI"; - - private final static String axisSystemName = "javaERISystemAxis"; - private final static String axisValidName = "javaERIValidAxis"; - - private final static String temporalCollectionName = "javaERITemporalCollection"; - private final static String bulktemporalCollectionName = "bulkjavaERITemporalCollection"; - private final static String temporalLsqtCollectionName = "javaERILsqtTemporalCollection"; - - private final static String systemNodeName = "System"; - private final static String validNodeName = "Valid"; - private final static String addressNodeName = "Address"; - private final static String uriNodeName = "uri"; - - private static String appServerHostname = null; - private static int restPort = 0; - private static String datasource = null; - - @BeforeAll - public static void setUpBeforeClass() throws Exception { - System.out.println("In setup"); - configureRESTServer(dbName, fNames); - appServerHostname = getRestAppServerHostName(); - restPort = getRestServerPort(); - datasource = getDataConfigDirPath() + "/data/optics/"; - - ConnectedRESTQA.addRangeElementIndex(dbName, dateTimeDataTypeString, "", - systemStartERIName); - ConnectedRESTQA.addRangeElementIndex(dbName, dateTimeDataTypeString, "", - systemEndERIName); - ConnectedRESTQA.addRangeElementIndex(dbName, dateTimeDataTypeString, "", - validStartERIName); - ConnectedRESTQA.addRangeElementIndex(dbName, dateTimeDataTypeString, "", - validEndERIName); - createDB(schemadbName); - createForest(schemafNames[0], schemadbName); - // Set the schemadbName database as the Schema database. - setDatabaseProperties(dbName, "schema-database", schemadbName); - addRangeElementAttributeIndex(dbName, "decimal", "http://cloudbank.com", - "price", "", "amt", "http://marklogic.com/collation/"); - - // Temporal axis must be created before temporal collection associated with - // those axes is created - ConnectedRESTQA.addElementRangeIndexTemporalAxis(dbName, axisSystemName, - "", systemStartERIName, "", systemEndERIName); - ConnectedRESTQA.addElementRangeIndexTemporalAxis(dbName, axisValidName, "", - validStartERIName, "", validEndERIName); - ConnectedRESTQA.addElementRangeIndexTemporalCollection(dbName, - temporalCollectionName, axisSystemName, axisValidName); - ConnectedRESTQA.addElementRangeIndexTemporalCollection(dbName, - bulktemporalCollectionName, axisSystemName, axisValidName); - ConnectedRESTQA.addElementRangeIndexTemporalCollection(dbName, - temporalLsqtCollectionName, axisSystemName, axisValidName); - ConnectedRESTQA.updateTemporalCollectionForLSQT(dbName, - temporalLsqtCollectionName, true); - - String[][] rangeElements = { - // { scalar-type, namespace-uri, localname, collation, - // range-value-positions, invalid-values } - // If there is a need to add additional fields, then add them to the end - // of each array - // and pass empty strings ("") into an array where the additional field - // does not have a value. - // For example : as in namespace, collections below. - // Add new RangeElementIndex as an array below. - { "string", "", "city", "http://marklogic.com/collation/", "false", - "reject" }, - { "int", "", "popularity", "", "false", "reject" }, - { "int", "", "id", "", "false", "reject" }, - { "double", "", "distance", "", "false", "reject" }, - { "date", "", "date", "", "false", "reject" }, - { "string", "", "cityName", "http://marklogic.com/collation/", "false", - "reject" }, - { "string", "", "cityTeam", "http://marklogic.com/collation/", "false", - "reject" }, { "long", "", "cityPopulation", "", "false", "reject" } }; - - // Insert the range indices - addRangeElementIndex(dbName, rangeElements); - - // Insert word lexicon. - ObjectMapper mapper = new ObjectMapper(); - ObjectNode mainNode = mapper.createObjectNode(); - ArrayNode childArray = mapper.createArrayNode(); - ObjectNode childNodeObject = mapper.createObjectNode(); - - childNodeObject.put("namespace-uri", ""); - childNodeObject.put("localname", "city"); - childNodeObject.put("collation", "http://marklogic.com/collation/"); - childArray.add(childNodeObject); - mainNode.withArray("element-word-lexicon").add(childArray); - - setDatabaseProperties(dbName, "element-word-lexicon", mainNode); - setupAppServicesGeoConstraint(dbName); - - // Add geo element index. - addGeospatialElementIndexes(dbName, "latLonPoint", "", "wgs84", "point", - false, "reject"); - // Enable triple index. - enableTripleIndex(dbName); - enableTrailingWildcardSearches(dbName); - // Enable collection lexicon. - enableCollectionLexicon(dbName); - // Enable uri lexicon. - setDatabaseProperties(dbName, "uri-lexicon", true); - // Create schema database - createDB(schemadbName); - createForest(schemafNames[0], schemadbName); - // Set the schemadbName database as the Schema database. - setDatabaseProperties(dbName, "schema-database", schemadbName); - - DatabaseClient schemaDBclient = DatabaseClientFactory.newClient( - appServerHostname, restPort, schemadbName, new DigestAuthContext( - "admin", "admin")); - - // You can enable the triple positions index for faster near searches using - // cts:triple-range-query. - DatabaseClient client = DatabaseClientFactory.newClient(appServerHostname, - restPort, new DigestAuthContext("admin", "admin")); - - // Install the TDE templates - // loadFileToDB(client, filename, docURI, collection, document format) - loadFileToDB(schemaDBclient, "masterDetail.tdex", - "/optic/view/test/masterDetail.tdex", "XML", - new String[] { "http://marklogic.com/xdmp/tde" }); - loadFileToDB(schemaDBclient, "masterDetail2.tdej", - "/optic/view/test/masterDetail2.tdej", "JSON", - new String[] { "http://marklogic.com/xdmp/tde" }); - loadFileToDB(schemaDBclient, "masterDetail3.tdej", - "/optic/view/test/masterDetail3.tdej", "JSON", - new String[] { "http://marklogic.com/xdmp/tde" }); - loadFileToDB(schemaDBclient, "masterDetail4.tdej", - "/optic/view/test/masterDetail4.tdej", "JSON", - new String[] { "http://marklogic.com/xdmp/tde" }); - - // Load XML data files. - loadFileToDB(client, "masterDetail.xml", - "/optic/view/test/masterDetail.xml", "XML", - new String[] { "/optic/view/test" }); - loadFileToDB(client, "playerTripleSet.xml", - "/optic/triple/test/playerTripleSet.xml", "XML", - new String[] { "/optic/player/triple/test" }); - loadFileToDB(client, "teamTripleSet.xml", - "/optic/triple/test/teamTripleSet.xml", "XML", - new String[] { "/optic/team/triple/test" }); - loadFileToDB(client, "otherPlayerTripleSet.xml", - "/optic/triple/test/otherPlayerTripleSet.xml", "XML", - new String[] { "/optic/other/player/triple/test" }); - loadFileToDB(client, "doc4.xml", "/optic/lexicon/test/doc4.xml", "XML", - new String[] { "/optic/lexicon/test" }); - loadFileToDB(client, "doc5.xml", "/optic/lexicon/test/doc5.xml", "XML", - new String[] { "/optic/lexicon/test" }); - - // Load JSON data files. - loadFileToDB(client, "masterDetail2.json", - "/optic/view/test/masterDetail2.json", "JSON", - new String[] { "/optic/view/test" }); - loadFileToDB(client, "masterDetail3.json", - "/optic/view/test/masterDetail3.json", "JSON", - new String[] { "/optic/view/test" }); - loadFileToDB(client, "masterDetail4.json", - "/optic/view/test/masterDetail4.json", "JSON", - new String[] { "/optic/view/test" }); - loadFileToDB(client, "masterDetail5.json", - "/optic/view/test/masterDetail5.json", "JSON", - new String[] { "/optic/view/test" }); - - loadFileToDB(client, "doc1.json", "/optic/lexicon/test/doc1.json", "JSON", - new String[] { "/other/coll1", "/other/coll2" }); - loadFileToDB(client, "doc2.json", "/optic/lexicon/test/doc2.json", "JSON", - new String[] { "/optic/lexicon/test" }); - loadFileToDB(client, "doc3.json", "/optic/lexicon/test/doc3.json", "JSON", - new String[] { "/optic/lexicon/test" }); - - loadFileToDB(client, "city1.json", "/optic/lexicon/test/city1.json", - "JSON", new String[] { "/optic/lexicon/test" }); - loadFileToDB(client, "city2.json", "/optic/lexicon/test/city2.json", - "JSON", new String[] { "/optic/lexicon/test" }); - loadFileToDB(client, "city3.json", "/optic/lexicon/test/city3.json", - "JSON", new String[] { "/optic/lexicon/test" }); - loadFileToDB(client, "city4.json", "/optic/lexicon/test/city4.json", - "JSON", new String[] { "/optic/lexicon/test" }); - loadFileToDB(client, "city5.json", "/optic/lexicon/test/city5.json", - "JSON", new String[] { "/optic/lexicon/test" }); - Thread.sleep(10000); - schemaDBclient.release(); - client.release(); - } - - @AfterAll - public static void tearDownAfterClass() throws Exception { - System.out.println("In tear down"); - - // Delete database first. Otherwise axis and collection cannot be deleted - cleanupRESTServer(dbName, fNames); - deleteRESTUser("eval-user"); - deleteRESTUser("eval-readeruser"); - deleteUserRole("test-eval"); - - // Temporal collection needs to be deleted before temporal axis associated - // with it can be deleted - ConnectedRESTQA.deleteElementRangeIndexTemporalCollection(dbName, - temporalLsqtCollectionName); - ConnectedRESTQA.deleteElementRangeIndexTemporalCollection(dbName, - temporalCollectionName); - ConnectedRESTQA.deleteElementRangeIndexTemporalCollection(dbName, - bulktemporalCollectionName); - ConnectedRESTQA.deleteElementRangeIndexTemporalAxis(dbName, axisValidName); - ConnectedRESTQA.deleteElementRangeIndexTemporalAxis(dbName, axisSystemName); - deleteDB(schemadbName); - deleteForest(schemafNames[0]); - } - - @BeforeEach - public void setUp() throws Exception { - createUserRolesWithPrevilages("test-eval", "xdbc:eval", "xdbc:eval-in", - "xdmp:eval-in", "any-uri", "xdbc:invoke", - "temporal:statement-set-system-time", "temporal-document-protect", - "temporal-document-wipe"); - - createRESTUser("eval-user", "x", "test-eval", "rest-admin", "rest-writer", - "rest-reader", "temporal-admin"); - createRESTUser("eval-readeruser", "x", "rest-reader"); - writerClient = getDatabaseClientOnDatabase(appServerHostname, restPort, - dbName, "eval-user", "x", getConnType()); - } - - @AfterEach - public void tearDown() throws Exception { - //clearDB(); - } - - public DocumentMetadataHandle setMetadata(boolean update) { - // create and initialize a handle on the meta-data - DocumentMetadataHandle metadataHandle = new DocumentMetadataHandle(); - - if (update) { - metadataHandle.getCollections().addAll("updateCollection"); - metadataHandle.getProperties().put("published", true); - - metadataHandle.getPermissions().add("app-user", Capability.UPDATE, - Capability.READ); - - metadataHandle.setQuality(99); - } else { - metadataHandle.getCollections().addAll("insertCollection"); - metadataHandle.getProperties().put("reviewed", true); - - metadataHandle.getPermissions().add("app-user", Capability.UPDATE, - Capability.READ, Capability.EXECUTE); - - metadataHandle.setQuality(11); - } - - metadataHandle.getProperties().put("myString", "foo"); - metadataHandle.getProperties().put("myInteger", 10); - metadataHandle.getProperties().put("myDecimal", 34.56678); - metadataHandle.getProperties().put("myCalendar", - Calendar.getInstance().get(Calendar.YEAR)); - - return metadataHandle; - } - - private JacksonDatabindHandle getJSONDocumentHandle( - String startValidTime, String endValidTime, String address, String uri) - throws Exception { - - // Setup for JSON document - /** - * - { "System": { systemStartERIName : "", systemEndERIName : "", }, "Valid": - * { validStartERIName: "2001-01-01T00:00:00", validEndERIName: - * "2011-12-31T23:59:59" }, "Address": "999 Skyway Park", "uri": - * "javaSingleDoc1.json" } - */ - - ObjectMapper mapper = new ObjectMapper(); - ObjectNode rootNode = mapper.createObjectNode(); - - // Set system time values - ObjectNode system = mapper.createObjectNode(); - - system.put(systemStartERIName, ""); - system.put(systemEndERIName, ""); - rootNode.set(systemNodeName, system); - - // Set valid time values - ObjectNode valid = mapper.createObjectNode(); - - valid.put(validStartERIName, startValidTime); - valid.put(validEndERIName, endValidTime); - rootNode.set(validNodeName, valid); - - // Set Address - rootNode.put(addressNodeName, address); - - // Set uri - rootNode.put(uriNodeName, uri); - - System.out.println(rootNode.toString()); - - JacksonDatabindHandle handle = new JacksonDatabindHandle<>( - ObjectNode.class).withFormat(Format.JSON); - handle.set(rootNode); - - return handle; - } - - @Test - /* - * Test bitemporal protections - NOUPDATE With transaction. - */ - public void testProtectUpdateInTransaction() throws Exception { - - System.out.println("Inside testProtectUpdateInTransaction"); - ConnectedRESTQA.updateTemporalCollectionForLSQT(dbName, - temporalLsqtCollectionName, true); - - Calendar insertTime = DatatypeConverter - .parseDateTime("2005-01-01T00:00:01"); - Calendar updateTime = DatatypeConverter - .parseDateTime("2005-01-01T00:00:11"); - - String docId = "javaSingleJSONDoc.json"; - JacksonDatabindHandle handle = getJSONDocumentHandle( - "2001-01-01T00:00:00", "2011-12-31T23:59:59", "999 Skyway Park - JSON", - docId); - - JSONDocumentManager docMgr = writerClient.newJSONDocumentManager(); - Transaction t1 = writerClient.openTransaction(); - Transaction t2 = null; - docMgr.write("javaSingleJSONDocV1.json", docId, null, handle, null, t1, - temporalLsqtCollectionName, insertTime); - - // Protect document for 30 sec from delete and update. Use Duration. - docMgr.protect(docId, temporalLsqtCollectionName, ProtectionLevel.NOUPDATE, - DatatypeFactory.newInstance().newDuration("PT30S"), t1); - JacksonDatabindHandle handleUpd = getJSONDocumentHandle( - "2003-01-01T00:00:00", "2008-12-31T23:59:59", - "1999 Skyway Park - Updated - JSON", docId); - StringBuilder str = new StringBuilder(); - try { - docMgr.write(docId, null, handleUpd, null, t1, - temporalLsqtCollectionName, updateTime); - } catch (Exception ex) { - str.append(ex.getMessage()); - System.out.println("Exception when update within 30 sec is " - + str.toString()); - } - assertTrue( - str.toString().contains( - "The document javaSingleJSONDoc.json is protected noUpdate")); - try { - // Sleep for 40 secs and try to update the same docId. - Thread.sleep(40000); - docMgr.write(docId, null, handleUpd, null, t1, - temporalLsqtCollectionName, updateTime); - Thread.sleep(5000); - - JSONDocumentManager jsonDocMgr = writerClient.newJSONDocumentManager(); - DocumentPage readResults = jsonDocMgr.read(t1, docId); - System.out.println("Number of results = " + readResults.size()); - assertEquals( 1, readResults.size()); - - QueryManager queryMgr = writerClient.newQueryManager(); - - StructuredQueryBuilder sqb = queryMgr.newStructuredQueryBuilder(); - StructuredQueryDefinition termQuery = sqb - .collection(temporalLsqtCollectionName); - t1.commit(); - - long start = 1; - t2 = writerClient.openTransaction(); - DocumentPage termQueryResults = docMgr.search(termQuery, start, t2); - System.out.println("Number of results = " - + termQueryResults.getTotalSize()); - assertEquals( 4, - termQueryResults.getTotalSize()); - } catch (Exception e) { - System.out.println("Exception when update within 30 sec is " - + e.getMessage()); - } finally { - if (t2 != null) - t2.rollback(); - writerClient.release(); - } - } - - @Test - public void testQueryByExampleXML() throws KeyManagementException, - NoSuchAlgorithmException, IOException, TransformerException, - XpathException { - System.out.println("Running testQueryByExampleXML"); - - String[] filenames = { "constraint1.xml", "constraint2.xml", - "constraint3.xml", "constraint4.xml", "constraint5.xml" }; - DatabaseClient client = null; - try { - - client = getDatabaseClient("rest-writer", "x", getConnType()); - - // write docs - for (String filename : filenames) { - writeDocumentUsingInputStreamHandle(client, filename, "/qbe/", "XML"); - } - - // get the combined query - File file = new File("src/test/java/com/marklogic/client/functionaltest/qbe/qbe1.xml"); - - String qbeQuery = convertFileToString(file); - StringHandle qbeHandle = new StringHandle(qbeQuery); - qbeHandle.setFormat(Format.XML); - - QueryManager queryMgr = client.newQueryManager(); - - RawQueryByExampleDefinition qbyex = queryMgr.newRawQueryByExampleDefinition(qbeHandle); - - Document resultDoc = queryMgr.search(qbyex, new DOMHandle()).get(); - - System.out.println("XML Result" + convertXMLDocumentToString(resultDoc)); - - assertXpathEvaluatesTo("1", - "string(//*[local-name()='result'][last()]//@*[local-name()='index'])", - resultDoc); - assertXpathEvaluatesTo("0011", - "string(//*[local-name()='result'][1]//*[local-name()='id'])", - resultDoc); - } - catch(Exception ex) { - System.out.println("Exceptions" + ex.getStackTrace()); - } - finally { - client.release(); - } - } - - @Test - public void testPointAndWord() throws KeyManagementException, - NoSuchAlgorithmException, IOException, ParserConfigurationException, - SAXException, XpathException, TransformerException { - System.out.println("Running testPointAndWord"); - - String queryOptionName = "geoConstraintOpt.xml"; - DatabaseClient client = null; - try { - - client = getDatabaseClient("rest-admin", "x", getConnType()); - - // write docs - for (int i = 1; i <= 9; i++) { - writeDocumentUsingInputStreamHandle(client, - "geo-constraint" + i + ".xml", "/geo-constraint/", "XML"); - } - - setQueryOption(client, queryOptionName); - - QueryManager queryMgr = client.newQueryManager(); - - // create query def - StringQueryDefinition querydef = queryMgr - .newStringDefinition(queryOptionName); - querydef.setCriteria("geo-elem:\"150,-140\" AND john"); - - // create handle - DOMHandle resultsHandle = new DOMHandle(); - queryMgr.search(querydef, resultsHandle); - - // get the result - Document resultDoc = resultsHandle.get(); - - assertXpathEvaluatesTo("1", - "string(//*[local-name()='result'][last()]//@*[local-name()='index'])", - resultDoc); - assertXpathEvaluatesTo("/geo-constraint/geo-constraint8.xml", - "string(//*[local-name()='result']//@*[local-name()='uri'])", resultDoc); - - } - catch(Exception ex) { - System.out.println("Exceptions" + ex.getStackTrace()); - } - finally { - client.release(); - } - } - - @Test - public void testSearchOnPropertiesBucketAndWord() - throws KeyManagementException, NoSuchAlgorithmException, IOException, - ParserConfigurationException, SAXException, XpathException, - TransformerException { - System.out.println("Running testSearchOnPropertiesBucketAndWord"); - - String filename1 = "property1.xml"; - String filename2 = "property2.xml"; - String filename3 = "property3.xml"; - String queryOptionName = "propertiesSearchWordOpt.xml"; - DatabaseClient client = null; - - try { - - client = getDatabaseClient("rest-admin", "x", getConnType()); - - // create and initialize a handle on the metadata - DocumentMetadataHandle metadataHandle1 = new DocumentMetadataHandle(); - DocumentMetadataHandle metadataHandle2 = new DocumentMetadataHandle(); - DocumentMetadataHandle metadataHandle3 = new DocumentMetadataHandle(); - - // set metadata properties - metadataHandle1.getProperties().put("popularity", 5); - metadataHandle2.getProperties().put("popularity", 9); - metadataHandle3.getProperties().put("popularity", 1); - metadataHandle1.getProperties().put("city", "Shanghai is a good one"); - metadataHandle2.getProperties().put("city", "Tokyo is hot in the summer"); - metadataHandle3.getProperties().put("city", - "The food in Seoul is similar in Shanghai"); - - // write docs - writeDocumentUsingInputStreamHandle(client, filename1, - "/properties-search/", metadataHandle1, "XML"); - writeDocumentUsingInputStreamHandle(client, filename2, - "/properties-search/", metadataHandle2, "XML"); - writeDocumentUsingInputStreamHandle(client, filename3, - "/properties-search/", metadataHandle3, "XML"); - - setQueryOption(client, queryOptionName); - - QueryManager queryMgr = client.newQueryManager(); - - // create query def - StringQueryDefinition querydef = queryMgr - .newStringDefinition(queryOptionName); - querydef.setCriteria("pop:medium AND city-property:Shanghai"); - - // create handle - DOMHandle resultsHandle = new DOMHandle(); - queryMgr.search(querydef, resultsHandle); - - // get the result - Document resultDoc = resultsHandle.get(); - - assertXpathEvaluatesTo("1", - "string(//*[local-name()='result'][last()]//@*[local-name()='index'])", - resultDoc); - assertXpathEvaluatesTo("/properties-search/property1.xml", - "string(//*[local-name()='result']//@*[local-name()='uri'])", resultDoc); - - } - catch(Exception ex) { - System.out.println("Exceptions" + ex.getStackTrace()); - } - finally { - client.release(); - } - } - - /* - * Checks for cts queries with options on fromLexicons TEST 14 - */ - @Test - public void testCtsQueriesWithOptions() throws KeyManagementException, - NoSuchAlgorithmException, IOException, SAXException, - ParserConfigurationException { - System.out.println("In testCtsQueriesWithOptions method"); - DatabaseClient client = null; - - try { - // Create a new Plan. - client = getDatabaseClient("rest-admin", "x", getConnType()); - RowManager rowMgr = client.newRowManager(); - PlanBuilder p = rowMgr.newPlanBuilder(); - Map index1 = new HashMap(); - index1.put("uri1", p.cts.uriReference()); - index1.put("city", p.cts.jsonPropertyReference("city")); - index1.put("popularity", p.cts.jsonPropertyReference("popularity")); - index1.put("date", p.cts.jsonPropertyReference("date")); - index1.put("distance", p.cts.jsonPropertyReference("distance")); - index1.put("point", p.cts.jsonPropertyReference("latLonPoint")); - - Map index2 = new HashMap(); - index2.put("uri2", p.cts.uriReference()); - index2.put("cityName", p.cts.jsonPropertyReference("cityName")); - index2.put("cityTeam", p.cts.jsonPropertyReference("cityTeam")); - - // plan1 - fromLexicons - ModifyPlan plan1 = p.fromLexicons(index1, "myCity", - p.fragmentIdCol("fragId1")); - // plan2 - fromLexicons - ModifyPlan plan2 = p.fromLexicons(index2, "myTeam", - p.fragmentIdCol("fragId2")); - - XsStringSeqVal propertyName = p.xs.string("city"); - XsStringSeqVal value = p.xs.string("*k"); - - XsStringSeqVal options = p.xs.stringSeq("wildcarded", "case-sensitive"); - - // ModifyPlan output = plan1.where(p.cts.jsonPropertyWordQuery(propertyName, - // value, options)) - ModifyPlan output = plan1 - .where( - p.cts.jsonPropertyWordQuery("city", "*k", "wildcarded", - "case-sensitive")).joinInner(plan2) - .where(p.eq(p.viewCol("myCity", "city"), p.col("cityName"))) - .orderBy(p.asc(p.col("date"))); - - JacksonHandle jacksonHandle = new JacksonHandle(); - jacksonHandle.setMimetype("application/json"); - - rowMgr.resultDoc(output, jacksonHandle); - JsonNode jsonResults = jacksonHandle.get(); - - JsonNode jsonBindingsNodes = jsonResults.path("rows"); - assertTrue( - 1 == jsonBindingsNodes.size()); - assertEquals( "new york", - jsonBindingsNodes.path(0).path("myCity.city").path("value").asText()); - } - catch(Exception ex) { - System.out.println("Exceptions" + ex.getStackTrace()); - } - finally { - client.release(); - } - } - - @Test - public void testWriteMultiJSONFilesDefaultMetadata() throws KeyManagementException, NoSuchAlgorithmException, Exception - { - DatabaseClient client = null; - try { - client = getDatabaseClient("rest-admin", "x", getConnType()); - String docId[] = { "/original.json", "/updated.json", "/constraint1.json" }; - String jsonFilename1 = "json-original.json"; - String jsonFilename2 = "json-updated.json"; - String jsonFilename3 = "constraint1.json"; - - File jsonFile1 = new File("src/test/java/com/marklogic/client/functionaltest/data/" + jsonFilename1); - File jsonFile2 = new File("src/test/java/com/marklogic/client/functionaltest/data/" + jsonFilename2); - File jsonFile3 = new File("src/test/java/com/marklogic/client/functionaltest/data/" + jsonFilename3); - - JSONDocumentManager docMgr = client.newJSONDocumentManager(); - docMgr.setMetadataCategories(Metadata.ALL); - DocumentWriteSet writeset = docMgr.newWriteSet(); - // put meta-data - DocumentMetadataHandle mh = setMetadata(); - DocumentMetadataHandle mhRead = new DocumentMetadataHandle(); - - ObjectMapper mapper = new ObjectMapper(); - - JacksonHandle jacksonHandle1 = new JacksonHandle(); - JacksonHandle jacksonHandle2 = new JacksonHandle(); - JacksonHandle jacksonHandle3 = new JacksonHandle(); - - JsonNode originalNode = mapper.readTree(jsonFile1); - jacksonHandle1.set(originalNode); - jacksonHandle1.withFormat(Format.JSON); - - JsonNode updatedNode = mapper.readTree(jsonFile2); - jacksonHandle2.set(updatedNode); - jacksonHandle2.withFormat(Format.JSON); - - JsonNode constraintNode = mapper.readTree(jsonFile3); - jacksonHandle3.set(constraintNode); - jacksonHandle3.withFormat(Format.JSON); - - writeset.addDefault(mh); - writeset.add(docId[0], jacksonHandle1); - writeset.add(docId[1], jacksonHandle2); - writeset.add(docId[2], jacksonHandle3); - - docMgr.write(writeset); - - DocumentPage page = docMgr.read(docId); - - while (page.hasNext()) { - DocumentRecord rec = page.next(); - docMgr.readMetadata(rec.getUri(), mhRead); - System.out.println(rec.getUri()); - validateMetadata(mhRead); - } - validateMetadata(mhRead); - mhRead = null; - } - catch(Exception ex) { - System.out.println("Exceptions" + ex.getStackTrace()); - } - finally { - client.release(); - } - } - - /* - * Write Triples of Type JSON Merge NTriples into the same graph and validate - * mergeGraphs with transactions. - * - * Merge within same write transaction Write and merge Triples within - * different transactions. Commit the merge transaction Write and merge - * Triples within different transactions. Rollback the merge transaction Write - * and merge Triples within different transactions. Rollback the merge - * transaction and then commit. - */ - @Test - public void testMergeGraphWithTransaction() throws InterruptedException, - KeyManagementException, NoSuchAlgorithmException, IOException { - String uri = "http://test.sem.quads/json-quads"; - DatabaseClient writerClient = getDatabaseClientOnDatabase( - appServerHostname, restPort, dbName, "rest-writer", "x", - getConnType()); - Transaction trxIn = writerClient.openTransaction(); - gmWriter = writerClient.newGraphManager(); - Transaction trxInMergeGraph = null; - Transaction trxDelIn = null; - try { - String ntriple6 = " ."; - File file = new File("src/test/java/com/marklogic/client/functionaltest/data/semantics/bug25348.json"); - FileHandle filehandle = new FileHandle(); - filehandle.set(file); - - // Using client write and merge Triples within same transaction. - gmWriter.write(uri, filehandle.withMimetype(RDFMimeTypes.RDFJSON), trxIn); - // Merge Graphs inside the transaction. - gmWriter.mergeGraphs( - new StringHandle(ntriple6).withMimetype(RDFMimeTypes.NQUADS), trxIn); - trxIn.commit(); - FileHandle handle = gmWriter.read(uri, new FileHandle()); - File readFile = handle.get(); - String expectedContent = convertFileToString(readFile); - assertTrue( - expectedContent.contains(" products = client.newPojoRepository(Artifact.class, Long.class); - PojoPage p; - this.loadSimplePojos(products); - QueryManager queryMgr = client.newQueryManager(); - StringQueryDefinition qd = queryMgr.newStringDefinition(); - qd.setCriteria("cogs"); - JacksonHandle results = new JacksonHandle(); - p = products.search(qd, 1, results); - products.setPageLength(11); - assertEquals( 3, p.getTotalPages()); - // System.out.println(p.getTotalPages()+results.get().toString()); - long pageNo = 1, count = 0; - do { - count = 0; - p = products.search(qd, pageNo, results); - - while (p.iterator().hasNext()) { - Artifact a = p.iterator().next(); - validateArtifact(a); - count++; - // System.out.println(a.getId()+" "+a.getManufacturer().getName() - // +" "+count); - } - assertEquals( count, p.size()); - pageNo = pageNo + p.getPageSize(); - - assertEquals( results.get().get("start").asLong(), p.getStart()); - assertEquals( "json", results.get().withArray("results").get(1).path("format").asText()); - assertTrue( results.get().withArray("results").get(1).path("uri").asText().contains("Artifact")); - // System.out.println(results.get().toString()); - } while (!p.isLastPage() && pageNo < p.getTotalSize()); - // assertTrue(results.get().has("metrics")); - assertEquals( "cogs", results.get().path("qtext").asText()); - assertEquals( 110, results.get().get("total").asInt()); - assertEquals( 10, p.getPageNumber()); - assertEquals( 10, p.getTotalPages()); - } - catch(Exception ex) { - System.out.println("Exceptions" + ex.getStackTrace()); - } - finally { - client.release(); - } - } - - /** - * Write document using DOMHandle - * - * @param client - * @param filename - * @param uri - * @param type - * @throws IOException - * @throws ParserConfigurationException - * @throws SAXException - */ - - public static void loadFileToDB(DatabaseClient client, String filename, - String uri, String type, String[] collections) throws IOException, - ParserConfigurationException, SAXException { - // create doc manager - DocumentManager docMgr = null; - docMgr = documentMgrSelector(client, docMgr, type); - - File file = new File(datasource + filename); - // create a handle on the content - FileHandle handle = new FileHandle(file); - handle.set(file); - - DocumentMetadataHandle metadataHandle = new DocumentMetadataHandle(); - for (String coll : collections) - metadataHandle.getCollections().addAll(coll.toString()); - - // write the document content - DocumentWriteSet writeset = docMgr.newWriteSet(); - writeset.addDefault(metadataHandle); - writeset.add(uri, handle); - - docMgr.write(writeset); - - System.out.println("Write " + uri + " to database"); - } - - /** - * Function to select and create document manager based on the type - * - * @param client - * @param docMgr - * @param type - * @return - */ - public static DocumentManager documentMgrSelector(DatabaseClient client, - DocumentManager docMgr, String type) { - // create doc manager - switch (type) { - case "XML": - docMgr = client.newXMLDocumentManager(); - break; - case "Text": - docMgr = client.newTextDocumentManager(); - break; - case "JSON": - docMgr = client.newJSONDocumentManager(); - break; - case "Binary": - docMgr = client.newBinaryDocumentManager(); - break; - case "JAXB": - docMgr = client.newXMLDocumentManager(); - break; - default: - System.out.println("Invalid type"); - break; - } - return docMgr; - } - - public DocumentMetadataHandle setMetadata() { - // create and initialize a handle on the meta-data - DocumentMetadataHandle metadataHandle = new DocumentMetadataHandle(); - metadataHandle.getCollections().addAll("my-collection1", "my-collection2"); - metadataHandle.getPermissions().add("app-user", Capability.UPDATE, Capability.READ); - metadataHandle.getProperties().put("reviewed", true); - metadataHandle.getProperties().put("myString", "foo"); - metadataHandle.getProperties().put("myInteger", 10); - metadataHandle.getProperties().put("myDecimal", 34.56678); - metadataHandle.getProperties().put("myCalendar", Calendar.getInstance().get(Calendar.YEAR)); - metadataHandle.setQuality(23); - return metadataHandle; - } - - public void validateMetadata(DocumentMetadataHandle mh) { - // get meta-data values - DocumentProperties properties = mh.getProperties(); - DocumentPermissions permissions = mh.getPermissions(); - DocumentCollections collections = mh.getCollections(); - - // Properties - // String expectedProperties = - // "size:5|reviewed:true|myInteger:10|myDecimal:34.56678|myCalendar:2014|myString:foo|"; - String actualProperties = getDocumentPropertiesString(properties); - boolean result = actualProperties.contains("size:5|"); - assertTrue( result); - - // Permissions - String actualPermissions = getDocumentPermissionsString(permissions); - System.out.println(actualPermissions); - - assertTrue( actualPermissions.contains("size:5")); - // assertTrue( - // actualPermissions.contains("flexrep-eval:[READ]")); - assertTrue( actualPermissions.contains("rest-reader:[READ]")); - assertTrue( actualPermissions.contains("rest-writer:[UPDATE]")); - assertTrue( - (actualPermissions.contains("app-user:[UPDATE, READ]") || actualPermissions.contains("app-user:[READ, UPDATE]"))); - - // Collections - String actualCollections = getDocumentCollectionsString(collections); - System.out.println(collections); - - assertTrue( actualCollections.contains("size:2")); - assertTrue( actualCollections.contains("my-collection1")); - assertTrue( actualCollections.contains("my-collection2")); - } - - public void validateArtifact(Artifact art) - { - assertNotNull(art); - assertNotNull(art.id); - assertTrue( art.getInventory() > 1000); - } - - public void loadSimplePojos(PojoRepository products) - { - for (int i = 1; i < 111; i++) { - if (i % 2 == 0) { - products.write(this.getArtifact(i), "even", "numbers"); - } - else { - products.write(this.getArtifact(i), "odd", "numbers"); - } - } - } - - public Artifact getArtifact(int counter) { - - Artifact cogs = new Artifact(); - cogs.setId(counter); - if (counter % 5 == 0) { - cogs.setName("Cogs special"); - if (counter % 2 == 0) { - Company acme = new Company(); - acme.setName("Acme special, Inc."); - acme.setWebsite("http://www.acme special.com"); - acme.setLatitude(41.998 + counter); - acme.setLongitude(-87.966 + counter); - cogs.setManufacturer(acme); - - } else { - Company widgets = new Company(); - widgets.setName("Widgets counter Inc."); - widgets.setWebsite("http://www.widgets counter.com"); - widgets.setLatitude(41.998 + counter); - widgets.setLongitude(-87.966 + counter); - cogs.setManufacturer(widgets); - } - } else { - cogs.setName("Cogs " + counter); - if (counter % 2 == 0) { - Company acme = new Company(); - acme.setName("Acme " + counter + ", Inc."); - acme.setWebsite("http://www.acme" + counter + ".com"); - acme.setLatitude(41.998 + counter); - acme.setLongitude(-87.966 + counter); - cogs.setManufacturer(acme); - - } else { - Company widgets = new Company(); - widgets.setName("Widgets " + counter + ", Inc."); - widgets.setWebsite("http://www.widgets" + counter + ".com"); - widgets.setLatitude(41.998 + counter); - widgets.setLongitude(-87.966 + counter); - cogs.setManufacturer(widgets); - } - } - cogs.setInventory(1000 + counter); - return cogs; - } -} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromSearchDocsWithOptionsTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromSearchDocsWithOptionsTest.java index e340ceaf4..56d28394d 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromSearchDocsWithOptionsTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromSearchDocsWithOptionsTest.java @@ -31,7 +31,6 @@ void bm25() { assertEquals(2, rows.size()); } - @Disabled("Waiting on fix for MLE-16147.") @Test void bm25ViaSearchOptions() { final String combinedQuery = "" + diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/RowManagerTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/RowManagerTest.java index 3e42c3251..36e86899f 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/RowManagerTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/RowManagerTest.java @@ -1470,7 +1470,7 @@ public void testSampleByNoArg() throws IOException { } @Test - @Disabled("Waiting on a fix for https://bugtrack.marklogic.com/58233") + @Disabled("Waiting on a fix for MLE-241") public void testBug58233() { RowManager rowMgr = Common.client.newRowManager(); From b59518cab37e3772ae4f4220a678b3696e5b992e Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 27 Nov 2024 13:40:31 -0500 Subject: [PATCH 29/39] MLE-17235 Removed weight from shortest-path Not yet supported by the server. --- .../fastfunctest/TestOpticOnTriples.java | 68 ------------------- .../client/expression/PlanBuilder.java | 22 ------ .../client/impl/PlanBuilderImpl.java | 20 ------ 3 files changed, 110 deletions(-) diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestOpticOnTriples.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestOpticOnTriples.java index 271f29179..775ca884a 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestOpticOnTriples.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestOpticOnTriples.java @@ -1487,72 +1487,4 @@ public void testShortestPathWithStringInputs() assertEquals("1",path.path("length").path("value").toString()); } } - - // TODO: Enable testShortestPathWithWeight and testShortestPathWithWeightColumn tests after server code starts - // accepting weight column and string. - @Disabled - @Test - public void testShortestPathWithWeight() - { - if(!isML12OrHigher){ - return; - } - - RowManager rowMgr = client.newRowManager(); - PlanBuilder p = rowMgr.newPlanBuilder(); - - PlanColumn teamIdCol = p.col("player_team"); - PlanColumn teamNameCol = p.col("team_name"); - PlanColumn teamCityCol = p.col("team_city"); - - PlanPrefixer team = p.prefixer("http://marklogic.com/mlb/team/"); - PlanBuilder.ModifyPlan team_plan = p.fromTriples( - p.pattern(teamIdCol, team.iri("name"), teamNameCol), - p.pattern(teamIdCol, team.iri("city"), teamCityCol) - ).shortestPath("team_name", "team_city", "path", "length", "weight"); - JacksonHandle jacksonHandle = new JacksonHandle().withMimetype("application/json"); - rowMgr.resultDoc(team_plan, jacksonHandle); - JsonNode jsonResults = jacksonHandle.get(); - JsonNode jsonBindingsNodes = jsonResults.path("rows"); - for (int i=0; i Date: Thu, 5 Dec 2024 12:05:52 -0500 Subject: [PATCH 30/39] Added support for zero and random --- gradle.properties | 2 +- .../client/type/PlanSearchOptions.java | 37 ++++++++++++++----- .../rows/FromSearchDocsWithOptionsTest.java | 33 ++++++++++++----- 3 files changed, 52 insertions(+), 20 deletions(-) diff --git a/gradle.properties b/gradle.properties index ed8022bc0..d2af75728 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ group=com.marklogic -version=7.0-SNAPSHOT +version=7.1-SNAPSHOT describedName=MarkLogic Java Client API publishUrl=file:../marklogic-java/releases diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanSearchOptions.java b/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanSearchOptions.java index ef07719c6..197163c84 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanSearchOptions.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanSearchOptions.java @@ -8,31 +8,50 @@ * for a row pipeline. */ public interface PlanSearchOptions { + /** * Changed in release 7.0.0 to return a float, as the server requires a float and throws an error on a double. */ XsFloatVal getQualityWeight(); - ScoreMethod getScoreMethod(); + + ScoreMethod getScoreMethod(); + /** * @since 7.0.0; requires MarkLogic 12 or higher. */ XsDoubleVal getBm25LengthWeight(); + /** * Changed in release 7.0.0 to return a float, as the server requires a float and throws an error on a double. */ - PlanSearchOptions withQualityWeight(float qualityWeight); + PlanSearchOptions withQualityWeight(float qualityWeight); + /** * Changed in release 7.0.0 to return a float, as the server requires a float and throws an error on a double. */ - PlanSearchOptions withQualityWeight(XsFloatVal qualityWeight); - PlanSearchOptions withScoreMethod(ScoreMethod scoreMethod); + PlanSearchOptions withQualityWeight(XsFloatVal qualityWeight); + + PlanSearchOptions withScoreMethod(ScoreMethod scoreMethod); + /** * @since 7.0.0; requires MarkLogic 12 or higher. */ PlanSearchOptions withBm25LengthWeight(double bm25LengthWeight); - enum ScoreMethod { - LOGTFIDF, LOGTF, SIMPLE, BM25; - // zero and random aren't in the 12 EA release. - //ZERO, RANDOM; - } + + enum ScoreMethod { + LOGTFIDF, + LOGTF, + SIMPLE, + BM25, + + /** + * @since 7.1.0; requires MarkLogic 12 EA2 or higher. + */ + ZERO, + + /** + * @since 7.1.0; requires MarkLogic 12 EA2 or higher. + */ + RANDOM; + } } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromSearchDocsWithOptionsTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromSearchDocsWithOptionsTest.java index 56d28394d..120e5c2db 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromSearchDocsWithOptionsTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromSearchDocsWithOptionsTest.java @@ -8,25 +8,39 @@ import com.marklogic.client.test.Common; import com.marklogic.client.test.junit5.RequiresML12; import com.marklogic.client.type.PlanSearchOptions; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; +/** + * These tests do not attempt to verify that a score method produces a particular ordering. Instead, they verify + * that each score method option is accepted by the server. + */ @ExtendWith(RequiresML12.class) class FromSearchDocsWithOptionsTest extends AbstractOpticUpdateTest { - @Test - void bm25() { -// Note that this does not actually test that the scoring is correct. -// It only tests that including the BM25 scoring option and a valid bm25LengthWeight do not cause any problems. + @BeforeEach + void setupTest() { rowManager.withUpdate(false); + } + + @ValueSource(strings = {"bm25", "zero", "random", "simple", "logtfidf", "logtf"}) + @ParameterizedTest + void scoreMethod(String scoreMethod) { PlanSearchOptions options = op.searchOptions() - .withScoreMethod(PlanSearchOptions.ScoreMethod.BM25) - .withBm25LengthWeight(0.25); + .withScoreMethod(PlanSearchOptions.ScoreMethod.valueOf(scoreMethod.toUpperCase())); + + if ("bm25".equalsIgnoreCase(scoreMethod)) { + options.withBm25LengthWeight(0.25); + } + List rows = resultRows(op.fromSearchDocs(op.cts.wordQuery("saxophone"), null, options)); assertEquals(2, rows.size()); } @@ -47,10 +61,9 @@ void bm25ViaSearchOptions() { @Test void qualityWeight() { -// Note that this does not actually test that the scoring is correct. -// It only tests that including a valid qualityWeight value does not cause any problems. - rowManager.withUpdate(false); - PlanSearchOptions options = op.searchOptions().withScoreMethod(PlanSearchOptions.ScoreMethod.LOGTFIDF).withQualityWeight(0.75F); + PlanSearchOptions options = op.searchOptions() + .withScoreMethod(PlanSearchOptions.ScoreMethod.LOGTFIDF) + .withQualityWeight(0.75F); List rows = resultRows(op.fromSearchDocs(op.cts.wordQuery("saxophone"), null, options)); assertEquals(2, rows.size()); } From 97091827734b5c3483fa0ac14c372f330cd3b408 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 20 Dec 2024 08:44:48 -0500 Subject: [PATCH 31/39] MLE-18458 Added support for try-with-resource --- .../com/marklogic/client/DatabaseClient.java | 13 +- .../client/test/DatabaseClientTest.java | 217 +++++++++--------- 2 files changed, 124 insertions(+), 106 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClient.java b/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClient.java index 18fe271d1..25735575f 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClient.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClient.java @@ -3,6 +3,7 @@ */ package com.marklogic.client; +import java.io.Closeable; import java.io.OutputStream; import java.io.Serializable; @@ -28,7 +29,7 @@ * A Database Client instantiates document and query managers and other objects * with shared access to a database. */ -public interface DatabaseClient { +public interface DatabaseClient extends Closeable { /** * Identifies whether the client connects directly to MarkLogic (the default) or * by means of a gateway such as a load balancer. @@ -241,4 +242,14 @@ static public interface ConnectionResult { String getDatabase(); SecurityContext getSecurityContext(); + + /** + * Overridden from the {@code Closeable} interface so that a user doesn't have to deal with a checked + * IOException. + * + * @since 7.1.0 + */ + default void close() { + release(); + } } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/DatabaseClientTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/DatabaseClientTest.java index d78ea1740..03e652d7f 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/DatabaseClientTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/DatabaseClientTest.java @@ -3,14 +3,11 @@ */ package com.marklogic.client.test; +import com.marklogic.client.DatabaseClient; import com.marklogic.client.DatabaseClient.ConnectionResult; import com.marklogic.client.admin.QueryOptionsManager; import com.marklogic.client.alerting.RuleManager; -import com.marklogic.client.document.BinaryDocumentManager; -import com.marklogic.client.document.GenericDocumentManager; -import com.marklogic.client.document.JSONDocumentManager; -import com.marklogic.client.document.TextDocumentManager; -import com.marklogic.client.document.XMLDocumentManager; +import com.marklogic.client.document.*; import com.marklogic.client.eval.ServerEvaluationCall; import com.marklogic.client.pojo.PojoRepository; import com.marklogic.client.query.QueryManager; @@ -19,105 +16,115 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class DatabaseClientTest { - @BeforeAll - public static void beforeClass() { - Common.connect(); - Common.connectRestAdmin(); - } - @AfterAll - public static void afterClass() { - } - - @Test - public void testNewDocument() { - GenericDocumentManager doc = Common.client.newDocumentManager(); - assertNotNull( doc); - } - - @Test - public void testNewBinaryDocument() { - BinaryDocumentManager doc = Common.client.newBinaryDocumentManager(); - assertNotNull( doc); - } - - @Test - public void testNewJSONDocument() { - JSONDocumentManager doc = Common.client.newJSONDocumentManager(); - assertNotNull( doc); - } - - @Test - public void testNewTextDocument() { - TextDocumentManager doc = Common.client.newTextDocumentManager(); - assertNotNull( doc); - } - - @Test - public void testNewXMLDocument() { - XMLDocumentManager doc = Common.client.newXMLDocumentManager(); - assertNotNull( doc); - } - - @Test - public void testNewLogger() { - RequestLogger logger = Common.client.newLogger(System.out); - assertNotNull( logger); - } - - @Test - public void testNewQueryManager() { - QueryManager mgr = Common.client.newQueryManager(); - assertNotNull( mgr); - } - - @Test - public void testNewRuleManager() { - RuleManager mgr = Common.client.newRuleManager(); - assertNotNull( mgr); - } - - @Test - public void testNewPojoRepository() { - PojoRepository mgr = Common.client.newPojoRepository(City.class, Integer.class); - assertNotNull( mgr); - } - - @Test - public void testNewServerEvaluationCall() { - ServerEvaluationCall mgr = Common.client.newServerEval(); - assertNotNull( mgr); - } - - @Test - public void testNewQueryOptionsManager() { - QueryOptionsManager mgr = Common.restAdminClient.newServerConfigManager().newQueryOptionsManager(); - assertNotNull( mgr); - } - - @Test - public void testGetClientImplementationObject() { - Object impl = Common.client.getClientImplementation(); - assertNotNull( impl); - assertTrue( impl instanceof okhttp3.OkHttpClient); - } - - @Test - public void testCheckConnectionWithValidUser() { - ConnectionResult connResult = Common.newClient().checkConnection(); - assertTrue(connResult.isConnected()); - } - - @Test - public void testCheckConnectionWithInvalidUser() { - ConnectionResult connResult = Common.newClientBuilder().withUsername("invalid").withPassword("invalid").build().checkConnection(); - assertFalse(connResult.isConnected()); - assertTrue(connResult.getStatusCode() == 401); - assertTrue(connResult.getErrorMessage().equalsIgnoreCase("Unauthorized")); - } +import static org.junit.jupiter.api.Assertions.*; + +class DatabaseClientTest { + + @BeforeAll + public static void beforeClass() { + Common.connect(); + Common.connectRestAdmin(); + } + + @AfterAll + public static void afterClass() { + } + + @Test + void tryWithResource() { + try (DatabaseClient client = Common.newClient()) { + ConnectionResult result = client.checkConnection(); + assertTrue(result.isConnected(), "This test is ensuring that a DatabaseClient, as of 7.1.0, can " + + "be used in a try-with-resources block without any error being thrown. We don't have a way of " + + "verifying that release() is actually called on the client though."); + } + } + + @Test + public void testNewDocument() { + GenericDocumentManager doc = Common.client.newDocumentManager(); + assertNotNull(doc); + } + + @Test + public void testNewBinaryDocument() { + BinaryDocumentManager doc = Common.client.newBinaryDocumentManager(); + assertNotNull(doc); + } + + @Test + public void testNewJSONDocument() { + JSONDocumentManager doc = Common.client.newJSONDocumentManager(); + assertNotNull(doc); + } + + @Test + public void testNewTextDocument() { + TextDocumentManager doc = Common.client.newTextDocumentManager(); + assertNotNull(doc); + } + + @Test + public void testNewXMLDocument() { + XMLDocumentManager doc = Common.client.newXMLDocumentManager(); + assertNotNull(doc); + } + + @Test + public void testNewLogger() { + RequestLogger logger = Common.client.newLogger(System.out); + assertNotNull(logger); + } + + @Test + public void testNewQueryManager() { + QueryManager mgr = Common.client.newQueryManager(); + assertNotNull(mgr); + } + + @Test + public void testNewRuleManager() { + RuleManager mgr = Common.client.newRuleManager(); + assertNotNull(mgr); + } + + @Test + public void testNewPojoRepository() { + PojoRepository mgr = Common.client.newPojoRepository(City.class, Integer.class); + assertNotNull(mgr); + } + + @Test + public void testNewServerEvaluationCall() { + ServerEvaluationCall mgr = Common.client.newServerEval(); + assertNotNull(mgr); + } + + @Test + public void testNewQueryOptionsManager() { + QueryOptionsManager mgr = Common.restAdminClient.newServerConfigManager().newQueryOptionsManager(); + assertNotNull(mgr); + } + + @Test + public void testGetClientImplementationObject() { + Object impl = Common.client.getClientImplementation(); + assertNotNull(impl); + assertTrue(impl instanceof okhttp3.OkHttpClient); + } + + @Test + public void testCheckConnectionWithValidUser() { + ConnectionResult connResult = Common.newClient().checkConnection(); + assertTrue(connResult.isConnected()); + } + + @Test + public void testCheckConnectionWithInvalidUser() { + ConnectionResult connResult = Common.newClientBuilder().withUsername("invalid").withPassword("invalid").build().checkConnection(); + assertFalse(connResult.isConnected()); + assertTrue(connResult.getStatusCode() == 401); + assertTrue(connResult.getErrorMessage().equalsIgnoreCase("Unauthorized")); + } } From b13710e45edd55ace849acc3bed44a8b76632f38 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 3 Jan 2025 09:52:36 -0500 Subject: [PATCH 32/39] Optimizing imports and formatting No functional changes. This is to prevent future PRs from having a bunch of formatting changes. --- .../marklogic/client/impl/OkHttpServices.java | 12933 ++++++++-------- 1 file changed, 6447 insertions(+), 6486 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java index 3b4fd88b7..120fbd600 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java @@ -3,71 +3,47 @@ */ package com.marklogic.client.impl; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.marklogic.client.*; import com.marklogic.client.DatabaseClient.ConnectionResult; import com.marklogic.client.DatabaseClientFactory.DigestAuthContext; import com.marklogic.client.DatabaseClientFactory.SecurityContext; import com.marklogic.client.bitemporal.TemporalDescriptor; import com.marklogic.client.bitemporal.TemporalDocumentManager.ProtectionLevel; -import com.marklogic.client.document.ContentDescriptor; -import com.marklogic.client.document.DocumentDescriptor; +import com.marklogic.client.document.*; import com.marklogic.client.document.DocumentManager.Metadata; -import com.marklogic.client.document.DocumentPage; -import com.marklogic.client.document.DocumentRecord; -import com.marklogic.client.document.DocumentUriTemplate; -import com.marklogic.client.document.DocumentWriteOperation; -import com.marklogic.client.document.DocumentWriteSet; -import com.marklogic.client.document.ServerTransform; import com.marklogic.client.eval.EvalResult; import com.marklogic.client.eval.EvalResultIterator; - -import com.marklogic.client.impl.okhttp.*; +import com.marklogic.client.impl.okhttp.HttpUrlBuilder; +import com.marklogic.client.impl.okhttp.OkHttpUtil; import com.marklogic.client.io.*; import com.marklogic.client.io.marker.*; import com.marklogic.client.query.*; import com.marklogic.client.query.QueryManager.QueryView; -import com.marklogic.client.semantics.Capability; -import com.marklogic.client.semantics.GraphManager; -import com.marklogic.client.semantics.GraphPermissions; -import com.marklogic.client.semantics.SPARQLBinding; -import com.marklogic.client.semantics.SPARQLBindings; -import com.marklogic.client.semantics.SPARQLQueryDefinition; -import com.marklogic.client.semantics.SPARQLRuleset; +import com.marklogic.client.semantics.*; import com.marklogic.client.util.EditableNamespaceContext; import com.marklogic.client.util.RequestLogger; import com.marklogic.client.util.RequestParameters; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; - +import jakarta.mail.BodyPart; +import jakarta.mail.Header; +import jakarta.mail.MessagingException; +import jakarta.mail.internet.MimeMultipart; +import jakarta.mail.util.ByteArrayDataSource; +import jakarta.xml.bind.DatatypeConverter; import okhttp3.*; import okhttp3.MultipartBody.Part; import okhttp3.logging.HttpLoggingInterceptor; import okio.BufferedSink; import okio.Okio; import okio.Source; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import jakarta.mail.BodyPart; -import jakarta.mail.Header; -import jakarta.mail.MessagingException; -import jakarta.mail.internet.MimeMultipart; -import jakarta.mail.util.ByteArrayDataSource; -import javax.net.ssl.*; -import jakarta.xml.bind.DatatypeConverter; -import java.io.ByteArrayInputStream; -import java.io.Closeable; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintStream; -import java.io.Reader; -import java.io.UnsupportedEncodingException; -import java.io.Writer; +import javax.net.ssl.SSLException; +import java.io.*; import java.math.BigDecimal; import java.net.URLEncoder; import java.nio.charset.Charset; @@ -84,7 +60,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -@SuppressWarnings({ "unchecked", "rawtypes" }) +@SuppressWarnings({"unchecked", "rawtypes"}) public class OkHttpServices implements RESTServices { static { @@ -94,66 +70,68 @@ public class OkHttpServices implements RESTServices { } } - static final private Logger logger = LoggerFactory.getLogger(OkHttpServices.class); + static final private Logger logger = LoggerFactory.getLogger(OkHttpServices.class); + + static final public String OKHTTP_LOGGINGINTERCEPTOR_LEVEL = "com.marklogic.client.okhttp.httplogginginterceptor.level"; + static final public String OKHTTP_LOGGINGINTERCEPTOR_OUTPUT = "com.marklogic.client.okhttp.httplogginginterceptor.output"; + + static final private String DOCUMENT_URI_PREFIX = "/documents?uri="; - static final public String OKHTTP_LOGGINGINTERCEPTOR_LEVEL = "com.marklogic.client.okhttp.httplogginginterceptor.level"; - static final public String OKHTTP_LOGGINGINTERCEPTOR_OUTPUT = "com.marklogic.client.okhttp.httplogginginterceptor.output"; + static final private int DELAY_FLOOR = 125; + static final private int DELAY_CEILING = 2000; + static final private int DELAY_MULTIPLIER = 20; + static final private int DEFAULT_MAX_DELAY = 120000; + static final private int DEFAULT_MIN_RETRY = 8; - static final private String DOCUMENT_URI_PREFIX = "/documents?uri="; + private final static MediaType URLENCODED_MIME_TYPE = MediaType.parse("application/x-www-form-urlencoded; charset=UTF-8"); + private final static String UTF8_ID = StandardCharsets.UTF_8.toString(); - static final private int DELAY_FLOOR = 125; - static final private int DELAY_CEILING = 2000; - static final private int DELAY_MULTIPLIER = 20; - static final private int DEFAULT_MAX_DELAY = 120000; - static final private int DEFAULT_MIN_RETRY = 8; + private DatabaseClient databaseClient; + private String database = null; + private HttpUrl baseUri; + private OkHttpClient client; + private boolean released = false; - private final static MediaType URLENCODED_MIME_TYPE = MediaType.parse("application/x-www-form-urlencoded; charset=UTF-8"); - private final static String UTF8_ID = StandardCharsets.UTF_8.toString(); + private final Random randRetry = new Random(); - private DatabaseClient databaseClient; - private String database = null; - private HttpUrl baseUri; - private OkHttpClient client; - private boolean released = false; + private int maxDelay = DEFAULT_MAX_DELAY; + private int minRetry = DEFAULT_MIN_RETRY; - private final Random randRetry = new Random(); + private boolean checkFirstRequest = true; - private int maxDelay = DEFAULT_MAX_DELAY; - private int minRetry = DEFAULT_MIN_RETRY; + private final Set retryStatus = new HashSet<>(); - private boolean checkFirstRequest = true; + static protected class ThreadState { + boolean isFirstRequest; - private final Set retryStatus = new HashSet<>(); + ThreadState(boolean value) { + isFirstRequest = value; + } + } - static protected class ThreadState { - boolean isFirstRequest; - ThreadState(boolean value) { - isFirstRequest = value; - } - } + private final ThreadLocal threadState = + ThreadLocal.withInitial(() -> new ThreadState(checkFirstRequest)); - private final ThreadLocal threadState = - ThreadLocal.withInitial(() -> new ThreadState(checkFirstRequest)); + public OkHttpServices() { + retryStatus.add(STATUS_BAD_GATEWAY); + retryStatus.add(STATUS_SERVICE_UNAVAILABLE); + retryStatus.add(STATUS_GATEWAY_TIMEOUT); + } - public OkHttpServices() { - retryStatus.add(STATUS_BAD_GATEWAY); - retryStatus.add(STATUS_SERVICE_UNAVAILABLE); - retryStatus.add(STATUS_GATEWAY_TIMEOUT); - } + @Override + public Set getRetryStatus() { + return retryStatus; + } - @Override - public Set getRetryStatus() { - return retryStatus; - } + @Override + public int getMaxDelay() { + return maxDelay; + } - @Override - public int getMaxDelay() { - return maxDelay; - } - @Override - public void setMaxDelay(int maxDelay) { - this.maxDelay = maxDelay; - } + @Override + public void setMaxDelay(int maxDelay) { + this.maxDelay = maxDelay; + } private FailedRequest extractErrorFields(Response response) { if (response == null) return null; @@ -196,27 +174,27 @@ private FailedRequest extractErrorFields(Response response) { } } - @Override - public void connect(String host, int port, String basePath, String database, SecurityContext securityContext){ - if (host == null) - throw new IllegalArgumentException("No host provided"); - if (securityContext == null) - throw new IllegalArgumentException("No security context provided"); + @Override + public void connect(String host, int port, String basePath, String database, SecurityContext securityContext) { + if (host == null) + throw new IllegalArgumentException("No host provided"); + if (securityContext == null) + throw new IllegalArgumentException("No security context provided"); - this.checkFirstRequest = securityContext instanceof DigestAuthContext; - this.database = database; - this.baseUri = HttpUrlBuilder.newBaseUrl(host, port, basePath, securityContext.getSSLContext()); + this.checkFirstRequest = securityContext instanceof DigestAuthContext; + this.database = database; + this.baseUri = HttpUrlBuilder.newBaseUrl(host, port, basePath, securityContext.getSSLContext()); - OkHttpClient.Builder clientBuilder = OkHttpUtil.newOkHttpClientBuilder(host, securityContext); + OkHttpClient.Builder clientBuilder = OkHttpUtil.newOkHttpClientBuilder(host, securityContext); - Properties props = System.getProperties(); - if (props.containsKey(OKHTTP_LOGGINGINTERCEPTOR_LEVEL)) { - configureOkHttpLogging(clientBuilder, props); - } - this.configureDelayAndRetry(props); + Properties props = System.getProperties(); + if (props.containsKey(OKHTTP_LOGGINGINTERCEPTOR_LEVEL)) { + configureOkHttpLogging(clientBuilder, props); + } + this.configureDelayAndRetry(props); - this.client = clientBuilder.build(); - } + this.client = clientBuilder.build(); + } /** * Based on the given properties, add a network interceptor to the given OkHttpClient.Builder to log HTTP @@ -226,6413 +204,6396 @@ public void connect(String host, int port, String basePath, String database, Sec * @param props */ private void configureOkHttpLogging(OkHttpClient.Builder clientBuilder, Properties props) { - final boolean useLogger = "LOGGER".equalsIgnoreCase(props.getProperty(OKHTTP_LOGGINGINTERCEPTOR_OUTPUT)); - final boolean useStdErr = "STDERR".equalsIgnoreCase(props.getProperty(OKHTTP_LOGGINGINTERCEPTOR_OUTPUT)); - HttpLoggingInterceptor networkInterceptor = new HttpLoggingInterceptor(message -> { - if (useLogger == true) { - logger.debug(message); - } else if (useStdErr == true) { - System.err.println(message); - } else { - System.out.println(message); - } - }); - if ("BASIC".equalsIgnoreCase(props.getProperty(OKHTTP_LOGGINGINTERCEPTOR_LEVEL))) { - networkInterceptor = networkInterceptor.setLevel(HttpLoggingInterceptor.Level.BASIC); - } else if ("BODY".equalsIgnoreCase(props.getProperty(OKHTTP_LOGGINGINTERCEPTOR_LEVEL))) { - networkInterceptor = networkInterceptor.setLevel(HttpLoggingInterceptor.Level.BODY); - } else if ("HEADERS".equalsIgnoreCase(props.getProperty(OKHTTP_LOGGINGINTERCEPTOR_LEVEL))) { - networkInterceptor = networkInterceptor.setLevel(HttpLoggingInterceptor.Level.HEADERS); - } else if ("NONE".equalsIgnoreCase(props.getProperty(OKHTTP_LOGGINGINTERCEPTOR_LEVEL))) { - networkInterceptor = networkInterceptor.setLevel(HttpLoggingInterceptor.Level.NONE); - } - clientBuilder.addNetworkInterceptor(networkInterceptor); - } - - private void configureDelayAndRetry(Properties props) { - if (props.containsKey(MAX_DELAY_PROP)) { - int max = Utilities.parseInt(props.getProperty(MAX_DELAY_PROP)); - if (max > 0) { - maxDelay = max * 1000; - } - } - if (props.containsKey(MIN_RETRY_PROP)) { - int min = Utilities.parseInt(props.getProperty(MIN_RETRY_PROP)); - if (min > 0) { - minRetry = min; - } - } - } - - @Override - public DatabaseClient getDatabaseClient() { - return databaseClient; - } - @Override - public void setDatabaseClient(DatabaseClient client) { - this.databaseClient = client; - } - - private OkHttpClient getConnection() { - if ( client != null ) { - return client; - } else if ( released ) { - throw new IllegalStateException( - "You cannot use this connected object anymore--connection has already been released"); - } else { - throw new MarkLogicInternalException("Cannot proceed--connection is null for unknown reason"); - } - } - - @Override - public void release() { - if ( client == null ) return; - try { - released = true; - client.dispatcher().executorService().shutdownNow(); - } finally { - try { - if ( client.cache() != null ) client.cache().close(); - } catch (IOException e) { - throw new MarkLogicIOException(e); - } finally { - client = null; - logger.debug("Releasing connection"); - } - } - } - - private boolean isFirstRequest() { - return threadState.get().isFirstRequest; - } - private void setFirstRequest(boolean value) { - threadState.get().isFirstRequest = value; - } - private void checkFirstRequest() { - if (checkFirstRequest) setFirstRequest(true); - } - - private int makeFirstRequest(int retry) { - return makeFirstRequest(baseUri, "ping", retry); - } - - private int makeFirstRequest(HttpUrl requestUri, String path, int retry) { - Response response = sendRequestOnce(setupRequest(requestUri, path, null).head()); - int statusCode = response.code(); - if (!retryStatus.contains(statusCode)) { - closeResponse(response); - return 0; - } - - String retryAfterRaw = response.header("Retry-After"); - closeResponse(response); - - int retryAfter = Utilities.parseInt(retryAfterRaw); - return Math.max(retryAfter, calculateDelay(randRetry, retry)); - } - - private RequestParameters addTemporalProtectionParams(RequestParameters params, String uri, ProtectionLevel level, - String duration, Calendar expiryTime, String archivePath) { - if (params == null) - params = new RequestParameters(); - params.add("uri", uri); - params.add("level", level.toString()); - if (duration != null) - params.add("duration", duration); - if (expiryTime != null) { - String formattedSystemTime = DatatypeConverter.printDateTime(expiryTime); - params.add("expireTime", formattedSystemTime); - } - if (archivePath != null) - params.add("archivePath", archivePath); - return params; - } - - @Override - public String advanceLsqt(RequestLogger reqlog, String temporalCollection, long lag) { - if (logger.isDebugEnabled()) - logger.debug("Advancing LSQT in temporal collection {}", temporalCollection); - logRequest(reqlog, "wiped %s document", temporalCollection); - RequestParameters params = new RequestParameters(); - params.add("result", "advance-lsqt"); - if ( lag > 0 ) params.add("lag", String.valueOf(lag)); - Map> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - postResource(reqlog, "temporal/collections/" + temporalCollection, - null, params, null, null, "advanceLsqt", headers); - List values = headers.get(HEADER_ML_LSQT); - if ( values != null && values.size() > 0 ) { - return values.get(0); - } else { - throw new FailedRequestException("Response missing header \"" + HEADER_ML_LSQT + "\""); - } - } - - @Override - public void protectDocument(RequestLogger requestLogger, String temporalDocumentURI, Transaction transaction, - RequestParameters extraParams, ProtectionLevel level, String duration, - Calendar expiryTime, String archivePath) { - if (temporalDocumentURI == null) - throw new IllegalArgumentException( - "Document protection for document identifier without uri"); - extraParams = addTemporalProtectionParams(extraParams, temporalDocumentURI, level, duration, expiryTime, archivePath); - if (logger.isDebugEnabled()) - logger.debug("Protecting {} in transaction {}", temporalDocumentURI, getTransactionId(transaction)); - - postResource(requestLogger, "documents/protection", transaction, extraParams, null, null, "protect"); - } - @Override - public void wipeDocument(RequestLogger reqlog, String temporalDocumentURI, Transaction transaction, - RequestParameters extraParams) { - if (logger.isDebugEnabled()) - logger.debug("Wiping {} in transaction {}", temporalDocumentURI, getTransactionId(transaction)); - extraParams.add("result", "wiped"); - extraParams.add("uri", temporalDocumentURI); - deleteResource(reqlog, "documents", transaction, extraParams, null); - logRequest(reqlog, "wiped %s document", temporalDocumentURI); - } - - @Override - public TemporalDescriptor deleteDocument(RequestLogger reqlog, DocumentDescriptor desc, - Transaction transaction, Set categories, RequestParameters extraParams) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - String uri = desc.getUri(); - if (uri == null) { - throw new IllegalArgumentException( - "Document delete for document identifier without uri"); - } - - logger.debug("Deleting {} in transaction {}", uri, getTransactionId(transaction)); - - Request.Builder requestBldr = makeDocumentResource(makeDocumentParams(uri, - categories, transaction, extraParams)); - - requestBldr = addVersionHeader(desc, requestBldr, "If-Match"); - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - - Function doDeleteFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return sendRequestOnce(funcBuilder.delete().build()); - } - }; - Response response = sendRequestWithRetry(requestBldr, (transaction == null), doDeleteFunction, null); - int status = response.code(); - - if (status == STATUS_NOT_FOUND) { - closeResponse(response); - throw new ResourceNotFoundException( - "Could not delete non-existent document"); - } - if (status == STATUS_PRECONDITION_REQUIRED) { - FailedRequest failure = extractErrorFields(response); - if (failure.getMessageCode().equals("RESTAPI-CONTENTNOVERSION")) { - throw new ContentNoVersionException( - "Content version required to delete document", failure); - } - throw new FailedRequestException( - "Precondition required to delete document", failure); - } else if (status == STATUS_FORBIDDEN) { - FailedRequest failure = extractErrorFields(response); - throw new ForbiddenUserException( - "User is not allowed to delete documents", failure); - } - if (status == STATUS_PRECONDITION_FAILED) { - FailedRequest failure = extractErrorFields(response); - if (failure.getMessageCode().equals("RESTAPI-CONTENTWRONGVERSION")) { - throw new ContentWrongVersionException("Content version must match to delete document", failure); - } else if (failure.getMessageCode().equals("RESTAPI-EMPTYBODY")) { - throw new FailedRequestException( - "Empty request body sent to server", failure); - } - throw new FailedRequestException("Precondition Failed", failure); - } - if (status != STATUS_NO_CONTENT) { - throw new FailedRequestException("delete failed: " - + getReasonPhrase(response), extractErrorFields(response)); - } - Headers responseHeaders = response.headers(); - TemporalDescriptor temporalDesc = updateTemporalSystemTime(desc, responseHeaders); - - closeResponse(response); - logRequest(reqlog, "deleted %s document", uri); - return temporalDesc; - } - - @Override - public boolean getDocument(RequestLogger reqlog, DocumentDescriptor desc, - Transaction transaction, Set categories, - RequestParameters extraParams, - DocumentMetadataReadHandle metadataHandle, - AbstractReadHandle contentHandle) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - - HandleImplementation metadataBase = HandleAccessor.checkHandle( - metadataHandle, "metadata"); - HandleImplementation contentBase = HandleAccessor.checkHandle( - contentHandle, "content"); - - String metadataFormat = null; - String metadataMimetype = null; - if (metadataBase != null) { - metadataFormat = metadataBase.getFormat().toString().toLowerCase(); - metadataMimetype = metadataBase.getMimetype(); - } - - String contentMimetype = null; - if (contentBase != null) { - contentMimetype = contentBase.getMimetype(); - } - - if (metadataBase != null && contentBase != null) { - return getDocumentImpl(reqlog, desc, transaction, categories, - extraParams, metadataFormat, metadataHandle, contentHandle); - } else if (metadataBase != null) { - return getDocumentImpl(reqlog, desc, transaction, categories, - extraParams, metadataMimetype, metadataHandle); - } else if (contentBase != null) { - return getDocumentImpl(reqlog, desc, transaction, null, - extraParams, contentMimetype, contentHandle); - } - - return false; - } - - private int getRetryAfterTime(Response response) { - return Utilities.parseInt(response.header("Retry-After")) ; - } - - private Response sendRequestOnce(Request.Builder requestBldr) { - return sendRequestOnce(requestBldr.build()); - } - - private Response sendRequestOnce(Request request) { - try { - return getConnection().newCall(request).execute(); - } catch (IOException e) { - if (e instanceof SSLException) { - String message = e.getMessage(); - if (message != null && message.contains("readHandshakeRecord")) { - throw new MarkLogicIOException(String.format("SSL error occurred: %s; ensure you are using a valid certificate " + - "if the MarkLogic app server requires a client certificate for SSL.", message)); - } - } - String message = String.format( - "Error occurred while calling %s; %s: %s " + - "; possible reasons for the error include that a MarkLogic app server may not be listening on the port, " + - "or MarkLogic was stopped or restarted during the request; check the MarkLogic server logs for more information.", - request.url(), e.getClass().getName(), e.getMessage() - ); - throw new MarkLogicIOException(message, e); - } - } - - private Response sendRequestWithRetry(Request.Builder requestBldr, Function doFunction, Consumer resendableConsumer) { - return sendRequestWithRetry(requestBldr, true, doFunction, resendableConsumer); - } - - private Response sendRequestWithRetry( - Request.Builder requestBldr, boolean isRetryable, Function doFunction, Consumer resendableConsumer - ) { - Response response = null; - int status = -1; - long startTime = System.currentTimeMillis(); - int nextDelay = 0; - int retry = 0; - /* - * This loop is for retrying the request if the service is unavailable - */ - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { - if (nextDelay > 0) { - try { Thread.sleep(nextDelay);} catch (InterruptedException e) {} - } - - /* - * Execute the function which is passed as an argument - * in order to get the Response - */ - response = doFunction.apply(requestBldr); - if (response == null) { - throw new MarkLogicInternalException( - "null response for: "+requestBldr.build().url().toString() - ); - } - status = response.code(); - if (!isRetryable || !retryStatus.contains(status)) { - if (isFirstRequest()) setFirstRequest(false); - /* - * If we don't get a service unavailable status or if the request - * is not retryable, we break from the retrying loop and return - * the response - */ - break; - } - /* - * This code will be executed whenever the service is unavailable. - * When the service becomes unavailable, we close the Response - * we got and retry it to try and get a new Response - */ - closeResponse(response); - /* - * There are scenarios where we don't want to retry and we just want to - * throw ResourceNotResendableException. In that case, we pass that code from - * the caller through the Consumer and execute it here. In the rest of the - * scenarios, we pass it as null and it is just a no-operation. - */ - if(resendableConsumer != null) resendableConsumer.accept(null); - /* - * Calculate the delay before which we shouldn't retry - */ - nextDelay = Math.max(getRetryAfterTime(response), calculateDelay(randRetry, retry)); - } - /* - * If the service is still unavailable after all the retries, we throw a - * FailedRetryException indicating that the service is unavailable. - */ - if (retryStatus.contains(status)) { - checkFirstRequest(); - closeResponse(response); - throw new FailedRetryException( - "Service unavailable and maximum retry period elapsed: "+ - ((System.currentTimeMillis() - startTime) / 1000)+ - " seconds after "+retry+" retries"); - } - /* - * Once we break from the retry loop, we just return the Response - * back to the caller in order to proceed with the flow - */ - return response; - } - - private boolean getDocumentImpl(RequestLogger reqlog, - DocumentDescriptor desc, Transaction transaction, - Set categories, RequestParameters extraParams, - String mimetype, AbstractReadHandle handle) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - String uri = desc.getUri(); - if (uri == null) { - throw new IllegalArgumentException( - "Document read for document identifier without uri"); - } - - logger.debug("Getting {} in transaction {}", uri, getTransactionId(transaction)); - - addPointInTimeQueryParam(extraParams, handle); - - Request.Builder requestBldr = makeDocumentResource( - makeDocumentParams(uri, categories, transaction, extraParams)); - if ( mimetype != null ) { - requestBldr = requestBldr.header(HEADER_ACCEPT, mimetype); - } - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - - if (extraParams != null && extraParams.containsKey("range")) { - requestBldr = requestBldr.header("range", extraParams.get("range").get(0)); - } - - requestBldr = addVersionHeader(desc, requestBldr, "If-None-Match"); - - Function doGetFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return sendRequestOnce(funcBuilder.get().build()); - } - }; - Response response = sendRequestWithRetry(requestBldr, (transaction == null), doGetFunction, null); - - int status = response.code(); - if (status == STATUS_NOT_FOUND) { - throw new ResourceNotFoundException( - "Could not read non-existent document", - extractErrorFields(response)); - } - if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException( - "User is not allowed to read documents", - extractErrorFields(response)); - } - if (status == STATUS_NOT_MODIFIED) { - closeResponse(response); - return false; - } - if (status != STATUS_OK && status != STATUS_PARTIAL_CONTENT) { - throw new FailedRequestException("read failed: " - + getReasonPhrase(response), extractErrorFields(response)); - } - - logRequest( - reqlog, - "read %s document from %s transaction with %s mime type and %s metadata categories", - uri, (transaction != null) ? transaction.getTransactionId() : "no", - (mimetype != null) ? mimetype : "no", - stringJoin(categories, ", ", "no")); - - HandleImplementation handleBase = HandleAccessor.as(handle); - - Headers responseHeaders = response.headers(); - if (isExternalDescriptor(desc)) { - updateVersion(desc, responseHeaders); - updateDescriptor(desc, responseHeaders); - copyDescriptor(desc, handleBase); - } else { - updateDescriptor(handleBase, responseHeaders); - } - - Class as = handleBase.receiveAs(); - ResponseBody body = response.body(); - Object entity = body.contentLength() != 0 ? getEntity(body, as) : null; - - if (entity == null || (!InputStream.class.isAssignableFrom(as) && !Reader.class.isAssignableFrom(as))) { - closeResponse(response); - } - - handleBase.receiveContent((reqlog != null) ? reqlog.copyContent(entity) : entity); - - return true; - } - - @Override - public DocumentPage getBulkDocuments(RequestLogger reqlog, long serverTimestamp, - Transaction transaction, Set categories, - Format format, RequestParameters extraParams, boolean withContent, String... uris) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - boolean hasMetadata = categories != null && categories.size() > 0; - OkHttpResultIterator iterator = - getBulkDocumentsImpl(reqlog, serverTimestamp, transaction, categories, format, extraParams, - withContent, uris); - return new OkHttpDocumentPage(iterator, withContent, hasMetadata); - } - - @Override - public DocumentPage getBulkDocuments(RequestLogger reqlog, long serverTimestamp, - SearchQueryDefinition querydef, - long start, long pageLength, - Transaction transaction, - SearchReadHandle searchHandle, QueryView view, - Set categories, Format format, ServerTransform responseTransform, - RequestParameters extraParams, String forestName) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - boolean hasMetadata = categories != null && categories.size() > 0; - boolean hasContent = true; - OkHttpResultIterator iterator = - getBulkDocumentsImpl(reqlog, serverTimestamp, querydef, start, pageLength, transaction, - searchHandle, view, categories, format, responseTransform, extraParams, forestName); - return new OkHttpDocumentPage(iterator, hasContent, hasMetadata); - } - - private class OkHttpDocumentPage extends BasicPage implements DocumentPage, Iterator { - private final OkHttpResultIterator iterator; - private final boolean hasMetadata; - private final boolean hasContent; - - OkHttpDocumentPage(OkHttpResultIterator iterator, boolean hasContent, boolean hasMetadata) { - super( - new ArrayList().iterator(), - iterator != null ? iterator.getStart() : 1, - iterator != null ? iterator.getPageSize() : 0, - iterator != null ? iterator.getTotalSize() : 0 - ); - this.iterator = iterator; - this.hasContent = hasContent; - this.hasMetadata = hasMetadata; - if ( iterator == null ) { - setSize(0); - } else if ( hasContent && hasMetadata ) { - setSize(iterator.getSize() / 2); - } else { - setSize(iterator.getSize()); - } - } - - @Override - public Iterator iterator() { - return this; - } - - @Override - public boolean hasNext() { - if ( iterator == null ) return false; - return iterator.hasNext(); - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - @Override - public DocumentRecord next() { - if ( iterator == null ) throw new NoSuchElementException("No documents available"); - OkHttpResult result = iterator.next(); - DocumentRecord record; - if ( hasContent && hasMetadata ) { - OkHttpResult metadata = result; - OkHttpResult content = iterator.next(); - record = new OkHttpDocumentRecord(content, metadata); - } else if ( hasContent ) { - OkHttpResult content = result; - record = new OkHttpDocumentRecord(content); - } else if ( hasMetadata ) { - OkHttpResult metadata = result; - record = new OkHttpDocumentRecord(null, metadata); - } else { - throw new IllegalStateException("Should never have neither content nor metadata"); - } - return record; - } - - @Override - public T nextContent(T contentHandle) { - return next().getContent(contentHandle); - } - - @Override - public void close() { - if ( iterator != null ) iterator.close(); - } - } + final boolean useLogger = "LOGGER".equalsIgnoreCase(props.getProperty(OKHTTP_LOGGINGINTERCEPTOR_OUTPUT)); + final boolean useStdErr = "STDERR".equalsIgnoreCase(props.getProperty(OKHTTP_LOGGINGINTERCEPTOR_OUTPUT)); + HttpLoggingInterceptor networkInterceptor = new HttpLoggingInterceptor(message -> { + if (useLogger == true) { + logger.debug(message); + } else if (useStdErr == true) { + System.err.println(message); + } else { + System.out.println(message); + } + }); + if ("BASIC".equalsIgnoreCase(props.getProperty(OKHTTP_LOGGINGINTERCEPTOR_LEVEL))) { + networkInterceptor = networkInterceptor.setLevel(HttpLoggingInterceptor.Level.BASIC); + } else if ("BODY".equalsIgnoreCase(props.getProperty(OKHTTP_LOGGINGINTERCEPTOR_LEVEL))) { + networkInterceptor = networkInterceptor.setLevel(HttpLoggingInterceptor.Level.BODY); + } else if ("HEADERS".equalsIgnoreCase(props.getProperty(OKHTTP_LOGGINGINTERCEPTOR_LEVEL))) { + networkInterceptor = networkInterceptor.setLevel(HttpLoggingInterceptor.Level.HEADERS); + } else if ("NONE".equalsIgnoreCase(props.getProperty(OKHTTP_LOGGINGINTERCEPTOR_LEVEL))) { + networkInterceptor = networkInterceptor.setLevel(HttpLoggingInterceptor.Level.NONE); + } + clientBuilder.addNetworkInterceptor(networkInterceptor); + } + + private void configureDelayAndRetry(Properties props) { + if (props.containsKey(MAX_DELAY_PROP)) { + int max = Utilities.parseInt(props.getProperty(MAX_DELAY_PROP)); + if (max > 0) { + maxDelay = max * 1000; + } + } + if (props.containsKey(MIN_RETRY_PROP)) { + int min = Utilities.parseInt(props.getProperty(MIN_RETRY_PROP)); + if (min > 0) { + minRetry = min; + } + } + } + + @Override + public DatabaseClient getDatabaseClient() { + return databaseClient; + } + + @Override + public void setDatabaseClient(DatabaseClient client) { + this.databaseClient = client; + } + + private OkHttpClient getConnection() { + if (client != null) { + return client; + } else if (released) { + throw new IllegalStateException( + "You cannot use this connected object anymore--connection has already been released"); + } else { + throw new MarkLogicInternalException("Cannot proceed--connection is null for unknown reason"); + } + } + + @Override + public void release() { + if (client == null) return; + try { + released = true; + client.dispatcher().executorService().shutdownNow(); + } finally { + try { + if (client.cache() != null) client.cache().close(); + } catch (IOException e) { + throw new MarkLogicIOException(e); + } finally { + client = null; + logger.debug("Releasing connection"); + } + } + } + + private boolean isFirstRequest() { + return threadState.get().isFirstRequest; + } + + private void setFirstRequest(boolean value) { + threadState.get().isFirstRequest = value; + } + + private void checkFirstRequest() { + if (checkFirstRequest) setFirstRequest(true); + } + + private int makeFirstRequest(int retry) { + return makeFirstRequest(baseUri, "ping", retry); + } + + private int makeFirstRequest(HttpUrl requestUri, String path, int retry) { + Response response = sendRequestOnce(setupRequest(requestUri, path, null).head()); + int statusCode = response.code(); + if (!retryStatus.contains(statusCode)) { + closeResponse(response); + return 0; + } + + String retryAfterRaw = response.header("Retry-After"); + closeResponse(response); + + int retryAfter = Utilities.parseInt(retryAfterRaw); + return Math.max(retryAfter, calculateDelay(randRetry, retry)); + } + + private RequestParameters addTemporalProtectionParams(RequestParameters params, String uri, ProtectionLevel level, + String duration, Calendar expiryTime, String archivePath) { + if (params == null) + params = new RequestParameters(); + params.add("uri", uri); + params.add("level", level.toString()); + if (duration != null) + params.add("duration", duration); + if (expiryTime != null) { + String formattedSystemTime = DatatypeConverter.printDateTime(expiryTime); + params.add("expireTime", formattedSystemTime); + } + if (archivePath != null) + params.add("archivePath", archivePath); + return params; + } + + @Override + public String advanceLsqt(RequestLogger reqlog, String temporalCollection, long lag) { + if (logger.isDebugEnabled()) + logger.debug("Advancing LSQT in temporal collection {}", temporalCollection); + logRequest(reqlog, "wiped %s document", temporalCollection); + RequestParameters params = new RequestParameters(); + params.add("result", "advance-lsqt"); + if (lag > 0) params.add("lag", String.valueOf(lag)); + Map> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + postResource(reqlog, "temporal/collections/" + temporalCollection, + null, params, null, null, "advanceLsqt", headers); + List values = headers.get(HEADER_ML_LSQT); + if (values != null && values.size() > 0) { + return values.get(0); + } else { + throw new FailedRequestException("Response missing header \"" + HEADER_ML_LSQT + "\""); + } + } + + @Override + public void protectDocument(RequestLogger requestLogger, String temporalDocumentURI, Transaction transaction, + RequestParameters extraParams, ProtectionLevel level, String duration, + Calendar expiryTime, String archivePath) { + if (temporalDocumentURI == null) + throw new IllegalArgumentException( + "Document protection for document identifier without uri"); + extraParams = addTemporalProtectionParams(extraParams, temporalDocumentURI, level, duration, expiryTime, archivePath); + if (logger.isDebugEnabled()) + logger.debug("Protecting {} in transaction {}", temporalDocumentURI, getTransactionId(transaction)); + + postResource(requestLogger, "documents/protection", transaction, extraParams, null, null, "protect"); + } + + @Override + public void wipeDocument(RequestLogger reqlog, String temporalDocumentURI, Transaction transaction, + RequestParameters extraParams) { + if (logger.isDebugEnabled()) + logger.debug("Wiping {} in transaction {}", temporalDocumentURI, getTransactionId(transaction)); + extraParams.add("result", "wiped"); + extraParams.add("uri", temporalDocumentURI); + deleteResource(reqlog, "documents", transaction, extraParams, null); + logRequest(reqlog, "wiped %s document", temporalDocumentURI); + } + + @Override + public TemporalDescriptor deleteDocument(RequestLogger reqlog, DocumentDescriptor desc, + Transaction transaction, Set categories, RequestParameters extraParams) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + String uri = desc.getUri(); + if (uri == null) { + throw new IllegalArgumentException( + "Document delete for document identifier without uri"); + } + + logger.debug("Deleting {} in transaction {}", uri, getTransactionId(transaction)); + + Request.Builder requestBldr = makeDocumentResource(makeDocumentParams(uri, + categories, transaction, extraParams)); + + requestBldr = addVersionHeader(desc, requestBldr, "If-Match"); + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + + Function doDeleteFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return sendRequestOnce(funcBuilder.delete().build()); + } + }; + Response response = sendRequestWithRetry(requestBldr, (transaction == null), doDeleteFunction, null); + int status = response.code(); + + if (status == STATUS_NOT_FOUND) { + closeResponse(response); + throw new ResourceNotFoundException( + "Could not delete non-existent document"); + } + if (status == STATUS_PRECONDITION_REQUIRED) { + FailedRequest failure = extractErrorFields(response); + if (failure.getMessageCode().equals("RESTAPI-CONTENTNOVERSION")) { + throw new ContentNoVersionException( + "Content version required to delete document", failure); + } + throw new FailedRequestException( + "Precondition required to delete document", failure); + } else if (status == STATUS_FORBIDDEN) { + FailedRequest failure = extractErrorFields(response); + throw new ForbiddenUserException( + "User is not allowed to delete documents", failure); + } + if (status == STATUS_PRECONDITION_FAILED) { + FailedRequest failure = extractErrorFields(response); + if (failure.getMessageCode().equals("RESTAPI-CONTENTWRONGVERSION")) { + throw new ContentWrongVersionException("Content version must match to delete document", failure); + } else if (failure.getMessageCode().equals("RESTAPI-EMPTYBODY")) { + throw new FailedRequestException( + "Empty request body sent to server", failure); + } + throw new FailedRequestException("Precondition Failed", failure); + } + if (status != STATUS_NO_CONTENT) { + throw new FailedRequestException("delete failed: " + + getReasonPhrase(response), extractErrorFields(response)); + } + Headers responseHeaders = response.headers(); + TemporalDescriptor temporalDesc = updateTemporalSystemTime(desc, responseHeaders); + + closeResponse(response); + logRequest(reqlog, "deleted %s document", uri); + return temporalDesc; + } + + @Override + public boolean getDocument(RequestLogger reqlog, DocumentDescriptor desc, + Transaction transaction, Set categories, + RequestParameters extraParams, + DocumentMetadataReadHandle metadataHandle, + AbstractReadHandle contentHandle) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + + HandleImplementation metadataBase = HandleAccessor.checkHandle( + metadataHandle, "metadata"); + HandleImplementation contentBase = HandleAccessor.checkHandle( + contentHandle, "content"); + + String metadataFormat = null; + String metadataMimetype = null; + if (metadataBase != null) { + metadataFormat = metadataBase.getFormat().toString().toLowerCase(); + metadataMimetype = metadataBase.getMimetype(); + } + + String contentMimetype = null; + if (contentBase != null) { + contentMimetype = contentBase.getMimetype(); + } + + if (metadataBase != null && contentBase != null) { + return getDocumentImpl(reqlog, desc, transaction, categories, + extraParams, metadataFormat, metadataHandle, contentHandle); + } else if (metadataBase != null) { + return getDocumentImpl(reqlog, desc, transaction, categories, + extraParams, metadataMimetype, metadataHandle); + } else if (contentBase != null) { + return getDocumentImpl(reqlog, desc, transaction, null, + extraParams, contentMimetype, contentHandle); + } + + return false; + } + + private int getRetryAfterTime(Response response) { + return Utilities.parseInt(response.header("Retry-After")); + } + + private Response sendRequestOnce(Request.Builder requestBldr) { + return sendRequestOnce(requestBldr.build()); + } + + private Response sendRequestOnce(Request request) { + try { + return getConnection().newCall(request).execute(); + } catch (IOException e) { + if (e instanceof SSLException) { + String message = e.getMessage(); + if (message != null && message.contains("readHandshakeRecord")) { + throw new MarkLogicIOException(String.format("SSL error occurred: %s; ensure you are using a valid certificate " + + "if the MarkLogic app server requires a client certificate for SSL.", message)); + } + } + String message = String.format( + "Error occurred while calling %s; %s: %s " + + "; possible reasons for the error include that a MarkLogic app server may not be listening on the port, " + + "or MarkLogic was stopped or restarted during the request; check the MarkLogic server logs for more information.", + request.url(), e.getClass().getName(), e.getMessage() + ); + throw new MarkLogicIOException(message, e); + } + } + + private Response sendRequestWithRetry(Request.Builder requestBldr, Function doFunction, Consumer resendableConsumer) { + return sendRequestWithRetry(requestBldr, true, doFunction, resendableConsumer); + } + + private Response sendRequestWithRetry( + Request.Builder requestBldr, boolean isRetryable, Function doFunction, Consumer resendableConsumer + ) { + Response response = null; + int status = -1; + long startTime = System.currentTimeMillis(); + int nextDelay = 0; + int retry = 0; + /* + * This loop is for retrying the request if the service is unavailable + */ + for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + if (nextDelay > 0) { + try { + Thread.sleep(nextDelay); + } catch (InterruptedException e) { + } + } + + /* + * Execute the function which is passed as an argument + * in order to get the Response + */ + response = doFunction.apply(requestBldr); + if (response == null) { + throw new MarkLogicInternalException( + "null response for: " + requestBldr.build().url().toString() + ); + } + status = response.code(); + if (!isRetryable || !retryStatus.contains(status)) { + if (isFirstRequest()) setFirstRequest(false); + /* + * If we don't get a service unavailable status or if the request + * is not retryable, we break from the retrying loop and return + * the response + */ + break; + } + /* + * This code will be executed whenever the service is unavailable. + * When the service becomes unavailable, we close the Response + * we got and retry it to try and get a new Response + */ + closeResponse(response); + /* + * There are scenarios where we don't want to retry and we just want to + * throw ResourceNotResendableException. In that case, we pass that code from + * the caller through the Consumer and execute it here. In the rest of the + * scenarios, we pass it as null and it is just a no-operation. + */ + if (resendableConsumer != null) resendableConsumer.accept(null); + /* + * Calculate the delay before which we shouldn't retry + */ + nextDelay = Math.max(getRetryAfterTime(response), calculateDelay(randRetry, retry)); + } + /* + * If the service is still unavailable after all the retries, we throw a + * FailedRetryException indicating that the service is unavailable. + */ + if (retryStatus.contains(status)) { + checkFirstRequest(); + closeResponse(response); + throw new FailedRetryException( + "Service unavailable and maximum retry period elapsed: " + + ((System.currentTimeMillis() - startTime) / 1000) + + " seconds after " + retry + " retries"); + } + /* + * Once we break from the retry loop, we just return the Response + * back to the caller in order to proceed with the flow + */ + return response; + } + + private boolean getDocumentImpl(RequestLogger reqlog, + DocumentDescriptor desc, Transaction transaction, + Set categories, RequestParameters extraParams, + String mimetype, AbstractReadHandle handle) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + String uri = desc.getUri(); + if (uri == null) { + throw new IllegalArgumentException( + "Document read for document identifier without uri"); + } + + logger.debug("Getting {} in transaction {}", uri, getTransactionId(transaction)); + + addPointInTimeQueryParam(extraParams, handle); + + Request.Builder requestBldr = makeDocumentResource( + makeDocumentParams(uri, categories, transaction, extraParams)); + if (mimetype != null) { + requestBldr = requestBldr.header(HEADER_ACCEPT, mimetype); + } + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + + if (extraParams != null && extraParams.containsKey("range")) { + requestBldr = requestBldr.header("range", extraParams.get("range").get(0)); + } + + requestBldr = addVersionHeader(desc, requestBldr, "If-None-Match"); + + Function doGetFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return sendRequestOnce(funcBuilder.get().build()); + } + }; + Response response = sendRequestWithRetry(requestBldr, (transaction == null), doGetFunction, null); + + int status = response.code(); + if (status == STATUS_NOT_FOUND) { + throw new ResourceNotFoundException( + "Could not read non-existent document", + extractErrorFields(response)); + } + if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException( + "User is not allowed to read documents", + extractErrorFields(response)); + } + if (status == STATUS_NOT_MODIFIED) { + closeResponse(response); + return false; + } + if (status != STATUS_OK && status != STATUS_PARTIAL_CONTENT) { + throw new FailedRequestException("read failed: " + + getReasonPhrase(response), extractErrorFields(response)); + } + + logRequest( + reqlog, + "read %s document from %s transaction with %s mime type and %s metadata categories", + uri, (transaction != null) ? transaction.getTransactionId() : "no", + (mimetype != null) ? mimetype : "no", + stringJoin(categories, ", ", "no")); + + HandleImplementation handleBase = HandleAccessor.as(handle); + + Headers responseHeaders = response.headers(); + if (isExternalDescriptor(desc)) { + updateVersion(desc, responseHeaders); + updateDescriptor(desc, responseHeaders); + copyDescriptor(desc, handleBase); + } else { + updateDescriptor(handleBase, responseHeaders); + } + + Class as = handleBase.receiveAs(); + ResponseBody body = response.body(); + Object entity = body.contentLength() != 0 ? getEntity(body, as) : null; + + if (entity == null || (!InputStream.class.isAssignableFrom(as) && !Reader.class.isAssignableFrom(as))) { + closeResponse(response); + } + + handleBase.receiveContent((reqlog != null) ? reqlog.copyContent(entity) : entity); + + return true; + } + + @Override + public DocumentPage getBulkDocuments(RequestLogger reqlog, long serverTimestamp, + Transaction transaction, Set categories, + Format format, RequestParameters extraParams, boolean withContent, String... uris) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + boolean hasMetadata = categories != null && categories.size() > 0; + OkHttpResultIterator iterator = + getBulkDocumentsImpl(reqlog, serverTimestamp, transaction, categories, format, extraParams, + withContent, uris); + return new OkHttpDocumentPage(iterator, withContent, hasMetadata); + } + + @Override + public DocumentPage getBulkDocuments(RequestLogger reqlog, long serverTimestamp, + SearchQueryDefinition querydef, + long start, long pageLength, + Transaction transaction, + SearchReadHandle searchHandle, QueryView view, + Set categories, Format format, ServerTransform responseTransform, + RequestParameters extraParams, String forestName) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + boolean hasMetadata = categories != null && categories.size() > 0; + boolean hasContent = true; + OkHttpResultIterator iterator = + getBulkDocumentsImpl(reqlog, serverTimestamp, querydef, start, pageLength, transaction, + searchHandle, view, categories, format, responseTransform, extraParams, forestName); + return new OkHttpDocumentPage(iterator, hasContent, hasMetadata); + } + + private class OkHttpDocumentPage extends BasicPage implements DocumentPage, Iterator { + private final OkHttpResultIterator iterator; + private final boolean hasMetadata; + private final boolean hasContent; + + OkHttpDocumentPage(OkHttpResultIterator iterator, boolean hasContent, boolean hasMetadata) { + super( + new ArrayList().iterator(), + iterator != null ? iterator.getStart() : 1, + iterator != null ? iterator.getPageSize() : 0, + iterator != null ? iterator.getTotalSize() : 0 + ); + this.iterator = iterator; + this.hasContent = hasContent; + this.hasMetadata = hasMetadata; + if (iterator == null) { + setSize(0); + } else if (hasContent && hasMetadata) { + setSize(iterator.getSize() / 2); + } else { + setSize(iterator.getSize()); + } + } + + @Override + public Iterator iterator() { + return this; + } + + @Override + public boolean hasNext() { + if (iterator == null) return false; + return iterator.hasNext(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public DocumentRecord next() { + if (iterator == null) throw new NoSuchElementException("No documents available"); + OkHttpResult result = iterator.next(); + DocumentRecord record; + if (hasContent && hasMetadata) { + OkHttpResult metadata = result; + OkHttpResult content = iterator.next(); + record = new OkHttpDocumentRecord(content, metadata); + } else if (hasContent) { + OkHttpResult content = result; + record = new OkHttpDocumentRecord(content); + } else if (hasMetadata) { + OkHttpResult metadata = result; + record = new OkHttpDocumentRecord(null, metadata); + } else { + throw new IllegalStateException("Should never have neither content nor metadata"); + } + return record; + } + + @Override + public T nextContent(T contentHandle) { + return next().getContent(contentHandle); + } + + @Override + public void close() { + if (iterator != null) iterator.close(); + } + } /** * Uses v1/documents. */ - private OkHttpResultIterator getBulkDocumentsImpl(RequestLogger reqlog, long serverTimestamp, - Transaction transaction, Set categories, - Format format, RequestParameters extraParams, boolean withContent, - String... uris) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - - String path = "documents"; - RequestParameters params = new RequestParameters(); - if ( extraParams != null ) params.putAll(extraParams); - if (serverTimestamp != -1) params.add("timestamp", Long.toString(serverTimestamp)); - addCategoryParams(categories, params, withContent); - if (format != null) params.add("format", format.toString().toLowerCase()); - for (String uri: uris) { - if ( uri != null && uri.length() > 0 ) { - params.add("uri", uri); - } - } - - OkHttpResultIterator iterator = getIteratedResourceImpl(DefaultOkHttpResultIterator::new, - reqlog, path, transaction, params); - if ( iterator != null ) { - if ( iterator.getStart() == -1 ) iterator.setStart(1); - if ( iterator.getSize() != -1 ) { - if ( iterator.getPageSize() == -1 ) iterator.setPageSize(iterator.getSize()); - if ( iterator.getTotalSize() == -1 ) iterator.setTotalSize(iterator.getSize()); - } - } - return iterator; - } + private OkHttpResultIterator getBulkDocumentsImpl(RequestLogger reqlog, long serverTimestamp, + Transaction transaction, Set categories, + Format format, RequestParameters extraParams, boolean withContent, + String... uris) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + + String path = "documents"; + RequestParameters params = new RequestParameters(); + if (extraParams != null) params.putAll(extraParams); + if (serverTimestamp != -1) params.add("timestamp", Long.toString(serverTimestamp)); + addCategoryParams(categories, params, withContent); + if (format != null) params.add("format", format.toString().toLowerCase()); + for (String uri : uris) { + if (uri != null && uri.length() > 0) { + params.add("uri", uri); + } + } - /** - * Uses v1/search. - */ - private OkHttpResultIterator getBulkDocumentsImpl(RequestLogger reqlog, long serverTimestamp, - SearchQueryDefinition querydef, long start, long pageLength, - Transaction transaction, SearchReadHandle searchHandle, QueryView view, - Set categories, Format format, ServerTransform responseTransform, - RequestParameters extraParams, String forestName) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - try { - RequestParameters params = new RequestParameters(); - if ( extraParams != null ) params.putAll(extraParams); - addCategoryParams(categories, params, true); - if ( searchHandle != null && view != null ) params.add("view", view.toString().toLowerCase()); - if ( start > 1 ) params.add("start", Long.toString(start)); - if ( pageLength >= 0 ) params.add("pageLength", Long.toString(pageLength)); - if (serverTimestamp != -1) params.add("timestamp", Long.toString(serverTimestamp)); - addPointInTimeQueryParam(params, searchHandle); - if ( format != null ) params.add("format", format.toString().toLowerCase()); - HandleImplementation handleBase = HandleAccessor.as(searchHandle); - if ( format == null && searchHandle != null ) { - if ( Format.XML == handleBase.getFormat() ) { - params.add("format", "xml"); - } else if ( Format.JSON == handleBase.getFormat() ) { - params.add("format", "json"); - } - } - - OkHttpSearchRequest request = - generateSearchRequest(reqlog, querydef, MIMETYPE_MULTIPART_MIXED, transaction, responseTransform, params, forestName); - Response response = request.getResponse(); - if ( response == null ) return null; - MimeMultipart entity = null; - if ( searchHandle != null ) { - updateServerTimestamp(handleBase, response.headers()); - ResponseBody body = response.body(); - if ( body.contentLength() != 0 ) { - entity = getEntity(body, MimeMultipart.class); - if ( entity != null ) { - List partList = getPartList(entity); - if ( entity.getCount() > 0 ) { - BodyPart searchResponsePart = entity.getBodyPart(0); - handleBase.receiveContent(getEntity(searchResponsePart, handleBase.receiveAs())); - partList = partList.subList(1, partList.size()); - } - Closeable closeable = response; - return makeResults(OkHttpServiceResultIterator::new, reqlog, "read", "resource", partList, response, - closeable); - } - } - } - return makeResults(OkHttpServiceResultIterator::new, reqlog, "read", "resource", response); - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } - } - - private boolean getDocumentImpl(RequestLogger reqlog, - DocumentDescriptor desc, Transaction transaction, - Set categories, RequestParameters extraParams, - String metadataFormat, DocumentMetadataReadHandle metadataHandle, - AbstractReadHandle contentHandle) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - String uri = desc.getUri(); - if (uri == null) { - throw new IllegalArgumentException( - "Document read for document identifier without uri"); - } - - assert metadataHandle != null : "metadataHandle is null"; - assert contentHandle != null : "contentHandle is null"; - - logger.debug("Getting multipart for {} in transaction {}", uri, getTransactionId(transaction)); - - addPointInTimeQueryParam(extraParams, contentHandle); - - RequestParameters docParams = makeDocumentParams(uri, categories, transaction, extraParams, true); - docParams.add("format", metadataFormat); - - Request.Builder requestBldr = makeDocumentResource(docParams); - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - requestBldr = addVersionHeader(desc, requestBldr, "If-None-Match"); - - Function doGetFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return sendRequestOnce(funcBuilder.addHeader(HEADER_ACCEPT, multipartMixedWithBoundary()).get()); - } - }; - Response response = sendRequestWithRetry(requestBldr, (transaction == null), doGetFunction, null); - int status = response.code(); - if (status == STATUS_NOT_FOUND) { - throw new ResourceNotFoundException( - "Could not read non-existent document", - extractErrorFields(response)); - } - if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException( - "User is not allowed to read documents", - extractErrorFields(response)); - } - if (status == STATUS_NOT_MODIFIED) { - closeResponse(response); - return false; - } - if (status != STATUS_OK) { - throw new FailedRequestException("read failed: " - + getReasonPhrase(response), extractErrorFields(response)); - } - - logRequest( - reqlog, - "read %s document from %s transaction with %s metadata categories and content", - uri, (transaction != null) ? transaction.getTransactionId() : "no", stringJoin(categories, ", ", "no")); - - try { - ResponseBody body = response.body(); - MimeMultipart entity = body.contentLength() != 0 ? - getEntity(body, MimeMultipart.class) : null; - if (entity == null) return false; - - int partCount = entity.getCount(); - if (partCount == 0) return false; - List partList = getPartList(entity); - - if (partCount != 2) { - throw new FailedRequestException("read expected 2 parts but got " + partCount + " parts", - extractErrorFields(response)); - } - - HandleImplementation metadataBase = HandleAccessor.as(metadataHandle); - HandleImplementation contentBase = HandleAccessor.as(contentHandle); - - BodyPart contentPart = partList.get(1); - - Headers responseHeaders = response.headers(); - if (isExternalDescriptor(desc)) { - updateVersion(desc, responseHeaders); - updateFormat(desc, responseHeaders); - updateMimetype(desc, getHeaderMimetype(getHeader(contentPart, HEADER_CONTENT_TYPE))); - updateLength(desc, getHeaderLength(getHeader(contentPart, HEADER_CONTENT_LENGTH))); - copyDescriptor(desc, contentBase); - } else { - updateDescriptor(contentBase, responseHeaders); - } - - metadataBase.receiveContent(getEntity(partList.get(0), - metadataBase.receiveAs())); - - Object contentEntity = getEntity(contentPart, contentBase.receiveAs()); - contentBase.receiveContent((reqlog != null) ? reqlog.copyContent(contentEntity) : contentEntity); - - closeResponse(response); - - return true; - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } - } - - @Override - public DocumentDescriptor head(RequestLogger reqlog, String uri, - Transaction transaction) - throws ForbiddenUserException, FailedRequestException - { - Response response = headImpl(reqlog, uri, transaction, makeDocumentResource(makeDocumentParams(uri, - null, transaction, null))); - - // 404 - if (response == null) return null; - - Headers responseHeaders = response.headers(); - - closeResponse(response); - logRequest(reqlog, "checked %s document from %s transaction", uri, - (transaction != null) ? transaction.getTransactionId() : "no"); - - DocumentDescriptorImpl desc = new DocumentDescriptorImpl(uri, false); - - updateVersion(desc, responseHeaders); - updateDescriptor(desc, responseHeaders); - - return desc; - } - - @Override - public boolean exists(String uri) throws ForbiddenUserException, FailedRequestException { - return headImpl(null, uri, null, setupRequest(uri, null)) == null ? false : true; - } - - @Override - public ConnectionResult checkConnection() { - Request.Builder request = new Request.Builder() - .url(this.baseUri); - Response response = headImplExec(null, this.baseUri.uri().toString(), null, request); - ConnectionResultImpl connectionResultImpl = new ConnectionResultImpl(); - int statusCode = response.code(); - if(statusCode < 300) { - connectionResultImpl.setConnected(true); - } - else { - connectionResultImpl.setConnected(false); - connectionResultImpl.setStatusCode(statusCode); - connectionResultImpl.setErrorMessage(getReasonPhrase(response)); - } - return connectionResultImpl; - } - - private Response headImpl(RequestLogger reqlog, String uri, - Transaction transaction, Request.Builder requestBldr) { - Response response = headImplExec(reqlog, uri, transaction, requestBldr); - int status = response.code(); - if (status != STATUS_OK) { - if (status == STATUS_NOT_FOUND) { - closeResponse(response); - return null; - } else if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException( - "User is not allowed to check the existence of documents", - extractErrorFields(response)); - } else { - throw new FailedRequestException( - "Document existence check failed: " - + getReasonPhrase(response), - extractErrorFields(response)); - } - } - return response; - } - - private Response headImplExec(RequestLogger reqlog, String uri, - Transaction transaction, Request.Builder requestBldr) { - if (uri == null) { - throw new IllegalArgumentException( - "Existence check for document identifier without uri"); - } - - logger.debug("Requesting head for {} in transaction {}", uri, getTransactionId(transaction)); - - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - - Function doHeadFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return sendRequestOnce(funcBuilder.head().build()); - } - }; - return sendRequestWithRetry(requestBldr, (transaction == null), doHeadFunction, null); - } - - @Override - public TemporalDescriptor putDocument(RequestLogger reqlog, DocumentDescriptor desc, - Transaction transaction, Set categories, - RequestParameters extraParams, - DocumentMetadataWriteHandle metadataHandle, - AbstractWriteHandle contentHandle) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - if (desc.getUri() == null) { - throw new IllegalArgumentException( - "Document write for document identifier without uri"); - } - - HandleImplementation metadataBase = HandleAccessor.checkHandle( - metadataHandle, "metadata"); - HandleImplementation contentBase = HandleAccessor.checkHandle( - contentHandle, "content"); - - String metadataMimetype = null; - if (metadataBase != null) { - metadataMimetype = metadataBase.getMimetype(); - } - - Format descFormat = desc.getFormat(); - String contentMimetype = (descFormat != null && descFormat != Format.UNKNOWN) ? desc.getMimetype() : null; - if (contentMimetype == null && contentBase != null) { - Format contentFormat = contentBase.getFormat(); - if (descFormat != null && descFormat != contentFormat) { - contentMimetype = descFormat.getDefaultMimetype(); - } else if (contentFormat != null && contentFormat != Format.UNKNOWN) { - contentMimetype = contentBase.getMimetype(); - } - } - - if (metadataBase != null && contentBase != null) { - return putPostDocumentImpl(reqlog, "put", desc, transaction, categories, - extraParams, metadataMimetype, metadataHandle, - contentMimetype, contentHandle); - } else if (metadataBase != null) { - return putPostDocumentImpl(reqlog, "put", desc, transaction, categories, false, - extraParams, metadataMimetype, metadataHandle); - } else if (contentBase != null) { - return putPostDocumentImpl(reqlog, "put", desc, transaction, null, true, - extraParams, contentMimetype, contentHandle); - } - throw new IllegalArgumentException("Either metadataHandle or contentHandle must not be null"); - } - - @Override - public DocumentDescriptorImpl postDocument(RequestLogger reqlog, DocumentUriTemplate template, - Transaction transaction, Set categories, RequestParameters extraParams, - DocumentMetadataWriteHandle metadataHandle, AbstractWriteHandle contentHandle) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - DocumentDescriptorImpl desc = new DocumentDescriptorImpl(false); - - HandleImplementation metadataBase = HandleAccessor.checkHandle( - metadataHandle, "metadata"); - HandleImplementation contentBase = HandleAccessor.checkHandle( - contentHandle, "content"); - - String metadataMimetype = null; - if (metadataBase != null) { - metadataMimetype = metadataBase.getMimetype(); - } - - Format templateFormat = template.getFormat(); - String contentMimetype = (templateFormat != null && templateFormat != Format.UNKNOWN) ? - template.getMimetype() : null; - if (contentMimetype == null && contentBase != null) { - Format contentFormat = contentBase.getFormat(); - if (templateFormat != null && templateFormat != contentFormat) { - contentMimetype = templateFormat.getDefaultMimetype(); - desc.setFormat(templateFormat); - } else if (contentFormat != null && contentFormat != Format.UNKNOWN) { - contentMimetype = contentBase.getMimetype(); - desc.setFormat(contentFormat); - } - } - desc.setMimetype(contentMimetype); - - if (extraParams == null) extraParams = new RequestParameters(); - - String extension = template.getExtension(); - if (extension != null) extraParams.add("extension", extension); - - String directory = template.getDirectory(); - if (directory != null) extraParams.add("directory", directory); - - if (metadataBase != null && contentBase != null) { - putPostDocumentImpl(reqlog, "post", desc, transaction, categories, extraParams, - metadataMimetype, metadataHandle, contentMimetype, contentHandle); - } else if (contentBase != null) { - putPostDocumentImpl(reqlog, "post", desc, transaction, null, true, extraParams, - contentMimetype, contentHandle); - } - - return desc; - } - - private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String method, DocumentDescriptor desc, - Transaction transaction, Set categories, boolean isOnContent, RequestParameters extraParams, - String mimetype, AbstractWriteHandle handle) - throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException - { - String uri = desc.getUri(); - - HandleImplementation handleBase = HandleAccessor.as(handle); - - logger.debug("Sending {} document in transaction {}", - (uri != null) ? uri : "new", getTransactionId(transaction)); - - logRequest( - reqlog, - "writing %s document from %s transaction with %s mime type and %s metadata categories", - (uri != null) ? uri : "new", - (transaction != null) ? transaction.getTransactionId() : "no", - (mimetype != null) ? mimetype : "no", - stringJoin(categories, ", ", "no")); - - Request.Builder requestBldr = makeDocumentResource( - makeDocumentParams( - uri, categories, transaction, extraParams, isOnContent - )); - - requestBldr = requestBldr.header(HEADER_CONTENT_TYPE, - (mimetype != null) ? mimetype : MIMETYPE_WILDCARD); - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - if (uri != null) { - requestBldr = addVersionHeader(desc, requestBldr, "If-Match"); - } - - if ("patch".equals(method)) { - requestBldr = requestBldr.header("X-HTTP-Method-Override", "PATCH"); - method = "post"; - } - boolean isResendable = handleBase.isResendable(); - - Response response = null; - int status = -1; - Headers responseHeaders = null; - long startTime = System.currentTimeMillis(); - int nextDelay = 0; - int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { - if (nextDelay > 0) { - try { - Thread.sleep(nextDelay); - } catch (InterruptedException e) { - } - } - - Object value = handleBase.sendContent(); - if (value == null) { - throw new IllegalArgumentException( - "Document write with null value for " + ((uri != null) ? uri : "new document")); - } - - if (isFirstRequest() && !isResendable && isStreaming(value)) { - nextDelay = makeFirstRequest(retry); - if (nextDelay != 0) continue; - } - - MediaType mediaType = makeType(requestBldr.build().header(HEADER_CONTENT_TYPE)); - if (value instanceof OutputStreamSender) { - StreamingOutputImpl sentStream = - new StreamingOutputImpl((OutputStreamSender) value, reqlog, mediaType); - requestBldr = - ("put".equals(method)) ? - requestBldr.put(sentStream) : - requestBldr.post(sentStream); - } else { - Object sentObj = (reqlog != null) ? - reqlog.copyContent(value) : value; - requestBldr = - ("put".equals(method)) ? - requestBldr.put(new ObjectRequestBody(sentObj, mediaType)) : - requestBldr.post(new ObjectRequestBody(sentObj, mediaType)); - } - response = sendRequestOnce(requestBldr); - - status = response.code(); - - responseHeaders = response.headers(); - if (transaction != null || !retryStatus.contains(status)) { - if (isFirstRequest()) setFirstRequest(false); - - break; - } - - String retryAfterRaw = response.header("Retry-After"); - closeResponse(response); - - if (!isResendable) { - checkFirstRequest(); - throw new ResourceNotResendableException( - "Cannot retry request for " + - ((uri != null) ? uri : "new document")); - } - - int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); - } - if (retryStatus.contains(status)) { - checkFirstRequest(); - closeResponse(response); - throw new FailedRetryException( - "Service unavailable and maximum retry period elapsed: "+ - ((System.currentTimeMillis() - startTime) / 1000)+ - " seconds after "+retry+" retries"); - } - if (status == STATUS_NOT_FOUND) { - throw new ResourceNotFoundException( - "Could not write non-existent document", - extractErrorFields(response)); - } - if (status == STATUS_PRECONDITION_REQUIRED) { - FailedRequest failure = extractErrorFields(response); - if (failure.getMessageCode().equals("RESTAPI-CONTENTNOVERSION")) { - throw new ContentNoVersionException("Content version required to write document", failure); - } - throw new FailedRequestException( - "Precondition required to write document", failure); - } else if (status == STATUS_FORBIDDEN) { - FailedRequest failure = extractErrorFields(response); - throw new ForbiddenUserException( - "User is not allowed to write documents", failure); - } - if (status == STATUS_PRECONDITION_FAILED) { - FailedRequest failure = extractErrorFields(response); - if (failure.getMessageCode().equals("RESTAPI-CONTENTWRONGVERSION")) { - throw new ContentWrongVersionException("Content version must match to write document", failure); - } else if (failure.getMessageCode().equals("RESTAPI-EMPTYBODY")) { - throw new FailedRequestException( - "Empty request body sent to server", failure); - } - throw new FailedRequestException("Precondition Failed", failure); - } - if (status == -1) { - throw new FailedRequestException("write failed: Unknown Reason", extractErrorFields(response)); - } - if (status != STATUS_CREATED && status != STATUS_NO_CONTENT) { - throw new FailedRequestException("write failed: " - + getReasonPhrase(response), extractErrorFields(response)); - } - - if (uri == null) { - String location = response.header("Location"); - if (location != null) { - int offset = location.indexOf(DOCUMENT_URI_PREFIX); - if (offset == -1) { - closeResponse(response); - throw new MarkLogicInternalException( - "document create produced invalid location: " + location); - } - uri = location.substring(offset + DOCUMENT_URI_PREFIX.length()); - if (uri == null) { - closeResponse(response); - throw new MarkLogicInternalException( - "document create produced location without uri: " + location); - } - desc.setUri(uri); - updateVersion(desc, responseHeaders); - updateDescriptor(desc, responseHeaders); - } - } - TemporalDescriptor temporalDesc = updateTemporalSystemTime(desc, responseHeaders); - closeResponse(response); - return temporalDesc; - } - - private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String method, DocumentDescriptor desc, - Transaction transaction, Set categories, RequestParameters extraParams, - String metadataMimetype, DocumentMetadataWriteHandle metadataHandle, String contentMimetype, - AbstractWriteHandle contentHandle) - throws ResourceNotFoundException, ResourceNotResendableException,ForbiddenUserException, FailedRequestException - { - String uri = desc.getUri(); - - logger.debug("Sending {} multipart document in transaction {}", - (uri != null) ? uri : "new", getTransactionId(transaction)); - - logRequest( - reqlog, - "writing %s document from %s transaction with %s metadata categories and content", - (uri != null) ? uri : "new", - (transaction != null) ? transaction.getTransactionId() : "no", - stringJoin(categories, ", ", "no")); - - RequestParameters docParams = - makeDocumentParams(uri, categories, transaction, extraParams, true); - - Request.Builder requestBldr = makeDocumentResource(docParams) - .addHeader(HEADER_ACCEPT, MIMETYPE_MULTIPART_MIXED); - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - if (uri != null) { - requestBldr = addVersionHeader(desc, requestBldr, "If-Match"); - } - - Response response = null; - int status = -1; - Headers responseHeaders = null; - long startTime = System.currentTimeMillis(); - int nextDelay = 0; - int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { - if (nextDelay > 0) { - try { - Thread.sleep(nextDelay); - } catch (InterruptedException e) { - } - } - - MultipartBody.Builder multiPart = new MultipartBody.Builder(); - boolean hasStreamingPart = addParts(multiPart, reqlog, - new String[] { metadataMimetype, contentMimetype }, - new AbstractWriteHandle[] { metadataHandle, contentHandle }); - - if (isFirstRequest() && hasStreamingPart) { - nextDelay = makeFirstRequest(retry); - if (nextDelay != 0) continue; - } - - requestBldr = ("put".equals(method)) ? requestBldr.put(multiPart.build()) : requestBldr.post(multiPart.build()); - response = sendRequestOnce(requestBldr); - status = response.code(); - - responseHeaders = response.headers(); - if (transaction != null || !retryStatus.contains(status)) { - if (isFirstRequest()) setFirstRequest(false); - - break; - } - String retryAfterRaw = response.header("Retry-After"); - closeResponse(response); - - if (hasStreamingPart) { - throw new ResourceNotResendableException( - "Cannot retry request for " + - ((uri != null) ? uri : "new document")); - } - - int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); - } - if (retryStatus.contains(status)) { - checkFirstRequest(); - closeResponse(response); - throw new FailedRetryException( - "Service unavailable and maximum retry period elapsed: "+ - ((System.currentTimeMillis() - startTime) / 1000)+ - " seconds after "+retry+" retries"); - } - if (status == STATUS_NOT_FOUND) { - closeResponse(response); - throw new ResourceNotFoundException( - "Could not write non-existent document"); - } - if (status == STATUS_PRECONDITION_REQUIRED) { - FailedRequest failure = extractErrorFields(response); - if (failure.getMessageCode().equals("RESTAPI-CONTENTNOVERSION")) { - throw new ContentNoVersionException("Content version required to write document", failure); - } - throw new FailedRequestException( - "Precondition required to write document", failure); - } else if (status == STATUS_FORBIDDEN) { - FailedRequest failure = extractErrorFields(response); - throw new ForbiddenUserException( - "User is not allowed to write documents", failure); - } - if (status == STATUS_PRECONDITION_FAILED) { - FailedRequest failure = extractErrorFields(response); - if (failure.getMessageCode().equals("RESTAPI-CONTENTWRONGVERSION")) { - throw new ContentWrongVersionException("Content version must match to write document", failure); - } else if (failure.getMessageCode().equals("RESTAPI-EMPTYBODY")) { - throw new FailedRequestException( - "Empty request body sent to server", failure); - } - throw new FailedRequestException("Precondition Failed", failure); - } - if (status != STATUS_CREATED && status != STATUS_NO_CONTENT) { - throw new FailedRequestException("write failed: " - + getReasonPhrase(response), extractErrorFields(response)); - } - - if (uri == null) { - String location = response.header("Location"); - if (location != null) { - int offset = location.indexOf(DOCUMENT_URI_PREFIX); - if (offset == -1) { - closeResponse(response); - throw new MarkLogicInternalException( - "document create produced invalid location: " + location); - } - uri = location.substring(offset + DOCUMENT_URI_PREFIX.length()); - if (uri == null) { - closeResponse(response); - throw new MarkLogicInternalException( - "document create produced location without uri: " + location); - } - desc.setUri(uri); - updateVersion(desc, responseHeaders); - updateDescriptor(desc, responseHeaders); - } - } - TemporalDescriptor temporalDesc = updateTemporalSystemTime(desc, responseHeaders); - closeResponse(response); - return temporalDesc; - } - - @Override - public void patchDocument(RequestLogger reqlog, DocumentDescriptor desc, Transaction transaction, - Set categories, boolean isOnContent, DocumentPatchHandle patchHandle) - throws ResourceNotFoundException, ResourceNotResendableException,ForbiddenUserException, FailedRequestException - { - patchDocument(reqlog, desc, transaction, categories, isOnContent, null, null, patchHandle); - } - - @Override - public void patchDocument(RequestLogger reqlog, DocumentDescriptor desc, Transaction transaction, - Set categories, boolean isOnContent, RequestParameters extraParams, String sourceDocumentURI, - DocumentPatchHandle patchHandle) - throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException - { - HandleImplementation patchBase = HandleAccessor.checkHandle(patchHandle, "patch"); - - if(sourceDocumentURI != null) - extraParams.add("source-document", sourceDocumentURI); - putPostDocumentImpl(reqlog, "patch", desc, transaction, categories, isOnContent, extraParams, patchBase.getMimetype(), - patchHandle); - } - - @Override - public Transaction openTransaction(String name, int timeLimit) throws ForbiddenUserException, FailedRequestException { - logger.debug("Opening transaction"); - - RequestParameters transParams = new RequestParameters(); - if ( name != null || timeLimit > 0 ) { - if ( name != null ) transParams.add("name", name); - if ( timeLimit > 0 ) transParams.add("timeLimit", String.valueOf(timeLimit)); - } - - Request.Builder requestBldr = setupRequest("transactions", transParams); - requestBldr = addTelemetryAgentId(requestBldr); - - Function doPostFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return sendRequestOnce(funcBuilder.post(RequestBody.create("", null))); - } - }; - Response response = sendRequestWithRetry(requestBldr, doPostFunction, null); - int status = response.code(); - if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException("User is not allowed to open transactions", extractErrorFields(response)); - } - if (status != STATUS_SEE_OTHER) { - throw new FailedRequestException("transaction open failed: " + - getReasonPhrase(response), extractErrorFields(response)); - } - - String location = response.headers().get("Location"); - List cookies = new ArrayList<>(); - for ( String setCookie : response.headers(HEADER_SET_COOKIE) ) { - ClientCookie cookie = ClientCookie.parse(requestBldr.build().url(), setCookie); - cookies.add(cookie); - } - closeResponse(response); - if (location == null) throw new MarkLogicInternalException("transaction open failed to provide location"); - if (!location.contains("/")) { - throw new MarkLogicInternalException("transaction open produced invalid location: " + location); - } - - String transactionId = location.substring(location.lastIndexOf("/") + 1); - return new TransactionImpl(this, transactionId, cookies); - } - - @Override - public void commitTransaction(Transaction transaction) throws ForbiddenUserException, FailedRequestException { - completeTransaction(transaction, "commit"); - } - - @Override - public void rollbackTransaction(Transaction transaction) throws ForbiddenUserException, FailedRequestException { - completeTransaction(transaction, "rollback"); - } - - private void completeTransaction(Transaction transaction, String result) - throws ForbiddenUserException, FailedRequestException - { - if (result == null) { - throw new MarkLogicInternalException( - "transaction completion without operation"); - } - if (transaction == null) { - throw new MarkLogicInternalException( - "transaction completion without id: " + result); - } - - logger.debug("Completing transaction {} with {}", transaction.getTransactionId(), result); - - RequestParameters transParams = new RequestParameters(); - transParams.add("result", result); - - Request.Builder requestBldr = setupRequest("transactions/" + transaction.getTransactionId(), transParams); - - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - - Function doPostFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return sendRequestOnce(funcBuilder.post(RequestBody.create("", null)).build()); - } - }; - Response response = sendRequestWithRetry(requestBldr, false, doPostFunction, null); - int status = response.code(); - - if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException( - "User is not allowed to complete transaction with " - + result, extractErrorFields(response)); - } - if (status != STATUS_NO_CONTENT) { - throw new FailedRequestException("transaction " + result - + " failed: " + getReasonPhrase(response), - extractErrorFields(response)); - } - - closeResponse(response); - } - - private void addCategoryParams(Set categories, RequestParameters params, boolean withContent) - { - if (withContent) - params.add("category", "content"); - if (categories != null && categories.size() > 0) { - if (categories.contains(Metadata.ALL)) { - addCategoryParam(params, "metadata"); - } else { - for (Metadata category : categories) { - addCategoryParam(params, category); - } - } - } - } - private void addCategoryParam(RequestParameters params, Metadata category) { - addCategoryParam(params, category.toString().toLowerCase()); - } - private void addCategoryParam(RequestParameters params, String category) { - params.add("category", category); - } - - private RequestParameters makeDocumentParams(String uri, - Set categories, Transaction transaction, - RequestParameters extraParams) { - return makeDocumentParams(uri, categories, transaction, extraParams, - false); - } - - private RequestParameters makeDocumentParams(String uri, Set categories, Transaction transaction, - RequestParameters extraParams, boolean withContent) - { - RequestParameters docParams = new RequestParameters(); - if (extraParams != null && extraParams.size() > 0) { - for ( Map.Entry> entry : extraParams.entrySet() ) { - for ( String value : entry.getValue() ) { - String extraKey = entry.getKey(); - if ( !"range".equalsIgnoreCase(extraKey) ) { - docParams.add(extraKey, value); - } - } - } - } - if ( uri != null ) docParams.add("uri", uri); - if (categories == null || categories.size() == 0) { - docParams.add("category", "content"); - } else { - if (withContent) { - docParams.add("category", "content"); - } - if (categories.contains(Metadata.ALL)) { - docParams.add("category", "metadata"); - } else { - for (Metadata category : categories) { - docParams.add("category", category.toString().toLowerCase()); - } - } - } - if (transaction != null) { - docParams.add("txid", transaction.getTransactionId()); - } - return docParams; - } - - private Request.Builder makeDocumentResource(RequestParameters queryParams) { - return setupRequest("documents", queryParams); - } - - static private boolean isExternalDescriptor(ContentDescriptor desc) { - return desc != null && desc instanceof DocumentDescriptorImpl - && !((DocumentDescriptorImpl) desc).isInternal(); - } - - static private void updateDescriptor(ContentDescriptor desc, - Headers headers) { - if (desc == null || headers == null) return; - - updateFormat(desc, headers); - updateMimetype(desc, headers); - updateLength(desc, headers); - updateServerTimestamp(desc, headers); - } - - static private TemporalDescriptor updateTemporalSystemTime(DocumentDescriptor desc, - Headers headers) - { - if (headers == null) return null; - - DocumentDescriptorImpl temporalDescriptor; - if ( desc instanceof DocumentDescriptorImpl ) { - temporalDescriptor = (DocumentDescriptorImpl) desc; - } else { - temporalDescriptor = new DocumentDescriptorImpl(desc.getUri(), false); - } - temporalDescriptor.setTemporalSystemTime(headers.get(HEADER_X_MARKLOGIC_SYSTEM_TIME)); - return temporalDescriptor; - } - - static private void copyDescriptor(DocumentDescriptor desc, - HandleImplementation handleBase) { - if (handleBase == null) return; - - if (desc.getFormat() != null) handleBase.setFormat(desc.getFormat()); - if (desc.getMimetype() != null) handleBase.setMimetype(desc.getMimetype()); - handleBase.setByteLength(desc.getByteLength()); - } - - static private void updateFormat(ContentDescriptor descriptor, - Headers headers) { - updateFormat(descriptor, getHeaderFormat(headers)); - } - - static private void updateFormat(ContentDescriptor descriptor, Format format) { - if (format != null) { - descriptor.setFormat(format); - } - } - - static private Format getHeaderFormat(Headers headers) { - String format = headers.get(HEADER_VND_MARKLOGIC_DOCUMENT_FORMAT); - if (format != null && format.length() > 0) { - return Format.valueOf(format.toUpperCase()); - } - String contentType = headers.get(HEADER_CONTENT_TYPE); - if (contentType != null && contentType.length() > 0) { - return Format.getFromMimetype(contentType); - } - return null; - } - - static private Format getHeaderFormat(BodyPart part) { - String contentDisposition = getHeader(part, HEADER_CONTENT_DISPOSITION); - String formatRegex = ".* format=(text|binary|xml|json).*"; - String format = getHeader(part, HEADER_VND_MARKLOGIC_DOCUMENT_FORMAT); - String contentType = getHeader(part, HEADER_CONTENT_TYPE); - if ( format != null && format.length() > 0 ) { - return Format.valueOf(format.toUpperCase()); - } else if ( contentDisposition != null && contentDisposition.matches(formatRegex) ) { - format = contentDisposition.replaceFirst("^.*" + formatRegex + ".*$", "$1"); - return Format.valueOf(format.toUpperCase()); - } else if ( contentType != null && contentType.length() > 0 ) { - return Format.getFromMimetype(contentType); - } - return null; - } - - static private void updateMimetype(ContentDescriptor descriptor, - Headers headers) { - updateMimetype(descriptor, getHeaderMimetype(headers.get(HEADER_CONTENT_TYPE))); - } - - static private void updateMimetype(ContentDescriptor descriptor, String mimetype) { - if (mimetype != null) { - descriptor.setMimetype(mimetype); - } - } - - static private String getHeader(Map> headers, String name) { - List values = headers.get(name); - if ( values != null && values.size() > 0 ) { - return values.get(0); - } - return null; - } - - static private String getHeader(BodyPart part, String name) { - if ( part == null ) throw new MarkLogicInternalException("part must not be null"); - try { - String[] values = part.getHeader(name); - if ( values != null && values.length > 0 ) { - return values[0]; - } - return null; - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } - } - - static private String getHeaderMimetype(String contentType) { - if (contentType != null) { - int offset = contentType.indexOf(";"); - String mimetype = (offset == -1) ? contentType : contentType.substring(0, offset); - // TODO: if "; charset=foo" set character set - if (mimetype != null && mimetype.length() > 0) { - return mimetype; - } - } - return null; - } - - static private void updateLength(ContentDescriptor descriptor, - Headers headers) { - updateLength(descriptor, getHeaderLength(headers.get(HEADER_CONTENT_LENGTH))); - } - - static private void updateLength(ContentDescriptor descriptor, long length) { - descriptor.setByteLength(length); - } - - static private void updateServerTimestamp(ContentDescriptor descriptor, - Headers headers) { - updateServerTimestamp(descriptor, getHeaderServerTimestamp(headers)); - } - - static private long getHeaderServerTimestamp(Headers headers) { - return Utilities.parseLong(headers.get(HEADER_ML_EFFECTIVE_TIMESTAMP)); - } - - static private void updateServerTimestamp(ContentDescriptor descriptor, long timestamp) { - if ( descriptor instanceof HandleImplementation ) { - if ( descriptor != null && timestamp != -1 ) { - ((HandleImplementation) descriptor).setResponseServerTimestamp(timestamp); - } - } - } - - static private long getHeaderLength(String length) { - return Utilities.parseLong(length, ContentDescriptor.UNKNOWN_LENGTH); - } - - static private String getHeaderUri(BodyPart part) { - try { - if ( part != null ) { - return part.getFileName(); - } - // if it's not found, just return null - return null; - } catch(MessagingException e) { - throw new MarkLogicIOException(e); - } - } - - static private void updateVersion(DocumentDescriptor descriptor, Headers headers) { - updateVersion(descriptor, extractVersion(headers.get(HEADER_ETAG))); - } - static private void updateVersion(DocumentDescriptor descriptor, String header) { - updateVersion(descriptor, extractVersion(header)); - } - static private void updateVersion(DocumentDescriptor descriptor, long version) { - descriptor.setVersion(version); - } - static private long extractVersion(String header) { - if (header != null && header.length() > 0) { - // trim the double quotes - return Long.parseLong(header.substring(1, header.length() - 1)); - } - return DocumentDescriptor.UNKNOWN_VERSION; - } - - static private Request.Builder addVersionHeader(DocumentDescriptor desc, Request.Builder requestBldr, String name) { - if ( desc != null && - desc instanceof DocumentDescriptorImpl && - !((DocumentDescriptorImpl) desc).isInternal()) - { - long version = desc.getVersion(); - if (version != DocumentDescriptor.UNKNOWN_VERSION) { - return requestBldr.header(name, "\"" + String.valueOf(version) + "\""); - } - } - return requestBldr; - } - - static private R updateHandle(BodyPart part, R handle) { - HandleImplementation handleBase = HandleAccessor.as(handle); - - updateFormat(handleBase, getHeaderFormat(part)); - updateMimetype(handleBase, getHeaderMimetype(OkHttpServices.getHeader(part, HEADER_CONTENT_TYPE))); - updateLength(handleBase, getHeaderLength(OkHttpServices.getHeader(part, HEADER_CONTENT_LENGTH))); - handleBase.receiveContent(getEntity(part, handleBase.receiveAs())); - - return handle; - } - static private R updateHandle(Headers headers, ResponseBody body, R handle) { - HandleImplementation handleBase = HandleAccessor.as(handle); - - updateFormat(handleBase, getHeaderFormat(headers)); - updateMimetype(handleBase, getHeaderMimetype(headers.get(HEADER_CONTENT_TYPE))); - updateLength(handleBase, getHeaderLength(headers.get(HEADER_CONTENT_LENGTH))); - handleBase.receiveContent(getEntity(body, handleBase.receiveAs())); - - return handle; - } - - @Override - public T search(RequestLogger reqlog, T searchHandle, - SearchQueryDefinition queryDef, long start, long len, QueryView view, - Transaction transaction, String forestName) - throws ForbiddenUserException, FailedRequestException - { - RequestParameters params = new RequestParameters(); - - if (start > 1) { - params.add("start", Long.toString(start)); - } - - if (len > -1) { - params.add("pageLength", Long.toString(len)); - } - - if (view != null && view != QueryView.DEFAULT) { - if (view == QueryView.ALL) { - params.add("view", "all"); - } else if (view == QueryView.RESULTS) { - params.add("view", "results"); - } else if (view == QueryView.FACETS) { - params.add("view", "facets"); - } else if (view == QueryView.METADATA) { - params.add("view", "metadata"); - } - } - - addPointInTimeQueryParam(params, searchHandle); - - @SuppressWarnings("rawtypes") - HandleImplementation searchBase = HandleAccessor.checkHandle(searchHandle, "search"); - - Format searchFormat = searchBase.getFormat(); - switch(searchFormat) { - case UNKNOWN: - searchFormat = Format.XML; - break; - case JSON: - case XML: - break; - default: - throw new UnsupportedOperationException("Only XML and JSON search results are possible."); - } - - String mimetype = searchFormat.getDefaultMimetype(); - - OkHttpSearchRequest request = generateSearchRequest(reqlog, queryDef, mimetype, transaction, null, params, forestName); - - Response response = request.getResponse(); - if ( response == null ) return null; - - Class as = searchBase.receiveAs(); - - - ResponseBody body = response.body(); - Object entity = body.contentLength() != 0 ? getEntity(body, as) : null; - if (entity == null || (as != InputStream.class && as != Reader.class)) { - closeResponse(response); - } - searchBase.receiveContent(entity); - updateDescriptor(searchBase, response.headers()); - - logRequest( reqlog, - "searched starting at %s with length %s in %s transaction with %s mime type", - start, len, getTransactionId(transaction), mimetype); - - return searchHandle; - } - - private OkHttpSearchRequest generateSearchRequest(RequestLogger reqlog, SearchQueryDefinition queryDef, - String mimetype, Transaction transaction, ServerTransform responseTransform, - RequestParameters params, String forestName) - { - if ( params == null ) params = new RequestParameters(); - if ( forestName != null ) params.add("forest-name", forestName); - return new OkHttpSearchRequest(reqlog, queryDef, mimetype, transaction, responseTransform, params); - } - - private class OkHttpSearchRequest { - RequestLogger reqlog; - SearchQueryDefinition queryDef; - String mimetype; - RequestParameters params; - ServerTransform responseTransform; - Transaction transaction; - - Request.Builder requestBldr = null; - String structure = null; - HandleImplementation baseHandle = null; - - OkHttpSearchRequest(RequestLogger reqlog, SearchQueryDefinition queryDef, String mimetype, - Transaction transaction, ServerTransform responseTransform, RequestParameters params) { - this.reqlog = reqlog; - this.queryDef = queryDef; - this.mimetype = mimetype; - this.transaction = transaction; - this.responseTransform = responseTransform; - this.params = params != null ? params : new RequestParameters(); - addParams(); - init(); - } - - void addParams() { - if (queryDef instanceof QueryDefinition) { - String directory = ((QueryDefinition) queryDef).getDirectory(); - if (directory != null) { - params.add("directory", directory); - } - - params.add("collection", ((QueryDefinition) queryDef).getCollections()); - } - - String optionsName = queryDef.getOptionsName(); - if (optionsName != null && optionsName.length() > 0) { - params.add("options", optionsName); - } - - ServerTransform transform = queryDef.getResponseTransform(); - if (transform != null) { - if (responseTransform != null) { - if ( ! transform.getName().equals(responseTransform.getName()) ) { - throw new IllegalStateException("QueryDefinition transform and DocumentManager transform have different names (" + - transform.getName() + ", " + responseTransform.getName() + ")"); - } - logger.warn("QueryDefinition and DocumentManager both specify a ServerTransform--using params from QueryDefinition"); - } - transform.merge(params); - } else if (responseTransform != null) { - responseTransform.merge(params); - } - - if (transaction != null) { - params.add("txid", transaction.getTransactionId()); - } - } - - void init() { - String text = null; - if (queryDef instanceof StringQueryDefinition) { - text = ((StringQueryDefinition) queryDef).getCriteria(); - } else if (queryDef instanceof StructuredQueryDefinition) { - text = ((StructuredQueryDefinition) queryDef).getCriteria(); - } else if (queryDef instanceof RawStructuredQueryDefinition) { - text = ((RawStructuredQueryDefinition) queryDef).getCriteria(); - } else if (queryDef instanceof RawCtsQueryDefinition) { - text = ((RawCtsQueryDefinition) queryDef).getCriteria(); - } - if (text != null) { - params.add("q", text); - } - if (queryDef instanceof StructuredQueryDefinition) { - structure = ((StructuredQueryDefinition) queryDef).serialize(); - - if (logger.isDebugEnabled()) { - String qtextMessage = text == null ? "" : " and string query \"" + text + "\""; - logger.debug("Searching for structure {}{}", structure, qtextMessage); - } - - requestBldr = setupRequest("search", params); - requestBldr = requestBldr.header(HEADER_CONTENT_TYPE, MIMETYPE_APPLICATION_XML); - requestBldr = requestBldr.header(HEADER_ACCEPT, mimetype); - } else if (queryDef instanceof RawQueryDefinition || queryDef instanceof RawCtsQueryDefinition) { - logger.debug("Raw search"); - - if (queryDef instanceof RawQueryDefinition) { - StructureWriteHandle handle = ((RawQueryDefinition) queryDef).getHandle(); - baseHandle = HandleAccessor.checkHandle(handle, "search"); - } else if (queryDef instanceof RawCtsQueryDefinition) { - CtsQueryWriteHandle handle = ((RawCtsQueryDefinition) queryDef).getHandle(); - baseHandle = HandleAccessor.checkHandle(handle, "search"); - } - - Format payloadFormat = getStructuredQueryFormat(baseHandle); - String payloadMimetype = getMimetypeWithDefaultXML(payloadFormat, baseHandle); - - String path = (queryDef instanceof RawQueryByExampleDefinition) ? - "qbe" : "search"; - - requestBldr = setupRequest(path, params); - if ( payloadMimetype != null ) { - requestBldr = requestBldr.header(HEADER_CONTENT_TYPE, payloadMimetype); - } - requestBldr = requestBldr.header(HEADER_ACCEPT, mimetype); - } else if (queryDef instanceof CombinedQueryDefinition) { - structure = ((CombinedQueryDefinition) queryDef).serialize(); - - logger.debug("Searching for combined query {}", structure); - - requestBldr = setupRequest("search", params); - requestBldr = requestBldr - .header(HEADER_CONTENT_TYPE, MIMETYPE_APPLICATION_XML) - .header(HEADER_ACCEPT, mimetype); - } else if (queryDef instanceof StringQueryDefinition) { - logger.debug("Searching for string [{}]", text); - - requestBldr = setupRequest("search", params); - requestBldr = requestBldr.header(HEADER_CONTENT_TYPE, MIMETYPE_APPLICATION_XML); - requestBldr = requestBldr.header(HEADER_ACCEPT, mimetype); - } else if (queryDef instanceof DeleteQueryDefinition) { - logger.debug("Searching for deletes"); - - requestBldr = setupRequest("search", params); - requestBldr = requestBldr.header(HEADER_ACCEPT, mimetype); - } else if (queryDef instanceof CtsQueryDefinition) { - structure = ((CtsQueryDefinition) queryDef).serialize(); - if (logger.isDebugEnabled()) { - logger.debug("Searching Cts Query: ", ((CtsQueryDefinition) queryDef).serialize()); - } - requestBldr = setupRequest("search", params); - requestBldr = requestBldr.header(HEADER_CONTENT_TYPE, MIMETYPE_APPLICATION_JSON); - requestBldr = requestBldr.header(HEADER_ACCEPT, mimetype); - }else { - throw new UnsupportedOperationException("Cannot search with " - + queryDef.getClass().getName()); - } - - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - } - - Response getResponse() { - Response response = null; - int status = -1; - long startTime = System.currentTimeMillis(); - int nextDelay = 0; - int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { - if (nextDelay > 0) { - try { - Thread.sleep(nextDelay); - } catch (InterruptedException e) { - } - } - - if (queryDef instanceof StructuredQueryDefinition && ! (queryDef instanceof RawQueryDefinition)) { - response = doPost(reqlog, requestBldr, structure); - } else if (queryDef instanceof CombinedQueryDefinition) { - response = doPost(reqlog, requestBldr, structure); - } else if (queryDef instanceof DeleteQueryDefinition) { - response = doGet(requestBldr); - } else if (queryDef instanceof RawQueryDefinition) { - response = doPost(reqlog, requestBldr, baseHandle.sendContent()); - } else if (queryDef instanceof RawCtsQueryDefinition) { - response = doPost(reqlog, requestBldr, baseHandle.sendContent()); - } else if (queryDef instanceof StringQueryDefinition) { - response = doGet(requestBldr); - } else if (queryDef instanceof CtsQueryDefinition) { - response = doPost(reqlog, requestBldr, structure); - } else { - throw new UnsupportedOperationException("Cannot search with " - + queryDef.getClass().getName()); - } - - status = response.code(); - - if (transaction != null || !retryStatus.contains(status)) { - if (isFirstRequest()) setFirstRequest(false); - - break; - } - - String retryAfterRaw = response.header("Retry-After"); - int retryAfter = Utilities.parseInt(retryAfterRaw); - - closeResponse(response); - - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); - } - if (retryStatus.contains(status)) { - checkFirstRequest(); - closeResponse(response); - throw new FailedRetryException( - "Service unavailable and maximum retry period elapsed: "+ - ((System.currentTimeMillis() - startTime) / 1000)+ - " seconds after "+retry+" retries"); - } - if (status == STATUS_NOT_FOUND) { - closeResponse(response); - return null; - } - if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException("User is not allowed to search", - extractErrorFields(response)); - } - if (status != STATUS_OK) { - throw new FailedRequestException("search failed: " - + getReasonPhrase(response), extractErrorFields(response)); - } - return response; - } - } - - private Format getStructuredQueryFormat(HandleImplementation baseHandle) { - Format payloadFormat = baseHandle.getFormat(); - if (payloadFormat == Format.UNKNOWN) { - payloadFormat = null; - } else if (payloadFormat != Format.XML && payloadFormat != Format.JSON) { - throw new IllegalArgumentException( - "Cannot perform raw search for format "+payloadFormat.name()); - } - return payloadFormat; - } - - private String getMimetypeWithDefaultXML(Format payloadFormat, HandleImplementation baseHandle) { - String payloadMimetype = baseHandle.getMimetype(); - if (payloadFormat != null) { - if (payloadMimetype == null) { - payloadMimetype = payloadFormat.getDefaultMimetype(); - } - } else if (payloadMimetype == null) { - payloadMimetype = MIMETYPE_APPLICATION_XML; - } - return payloadMimetype; - } - - @Override - public void deleteSearch(RequestLogger reqlog, DeleteQueryDefinition queryDef, - Transaction transaction) - throws ForbiddenUserException, FailedRequestException - { - RequestParameters params = new RequestParameters(); - - if (queryDef.getDirectory() != null) { - params.add("directory", queryDef.getDirectory()); - } - - params.add("collection", queryDef.getCollections()); - - if (transaction != null) { - params.add("txid", transaction.getTransactionId()); - } - - Request.Builder requestBldr = setupRequest("search", params); - - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - - Function doDeleteFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return sendRequestOnce(funcBuilder.delete().build()); - } - }; - Response response = sendRequestWithRetry(requestBldr, (transaction == null), doDeleteFunction, null); - int status = response.code(); - if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException("User is not allowed to delete", - extractErrorFields(response)); - } - - if (status != STATUS_NO_CONTENT) { - throw new FailedRequestException("delete failed: " - + getReasonPhrase(response), extractErrorFields(response)); - } - - closeResponse(response); - - logRequest( - reqlog, - "deleted search results in %s transaction", - getTransactionId(transaction)); - } - - @Override - public void delete(RequestLogger logger, Transaction transaction, Set categories, String... uris) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - RequestParameters params = new RequestParameters(); - addCategoryParams(categories, params, false); - for ( String uri : uris ) { - params.add("uri", uri); - } - deleteResource(logger, "documents", transaction, params, null); - } - - @Override - public T values(Class as, ValuesDefinition valDef, String mimetype, - long start, long pageLength, Transaction transaction) - throws ForbiddenUserException, FailedRequestException - { - RequestParameters docParams = new RequestParameters(); - - String optionsName = valDef.getOptionsName(); - if (optionsName != null && optionsName.length() > 0) { - docParams.add("options", optionsName); - } - - if (valDef.getAggregate() != null) { - docParams.add("aggregate", valDef.getAggregate()); - } - - if (valDef.getAggregatePath() != null) { - docParams.add("aggregatePath", - valDef.getAggregatePath()); - } - - if (valDef.getView() != null) { - docParams.add("view", valDef.getView()); - } - - if (valDef.getDirection() != null) { - if (valDef.getDirection() == ValuesDefinition.Direction.ASCENDING) { - docParams.add("direction", "ascending"); - } else { - docParams.add("direction", "descending"); - } - } - - if (valDef.getFrequency() != null) { - if (valDef.getFrequency() == ValuesDefinition.Frequency.FRAGMENT) { - docParams.add("frequency", "fragment"); - } else { - docParams.add("frequency", "item"); - } - } - - if (start > 0) { - docParams.add("start", Long.toString(start)); - if (pageLength > 0) { - docParams.add("pageLength", Long.toString(pageLength)); - } - } - - HandleImplementation baseHandle = null; - - if (valDef.getQueryDefinition() != null) { - ValueQueryDefinition queryDef = valDef.getQueryDefinition(); - - if (optionsName == null) { - optionsName = queryDef.getOptionsName(); - if (optionsName != null) { - docParams.add("options", optionsName); - } - } else if (queryDef.getOptionsName() != null) { - if (!queryDef.getOptionsName().equals(optionsName)) { - logger.warn("values definition options take precedence over query definition options"); - } - } - - if (queryDef.getCollections().length > 0) { - logger.warn("collections scope ignored for values query"); - } - if (queryDef.getDirectory() != null) { - logger.warn("directory scope ignored for values query"); - } - - String text = null; - if (queryDef instanceof StringQueryDefinition) { - text = ((StringQueryDefinition) queryDef).getCriteria(); - } else if (queryDef instanceof StructuredQueryDefinition) { - text = ((StructuredQueryDefinition) queryDef).getCriteria(); - } else if (queryDef instanceof RawStructuredQueryDefinition) { - text = ((RawStructuredQueryDefinition) queryDef).getCriteria(); - } else if (queryDef instanceof RawCtsQueryDefinition) { - text = ((RawCtsQueryDefinition) queryDef).getCriteria(); - } - if (text != null) { - docParams.add("q", text); - } - if (queryDef instanceof StructuredQueryDefinition ) { - String structure = ((StructuredQueryDefinition) queryDef) - .serialize(); - if (structure != null) { - docParams.add("structuredQuery", structure); - } - } else if (queryDef instanceof RawQueryDefinition) { - StructureWriteHandle handle = ((RawQueryDefinition) queryDef).getHandle(); - baseHandle = HandleAccessor.checkHandle(handle, "values"); - } else if (queryDef instanceof RawCtsQueryDefinition) { - CtsQueryWriteHandle handle = ((RawCtsQueryDefinition) queryDef).getHandle(); - baseHandle = HandleAccessor.checkHandle(handle, "values"); - } else if (queryDef instanceof StringQueryDefinition) { - } else { - logger.warn("unsupported query definition: {}", queryDef.getClass().getName()); - } - - ServerTransform transform = queryDef.getResponseTransform(); - if (transform != null) { - transform.merge(docParams); - } - } - - if (transaction != null) { - docParams.add("txid", transaction.getTransactionId()); - } - - String uri = "values"; - if (valDef.getName() != null) { - uri += "/" + valDef.getName(); - } - - Request.Builder requestBldr = setupRequest(uri, docParams); - requestBldr = setupRequest(requestBldr, null, mimetype); - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - - final HandleImplementation tempBaseHandle = baseHandle; - - Function doFunction = (baseHandle == null) ? - new Function() { - public Response apply(Request.Builder funcBuilder) { - return doGet(funcBuilder); - } - } : - new Function() { - public Response apply(Request.Builder funcBuilder) { - String contentType = tempBaseHandle.getMimetype(); - return doPost(null, - (contentType == null) ? funcBuilder : funcBuilder.header(HEADER_CONTENT_TYPE, contentType), - tempBaseHandle.sendContent()); - } - }; - - Response response = sendRequestWithRetry(requestBldr, (transaction == null), doFunction, null); - int status = response.code(); - - if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException("User is not allowed to search", - extractErrorFields(response)); - } - if (status != STATUS_OK) { - throw new FailedRequestException("search failed: " - + getReasonPhrase(response), extractErrorFields(response)); - } - - ResponseBody body = response.body(); - T entity = body.contentLength() != 0 ? getEntity(body, as) : null; - if (entity == null || (as != InputStream.class && as != Reader.class)) { - closeResponse(response); - } - - return entity; - } - - @Override - public T valuesList(Class as, ValuesListDefinition valDef, - String mimetype, Transaction transaction) - throws ForbiddenUserException, FailedRequestException - { - RequestParameters docParams = new RequestParameters(); - - String optionsName = valDef.getOptionsName(); - if (optionsName != null && optionsName.length() > 0) { - docParams.add("options", optionsName); - } - - if (transaction != null) { - docParams.add("txid", transaction.getTransactionId()); - } - - String uri = "values"; - - Request.Builder requestBldr = setupRequest(uri, docParams); - requestBldr = setupRequest(requestBldr, null, mimetype); - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - - Function doGetFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return sendRequestOnce(funcBuilder.get().build()); - } - }; - Response response = sendRequestWithRetry(requestBldr, (transaction == null), doGetFunction, null); - int status = response.code(); - - if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException("User is not allowed to search", - extractErrorFields(response)); - } - if (status != STATUS_OK) { - throw new FailedRequestException("search failed: " - + getReasonPhrase(response), extractErrorFields(response)); - } - - ResponseBody body = response.body(); - T entity = body.contentLength() != 0 ? getEntity(body, as) : null; - if (entity == null || (as != InputStream.class && as != Reader.class)) { - closeResponse(response); - } - - return entity; - } - - @Override - public T optionsList(Class as, String mimetype, Transaction transaction) - throws ForbiddenUserException, FailedRequestException - { - RequestParameters docParams = new RequestParameters(); - - if (transaction != null) { - docParams.add("txid", transaction.getTransactionId()); - } - - String uri = "config/query"; - - Request.Builder requestBldr = setupRequest(uri, docParams); - requestBldr = requestBldr.header(HEADER_ACCEPT, mimetype); - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - - Function doGetFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return sendRequestOnce(funcBuilder.get().build()); - } - }; - Response response = sendRequestWithRetry(requestBldr, (transaction == null), doGetFunction, null); - int status = response.code(); - - if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException("User is not allowed to search", - extractErrorFields(response)); - } - if (status != STATUS_OK) { - throw new FailedRequestException("search failed: " - + getReasonPhrase(response), extractErrorFields(response)); - } - - ResponseBody body = response.body(); - T entity = body.contentLength() != 0 ? getEntity(body, as) : null; - if (entity == null || (as != InputStream.class && as != Reader.class)) { - closeResponse(response); - } - - return entity; - } - - // namespaces, search options etc. - @Override - public T getValue(RequestLogger reqlog, String type, String key, - boolean isNullable, String mimetype, Class as) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - return getValue(reqlog, type, key, null, isNullable, mimetype, as); - } - @Override - public T getValue(RequestLogger reqlog, String type, String key, Transaction transaction, - boolean isNullable, String mimetype, Class as) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - logger.debug("Getting {}/{}", type, key); - - Request.Builder requestBldr = setupRequest(type + "/" + key, null, null, mimetype); - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - - Function doGetFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return sendRequestOnce(funcBuilder.get().build()); - } - }; - Response response = sendRequestWithRetry(requestBldr, (transaction == null), doGetFunction, null); - int status = response.code(); - - if (status != STATUS_OK) { - if (status == STATUS_NOT_FOUND) { - closeResponse(response); - if (!isNullable) { - throw new ResourceNotFoundException("Could not get " + type + "/" + key); - } - return null; - } else if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException("User is not allowed to read " - + type, extractErrorFields(response)); - } else { - throw new FailedRequestException(type + " read failed: " - + getReasonPhrase(response), - extractErrorFields(response)); - } - } - - logRequest(reqlog, "read %s value with %s key and %s mime type", type, - key, mimetype); - - ResponseBody body = response.body(); - T entity = body.contentLength() != 0 ? getEntity(body, as) : null; - if (entity == null || (as != InputStream.class && as != Reader.class)) { - closeResponse(response); - } - - return (reqlog != null) ? reqlog.copyContent(entity) : entity; - } - - @Override - public T getValues(RequestLogger reqlog, String type, String mimetype, Class as) - throws ForbiddenUserException, FailedRequestException - { - return getValues(reqlog, type, null, mimetype, as); - } - @Override - public T getValues(RequestLogger reqlog, String type, RequestParameters extraParams, - String mimetype, Class as) - throws ForbiddenUserException, FailedRequestException - { - logger.debug("Getting {}", type); - - Request.Builder requestBldr = setupRequest(type, extraParams).header(HEADER_ACCEPT, mimetype); - requestBldr = addTelemetryAgentId(requestBldr); - - Function doGetFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return sendRequestOnce(funcBuilder.get().build()); - } - }; - Response response = sendRequestWithRetry(requestBldr, doGetFunction, null); - int status = response.code(); - if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException("User is not allowed to read " - + type, extractErrorFields(response)); - } - if (status != STATUS_OK) { - throw new FailedRequestException(type + " read failed: " - + getReasonPhrase(response), extractErrorFields(response)); - } - - logRequest(reqlog, "read %s values with %s mime type", type, mimetype); - - ResponseBody body = response.body(); - T entity = body.contentLength() != 0 ? getEntity(body, as) : null; - if (entity == null || (as != InputStream.class && as != Reader.class)) { - closeResponse(response); - } - - return (reqlog != null) ? reqlog.copyContent(entity) : entity; - } - - @Override - public void postValue(RequestLogger reqlog, String type, String key, - String mimetype, Object value) - throws ResourceNotResendableException, ForbiddenUserException, FailedRequestException - { - logger.debug("Posting {}/{}", type, key); - - putPostValueImpl(reqlog, "post", type, key, null, mimetype, value, STATUS_CREATED); - } - @Override - public void postValue(RequestLogger reqlog, String type, String key, - RequestParameters extraParams) - throws ResourceNotResendableException, ForbiddenUserException, FailedRequestException - { - logger.debug("Posting {}/{}", type, key); - - putPostValueImpl(reqlog, "post", type, key, extraParams, null, null, STATUS_NO_CONTENT); - } - - - @Override - public void putValue(RequestLogger reqlog, String type, String key, - String mimetype, Object value) - throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException - { - logger.debug("Putting {}/{}", type, key); - - putPostValueImpl(reqlog, "put", type, key, null, mimetype, value, STATUS_NO_CONTENT, STATUS_CREATED); - } - - @Override - public void putValue(RequestLogger reqlog, String type, String key, - RequestParameters extraParams, String mimetype, Object value) - throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException - { - logger.debug("Putting {}/{}", type, key); - - putPostValueImpl(reqlog, "put", type, key, extraParams, mimetype, value, STATUS_NO_CONTENT); - } - - private void putPostValueImpl(RequestLogger reqlog, String method, - String type, String key, RequestParameters extraParams, - String mimetype, Object value, - int... expectedStatuses) { - if (key != null) { - logRequest(reqlog, "writing %s value with %s key and %s mime type", - type, key, mimetype); - } else { - logRequest(reqlog, "writing %s values with %s mime type", type, mimetype); - } - - HandleImplementation handle = (value instanceof HandleImplementation) ? - (HandleImplementation) value : null; - - MediaType mediaType = makeType(mimetype); - - String connectPath = null; - Request.Builder requestBldr = null; - - Response response = null; - int status = -1; - long startTime = System.currentTimeMillis(); - int nextDelay = 0; - int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { - if (nextDelay > 0) { - try { - Thread.sleep(nextDelay); - } catch (InterruptedException e) { - } - } - - Object nextValue = (handle != null) ? handle.sendContent() : value; - - RequestBody sentValue = null; - if (nextValue instanceof OutputStreamSender) { - sentValue = new StreamingOutputImpl( - (OutputStreamSender) nextValue, reqlog, mediaType); - } else { - if (reqlog != null && retry == 0) { - sentValue = new ObjectRequestBody(reqlog.copyContent(nextValue), mediaType); - } else { - sentValue = new ObjectRequestBody(nextValue, mediaType); - } - } - - boolean isStreaming = (isFirstRequest() || handle == null) ? isStreaming(sentValue) : false; - - boolean isResendable = (handle == null) ? !isStreaming : handle.isResendable(); - - if (isFirstRequest() && !isResendable && isStreaming) { - nextDelay = makeFirstRequest(retry); - if (nextDelay != 0) continue; - } - - if ("put".equals(method)) { - if (requestBldr == null) { - connectPath = (key != null) ? type + "/" + key : type; - Request.Builder resource = setupRequest(connectPath, extraParams); - requestBldr = (mimetype == null) ? - resource : resource.header(HEADER_CONTENT_TYPE, mimetype); - requestBldr = addTelemetryAgentId(requestBldr); - } - - response = (sentValue == null) ? - sendRequestOnce(requestBldr.put(null).build()) : - sendRequestOnce(requestBldr.put(sentValue).build()); - } else if ("post".equals(method)) { - if (requestBldr == null) { - connectPath = type; - Request.Builder resource = setupRequest(connectPath, extraParams); - requestBldr = (mimetype == null) ? - resource : resource.header(HEADER_CONTENT_TYPE, mimetype); - requestBldr = addTelemetryAgentId(requestBldr); - } - - response = (sentValue == null) ? - sendRequestOnce(requestBldr.post(RequestBody.create("", null)).build()) : - sendRequestOnce(requestBldr.post(sentValue).build()); - } else { - throw new MarkLogicInternalException("unknown method type " - + method); - } - - status = response.code(); - - if (!retryStatus.contains(status)) { - if (isFirstRequest()) setFirstRequest(false); - break; - } - - String retryAfterRaw = response.header("Retry-After"); - closeResponse(response); - - if (!isResendable) { - checkFirstRequest(); - throw new ResourceNotResendableException( - "Cannot retry request for " + connectPath); - } - - int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); - } - if (retryStatus.contains(status)) { - checkFirstRequest(); - closeResponse(response); - throw new FailedRetryException( - "Service unavailable and maximum retry period elapsed: "+ - ((System.currentTimeMillis() - startTime) / 1000)+ - " seconds after "+retry+" retries"); - } - if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException("User is not allowed to write " - + type, extractErrorFields(response)); - } - if (status == STATUS_NOT_FOUND) { - throw new ResourceNotFoundException(type + " not found for write", - extractErrorFields(response)); - } - boolean statusOk = false; - for (int expectedStatus : expectedStatuses) { - statusOk = statusOk || (status == expectedStatus); - if (statusOk) { - break; - } - } - - if (!statusOk) { - throw new FailedRequestException(type + " write failed: " - + getReasonPhrase(response), extractErrorFields(response)); - } - closeResponse(response); - } - - @Override - public void deleteValue(RequestLogger reqlog, String type, String key) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - logger.debug("Deleting {}/{}", type, key); - - Request.Builder requestBldr = setupRequest(type + "/" + key, null); - requestBldr = addTelemetryAgentId(requestBldr); - - Function doDeleteFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return sendRequestOnce(funcBuilder.delete().build()); - } - }; - Response response = sendRequestWithRetry(requestBldr, doDeleteFunction, null); - int status = response.code(); - if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException("User is not allowed to delete " - + type, extractErrorFields(response)); - } - if (status == STATUS_NOT_FOUND) { - throw new ResourceNotFoundException(type + " not found for delete", - extractErrorFields(response)); - } - if (status != STATUS_NO_CONTENT) { - throw new FailedRequestException("delete failed: " - + getReasonPhrase(response), extractErrorFields(response)); - } - - closeResponse(response); - - logRequest(reqlog, "deleted %s value with %s key", type, key); - } - - @Override - public void deleteValues(RequestLogger reqlog, String type) - throws ForbiddenUserException, FailedRequestException - { - logger.debug("Deleting {}", type); - - Request.Builder requestBldr = setupRequest(type, null); - requestBldr = addTelemetryAgentId(requestBldr); - - Function doDeleteFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return sendRequestOnce(funcBuilder.delete().build()); - } - }; - Response response = sendRequestWithRetry(requestBldr, doDeleteFunction, null); - int status = response.code(); - if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException("User is not allowed to delete " - + type, extractErrorFields(response)); - } - if (status != STATUS_NO_CONTENT) { - throw new FailedRequestException("delete failed: " - + getReasonPhrase(response), extractErrorFields(response)); - } - closeResponse(response); - - logRequest(reqlog, "deleted %s values", type); - } - - @Override - public R getSystemSchema(RequestLogger reqlog, String schemaName, R output) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - RequestParameters params = new RequestParameters(); - params.add("system", schemaName); - return getResource(reqlog, "internal/schemas", null, params, output); - } - @Override - public R uris(RequestLogger reqlog, String method, SearchQueryDefinition qdef, - Boolean filtered, long start, String afterUri, long pageLength, String forestName, R output - ) throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { - logger.debug("Querying for uris"); - RequestParameters params = new RequestParameters(); - if (filtered != null) params.add("filtered", filtered.toString()); - if (forestName != null) params.add("forest-name", forestName); - if (start > 1) params.add("start", Long.toString(start)); - if (afterUri != null) params.add("after", afterUri); - if (pageLength >= 1) params.add("pageLength", Long.toString(pageLength)); - return processQuery(reqlog, "internal/uris", method, params, qdef, output); - } - @Override - public R forestInfo(RequestLogger reqlog, - String method, RequestParameters params, SearchQueryDefinition qdef, R output - ) throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { - return processQuery(reqlog, "internal/forestinfo", method, params, qdef, output); - } - private R processQuery(RequestLogger reqlog, String path, - String method, RequestParameters params, SearchQueryDefinition qdef, R output - ) throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { - if (qdef instanceof QueryDefinition) { - if (((QueryDefinition)qdef).getDirectory() != null) { - params.add("directory", ((QueryDefinition)qdef).getDirectory()); - } - - if (((QueryDefinition)qdef).getCollections() != null ) { - for ( String collection : ((QueryDefinition)qdef).getCollections() ) { - params.add("collection", collection); - } - } - } - - if (qdef.getOptionsName()!= null && qdef.getOptionsName().length() > 0) { - params.add("options", qdef.getOptionsName()); - } - - if (qdef instanceof RawQueryByExampleDefinition) { - throw new UnsupportedOperationException(path+" cannot process RawQueryByExampleDefinition"); - } - - boolean sendQueryAsPayload = "POST".equals(method); - - String text = null; - String structure = null; - StructureWriteHandle input = null; - if (qdef instanceof RawCtsQueryDefinition) { - if (!(qdef instanceof RawQueryDefinitionImpl.CtsQuery)) { - throw new IllegalArgumentException( - "unknown implementation of RawCtsQueryDefinition: "+qdef.getClass().getName()); - } - RawQueryDefinitionImpl.CtsQuery ctsQuery = (RawQueryDefinitionImpl.CtsQuery) qdef; - text = ctsQuery.getCriteria(); - structure = ctsQuery.serialize(); - logger.debug("{} processing raw cts query {} and string query \"{}\"", path, structure, text); - if (structure != null) { - input = checkStructure(structure, ctsQuery.getHandle()); - } - } else if (qdef instanceof StructuredQueryDefinition) { - StructuredQueryDefinition builtStructuredQuery = (StructuredQueryDefinition) qdef; - text = builtStructuredQuery.getCriteria(); - structure = builtStructuredQuery.serialize(); - logger.debug("{} processing structured query {} and string query \"{}\"", path, structure, text); - if (sendQueryAsPayload && structure != null) { - input = new StringHandle(structure).withFormat(Format.XML); - } - } else if (qdef instanceof RawStructuredQueryDefinition) { - RawStructuredQueryDefinition rawStructuredQuery = (RawStructuredQueryDefinition) qdef; - text = rawStructuredQuery.getCriteria(); - structure = rawStructuredQuery.serialize(); - logger.debug("{} processing raw structured query {} and string query \"{}\"", path, structure, text); - if (sendQueryAsPayload && structure != null) { - input = checkStructure(structure, rawStructuredQuery.getHandle()); - } - } else if (qdef instanceof CombinedQueryDefinition) { - CombinedQueryDefinition combinedQuery = (CombinedQueryDefinition) qdef; - structure = combinedQuery.serialize(); - logger.debug("{} processing combined query {}", path, structure); - if (sendQueryAsPayload && structure != null) { - input = checkStructure(structure, combinedQuery.getFormat()); - } - } else if (qdef instanceof StringQueryDefinition) { - StringQueryDefinition stringQuery = (StringQueryDefinition) qdef; - text = stringQuery.getCriteria(); - logger.debug("{} processing string query \"{}\"", path, text); - } else if (qdef instanceof RawQueryDefinitionImpl) { - RawQueryDefinitionImpl rawQueryImpl = (RawQueryDefinitionImpl) qdef; - structure = rawQueryImpl.serialize(); - logger.debug("{} processing raw query implementation {}", path, structure); - input = checkStructure(structure, rawQueryImpl.getHandle()); - } else if (qdef instanceof RawQueryDefinition) { - RawQueryDefinition rawQuery = (RawQueryDefinition) qdef; - logger.debug("{} processing raw query", path); - input = checkFormat(rawQuery.getHandle()); - } else if (qdef instanceof CtsQueryDefinition) { - CtsQueryDefinition builtCtsQuery = (CtsQueryDefinition) qdef; - structure = builtCtsQuery.serialize(); - logger.debug("{} processing cts query {}", path, structure); - if (sendQueryAsPayload && structure != null) { - input = new StringHandle(structure).withFormat(Format.JSON); - } - }else { - throw new UnsupportedOperationException(path+" cannot process query of "+qdef.getClass().getName()); - } - - if (text != null) { - params.add("q", text); - } - - if (input != null) { - return postResource(reqlog, path, null, params, input, output); - } else if (structure != null) { - params.add("structuredQuery", structure); - } - return getResource(reqlog, path, null, params, output); - } - private StructureWriteHandle checkStructure(String structure, StructureWriteHandle handle) { - return checkStructure(structure, - (handle == null || !(handle instanceof HandleImplementation)) ? Format.UNKNOWN : - ((HandleImplementation) handle).getFormat()); - } - private StructureWriteHandle checkStructure(String structure, Format format) { - return new StringHandle(structure).withFormat( - (format == null || format == Format.UNKNOWN) ? Format.TEXT : format); - } - private StructureWriteHandle checkFormat(StructureWriteHandle handle) { - if (handle != null && handle instanceof HandleImplementation) { - HandleImplementation handleImpl = (HandleImplementation) handle; - Format format = handleImpl.getFormat(); - if (format == null || format == Format.UNKNOWN) { - handleImpl.setFormat(Format.TEXT); - handleImpl.setMimetype(Format.TEXT.getDefaultMimetype()); - } - } - return handle; - } - - @Override - public R getResource(RequestLogger reqlog, - String path, Transaction transaction, RequestParameters params, R output) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - if ( params == null ) params = new RequestParameters(); - if ( transaction != null ) params.add("txid", transaction.getTransactionId()); - addPointInTimeQueryParam(params, output); - HandleImplementation outputBase = HandleAccessor.checkHandle(output, - "read"); - - String mimetype = outputBase.getMimetype(); - Class as = outputBase.receiveAs(); - - Request.Builder requestBldr = makeGetWebResource(path, params, mimetype); - requestBldr = setupRequest(requestBldr, null, mimetype); - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - - Function doGetFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return doGet(funcBuilder); - } - }; - Response response = sendRequestWithRetry(requestBldr, (transaction == null), doGetFunction, null); - int status = response.code(); - checkStatus(response, status, "read", "resource", path, - ResponseStatus.OK_OR_NO_CONTENT); - - updateDescriptor(outputBase, response.headers()); - if (as != null) { - outputBase.receiveContent(makeResult(reqlog, "read", "resource", - response, as)); - } else { - closeResponse(response); - } - - return output; - } - - @Override - public RESTServiceResultIterator getIteratedResource(RequestLogger reqlog, - String path, Transaction transaction, RequestParameters params) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - return getIteratedResourceImpl(OkHttpServiceResultIterator::new, reqlog, path, transaction, params); - } - - private U getIteratedResourceImpl(ResultIteratorConstructor constructor, - RequestLogger reqlog, String path, Transaction transaction, RequestParameters params) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - if ( params == null ) params = new RequestParameters(); - if (transaction != null) params.add("txid", transaction.getTransactionId()); - - Request.Builder requestBldr = makeGetWebResource(path, params, null); - requestBldr = setupRequest(requestBldr, null, null); - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - - requestBldr = requestBldr.header(HEADER_ACCEPT, multipartMixedWithBoundary()); - Function doGetFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return doGet(funcBuilder); - } - }; - Response response = sendRequestWithRetry(requestBldr, (transaction == null), doGetFunction, null); - int status = response.code(); - checkStatus(response, status, "read", "resource", path, - ResponseStatus.OK_OR_NO_CONTENT); - - return makeResults(constructor, reqlog, "read", "resource", response); - } - - @Override - public R putResource(RequestLogger reqlog, - String path, Transaction transaction, RequestParameters params, - AbstractWriteHandle input, R output) - throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException - { - if ( params == null ) params = new RequestParameters(); - if ( transaction != null ) params.add("txid", transaction.getTransactionId()); - HandleImplementation inputBase = HandleAccessor.checkHandle(input, - "write"); - HandleImplementation outputBase = HandleAccessor.checkHandle(output, - "read"); - - String inputMimetype = inputBase.getMimetype(); - boolean isResendable = inputBase.isResendable(); - String outputMimeType = null; - Class as = null; - if (outputBase != null) { - outputMimeType = outputBase.getMimetype(); - as = outputBase.receiveAs(); - } - Request.Builder requestBldr = makePutWebResource(path, params); - requestBldr = setupRequest(requestBldr, inputMimetype, outputMimeType); - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - - Consumer resendableConsumer = (resendable) -> { - if (!isResendable) { - checkFirstRequest(); - throw new ResourceNotResendableException( - "Cannot retry request for " + path); - } - }; - - Function doPutFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return doPut(reqlog, funcBuilder, inputBase.sendContent()); - } - }; - Response response = sendRequestWithRetry(requestBldr, (transaction == null), doPutFunction, resendableConsumer); - int status = response.code(); - - checkStatus(response, status, "write", "resource", path, - ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); - - if (as != null) { - outputBase.receiveContent(makeResult(reqlog, "write", "resource", - response, as)); - } else { - closeResponse(response); - } - - return output; - } - - @Override - public R putResource( - RequestLogger reqlog, String path, Transaction transaction, RequestParameters params, - W[] input, R output) - throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException - { - if (input == null || input.length == 0) { - throw new IllegalArgumentException("input not specified for multipart"); - } - if ( params == null ) params = new RequestParameters(); - if ( transaction != null ) params.add("txid", transaction.getTransactionId()); - - HandleImplementation outputBase = HandleAccessor.checkHandle(output, - "read"); - - String outputMimetype = outputBase.getMimetype(); - Class as = outputBase.receiveAs(); - - Response response = null; - int status = -1; - long startTime = System.currentTimeMillis(); - int nextDelay = 0; - int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { - if (nextDelay > 0) { - try { - Thread.sleep(nextDelay); - } catch (InterruptedException e) { - } - } - - MultipartBody.Builder multiPart = new MultipartBody.Builder(); - boolean hasStreamingPart = addParts(multiPart, reqlog, input); - - Request.Builder requestBldr = makePutWebResource(path, params); - requestBldr = setupRequest(requestBldr, multiPart, outputMimetype); - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - - response = doPut(requestBldr, multiPart, hasStreamingPart); - status = response.code(); - - if (transaction != null || !retryStatus.contains(status)) { - if (isFirstRequest()) setFirstRequest(false); - - break; - } - - String retryAfterRaw = response.header("Retry-After"); - closeResponse(response); - - if (hasStreamingPart) { - throw new ResourceNotResendableException( - "Cannot retry request for " + path); - } - - int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); - } - if (retryStatus.contains(status)) { - checkFirstRequest(); - closeResponse(response); - throw new FailedRetryException( - "Service unavailable and maximum retry period elapsed: "+ - ((System.currentTimeMillis() - startTime) / 1000)+ - " seconds after "+retry+" retries"); - } - - checkStatus(response, status, "write", "resource", path, - ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); - - if (as != null) { - outputBase.receiveContent(makeResult(reqlog, "write", "resource", - response, as)); - } else { - closeResponse(response); - } - - return output; - } - - @Override - public R postResource(RequestLogger reqlog, - String path, Transaction transaction, RequestParameters params, - AbstractWriteHandle input, R output) - throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException - { - return postResource(reqlog, path, transaction, params, input, output, "apply"); - } - - @Override - public R postResource(RequestLogger reqlog, - String path, Transaction transaction, RequestParameters params, - AbstractWriteHandle input, R output, String operation) - throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException - { - return postResource(reqlog, path, transaction, params, input, output, operation, null); - } - - @Override - public R postResource(RequestLogger reqlog, - String path, Transaction transaction, RequestParameters params, - AbstractWriteHandle input, R output, String operation, - Map> responseHeaders) - throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException - { - if ( params == null ) params = new RequestParameters(); - if ( transaction != null ) params.add("txid", transaction.getTransactionId()); - - HandleImplementation inputBase = HandleAccessor.checkHandle(input, - "write"); - HandleImplementation outputBase = HandleAccessor.checkHandle(output, - "read"); - - addPointInTimeQueryParam(params, outputBase); - - String inputMimetype = null; - if(inputBase != null) { - inputMimetype = inputBase.getMimetype(); - if ( inputMimetype == null && - ( Format.JSON == inputBase.getFormat() || - Format.XML == inputBase.getFormat() ) ) - { - inputMimetype = inputBase.getFormat().getDefaultMimetype(); - } - } - String outputMimetype = outputBase == null ? null : outputBase.getMimetype(); - boolean isResendable = inputBase == null ? true : inputBase.isResendable(); - Class as = outputBase == null ? null : outputBase.receiveAs(); - - Request.Builder requestBldr = makePostWebResource(path, params); - requestBldr = setupRequest(requestBldr, inputMimetype, outputMimetype); - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - - Consumer resendableConsumer = new Consumer() { - public void accept(Boolean resendable) { - if (!isResendable) { - checkFirstRequest(); - throw new ResourceNotResendableException("Cannot retry request for " + path); - } - } - }; - final Object value = inputBase == null ? null :inputBase.sendContent(); - Function doPostFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return doPost(reqlog, funcBuilder, value); - } - }; - - Response response = sendRequestWithRetry(requestBldr, (transaction == null), doPostFunction, resendableConsumer); - int status = response.code(); - checkStatus(response, status, operation, "resource", path, - ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); - - Headers headers = response.headers(); - if ( responseHeaders != null ) { - // add all the headers from the OkHttp Headers object to the caller-provided map - responseHeaders.putAll(headers.toMultimap()); - } else if (outputBase != null){ - updateLength(outputBase, headers); - updateServerTimestamp(outputBase, headers); - } - - if (as != null) { - outputBase.receiveContent(makeResult(reqlog, operation, "resource", - response, as)); - } else { - closeResponse(response); - } - - return output; - } - - @Override - public R postResource( - RequestLogger reqlog, String path, Transaction transaction, RequestParameters params, - W[] input, R output) - throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException - { - return postResource(reqlog, path, transaction, params, input, null, output); - } - - @Override - public R postResource( - RequestLogger reqlog, String path, Transaction transaction, RequestParameters params, - W[] input, Map>[] requestHeaders, R output) - throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException - { - if ( params == null ) params = new RequestParameters(); - if ( transaction != null ) params.add("txid", transaction.getTransactionId()); - - HandleImplementation outputBase = HandleAccessor.checkHandle(output, "read"); - - String outputMimetype = outputBase != null ? outputBase.getMimetype() : null; - Class as = outputBase != null ? outputBase.receiveAs() : null; - - Response response = null; - int status = -1; - long startTime = System.currentTimeMillis(); - int nextDelay = 0; - int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { - if (nextDelay > 0) { - try { - Thread.sleep(nextDelay); - } catch (InterruptedException e) { - } - } - - MultipartBody.Builder multiPart = new MultipartBody.Builder(); - boolean hasStreamingPart = addParts(multiPart, reqlog, null, input, requestHeaders); - - Request.Builder requestBldr = makePostWebResource(path, params); - requestBldr = setupRequest(requestBldr, multiPart, outputMimetype); - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - - response = doPost(requestBldr, multiPart, hasStreamingPart); - status = response.code(); - - if (transaction != null || !retryStatus.contains(status)) { - if (isFirstRequest()) setFirstRequest(false); - - break; - } - - String retryAfterRaw = response.header("Retry-After"); - closeResponse(response); - - if (hasStreamingPart) { - throw new ResourceNotResendableException( - "Cannot retry request for " + path); - } - - int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); - } - if (retryStatus.contains(status)) { - checkFirstRequest(); - closeResponse(response); - throw new FailedRetryException( - "Service unavailable and maximum retry period elapsed: "+ - ((System.currentTimeMillis() - startTime) / 1000)+ - " seconds after "+retry+" retries"); - } - - checkStatus(response, status, "apply", "resource", path, - ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); - - if (as != null) { - outputBase.receiveContent(makeResult(reqlog, "apply", "resource", - response, as)); - } else { - closeResponse(response); - } - - return output; - } - - @Override - public R postBulkDocuments( - RequestLogger reqlog, DocumentWriteSet writeSet, - ServerTransform transform, Transaction transaction, Format defaultFormat, R output, - String temporalCollection, String extraContentDispositionParams) - throws ForbiddenUserException, FailedRequestException - { - CharsetEncoder asciiEncoder = java.nio.charset.StandardCharsets.US_ASCII.newEncoder(); - - List writeHandles = new ArrayList(); - List headerList = new ArrayList(); - for ( DocumentWriteOperation write : writeSet ) { - HandleImplementation metadata = HandleAccessor.checkHandle(write.getMetadata(), "write"); - HandleImplementation content = HandleAccessor.checkHandle(write.getContent(), "write"); - - String dispositionFilename = (write.getUri() == null) ? "" : - ("; " + DISPOSITION_PARAM_FILENAME + Utilities.escapeMultipartParamAssignment(asciiEncoder, write.getUri())); - String dispositionTemporalDoc = (write.getTemporalDocumentURI() == null) ? "" : - ("; " + DISPOSITION_PARAM_TEMPORALDOC + Utilities.escapeMultipartParamAssignment(asciiEncoder, write.getTemporalDocumentURI())); - - if ( write.getOperationType() == DocumentWriteOperation.OperationType.DISABLE_METADATA_DEFAULT ) { - RequestParameters headers = new RequestParameters(); - headers.add(HEADER_CONTENT_TYPE, metadata.getMimetype()); - headers.add(HEADER_CONTENT_DISPOSITION, - DISPOSITION_TYPE_INLINE + "; "+DISPOSITION_PARAM_CATEGORY+"=metadata"); - headerList.add(headers); - writeHandles.add(write.getMetadata()); - } else if ( metadata != null ) { - RequestParameters headers = new RequestParameters(); - headers.add(HEADER_CONTENT_TYPE, metadata.getMimetype()); - if ( write.getOperationType() == DocumentWriteOperation.OperationType.METADATA_DEFAULT ) { - headers.add(HEADER_CONTENT_DISPOSITION, - DISPOSITION_TYPE_INLINE + "; "+DISPOSITION_PARAM_CATEGORY+"=metadata"); - } else { - headers.add(HEADER_CONTENT_DISPOSITION, - DISPOSITION_TYPE_ATTACHMENT + dispositionFilename + dispositionTemporalDoc + - "; " + DISPOSITION_PARAM_CATEGORY + "=metadata"); - } - headerList.add(headers); - writeHandles.add(write.getMetadata()); - } - - if ( content != null ) { - RequestParameters headers = new RequestParameters(); - String mimeType = content.getMimetype(); - if ( mimeType == null && defaultFormat != null ) { - mimeType = defaultFormat.getDefaultMimetype(); - } - headers.add(HEADER_CONTENT_TYPE, mimeType); - headers.add(HEADER_CONTENT_DISPOSITION, - DISPOSITION_TYPE_ATTACHMENT + dispositionFilename + dispositionTemporalDoc + - extraContentDispositionParams); - headerList.add(headers); - writeHandles.add(write.getContent()); - } - } - RequestParameters params = new RequestParameters(); - if ( transform != null ) { - transform.merge(params); - } - if ( temporalCollection != null ) params.add("temporal-collection", temporalCollection); - return postResource(reqlog, "documents", transaction, params, - (AbstractWriteHandle[]) writeHandles.toArray(new AbstractWriteHandle[0]), - (RequestParameters[]) headerList.toArray(new RequestParameters[0]), - output); - } - - public class OkHttpEvalResultIterator implements EvalResultIterator { - private OkHttpResultIterator iterator; - - OkHttpEvalResultIterator(OkHttpResultIterator iterator) { - this.iterator = iterator; - } - - @Override - public Iterator iterator() { - return this; - } - - @Override - public boolean hasNext() { - if ( iterator == null ) return false; - return iterator.hasNext(); - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - @Override - public EvalResult next() { - if ( iterator == null ) throw new NoSuchElementException("No results available"); - OkHttpResult jerseyResult = iterator.next(); - EvalResult result = new OkHttpEvalResult(jerseyResult); - return result; - } - - @Override - public void close() { - if ( iterator != null ) iterator.close(); - } - } - public class OkHttpEvalResult implements EvalResult { - private OkHttpResult content; - - public OkHttpEvalResult(OkHttpResult content) { - this.content = content; - } - - @Override - public Format getFormat() { - return content.getFormat(); - } - - @Override - public EvalResult.Type getType() { - String contentType = content.getHeader(HEADER_CONTENT_TYPE); - String xPrimitive = content.getHeader(HEADER_X_PRIMITIVE); - if ( contentType != null ) { - if ( MIMETYPE_APPLICATION_JSON.equals(contentType) ) { - if ( "null-node()".equals(xPrimitive) ) { - return EvalResult.Type.NULL; - } else { - return EvalResult.Type.JSON; - } - } else if ( MIMETYPE_TEXT_JSON.equals(contentType) ) { - return EvalResult.Type.JSON; - } else if ( MIMETYPE_APPLICATION_XML.equals(contentType) ) { - return EvalResult.Type.XML; - } else if ( MIMETYPE_TEXT_XML.equals(contentType) ) { - return EvalResult.Type.XML; - } else if ( "application/x-unknown-content-type".equals(contentType) && "binary()".equals(xPrimitive) ) { - return EvalResult.Type.BINARY; - } else if ( "application/octet-stream".equals(contentType) && "node()".equals(xPrimitive) ) { - return EvalResult.Type.BINARY; - } - } - if ( xPrimitive == null ) { - return EvalResult.Type.OTHER; - } else if ( "string".equals(xPrimitive) || "untypedAtomic".equals(xPrimitive) ) { - return EvalResult.Type.STRING; - } else if ( "boolean".equals(xPrimitive) ) { - return EvalResult.Type.BOOLEAN; - } else if ( "attribute()".equals(xPrimitive) ) { - return EvalResult.Type.ATTRIBUTE; - } else if ( "comment()".equals(xPrimitive) ) { - return EvalResult.Type.COMMENT; - } else if ( "processing-instruction()".equals(xPrimitive) ) { - return EvalResult.Type.PROCESSINGINSTRUCTION; - } else if ( "text()".equals(xPrimitive) ) { - return EvalResult.Type.TEXTNODE; - } else if ( "binary()".equals(xPrimitive) ) { - return EvalResult.Type.BINARY; - } else if ( "duration".equals(xPrimitive) ) { - return EvalResult.Type.DURATION; - } else if ( "date".equals(xPrimitive) ) { - return EvalResult.Type.DATE; - } else if ( "anyURI".equals(xPrimitive) ) { - return EvalResult.Type.ANYURI; - } else if ( "hexBinary".equals(xPrimitive) ) { - return EvalResult.Type.HEXBINARY; - } else if ( "base64Binary".equals(xPrimitive) ) { - return EvalResult.Type.BASE64BINARY; - } else if ( "dateTime".equals(xPrimitive) ) { - return EvalResult.Type.DATETIME; - } else if ( "decimal".equals(xPrimitive) ) { - return EvalResult.Type.DECIMAL; - } else if ( "double".equals(xPrimitive) ) { - return EvalResult.Type.DOUBLE; - } else if ( "float".equals(xPrimitive) ) { - return EvalResult.Type.FLOAT; - } else if ( "gDay".equals(xPrimitive) ) { - return EvalResult.Type.GDAY; - } else if ( "gMonth".equals(xPrimitive) ) { - return EvalResult.Type.GMONTH; - } else if ( "gMonthDay".equals(xPrimitive) ) { - return EvalResult.Type.GMONTHDAY; - } else if ( "gYear".equals(xPrimitive) ) { - return EvalResult.Type.GYEAR; - } else if ( "gYearMonth".equals(xPrimitive) ) { - return EvalResult.Type.GYEARMONTH; - } else if ( "integer".equals(xPrimitive) ) { - return EvalResult.Type.INTEGER; - } else if ( "QName".equals(xPrimitive) ) { - return EvalResult.Type.QNAME; - } else if ( "time".equals(xPrimitive) ) { - return EvalResult.Type.TIME; - } - return EvalResult.Type.OTHER; - } - - @Override - public H get(H handle) { - if ( getType() == EvalResult.Type.NULL && handle instanceof StringHandle ) { - return (H) ((StringHandle) handle).with(null); - } else if ( getType() == EvalResult.Type.NULL && handle instanceof BytesHandle ) { - return (H) ((BytesHandle) handle).with(null); - } else { - return content.getContent(handle); - } - } - - @Override - public T getAs(Class as) { - if ( getType() == EvalResult.Type.NULL ) return null; - if (as == null) throw new IllegalArgumentException("class cannot be null"); - - ContentHandle readHandle = DatabaseClientFactory.getHandleRegistry().makeHandle(as); - if ( readHandle == null ) return null; - readHandle = get(readHandle); - if ( readHandle == null ) return null; - return readHandle.get(); - } - - @Override - public String getString() { - if ( getType() == EvalResult.Type.NULL ) { - return null; - } else { - return content.getContentAs(String.class); - } - } - - @Override - public Number getNumber() { - String value = getString(); - if ( value == null ) return null; - if ( getType() == EvalResult.Type.DECIMAL ) return new BigDecimal(value); - else if ( getType() == EvalResult.Type.DOUBLE ) return new Double(value); - else if ( getType() == EvalResult.Type.FLOAT ) return new Float(value); - // MarkLogic integers can be much larger than Java integers, so we'll use Long instead - else if ( getType() == EvalResult.Type.INTEGER ) return new Long(value); - else return new BigDecimal(value); - } - - @Override - public Boolean getBoolean() { - // converts null to false - return Boolean.valueOf(getString()); - } - } - - @Override - public EvalResultIterator postEvalInvoke( - RequestLogger reqlog, String code, String modulePath, - ServerEvaluationCallImpl.Context context, - Map variables, EditableNamespaceContext namespaces, - Transaction transaction) - throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException - { - String formUrlEncodedPayload; - String path; - RequestParameters params = new RequestParameters(); - try { - StringBuffer sb = new StringBuffer(); - if ( context == ServerEvaluationCallImpl.Context.ADHOC_XQUERY ) { - path = "eval"; - sb.append("xquery="); - sb.append(URLEncoder.encode(code, "UTF-8")); - } else if ( context == ServerEvaluationCallImpl.Context.ADHOC_JAVASCRIPT ) { - path = "eval"; - sb.append("javascript="); - sb.append(URLEncoder.encode(code, "UTF-8")); - } else if ( context == ServerEvaluationCallImpl.Context.INVOKE ) { - path = "invoke"; - sb.append("module="); - sb.append(URLEncoder.encode(modulePath, "UTF-8")); - } else { - throw new IllegalStateException("Invalid eval context: " + context); - } - if ( variables != null && variables.size() > 0 ) { - int i=0; - for ( String name : variables.keySet() ) { - String namespace = ""; - String localname = name; - if ( namespaces != null ) { - for ( String prefix : namespaces.keySet() ) { - if ( name != null && prefix != null && name.startsWith(prefix + ":") ) { - localname = name.substring(prefix.length() + 1); - namespace = namespaces.get(prefix); - } - } - } - // set the variable namespace - sb.append("&evn" + i + "="); - sb.append(URLEncoder.encode(namespace, "UTF-8")); - // set the variable localname - sb.append("&evl" + i + "="); - sb.append(URLEncoder.encode(localname, "UTF-8")); - - String value; - String type = null; - Object valueObject = variables.get(name); - if ( valueObject == null ) { - value = "null"; - type = "null-node()"; - } else if ( valueObject instanceof JacksonHandle || - valueObject instanceof JacksonParserHandle ) - { - JsonNode jsonNode = null; - if ( valueObject instanceof JacksonHandle ) { - jsonNode = ((JacksonHandle) valueObject).get(); - } else if ( valueObject instanceof JacksonParserHandle ) { - jsonNode = ((JacksonParserHandle) valueObject).get().readValueAs(JsonNode.class); - } - value = jsonNode.toString(); - type = getJsonType(jsonNode); - } else if ( valueObject instanceof AbstractWriteHandle ) { - value = HandleAccessor.contentAsString((AbstractWriteHandle) valueObject); - HandleImplementation valueBase = HandleAccessor.as((AbstractWriteHandle) valueObject); - Format format = valueBase.getFormat(); - //TODO: figure out what type should be - // I see element() and document-node() are two valid types - if ( format == Format.XML ) { - type = "document-node()"; - } else if ( format == Format.JSON ) { - try ( JacksonParserHandle handle = new JacksonParserHandle() ) { - JsonNode jsonNode = handle.getMapper().readTree(value); - type = getJsonType(jsonNode); - } - } else if ( format == Format.TEXT ) { - /* Comment next line until 32608 is resolved - type = "text()"; - // until then, use the following line */ - type = "xs:untypedAtomic"; - } else if ( format == Format.BINARY ) { - throw new UnsupportedOperationException("Binary format is not supported for variables"); - } else { - throw new UnsupportedOperationException("Undefined format is not supported for variables. " + - "Please set the format on your handle for variable " + name + "."); - } - } else if ( valueObject instanceof String || - valueObject instanceof Boolean || - valueObject instanceof Number ) - { - value = valueObject.toString(); - // when we send type "xs:untypedAtomic" via XDBC, the server attempts to intelligently decide - // how to cast the type - type = "xs:untypedAtomic"; - } else { - throw new IllegalArgumentException("Variable with name=" + - name + " is of unsupported type" + - valueObject.getClass() + ". Supported types are String, Boolean, Number, " + - "or AbstractWriteHandle"); - } - - // set the variable value - sb.append("&evv" + i + "="); - sb.append(URLEncoder.encode(value, "UTF-8")); - // set the variable type - sb.append("&evt" + i + "=" + type); - i++; - } - } - formUrlEncodedPayload = sb.toString(); - } catch (UnsupportedEncodingException e) { - throw new IllegalStateException("UTF-8 is unsupported", e); - } catch (IOException e) { - throw new MarkLogicIOException(e); - } - StringHandle input = new StringHandle(formUrlEncodedPayload) - .withMimetype("application/x-www-form-urlencoded"); - return new OkHttpEvalResultIterator( postIteratedResourceImpl(DefaultOkHttpResultIterator::new, - reqlog, path, transaction, params, input) ); - } - - private String getJsonType(JsonNode jsonNode) { - if ( jsonNode instanceof ArrayNode ) { - return "json:array"; - } else if ( jsonNode instanceof ObjectNode ) { - return "json:object"; - } else { - throw new IllegalArgumentException("When using JacksonHandle or " + - "JacksonParserHandle with ServerEvaluationCall the content must be " + - "a valid array or object"); - } - } - - @Override - public RESTServiceResultIterator postIteratedResource(RequestLogger reqlog, - String path, Transaction transaction, RequestParameters params, AbstractWriteHandle input) - throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException - { - return postIteratedResourceImpl(OkHttpServiceResultIterator::new, - reqlog, path, transaction, params, input); - } - - public RESTServiceResultIterator postMultipartForm( - RequestLogger reqlog, String path, Transaction transaction, RequestParameters params, List contentParams) - throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException - { - if ( transaction != null ) { - params.add("txid", transaction.getTransactionId()); - } - - // Don't include incoming request params, those all have to be form-data inputs - Request.Builder requestBldr = makePostWebResource(path, new RequestParameters()); - - MultipartBody.Builder multiBuilder = new MultipartBody.Builder().setType(MediaType.parse("multipart/form-data")); - for (String key : params.keySet()) { - for (String value : params.get(key)) { - if (value != null) { - multiBuilder.addFormDataPart(key, value); - } - } - } - - contentParams.forEach(contentParam -> { - String name = contentParam.getPlanParam().getName(); - multiBuilder.addFormDataPart(name, name, makeRequestBodyForContent(contentParam.getContent())); - }); - this.addContentParamAttachments(multiBuilder, contentParams); - - requestBldr = setupRequest(requestBldr, multiBuilder, null); - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - requestBldr = addTrailerHeadersIfNecessary(requestBldr, path); - - Function doPostFunction = funcBuilder -> - doPost(reqlog, funcBuilder.header(HEADER_ACCEPT, multipartMixedWithBoundary()), multiBuilder.build()); - - // The construction of this was based on whether the primary input was resendable or not. We don't have a primary - // input with a multipart request. So keeping this as null for now. - Consumer resendableConsumer = null; - - Response response = sendRequestWithRetry(requestBldr, (transaction == null), doPostFunction, resendableConsumer); - int status = response.code(); - checkStatus(response, status, "apply", "resource", path, ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); - return makeResults(OkHttpServiceResultIterator::new, reqlog, "apply", "resource", response); - } - - /** - * The REST endpoint checks for a 'metadata' parameter that, if it exists, is expected to be a JSON object with - * an 'attachment/docs' array. Each object in the array is expected to have two fields - 'rowsField' and 'column'. - * The expectation is that 'rowsField' identifies a bound parameter name that is associated with a JSON array, and - * 'column' identifies a particular column in each object in the JSON array. - *

- * To provide support for this parameter, each object in the given {@code contentParamAttachments} array results in - * a new object in the 'rowsField' array. Then, each entry in the map of attachments in each such object is added - * as a new multipart form data part, with the map key being used as the form data part's filename. The value of - * the 'column' name in each row is then expected to be one of these map keys. - * - * @param multiBuilder - * @param contentParams - */ - private void addContentParamAttachments(MultipartBody.Builder multiBuilder, List contentParams) { - ObjectNode metadata = new ObjectMapper().createObjectNode(); - ArrayNode docsArray = metadata.putObject("attachments").putArray("docs"); - contentParams.stream().filter(contentParam -> contentParam.getColumnAttachments() != null).forEach(contentParam -> { - Map> attachments = contentParam.getColumnAttachments(); - attachments.keySet().forEach(columnName -> { - docsArray.addObject().put("rowsField", contentParam.getPlanParam().getName()).put("column", columnName); - attachments.get(columnName).keySet().forEach(filename -> { - multiBuilder.addFormDataPart(columnName, filename, makeRequestBodyForContent(attachments.get(columnName).get(filename))); - }); - }); - }); - if (docsArray.size() > 0) { - multiBuilder.addFormDataPart("metadata", "metadata", makeRequestBodyForContent(new JacksonHandle(metadata))); - } - } - - private U postIteratedResourceImpl( - ResultIteratorConstructor constructor, final RequestLogger reqlog, - final String path, Transaction transaction, RequestParameters params, - AbstractWriteHandle input) - throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException - { - if ( params == null ) params = new RequestParameters(); - if ( transaction != null ) params.add("txid", transaction.getTransactionId()); - HandleImplementation inputBase = HandleAccessor.checkHandle(input, "write"); - - String inputMimetype = inputBase.getMimetype(); - boolean isResendable = inputBase.isResendable(); - - Request.Builder requestBldr = makePostWebResource(path, params); - requestBldr = setupRequest(requestBldr, inputMimetype, null); - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - requestBldr = addTrailerHeadersIfNecessary(requestBldr, path); - requestBldr = setErrorFormatIfNecessary(requestBldr, path); - - Consumer resendableConsumer = resendable -> { - if (!isResendable) { - checkFirstRequest(); - throw new ResourceNotResendableException( - "Cannot retry request for " + path); - } - }; - - Function doPostFunction = requestBuilder -> doPost( - reqlog, - requestBuilder.header(HEADER_ACCEPT, multipartMixedWithBoundary()), - inputBase.sendContent() - ); - - Response response = sendRequestWithRetry(requestBldr, (transaction == null), doPostFunction, resendableConsumer); - checkStatus(response, response.code(), "apply", "resource", path, ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); - return makeResults(constructor, reqlog, "apply", "resource", response); - } - - @Override - public RESTServiceResultIterator postIteratedResource( - RequestLogger reqlog, String path, Transaction transaction, RequestParameters params, - W[] input) - throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException - { - return postIteratedResourceImpl(OkHttpServiceResultIterator::new, - reqlog, path, transaction, params, input); - } - - private U postIteratedResourceImpl( - ResultIteratorConstructor constructor, RequestLogger reqlog, String path, Transaction transaction, - RequestParameters params, W[] input) - throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException - { - if ( params == null ) params = new RequestParameters(); - if ( transaction != null ) params.add("txid", transaction.getTransactionId()); - Response response = null; - int status = -1; - long startTime = System.currentTimeMillis(); - int nextDelay = 0; - int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { - if (nextDelay > 0) { - try { - Thread.sleep(nextDelay); - } catch (InterruptedException e) { - } - } - - MultipartBody.Builder multiPart = new MultipartBody.Builder(); - boolean hasStreamingPart = addParts(multiPart, reqlog, input); - - Request.Builder requestBldr = makePostWebResource(path, params); - requestBldr = setupRequest( - requestBldr, - multiPart, - multipartMixedWithBoundary()); - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - - response = doPost(requestBldr, multiPart, hasStreamingPart); - status = response.code(); - - if (transaction != null || !retryStatus.contains(status)) { - if (isFirstRequest()) setFirstRequest(false); - - break; - } - - String retryAfterRaw = response.header("Retry-After"); - closeResponse(response); - - if (hasStreamingPart) { - throw new ResourceNotResendableException( - "Cannot retry request for " + path); - } - - int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); - } - if (retryStatus.contains(status)) { - checkFirstRequest(); - closeResponse(response); - throw new FailedRetryException( - "Service unavailable and maximum retry period elapsed: "+ - ((System.currentTimeMillis() - startTime) / 1000)+ - " seconds after "+retry+" retries"); - } - - checkStatus(response, status, "apply", "resource", path, - ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); - - return makeResults(constructor, reqlog, "apply", "resource", response); - } - - @Override - public R deleteResource( - RequestLogger reqlog, String path, Transaction transaction, RequestParameters params, - R output) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - if ( params == null ) params = new RequestParameters(); - if ( transaction != null ) params.add("txid", transaction.getTransactionId()); - HandleImplementation outputBase = HandleAccessor.checkHandle(output, - "read"); - - String outputMimeType = null; - Class as = null; - if (outputBase != null) { - outputMimeType = outputBase.getMimetype(); - as = outputBase.receiveAs(); - } - Request.Builder requestBldr = makeDeleteWebResource(path, params); - requestBldr = setupRequest(requestBldr, null, outputMimeType); - requestBldr = addTransactionScopedCookies(requestBldr, transaction); - requestBldr = addTelemetryAgentId(requestBldr); - - Function doDeleteFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return doDelete(funcBuilder); - } - }; - Response response = sendRequestWithRetry(requestBldr, (transaction == null), doDeleteFunction, null); - int status = response.code(); - checkStatus(response, status, "delete", "resource", path, - ResponseStatus.OK_OR_NO_CONTENT); - - if (as != null) { - outputBase.receiveContent(makeResult(reqlog, "delete", "resource", - response, as)); - } else { - closeResponse(response); - } - - return output; - } - - private Request.Builder makeGetWebResource(String path, - RequestParameters params, Object mimetype) { - if (path == null) throw new IllegalArgumentException("Read with null path"); - - logger.debug(String.format("Getting %s as %s", path, mimetype)); - - return setupRequest(path, params); - } - - private Response doGet(Request.Builder requestBldr) { - requestBldr = requestBldr.get(); - Response response = sendRequestOnce(requestBldr); - - if (isFirstRequest()) setFirstRequest(false); - - return response; - } - - private Request.Builder makePutWebResource(String path, - RequestParameters params) { - if (path == null) throw new IllegalArgumentException("Write with null path"); - - logger.debug("Putting {}", path); - - return setupRequest(path, params); - } - - private Response doPut(RequestLogger reqlog, Request.Builder requestBldr, Object value) { - if (value == null) throw new IllegalArgumentException("Resource write with null value"); - - if (isFirstRequest() && isStreaming(value)) makeFirstRequest(0); - - MediaType mediaType = makeType(requestBldr.build().header(HEADER_CONTENT_TYPE)); - if (value instanceof OutputStreamSender) { - requestBldr = requestBldr.put(new StreamingOutputImpl((OutputStreamSender) value, reqlog, mediaType)); - } else { - if (reqlog != null) { - requestBldr = requestBldr.put(new ObjectRequestBody(reqlog.copyContent(value), mediaType)); - } else { - requestBldr = requestBldr.put(new ObjectRequestBody(value, mediaType)); - } - } - Response response = sendRequestOnce(requestBldr); - - if (isFirstRequest()) setFirstRequest(false); - - return response; - } - - private Response doPut(Request.Builder requestBldr, - MultipartBody.Builder multiPart, boolean hasStreamingPart) { - if (isFirstRequest() && hasStreamingPart) makeFirstRequest(0); - - requestBldr = requestBldr.put(multiPart.build()); - Response response = sendRequestOnce(requestBldr); - - if (isFirstRequest()) setFirstRequest(false); - - return response; - } - - private Request.Builder makePostWebResource(String path, RequestParameters params) { - if (path == null) throw new IllegalArgumentException("Apply with null path"); - - logger.debug("Posting {}", path); - - return setupRequest(path, params); - } - - private Response doPost(RequestLogger reqlog, Request.Builder requestBldr, Object value) { - if (isFirstRequest() && isStreaming(value)) { - makeFirstRequest(0); - } - - MediaType mediaType = makeType(requestBldr.build().header(HEADER_CONTENT_TYPE)); - if(value == null) { - requestBldr = requestBldr.post(new ObjectRequestBody(null, null)); - } else if (value instanceof MultipartBody) { - requestBldr = requestBldr.post((MultipartBody)value); - } else if (value instanceof OutputStreamSender) { - requestBldr = requestBldr - .post(new StreamingOutputImpl((OutputStreamSender) value, reqlog, mediaType)); - } else { - if (reqlog != null) { - requestBldr = requestBldr.post(new ObjectRequestBody(reqlog.copyContent(value), mediaType)); - } else { - requestBldr = requestBldr.post(new ObjectRequestBody(value, mediaType)); - } - } - Response response = sendRequestOnce(requestBldr); - - if (isFirstRequest()) setFirstRequest(false); - - return response; - } - - private Response doPost(Request.Builder requestBldr, - MultipartBody.Builder multiPart, boolean hasStreamingPart) { - if (isFirstRequest() && hasStreamingPart) makeFirstRequest(0); - - Response response = sendRequestOnce(requestBldr.post(multiPart.build())); - - if (isFirstRequest()) setFirstRequest(false); - - return response; - } - - private Request.Builder makeDeleteWebResource(String path, RequestParameters params) { - if (path == null) throw new IllegalArgumentException("Delete with null path"); - - logger.debug("Deleting {}", path); - - return setupRequest(path, params); - } - - private Response doDelete(Request.Builder requestBldr) { - Response response = sendRequestOnce(requestBldr.delete().build()); - - if (isFirstRequest()) setFirstRequest(false); - - return response; - } - - private void addPointInTimeQueryParam(RequestParameters params, Object outputHandle) { - addPointInTimeQueryParam(params, HandleAccessor.as(outputHandle)); - } - private void addPointInTimeQueryParam(RequestParameters params, HandleImplementation handleBase) { - if (params != null && handleBase != null && handleBase.getPointInTimeQueryTimestamp() != -1) { - logger.trace("param timestamp=[" + handleBase.getPointInTimeQueryTimestamp() + "]"); - params.add("timestamp", Long.toString(handleBase.getPointInTimeQueryTimestamp())); - } - } - - private Request.Builder addTransactionScopedCookies(Request.Builder requestBldr, Transaction transaction) { - if ( transaction != null && transaction.getCookies() != null ) { - if ( requestBldr == null ) { - throw new MarkLogicInternalException("no requestBldr available to get the URI"); - } - requestBldr = addCookies( - requestBldr, transaction.getCookies(), ((TransactionImpl) transaction).getCreatedTimestamp() - ); - } - return requestBldr; - } - - private Request.Builder addCookies(Request.Builder requestBldr, List cookies, Calendar creation) { - HttpUrl uri = requestBldr.build().url(); - for (ClientCookie cookie : cookies) { - // don't forward the cookie if it requires https and we're not using https - if ( cookie.isSecure() && ! uri.isHttps() ) { - continue; - } - // don't forward the cookie if it requires a path and we're using a different path - if ( cookie.getPath() != null ) { - String path = uri.encodedPath(); - if ( path == null || ! path.startsWith(cookie.getPath()) ) { - continue; - } - } - // don't forward the cookie if it requires a domain and we're using a different domain - if ( cookie.getDomain() != null ) { - if ( uri.host() == null || ! uri.host().equals(cookie.getDomain()) ) { - continue; - } - } - // don't forward the cookie if it has 0 for max age - if ( cookie.getMaxAge() == 0 ) { - continue; - } - // TODO: determine if we need handling for MIN_VALUE - // else if ( cookie.getMaxAge() == Integer.MIN_VALUE ) { - // don't forward the cookie if it has a max age and we're past the max age - if ( creation != null && cookie.getMaxAge() > 0 ) { - int currentAge = (int) TimeUnit.MILLISECONDS.toSeconds( - System.currentTimeMillis() - creation.getTimeInMillis() - ); - if ( currentAge > cookie.getMaxAge() ) { - logger.warn( - cookie.getName()+" cookie expired after "+cookie.getMaxAge()+" seconds: "+cookie.getValue() - ); - continue; - } - } - requestBldr = requestBldr.addHeader(HEADER_COOKIE, cookie.toString()); - } - return requestBldr; - } - - private Request.Builder addTelemetryAgentId(Request.Builder requestBldr) { - if ( requestBldr == null ) throw new MarkLogicInternalException("no requestBldr available to set ML-Agent-ID header"); - return requestBldr.header("ML-Agent-ID", "java"); - } + OkHttpResultIterator iterator = getIteratedResourceImpl(DefaultOkHttpResultIterator::new, + reqlog, path, transaction, params); + if (iterator != null) { + if (iterator.getStart() == -1) iterator.setStart(1); + if (iterator.getSize() != -1) { + if (iterator.getPageSize() == -1) iterator.setPageSize(iterator.getSize()); + if (iterator.getTotalSize() == -1) iterator.setTotalSize(iterator.getSize()); + } + } + return iterator; + } /** - * Per https://docs.marklogic.com/10.0/guide/relnotes/chap3#id_73268 , support for ML-Check-ML11-Headers was added - * for MarkLogic 10.0-9. It is no longer needed in MarkLogic 11 or later. The addition of it will not cause any - * harm, but it can be removed once the Java client no longer needs to support MarkLogic 10. - * - * @param requestBldr - * @param path - * @return + * Uses v1/search. */ - private Request.Builder addTrailerHeadersIfNecessary(Request.Builder requestBldr, String path) { - if ("rows".equals(path)) { - requestBldr.addHeader("TE", "trailers"); - requestBldr.addHeader("ML-Check-ML11-Headers", "true"); - } - return requestBldr; - } + private OkHttpResultIterator getBulkDocumentsImpl(RequestLogger reqlog, long serverTimestamp, + SearchQueryDefinition querydef, long start, long pageLength, + Transaction transaction, SearchReadHandle searchHandle, QueryView view, + Set categories, Format format, ServerTransform responseTransform, + RequestParameters extraParams, String forestName) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + try { + RequestParameters params = new RequestParameters(); + if (extraParams != null) params.putAll(extraParams); + addCategoryParams(categories, params, true); + if (searchHandle != null && view != null) params.add("view", view.toString().toLowerCase()); + if (start > 1) params.add("start", Long.toString(start)); + if (pageLength >= 0) params.add("pageLength", Long.toString(pageLength)); + if (serverTimestamp != -1) params.add("timestamp", Long.toString(serverTimestamp)); + addPointInTimeQueryParam(params, searchHandle); + if (format != null) params.add("format", format.toString().toLowerCase()); + HandleImplementation handleBase = HandleAccessor.as(searchHandle); + if (format == null && searchHandle != null) { + if (Format.XML == handleBase.getFormat()) { + params.add("format", "xml"); + } else if (Format.JSON == handleBase.getFormat()) { + params.add("format", "json"); + } + } - private Request.Builder setErrorFormatIfNecessary(Request.Builder requestBuilder, String path) { - // Slightly dirty hack; per https://docs.marklogic.com/guide/rest-dev/intro#id_34966, the X-Error-Accept header - // should be used to specify the error format. A REST API server defaults to 'json', though the App-Services app - // server defaults to 'compatible'. If the error format is 'compatible', a block of HTML is sent back which - // causes an error that prevents the user from seeing the actual error from the server. So for all eval calls, - // X-Error-Accept is used to request any errors back as JSON so that they can be handled correctly. - if ("eval".equals(path) || ("invoke".equals(path))) { - requestBuilder.addHeader(HEADER_ERROR_FORMAT, "application/json"); + OkHttpSearchRequest request = + generateSearchRequest(reqlog, querydef, MIMETYPE_MULTIPART_MIXED, transaction, responseTransform, params, forestName); + Response response = request.getResponse(); + if (response == null) return null; + MimeMultipart entity = null; + if (searchHandle != null) { + updateServerTimestamp(handleBase, response.headers()); + ResponseBody body = response.body(); + if (body.contentLength() != 0) { + entity = getEntity(body, MimeMultipart.class); + if (entity != null) { + List partList = getPartList(entity); + if (entity.getCount() > 0) { + BodyPart searchResponsePart = entity.getBodyPart(0); + handleBase.receiveContent(getEntity(searchResponsePart, handleBase.receiveAs())); + partList = partList.subList(1, partList.size()); + } + Closeable closeable = response; + return makeResults(OkHttpServiceResultIterator::new, reqlog, "read", "resource", partList, response, + closeable); + } + } + } + return makeResults(OkHttpServiceResultIterator::new, reqlog, "read", "resource", response); + } catch (MessagingException e) { + throw new MarkLogicIOException(e); } - return requestBuilder; } - private boolean addParts( - MultipartBody.Builder multiPart, RequestLogger reqlog, W[] input) - { - return addParts(multiPart, reqlog, null, input, null); - } - - private boolean addParts( - MultipartBody.Builder multiPart, RequestLogger reqlog, String[] mimetypes, W[] input) - { - return addParts(multiPart, reqlog, null, input, null); - } - - private boolean addParts( - MultipartBody.Builder multiPart, RequestLogger reqlog, String[] mimetypes, - W[] input, Map>[] headers) - { - if (mimetypes != null && mimetypes.length != input.length) { - throw new IllegalArgumentException( - "Mismatch between count of mimetypes and input"); - } - if (headers != null && headers.length != input.length) { - throw new IllegalArgumentException( - "Mismatch between count of headers and input"); - } - - multiPart.setType(MediaType.parse(MIMETYPE_MULTIPART_MIXED)); - - boolean hasStreamingPart = false; - for (int i = 0; i < input.length; i++) { - AbstractWriteHandle handle = input[i]; - HandleImplementation handleBase = HandleAccessor.checkHandle( - handle, "write"); - - if (!hasStreamingPart) { - hasStreamingPart = !handleBase.isResendable(); - } - - Object value = handleBase.sendContent(); - - String inputMimetype = null; - if ( mimetypes != null ) inputMimetype = mimetypes[i]; - if ( inputMimetype == null && headers != null ) { - inputMimetype = getHeaderMimetype(getHeader(headers[i], HEADER_CONTENT_TYPE)); - } - if ( inputMimetype == null ) inputMimetype = handleBase.getMimetype(); - - MediaType mediaType = (inputMimetype != null) - ? MediaType.parse(inputMimetype) - : MediaType.parse(MIMETYPE_WILDCARD); - - Headers.Builder partHeaders = new Headers.Builder(); - if ( headers != null ) { - for ( String key : headers[i].keySet() ) { - // OkHttp wants me to skip the Content-Type header - if ( HEADER_CONTENT_TYPE.equalsIgnoreCase(key) ) continue; - for ( String headerValue : headers[i].get(key) ) { - partHeaders.add(key, headerValue); - } - } - } - - Part bodyPart = null; - if (value instanceof OutputStreamSender) { - bodyPart = Part.create(partHeaders.build(), new StreamingOutputImpl( - (OutputStreamSender) value, reqlog, mediaType)); - } else { - if (reqlog != null) { - bodyPart = Part.create(partHeaders.build(), new ObjectRequestBody(reqlog.copyContent(value), mediaType)); - } else { - bodyPart = Part.create(partHeaders.build(), new ObjectRequestBody(value, mediaType)); - } - } - - multiPart = multiPart.addPart(bodyPart); - } - - return hasStreamingPart; - } - - private String multipartMixedWithBoundary() { - return MIMETYPE_MULTIPART_MIXED + "; boundary=" + UUID.randomUUID().toString(); - } - - private Request.Builder setupRequest(HttpUrl requestUri, String path, RequestParameters params) { - if ( requestUri == null ) throw new IllegalArgumentException("request URI cannot be null"); - if ( path == null ) throw new IllegalArgumentException("path cannot be null"); - if ( path.startsWith("/") ) path = path.substring(1); - HttpUrl.Builder uri = requestUri.resolve(path).newBuilder(); - if ( params != null ) { - for ( String key : params.keySet() ) { - for ( String value : params.get(key) ) { - uri.addQueryParameter(key, value); - } - } - } - if ( database != null && ! path.startsWith("config/") ) { - uri.addQueryParameter("database", database); - } - HttpUrl httpUrl = uri.build(); - return new Request.Builder().url(httpUrl); - } - - private Request.Builder setupRequest(String path, RequestParameters params) { - return setupRequest(baseUri, path, params); - } - - private Request.Builder setupRequest(Request.Builder requestBldr, - Object inputMimetype, Object outputMimetype) { - if (inputMimetype == null) { - } else if (inputMimetype instanceof String) { - requestBldr = requestBldr.header(HEADER_CONTENT_TYPE, (String) inputMimetype); - } else if (inputMimetype instanceof MediaType) { - requestBldr = requestBldr.header(HEADER_CONTENT_TYPE, inputMimetype.toString()); - } else if (inputMimetype instanceof MultipartBody.Builder) { - requestBldr = requestBldr.header(HEADER_CONTENT_TYPE, MIMETYPE_MULTIPART_MIXED); - logger.debug("Sending multipart for {}", requestBldr.build().url().encodedPath()); - } else { - throw new IllegalArgumentException( - "Unknown input mimetype specifier " - + inputMimetype.getClass().getName()); - } - - if (outputMimetype == null) { - } else if (outputMimetype instanceof String) { - requestBldr = requestBldr.header(HEADER_ACCEPT, (String) outputMimetype); - } else if (outputMimetype instanceof MediaType) { - requestBldr = requestBldr.header(HEADER_ACCEPT, outputMimetype.toString()); - } else { - throw new IllegalArgumentException( - "Unknown output mimetype specifier " - + outputMimetype.getClass().getName()); - } - - return requestBldr; - } - - private Request.Builder setupRequest(String path, RequestParameters params, Object inputMimetype, - Object outputMimetype) - { - return setupRequest(setupRequest(path, params), inputMimetype, outputMimetype); - } - - private void checkStatus(Response response, int status, String operation, String entityType, - String path, ResponseStatus expected) - { - if (!expected.isExpected(status)) { - FailedRequest failure = extractErrorFields(response); - if (status == STATUS_NOT_FOUND) { - throw new ResourceNotFoundException("Could not " + operation - + " " + entityType + " at " + path, - failure); - } - if ("RESTAPI-CONTENTNOVERSION".equals(failure.getMessageCode())) { - throw new ContentNoVersionException("Content version required to " + - operation + " " + entityType + " at " + path, failure); - } else if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException("User is not allowed to " - + operation + " " + entityType + " at " + path, - failure); - } - throw new FailedRequestException("failed to " + operation + " " - + entityType + " at " + path + ": " - + getReasonPhrase(response), failure); - } - } - - private T makeResult(RequestLogger reqlog, String operation, - String entityType, Response response, Class as) { - if (as == null) { - return null; - } - - logRequest(reqlog, "%s for %s", operation, entityType); - - ResponseBody body = response.body(); - T entity = body.contentLength() != 0 ? getEntity(body, as) : null; - if (entity == null || (as != InputStream.class && as != Reader.class)) { - closeResponse(response); - } - - return (reqlog != null) ? reqlog.copyContent(entity) : entity; - } - - static private List readMultipartBodyParts(ResponseBody body) { - long length = body.contentLength(); - MimeMultipart entity = length != 0 ? getEntity(body, MimeMultipart.class) : null; - try { - if (length == -1 && entity != null) entity.getCount(); - } catch (MessagingException e) { - entity = null; - } - return getPartList(entity); - } - - private U makeResults(ResultIteratorConstructor constructor, RequestLogger reqlog, - String operation, String entityType, Response response) { - if ( response == null ) return null; - final List partList = readMultipartBodyParts(response.body()); - throwExceptionIfErrorInTrailers(operation, entityType, response); - return makeResults(constructor, reqlog, operation, entityType, partList, response, response); - } - - static private void throwExceptionIfErrorInTrailers(String operation, String entityType, Response response) { - String mlErrorCode = null; - String mlErrorMessage = null; - try { - Headers trailers = response.trailers(); - mlErrorCode = trailers.get("ml-error-code"); - mlErrorMessage = trailers.get("ml-error-message"); - } catch (IOException e) { - // This does not seem worthy of causing the entire operation to fail; we also don't expect this to occur, as it - // should only occur due to a programming error where the response body has already been consumed - logger.warn("Unexpected IO error while getting HTTP response trailers: " + e.getMessage()); - } - - if (mlErrorCode != null && !"N/A".equals(mlErrorCode)) { - FailedRequest failure = new FailedRequest(); - failure.setMessageString(mlErrorCode); - failure.setStatusString(mlErrorMessage); - failure.setStatusCode(500); - String message = String.format("failed to %s %s at rows: %s, %s", operation, entityType, mlErrorCode, mlErrorMessage); - throw new FailedRequestException(message, failure); - } - } - - private U makeResults( - ResultIteratorConstructor constructor, RequestLogger reqlog, - String operation, String entityType, List partList, Response response, - Closeable closeable) { - logRequest(reqlog, "%s for %s", operation, entityType); - - if ( response == null ) return null; - - try { - OkHttpResultIterator result = constructor.construct(reqlog, partList, closeable); - Headers headers = response.headers(); - long pageStart = Utilities.parseLong(headers.get(HEADER_VND_MARKLOGIC_START)); - if (pageStart > -1l) { - result.setStart(pageStart); - } - long pageLength = Utilities.parseLong(headers.get(HEADER_VND_MARKLOGIC_PAGELENGTH)); - if (pageLength > -1l) { - result.setPageSize(pageLength); - } - long totalSize = Utilities.parseLong(headers.get(HEADER_VND_MARKLOGIC_RESULT_ESTIMATE)); - if (totalSize > -1l) { - result.setTotalSize(totalSize); - } - return (U) result; - } catch (Throwable t) { - throw new MarkLogicInternalException("Error constructing iterator", t); - } - } - - private boolean isStreaming(Object value) { - return !(value instanceof String || value instanceof byte[] || value instanceof File); - } - - private void logRequest(RequestLogger reqlog, String message, - Object... params) { - if (reqlog == null) return; - - PrintStream out = reqlog.getPrintStream(); - if (out == null) return; - - if (params == null || params.length == 0) { - out.println(message); - } else { - out.format(message, params); - out.println(); - } - } - - private String stringJoin(Collection collection, String separator, - String defaultValue) { - if (collection == null || collection.size() == 0) return defaultValue; - - StringBuilder builder = null; - for (Object value : collection) { - if (builder == null) { - builder = new StringBuilder(); - } else { - builder.append(separator); - } - - builder.append(value); - } - - return (builder != null) ? builder.toString() : null; - } - - private int calculateDelay(Random rand, int i) { - int min = - (i > 6) ? DELAY_CEILING : - (i == 0) ? DELAY_FLOOR : - DELAY_FLOOR + (1 << i) * DELAY_MULTIPLIER; - int range = - (i > 6) ? DELAY_FLOOR : - (i == 0) ? 2 * DELAY_MULTIPLIER : - (i == 6) ? DELAY_CEILING - min : - (1 << i) * DELAY_MULTIPLIER; - return min + randRetry.nextInt(range); - } - - static class OkHttpResult { - private RequestLogger reqlog; - private BodyPart part; - private boolean extractedHeaders = false; - private String uri; - private RequestParameters headers = new RequestParameters(); - private Format format; - private String mimetype; - private long length; - - OkHttpResult(RequestLogger reqlog, BodyPart part) { - this.reqlog = reqlog; - this.part = part; - } - - public R getContent(R handle) { - if (part == null) throw new IllegalStateException("Content already retrieved"); - - HandleImplementation handleBase = HandleAccessor.as(handle); - - extractHeaders(); - updateFormat(handleBase, format); - updateMimetype(handleBase, mimetype); - updateLength(handleBase, length); - - try { - Object contentEntity = getEntity(part, handleBase.receiveAs()); - handleBase.receiveContent((reqlog != null) ? reqlog.copyContent(contentEntity) : contentEntity); - - return handle; - } finally { - part = null; - reqlog = null; - } - } - - public T getContentAs(Class as) { - ContentHandle readHandle = DatabaseClientFactory.getHandleRegistry().makeHandle(as); - readHandle = getContent(readHandle); - if ( readHandle == null ) return null; - return readHandle.get(); - } - - public String getUri() { - extractHeaders(); - return uri; - } - public Format getFormat() { - extractHeaders(); - return format; - } - - public String getMimetype() { - extractHeaders(); - return mimetype; - } - - public long getLength() { - extractHeaders(); - return length; - } - - public String getHeader(String name) { - extractHeaders(); - List values = headers.get(name); - if ( values != null && values.size() > 0 ) { - return values.get(0); - } - return null; - } - - public Map> getHeaders() { - extractHeaders(); - return headers.getMap(); - } - - private void extractHeaders() { - if (part == null || extractedHeaders) return; - try { - for ( Enumeration

e = part.getAllHeaders(); e.hasMoreElements(); ) { - Header header = e.nextElement(); - headers.put(header.getName(), header.getValue()); - } - format = getHeaderFormat(part); - mimetype = getHeaderMimetype(OkHttpServices.getHeader(part, HEADER_CONTENT_TYPE)); - length = getHeaderLength(OkHttpServices.getHeader(part, HEADER_CONTENT_LENGTH)); - uri = getHeaderUri(part); - extractedHeaders = true; - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } - } - } - - static class OkHttpServiceResult extends OkHttpResult implements RESTServices.RESTServiceResult { - OkHttpServiceResult(RequestLogger reqlog, BodyPart part) { - super(reqlog, part); - } - } - - static abstract class OkHttpResultIterator { - private RequestLogger reqlog; - private Iterator partQueue; - private long start = -1; - private long size = -1; - private long pageSize = -1; - private long totalSize = -1; - private Closeable closeable; - - OkHttpResultIterator(RequestLogger reqlog, List partList, Closeable closeable) { - this.reqlog = reqlog; - if (partList != null && partList.size() > 0) { - this.size = partList.size(); - this.partQueue = new ConcurrentLinkedQueue<>( - partList).iterator(); - } else { - this.size = 0; - } - this.closeable = closeable; - } - - public long getStart() { - return start; - } - - public OkHttpResultIterator setStart(long start) { - this.start = start; - return this; - } - - public long getSize() { - return size; - } - - public OkHttpResultIterator setSize(long size) { - this.size = size; - return this; - } - - public long getPageSize() { - return pageSize; - } - - public OkHttpResultIterator setPageSize(long pageSize) { - this.pageSize = pageSize; - return this; - } - - public long getTotalSize() { - return totalSize; - } - - public OkHttpResultIterator setTotalSize(long totalSize) { - this.totalSize = totalSize; - return this; - } - - public boolean hasNext() { - if (partQueue == null) return false; - boolean hasNext = partQueue.hasNext(); - return hasNext; - } - - public T next() { - if (partQueue == null) return null; - - try { - return constructNext(reqlog, partQueue.next()); - } catch (Throwable t) { - throw new IllegalStateException("Error instantiating iterated result", t); - } - } - - abstract T constructNext(RequestLogger logger, BodyPart part); - - public void remove() { - if (partQueue == null) return; - partQueue.remove(); - if (!partQueue.hasNext()) close(); - } - - public void close() { - partQueue = null; - reqlog = null; - if ( closeable != null ) { - try { - closeable.close(); - } catch (IOException e) { - throw new MarkLogicIOException(e); - } - } - } - } - - static class OkHttpServiceResultIterator - extends OkHttpResultIterator - implements RESTServiceResultIterator - { - OkHttpServiceResultIterator(RequestLogger reqlog, - List partList, Closeable closeable) { - super(reqlog, partList, closeable); - } - OkHttpServiceResult constructNext(RequestLogger logger, BodyPart part) { - return new OkHttpServiceResult(logger, part); - } - } - - static class DefaultOkHttpResultIterator - extends OkHttpResultIterator - implements Iterator - { - DefaultOkHttpResultIterator(RequestLogger reqlog, - List partList, Closeable closeable) { - super(reqlog, partList, closeable); - } - OkHttpResult constructNext(RequestLogger logger, BodyPart part) { - return new OkHttpResult(logger, part); - } - } - - static class OkHttpDocumentRecord implements DocumentRecord { - private OkHttpResult content; - private OkHttpResult metadata; - - OkHttpDocumentRecord(OkHttpResult content, OkHttpResult metadata) { - this.content = content; - this.metadata = metadata; - } - - OkHttpDocumentRecord(OkHttpResult content) { - this.content = content; - } - - @Override - public String getUri() { - if ( content == null && metadata != null ) { - return metadata.getUri(); - } else if ( content != null ) { - return content.getUri(); - } else { - throw new IllegalStateException("Missing both content and metadata!"); - } - } - - @Override - public DocumentDescriptor getDescriptor() { - if (content == null) { - throw new IllegalStateException("getDescriptor() called when no content is available"); - } - DocumentDescriptorImpl descriptor = new DocumentDescriptorImpl(getUri(), false); - updateFormat(descriptor, getFormat()); - updateMimetype(descriptor, getMimetype()); - updateLength(descriptor, getLength()); - updateVersion(descriptor, content.getHeader(HEADER_ETAG)); - return descriptor; - } - - @Override - public Format getFormat() { - if (content == null) { - throw new IllegalStateException("getFormat() called when no content is available"); - } - return content.getFormat(); - } - - @Override - public String getMimetype() { - if (content == null) { - throw new IllegalStateException("getMimetype() called when no content is available"); - } - return content.getMimetype(); - } - - @Override - public long getLength() { - if (content == null) { - throw new IllegalStateException("getLenth() called when no content is available"); - } - return content.getLength(); - } - - @Override - public T getMetadata(T metadataHandle) { - if (metadata == null) { - throw new IllegalStateException("getMetadata called when no metadata is available"); - } - return metadata.getContent(metadataHandle); - } - - @Override - public T getMetadataAs(Class as) { - if ( as == null ) { - throw new IllegalStateException("getMetadataAs cannot accept null"); - } - return metadata.getContentAs(as); - } - - @Override - public T getContent(T contentHandle) { - if ( content == null ) { - throw new IllegalStateException("getContent called when no content is available"); - } - return content.getContent(contentHandle); - } - - @Override - public T getContentAs(Class as) { - if ( as == null ) { - throw new IllegalStateException("getContentAs cannot accept null"); - } - return content.getContentAs(as); - } - } - - @Override - public OkHttpClient getClientImplementation() { - if (client == null) return null; - return client; - } - - public void setClientImplementation(OkHttpClient client) { - this.client = client; - } - - @Override - public T suggest(Class as, SuggestDefinition suggestionDef) { - RequestParameters params = new RequestParameters(); - - String suggestCriteria = suggestionDef.getStringCriteria(); - String[] queries = suggestionDef.getQueryStrings(); - String optionsName = suggestionDef.getOptionsName(); - Integer limit = suggestionDef.getLimit(); - Integer cursorPosition = suggestionDef.getCursorPosition(); - - if (suggestCriteria != null) { - params.add("partial-q", suggestCriteria); - } - if (optionsName != null) { - params.add("options", optionsName); - } - if (limit != null) { - params.add("limit", Long.toString(limit)); - } - if (cursorPosition != null) { - params.add("cursor-position", Long.toString(cursorPosition)); - } - if (queries != null) { - for (String stringQuery : queries) { - params.add("q", stringQuery); - } - } - Request.Builder requestBldr = null; - requestBldr = setupRequest("suggest", params, null, MIMETYPE_APPLICATION_XML); - requestBldr = addTelemetryAgentId(requestBldr); - - Function doGetFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return sendRequestOnce(funcBuilder.get().build()); - } - }; - Response response = sendRequestWithRetry(requestBldr, doGetFunction, null); - int status = response.code(); - if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException( - "User is not allowed to get suggestions", - extractErrorFields(response)); - } - if (status != STATUS_OK) { - throw new FailedRequestException("Suggest call failed: " - + getReasonPhrase(response), extractErrorFields(response)); - } - - ResponseBody body = response.body(); - T entity = body.contentLength() != 0 ? getEntity(body, as) : null; - if (entity == null || (as != InputStream.class && as != Reader.class)) { - closeResponse(response); - } - - return entity; - } - - @Override - public InputStream match(StructureWriteHandle document, - String[] candidateRules, String mimeType, ServerTransform transform) { - RequestParameters params = new RequestParameters(); - - HandleImplementation baseHandle = HandleAccessor.checkHandle(document, "match"); - if (candidateRules != null) { - for (String candidateRule : candidateRules) { - params.add("rule", candidateRule); - } - } - if (transform != null) { - transform.merge(params); - } - Request.Builder requestBldr = null; - requestBldr = setupRequest("alert/match", params, MIMETYPE_APPLICATION_XML, mimeType); - requestBldr = addTelemetryAgentId(requestBldr); - - Function doPostFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return doPost(null, funcBuilder, baseHandle.sendContent()); - } - }; - Response response = sendRequestWithRetry(requestBldr, doPostFunction, null); - int status = response.code(); - - if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException("User is not allowed to match", - extractErrorFields(response)); - } - if (status != STATUS_OK) { - throw new FailedRequestException("match failed: " - + getReasonPhrase(response), extractErrorFields(response)); - } - - ResponseBody body = response.body(); - InputStream entity = body.contentLength() != 0 ? - getEntity(body, InputStream.class) : null; - if (entity == null) closeResponse(response); - - return entity; - } - - @Override - public InputStream match(QueryDefinition queryDef, - long start, long pageLength, String[] candidateRules, ServerTransform transform) { - if (queryDef == null) { - throw new IllegalArgumentException("Cannot match null query"); - } - - RequestParameters params = new RequestParameters(); - - if (start > 1) { - params.add("start", Long.toString(start)); - } - if (pageLength >= 0) { - params.add("pageLength", Long.toString(pageLength)); - } - if (transform != null) { - transform.merge(params); - } - if (candidateRules.length > 0) { - for (String candidateRule : candidateRules) { - params.add("rule", candidateRule); - } - } - - if (queryDef.getOptionsName() != null) { - params.add("options", queryDef.getOptionsName()); - } - - Request.Builder requestBldr = null; - String structure = null; - HandleImplementation baseHandle = null; - - String text = null; - if (queryDef instanceof StringQueryDefinition) { - text = ((StringQueryDefinition) queryDef).getCriteria(); - } else if (queryDef instanceof StructuredQueryDefinition) { - text = ((StructuredQueryDefinition) queryDef).getCriteria(); - } else if (queryDef instanceof RawStructuredQueryDefinition) { - text = ((RawStructuredQueryDefinition) queryDef).getCriteria(); - } - if (text != null) { - params.add("q", text); - } - if (queryDef instanceof StructuredQueryDefinition) { - structure = ((StructuredQueryDefinition) queryDef).serialize(); - - logger.debug("Searching with structured query {}", structure); - - requestBldr = setupRequest("alert/match", params, MIMETYPE_APPLICATION_XML, MIMETYPE_APPLICATION_XML); - } else if (queryDef instanceof RawQueryDefinition) { - StructureWriteHandle handle = ((RawQueryDefinition) queryDef).getHandle(); - baseHandle = HandleAccessor.checkHandle(handle, "match"); - - logger.debug("Searching with raw query"); - - requestBldr = setupRequest("alert/match", params, MIMETYPE_APPLICATION_XML, MIMETYPE_APPLICATION_XML); - } else if (queryDef instanceof StringQueryDefinition) { - logger.debug("Searching with string query [{}]", text); - - requestBldr = setupRequest("alert/match", params, null, MIMETYPE_APPLICATION_XML); - } else { - throw new UnsupportedOperationException("Cannot match with " - + queryDef.getClass().getName()); - } - requestBldr = addTelemetryAgentId(requestBldr); - - MediaType mediaType = makeType(requestBldr.build().header(HEADER_CONTENT_TYPE)); - - Response response = null; - int status = -1; - long startTime = System.currentTimeMillis(); - int nextDelay = 0; - int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { - if (nextDelay > 0) { - try { - Thread.sleep(nextDelay); - } catch (InterruptedException e) { - } - } - - if (queryDef instanceof StructuredQueryDefinition) { - response = doPost(null, requestBldr, structure); - } else if (queryDef instanceof RawQueryDefinition) { - response = doPost(null, requestBldr, baseHandle.sendContent()); - } else if (queryDef instanceof StringQueryDefinition) { - response = sendRequestOnce(requestBldr.get()); - } else { - throw new UnsupportedOperationException("Cannot match with " - + queryDef.getClass().getName()); - } - status = response.code(); - - if (!retryStatus.contains(status)) { - if (isFirstRequest()) setFirstRequest(false); - - break; - } - - String retryAfterRaw = response.header("Retry-After"); - int retryAfter = Utilities.parseInt(retryAfterRaw); - - closeResponse(response); - - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); - } - if (retryStatus.contains(status)) { - checkFirstRequest(); - closeResponse(response); - throw new FailedRetryException( - "Service unavailable and maximum retry period elapsed: "+ - ((System.currentTimeMillis() - startTime) / 1000)+ - " seconds after "+retry+" retries"); - } - if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException("User is not allowed to match", - extractErrorFields(response)); - } - if (status != STATUS_OK) { - throw new FailedRequestException("match failed: " - + getReasonPhrase(response), extractErrorFields(response)); - } - - ResponseBody body = response.body(); - InputStream entity = body.contentLength() != 0 ? - getEntity(body, InputStream.class) : null; - if (entity == null) closeResponse(response); - - return entity; - } - - @Override - public InputStream match(String[] docIds, String[] candidateRules, ServerTransform transform) { - RequestParameters params = new RequestParameters(); - - if (docIds.length > 0) { - for (String docId : docIds) { - params.add("uri", docId); - } - } - if (candidateRules.length > 0) { - for (String candidateRule : candidateRules) { - params.add("rule", candidateRule); - } - } - if (transform != null) { - transform.merge(params); - } - Request.Builder requestBldr = setupRequest("alert/match", params, MIMETYPE_APPLICATION_XML, MIMETYPE_APPLICATION_XML); - requestBldr = addTelemetryAgentId(requestBldr); - - Function doGetFunction = new Function() { - public Response apply(Request.Builder funcBuilder) { - return doGet(funcBuilder); - } - }; - Response response = sendRequestWithRetry(requestBldr, doGetFunction, null); - int status = response.code(); - if (status == STATUS_FORBIDDEN) { - throw new ForbiddenUserException("User is not allowed to match", - extractErrorFields(response)); - } - if (status != STATUS_OK) { - throw new FailedRequestException("match failed: " - + getReasonPhrase(response), extractErrorFields(response)); - } - - ResponseBody body = response.body(); - InputStream entity = body.contentLength() != 0 ? - getEntity(body, InputStream.class) : null; - if (entity == null) closeResponse(response); - - return entity; - } - - private void addGraphUriParam(RequestParameters params, String uri) { - if ( uri == null || uri.equals(GraphManager.DEFAULT_GRAPH) ) { - params.add("default", ""); - } else { - params.add("graph", uri); - } - } - - private void addPermsParams(RequestParameters params, GraphPermissions permissions) { - if ( permissions != null ) { - for ( Map.Entry> entry : permissions.entrySet() ) { - if ( entry.getValue() != null ) { - for ( Capability capability : entry.getValue() ) { - params.add("perm:" + entry.getKey(), capability.toString().toLowerCase()); - } - } - } - } - } - - @Override - public R getGraphUris(RequestLogger reqlog, R output) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - return getResource(reqlog, "graphs", null, null, output); - } - - @Override - public R readGraph(RequestLogger reqlog, String uri, R output, - Transaction transaction) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - RequestParameters params = new RequestParameters(); - addGraphUriParam(params, uri); - return getResource(reqlog, "graphs", transaction, params, output); - } - - @Override - public void writeGraph(RequestLogger reqlog, String uri, - AbstractWriteHandle input, GraphPermissions permissions, Transaction transaction) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - RequestParameters params = new RequestParameters(); - addGraphUriParam(params, uri); - addPermsParams(params, permissions); - putResource(reqlog, "graphs", transaction, params, input, null); - } - - @Override - public void writeGraphs(RequestLogger reqlog, AbstractWriteHandle input, Transaction transaction) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - RequestParameters params = new RequestParameters(); - putResource(reqlog, "graphs", transaction, params, input, null); - } - - @Override - public void mergeGraph(RequestLogger reqlog, String uri, - AbstractWriteHandle input, GraphPermissions permissions, Transaction transaction) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - RequestParameters params = new RequestParameters(); - addGraphUriParam(params, uri); - addPermsParams(params, permissions); - postResource(reqlog, "graphs", transaction, params, input, null); - } - - @Override - public void mergeGraphs(RequestLogger reqlog, AbstractWriteHandle input, Transaction transaction) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - RequestParameters params = new RequestParameters(); - postResource(reqlog, "graphs", transaction, params, input, null); - } - - @Override - public R getPermissions(RequestLogger reqlog, String uri, - R output,Transaction transaction) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - RequestParameters params = new RequestParameters(); - addGraphUriParam(params, uri); - params.add("category", "permissions"); - return getResource(reqlog, "graphs", transaction, params, output); - } - - @Override - public void deletePermissions(RequestLogger reqlog, String uri, Transaction transaction) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - RequestParameters params = new RequestParameters(); - addGraphUriParam(params, uri); - params.add("category", "permissions"); - deleteResource(reqlog, "graphs", transaction, params, null); - } - - @Override - public void writePermissions(RequestLogger reqlog, String uri, - AbstractWriteHandle permissions, Transaction transaction) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - RequestParameters params = new RequestParameters(); - addGraphUriParam(params, uri); - params.add("category", "permissions"); - putResource(reqlog, "graphs", transaction, params, permissions, null); - } - - @Override - public void mergePermissions(RequestLogger reqlog, String uri, - AbstractWriteHandle permissions, Transaction transaction) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - RequestParameters params = new RequestParameters(); - addGraphUriParam(params, uri); - params.add("category", "permissions"); - postResource(reqlog, "graphs", transaction, params, permissions, null); - } - - @Override - public Object deleteGraph(RequestLogger reqlog, String uri, Transaction transaction) - throws ForbiddenUserException, FailedRequestException - { - RequestParameters params = new RequestParameters(); - addGraphUriParam(params, uri); - return deleteResource(reqlog, "graphs", transaction, params, null); - - } - - @Override - public void deleteGraphs(RequestLogger reqlog, Transaction transaction) - throws ForbiddenUserException, FailedRequestException - { - deleteResource(reqlog, "graphs", transaction, null, null); - } - - @Override - public R getThings(RequestLogger reqlog, String[] iris, R output) - throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException - { - if ( iris == null ) throw new IllegalArgumentException("iris cannot be null"); - RequestParameters params = new RequestParameters(); - for ( String iri : iris ) { - params.add("iri", iri); - } - return getResource(reqlog, "graphs/things", null, params, output); - } - - @Override - public R executeSparql(RequestLogger reqlog, - SPARQLQueryDefinition qdef, R output, long start, long pageLength, - Transaction transaction, boolean isUpdate) - { - if ( qdef == null ) throw new IllegalArgumentException("qdef cannot be null"); - if ( output == null ) throw new IllegalArgumentException("output cannot be null"); - RequestParameters params = new RequestParameters(); - if (start > 1) params.add("start", Long.toString(start)); - if (pageLength >= 0) params.add("pageLength", Long.toString(pageLength)); - if (qdef.getOptimizeLevel() >= 0) { - params.add("optimize", Integer.toString(qdef.getOptimizeLevel())); - } - if (qdef.getCollections() != null ) { - for ( String collection : qdef.getCollections() ) { - params.add("collection", collection); - } - } - addPermsParams(params, qdef.getUpdatePermissions()); - String sparql = qdef.getSparql(); - SPARQLBindings bindings = qdef.getBindings(); - for ( Map.Entry> entry : bindings.entrySet() ) { - String paramName = "bind:" + entry.getKey(); - String typeOrLang = ""; - for ( SPARQLBinding binding : entry.getValue() ) { - if ( binding.getDatatype() != null ) { - typeOrLang = ":" + binding.getDatatype(); - } else if ( binding.getLanguageTag() != null ) { - typeOrLang = "@" + binding.getLanguageTag().toLanguageTag(); - } - params.add(paramName + typeOrLang, binding.getValue()); - } - } - QueryDefinition constrainingQuery = qdef.getConstrainingQueryDefinition(); - StructureWriteHandle input; - if ( constrainingQuery != null ) { - if (qdef.getOptionsName()!= null && qdef.getOptionsName().length() > 0) { - params.add("options", qdef.getOptionsName()); - } - if ( constrainingQuery instanceof RawCombinedQueryDefinition ) { - CombinedQueryDefinition combinedQdef = new CombinedQueryBuilderImpl().combine( - (RawCombinedQueryDefinition) constrainingQuery, null, null, sparql); - Format format = combinedQdef.getFormat(); - input = new StringHandle(combinedQdef.serialize()).withFormat(format); - } else if ( constrainingQuery instanceof RawStructuredQueryDefinition ) { - CombinedQueryDefinition combinedQdef = new CombinedQueryBuilderImpl().combine( - (RawStructuredQueryDefinition) constrainingQuery, null, null, sparql); - Format format = combinedQdef.getFormat(); - input = new StringHandle(combinedQdef.serialize()).withFormat(format); - } else if ( constrainingQuery instanceof StringQueryDefinition || - constrainingQuery instanceof StructuredQueryDefinition ) - { - String stringQuery = constrainingQuery instanceof StringQueryDefinition ? - ((StringQueryDefinition) constrainingQuery).getCriteria() : null; - StructuredQueryDefinition structuredQuery = - constrainingQuery instanceof StructuredQueryDefinition ? - (StructuredQueryDefinition) constrainingQuery : null; - CombinedQueryDefinition combinedQdef = new CombinedQueryBuilderImpl().combine( - structuredQuery, null, stringQuery, sparql); - input = new StringHandle(combinedQdef.serialize()).withMimetype(MIMETYPE_APPLICATION_XML); - } else { - throw new IllegalArgumentException( - "Constraining query must be of type SPARQLConstrainingQueryDefinition"); - } - } else { - String mimetype = isUpdate ? "application/sparql-update" : "application/sparql-query"; - input = new StringHandle(sparql).withMimetype(mimetype); - } - if (qdef.getBaseUri() != null) { - params.add("base", qdef.getBaseUri()); - } - if (qdef.getDefaultGraphUris() != null) { - for (String defaultGraphUri : qdef.getDefaultGraphUris()) { - params.add("default-graph-uri", defaultGraphUri); - } - } - if (qdef.getNamedGraphUris() != null) { - for (String namedGraphUri : qdef.getNamedGraphUris()) { - params.add("named-graph-uri", namedGraphUri); - } - } - if (qdef.getUsingGraphUris() != null) { - for (String usingGraphUri : qdef.getUsingGraphUris()) { - params.add("using-graph-uri", usingGraphUri); - } - } - if (qdef.getUsingNamedGraphUris() != null) { - for (String usingNamedGraphUri : qdef.getUsingNamedGraphUris()) { - params.add("using-named-graph-uri", usingNamedGraphUri); - } - } - - // rulesets - if (qdef.getRulesets() != null) { - for (SPARQLRuleset ruleset : qdef.getRulesets()) { - params.add("ruleset", ruleset.getName()); - } - } - if (qdef.getIncludeDefaultRulesets() != null) { - params.add("default-rulesets", qdef.getIncludeDefaultRulesets() ? "include" : "exclude"); - } - - return postResource(reqlog, "/graphs/sparql", transaction, params, input, output); - } - - static private String getTransactionId(Transaction transaction) { - if ( transaction == null ) return null; - return transaction.getTransactionId(); - } - - static private String getReasonPhrase(Response response) { - if (response == null || response.message() == null) return ""; - // strip off the number part of the reason phrase - return response.message().replaceFirst("^\\d+ ", ""); - } - - static private T getEntity(BodyPart part, Class as) { - try { - String contentType = part.getContentType(); - return getEntity( - ResponseBody.create(Okio.buffer(Okio.source(part.getInputStream())), MediaType.parse(contentType), part.getSize()), - as); - } catch (IOException e) { - throw new MarkLogicIOException(e); - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } - } - - static private MediaType makeType(String mimetype) { - if ( mimetype == null ) return null; - MediaType type = MediaType.parse(mimetype); - if ( type == null ) throw new IllegalArgumentException("Invalid mime-type: " + mimetype); - return type; - } - - static private T getEntity(ResponseBody body, Class as) { - try { - if ( as == InputStream.class ) { - return (T) body.byteStream(); - } else if ( as == byte[].class ) { - return (T) body.bytes(); - } else if ( as == Reader.class ) { - return (T) body.charStream(); - } else if ( as == String.class ) { - return (T) body.string(); - } else if ( as == MimeMultipart.class ) { - MediaType mediaType = body.contentType(); - String contentType = (mediaType != null) ? mediaType.toString() : "application/x-unknown-content-type"; - ByteArrayDataSource dataSource = new ByteArrayDataSource(body.byteStream(), contentType); - return (T) new MimeMultipart(dataSource); - } else if ( as == File.class ) { - // write out the response body to a temp file in the system temp folder - // then return the path to that file as a File object - String suffix = ".unknown"; - boolean isBinary = true; - MediaType mediaType = body.contentType(); - if (mediaType != null) { - String subtype = mediaType.subtype(); - if (subtype != null) { - subtype = subtype.toLowerCase(); - if (subtype.endsWith("json")) { - suffix = ".json"; - isBinary = false; - } else if (subtype.endsWith("xml")) { - suffix = ".xml"; - isBinary = false; - } else if (subtype.equals("vnd.marklogic-js-module")) { - suffix = ".mjs"; - isBinary = false; - } else if (subtype.equals("vnd.marklogic-javascript")) { - suffix = ".sjs"; - isBinary = false; - } else if (subtype.equals("vnd.marklogic-xdmp") || subtype.endsWith("xquery")) { - suffix = ".xqy"; - isBinary = false; - } else if (subtype.endsWith("javascript")) { - suffix = ".js"; - isBinary = false; - } else if (subtype.endsWith("html")) { - suffix = ".html"; - isBinary = false; - } else if (mediaType.type().equalsIgnoreCase("text")) { - suffix = ".txt"; - isBinary = false; - } else { - suffix = "." + subtype; - } - } - } - Path path = Files.createTempFile("tmp", suffix); - if ( isBinary == true ) { - Files.copy(body.byteStream(), path, StandardCopyOption.REPLACE_EXISTING); - } else { - try(Writer out = Files.newBufferedWriter(path, Charset.forName("UTF-8"))) { - Utilities.write(body.charStream(), out); - } - } - return (T) path.toFile(); - } else { - throw new IllegalArgumentException( - "Handle recieveAs returned " + as + " which is not a supported type. " + - "Try InputStream, Reader, String, byte[], File."); - } - } catch (IOException e) { - throw new MarkLogicIOException(e); - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } - } - - static private List getPartList(MimeMultipart multipart) { - try { - if ( multipart == null ) return null; - List partList = new ArrayList(); - for ( int i = 0; i < multipart.getCount(); i++ ) { - partList.add(multipart.getBodyPart(i)); - } - return partList; - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } - } - - static private class ObjectRequestBody extends RequestBody { - private Object obj; - private MediaType contentType; - - ObjectRequestBody(Object obj, MediaType contentType) { - super(); - this.obj = obj; - this.contentType = contentType; - } - - @Override - public MediaType contentType() { - return contentType; - } - - @Override - public void writeTo(BufferedSink sink) throws IOException { - if ( obj instanceof InputStream ) { - sink.writeAll(Okio.source((InputStream) obj)); - } else if ( obj instanceof File ) { - try ( Source source = Okio.source((File) obj) ) { - sink.writeAll(source); - } - } else if ( obj instanceof byte[] ) { - sink.write((byte[]) obj); - } else if ( obj instanceof String) { - sink.write(((String) obj).getBytes(StandardCharsets.UTF_8)); - } else if ( obj == null ) { - } else { - throw new IllegalStateException("Cannot write object of type: " + obj.getClass()); - } - } - } - - // API First Changes - static private class EmptyRequestBody extends RequestBody { - @Override - public MediaType contentType() { - return null; - } - @Override - public void writeTo(BufferedSink sink) { - } - } - - static class AtomicRequestBody extends RequestBody { - private MediaType contentType; - private String value; - AtomicRequestBody(String value, MediaType contentType) { - super(); - this.value = value; - this.contentType = contentType; - } - @Override - public MediaType contentType() { - return contentType; - } - @Override - public void writeTo(BufferedSink sink) throws IOException { - sink.writeUtf8(value); - } - } - - public static RequestBody makeRequestBodyForContent(AbstractWriteHandle content) { - if (content == null) { - return new EmptyRequestBody(); - } - HandleImplementation handleBase = HandleAccessor.as(content); - Format format = handleBase.getFormat(); - String mimetype = (format == Format.BINARY) ? null : handleBase.getMimetype(); - MediaType mediaType = MediaType.parse( - (mimetype != null) ? mimetype : "application/x-unknown-content-type" - ); - return (content instanceof OutputStreamSender) ? - new StreamingOutputImpl((OutputStreamSender) content, null, mediaType) : - new ObjectRequestBody(HandleAccessor.sendContent(content), mediaType); - } - - class CallRequestImpl implements CallRequest { - private SessionStateImpl session; - private Request.Builder requestBldr; - private RequestBody requestBody; - private boolean hasStreamingPart; - private HttpMethod method; - private String endpoint; - private HttpUrl callBaseUri; - - CallRequestImpl(String endpoint, HttpMethod method, SessionState session) { - if (session != null && !(session instanceof SessionStateImpl)) { - throw new IllegalArgumentException("Session state must be implemented by internal class: "+session.getClass().getName()); - } - this.endpoint = endpoint; - this.method = method; - this.session = (SessionStateImpl) session; - this.hasStreamingPart = false; - this.callBaseUri = HttpUrlBuilder.newDataServicesBaseUri(baseUri); - } - - @Override - public CallResponse withEmptyResponse() { - prepareRequestBuilder(); - CallResponseImpl responseImpl = new CallResponseImpl(); - executeRequest(responseImpl); - return responseImpl; - } - - @Override - public SingleCallResponse withDocumentResponse(Format format) { - prepareRequestBuilder(); - SingleCallResponseImpl responseImpl = new SingleCallResponseImpl(format); - this.requestBldr = forDocumentResponse(requestBldr, format); - executeRequest(responseImpl); - return responseImpl; - } - - @Override - public MultipleCallResponse withMultipartMixedResponse(Format format) { - prepareRequestBuilder(); - MultipleCallResponseImpl responseImpl = new MultipleCallResponseImpl(format); - this.requestBldr = forMultipartMixedResponse(requestBldr); - executeRequest(responseImpl); - return responseImpl; - } - - @Override - public boolean hasStreamingPart() { - return this.hasStreamingPart; - } - - @Override - public SessionState getSession() { - return this.session; - } - - @Override - public String getEndpoint() { - return this.endpoint; - } - - @Override - public HttpMethod getHttpMethod() { - return this.method; - } - - private void prepareRequestBuilder() { - this.requestBldr = setupRequest(callBaseUri, endpoint, null); - if (session != null) { - this.requestBldr = addCookies(this.requestBldr, session.getCookies(), session.getCreatedTimestamp()); - // Add the Cookie header for SessionId if we have a session object passed - this.requestBldr.addHeader(HEADER_COOKIE, "SessionID="+session.getSessionId()); - } - addHttpMethod(); - this.requestBldr.addHeader(HEADER_ERROR_FORMAT, MIMETYPE_APPLICATION_JSON); - } - - private void addHttpMethod() { - if(method != null && method == HttpMethod.POST) { - if(requestBody == null) { - throw new IllegalStateException("Request Body is null!"); - } - this.requestBldr.post(requestBody); - } else { - throw new IllegalStateException("HTTP method is null or invalid!"); - } - } - - private void executeRequest(CallResponseImpl responseImpl) { - SessionState session = getSession(); - //TODO: Add a telemetry agent if needed - // requestBuilder = addTelemetryAgentId(requestBuilder); - - boolean hasStreamingPart = hasStreamingPart(); - Consumer resendableConsumer = resendable -> { - if (hasStreamingPart) { - checkFirstRequest(); - throw new ResourceNotResendableException( - "Cannot retry request for " + getEndpoint()); - } - }; - - Function sendRequestFunction = requestBldr -> { - if (isFirstRequest() && hasStreamingPart) makeFirstRequest(callBaseUri, "", 0); - Response response = sendRequestOnce(requestBldr); - if (isFirstRequest()) setFirstRequest(false); - return response; - }; - - Response response = sendRequestWithRetry(requestBldr, sendRequestFunction, resendableConsumer); - - if(session != null) { - List cookies = new ArrayList<>(); - for ( String setCookie : response.headers(HEADER_SET_COOKIE) ) { - ClientCookie cookie = ClientCookie.parse(requestBldr.build().url(), setCookie); - cookies.add(cookie); - } - ((SessionStateImpl)session).setCookies(cookies); - } - checkStatus(response); - responseImpl.setResponse(response); - } - - private void checkStatus(Response response) { - int statusCode = response.code(); - if (statusCode >= 300) { - FailedRequest failure = null; - String contentType = response.header(HEADER_CONTENT_TYPE); - MediaType mediaType = MediaType.parse( - (contentType != null) ? contentType : "application/x-unknown-content-type" - ); - String subtype = (mediaType != null) ? mediaType.subtype() : null; - if (subtype != null) { - subtype = subtype.toLowerCase(); - if (subtype.endsWith("json") || subtype.endsWith("xml")) { - failure = extractErrorFields(response); - } - } - if (failure == null) { - closeResponse(response); - if (statusCode == STATUS_UNAUTHORIZED) { - failure = new FailedRequest(); - failure.setMessageString("Unauthorized"); - failure.setStatusString("Failed Auth"); - } else if (statusCode == STATUS_NOT_FOUND) { - throw new ResourceNotFoundException("Could not " + method + " at " + endpoint); - } else if (statusCode == STATUS_FORBIDDEN) { - throw new ForbiddenUserException("User is not allowed to " + method + " at " + endpoint); - } else { - failure = new FailedRequest(); - failure.setStatusCode(statusCode); - failure.setMessageCode("UNKNOWN"); - failure.setMessageString("Server did not respond with an expected Error message."); - failure.setStatusString("UNKNOWN"); - } - } - FailedRequestException ex = failure == null ? new FailedRequestException("failed to " + method + " at " + endpoint + ": " - + getReasonPhrase(response)) : new FailedRequestException("failed to " + method + " at " + endpoint + ": " - + getReasonPhrase(response), failure); - throw ex; - } - } - - public CallRequest withEmptyRequest() { - requestBody = new EmptyRequestBody(); - return this; - } - - public CallRequest withAtomicBodyRequest(CallField... params) { - String atomics = Stream.of(params) - .map(param -> encodeParamValue(param)) - .filter(param -> param != null) - .collect(Collectors.joining("&")); - requestBody = RequestBody.create((atomics == null) ? "" : atomics, URLENCODED_MIME_TYPE); - return this; - } - - public CallRequest withNodeBodyRequest(CallField... params) { - this.requestBody = makeRequestBody(params); - return this; - } - - private RequestBody makeRequestBody(String value) { - if (value == null) { - return new EmptyRequestBody(); - } - return new AtomicRequestBody(value, MediaType.parse("text/plain")); - } - - private RequestBody makeRequestBody(AbstractWriteHandle document) { - if (document == null) { - return new EmptyRequestBody(); - } - HandleImplementation handleBase = HandleAccessor.as(document); - Format format = handleBase.getFormat(); - String mimetype = (format == Format.BINARY) ? null : handleBase.getMimetype(); - MediaType mediaType = MediaType.parse( - (mimetype != null) ? mimetype : "application/x-unknown-content-type" - ); - return (document instanceof OutputStreamSender) ? - new StreamingOutputImpl((OutputStreamSender) document, null, mediaType) : - new ObjectRequestBody(HandleAccessor.sendContent(document), mediaType); - } - - private RequestBody makeRequestBody(CallField[] params) { - if (params == null || params.length == 0) { - return new EmptyRequestBody(); - } - - MultipartBody.Builder multiBldr = new MultipartBody.Builder(); - multiBldr.setType(MultipartBody.FORM); - - Condition hasValue = new Condition(); - Condition hasStreamingPartCondition = new Condition(); - for (CallField param: params) { - if (param == null) { - continue; - } - - final String paramName = param.getParamName(); - if (param instanceof SingleAtomicCallField) { - String paramValue = ((SingleAtomicCallField) param).getParamValue(); - if (paramValue != null) { - hasValue.set(); - multiBldr.addFormDataPart(paramName, null, makeRequestBody(paramValue)); - } - } else if (param instanceof MultipleAtomicCallField) { - Stream paramValues = ((MultipleAtomicCallField) param).getParamValues(); - if (paramValues != null) { - paramValues - .filter(paramValue -> paramValue != null) - .forEachOrdered(paramValue -> { - hasValue.set(); - multiBldr.addFormDataPart(paramName, null, makeRequestBody(paramValue)); - }); - } - } else if (param instanceof SingleNodeCallField) { - SingleNodeCallField singleNodeParam = (SingleNodeCallField) param; - BufferableHandle paramValue = singleNodeParam.getParamValue(); - if (paramValue != null) { - HandleImplementation handleBase = HandleAccessor.as(paramValue); - if(! handleBase.isResendable()) { - BytesHandle bytesHandle = new BytesHandle(paramValue); - singleNodeParam.setParamValue(bytesHandle); - paramValue = bytesHandle; - } - hasValue.set(); - multiBldr.addFormDataPart(paramName, null, makeRequestBodyForContent(paramValue)); - } - } else if (param instanceof UnbufferedMultipleNodeCallField) { - Stream paramValues = ((UnbufferedMultipleNodeCallField) param).getParamValues(); - if (paramValues != null) { - paramValues - .filter(paramValue -> paramValue != null) - .forEachOrdered(paramValue -> { - HandleImplementation handleBase = HandleAccessor.as(paramValue); - if(!handleBase.isResendable()) { - hasStreamingPartCondition.set(); - } - hasValue.set(); - multiBldr.addFormDataPart(paramName, null, makeRequestBody(paramValue)); - }); - } - } else if (param instanceof BufferedMultipleNodeCallField) { - BufferableHandle[] paramValues = ((BufferedMultipleNodeCallField) param).getParamValuesArray(); - if(paramValues != null) { - boolean checkedBuffer = false; - for(int i=0; i < paramValues.length; i++) { - BufferableHandle paramValue = paramValues[i]; - if (paramValue != null) { - HandleImplementation handleBase = HandleAccessor.as(paramValue); - if (!handleBase.isResendable()) { - paramValue = new BytesHandle(paramValue); - if (!checkedBuffer) { - Class actualClass = paramValues.getClass().getComponentType(); - if (actualClass != BufferableHandle.class && actualClass != BytesHandle.class) { - paramValues = - Arrays.copyOf(paramValues, paramValues.length, BufferableHandle[].class); - } - checkedBuffer = true; - } - paramValues[i] = paramValue; - } - hasValue.set(); - multiBldr.addFormDataPart(paramName, null, makeRequestBody(paramValue)); - } - } - } - } - else { - throw new IllegalStateException( - "unknown multipart "+paramName+" param of: "+param.getClass().getName() - ); - } - } - - if (!hasValue.get()) { - return new EmptyRequestBody(); - } - this.hasStreamingPart = hasStreamingPartCondition.get(); - return multiBldr.build(); - } - - } - - @Override - public CallRequest makeEmptyRequest(String endpoint, HttpMethod method, SessionState session) { - return new CallRequestImpl(endpoint, method, session).withEmptyRequest(); - } - - @Override - public CallRequest makeAtomicBodyRequest(String endpoint, HttpMethod method, SessionState session, CallField... params) { - if (params == null || params.length == 0) { - return makeEmptyRequest(endpoint, method, session); - } - return new CallRequestImpl(endpoint, method, session).withAtomicBodyRequest(params); - } - - @Override - public CallRequest makeNodeBodyRequest(String endpoint, HttpMethod method, SessionState session, CallField... params) { - if (params == null || params.length == 0) { - return makeEmptyRequest(endpoint, method, session); - } - return new CallRequestImpl(endpoint, method, session).withNodeBodyRequest(params); - } - - static private String encodeParamValue(String paramName, String value) { - if (value == null) { - return null; - } - try { - return paramName+"="+URLEncoder.encode(value, UTF8_ID); - } catch(UnsupportedEncodingException e) { - throw new IllegalStateException("UTF-8 is unsupported", e); - } - } - - static private String encodeParamValue(SingleAtomicCallField param) { - if (param == null) { - return null; - } - return encodeParamValue(param.getParamName(), param.getParamValue()); - } - - static private String encodeParamValue(MultipleAtomicCallField param) { - if (param == null) { - return null; - } - String paramName = param.getParamName(); - Stream paramValues = param.getParamValues(); - if (paramValues == null) { - return null; - } - String encodedParamValues = paramValues - .map(paramValue -> encodeParamValue(paramName, paramValue)) - .filter(paramValue -> (paramValue != null)) - .collect(Collectors.joining("&")); - if (encodedParamValues == null || encodedParamValues.length() == 0) { - return null; - } - return encodedParamValues; - } - - static private String encodeParamValue(CallField param) { - if (param == null) { - return null; - } else if (param instanceof SingleAtomicCallField) { - return encodeParamValue((SingleAtomicCallField) param); - } else if (param instanceof MultipleAtomicCallField) { - return encodeParamValue((MultipleAtomicCallField) param); - } - throw new IllegalStateException( - "could not encode parameter "+param.getParamName()+" of type: "+param.getClass().getName() - ); - } - - static class CallResponseImpl implements CallResponse { - private boolean isNull = true; - private Response response; - - Response getResponse() { - return response; - } - void setResponse(Response response) { - this.response = response; - } - - @Override - public boolean isNull() { - return isNull; - } - - void setNull(boolean isNull) { - this.isNull = isNull; - } - - @Override - public int getStatusCode() { - return response.code(); - } - - @Override - public String getStatusMsg() { - return response.message(); - } - - //TODO: Check if this is needed since we are parsing it in the checkStatus(Respose). - //TODO: It might throw a closed exception since the response would be closed. Remove it after some testing - @Override - public String getErrorBody() { - try (ResponseBody errorBody = response.body()) { - if (errorBody.contentLength() > 0) { - MediaType errorType = errorBody.contentType(); - if (errorType != null) { - String subtype = errorType.subtype(); - if (subtype != null && subtype.toLowerCase().endsWith("json")) { - return errorBody.string(); - } - } - } - } catch (IOException e) { - throw new MarkLogicIOException(e); - } - return null; - } - - @Override - public void close() { - } - } - - static class SingleCallResponseImpl extends CallResponseImpl implements SingleCallResponse, AutoCloseable { - private Format format; - private ResponseBody responseBody; - SingleCallResponseImpl(Format format) { - this.format = format; - } - void setResponse(Response response) { - super.setResponse(response); - setResponseBody(response.body()); - } - void setResponseBody(ResponseBody responseBody) { - if (!checkNull(responseBody, format)) { - this.responseBody = responseBody; - setNull(false); - } - } - - @Override - public byte[] asBytes() { - try { - if (responseBody == null) { - return null; - } - byte[] value = responseBody.bytes(); - closeImpl(); - return value; - } catch (IOException e) { - throw new MarkLogicIOException(e); - } - } - @Override - public C asContent(BufferableContentHandle outputHandle) { - if (responseBody == null) return null; - HandleImplementation handleImpl = (HandleImplementation) outputHandle; - Class receiveClass = handleImpl.receiveAs(); - C content= outputHandle.toContent(getEntity(responseBody, receiveClass)); - return content; - } - @Override - public > T asHandle(T outputHandle) { - if (responseBody == null) return null; - - return updateHandle(getResponse().headers(), responseBody, outputHandle); - } - @Override - public InputStream asInputStream() { - return (responseBody == null) ? null : responseBody.byteStream(); - } - @Override - public InputStreamHandle asInputStreamHandle() { - return (responseBody == null) ? null : - updateHandle(getResponse().headers(), responseBody, new InputStreamHandle()); - } - @Override - public Reader asReader() { - return (responseBody == null) ? null : responseBody.charStream(); - } - @Override - public ReaderHandle asReaderHandle() { - return (responseBody == null) ? null : - updateHandle(getResponse().headers(), responseBody, new ReaderHandle(asReader())); - } - @Override - public String asString() { - try { - if (responseBody == null) { - return null; - } - String value = responseBody.string(); - closeImpl(); - return value; - } catch (IOException e) { - throw new MarkLogicIOException(e); - } - } - @Override - public boolean asEndpointState(BytesHandle endpointStateHandle) { - try { - if (endpointStateHandle == null || responseBody == null) - return false; - byte[] value = responseBody.bytes(); - closeImpl(); - endpointStateHandle.set(value); - return true; - } catch (IOException e) { - throw new MarkLogicIOException(e); - } - } - - @Override - public void close() { - if (responseBody != null) { - closeImpl(); - } - } - - private void closeImpl() { - responseBody.close(); - responseBody = null; - } - } - - static class MultipleCallResponseImpl extends CallResponseImpl implements MultipleCallResponse { - private Format format; - private MimeMultipart multipart; - MultipleCallResponseImpl(Format format){ - this.format = format; - } - void setResponse(Response response) { - try { - super.setResponse(response); - ResponseBody responseBody = response.body(); - if (responseBody == null) { - setNull(true); - return; - } - MediaType contentType = responseBody.contentType(); - if (contentType == null) { - setNull(true); - return; - } - ByteArrayDataSource dataSource = new ByteArrayDataSource( - responseBody.byteStream(), contentType.toString() - ); - setMultipart(new MimeMultipart(dataSource)); - } catch (IOException e) { - throw new MarkLogicIOException(e); - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } - } - void setMultipart(MimeMultipart multipart) { - if (!checkNull(multipart, format)) { - this.multipart = multipart; - setNull(false); - } - } - - @Override - public Stream asStreamOfBytes() { - try { - if (multipart == null) { - return Stream.empty(); - } - int partCount = multipart.getCount(); - - Stream.Builder builder = Stream.builder(); - for (int i=0; i < partCount; i++) { - BodyPart bodyPart = multipart.getBodyPart(i); - builder.accept(NodeConverter.InputStreamToBytes(bodyPart.getInputStream())); - } - return builder.build(); - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } catch (IOException e) { - throw new MarkLogicIOException(e); - } - } - @Override - public Stream asStreamOfContent( - BytesHandle endpointStateHandle, BufferableContentHandle outputHandle) { - try { - if (multipart == null) { - return Stream.empty(); - } - - boolean hasEndpointState = (endpointStateHandle != null); - - HandleImplementation handleImpl = (HandleImplementation) outputHandle; - Class receiveClass = handleImpl.receiveAs(); - - int partCount = multipart.getCount(); - Stream.Builder builder = Stream.builder(); - - for (int i=0; i < partCount; i++) { - BodyPart bodyPart = multipart.getBodyPart(i); - if (hasEndpointState && i == 0) { - updateHandle(bodyPart, endpointStateHandle); - } else { - C value = responsePartToContent(outputHandle, bodyPart, receiveClass); - builder.accept(value); - } - } - - return builder.build(); - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } finally { - outputHandle.set(null); - } - } - @Override - public > Stream asStreamOfHandles( - BytesHandle endpointStateHandle, T outputHandle - ) { - try { - if (multipart == null) { - return Stream.empty(); - } - - boolean hasEndpointState = (endpointStateHandle != null); - - Stream.Builder builder = Stream.builder(); - - int partCount = multipart.getCount(); - for (int i=0; i < partCount; i++) { - BodyPart bodyPart = multipart.getBodyPart(i); - if (hasEndpointState && i == 0) { - updateHandle(bodyPart, endpointStateHandle); - } else { - builder.accept(updateHandle(bodyPart, (T) outputHandle.newHandle())); - } - } - return builder.build(); - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } finally { - outputHandle.set(null); - } - } - @Override - public Stream asStreamOfInputStreamHandle() { - try { - if (multipart == null) { - return Stream.empty(); - } - int partCount = multipart.getCount(); - - Stream.Builder builder = Stream.builder(); - for (int i=0; i < partCount; i++) { - BodyPart bodyPart = multipart.getBodyPart(i); - builder.accept(updateHandle(bodyPart, new InputStreamHandle())); - } - return builder.build(); - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } - } - @Override - public Stream asStreamOfInputStream() { - try { - if (multipart == null) { - return Stream.empty(); - } - int partCount = multipart.getCount(); - - Stream.Builder builder = Stream.builder(); - for (int i=0; i < partCount; i++) { - BodyPart bodyPart = multipart.getBodyPart(i); - builder.accept(bodyPart.getInputStream()); - } - return builder.build(); - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } catch (IOException e) { - throw new MarkLogicIOException(e); - } - } - @Override - public Stream asStreamOfReader() { - try { - if (multipart == null) { - return Stream.empty(); - } - int partCount = multipart.getCount(); - - Stream.Builder builder = Stream.builder(); - for (int i=0; i < partCount; i++) { - BodyPart bodyPart = multipart.getBodyPart(i); - builder.accept(NodeConverter.InputStreamToReader(bodyPart.getInputStream())); - } - return builder.build(); - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } catch (IOException e) { - throw new MarkLogicIOException(e); - } - } - @Override - public Stream asStreamOfReaderHandle() { - try { - if (multipart == null) { - return Stream.empty(); - } - int partCount = multipart.getCount(); - - Stream.Builder builder = Stream.builder(); - for (int i=0; i < partCount; i++) { - BodyPart bodyPart = multipart.getBodyPart(i); - builder.accept(updateHandle(bodyPart, new ReaderHandle())); - } - return builder.build(); - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } - } - @Override - public Stream asStreamOfString() { - try { - if (multipart == null) { - return Stream.empty(); - } - int partCount = multipart.getCount(); - - Stream.Builder builder = Stream.builder(); - for (int i=0; i < partCount; i++) { - BodyPart bodyPart = multipart.getBodyPart(i); - builder.accept(NodeConverter.InputStreamToString(bodyPart.getInputStream())); - } - return builder.build(); - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } catch (IOException e) { - throw new MarkLogicIOException(e); - } - } - @Override - public byte[][] asArrayOfBytes() { - try { - if (multipart == null) { - return new byte[0][]; - } - int partCount = multipart.getCount(); - - byte[][] result = new byte[partCount][]; - for (int i=0; i < partCount; i++) { - BodyPart bodyPart = multipart.getBodyPart(i); - result[i] = NodeConverter.InputStreamToBytes(bodyPart.getInputStream()); - } - return result; - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } catch (IOException e) { - throw new MarkLogicIOException(e); - } - } - @Override - public C[] asArrayOfContent( - BytesHandle endpointStateHandle, BufferableContentHandle outputHandle - ) { - try { - if (multipart == null) { - return outputHandle.newArray(0); - } - - boolean hasEndpointState = (endpointStateHandle != null); - - HandleImplementation handleImpl = (HandleImplementation) outputHandle; - Class receiveClass = handleImpl.receiveAs(); - - int partCount = multipart.getCount(); - C[] result = outputHandle.newArray(hasEndpointState ? (partCount - 1) : partCount); - for (int i=0; i < partCount; i++) { - BodyPart bodyPart = multipart.getBodyPart(i); - if (hasEndpointState && i == 0) { - updateHandle(bodyPart, endpointStateHandle); - } else { - C value = responsePartToContent(outputHandle, bodyPart, receiveClass); - result[hasEndpointState ? (i - 1) : i] = value; - } - } - - return result; - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } finally { - outputHandle.set(null); - } - } - @Override - public BufferableContentHandle[] asArrayOfHandles( - BytesHandle endpointStateHandle, BufferableContentHandle outputHandle - ) { - try { - if (multipart == null) { - return outputHandle.newHandleArray(0); - } - - boolean hasEndpointState = (endpointStateHandle != null); - - int partCount = multipart.getCount(); - BufferableContentHandle[] result = outputHandle.newHandleArray(hasEndpointState ? (partCount - 1) : partCount); - for (int i=0; i < partCount; i++) { - BodyPart bodyPart = multipart.getBodyPart(i); - if (hasEndpointState && i == 0) { - updateHandle(bodyPart, endpointStateHandle); - } else { - result[hasEndpointState ? (i - 1) : i] = updateHandle(bodyPart, outputHandle.newHandle()); - } - } - - return result; - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } finally { - outputHandle.set(null); - } - } - private C responsePartToContent(BufferableContentHandle handle, BodyPart bodyPart, Class as) { - return handle.toContent(getEntity(bodyPart, as)); - } - @Override - public InputStream[] asArrayOfInputStream() { - try { - if (multipart == null) { - return new InputStream[0]; - } - int partCount = multipart.getCount(); - - InputStream[] result = new InputStream[partCount]; - for (int i=0; i < partCount; i++) { - BodyPart bodyPart = multipart.getBodyPart(i); - result[i] = bodyPart.getInputStream(); - } - return result; - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } catch (IOException e) { - throw new MarkLogicIOException(e); - } - } - @Override - public InputStreamHandle[] asArrayOfInputStreamHandle() { - try { - if (multipart == null) { - return new InputStreamHandle[0]; - } - int partCount = multipart.getCount(); - - InputStreamHandle[] result = new InputStreamHandle[partCount]; - for (int i=0; i < partCount; i++) { - BodyPart bodyPart = multipart.getBodyPart(i); - result[i] = updateHandle(bodyPart, new InputStreamHandle()); - } - return result; - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } - } - @Override - public ReaderHandle[] asArrayOfReaderHandle() { - try { - if (multipart == null) { - return new ReaderHandle[0]; - } - int partCount = multipart.getCount(); - - ReaderHandle[] result = new ReaderHandle[partCount]; - for (int i=0; i < partCount; i++) { - BodyPart bodyPart = multipart.getBodyPart(i); - result[i] = updateHandle(bodyPart, new ReaderHandle()); - } - return result; - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } - } - @Override - public Reader[] asArrayOfReader() { - try { - if (multipart == null) { - return new Reader[0]; - } - int partCount = multipart.getCount(); - - Reader[] result = new Reader[partCount]; - for (int i=0; i < partCount; i++) { - BodyPart bodyPart = multipart.getBodyPart(i); - result[i] = NodeConverter.InputStreamToReader(bodyPart.getInputStream()); - } - return result; - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } catch (IOException e) { - throw new MarkLogicIOException(e); - } - } - @Override - public String[] asArrayOfString() { - try { - if (multipart == null) { - return new String[0]; - } - int partCount = multipart.getCount(); - - String[] result = new String[partCount]; - for (int i=0; i < partCount; i++) { - BodyPart bodyPart = multipart.getBodyPart(i); - result[i] = NodeConverter.InputStreamToString(bodyPart.getInputStream()); - } - return result; - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } catch (IOException e) { - throw new MarkLogicIOException(e); - } - } - } - - static protected boolean checkNull(ResponseBody body, Format expectedFormat) { - if (body != null) { - if (body.contentLength() == 0) { - body.close(); - } else { - MediaType actualType = body.contentType(); - if (actualType == null) { - body.close(); - throw new RuntimeException( - "Returned document with unknown mime type instead of "+expectedFormat.getDefaultMimetype() - ); - } - if (expectedFormat != Format.UNKNOWN) { - Format actualFormat = Format.getFromMimetype(actualType.toString()); - if (expectedFormat != actualFormat) { - body.close(); - throw new RuntimeException( - "Mime type "+actualType.toString()+" for returned document not recognized for "+expectedFormat.name() - ); - } - } - return false; - } - } - return true; - } - static protected boolean checkNull(MimeMultipart multipart, Format expectedFormat) { - if (multipart != null) { - try { - if (multipart.getCount() != 0) { - BodyPart firstPart = multipart.getBodyPart(0); - String actualType = (firstPart == null) ? null : firstPart.getContentType(); - if (actualType == null) { - throw new RuntimeException( - "Returned document with unknown mime type instead of "+expectedFormat.getDefaultMimetype() - ); - } - if (expectedFormat != Format.UNKNOWN) { - Format actualFormat = Format.getFromMimetype(actualType); - if (expectedFormat != actualFormat) { - throw new RuntimeException( - "Mime type "+actualType+" for returned document not recognized for "+expectedFormat.name() - ); - } - } - return false; - } - } catch (MessagingException e) { - throw new MarkLogicIOException(e); - } - } - return true; - } - - static private void closeResponse(Response response) { - if (response == null || response.body() == null) return; - response.close(); - } - - Request.Builder forDocumentResponse(Request.Builder requestBldr, Format format) { - return requestBldr.addHeader( - HEADER_ACCEPT, - (format == null || format == Format.BINARY || format == Format.UNKNOWN) ? - "application/x-unknown-content-type" : format.getDefaultMimetype()); - } - - Request.Builder forMultipartMixedResponse(Request.Builder requestBldr) { - return requestBldr.addHeader(HEADER_ACCEPT, multipartMixedWithBoundary()); - } - - static protected class Condition { - private boolean is = false; - protected boolean get() { - return is; - } - protected void set() { - if (!is) - is = true; - } - } - - static class ConnectionResultImpl implements ConnectionResult { - private boolean connected = false; - private int statusCode; - private String errorMessage; + private boolean getDocumentImpl(RequestLogger reqlog, + DocumentDescriptor desc, Transaction transaction, + Set categories, RequestParameters extraParams, + String metadataFormat, DocumentMetadataReadHandle metadataHandle, + AbstractReadHandle contentHandle) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + String uri = desc.getUri(); + if (uri == null) { + throw new IllegalArgumentException( + "Document read for document identifier without uri"); + } - @Override - public boolean isConnected() { - return connected; - } + assert metadataHandle != null : "metadataHandle is null"; + assert contentHandle != null : "contentHandle is null"; + + logger.debug("Getting multipart for {} in transaction {}", uri, getTransactionId(transaction)); + + addPointInTimeQueryParam(extraParams, contentHandle); + + RequestParameters docParams = makeDocumentParams(uri, categories, transaction, extraParams, true); + docParams.add("format", metadataFormat); + + Request.Builder requestBldr = makeDocumentResource(docParams); + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + requestBldr = addVersionHeader(desc, requestBldr, "If-None-Match"); + + Function doGetFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return sendRequestOnce(funcBuilder.addHeader(HEADER_ACCEPT, multipartMixedWithBoundary()).get()); + } + }; + Response response = sendRequestWithRetry(requestBldr, (transaction == null), doGetFunction, null); + int status = response.code(); + if (status == STATUS_NOT_FOUND) { + throw new ResourceNotFoundException( + "Could not read non-existent document", + extractErrorFields(response)); + } + if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException( + "User is not allowed to read documents", + extractErrorFields(response)); + } + if (status == STATUS_NOT_MODIFIED) { + closeResponse(response); + return false; + } + if (status != STATUS_OK) { + throw new FailedRequestException("read failed: " + + getReasonPhrase(response), extractErrorFields(response)); + } + + logRequest( + reqlog, + "read %s document from %s transaction with %s metadata categories and content", + uri, (transaction != null) ? transaction.getTransactionId() : "no", stringJoin(categories, ", ", "no")); + + try { + ResponseBody body = response.body(); + MimeMultipart entity = body.contentLength() != 0 ? + getEntity(body, MimeMultipart.class) : null; + if (entity == null) return false; + + int partCount = entity.getCount(); + if (partCount == 0) return false; + List partList = getPartList(entity); + + if (partCount != 2) { + throw new FailedRequestException("read expected 2 parts but got " + partCount + " parts", + extractErrorFields(response)); + } + + HandleImplementation metadataBase = HandleAccessor.as(metadataHandle); + HandleImplementation contentBase = HandleAccessor.as(contentHandle); + + BodyPart contentPart = partList.get(1); + + Headers responseHeaders = response.headers(); + if (isExternalDescriptor(desc)) { + updateVersion(desc, responseHeaders); + updateFormat(desc, responseHeaders); + updateMimetype(desc, getHeaderMimetype(getHeader(contentPart, HEADER_CONTENT_TYPE))); + updateLength(desc, getHeaderLength(getHeader(contentPart, HEADER_CONTENT_LENGTH))); + copyDescriptor(desc, contentBase); + } else { + updateDescriptor(contentBase, responseHeaders); + } + + metadataBase.receiveContent(getEntity(partList.get(0), + metadataBase.receiveAs())); + + Object contentEntity = getEntity(contentPart, contentBase.receiveAs()); + contentBase.receiveContent((reqlog != null) ? reqlog.copyContent(contentEntity) : contentEntity); - private void setConnected(boolean connected) { - this.connected = connected; + closeResponse(response); + + return true; + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } } @Override - public Integer getStatusCode() { - return statusCode; + public DocumentDescriptor head(RequestLogger reqlog, String uri, + Transaction transaction) + throws ForbiddenUserException, FailedRequestException { + Response response = headImpl(reqlog, uri, transaction, makeDocumentResource(makeDocumentParams(uri, + null, transaction, null))); + + // 404 + if (response == null) return null; + + Headers responseHeaders = response.headers(); + + closeResponse(response); + logRequest(reqlog, "checked %s document from %s transaction", uri, + (transaction != null) ? transaction.getTransactionId() : "no"); + + DocumentDescriptorImpl desc = new DocumentDescriptorImpl(uri, false); + + updateVersion(desc, responseHeaders); + updateDescriptor(desc, responseHeaders); + + return desc; } - private void setStatusCode(int statusCode) { - this.statusCode = statusCode; + @Override + public boolean exists(String uri) throws ForbiddenUserException, FailedRequestException { + return headImpl(null, uri, null, setupRequest(uri, null)) == null ? false : true; } @Override - public String getErrorMessage() { - return errorMessage; + public ConnectionResult checkConnection() { + Request.Builder request = new Request.Builder() + .url(this.baseUri); + Response response = headImplExec(null, this.baseUri.uri().toString(), null, request); + ConnectionResultImpl connectionResultImpl = new ConnectionResultImpl(); + int statusCode = response.code(); + if (statusCode < 300) { + connectionResultImpl.setConnected(true); + } else { + connectionResultImpl.setConnected(false); + connectionResultImpl.setStatusCode(statusCode); + connectionResultImpl.setErrorMessage(getReasonPhrase(response)); + } + return connectionResultImpl; } - private void setErrorMessage(String errorMessage) { - this.errorMessage = errorMessage; + private Response headImpl(RequestLogger reqlog, String uri, + Transaction transaction, Request.Builder requestBldr) { + Response response = headImplExec(reqlog, uri, transaction, requestBldr); + int status = response.code(); + if (status != STATUS_OK) { + if (status == STATUS_NOT_FOUND) { + closeResponse(response); + return null; + } else if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException( + "User is not allowed to check the existence of documents", + extractErrorFields(response)); + } else { + throw new FailedRequestException( + "Document existence check failed: " + + getReasonPhrase(response), + extractErrorFields(response)); + } + } + return response; } - } - @FunctionalInterface - private interface ResultIteratorConstructor { - T construct(RequestLogger logger, List list, Closeable closeable); - } + private Response headImplExec(RequestLogger reqlog, String uri, + Transaction transaction, Request.Builder requestBldr) { + if (uri == null) { + throw new IllegalArgumentException( + "Existence check for document identifier without uri"); + } + + logger.debug("Requesting head for {} in transaction {}", uri, getTransactionId(transaction)); + + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + + Function doHeadFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return sendRequestOnce(funcBuilder.head().build()); + } + }; + return sendRequestWithRetry(requestBldr, (transaction == null), doHeadFunction, null); + } + + @Override + public TemporalDescriptor putDocument(RequestLogger reqlog, DocumentDescriptor desc, + Transaction transaction, Set categories, + RequestParameters extraParams, + DocumentMetadataWriteHandle metadataHandle, + AbstractWriteHandle contentHandle) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + if (desc.getUri() == null) { + throw new IllegalArgumentException( + "Document write for document identifier without uri"); + } + + HandleImplementation metadataBase = HandleAccessor.checkHandle( + metadataHandle, "metadata"); + HandleImplementation contentBase = HandleAccessor.checkHandle( + contentHandle, "content"); + + String metadataMimetype = null; + if (metadataBase != null) { + metadataMimetype = metadataBase.getMimetype(); + } + + Format descFormat = desc.getFormat(); + String contentMimetype = (descFormat != null && descFormat != Format.UNKNOWN) ? desc.getMimetype() : null; + if (contentMimetype == null && contentBase != null) { + Format contentFormat = contentBase.getFormat(); + if (descFormat != null && descFormat != contentFormat) { + contentMimetype = descFormat.getDefaultMimetype(); + } else if (contentFormat != null && contentFormat != Format.UNKNOWN) { + contentMimetype = contentBase.getMimetype(); + } + } + + if (metadataBase != null && contentBase != null) { + return putPostDocumentImpl(reqlog, "put", desc, transaction, categories, + extraParams, metadataMimetype, metadataHandle, + contentMimetype, contentHandle); + } else if (metadataBase != null) { + return putPostDocumentImpl(reqlog, "put", desc, transaction, categories, false, + extraParams, metadataMimetype, metadataHandle); + } else if (contentBase != null) { + return putPostDocumentImpl(reqlog, "put", desc, transaction, null, true, + extraParams, contentMimetype, contentHandle); + } + throw new IllegalArgumentException("Either metadataHandle or contentHandle must not be null"); + } + + @Override + public DocumentDescriptorImpl postDocument(RequestLogger reqlog, DocumentUriTemplate template, + Transaction transaction, Set categories, RequestParameters extraParams, + DocumentMetadataWriteHandle metadataHandle, AbstractWriteHandle contentHandle) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + DocumentDescriptorImpl desc = new DocumentDescriptorImpl(false); + + HandleImplementation metadataBase = HandleAccessor.checkHandle( + metadataHandle, "metadata"); + HandleImplementation contentBase = HandleAccessor.checkHandle( + contentHandle, "content"); + + String metadataMimetype = null; + if (metadataBase != null) { + metadataMimetype = metadataBase.getMimetype(); + } + + Format templateFormat = template.getFormat(); + String contentMimetype = (templateFormat != null && templateFormat != Format.UNKNOWN) ? + template.getMimetype() : null; + if (contentMimetype == null && contentBase != null) { + Format contentFormat = contentBase.getFormat(); + if (templateFormat != null && templateFormat != contentFormat) { + contentMimetype = templateFormat.getDefaultMimetype(); + desc.setFormat(templateFormat); + } else if (contentFormat != null && contentFormat != Format.UNKNOWN) { + contentMimetype = contentBase.getMimetype(); + desc.setFormat(contentFormat); + } + } + desc.setMimetype(contentMimetype); + + if (extraParams == null) extraParams = new RequestParameters(); + + String extension = template.getExtension(); + if (extension != null) extraParams.add("extension", extension); + + String directory = template.getDirectory(); + if (directory != null) extraParams.add("directory", directory); + + if (metadataBase != null && contentBase != null) { + putPostDocumentImpl(reqlog, "post", desc, transaction, categories, extraParams, + metadataMimetype, metadataHandle, contentMimetype, contentHandle); + } else if (contentBase != null) { + putPostDocumentImpl(reqlog, "post", desc, transaction, null, true, extraParams, + contentMimetype, contentHandle); + } + + return desc; + } + + private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String method, DocumentDescriptor desc, + Transaction transaction, Set categories, boolean isOnContent, RequestParameters extraParams, + String mimetype, AbstractWriteHandle handle) + throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + String uri = desc.getUri(); + + HandleImplementation handleBase = HandleAccessor.as(handle); + + logger.debug("Sending {} document in transaction {}", + (uri != null) ? uri : "new", getTransactionId(transaction)); + + logRequest( + reqlog, + "writing %s document from %s transaction with %s mime type and %s metadata categories", + (uri != null) ? uri : "new", + (transaction != null) ? transaction.getTransactionId() : "no", + (mimetype != null) ? mimetype : "no", + stringJoin(categories, ", ", "no")); + + Request.Builder requestBldr = makeDocumentResource( + makeDocumentParams( + uri, categories, transaction, extraParams, isOnContent + )); + + requestBldr = requestBldr.header(HEADER_CONTENT_TYPE, + (mimetype != null) ? mimetype : MIMETYPE_WILDCARD); + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + if (uri != null) { + requestBldr = addVersionHeader(desc, requestBldr, "If-Match"); + } + + if ("patch".equals(method)) { + requestBldr = requestBldr.header("X-HTTP-Method-Override", "PATCH"); + method = "post"; + } + boolean isResendable = handleBase.isResendable(); + + Response response = null; + int status = -1; + Headers responseHeaders = null; + long startTime = System.currentTimeMillis(); + int nextDelay = 0; + int retry = 0; + for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + if (nextDelay > 0) { + try { + Thread.sleep(nextDelay); + } catch (InterruptedException e) { + } + } + + Object value = handleBase.sendContent(); + if (value == null) { + throw new IllegalArgumentException( + "Document write with null value for " + ((uri != null) ? uri : "new document")); + } + + if (isFirstRequest() && !isResendable && isStreaming(value)) { + nextDelay = makeFirstRequest(retry); + if (nextDelay != 0) continue; + } + + MediaType mediaType = makeType(requestBldr.build().header(HEADER_CONTENT_TYPE)); + if (value instanceof OutputStreamSender) { + StreamingOutputImpl sentStream = + new StreamingOutputImpl((OutputStreamSender) value, reqlog, mediaType); + requestBldr = + ("put".equals(method)) ? + requestBldr.put(sentStream) : + requestBldr.post(sentStream); + } else { + Object sentObj = (reqlog != null) ? + reqlog.copyContent(value) : value; + requestBldr = + ("put".equals(method)) ? + requestBldr.put(new ObjectRequestBody(sentObj, mediaType)) : + requestBldr.post(new ObjectRequestBody(sentObj, mediaType)); + } + response = sendRequestOnce(requestBldr); + + status = response.code(); + + responseHeaders = response.headers(); + if (transaction != null || !retryStatus.contains(status)) { + if (isFirstRequest()) setFirstRequest(false); + + break; + } + + String retryAfterRaw = response.header("Retry-After"); + closeResponse(response); + + if (!isResendable) { + checkFirstRequest(); + throw new ResourceNotResendableException( + "Cannot retry request for " + + ((uri != null) ? uri : "new document")); + } + + int retryAfter = Utilities.parseInt(retryAfterRaw); + nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + } + if (retryStatus.contains(status)) { + checkFirstRequest(); + closeResponse(response); + throw new FailedRetryException( + "Service unavailable and maximum retry period elapsed: " + + ((System.currentTimeMillis() - startTime) / 1000) + + " seconds after " + retry + " retries"); + } + if (status == STATUS_NOT_FOUND) { + throw new ResourceNotFoundException( + "Could not write non-existent document", + extractErrorFields(response)); + } + if (status == STATUS_PRECONDITION_REQUIRED) { + FailedRequest failure = extractErrorFields(response); + if (failure.getMessageCode().equals("RESTAPI-CONTENTNOVERSION")) { + throw new ContentNoVersionException("Content version required to write document", failure); + } + throw new FailedRequestException( + "Precondition required to write document", failure); + } else if (status == STATUS_FORBIDDEN) { + FailedRequest failure = extractErrorFields(response); + throw new ForbiddenUserException( + "User is not allowed to write documents", failure); + } + if (status == STATUS_PRECONDITION_FAILED) { + FailedRequest failure = extractErrorFields(response); + if (failure.getMessageCode().equals("RESTAPI-CONTENTWRONGVERSION")) { + throw new ContentWrongVersionException("Content version must match to write document", failure); + } else if (failure.getMessageCode().equals("RESTAPI-EMPTYBODY")) { + throw new FailedRequestException( + "Empty request body sent to server", failure); + } + throw new FailedRequestException("Precondition Failed", failure); + } + if (status == -1) { + throw new FailedRequestException("write failed: Unknown Reason", extractErrorFields(response)); + } + if (status != STATUS_CREATED && status != STATUS_NO_CONTENT) { + throw new FailedRequestException("write failed: " + + getReasonPhrase(response), extractErrorFields(response)); + } + + if (uri == null) { + String location = response.header("Location"); + if (location != null) { + int offset = location.indexOf(DOCUMENT_URI_PREFIX); + if (offset == -1) { + closeResponse(response); + throw new MarkLogicInternalException( + "document create produced invalid location: " + location); + } + uri = location.substring(offset + DOCUMENT_URI_PREFIX.length()); + if (uri == null) { + closeResponse(response); + throw new MarkLogicInternalException( + "document create produced location without uri: " + location); + } + desc.setUri(uri); + updateVersion(desc, responseHeaders); + updateDescriptor(desc, responseHeaders); + } + } + TemporalDescriptor temporalDesc = updateTemporalSystemTime(desc, responseHeaders); + closeResponse(response); + return temporalDesc; + } + + private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String method, DocumentDescriptor desc, + Transaction transaction, Set categories, RequestParameters extraParams, + String metadataMimetype, DocumentMetadataWriteHandle metadataHandle, String contentMimetype, + AbstractWriteHandle contentHandle) + throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + String uri = desc.getUri(); + + logger.debug("Sending {} multipart document in transaction {}", + (uri != null) ? uri : "new", getTransactionId(transaction)); + + logRequest( + reqlog, + "writing %s document from %s transaction with %s metadata categories and content", + (uri != null) ? uri : "new", + (transaction != null) ? transaction.getTransactionId() : "no", + stringJoin(categories, ", ", "no")); + + RequestParameters docParams = + makeDocumentParams(uri, categories, transaction, extraParams, true); + + Request.Builder requestBldr = makeDocumentResource(docParams) + .addHeader(HEADER_ACCEPT, MIMETYPE_MULTIPART_MIXED); + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + if (uri != null) { + requestBldr = addVersionHeader(desc, requestBldr, "If-Match"); + } + + Response response = null; + int status = -1; + Headers responseHeaders = null; + long startTime = System.currentTimeMillis(); + int nextDelay = 0; + int retry = 0; + for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + if (nextDelay > 0) { + try { + Thread.sleep(nextDelay); + } catch (InterruptedException e) { + } + } + + MultipartBody.Builder multiPart = new MultipartBody.Builder(); + boolean hasStreamingPart = addParts(multiPart, reqlog, + new String[]{metadataMimetype, contentMimetype}, + new AbstractWriteHandle[]{metadataHandle, contentHandle}); + + if (isFirstRequest() && hasStreamingPart) { + nextDelay = makeFirstRequest(retry); + if (nextDelay != 0) continue; + } + + requestBldr = ("put".equals(method)) ? requestBldr.put(multiPart.build()) : requestBldr.post(multiPart.build()); + response = sendRequestOnce(requestBldr); + status = response.code(); + + responseHeaders = response.headers(); + if (transaction != null || !retryStatus.contains(status)) { + if (isFirstRequest()) setFirstRequest(false); + + break; + } + String retryAfterRaw = response.header("Retry-After"); + closeResponse(response); + + if (hasStreamingPart) { + throw new ResourceNotResendableException( + "Cannot retry request for " + + ((uri != null) ? uri : "new document")); + } + + int retryAfter = Utilities.parseInt(retryAfterRaw); + nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + } + if (retryStatus.contains(status)) { + checkFirstRequest(); + closeResponse(response); + throw new FailedRetryException( + "Service unavailable and maximum retry period elapsed: " + + ((System.currentTimeMillis() - startTime) / 1000) + + " seconds after " + retry + " retries"); + } + if (status == STATUS_NOT_FOUND) { + closeResponse(response); + throw new ResourceNotFoundException( + "Could not write non-existent document"); + } + if (status == STATUS_PRECONDITION_REQUIRED) { + FailedRequest failure = extractErrorFields(response); + if (failure.getMessageCode().equals("RESTAPI-CONTENTNOVERSION")) { + throw new ContentNoVersionException("Content version required to write document", failure); + } + throw new FailedRequestException( + "Precondition required to write document", failure); + } else if (status == STATUS_FORBIDDEN) { + FailedRequest failure = extractErrorFields(response); + throw new ForbiddenUserException( + "User is not allowed to write documents", failure); + } + if (status == STATUS_PRECONDITION_FAILED) { + FailedRequest failure = extractErrorFields(response); + if (failure.getMessageCode().equals("RESTAPI-CONTENTWRONGVERSION")) { + throw new ContentWrongVersionException("Content version must match to write document", failure); + } else if (failure.getMessageCode().equals("RESTAPI-EMPTYBODY")) { + throw new FailedRequestException( + "Empty request body sent to server", failure); + } + throw new FailedRequestException("Precondition Failed", failure); + } + if (status != STATUS_CREATED && status != STATUS_NO_CONTENT) { + throw new FailedRequestException("write failed: " + + getReasonPhrase(response), extractErrorFields(response)); + } + + if (uri == null) { + String location = response.header("Location"); + if (location != null) { + int offset = location.indexOf(DOCUMENT_URI_PREFIX); + if (offset == -1) { + closeResponse(response); + throw new MarkLogicInternalException( + "document create produced invalid location: " + location); + } + uri = location.substring(offset + DOCUMENT_URI_PREFIX.length()); + if (uri == null) { + closeResponse(response); + throw new MarkLogicInternalException( + "document create produced location without uri: " + location); + } + desc.setUri(uri); + updateVersion(desc, responseHeaders); + updateDescriptor(desc, responseHeaders); + } + } + TemporalDescriptor temporalDesc = updateTemporalSystemTime(desc, responseHeaders); + closeResponse(response); + return temporalDesc; + } + + @Override + public void patchDocument(RequestLogger reqlog, DocumentDescriptor desc, Transaction transaction, + Set categories, boolean isOnContent, DocumentPatchHandle patchHandle) + throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + patchDocument(reqlog, desc, transaction, categories, isOnContent, null, null, patchHandle); + } + + @Override + public void patchDocument(RequestLogger reqlog, DocumentDescriptor desc, Transaction transaction, + Set categories, boolean isOnContent, RequestParameters extraParams, String sourceDocumentURI, + DocumentPatchHandle patchHandle) + throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + HandleImplementation patchBase = HandleAccessor.checkHandle(patchHandle, "patch"); + + if (sourceDocumentURI != null) + extraParams.add("source-document", sourceDocumentURI); + putPostDocumentImpl(reqlog, "patch", desc, transaction, categories, isOnContent, extraParams, patchBase.getMimetype(), + patchHandle); + } + + @Override + public Transaction openTransaction(String name, int timeLimit) throws ForbiddenUserException, FailedRequestException { + logger.debug("Opening transaction"); + + RequestParameters transParams = new RequestParameters(); + if (name != null || timeLimit > 0) { + if (name != null) transParams.add("name", name); + if (timeLimit > 0) transParams.add("timeLimit", String.valueOf(timeLimit)); + } + + Request.Builder requestBldr = setupRequest("transactions", transParams); + requestBldr = addTelemetryAgentId(requestBldr); + + Function doPostFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return sendRequestOnce(funcBuilder.post(RequestBody.create("", null))); + } + }; + Response response = sendRequestWithRetry(requestBldr, doPostFunction, null); + int status = response.code(); + if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException("User is not allowed to open transactions", extractErrorFields(response)); + } + if (status != STATUS_SEE_OTHER) { + throw new FailedRequestException("transaction open failed: " + + getReasonPhrase(response), extractErrorFields(response)); + } + + String location = response.headers().get("Location"); + List cookies = new ArrayList<>(); + for (String setCookie : response.headers(HEADER_SET_COOKIE)) { + ClientCookie cookie = ClientCookie.parse(requestBldr.build().url(), setCookie); + cookies.add(cookie); + } + closeResponse(response); + if (location == null) throw new MarkLogicInternalException("transaction open failed to provide location"); + if (!location.contains("/")) { + throw new MarkLogicInternalException("transaction open produced invalid location: " + location); + } + + String transactionId = location.substring(location.lastIndexOf("/") + 1); + return new TransactionImpl(this, transactionId, cookies); + } + + @Override + public void commitTransaction(Transaction transaction) throws ForbiddenUserException, FailedRequestException { + completeTransaction(transaction, "commit"); + } + + @Override + public void rollbackTransaction(Transaction transaction) throws ForbiddenUserException, FailedRequestException { + completeTransaction(transaction, "rollback"); + } + + private void completeTransaction(Transaction transaction, String result) + throws ForbiddenUserException, FailedRequestException { + if (result == null) { + throw new MarkLogicInternalException( + "transaction completion without operation"); + } + if (transaction == null) { + throw new MarkLogicInternalException( + "transaction completion without id: " + result); + } + + logger.debug("Completing transaction {} with {}", transaction.getTransactionId(), result); + + RequestParameters transParams = new RequestParameters(); + transParams.add("result", result); + + Request.Builder requestBldr = setupRequest("transactions/" + transaction.getTransactionId(), transParams); + + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + + Function doPostFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return sendRequestOnce(funcBuilder.post(RequestBody.create("", null)).build()); + } + }; + Response response = sendRequestWithRetry(requestBldr, false, doPostFunction, null); + int status = response.code(); + + if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException( + "User is not allowed to complete transaction with " + + result, extractErrorFields(response)); + } + if (status != STATUS_NO_CONTENT) { + throw new FailedRequestException("transaction " + result + + " failed: " + getReasonPhrase(response), + extractErrorFields(response)); + } + + closeResponse(response); + } + + private void addCategoryParams(Set categories, RequestParameters params, boolean withContent) { + if (withContent) + params.add("category", "content"); + if (categories != null && categories.size() > 0) { + if (categories.contains(Metadata.ALL)) { + addCategoryParam(params, "metadata"); + } else { + for (Metadata category : categories) { + addCategoryParam(params, category); + } + } + } + } + + private void addCategoryParam(RequestParameters params, Metadata category) { + addCategoryParam(params, category.toString().toLowerCase()); + } + + private void addCategoryParam(RequestParameters params, String category) { + params.add("category", category); + } + + private RequestParameters makeDocumentParams(String uri, + Set categories, Transaction transaction, + RequestParameters extraParams) { + return makeDocumentParams(uri, categories, transaction, extraParams, + false); + } + + private RequestParameters makeDocumentParams(String uri, Set categories, Transaction transaction, + RequestParameters extraParams, boolean withContent) { + RequestParameters docParams = new RequestParameters(); + if (extraParams != null && extraParams.size() > 0) { + for (Map.Entry> entry : extraParams.entrySet()) { + for (String value : entry.getValue()) { + String extraKey = entry.getKey(); + if (!"range".equalsIgnoreCase(extraKey)) { + docParams.add(extraKey, value); + } + } + } + } + if (uri != null) docParams.add("uri", uri); + if (categories == null || categories.size() == 0) { + docParams.add("category", "content"); + } else { + if (withContent) { + docParams.add("category", "content"); + } + if (categories.contains(Metadata.ALL)) { + docParams.add("category", "metadata"); + } else { + for (Metadata category : categories) { + docParams.add("category", category.toString().toLowerCase()); + } + } + } + if (transaction != null) { + docParams.add("txid", transaction.getTransactionId()); + } + return docParams; + } + + private Request.Builder makeDocumentResource(RequestParameters queryParams) { + return setupRequest("documents", queryParams); + } + + static private boolean isExternalDescriptor(ContentDescriptor desc) { + return desc != null && desc instanceof DocumentDescriptorImpl + && !((DocumentDescriptorImpl) desc).isInternal(); + } + + static private void updateDescriptor(ContentDescriptor desc, + Headers headers) { + if (desc == null || headers == null) return; + + updateFormat(desc, headers); + updateMimetype(desc, headers); + updateLength(desc, headers); + updateServerTimestamp(desc, headers); + } + + static private TemporalDescriptor updateTemporalSystemTime(DocumentDescriptor desc, + Headers headers) { + if (headers == null) return null; + + DocumentDescriptorImpl temporalDescriptor; + if (desc instanceof DocumentDescriptorImpl) { + temporalDescriptor = (DocumentDescriptorImpl) desc; + } else { + temporalDescriptor = new DocumentDescriptorImpl(desc.getUri(), false); + } + temporalDescriptor.setTemporalSystemTime(headers.get(HEADER_X_MARKLOGIC_SYSTEM_TIME)); + return temporalDescriptor; + } + + static private void copyDescriptor(DocumentDescriptor desc, + HandleImplementation handleBase) { + if (handleBase == null) return; + + if (desc.getFormat() != null) handleBase.setFormat(desc.getFormat()); + if (desc.getMimetype() != null) handleBase.setMimetype(desc.getMimetype()); + handleBase.setByteLength(desc.getByteLength()); + } + + static private void updateFormat(ContentDescriptor descriptor, + Headers headers) { + updateFormat(descriptor, getHeaderFormat(headers)); + } + + static private void updateFormat(ContentDescriptor descriptor, Format format) { + if (format != null) { + descriptor.setFormat(format); + } + } + + static private Format getHeaderFormat(Headers headers) { + String format = headers.get(HEADER_VND_MARKLOGIC_DOCUMENT_FORMAT); + if (format != null && format.length() > 0) { + return Format.valueOf(format.toUpperCase()); + } + String contentType = headers.get(HEADER_CONTENT_TYPE); + if (contentType != null && contentType.length() > 0) { + return Format.getFromMimetype(contentType); + } + return null; + } + + static private Format getHeaderFormat(BodyPart part) { + String contentDisposition = getHeader(part, HEADER_CONTENT_DISPOSITION); + String formatRegex = ".* format=(text|binary|xml|json).*"; + String format = getHeader(part, HEADER_VND_MARKLOGIC_DOCUMENT_FORMAT); + String contentType = getHeader(part, HEADER_CONTENT_TYPE); + if (format != null && format.length() > 0) { + return Format.valueOf(format.toUpperCase()); + } else if (contentDisposition != null && contentDisposition.matches(formatRegex)) { + format = contentDisposition.replaceFirst("^.*" + formatRegex + ".*$", "$1"); + return Format.valueOf(format.toUpperCase()); + } else if (contentType != null && contentType.length() > 0) { + return Format.getFromMimetype(contentType); + } + return null; + } + + static private void updateMimetype(ContentDescriptor descriptor, + Headers headers) { + updateMimetype(descriptor, getHeaderMimetype(headers.get(HEADER_CONTENT_TYPE))); + } + + static private void updateMimetype(ContentDescriptor descriptor, String mimetype) { + if (mimetype != null) { + descriptor.setMimetype(mimetype); + } + } + + static private String getHeader(Map> headers, String name) { + List values = headers.get(name); + if (values != null && values.size() > 0) { + return values.get(0); + } + return null; + } + + static private String getHeader(BodyPart part, String name) { + if (part == null) throw new MarkLogicInternalException("part must not be null"); + try { + String[] values = part.getHeader(name); + if (values != null && values.length > 0) { + return values[0]; + } + return null; + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } + } + + static private String getHeaderMimetype(String contentType) { + if (contentType != null) { + int offset = contentType.indexOf(";"); + String mimetype = (offset == -1) ? contentType : contentType.substring(0, offset); + // TODO: if "; charset=foo" set character set + if (mimetype != null && mimetype.length() > 0) { + return mimetype; + } + } + return null; + } + + static private void updateLength(ContentDescriptor descriptor, + Headers headers) { + updateLength(descriptor, getHeaderLength(headers.get(HEADER_CONTENT_LENGTH))); + } + + static private void updateLength(ContentDescriptor descriptor, long length) { + descriptor.setByteLength(length); + } + + static private void updateServerTimestamp(ContentDescriptor descriptor, + Headers headers) { + updateServerTimestamp(descriptor, getHeaderServerTimestamp(headers)); + } + + static private long getHeaderServerTimestamp(Headers headers) { + return Utilities.parseLong(headers.get(HEADER_ML_EFFECTIVE_TIMESTAMP)); + } + + static private void updateServerTimestamp(ContentDescriptor descriptor, long timestamp) { + if (descriptor instanceof HandleImplementation) { + if (descriptor != null && timestamp != -1) { + ((HandleImplementation) descriptor).setResponseServerTimestamp(timestamp); + } + } + } + + static private long getHeaderLength(String length) { + return Utilities.parseLong(length, ContentDescriptor.UNKNOWN_LENGTH); + } + + static private String getHeaderUri(BodyPart part) { + try { + if (part != null) { + return part.getFileName(); + } + // if it's not found, just return null + return null; + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } + } + + static private void updateVersion(DocumentDescriptor descriptor, Headers headers) { + updateVersion(descriptor, extractVersion(headers.get(HEADER_ETAG))); + } + + static private void updateVersion(DocumentDescriptor descriptor, String header) { + updateVersion(descriptor, extractVersion(header)); + } + + static private void updateVersion(DocumentDescriptor descriptor, long version) { + descriptor.setVersion(version); + } + + static private long extractVersion(String header) { + if (header != null && header.length() > 0) { + // trim the double quotes + return Long.parseLong(header.substring(1, header.length() - 1)); + } + return DocumentDescriptor.UNKNOWN_VERSION; + } + + static private Request.Builder addVersionHeader(DocumentDescriptor desc, Request.Builder requestBldr, String name) { + if (desc != null && + desc instanceof DocumentDescriptorImpl && + !((DocumentDescriptorImpl) desc).isInternal()) { + long version = desc.getVersion(); + if (version != DocumentDescriptor.UNKNOWN_VERSION) { + return requestBldr.header(name, "\"" + String.valueOf(version) + "\""); + } + } + return requestBldr; + } + + static private R updateHandle(BodyPart part, R handle) { + HandleImplementation handleBase = HandleAccessor.as(handle); + + updateFormat(handleBase, getHeaderFormat(part)); + updateMimetype(handleBase, getHeaderMimetype(OkHttpServices.getHeader(part, HEADER_CONTENT_TYPE))); + updateLength(handleBase, getHeaderLength(OkHttpServices.getHeader(part, HEADER_CONTENT_LENGTH))); + handleBase.receiveContent(getEntity(part, handleBase.receiveAs())); + + return handle; + } + + static private R updateHandle(Headers headers, ResponseBody body, R handle) { + HandleImplementation handleBase = HandleAccessor.as(handle); + + updateFormat(handleBase, getHeaderFormat(headers)); + updateMimetype(handleBase, getHeaderMimetype(headers.get(HEADER_CONTENT_TYPE))); + updateLength(handleBase, getHeaderLength(headers.get(HEADER_CONTENT_LENGTH))); + handleBase.receiveContent(getEntity(body, handleBase.receiveAs())); + + return handle; + } + + @Override + public T search(RequestLogger reqlog, T searchHandle, + SearchQueryDefinition queryDef, long start, long len, QueryView view, + Transaction transaction, String forestName) + throws ForbiddenUserException, FailedRequestException { + RequestParameters params = new RequestParameters(); + + if (start > 1) { + params.add("start", Long.toString(start)); + } + + if (len > -1) { + params.add("pageLength", Long.toString(len)); + } + + if (view != null && view != QueryView.DEFAULT) { + if (view == QueryView.ALL) { + params.add("view", "all"); + } else if (view == QueryView.RESULTS) { + params.add("view", "results"); + } else if (view == QueryView.FACETS) { + params.add("view", "facets"); + } else if (view == QueryView.METADATA) { + params.add("view", "metadata"); + } + } + + addPointInTimeQueryParam(params, searchHandle); + + @SuppressWarnings("rawtypes") + HandleImplementation searchBase = HandleAccessor.checkHandle(searchHandle, "search"); + + Format searchFormat = searchBase.getFormat(); + switch (searchFormat) { + case UNKNOWN: + searchFormat = Format.XML; + break; + case JSON: + case XML: + break; + default: + throw new UnsupportedOperationException("Only XML and JSON search results are possible."); + } + + String mimetype = searchFormat.getDefaultMimetype(); + + OkHttpSearchRequest request = generateSearchRequest(reqlog, queryDef, mimetype, transaction, null, params, forestName); + + Response response = request.getResponse(); + if (response == null) return null; + + Class as = searchBase.receiveAs(); + + + ResponseBody body = response.body(); + Object entity = body.contentLength() != 0 ? getEntity(body, as) : null; + if (entity == null || (as != InputStream.class && as != Reader.class)) { + closeResponse(response); + } + searchBase.receiveContent(entity); + updateDescriptor(searchBase, response.headers()); + + logRequest(reqlog, + "searched starting at %s with length %s in %s transaction with %s mime type", + start, len, getTransactionId(transaction), mimetype); + + return searchHandle; + } + + private OkHttpSearchRequest generateSearchRequest(RequestLogger reqlog, SearchQueryDefinition queryDef, + String mimetype, Transaction transaction, ServerTransform responseTransform, + RequestParameters params, String forestName) { + if (params == null) params = new RequestParameters(); + if (forestName != null) params.add("forest-name", forestName); + return new OkHttpSearchRequest(reqlog, queryDef, mimetype, transaction, responseTransform, params); + } + + private class OkHttpSearchRequest { + RequestLogger reqlog; + SearchQueryDefinition queryDef; + String mimetype; + RequestParameters params; + ServerTransform responseTransform; + Transaction transaction; + + Request.Builder requestBldr = null; + String structure = null; + HandleImplementation baseHandle = null; + + OkHttpSearchRequest(RequestLogger reqlog, SearchQueryDefinition queryDef, String mimetype, + Transaction transaction, ServerTransform responseTransform, RequestParameters params) { + this.reqlog = reqlog; + this.queryDef = queryDef; + this.mimetype = mimetype; + this.transaction = transaction; + this.responseTransform = responseTransform; + this.params = params != null ? params : new RequestParameters(); + addParams(); + init(); + } + + void addParams() { + if (queryDef instanceof QueryDefinition) { + String directory = ((QueryDefinition) queryDef).getDirectory(); + if (directory != null) { + params.add("directory", directory); + } + + params.add("collection", ((QueryDefinition) queryDef).getCollections()); + } + + String optionsName = queryDef.getOptionsName(); + if (optionsName != null && optionsName.length() > 0) { + params.add("options", optionsName); + } + + ServerTransform transform = queryDef.getResponseTransform(); + if (transform != null) { + if (responseTransform != null) { + if (!transform.getName().equals(responseTransform.getName())) { + throw new IllegalStateException("QueryDefinition transform and DocumentManager transform have different names (" + + transform.getName() + ", " + responseTransform.getName() + ")"); + } + logger.warn("QueryDefinition and DocumentManager both specify a ServerTransform--using params from QueryDefinition"); + } + transform.merge(params); + } else if (responseTransform != null) { + responseTransform.merge(params); + } + + if (transaction != null) { + params.add("txid", transaction.getTransactionId()); + } + } + + void init() { + String text = null; + if (queryDef instanceof StringQueryDefinition) { + text = ((StringQueryDefinition) queryDef).getCriteria(); + } else if (queryDef instanceof StructuredQueryDefinition) { + text = ((StructuredQueryDefinition) queryDef).getCriteria(); + } else if (queryDef instanceof RawStructuredQueryDefinition) { + text = ((RawStructuredQueryDefinition) queryDef).getCriteria(); + } else if (queryDef instanceof RawCtsQueryDefinition) { + text = ((RawCtsQueryDefinition) queryDef).getCriteria(); + } + if (text != null) { + params.add("q", text); + } + if (queryDef instanceof StructuredQueryDefinition) { + structure = ((StructuredQueryDefinition) queryDef).serialize(); + + if (logger.isDebugEnabled()) { + String qtextMessage = text == null ? "" : " and string query \"" + text + "\""; + logger.debug("Searching for structure {}{}", structure, qtextMessage); + } + + requestBldr = setupRequest("search", params); + requestBldr = requestBldr.header(HEADER_CONTENT_TYPE, MIMETYPE_APPLICATION_XML); + requestBldr = requestBldr.header(HEADER_ACCEPT, mimetype); + } else if (queryDef instanceof RawQueryDefinition || queryDef instanceof RawCtsQueryDefinition) { + logger.debug("Raw search"); + + if (queryDef instanceof RawQueryDefinition) { + StructureWriteHandle handle = ((RawQueryDefinition) queryDef).getHandle(); + baseHandle = HandleAccessor.checkHandle(handle, "search"); + } else if (queryDef instanceof RawCtsQueryDefinition) { + CtsQueryWriteHandle handle = ((RawCtsQueryDefinition) queryDef).getHandle(); + baseHandle = HandleAccessor.checkHandle(handle, "search"); + } + + Format payloadFormat = getStructuredQueryFormat(baseHandle); + String payloadMimetype = getMimetypeWithDefaultXML(payloadFormat, baseHandle); + + String path = (queryDef instanceof RawQueryByExampleDefinition) ? + "qbe" : "search"; + + requestBldr = setupRequest(path, params); + if (payloadMimetype != null) { + requestBldr = requestBldr.header(HEADER_CONTENT_TYPE, payloadMimetype); + } + requestBldr = requestBldr.header(HEADER_ACCEPT, mimetype); + } else if (queryDef instanceof CombinedQueryDefinition) { + structure = ((CombinedQueryDefinition) queryDef).serialize(); + + logger.debug("Searching for combined query {}", structure); + + requestBldr = setupRequest("search", params); + requestBldr = requestBldr + .header(HEADER_CONTENT_TYPE, MIMETYPE_APPLICATION_XML) + .header(HEADER_ACCEPT, mimetype); + } else if (queryDef instanceof StringQueryDefinition) { + logger.debug("Searching for string [{}]", text); + + requestBldr = setupRequest("search", params); + requestBldr = requestBldr.header(HEADER_CONTENT_TYPE, MIMETYPE_APPLICATION_XML); + requestBldr = requestBldr.header(HEADER_ACCEPT, mimetype); + } else if (queryDef instanceof DeleteQueryDefinition) { + logger.debug("Searching for deletes"); + + requestBldr = setupRequest("search", params); + requestBldr = requestBldr.header(HEADER_ACCEPT, mimetype); + } else if (queryDef instanceof CtsQueryDefinition) { + structure = ((CtsQueryDefinition) queryDef).serialize(); + if (logger.isDebugEnabled()) { + logger.debug("Searching Cts Query: ", ((CtsQueryDefinition) queryDef).serialize()); + } + requestBldr = setupRequest("search", params); + requestBldr = requestBldr.header(HEADER_CONTENT_TYPE, MIMETYPE_APPLICATION_JSON); + requestBldr = requestBldr.header(HEADER_ACCEPT, mimetype); + } else { + throw new UnsupportedOperationException("Cannot search with " + + queryDef.getClass().getName()); + } + + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + } + + Response getResponse() { + Response response = null; + int status = -1; + long startTime = System.currentTimeMillis(); + int nextDelay = 0; + int retry = 0; + for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + if (nextDelay > 0) { + try { + Thread.sleep(nextDelay); + } catch (InterruptedException e) { + } + } + + if (queryDef instanceof StructuredQueryDefinition && !(queryDef instanceof RawQueryDefinition)) { + response = doPost(reqlog, requestBldr, structure); + } else if (queryDef instanceof CombinedQueryDefinition) { + response = doPost(reqlog, requestBldr, structure); + } else if (queryDef instanceof DeleteQueryDefinition) { + response = doGet(requestBldr); + } else if (queryDef instanceof RawQueryDefinition) { + response = doPost(reqlog, requestBldr, baseHandle.sendContent()); + } else if (queryDef instanceof RawCtsQueryDefinition) { + response = doPost(reqlog, requestBldr, baseHandle.sendContent()); + } else if (queryDef instanceof StringQueryDefinition) { + response = doGet(requestBldr); + } else if (queryDef instanceof CtsQueryDefinition) { + response = doPost(reqlog, requestBldr, structure); + } else { + throw new UnsupportedOperationException("Cannot search with " + + queryDef.getClass().getName()); + } + + status = response.code(); + + if (transaction != null || !retryStatus.contains(status)) { + if (isFirstRequest()) setFirstRequest(false); + + break; + } + + String retryAfterRaw = response.header("Retry-After"); + int retryAfter = Utilities.parseInt(retryAfterRaw); + + closeResponse(response); + + nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + } + if (retryStatus.contains(status)) { + checkFirstRequest(); + closeResponse(response); + throw new FailedRetryException( + "Service unavailable and maximum retry period elapsed: " + + ((System.currentTimeMillis() - startTime) / 1000) + + " seconds after " + retry + " retries"); + } + if (status == STATUS_NOT_FOUND) { + closeResponse(response); + return null; + } + if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException("User is not allowed to search", + extractErrorFields(response)); + } + if (status != STATUS_OK) { + throw new FailedRequestException("search failed: " + + getReasonPhrase(response), extractErrorFields(response)); + } + return response; + } + } + + private Format getStructuredQueryFormat(HandleImplementation baseHandle) { + Format payloadFormat = baseHandle.getFormat(); + if (payloadFormat == Format.UNKNOWN) { + payloadFormat = null; + } else if (payloadFormat != Format.XML && payloadFormat != Format.JSON) { + throw new IllegalArgumentException( + "Cannot perform raw search for format " + payloadFormat.name()); + } + return payloadFormat; + } + + private String getMimetypeWithDefaultXML(Format payloadFormat, HandleImplementation baseHandle) { + String payloadMimetype = baseHandle.getMimetype(); + if (payloadFormat != null) { + if (payloadMimetype == null) { + payloadMimetype = payloadFormat.getDefaultMimetype(); + } + } else if (payloadMimetype == null) { + payloadMimetype = MIMETYPE_APPLICATION_XML; + } + return payloadMimetype; + } + + @Override + public void deleteSearch(RequestLogger reqlog, DeleteQueryDefinition queryDef, + Transaction transaction) + throws ForbiddenUserException, FailedRequestException { + RequestParameters params = new RequestParameters(); + + if (queryDef.getDirectory() != null) { + params.add("directory", queryDef.getDirectory()); + } + + params.add("collection", queryDef.getCollections()); + + if (transaction != null) { + params.add("txid", transaction.getTransactionId()); + } + + Request.Builder requestBldr = setupRequest("search", params); + + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + + Function doDeleteFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return sendRequestOnce(funcBuilder.delete().build()); + } + }; + Response response = sendRequestWithRetry(requestBldr, (transaction == null), doDeleteFunction, null); + int status = response.code(); + if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException("User is not allowed to delete", + extractErrorFields(response)); + } + + if (status != STATUS_NO_CONTENT) { + throw new FailedRequestException("delete failed: " + + getReasonPhrase(response), extractErrorFields(response)); + } + + closeResponse(response); + + logRequest( + reqlog, + "deleted search results in %s transaction", + getTransactionId(transaction)); + } + + @Override + public void delete(RequestLogger logger, Transaction transaction, Set categories, String... uris) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + RequestParameters params = new RequestParameters(); + addCategoryParams(categories, params, false); + for (String uri : uris) { + params.add("uri", uri); + } + deleteResource(logger, "documents", transaction, params, null); + } + + @Override + public T values(Class as, ValuesDefinition valDef, String mimetype, + long start, long pageLength, Transaction transaction) + throws ForbiddenUserException, FailedRequestException { + RequestParameters docParams = new RequestParameters(); + + String optionsName = valDef.getOptionsName(); + if (optionsName != null && optionsName.length() > 0) { + docParams.add("options", optionsName); + } + + if (valDef.getAggregate() != null) { + docParams.add("aggregate", valDef.getAggregate()); + } + + if (valDef.getAggregatePath() != null) { + docParams.add("aggregatePath", + valDef.getAggregatePath()); + } + + if (valDef.getView() != null) { + docParams.add("view", valDef.getView()); + } + + if (valDef.getDirection() != null) { + if (valDef.getDirection() == ValuesDefinition.Direction.ASCENDING) { + docParams.add("direction", "ascending"); + } else { + docParams.add("direction", "descending"); + } + } + + if (valDef.getFrequency() != null) { + if (valDef.getFrequency() == ValuesDefinition.Frequency.FRAGMENT) { + docParams.add("frequency", "fragment"); + } else { + docParams.add("frequency", "item"); + } + } + + if (start > 0) { + docParams.add("start", Long.toString(start)); + if (pageLength > 0) { + docParams.add("pageLength", Long.toString(pageLength)); + } + } + + HandleImplementation baseHandle = null; + + if (valDef.getQueryDefinition() != null) { + ValueQueryDefinition queryDef = valDef.getQueryDefinition(); + + if (optionsName == null) { + optionsName = queryDef.getOptionsName(); + if (optionsName != null) { + docParams.add("options", optionsName); + } + } else if (queryDef.getOptionsName() != null) { + if (!queryDef.getOptionsName().equals(optionsName)) { + logger.warn("values definition options take precedence over query definition options"); + } + } + + if (queryDef.getCollections().length > 0) { + logger.warn("collections scope ignored for values query"); + } + if (queryDef.getDirectory() != null) { + logger.warn("directory scope ignored for values query"); + } + + String text = null; + if (queryDef instanceof StringQueryDefinition) { + text = ((StringQueryDefinition) queryDef).getCriteria(); + } else if (queryDef instanceof StructuredQueryDefinition) { + text = ((StructuredQueryDefinition) queryDef).getCriteria(); + } else if (queryDef instanceof RawStructuredQueryDefinition) { + text = ((RawStructuredQueryDefinition) queryDef).getCriteria(); + } else if (queryDef instanceof RawCtsQueryDefinition) { + text = ((RawCtsQueryDefinition) queryDef).getCriteria(); + } + if (text != null) { + docParams.add("q", text); + } + if (queryDef instanceof StructuredQueryDefinition) { + String structure = ((StructuredQueryDefinition) queryDef) + .serialize(); + if (structure != null) { + docParams.add("structuredQuery", structure); + } + } else if (queryDef instanceof RawQueryDefinition) { + StructureWriteHandle handle = ((RawQueryDefinition) queryDef).getHandle(); + baseHandle = HandleAccessor.checkHandle(handle, "values"); + } else if (queryDef instanceof RawCtsQueryDefinition) { + CtsQueryWriteHandle handle = ((RawCtsQueryDefinition) queryDef).getHandle(); + baseHandle = HandleAccessor.checkHandle(handle, "values"); + } else if (queryDef instanceof StringQueryDefinition) { + } else { + logger.warn("unsupported query definition: {}", queryDef.getClass().getName()); + } + + ServerTransform transform = queryDef.getResponseTransform(); + if (transform != null) { + transform.merge(docParams); + } + } + + if (transaction != null) { + docParams.add("txid", transaction.getTransactionId()); + } + + String uri = "values"; + if (valDef.getName() != null) { + uri += "/" + valDef.getName(); + } + + Request.Builder requestBldr = setupRequest(uri, docParams); + requestBldr = setupRequest(requestBldr, null, mimetype); + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + + final HandleImplementation tempBaseHandle = baseHandle; + + Function doFunction = (baseHandle == null) ? + new Function() { + public Response apply(Request.Builder funcBuilder) { + return doGet(funcBuilder); + } + } : + new Function() { + public Response apply(Request.Builder funcBuilder) { + String contentType = tempBaseHandle.getMimetype(); + return doPost(null, + (contentType == null) ? funcBuilder : funcBuilder.header(HEADER_CONTENT_TYPE, contentType), + tempBaseHandle.sendContent()); + } + }; + + Response response = sendRequestWithRetry(requestBldr, (transaction == null), doFunction, null); + int status = response.code(); + + if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException("User is not allowed to search", + extractErrorFields(response)); + } + if (status != STATUS_OK) { + throw new FailedRequestException("search failed: " + + getReasonPhrase(response), extractErrorFields(response)); + } + + ResponseBody body = response.body(); + T entity = body.contentLength() != 0 ? getEntity(body, as) : null; + if (entity == null || (as != InputStream.class && as != Reader.class)) { + closeResponse(response); + } + + return entity; + } + + @Override + public T valuesList(Class as, ValuesListDefinition valDef, + String mimetype, Transaction transaction) + throws ForbiddenUserException, FailedRequestException { + RequestParameters docParams = new RequestParameters(); + + String optionsName = valDef.getOptionsName(); + if (optionsName != null && optionsName.length() > 0) { + docParams.add("options", optionsName); + } + + if (transaction != null) { + docParams.add("txid", transaction.getTransactionId()); + } + + String uri = "values"; + + Request.Builder requestBldr = setupRequest(uri, docParams); + requestBldr = setupRequest(requestBldr, null, mimetype); + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + + Function doGetFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return sendRequestOnce(funcBuilder.get().build()); + } + }; + Response response = sendRequestWithRetry(requestBldr, (transaction == null), doGetFunction, null); + int status = response.code(); + + if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException("User is not allowed to search", + extractErrorFields(response)); + } + if (status != STATUS_OK) { + throw new FailedRequestException("search failed: " + + getReasonPhrase(response), extractErrorFields(response)); + } + + ResponseBody body = response.body(); + T entity = body.contentLength() != 0 ? getEntity(body, as) : null; + if (entity == null || (as != InputStream.class && as != Reader.class)) { + closeResponse(response); + } + + return entity; + } + + @Override + public T optionsList(Class as, String mimetype, Transaction transaction) + throws ForbiddenUserException, FailedRequestException { + RequestParameters docParams = new RequestParameters(); + + if (transaction != null) { + docParams.add("txid", transaction.getTransactionId()); + } + + String uri = "config/query"; + + Request.Builder requestBldr = setupRequest(uri, docParams); + requestBldr = requestBldr.header(HEADER_ACCEPT, mimetype); + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + + Function doGetFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return sendRequestOnce(funcBuilder.get().build()); + } + }; + Response response = sendRequestWithRetry(requestBldr, (transaction == null), doGetFunction, null); + int status = response.code(); + + if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException("User is not allowed to search", + extractErrorFields(response)); + } + if (status != STATUS_OK) { + throw new FailedRequestException("search failed: " + + getReasonPhrase(response), extractErrorFields(response)); + } + + ResponseBody body = response.body(); + T entity = body.contentLength() != 0 ? getEntity(body, as) : null; + if (entity == null || (as != InputStream.class && as != Reader.class)) { + closeResponse(response); + } + + return entity; + } + + // namespaces, search options etc. + @Override + public T getValue(RequestLogger reqlog, String type, String key, + boolean isNullable, String mimetype, Class as) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + return getValue(reqlog, type, key, null, isNullable, mimetype, as); + } + + @Override + public T getValue(RequestLogger reqlog, String type, String key, Transaction transaction, + boolean isNullable, String mimetype, Class as) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + logger.debug("Getting {}/{}", type, key); + + Request.Builder requestBldr = setupRequest(type + "/" + key, null, null, mimetype); + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + + Function doGetFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return sendRequestOnce(funcBuilder.get().build()); + } + }; + Response response = sendRequestWithRetry(requestBldr, (transaction == null), doGetFunction, null); + int status = response.code(); + + if (status != STATUS_OK) { + if (status == STATUS_NOT_FOUND) { + closeResponse(response); + if (!isNullable) { + throw new ResourceNotFoundException("Could not get " + type + "/" + key); + } + return null; + } else if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException("User is not allowed to read " + + type, extractErrorFields(response)); + } else { + throw new FailedRequestException(type + " read failed: " + + getReasonPhrase(response), + extractErrorFields(response)); + } + } + + logRequest(reqlog, "read %s value with %s key and %s mime type", type, + key, mimetype); + + ResponseBody body = response.body(); + T entity = body.contentLength() != 0 ? getEntity(body, as) : null; + if (entity == null || (as != InputStream.class && as != Reader.class)) { + closeResponse(response); + } + + return (reqlog != null) ? reqlog.copyContent(entity) : entity; + } + + @Override + public T getValues(RequestLogger reqlog, String type, String mimetype, Class as) + throws ForbiddenUserException, FailedRequestException { + return getValues(reqlog, type, null, mimetype, as); + } + + @Override + public T getValues(RequestLogger reqlog, String type, RequestParameters extraParams, + String mimetype, Class as) + throws ForbiddenUserException, FailedRequestException { + logger.debug("Getting {}", type); + + Request.Builder requestBldr = setupRequest(type, extraParams).header(HEADER_ACCEPT, mimetype); + requestBldr = addTelemetryAgentId(requestBldr); + + Function doGetFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return sendRequestOnce(funcBuilder.get().build()); + } + }; + Response response = sendRequestWithRetry(requestBldr, doGetFunction, null); + int status = response.code(); + if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException("User is not allowed to read " + + type, extractErrorFields(response)); + } + if (status != STATUS_OK) { + throw new FailedRequestException(type + " read failed: " + + getReasonPhrase(response), extractErrorFields(response)); + } + + logRequest(reqlog, "read %s values with %s mime type", type, mimetype); + + ResponseBody body = response.body(); + T entity = body.contentLength() != 0 ? getEntity(body, as) : null; + if (entity == null || (as != InputStream.class && as != Reader.class)) { + closeResponse(response); + } + + return (reqlog != null) ? reqlog.copyContent(entity) : entity; + } + + @Override + public void postValue(RequestLogger reqlog, String type, String key, + String mimetype, Object value) + throws ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + logger.debug("Posting {}/{}", type, key); + + putPostValueImpl(reqlog, "post", type, key, null, mimetype, value, STATUS_CREATED); + } + + @Override + public void postValue(RequestLogger reqlog, String type, String key, + RequestParameters extraParams) + throws ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + logger.debug("Posting {}/{}", type, key); + + putPostValueImpl(reqlog, "post", type, key, extraParams, null, null, STATUS_NO_CONTENT); + } + + + @Override + public void putValue(RequestLogger reqlog, String type, String key, + String mimetype, Object value) + throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + logger.debug("Putting {}/{}", type, key); + + putPostValueImpl(reqlog, "put", type, key, null, mimetype, value, STATUS_NO_CONTENT, STATUS_CREATED); + } + + @Override + public void putValue(RequestLogger reqlog, String type, String key, + RequestParameters extraParams, String mimetype, Object value) + throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + logger.debug("Putting {}/{}", type, key); + + putPostValueImpl(reqlog, "put", type, key, extraParams, mimetype, value, STATUS_NO_CONTENT); + } + + private void putPostValueImpl(RequestLogger reqlog, String method, + String type, String key, RequestParameters extraParams, + String mimetype, Object value, + int... expectedStatuses) { + if (key != null) { + logRequest(reqlog, "writing %s value with %s key and %s mime type", + type, key, mimetype); + } else { + logRequest(reqlog, "writing %s values with %s mime type", type, mimetype); + } + + HandleImplementation handle = (value instanceof HandleImplementation) ? + (HandleImplementation) value : null; + + MediaType mediaType = makeType(mimetype); + + String connectPath = null; + Request.Builder requestBldr = null; + + Response response = null; + int status = -1; + long startTime = System.currentTimeMillis(); + int nextDelay = 0; + int retry = 0; + for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + if (nextDelay > 0) { + try { + Thread.sleep(nextDelay); + } catch (InterruptedException e) { + } + } + + Object nextValue = (handle != null) ? handle.sendContent() : value; + + RequestBody sentValue = null; + if (nextValue instanceof OutputStreamSender) { + sentValue = new StreamingOutputImpl( + (OutputStreamSender) nextValue, reqlog, mediaType); + } else { + if (reqlog != null && retry == 0) { + sentValue = new ObjectRequestBody(reqlog.copyContent(nextValue), mediaType); + } else { + sentValue = new ObjectRequestBody(nextValue, mediaType); + } + } + + boolean isStreaming = (isFirstRequest() || handle == null) ? isStreaming(sentValue) : false; + + boolean isResendable = (handle == null) ? !isStreaming : handle.isResendable(); + + if (isFirstRequest() && !isResendable && isStreaming) { + nextDelay = makeFirstRequest(retry); + if (nextDelay != 0) continue; + } + + if ("put".equals(method)) { + if (requestBldr == null) { + connectPath = (key != null) ? type + "/" + key : type; + Request.Builder resource = setupRequest(connectPath, extraParams); + requestBldr = (mimetype == null) ? + resource : resource.header(HEADER_CONTENT_TYPE, mimetype); + requestBldr = addTelemetryAgentId(requestBldr); + } + + response = (sentValue == null) ? + sendRequestOnce(requestBldr.put(null).build()) : + sendRequestOnce(requestBldr.put(sentValue).build()); + } else if ("post".equals(method)) { + if (requestBldr == null) { + connectPath = type; + Request.Builder resource = setupRequest(connectPath, extraParams); + requestBldr = (mimetype == null) ? + resource : resource.header(HEADER_CONTENT_TYPE, mimetype); + requestBldr = addTelemetryAgentId(requestBldr); + } + + response = (sentValue == null) ? + sendRequestOnce(requestBldr.post(RequestBody.create("", null)).build()) : + sendRequestOnce(requestBldr.post(sentValue).build()); + } else { + throw new MarkLogicInternalException("unknown method type " + + method); + } + + status = response.code(); + + if (!retryStatus.contains(status)) { + if (isFirstRequest()) setFirstRequest(false); + break; + } + + String retryAfterRaw = response.header("Retry-After"); + closeResponse(response); + + if (!isResendable) { + checkFirstRequest(); + throw new ResourceNotResendableException( + "Cannot retry request for " + connectPath); + } + + int retryAfter = Utilities.parseInt(retryAfterRaw); + nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + } + if (retryStatus.contains(status)) { + checkFirstRequest(); + closeResponse(response); + throw new FailedRetryException( + "Service unavailable and maximum retry period elapsed: " + + ((System.currentTimeMillis() - startTime) / 1000) + + " seconds after " + retry + " retries"); + } + if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException("User is not allowed to write " + + type, extractErrorFields(response)); + } + if (status == STATUS_NOT_FOUND) { + throw new ResourceNotFoundException(type + " not found for write", + extractErrorFields(response)); + } + boolean statusOk = false; + for (int expectedStatus : expectedStatuses) { + statusOk = statusOk || (status == expectedStatus); + if (statusOk) { + break; + } + } + + if (!statusOk) { + throw new FailedRequestException(type + " write failed: " + + getReasonPhrase(response), extractErrorFields(response)); + } + closeResponse(response); + } + + @Override + public void deleteValue(RequestLogger reqlog, String type, String key) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + logger.debug("Deleting {}/{}", type, key); + + Request.Builder requestBldr = setupRequest(type + "/" + key, null); + requestBldr = addTelemetryAgentId(requestBldr); + + Function doDeleteFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return sendRequestOnce(funcBuilder.delete().build()); + } + }; + Response response = sendRequestWithRetry(requestBldr, doDeleteFunction, null); + int status = response.code(); + if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException("User is not allowed to delete " + + type, extractErrorFields(response)); + } + if (status == STATUS_NOT_FOUND) { + throw new ResourceNotFoundException(type + " not found for delete", + extractErrorFields(response)); + } + if (status != STATUS_NO_CONTENT) { + throw new FailedRequestException("delete failed: " + + getReasonPhrase(response), extractErrorFields(response)); + } + + closeResponse(response); + + logRequest(reqlog, "deleted %s value with %s key", type, key); + } + + @Override + public void deleteValues(RequestLogger reqlog, String type) + throws ForbiddenUserException, FailedRequestException { + logger.debug("Deleting {}", type); + + Request.Builder requestBldr = setupRequest(type, null); + requestBldr = addTelemetryAgentId(requestBldr); + + Function doDeleteFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return sendRequestOnce(funcBuilder.delete().build()); + } + }; + Response response = sendRequestWithRetry(requestBldr, doDeleteFunction, null); + int status = response.code(); + if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException("User is not allowed to delete " + + type, extractErrorFields(response)); + } + if (status != STATUS_NO_CONTENT) { + throw new FailedRequestException("delete failed: " + + getReasonPhrase(response), extractErrorFields(response)); + } + closeResponse(response); + + logRequest(reqlog, "deleted %s values", type); + } + + @Override + public R getSystemSchema(RequestLogger reqlog, String schemaName, R output) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + RequestParameters params = new RequestParameters(); + params.add("system", schemaName); + return getResource(reqlog, "internal/schemas", null, params, output); + } + + @Override + public R uris(RequestLogger reqlog, String method, SearchQueryDefinition qdef, + Boolean filtered, long start, String afterUri, long pageLength, String forestName, R output + ) throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + logger.debug("Querying for uris"); + RequestParameters params = new RequestParameters(); + if (filtered != null) params.add("filtered", filtered.toString()); + if (forestName != null) params.add("forest-name", forestName); + if (start > 1) params.add("start", Long.toString(start)); + if (afterUri != null) params.add("after", afterUri); + if (pageLength >= 1) params.add("pageLength", Long.toString(pageLength)); + return processQuery(reqlog, "internal/uris", method, params, qdef, output); + } + + @Override + public R forestInfo(RequestLogger reqlog, + String method, RequestParameters params, SearchQueryDefinition qdef, R output + ) throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + return processQuery(reqlog, "internal/forestinfo", method, params, qdef, output); + } + + private R processQuery(RequestLogger reqlog, String path, + String method, RequestParameters params, SearchQueryDefinition qdef, R output + ) throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + if (qdef instanceof QueryDefinition) { + if (((QueryDefinition) qdef).getDirectory() != null) { + params.add("directory", ((QueryDefinition) qdef).getDirectory()); + } + + if (((QueryDefinition) qdef).getCollections() != null) { + for (String collection : ((QueryDefinition) qdef).getCollections()) { + params.add("collection", collection); + } + } + } + + if (qdef.getOptionsName() != null && qdef.getOptionsName().length() > 0) { + params.add("options", qdef.getOptionsName()); + } + + if (qdef instanceof RawQueryByExampleDefinition) { + throw new UnsupportedOperationException(path + " cannot process RawQueryByExampleDefinition"); + } + + boolean sendQueryAsPayload = "POST".equals(method); + + String text = null; + String structure = null; + StructureWriteHandle input = null; + if (qdef instanceof RawCtsQueryDefinition) { + if (!(qdef instanceof RawQueryDefinitionImpl.CtsQuery)) { + throw new IllegalArgumentException( + "unknown implementation of RawCtsQueryDefinition: " + qdef.getClass().getName()); + } + RawQueryDefinitionImpl.CtsQuery ctsQuery = (RawQueryDefinitionImpl.CtsQuery) qdef; + text = ctsQuery.getCriteria(); + structure = ctsQuery.serialize(); + logger.debug("{} processing raw cts query {} and string query \"{}\"", path, structure, text); + if (structure != null) { + input = checkStructure(structure, ctsQuery.getHandle()); + } + } else if (qdef instanceof StructuredQueryDefinition) { + StructuredQueryDefinition builtStructuredQuery = (StructuredQueryDefinition) qdef; + text = builtStructuredQuery.getCriteria(); + structure = builtStructuredQuery.serialize(); + logger.debug("{} processing structured query {} and string query \"{}\"", path, structure, text); + if (sendQueryAsPayload && structure != null) { + input = new StringHandle(structure).withFormat(Format.XML); + } + } else if (qdef instanceof RawStructuredQueryDefinition) { + RawStructuredQueryDefinition rawStructuredQuery = (RawStructuredQueryDefinition) qdef; + text = rawStructuredQuery.getCriteria(); + structure = rawStructuredQuery.serialize(); + logger.debug("{} processing raw structured query {} and string query \"{}\"", path, structure, text); + if (sendQueryAsPayload && structure != null) { + input = checkStructure(structure, rawStructuredQuery.getHandle()); + } + } else if (qdef instanceof CombinedQueryDefinition) { + CombinedQueryDefinition combinedQuery = (CombinedQueryDefinition) qdef; + structure = combinedQuery.serialize(); + logger.debug("{} processing combined query {}", path, structure); + if (sendQueryAsPayload && structure != null) { + input = checkStructure(structure, combinedQuery.getFormat()); + } + } else if (qdef instanceof StringQueryDefinition) { + StringQueryDefinition stringQuery = (StringQueryDefinition) qdef; + text = stringQuery.getCriteria(); + logger.debug("{} processing string query \"{}\"", path, text); + } else if (qdef instanceof RawQueryDefinitionImpl) { + RawQueryDefinitionImpl rawQueryImpl = (RawQueryDefinitionImpl) qdef; + structure = rawQueryImpl.serialize(); + logger.debug("{} processing raw query implementation {}", path, structure); + input = checkStructure(structure, rawQueryImpl.getHandle()); + } else if (qdef instanceof RawQueryDefinition) { + RawQueryDefinition rawQuery = (RawQueryDefinition) qdef; + logger.debug("{} processing raw query", path); + input = checkFormat(rawQuery.getHandle()); + } else if (qdef instanceof CtsQueryDefinition) { + CtsQueryDefinition builtCtsQuery = (CtsQueryDefinition) qdef; + structure = builtCtsQuery.serialize(); + logger.debug("{} processing cts query {}", path, structure); + if (sendQueryAsPayload && structure != null) { + input = new StringHandle(structure).withFormat(Format.JSON); + } + } else { + throw new UnsupportedOperationException(path + " cannot process query of " + qdef.getClass().getName()); + } + + if (text != null) { + params.add("q", text); + } + + if (input != null) { + return postResource(reqlog, path, null, params, input, output); + } else if (structure != null) { + params.add("structuredQuery", structure); + } + return getResource(reqlog, path, null, params, output); + } + + private StructureWriteHandle checkStructure(String structure, StructureWriteHandle handle) { + return checkStructure(structure, + (handle == null || !(handle instanceof HandleImplementation)) ? Format.UNKNOWN : + ((HandleImplementation) handle).getFormat()); + } + + private StructureWriteHandle checkStructure(String structure, Format format) { + return new StringHandle(structure).withFormat( + (format == null || format == Format.UNKNOWN) ? Format.TEXT : format); + } + + private StructureWriteHandle checkFormat(StructureWriteHandle handle) { + if (handle != null && handle instanceof HandleImplementation) { + HandleImplementation handleImpl = (HandleImplementation) handle; + Format format = handleImpl.getFormat(); + if (format == null || format == Format.UNKNOWN) { + handleImpl.setFormat(Format.TEXT); + handleImpl.setMimetype(Format.TEXT.getDefaultMimetype()); + } + } + return handle; + } + + @Override + public R getResource(RequestLogger reqlog, + String path, Transaction transaction, RequestParameters params, R output) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + if (params == null) params = new RequestParameters(); + if (transaction != null) params.add("txid", transaction.getTransactionId()); + addPointInTimeQueryParam(params, output); + HandleImplementation outputBase = HandleAccessor.checkHandle(output, + "read"); + + String mimetype = outputBase.getMimetype(); + Class as = outputBase.receiveAs(); + + Request.Builder requestBldr = makeGetWebResource(path, params, mimetype); + requestBldr = setupRequest(requestBldr, null, mimetype); + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + + Function doGetFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return doGet(funcBuilder); + } + }; + Response response = sendRequestWithRetry(requestBldr, (transaction == null), doGetFunction, null); + int status = response.code(); + checkStatus(response, status, "read", "resource", path, + ResponseStatus.OK_OR_NO_CONTENT); + + updateDescriptor(outputBase, response.headers()); + if (as != null) { + outputBase.receiveContent(makeResult(reqlog, "read", "resource", + response, as)); + } else { + closeResponse(response); + } + + return output; + } + + @Override + public RESTServiceResultIterator getIteratedResource(RequestLogger reqlog, + String path, Transaction transaction, RequestParameters params) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + return getIteratedResourceImpl(OkHttpServiceResultIterator::new, reqlog, path, transaction, params); + } + + private U getIteratedResourceImpl(ResultIteratorConstructor constructor, + RequestLogger reqlog, String path, Transaction transaction, RequestParameters params) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + if (params == null) params = new RequestParameters(); + if (transaction != null) params.add("txid", transaction.getTransactionId()); + + Request.Builder requestBldr = makeGetWebResource(path, params, null); + requestBldr = setupRequest(requestBldr, null, null); + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + + requestBldr = requestBldr.header(HEADER_ACCEPT, multipartMixedWithBoundary()); + Function doGetFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return doGet(funcBuilder); + } + }; + Response response = sendRequestWithRetry(requestBldr, (transaction == null), doGetFunction, null); + int status = response.code(); + checkStatus(response, status, "read", "resource", path, + ResponseStatus.OK_OR_NO_CONTENT); + + return makeResults(constructor, reqlog, "read", "resource", response); + } + + @Override + public R putResource(RequestLogger reqlog, + String path, Transaction transaction, RequestParameters params, + AbstractWriteHandle input, R output) + throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + if (params == null) params = new RequestParameters(); + if (transaction != null) params.add("txid", transaction.getTransactionId()); + HandleImplementation inputBase = HandleAccessor.checkHandle(input, + "write"); + HandleImplementation outputBase = HandleAccessor.checkHandle(output, + "read"); + + String inputMimetype = inputBase.getMimetype(); + boolean isResendable = inputBase.isResendable(); + String outputMimeType = null; + Class as = null; + if (outputBase != null) { + outputMimeType = outputBase.getMimetype(); + as = outputBase.receiveAs(); + } + Request.Builder requestBldr = makePutWebResource(path, params); + requestBldr = setupRequest(requestBldr, inputMimetype, outputMimeType); + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + + Consumer resendableConsumer = (resendable) -> { + if (!isResendable) { + checkFirstRequest(); + throw new ResourceNotResendableException( + "Cannot retry request for " + path); + } + }; + + Function doPutFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return doPut(reqlog, funcBuilder, inputBase.sendContent()); + } + }; + Response response = sendRequestWithRetry(requestBldr, (transaction == null), doPutFunction, resendableConsumer); + int status = response.code(); + + checkStatus(response, status, "write", "resource", path, + ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); + + if (as != null) { + outputBase.receiveContent(makeResult(reqlog, "write", "resource", + response, as)); + } else { + closeResponse(response); + } + + return output; + } + + @Override + public R putResource( + RequestLogger reqlog, String path, Transaction transaction, RequestParameters params, + W[] input, R output) + throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + if (input == null || input.length == 0) { + throw new IllegalArgumentException("input not specified for multipart"); + } + if (params == null) params = new RequestParameters(); + if (transaction != null) params.add("txid", transaction.getTransactionId()); + + HandleImplementation outputBase = HandleAccessor.checkHandle(output, + "read"); + + String outputMimetype = outputBase.getMimetype(); + Class as = outputBase.receiveAs(); + + Response response = null; + int status = -1; + long startTime = System.currentTimeMillis(); + int nextDelay = 0; + int retry = 0; + for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + if (nextDelay > 0) { + try { + Thread.sleep(nextDelay); + } catch (InterruptedException e) { + } + } + + MultipartBody.Builder multiPart = new MultipartBody.Builder(); + boolean hasStreamingPart = addParts(multiPart, reqlog, input); + + Request.Builder requestBldr = makePutWebResource(path, params); + requestBldr = setupRequest(requestBldr, multiPart, outputMimetype); + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + + response = doPut(requestBldr, multiPart, hasStreamingPart); + status = response.code(); + + if (transaction != null || !retryStatus.contains(status)) { + if (isFirstRequest()) setFirstRequest(false); + + break; + } + + String retryAfterRaw = response.header("Retry-After"); + closeResponse(response); + + if (hasStreamingPart) { + throw new ResourceNotResendableException( + "Cannot retry request for " + path); + } + + int retryAfter = Utilities.parseInt(retryAfterRaw); + nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + } + if (retryStatus.contains(status)) { + checkFirstRequest(); + closeResponse(response); + throw new FailedRetryException( + "Service unavailable and maximum retry period elapsed: " + + ((System.currentTimeMillis() - startTime) / 1000) + + " seconds after " + retry + " retries"); + } + + checkStatus(response, status, "write", "resource", path, + ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); + + if (as != null) { + outputBase.receiveContent(makeResult(reqlog, "write", "resource", + response, as)); + } else { + closeResponse(response); + } + + return output; + } + + @Override + public R postResource(RequestLogger reqlog, + String path, Transaction transaction, RequestParameters params, + AbstractWriteHandle input, R output) + throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + return postResource(reqlog, path, transaction, params, input, output, "apply"); + } + + @Override + public R postResource(RequestLogger reqlog, + String path, Transaction transaction, RequestParameters params, + AbstractWriteHandle input, R output, String operation) + throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + return postResource(reqlog, path, transaction, params, input, output, operation, null); + } + + @Override + public R postResource(RequestLogger reqlog, + String path, Transaction transaction, RequestParameters params, + AbstractWriteHandle input, R output, String operation, + Map> responseHeaders) + throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + if (params == null) params = new RequestParameters(); + if (transaction != null) params.add("txid", transaction.getTransactionId()); + + HandleImplementation inputBase = HandleAccessor.checkHandle(input, + "write"); + HandleImplementation outputBase = HandleAccessor.checkHandle(output, + "read"); + + addPointInTimeQueryParam(params, outputBase); + + String inputMimetype = null; + if (inputBase != null) { + inputMimetype = inputBase.getMimetype(); + if (inputMimetype == null && + (Format.JSON == inputBase.getFormat() || + Format.XML == inputBase.getFormat())) { + inputMimetype = inputBase.getFormat().getDefaultMimetype(); + } + } + String outputMimetype = outputBase == null ? null : outputBase.getMimetype(); + boolean isResendable = inputBase == null ? true : inputBase.isResendable(); + Class as = outputBase == null ? null : outputBase.receiveAs(); + + Request.Builder requestBldr = makePostWebResource(path, params); + requestBldr = setupRequest(requestBldr, inputMimetype, outputMimetype); + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + + Consumer resendableConsumer = new Consumer() { + public void accept(Boolean resendable) { + if (!isResendable) { + checkFirstRequest(); + throw new ResourceNotResendableException("Cannot retry request for " + path); + } + } + }; + final Object value = inputBase == null ? null : inputBase.sendContent(); + Function doPostFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return doPost(reqlog, funcBuilder, value); + } + }; + + Response response = sendRequestWithRetry(requestBldr, (transaction == null), doPostFunction, resendableConsumer); + int status = response.code(); + checkStatus(response, status, operation, "resource", path, + ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); + + Headers headers = response.headers(); + if (responseHeaders != null) { + // add all the headers from the OkHttp Headers object to the caller-provided map + responseHeaders.putAll(headers.toMultimap()); + } else if (outputBase != null) { + updateLength(outputBase, headers); + updateServerTimestamp(outputBase, headers); + } + + if (as != null) { + outputBase.receiveContent(makeResult(reqlog, operation, "resource", + response, as)); + } else { + closeResponse(response); + } + + return output; + } + + @Override + public R postResource( + RequestLogger reqlog, String path, Transaction transaction, RequestParameters params, + W[] input, R output) + throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + return postResource(reqlog, path, transaction, params, input, null, output); + } + + @Override + public R postResource( + RequestLogger reqlog, String path, Transaction transaction, RequestParameters params, + W[] input, Map>[] requestHeaders, R output) + throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + if (params == null) params = new RequestParameters(); + if (transaction != null) params.add("txid", transaction.getTransactionId()); + + HandleImplementation outputBase = HandleAccessor.checkHandle(output, "read"); + + String outputMimetype = outputBase != null ? outputBase.getMimetype() : null; + Class as = outputBase != null ? outputBase.receiveAs() : null; + + Response response = null; + int status = -1; + long startTime = System.currentTimeMillis(); + int nextDelay = 0; + int retry = 0; + for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + if (nextDelay > 0) { + try { + Thread.sleep(nextDelay); + } catch (InterruptedException e) { + } + } + + MultipartBody.Builder multiPart = new MultipartBody.Builder(); + boolean hasStreamingPart = addParts(multiPart, reqlog, null, input, requestHeaders); + + Request.Builder requestBldr = makePostWebResource(path, params); + requestBldr = setupRequest(requestBldr, multiPart, outputMimetype); + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + + response = doPost(requestBldr, multiPart, hasStreamingPart); + status = response.code(); + + if (transaction != null || !retryStatus.contains(status)) { + if (isFirstRequest()) setFirstRequest(false); + + break; + } + + String retryAfterRaw = response.header("Retry-After"); + closeResponse(response); + + if (hasStreamingPart) { + throw new ResourceNotResendableException( + "Cannot retry request for " + path); + } + + int retryAfter = Utilities.parseInt(retryAfterRaw); + nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + } + if (retryStatus.contains(status)) { + checkFirstRequest(); + closeResponse(response); + throw new FailedRetryException( + "Service unavailable and maximum retry period elapsed: " + + ((System.currentTimeMillis() - startTime) / 1000) + + " seconds after " + retry + " retries"); + } + + checkStatus(response, status, "apply", "resource", path, + ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); + + if (as != null) { + outputBase.receiveContent(makeResult(reqlog, "apply", "resource", + response, as)); + } else { + closeResponse(response); + } + + return output; + } + + @Override + public R postBulkDocuments( + RequestLogger reqlog, DocumentWriteSet writeSet, + ServerTransform transform, Transaction transaction, Format defaultFormat, R output, + String temporalCollection, String extraContentDispositionParams) + throws ForbiddenUserException, FailedRequestException { + CharsetEncoder asciiEncoder = java.nio.charset.StandardCharsets.US_ASCII.newEncoder(); + + List writeHandles = new ArrayList(); + List headerList = new ArrayList(); + for (DocumentWriteOperation write : writeSet) { + HandleImplementation metadata = HandleAccessor.checkHandle(write.getMetadata(), "write"); + HandleImplementation content = HandleAccessor.checkHandle(write.getContent(), "write"); + + String dispositionFilename = (write.getUri() == null) ? "" : + ("; " + DISPOSITION_PARAM_FILENAME + Utilities.escapeMultipartParamAssignment(asciiEncoder, write.getUri())); + String dispositionTemporalDoc = (write.getTemporalDocumentURI() == null) ? "" : + ("; " + DISPOSITION_PARAM_TEMPORALDOC + Utilities.escapeMultipartParamAssignment(asciiEncoder, write.getTemporalDocumentURI())); + + if (write.getOperationType() == DocumentWriteOperation.OperationType.DISABLE_METADATA_DEFAULT) { + RequestParameters headers = new RequestParameters(); + headers.add(HEADER_CONTENT_TYPE, metadata.getMimetype()); + headers.add(HEADER_CONTENT_DISPOSITION, + DISPOSITION_TYPE_INLINE + "; " + DISPOSITION_PARAM_CATEGORY + "=metadata"); + headerList.add(headers); + writeHandles.add(write.getMetadata()); + } else if (metadata != null) { + RequestParameters headers = new RequestParameters(); + headers.add(HEADER_CONTENT_TYPE, metadata.getMimetype()); + if (write.getOperationType() == DocumentWriteOperation.OperationType.METADATA_DEFAULT) { + headers.add(HEADER_CONTENT_DISPOSITION, + DISPOSITION_TYPE_INLINE + "; " + DISPOSITION_PARAM_CATEGORY + "=metadata"); + } else { + headers.add(HEADER_CONTENT_DISPOSITION, + DISPOSITION_TYPE_ATTACHMENT + dispositionFilename + dispositionTemporalDoc + + "; " + DISPOSITION_PARAM_CATEGORY + "=metadata"); + } + headerList.add(headers); + writeHandles.add(write.getMetadata()); + } + + if (content != null) { + RequestParameters headers = new RequestParameters(); + String mimeType = content.getMimetype(); + if (mimeType == null && defaultFormat != null) { + mimeType = defaultFormat.getDefaultMimetype(); + } + headers.add(HEADER_CONTENT_TYPE, mimeType); + headers.add(HEADER_CONTENT_DISPOSITION, + DISPOSITION_TYPE_ATTACHMENT + dispositionFilename + dispositionTemporalDoc + + extraContentDispositionParams); + headerList.add(headers); + writeHandles.add(write.getContent()); + } + } + RequestParameters params = new RequestParameters(); + if (transform != null) { + transform.merge(params); + } + if (temporalCollection != null) params.add("temporal-collection", temporalCollection); + return postResource(reqlog, "documents", transaction, params, + (AbstractWriteHandle[]) writeHandles.toArray(new AbstractWriteHandle[0]), + (RequestParameters[]) headerList.toArray(new RequestParameters[0]), + output); + } + + public class OkHttpEvalResultIterator implements EvalResultIterator { + private OkHttpResultIterator iterator; + + OkHttpEvalResultIterator(OkHttpResultIterator iterator) { + this.iterator = iterator; + } + + @Override + public Iterator iterator() { + return this; + } + + @Override + public boolean hasNext() { + if (iterator == null) return false; + return iterator.hasNext(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public EvalResult next() { + if (iterator == null) throw new NoSuchElementException("No results available"); + OkHttpResult jerseyResult = iterator.next(); + EvalResult result = new OkHttpEvalResult(jerseyResult); + return result; + } + + @Override + public void close() { + if (iterator != null) iterator.close(); + } + } + + public class OkHttpEvalResult implements EvalResult { + private OkHttpResult content; + + public OkHttpEvalResult(OkHttpResult content) { + this.content = content; + } + + @Override + public Format getFormat() { + return content.getFormat(); + } + + @Override + public EvalResult.Type getType() { + String contentType = content.getHeader(HEADER_CONTENT_TYPE); + String xPrimitive = content.getHeader(HEADER_X_PRIMITIVE); + if (contentType != null) { + if (MIMETYPE_APPLICATION_JSON.equals(contentType)) { + if ("null-node()".equals(xPrimitive)) { + return EvalResult.Type.NULL; + } else { + return EvalResult.Type.JSON; + } + } else if (MIMETYPE_TEXT_JSON.equals(contentType)) { + return EvalResult.Type.JSON; + } else if (MIMETYPE_APPLICATION_XML.equals(contentType)) { + return EvalResult.Type.XML; + } else if (MIMETYPE_TEXT_XML.equals(contentType)) { + return EvalResult.Type.XML; + } else if ("application/x-unknown-content-type".equals(contentType) && "binary()".equals(xPrimitive)) { + return EvalResult.Type.BINARY; + } else if ("application/octet-stream".equals(contentType) && "node()".equals(xPrimitive)) { + return EvalResult.Type.BINARY; + } + } + if (xPrimitive == null) { + return EvalResult.Type.OTHER; + } else if ("string".equals(xPrimitive) || "untypedAtomic".equals(xPrimitive)) { + return EvalResult.Type.STRING; + } else if ("boolean".equals(xPrimitive)) { + return EvalResult.Type.BOOLEAN; + } else if ("attribute()".equals(xPrimitive)) { + return EvalResult.Type.ATTRIBUTE; + } else if ("comment()".equals(xPrimitive)) { + return EvalResult.Type.COMMENT; + } else if ("processing-instruction()".equals(xPrimitive)) { + return EvalResult.Type.PROCESSINGINSTRUCTION; + } else if ("text()".equals(xPrimitive)) { + return EvalResult.Type.TEXTNODE; + } else if ("binary()".equals(xPrimitive)) { + return EvalResult.Type.BINARY; + } else if ("duration".equals(xPrimitive)) { + return EvalResult.Type.DURATION; + } else if ("date".equals(xPrimitive)) { + return EvalResult.Type.DATE; + } else if ("anyURI".equals(xPrimitive)) { + return EvalResult.Type.ANYURI; + } else if ("hexBinary".equals(xPrimitive)) { + return EvalResult.Type.HEXBINARY; + } else if ("base64Binary".equals(xPrimitive)) { + return EvalResult.Type.BASE64BINARY; + } else if ("dateTime".equals(xPrimitive)) { + return EvalResult.Type.DATETIME; + } else if ("decimal".equals(xPrimitive)) { + return EvalResult.Type.DECIMAL; + } else if ("double".equals(xPrimitive)) { + return EvalResult.Type.DOUBLE; + } else if ("float".equals(xPrimitive)) { + return EvalResult.Type.FLOAT; + } else if ("gDay".equals(xPrimitive)) { + return EvalResult.Type.GDAY; + } else if ("gMonth".equals(xPrimitive)) { + return EvalResult.Type.GMONTH; + } else if ("gMonthDay".equals(xPrimitive)) { + return EvalResult.Type.GMONTHDAY; + } else if ("gYear".equals(xPrimitive)) { + return EvalResult.Type.GYEAR; + } else if ("gYearMonth".equals(xPrimitive)) { + return EvalResult.Type.GYEARMONTH; + } else if ("integer".equals(xPrimitive)) { + return EvalResult.Type.INTEGER; + } else if ("QName".equals(xPrimitive)) { + return EvalResult.Type.QNAME; + } else if ("time".equals(xPrimitive)) { + return EvalResult.Type.TIME; + } + return EvalResult.Type.OTHER; + } + + @Override + public H get(H handle) { + if (getType() == EvalResult.Type.NULL && handle instanceof StringHandle) { + return (H) ((StringHandle) handle).with(null); + } else if (getType() == EvalResult.Type.NULL && handle instanceof BytesHandle) { + return (H) ((BytesHandle) handle).with(null); + } else { + return content.getContent(handle); + } + } + + @Override + public T getAs(Class as) { + if (getType() == EvalResult.Type.NULL) return null; + if (as == null) throw new IllegalArgumentException("class cannot be null"); + + ContentHandle readHandle = DatabaseClientFactory.getHandleRegistry().makeHandle(as); + if (readHandle == null) return null; + readHandle = get(readHandle); + if (readHandle == null) return null; + return readHandle.get(); + } + + @Override + public String getString() { + if (getType() == EvalResult.Type.NULL) { + return null; + } else { + return content.getContentAs(String.class); + } + } + + @Override + public Number getNumber() { + String value = getString(); + if (value == null) return null; + if (getType() == EvalResult.Type.DECIMAL) return new BigDecimal(value); + else if (getType() == EvalResult.Type.DOUBLE) return new Double(value); + else if (getType() == EvalResult.Type.FLOAT) return new Float(value); + // MarkLogic integers can be much larger than Java integers, so we'll use Long instead + else if (getType() == EvalResult.Type.INTEGER) return new Long(value); + else return new BigDecimal(value); + } + + @Override + public Boolean getBoolean() { + // converts null to false + return Boolean.valueOf(getString()); + } + } + + @Override + public EvalResultIterator postEvalInvoke( + RequestLogger reqlog, String code, String modulePath, + ServerEvaluationCallImpl.Context context, + Map variables, EditableNamespaceContext namespaces, + Transaction transaction) + throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + String formUrlEncodedPayload; + String path; + RequestParameters params = new RequestParameters(); + try { + StringBuffer sb = new StringBuffer(); + if (context == ServerEvaluationCallImpl.Context.ADHOC_XQUERY) { + path = "eval"; + sb.append("xquery="); + sb.append(URLEncoder.encode(code, "UTF-8")); + } else if (context == ServerEvaluationCallImpl.Context.ADHOC_JAVASCRIPT) { + path = "eval"; + sb.append("javascript="); + sb.append(URLEncoder.encode(code, "UTF-8")); + } else if (context == ServerEvaluationCallImpl.Context.INVOKE) { + path = "invoke"; + sb.append("module="); + sb.append(URLEncoder.encode(modulePath, "UTF-8")); + } else { + throw new IllegalStateException("Invalid eval context: " + context); + } + if (variables != null && variables.size() > 0) { + int i = 0; + for (String name : variables.keySet()) { + String namespace = ""; + String localname = name; + if (namespaces != null) { + for (String prefix : namespaces.keySet()) { + if (name != null && prefix != null && name.startsWith(prefix + ":")) { + localname = name.substring(prefix.length() + 1); + namespace = namespaces.get(prefix); + } + } + } + // set the variable namespace + sb.append("&evn" + i + "="); + sb.append(URLEncoder.encode(namespace, "UTF-8")); + // set the variable localname + sb.append("&evl" + i + "="); + sb.append(URLEncoder.encode(localname, "UTF-8")); + + String value; + String type = null; + Object valueObject = variables.get(name); + if (valueObject == null) { + value = "null"; + type = "null-node()"; + } else if (valueObject instanceof JacksonHandle || + valueObject instanceof JacksonParserHandle) { + JsonNode jsonNode = null; + if (valueObject instanceof JacksonHandle) { + jsonNode = ((JacksonHandle) valueObject).get(); + } else if (valueObject instanceof JacksonParserHandle) { + jsonNode = ((JacksonParserHandle) valueObject).get().readValueAs(JsonNode.class); + } + value = jsonNode.toString(); + type = getJsonType(jsonNode); + } else if (valueObject instanceof AbstractWriteHandle) { + value = HandleAccessor.contentAsString((AbstractWriteHandle) valueObject); + HandleImplementation valueBase = HandleAccessor.as((AbstractWriteHandle) valueObject); + Format format = valueBase.getFormat(); + //TODO: figure out what type should be + // I see element() and document-node() are two valid types + if (format == Format.XML) { + type = "document-node()"; + } else if (format == Format.JSON) { + try (JacksonParserHandle handle = new JacksonParserHandle()) { + JsonNode jsonNode = handle.getMapper().readTree(value); + type = getJsonType(jsonNode); + } + } else if (format == Format.TEXT) { + /* Comment next line until 32608 is resolved + type = "text()"; + // until then, use the following line */ + type = "xs:untypedAtomic"; + } else if (format == Format.BINARY) { + throw new UnsupportedOperationException("Binary format is not supported for variables"); + } else { + throw new UnsupportedOperationException("Undefined format is not supported for variables. " + + "Please set the format on your handle for variable " + name + "."); + } + } else if (valueObject instanceof String || + valueObject instanceof Boolean || + valueObject instanceof Number) { + value = valueObject.toString(); + // when we send type "xs:untypedAtomic" via XDBC, the server attempts to intelligently decide + // how to cast the type + type = "xs:untypedAtomic"; + } else { + throw new IllegalArgumentException("Variable with name=" + + name + " is of unsupported type" + + valueObject.getClass() + ". Supported types are String, Boolean, Number, " + + "or AbstractWriteHandle"); + } + + // set the variable value + sb.append("&evv" + i + "="); + sb.append(URLEncoder.encode(value, "UTF-8")); + // set the variable type + sb.append("&evt" + i + "=" + type); + i++; + } + } + formUrlEncodedPayload = sb.toString(); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("UTF-8 is unsupported", e); + } catch (IOException e) { + throw new MarkLogicIOException(e); + } + StringHandle input = new StringHandle(formUrlEncodedPayload) + .withMimetype("application/x-www-form-urlencoded"); + return new OkHttpEvalResultIterator(postIteratedResourceImpl(DefaultOkHttpResultIterator::new, + reqlog, path, transaction, params, input)); + } + + private String getJsonType(JsonNode jsonNode) { + if (jsonNode instanceof ArrayNode) { + return "json:array"; + } else if (jsonNode instanceof ObjectNode) { + return "json:object"; + } else { + throw new IllegalArgumentException("When using JacksonHandle or " + + "JacksonParserHandle with ServerEvaluationCall the content must be " + + "a valid array or object"); + } + } + + @Override + public RESTServiceResultIterator postIteratedResource(RequestLogger reqlog, + String path, Transaction transaction, RequestParameters params, AbstractWriteHandle input) + throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + return postIteratedResourceImpl(OkHttpServiceResultIterator::new, + reqlog, path, transaction, params, input); + } + + public RESTServiceResultIterator postMultipartForm( + RequestLogger reqlog, String path, Transaction transaction, RequestParameters params, List contentParams) + throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + if (transaction != null) { + params.add("txid", transaction.getTransactionId()); + } + + // Don't include incoming request params, those all have to be form-data inputs + Request.Builder requestBldr = makePostWebResource(path, new RequestParameters()); + + MultipartBody.Builder multiBuilder = new MultipartBody.Builder().setType(MediaType.parse("multipart/form-data")); + for (String key : params.keySet()) { + for (String value : params.get(key)) { + if (value != null) { + multiBuilder.addFormDataPart(key, value); + } + } + } + + contentParams.forEach(contentParam -> { + String name = contentParam.getPlanParam().getName(); + multiBuilder.addFormDataPart(name, name, makeRequestBodyForContent(contentParam.getContent())); + }); + this.addContentParamAttachments(multiBuilder, contentParams); + + requestBldr = setupRequest(requestBldr, multiBuilder, null); + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + requestBldr = addTrailerHeadersIfNecessary(requestBldr, path); + + Function doPostFunction = funcBuilder -> + doPost(reqlog, funcBuilder.header(HEADER_ACCEPT, multipartMixedWithBoundary()), multiBuilder.build()); + + // The construction of this was based on whether the primary input was resendable or not. We don't have a primary + // input with a multipart request. So keeping this as null for now. + Consumer resendableConsumer = null; + + Response response = sendRequestWithRetry(requestBldr, (transaction == null), doPostFunction, resendableConsumer); + int status = response.code(); + checkStatus(response, status, "apply", "resource", path, ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); + return makeResults(OkHttpServiceResultIterator::new, reqlog, "apply", "resource", response); + } + + /** + * The REST endpoint checks for a 'metadata' parameter that, if it exists, is expected to be a JSON object with + * an 'attachment/docs' array. Each object in the array is expected to have two fields - 'rowsField' and 'column'. + * The expectation is that 'rowsField' identifies a bound parameter name that is associated with a JSON array, and + * 'column' identifies a particular column in each object in the JSON array. + *

+ * To provide support for this parameter, each object in the given {@code contentParamAttachments} array results in + * a new object in the 'rowsField' array. Then, each entry in the map of attachments in each such object is added + * as a new multipart form data part, with the map key being used as the form data part's filename. The value of + * the 'column' name in each row is then expected to be one of these map keys. + * + * @param multiBuilder + * @param contentParams + */ + private void addContentParamAttachments(MultipartBody.Builder multiBuilder, List contentParams) { + ObjectNode metadata = new ObjectMapper().createObjectNode(); + ArrayNode docsArray = metadata.putObject("attachments").putArray("docs"); + contentParams.stream().filter(contentParam -> contentParam.getColumnAttachments() != null).forEach(contentParam -> { + Map> attachments = contentParam.getColumnAttachments(); + attachments.keySet().forEach(columnName -> { + docsArray.addObject().put("rowsField", contentParam.getPlanParam().getName()).put("column", columnName); + attachments.get(columnName).keySet().forEach(filename -> { + multiBuilder.addFormDataPart(columnName, filename, makeRequestBodyForContent(attachments.get(columnName).get(filename))); + }); + }); + }); + if (docsArray.size() > 0) { + multiBuilder.addFormDataPart("metadata", "metadata", makeRequestBodyForContent(new JacksonHandle(metadata))); + } + } + + private U postIteratedResourceImpl( + ResultIteratorConstructor constructor, final RequestLogger reqlog, + final String path, Transaction transaction, RequestParameters params, + AbstractWriteHandle input) + throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + if (params == null) params = new RequestParameters(); + if (transaction != null) params.add("txid", transaction.getTransactionId()); + HandleImplementation inputBase = HandleAccessor.checkHandle(input, "write"); + + String inputMimetype = inputBase.getMimetype(); + boolean isResendable = inputBase.isResendable(); + + Request.Builder requestBldr = makePostWebResource(path, params); + requestBldr = setupRequest(requestBldr, inputMimetype, null); + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + requestBldr = addTrailerHeadersIfNecessary(requestBldr, path); + requestBldr = setErrorFormatIfNecessary(requestBldr, path); + + Consumer resendableConsumer = resendable -> { + if (!isResendable) { + checkFirstRequest(); + throw new ResourceNotResendableException( + "Cannot retry request for " + path); + } + }; + + Function doPostFunction = requestBuilder -> doPost( + reqlog, + requestBuilder.header(HEADER_ACCEPT, multipartMixedWithBoundary()), + inputBase.sendContent() + ); + + Response response = sendRequestWithRetry(requestBldr, (transaction == null), doPostFunction, resendableConsumer); + checkStatus(response, response.code(), "apply", "resource", path, ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); + return makeResults(constructor, reqlog, "apply", "resource", response); + } + + @Override + public RESTServiceResultIterator postIteratedResource( + RequestLogger reqlog, String path, Transaction transaction, RequestParameters params, + W[] input) + throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + return postIteratedResourceImpl(OkHttpServiceResultIterator::new, + reqlog, path, transaction, params, input); + } + + private U postIteratedResourceImpl( + ResultIteratorConstructor constructor, RequestLogger reqlog, String path, Transaction transaction, + RequestParameters params, W[] input) + throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { + if (params == null) params = new RequestParameters(); + if (transaction != null) params.add("txid", transaction.getTransactionId()); + Response response = null; + int status = -1; + long startTime = System.currentTimeMillis(); + int nextDelay = 0; + int retry = 0; + for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + if (nextDelay > 0) { + try { + Thread.sleep(nextDelay); + } catch (InterruptedException e) { + } + } + + MultipartBody.Builder multiPart = new MultipartBody.Builder(); + boolean hasStreamingPart = addParts(multiPart, reqlog, input); + + Request.Builder requestBldr = makePostWebResource(path, params); + requestBldr = setupRequest( + requestBldr, + multiPart, + multipartMixedWithBoundary()); + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + + response = doPost(requestBldr, multiPart, hasStreamingPart); + status = response.code(); + + if (transaction != null || !retryStatus.contains(status)) { + if (isFirstRequest()) setFirstRequest(false); + + break; + } + + String retryAfterRaw = response.header("Retry-After"); + closeResponse(response); + + if (hasStreamingPart) { + throw new ResourceNotResendableException( + "Cannot retry request for " + path); + } + + int retryAfter = Utilities.parseInt(retryAfterRaw); + nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + } + if (retryStatus.contains(status)) { + checkFirstRequest(); + closeResponse(response); + throw new FailedRetryException( + "Service unavailable and maximum retry period elapsed: " + + ((System.currentTimeMillis() - startTime) / 1000) + + " seconds after " + retry + " retries"); + } + + checkStatus(response, status, "apply", "resource", path, + ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); + + return makeResults(constructor, reqlog, "apply", "resource", response); + } + + @Override + public R deleteResource( + RequestLogger reqlog, String path, Transaction transaction, RequestParameters params, + R output) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + if (params == null) params = new RequestParameters(); + if (transaction != null) params.add("txid", transaction.getTransactionId()); + HandleImplementation outputBase = HandleAccessor.checkHandle(output, + "read"); + + String outputMimeType = null; + Class as = null; + if (outputBase != null) { + outputMimeType = outputBase.getMimetype(); + as = outputBase.receiveAs(); + } + Request.Builder requestBldr = makeDeleteWebResource(path, params); + requestBldr = setupRequest(requestBldr, null, outputMimeType); + requestBldr = addTransactionScopedCookies(requestBldr, transaction); + requestBldr = addTelemetryAgentId(requestBldr); + + Function doDeleteFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return doDelete(funcBuilder); + } + }; + Response response = sendRequestWithRetry(requestBldr, (transaction == null), doDeleteFunction, null); + int status = response.code(); + checkStatus(response, status, "delete", "resource", path, + ResponseStatus.OK_OR_NO_CONTENT); + + if (as != null) { + outputBase.receiveContent(makeResult(reqlog, "delete", "resource", + response, as)); + } else { + closeResponse(response); + } + + return output; + } + + private Request.Builder makeGetWebResource(String path, + RequestParameters params, Object mimetype) { + if (path == null) throw new IllegalArgumentException("Read with null path"); + + logger.debug(String.format("Getting %s as %s", path, mimetype)); + + return setupRequest(path, params); + } + + private Response doGet(Request.Builder requestBldr) { + requestBldr = requestBldr.get(); + Response response = sendRequestOnce(requestBldr); + + if (isFirstRequest()) setFirstRequest(false); + + return response; + } + + private Request.Builder makePutWebResource(String path, + RequestParameters params) { + if (path == null) throw new IllegalArgumentException("Write with null path"); + + logger.debug("Putting {}", path); + + return setupRequest(path, params); + } + + private Response doPut(RequestLogger reqlog, Request.Builder requestBldr, Object value) { + if (value == null) throw new IllegalArgumentException("Resource write with null value"); + + if (isFirstRequest() && isStreaming(value)) makeFirstRequest(0); + + MediaType mediaType = makeType(requestBldr.build().header(HEADER_CONTENT_TYPE)); + if (value instanceof OutputStreamSender) { + requestBldr = requestBldr.put(new StreamingOutputImpl((OutputStreamSender) value, reqlog, mediaType)); + } else { + if (reqlog != null) { + requestBldr = requestBldr.put(new ObjectRequestBody(reqlog.copyContent(value), mediaType)); + } else { + requestBldr = requestBldr.put(new ObjectRequestBody(value, mediaType)); + } + } + Response response = sendRequestOnce(requestBldr); + + if (isFirstRequest()) setFirstRequest(false); + + return response; + } + + private Response doPut(Request.Builder requestBldr, + MultipartBody.Builder multiPart, boolean hasStreamingPart) { + if (isFirstRequest() && hasStreamingPart) makeFirstRequest(0); + + requestBldr = requestBldr.put(multiPart.build()); + Response response = sendRequestOnce(requestBldr); + + if (isFirstRequest()) setFirstRequest(false); + + return response; + } + + private Request.Builder makePostWebResource(String path, RequestParameters params) { + if (path == null) throw new IllegalArgumentException("Apply with null path"); + + logger.debug("Posting {}", path); + + return setupRequest(path, params); + } + + private Response doPost(RequestLogger reqlog, Request.Builder requestBldr, Object value) { + if (isFirstRequest() && isStreaming(value)) { + makeFirstRequest(0); + } + + MediaType mediaType = makeType(requestBldr.build().header(HEADER_CONTENT_TYPE)); + if (value == null) { + requestBldr = requestBldr.post(new ObjectRequestBody(null, null)); + } else if (value instanceof MultipartBody) { + requestBldr = requestBldr.post((MultipartBody) value); + } else if (value instanceof OutputStreamSender) { + requestBldr = requestBldr + .post(new StreamingOutputImpl((OutputStreamSender) value, reqlog, mediaType)); + } else { + if (reqlog != null) { + requestBldr = requestBldr.post(new ObjectRequestBody(reqlog.copyContent(value), mediaType)); + } else { + requestBldr = requestBldr.post(new ObjectRequestBody(value, mediaType)); + } + } + Response response = sendRequestOnce(requestBldr); + + if (isFirstRequest()) setFirstRequest(false); + + return response; + } + + private Response doPost(Request.Builder requestBldr, + MultipartBody.Builder multiPart, boolean hasStreamingPart) { + if (isFirstRequest() && hasStreamingPart) makeFirstRequest(0); + + Response response = sendRequestOnce(requestBldr.post(multiPart.build())); + + if (isFirstRequest()) setFirstRequest(false); + + return response; + } + + private Request.Builder makeDeleteWebResource(String path, RequestParameters params) { + if (path == null) throw new IllegalArgumentException("Delete with null path"); + + logger.debug("Deleting {}", path); + + return setupRequest(path, params); + } + + private Response doDelete(Request.Builder requestBldr) { + Response response = sendRequestOnce(requestBldr.delete().build()); + + if (isFirstRequest()) setFirstRequest(false); + + return response; + } + + private void addPointInTimeQueryParam(RequestParameters params, Object outputHandle) { + addPointInTimeQueryParam(params, HandleAccessor.as(outputHandle)); + } + + private void addPointInTimeQueryParam(RequestParameters params, HandleImplementation handleBase) { + if (params != null && handleBase != null && handleBase.getPointInTimeQueryTimestamp() != -1) { + logger.trace("param timestamp=[" + handleBase.getPointInTimeQueryTimestamp() + "]"); + params.add("timestamp", Long.toString(handleBase.getPointInTimeQueryTimestamp())); + } + } + + private Request.Builder addTransactionScopedCookies(Request.Builder requestBldr, Transaction transaction) { + if (transaction != null && transaction.getCookies() != null) { + if (requestBldr == null) { + throw new MarkLogicInternalException("no requestBldr available to get the URI"); + } + requestBldr = addCookies( + requestBldr, transaction.getCookies(), ((TransactionImpl) transaction).getCreatedTimestamp() + ); + } + return requestBldr; + } + + private Request.Builder addCookies(Request.Builder requestBldr, List cookies, Calendar creation) { + HttpUrl uri = requestBldr.build().url(); + for (ClientCookie cookie : cookies) { + // don't forward the cookie if it requires https and we're not using https + if (cookie.isSecure() && !uri.isHttps()) { + continue; + } + // don't forward the cookie if it requires a path and we're using a different path + if (cookie.getPath() != null) { + String path = uri.encodedPath(); + if (path == null || !path.startsWith(cookie.getPath())) { + continue; + } + } + // don't forward the cookie if it requires a domain and we're using a different domain + if (cookie.getDomain() != null) { + if (uri.host() == null || !uri.host().equals(cookie.getDomain())) { + continue; + } + } + // don't forward the cookie if it has 0 for max age + if (cookie.getMaxAge() == 0) { + continue; + } + // TODO: determine if we need handling for MIN_VALUE + // else if ( cookie.getMaxAge() == Integer.MIN_VALUE ) { + // don't forward the cookie if it has a max age and we're past the max age + if (creation != null && cookie.getMaxAge() > 0) { + int currentAge = (int) TimeUnit.MILLISECONDS.toSeconds( + System.currentTimeMillis() - creation.getTimeInMillis() + ); + if (currentAge > cookie.getMaxAge()) { + logger.warn( + cookie.getName() + " cookie expired after " + cookie.getMaxAge() + " seconds: " + cookie.getValue() + ); + continue; + } + } + requestBldr = requestBldr.addHeader(HEADER_COOKIE, cookie.toString()); + } + return requestBldr; + } + + private Request.Builder addTelemetryAgentId(Request.Builder requestBldr) { + if (requestBldr == null) + throw new MarkLogicInternalException("no requestBldr available to set ML-Agent-ID header"); + return requestBldr.header("ML-Agent-ID", "java"); + } + + /** + * Per https://docs.marklogic.com/10.0/guide/relnotes/chap3#id_73268 , support for ML-Check-ML11-Headers was added + * for MarkLogic 10.0-9. It is no longer needed in MarkLogic 11 or later. The addition of it will not cause any + * harm, but it can be removed once the Java client no longer needs to support MarkLogic 10. + * + * @param requestBldr + * @param path + * @return + */ + private Request.Builder addTrailerHeadersIfNecessary(Request.Builder requestBldr, String path) { + if ("rows".equals(path)) { + requestBldr.addHeader("TE", "trailers"); + requestBldr.addHeader("ML-Check-ML11-Headers", "true"); + } + return requestBldr; + } + + private Request.Builder setErrorFormatIfNecessary(Request.Builder requestBuilder, String path) { + // Slightly dirty hack; per https://docs.marklogic.com/guide/rest-dev/intro#id_34966, the X-Error-Accept header + // should be used to specify the error format. A REST API server defaults to 'json', though the App-Services app + // server defaults to 'compatible'. If the error format is 'compatible', a block of HTML is sent back which + // causes an error that prevents the user from seeing the actual error from the server. So for all eval calls, + // X-Error-Accept is used to request any errors back as JSON so that they can be handled correctly. + if ("eval".equals(path) || ("invoke".equals(path))) { + requestBuilder.addHeader(HEADER_ERROR_FORMAT, "application/json"); + } + return requestBuilder; + } + + private boolean addParts( + MultipartBody.Builder multiPart, RequestLogger reqlog, W[] input) { + return addParts(multiPart, reqlog, null, input, null); + } + + private boolean addParts( + MultipartBody.Builder multiPart, RequestLogger reqlog, String[] mimetypes, W[] input) { + return addParts(multiPart, reqlog, null, input, null); + } + + private boolean addParts( + MultipartBody.Builder multiPart, RequestLogger reqlog, String[] mimetypes, + W[] input, Map>[] headers) { + if (mimetypes != null && mimetypes.length != input.length) { + throw new IllegalArgumentException( + "Mismatch between count of mimetypes and input"); + } + if (headers != null && headers.length != input.length) { + throw new IllegalArgumentException( + "Mismatch between count of headers and input"); + } + + multiPart.setType(MediaType.parse(MIMETYPE_MULTIPART_MIXED)); + + boolean hasStreamingPart = false; + for (int i = 0; i < input.length; i++) { + AbstractWriteHandle handle = input[i]; + HandleImplementation handleBase = HandleAccessor.checkHandle( + handle, "write"); + + if (!hasStreamingPart) { + hasStreamingPart = !handleBase.isResendable(); + } + + Object value = handleBase.sendContent(); + + String inputMimetype = null; + if (mimetypes != null) inputMimetype = mimetypes[i]; + if (inputMimetype == null && headers != null) { + inputMimetype = getHeaderMimetype(getHeader(headers[i], HEADER_CONTENT_TYPE)); + } + if (inputMimetype == null) inputMimetype = handleBase.getMimetype(); + + MediaType mediaType = (inputMimetype != null) + ? MediaType.parse(inputMimetype) + : MediaType.parse(MIMETYPE_WILDCARD); + + Headers.Builder partHeaders = new Headers.Builder(); + if (headers != null) { + for (String key : headers[i].keySet()) { + // OkHttp wants me to skip the Content-Type header + if (HEADER_CONTENT_TYPE.equalsIgnoreCase(key)) continue; + for (String headerValue : headers[i].get(key)) { + partHeaders.add(key, headerValue); + } + } + } + + Part bodyPart = null; + if (value instanceof OutputStreamSender) { + bodyPart = Part.create(partHeaders.build(), new StreamingOutputImpl( + (OutputStreamSender) value, reqlog, mediaType)); + } else { + if (reqlog != null) { + bodyPart = Part.create(partHeaders.build(), new ObjectRequestBody(reqlog.copyContent(value), mediaType)); + } else { + bodyPart = Part.create(partHeaders.build(), new ObjectRequestBody(value, mediaType)); + } + } + + multiPart = multiPart.addPart(bodyPart); + } + + return hasStreamingPart; + } + + private String multipartMixedWithBoundary() { + return MIMETYPE_MULTIPART_MIXED + "; boundary=" + UUID.randomUUID().toString(); + } + + private Request.Builder setupRequest(HttpUrl requestUri, String path, RequestParameters params) { + if (requestUri == null) throw new IllegalArgumentException("request URI cannot be null"); + if (path == null) throw new IllegalArgumentException("path cannot be null"); + if (path.startsWith("/")) path = path.substring(1); + HttpUrl.Builder uri = requestUri.resolve(path).newBuilder(); + if (params != null) { + for (String key : params.keySet()) { + for (String value : params.get(key)) { + uri.addQueryParameter(key, value); + } + } + } + if (database != null && !path.startsWith("config/")) { + uri.addQueryParameter("database", database); + } + HttpUrl httpUrl = uri.build(); + return new Request.Builder().url(httpUrl); + } + + private Request.Builder setupRequest(String path, RequestParameters params) { + return setupRequest(baseUri, path, params); + } + + private Request.Builder setupRequest(Request.Builder requestBldr, + Object inputMimetype, Object outputMimetype) { + if (inputMimetype == null) { + } else if (inputMimetype instanceof String) { + requestBldr = requestBldr.header(HEADER_CONTENT_TYPE, (String) inputMimetype); + } else if (inputMimetype instanceof MediaType) { + requestBldr = requestBldr.header(HEADER_CONTENT_TYPE, inputMimetype.toString()); + } else if (inputMimetype instanceof MultipartBody.Builder) { + requestBldr = requestBldr.header(HEADER_CONTENT_TYPE, MIMETYPE_MULTIPART_MIXED); + logger.debug("Sending multipart for {}", requestBldr.build().url().encodedPath()); + } else { + throw new IllegalArgumentException( + "Unknown input mimetype specifier " + + inputMimetype.getClass().getName()); + } + + if (outputMimetype == null) { + } else if (outputMimetype instanceof String) { + requestBldr = requestBldr.header(HEADER_ACCEPT, (String) outputMimetype); + } else if (outputMimetype instanceof MediaType) { + requestBldr = requestBldr.header(HEADER_ACCEPT, outputMimetype.toString()); + } else { + throw new IllegalArgumentException( + "Unknown output mimetype specifier " + + outputMimetype.getClass().getName()); + } + + return requestBldr; + } + + private Request.Builder setupRequest(String path, RequestParameters params, Object inputMimetype, + Object outputMimetype) { + return setupRequest(setupRequest(path, params), inputMimetype, outputMimetype); + } + + private void checkStatus(Response response, int status, String operation, String entityType, + String path, ResponseStatus expected) { + if (!expected.isExpected(status)) { + FailedRequest failure = extractErrorFields(response); + if (status == STATUS_NOT_FOUND) { + throw new ResourceNotFoundException("Could not " + operation + + " " + entityType + " at " + path, + failure); + } + if ("RESTAPI-CONTENTNOVERSION".equals(failure.getMessageCode())) { + throw new ContentNoVersionException("Content version required to " + + operation + " " + entityType + " at " + path, failure); + } else if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException("User is not allowed to " + + operation + " " + entityType + " at " + path, + failure); + } + throw new FailedRequestException("failed to " + operation + " " + + entityType + " at " + path + ": " + + getReasonPhrase(response), failure); + } + } + + private T makeResult(RequestLogger reqlog, String operation, + String entityType, Response response, Class as) { + if (as == null) { + return null; + } + + logRequest(reqlog, "%s for %s", operation, entityType); + + ResponseBody body = response.body(); + T entity = body.contentLength() != 0 ? getEntity(body, as) : null; + if (entity == null || (as != InputStream.class && as != Reader.class)) { + closeResponse(response); + } + + return (reqlog != null) ? reqlog.copyContent(entity) : entity; + } + + static private List readMultipartBodyParts(ResponseBody body) { + long length = body.contentLength(); + MimeMultipart entity = length != 0 ? getEntity(body, MimeMultipart.class) : null; + try { + if (length == -1 && entity != null) entity.getCount(); + } catch (MessagingException e) { + entity = null; + } + return getPartList(entity); + } + + private U makeResults(ResultIteratorConstructor constructor, RequestLogger reqlog, + String operation, String entityType, Response response) { + if (response == null) return null; + final List partList = readMultipartBodyParts(response.body()); + throwExceptionIfErrorInTrailers(operation, entityType, response); + return makeResults(constructor, reqlog, operation, entityType, partList, response, response); + } + + static private void throwExceptionIfErrorInTrailers(String operation, String entityType, Response response) { + String mlErrorCode = null; + String mlErrorMessage = null; + try { + Headers trailers = response.trailers(); + mlErrorCode = trailers.get("ml-error-code"); + mlErrorMessage = trailers.get("ml-error-message"); + } catch (IOException e) { + // This does not seem worthy of causing the entire operation to fail; we also don't expect this to occur, as it + // should only occur due to a programming error where the response body has already been consumed + logger.warn("Unexpected IO error while getting HTTP response trailers: " + e.getMessage()); + } + + if (mlErrorCode != null && !"N/A".equals(mlErrorCode)) { + FailedRequest failure = new FailedRequest(); + failure.setMessageString(mlErrorCode); + failure.setStatusString(mlErrorMessage); + failure.setStatusCode(500); + String message = String.format("failed to %s %s at rows: %s, %s", operation, entityType, mlErrorCode, mlErrorMessage); + throw new FailedRequestException(message, failure); + } + } + + private U makeResults( + ResultIteratorConstructor constructor, RequestLogger reqlog, + String operation, String entityType, List partList, Response response, + Closeable closeable) { + logRequest(reqlog, "%s for %s", operation, entityType); + + if (response == null) return null; + + try { + OkHttpResultIterator result = constructor.construct(reqlog, partList, closeable); + Headers headers = response.headers(); + long pageStart = Utilities.parseLong(headers.get(HEADER_VND_MARKLOGIC_START)); + if (pageStart > -1l) { + result.setStart(pageStart); + } + long pageLength = Utilities.parseLong(headers.get(HEADER_VND_MARKLOGIC_PAGELENGTH)); + if (pageLength > -1l) { + result.setPageSize(pageLength); + } + long totalSize = Utilities.parseLong(headers.get(HEADER_VND_MARKLOGIC_RESULT_ESTIMATE)); + if (totalSize > -1l) { + result.setTotalSize(totalSize); + } + return (U) result; + } catch (Throwable t) { + throw new MarkLogicInternalException("Error constructing iterator", t); + } + } + + private boolean isStreaming(Object value) { + return !(value instanceof String || value instanceof byte[] || value instanceof File); + } + + private void logRequest(RequestLogger reqlog, String message, + Object... params) { + if (reqlog == null) return; + + PrintStream out = reqlog.getPrintStream(); + if (out == null) return; + + if (params == null || params.length == 0) { + out.println(message); + } else { + out.format(message, params); + out.println(); + } + } + + private String stringJoin(Collection collection, String separator, + String defaultValue) { + if (collection == null || collection.size() == 0) return defaultValue; + + StringBuilder builder = null; + for (Object value : collection) { + if (builder == null) { + builder = new StringBuilder(); + } else { + builder.append(separator); + } + + builder.append(value); + } + + return (builder != null) ? builder.toString() : null; + } + + private int calculateDelay(Random rand, int i) { + int min = + (i > 6) ? DELAY_CEILING : + (i == 0) ? DELAY_FLOOR : + DELAY_FLOOR + (1 << i) * DELAY_MULTIPLIER; + int range = + (i > 6) ? DELAY_FLOOR : + (i == 0) ? 2 * DELAY_MULTIPLIER : + (i == 6) ? DELAY_CEILING - min : + (1 << i) * DELAY_MULTIPLIER; + return min + randRetry.nextInt(range); + } + + static class OkHttpResult { + private RequestLogger reqlog; + private BodyPart part; + private boolean extractedHeaders = false; + private String uri; + private RequestParameters headers = new RequestParameters(); + private Format format; + private String mimetype; + private long length; + + OkHttpResult(RequestLogger reqlog, BodyPart part) { + this.reqlog = reqlog; + this.part = part; + } + + public R getContent(R handle) { + if (part == null) throw new IllegalStateException("Content already retrieved"); + + HandleImplementation handleBase = HandleAccessor.as(handle); + + extractHeaders(); + updateFormat(handleBase, format); + updateMimetype(handleBase, mimetype); + updateLength(handleBase, length); + + try { + Object contentEntity = getEntity(part, handleBase.receiveAs()); + handleBase.receiveContent((reqlog != null) ? reqlog.copyContent(contentEntity) : contentEntity); + + return handle; + } finally { + part = null; + reqlog = null; + } + } + + public T getContentAs(Class as) { + ContentHandle readHandle = DatabaseClientFactory.getHandleRegistry().makeHandle(as); + readHandle = getContent(readHandle); + if (readHandle == null) return null; + return readHandle.get(); + } + + public String getUri() { + extractHeaders(); + return uri; + } + + public Format getFormat() { + extractHeaders(); + return format; + } + + public String getMimetype() { + extractHeaders(); + return mimetype; + } + + public long getLength() { + extractHeaders(); + return length; + } + + public String getHeader(String name) { + extractHeaders(); + List values = headers.get(name); + if (values != null && values.size() > 0) { + return values.get(0); + } + return null; + } + + public Map> getHeaders() { + extractHeaders(); + return headers.getMap(); + } + + private void extractHeaders() { + if (part == null || extractedHeaders) return; + try { + for (Enumeration

e = part.getAllHeaders(); e.hasMoreElements(); ) { + Header header = e.nextElement(); + headers.put(header.getName(), header.getValue()); + } + format = getHeaderFormat(part); + mimetype = getHeaderMimetype(OkHttpServices.getHeader(part, HEADER_CONTENT_TYPE)); + length = getHeaderLength(OkHttpServices.getHeader(part, HEADER_CONTENT_LENGTH)); + uri = getHeaderUri(part); + extractedHeaders = true; + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } + } + } + + static class OkHttpServiceResult extends OkHttpResult implements RESTServices.RESTServiceResult { + OkHttpServiceResult(RequestLogger reqlog, BodyPart part) { + super(reqlog, part); + } + } + + static abstract class OkHttpResultIterator { + private RequestLogger reqlog; + private Iterator partQueue; + private long start = -1; + private long size = -1; + private long pageSize = -1; + private long totalSize = -1; + private Closeable closeable; + + OkHttpResultIterator(RequestLogger reqlog, List partList, Closeable closeable) { + this.reqlog = reqlog; + if (partList != null && partList.size() > 0) { + this.size = partList.size(); + this.partQueue = new ConcurrentLinkedQueue<>( + partList).iterator(); + } else { + this.size = 0; + } + this.closeable = closeable; + } + + public long getStart() { + return start; + } + + public OkHttpResultIterator setStart(long start) { + this.start = start; + return this; + } + + public long getSize() { + return size; + } + + public OkHttpResultIterator setSize(long size) { + this.size = size; + return this; + } + + public long getPageSize() { + return pageSize; + } + + public OkHttpResultIterator setPageSize(long pageSize) { + this.pageSize = pageSize; + return this; + } + + public long getTotalSize() { + return totalSize; + } + + public OkHttpResultIterator setTotalSize(long totalSize) { + this.totalSize = totalSize; + return this; + } + + public boolean hasNext() { + if (partQueue == null) return false; + boolean hasNext = partQueue.hasNext(); + return hasNext; + } + + public T next() { + if (partQueue == null) return null; + + try { + return constructNext(reqlog, partQueue.next()); + } catch (Throwable t) { + throw new IllegalStateException("Error instantiating iterated result", t); + } + } + + abstract T constructNext(RequestLogger logger, BodyPart part); + + public void remove() { + if (partQueue == null) return; + partQueue.remove(); + if (!partQueue.hasNext()) close(); + } + + public void close() { + partQueue = null; + reqlog = null; + if (closeable != null) { + try { + closeable.close(); + } catch (IOException e) { + throw new MarkLogicIOException(e); + } + } + } + } + + static class OkHttpServiceResultIterator + extends OkHttpResultIterator + implements RESTServiceResultIterator { + OkHttpServiceResultIterator(RequestLogger reqlog, + List partList, Closeable closeable) { + super(reqlog, partList, closeable); + } + + OkHttpServiceResult constructNext(RequestLogger logger, BodyPart part) { + return new OkHttpServiceResult(logger, part); + } + } + + static class DefaultOkHttpResultIterator + extends OkHttpResultIterator + implements Iterator { + DefaultOkHttpResultIterator(RequestLogger reqlog, + List partList, Closeable closeable) { + super(reqlog, partList, closeable); + } + + OkHttpResult constructNext(RequestLogger logger, BodyPart part) { + return new OkHttpResult(logger, part); + } + } + + static class OkHttpDocumentRecord implements DocumentRecord { + private OkHttpResult content; + private OkHttpResult metadata; + + OkHttpDocumentRecord(OkHttpResult content, OkHttpResult metadata) { + this.content = content; + this.metadata = metadata; + } + + OkHttpDocumentRecord(OkHttpResult content) { + this.content = content; + } + + @Override + public String getUri() { + if (content == null && metadata != null) { + return metadata.getUri(); + } else if (content != null) { + return content.getUri(); + } else { + throw new IllegalStateException("Missing both content and metadata!"); + } + } + + @Override + public DocumentDescriptor getDescriptor() { + if (content == null) { + throw new IllegalStateException("getDescriptor() called when no content is available"); + } + DocumentDescriptorImpl descriptor = new DocumentDescriptorImpl(getUri(), false); + updateFormat(descriptor, getFormat()); + updateMimetype(descriptor, getMimetype()); + updateLength(descriptor, getLength()); + updateVersion(descriptor, content.getHeader(HEADER_ETAG)); + return descriptor; + } + + @Override + public Format getFormat() { + if (content == null) { + throw new IllegalStateException("getFormat() called when no content is available"); + } + return content.getFormat(); + } + + @Override + public String getMimetype() { + if (content == null) { + throw new IllegalStateException("getMimetype() called when no content is available"); + } + return content.getMimetype(); + } + + @Override + public long getLength() { + if (content == null) { + throw new IllegalStateException("getLenth() called when no content is available"); + } + return content.getLength(); + } + + @Override + public T getMetadata(T metadataHandle) { + if (metadata == null) { + throw new IllegalStateException("getMetadata called when no metadata is available"); + } + return metadata.getContent(metadataHandle); + } + + @Override + public T getMetadataAs(Class as) { + if (as == null) { + throw new IllegalStateException("getMetadataAs cannot accept null"); + } + return metadata.getContentAs(as); + } + + @Override + public T getContent(T contentHandle) { + if (content == null) { + throw new IllegalStateException("getContent called when no content is available"); + } + return content.getContent(contentHandle); + } + + @Override + public T getContentAs(Class as) { + if (as == null) { + throw new IllegalStateException("getContentAs cannot accept null"); + } + return content.getContentAs(as); + } + } + + @Override + public OkHttpClient getClientImplementation() { + if (client == null) return null; + return client; + } + + public void setClientImplementation(OkHttpClient client) { + this.client = client; + } + + @Override + public T suggest(Class as, SuggestDefinition suggestionDef) { + RequestParameters params = new RequestParameters(); + + String suggestCriteria = suggestionDef.getStringCriteria(); + String[] queries = suggestionDef.getQueryStrings(); + String optionsName = suggestionDef.getOptionsName(); + Integer limit = suggestionDef.getLimit(); + Integer cursorPosition = suggestionDef.getCursorPosition(); + + if (suggestCriteria != null) { + params.add("partial-q", suggestCriteria); + } + if (optionsName != null) { + params.add("options", optionsName); + } + if (limit != null) { + params.add("limit", Long.toString(limit)); + } + if (cursorPosition != null) { + params.add("cursor-position", Long.toString(cursorPosition)); + } + if (queries != null) { + for (String stringQuery : queries) { + params.add("q", stringQuery); + } + } + Request.Builder requestBldr = null; + requestBldr = setupRequest("suggest", params, null, MIMETYPE_APPLICATION_XML); + requestBldr = addTelemetryAgentId(requestBldr); + + Function doGetFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return sendRequestOnce(funcBuilder.get().build()); + } + }; + Response response = sendRequestWithRetry(requestBldr, doGetFunction, null); + int status = response.code(); + if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException( + "User is not allowed to get suggestions", + extractErrorFields(response)); + } + if (status != STATUS_OK) { + throw new FailedRequestException("Suggest call failed: " + + getReasonPhrase(response), extractErrorFields(response)); + } + + ResponseBody body = response.body(); + T entity = body.contentLength() != 0 ? getEntity(body, as) : null; + if (entity == null || (as != InputStream.class && as != Reader.class)) { + closeResponse(response); + } + + return entity; + } + + @Override + public InputStream match(StructureWriteHandle document, + String[] candidateRules, String mimeType, ServerTransform transform) { + RequestParameters params = new RequestParameters(); + + HandleImplementation baseHandle = HandleAccessor.checkHandle(document, "match"); + if (candidateRules != null) { + for (String candidateRule : candidateRules) { + params.add("rule", candidateRule); + } + } + if (transform != null) { + transform.merge(params); + } + Request.Builder requestBldr = null; + requestBldr = setupRequest("alert/match", params, MIMETYPE_APPLICATION_XML, mimeType); + requestBldr = addTelemetryAgentId(requestBldr); + + Function doPostFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return doPost(null, funcBuilder, baseHandle.sendContent()); + } + }; + Response response = sendRequestWithRetry(requestBldr, doPostFunction, null); + int status = response.code(); + + if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException("User is not allowed to match", + extractErrorFields(response)); + } + if (status != STATUS_OK) { + throw new FailedRequestException("match failed: " + + getReasonPhrase(response), extractErrorFields(response)); + } + + ResponseBody body = response.body(); + InputStream entity = body.contentLength() != 0 ? + getEntity(body, InputStream.class) : null; + if (entity == null) closeResponse(response); + + return entity; + } + + @Override + public InputStream match(QueryDefinition queryDef, + long start, long pageLength, String[] candidateRules, ServerTransform transform) { + if (queryDef == null) { + throw new IllegalArgumentException("Cannot match null query"); + } + + RequestParameters params = new RequestParameters(); + + if (start > 1) { + params.add("start", Long.toString(start)); + } + if (pageLength >= 0) { + params.add("pageLength", Long.toString(pageLength)); + } + if (transform != null) { + transform.merge(params); + } + if (candidateRules.length > 0) { + for (String candidateRule : candidateRules) { + params.add("rule", candidateRule); + } + } + + if (queryDef.getOptionsName() != null) { + params.add("options", queryDef.getOptionsName()); + } + + Request.Builder requestBldr = null; + String structure = null; + HandleImplementation baseHandle = null; + + String text = null; + if (queryDef instanceof StringQueryDefinition) { + text = ((StringQueryDefinition) queryDef).getCriteria(); + } else if (queryDef instanceof StructuredQueryDefinition) { + text = ((StructuredQueryDefinition) queryDef).getCriteria(); + } else if (queryDef instanceof RawStructuredQueryDefinition) { + text = ((RawStructuredQueryDefinition) queryDef).getCriteria(); + } + if (text != null) { + params.add("q", text); + } + if (queryDef instanceof StructuredQueryDefinition) { + structure = ((StructuredQueryDefinition) queryDef).serialize(); + + logger.debug("Searching with structured query {}", structure); + + requestBldr = setupRequest("alert/match", params, MIMETYPE_APPLICATION_XML, MIMETYPE_APPLICATION_XML); + } else if (queryDef instanceof RawQueryDefinition) { + StructureWriteHandle handle = ((RawQueryDefinition) queryDef).getHandle(); + baseHandle = HandleAccessor.checkHandle(handle, "match"); + + logger.debug("Searching with raw query"); + + requestBldr = setupRequest("alert/match", params, MIMETYPE_APPLICATION_XML, MIMETYPE_APPLICATION_XML); + } else if (queryDef instanceof StringQueryDefinition) { + logger.debug("Searching with string query [{}]", text); + + requestBldr = setupRequest("alert/match", params, null, MIMETYPE_APPLICATION_XML); + } else { + throw new UnsupportedOperationException("Cannot match with " + + queryDef.getClass().getName()); + } + requestBldr = addTelemetryAgentId(requestBldr); + + MediaType mediaType = makeType(requestBldr.build().header(HEADER_CONTENT_TYPE)); + + Response response = null; + int status = -1; + long startTime = System.currentTimeMillis(); + int nextDelay = 0; + int retry = 0; + for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + if (nextDelay > 0) { + try { + Thread.sleep(nextDelay); + } catch (InterruptedException e) { + } + } + + if (queryDef instanceof StructuredQueryDefinition) { + response = doPost(null, requestBldr, structure); + } else if (queryDef instanceof RawQueryDefinition) { + response = doPost(null, requestBldr, baseHandle.sendContent()); + } else if (queryDef instanceof StringQueryDefinition) { + response = sendRequestOnce(requestBldr.get()); + } else { + throw new UnsupportedOperationException("Cannot match with " + + queryDef.getClass().getName()); + } + status = response.code(); + + if (!retryStatus.contains(status)) { + if (isFirstRequest()) setFirstRequest(false); + + break; + } + + String retryAfterRaw = response.header("Retry-After"); + int retryAfter = Utilities.parseInt(retryAfterRaw); + + closeResponse(response); + + nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + } + if (retryStatus.contains(status)) { + checkFirstRequest(); + closeResponse(response); + throw new FailedRetryException( + "Service unavailable and maximum retry period elapsed: " + + ((System.currentTimeMillis() - startTime) / 1000) + + " seconds after " + retry + " retries"); + } + if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException("User is not allowed to match", + extractErrorFields(response)); + } + if (status != STATUS_OK) { + throw new FailedRequestException("match failed: " + + getReasonPhrase(response), extractErrorFields(response)); + } + + ResponseBody body = response.body(); + InputStream entity = body.contentLength() != 0 ? + getEntity(body, InputStream.class) : null; + if (entity == null) closeResponse(response); + + return entity; + } + + @Override + public InputStream match(String[] docIds, String[] candidateRules, ServerTransform transform) { + RequestParameters params = new RequestParameters(); + + if (docIds.length > 0) { + for (String docId : docIds) { + params.add("uri", docId); + } + } + if (candidateRules.length > 0) { + for (String candidateRule : candidateRules) { + params.add("rule", candidateRule); + } + } + if (transform != null) { + transform.merge(params); + } + Request.Builder requestBldr = setupRequest("alert/match", params, MIMETYPE_APPLICATION_XML, MIMETYPE_APPLICATION_XML); + requestBldr = addTelemetryAgentId(requestBldr); + + Function doGetFunction = new Function() { + public Response apply(Request.Builder funcBuilder) { + return doGet(funcBuilder); + } + }; + Response response = sendRequestWithRetry(requestBldr, doGetFunction, null); + int status = response.code(); + if (status == STATUS_FORBIDDEN) { + throw new ForbiddenUserException("User is not allowed to match", + extractErrorFields(response)); + } + if (status != STATUS_OK) { + throw new FailedRequestException("match failed: " + + getReasonPhrase(response), extractErrorFields(response)); + } + + ResponseBody body = response.body(); + InputStream entity = body.contentLength() != 0 ? + getEntity(body, InputStream.class) : null; + if (entity == null) closeResponse(response); + + return entity; + } + + private void addGraphUriParam(RequestParameters params, String uri) { + if (uri == null || uri.equals(GraphManager.DEFAULT_GRAPH)) { + params.add("default", ""); + } else { + params.add("graph", uri); + } + } + + private void addPermsParams(RequestParameters params, GraphPermissions permissions) { + if (permissions != null) { + for (Map.Entry> entry : permissions.entrySet()) { + if (entry.getValue() != null) { + for (Capability capability : entry.getValue()) { + params.add("perm:" + entry.getKey(), capability.toString().toLowerCase()); + } + } + } + } + } + + @Override + public R getGraphUris(RequestLogger reqlog, R output) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + return getResource(reqlog, "graphs", null, null, output); + } + + @Override + public R readGraph(RequestLogger reqlog, String uri, R output, + Transaction transaction) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + RequestParameters params = new RequestParameters(); + addGraphUriParam(params, uri); + return getResource(reqlog, "graphs", transaction, params, output); + } + + @Override + public void writeGraph(RequestLogger reqlog, String uri, + AbstractWriteHandle input, GraphPermissions permissions, Transaction transaction) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + RequestParameters params = new RequestParameters(); + addGraphUriParam(params, uri); + addPermsParams(params, permissions); + putResource(reqlog, "graphs", transaction, params, input, null); + } + + @Override + public void writeGraphs(RequestLogger reqlog, AbstractWriteHandle input, Transaction transaction) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + RequestParameters params = new RequestParameters(); + putResource(reqlog, "graphs", transaction, params, input, null); + } + + @Override + public void mergeGraph(RequestLogger reqlog, String uri, + AbstractWriteHandle input, GraphPermissions permissions, Transaction transaction) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + RequestParameters params = new RequestParameters(); + addGraphUriParam(params, uri); + addPermsParams(params, permissions); + postResource(reqlog, "graphs", transaction, params, input, null); + } + + @Override + public void mergeGraphs(RequestLogger reqlog, AbstractWriteHandle input, Transaction transaction) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + RequestParameters params = new RequestParameters(); + postResource(reqlog, "graphs", transaction, params, input, null); + } + + @Override + public R getPermissions(RequestLogger reqlog, String uri, + R output, Transaction transaction) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + RequestParameters params = new RequestParameters(); + addGraphUriParam(params, uri); + params.add("category", "permissions"); + return getResource(reqlog, "graphs", transaction, params, output); + } + + @Override + public void deletePermissions(RequestLogger reqlog, String uri, Transaction transaction) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + RequestParameters params = new RequestParameters(); + addGraphUriParam(params, uri); + params.add("category", "permissions"); + deleteResource(reqlog, "graphs", transaction, params, null); + } + + @Override + public void writePermissions(RequestLogger reqlog, String uri, + AbstractWriteHandle permissions, Transaction transaction) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + RequestParameters params = new RequestParameters(); + addGraphUriParam(params, uri); + params.add("category", "permissions"); + putResource(reqlog, "graphs", transaction, params, permissions, null); + } + + @Override + public void mergePermissions(RequestLogger reqlog, String uri, + AbstractWriteHandle permissions, Transaction transaction) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + RequestParameters params = new RequestParameters(); + addGraphUriParam(params, uri); + params.add("category", "permissions"); + postResource(reqlog, "graphs", transaction, params, permissions, null); + } + + @Override + public Object deleteGraph(RequestLogger reqlog, String uri, Transaction transaction) + throws ForbiddenUserException, FailedRequestException { + RequestParameters params = new RequestParameters(); + addGraphUriParam(params, uri); + return deleteResource(reqlog, "graphs", transaction, params, null); + + } + + @Override + public void deleteGraphs(RequestLogger reqlog, Transaction transaction) + throws ForbiddenUserException, FailedRequestException { + deleteResource(reqlog, "graphs", transaction, null, null); + } + + @Override + public R getThings(RequestLogger reqlog, String[] iris, R output) + throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { + if (iris == null) throw new IllegalArgumentException("iris cannot be null"); + RequestParameters params = new RequestParameters(); + for (String iri : iris) { + params.add("iri", iri); + } + return getResource(reqlog, "graphs/things", null, params, output); + } + + @Override + public R executeSparql(RequestLogger reqlog, + SPARQLQueryDefinition qdef, R output, long start, long pageLength, + Transaction transaction, boolean isUpdate) { + if (qdef == null) throw new IllegalArgumentException("qdef cannot be null"); + if (output == null) throw new IllegalArgumentException("output cannot be null"); + RequestParameters params = new RequestParameters(); + if (start > 1) params.add("start", Long.toString(start)); + if (pageLength >= 0) params.add("pageLength", Long.toString(pageLength)); + if (qdef.getOptimizeLevel() >= 0) { + params.add("optimize", Integer.toString(qdef.getOptimizeLevel())); + } + if (qdef.getCollections() != null) { + for (String collection : qdef.getCollections()) { + params.add("collection", collection); + } + } + addPermsParams(params, qdef.getUpdatePermissions()); + String sparql = qdef.getSparql(); + SPARQLBindings bindings = qdef.getBindings(); + for (Map.Entry> entry : bindings.entrySet()) { + String paramName = "bind:" + entry.getKey(); + String typeOrLang = ""; + for (SPARQLBinding binding : entry.getValue()) { + if (binding.getDatatype() != null) { + typeOrLang = ":" + binding.getDatatype(); + } else if (binding.getLanguageTag() != null) { + typeOrLang = "@" + binding.getLanguageTag().toLanguageTag(); + } + params.add(paramName + typeOrLang, binding.getValue()); + } + } + QueryDefinition constrainingQuery = qdef.getConstrainingQueryDefinition(); + StructureWriteHandle input; + if (constrainingQuery != null) { + if (qdef.getOptionsName() != null && qdef.getOptionsName().length() > 0) { + params.add("options", qdef.getOptionsName()); + } + if (constrainingQuery instanceof RawCombinedQueryDefinition) { + CombinedQueryDefinition combinedQdef = new CombinedQueryBuilderImpl().combine( + (RawCombinedQueryDefinition) constrainingQuery, null, null, sparql); + Format format = combinedQdef.getFormat(); + input = new StringHandle(combinedQdef.serialize()).withFormat(format); + } else if (constrainingQuery instanceof RawStructuredQueryDefinition) { + CombinedQueryDefinition combinedQdef = new CombinedQueryBuilderImpl().combine( + (RawStructuredQueryDefinition) constrainingQuery, null, null, sparql); + Format format = combinedQdef.getFormat(); + input = new StringHandle(combinedQdef.serialize()).withFormat(format); + } else if (constrainingQuery instanceof StringQueryDefinition || + constrainingQuery instanceof StructuredQueryDefinition) { + String stringQuery = constrainingQuery instanceof StringQueryDefinition ? + ((StringQueryDefinition) constrainingQuery).getCriteria() : null; + StructuredQueryDefinition structuredQuery = + constrainingQuery instanceof StructuredQueryDefinition ? + (StructuredQueryDefinition) constrainingQuery : null; + CombinedQueryDefinition combinedQdef = new CombinedQueryBuilderImpl().combine( + structuredQuery, null, stringQuery, sparql); + input = new StringHandle(combinedQdef.serialize()).withMimetype(MIMETYPE_APPLICATION_XML); + } else { + throw new IllegalArgumentException( + "Constraining query must be of type SPARQLConstrainingQueryDefinition"); + } + } else { + String mimetype = isUpdate ? "application/sparql-update" : "application/sparql-query"; + input = new StringHandle(sparql).withMimetype(mimetype); + } + if (qdef.getBaseUri() != null) { + params.add("base", qdef.getBaseUri()); + } + if (qdef.getDefaultGraphUris() != null) { + for (String defaultGraphUri : qdef.getDefaultGraphUris()) { + params.add("default-graph-uri", defaultGraphUri); + } + } + if (qdef.getNamedGraphUris() != null) { + for (String namedGraphUri : qdef.getNamedGraphUris()) { + params.add("named-graph-uri", namedGraphUri); + } + } + if (qdef.getUsingGraphUris() != null) { + for (String usingGraphUri : qdef.getUsingGraphUris()) { + params.add("using-graph-uri", usingGraphUri); + } + } + if (qdef.getUsingNamedGraphUris() != null) { + for (String usingNamedGraphUri : qdef.getUsingNamedGraphUris()) { + params.add("using-named-graph-uri", usingNamedGraphUri); + } + } + + // rulesets + if (qdef.getRulesets() != null) { + for (SPARQLRuleset ruleset : qdef.getRulesets()) { + params.add("ruleset", ruleset.getName()); + } + } + if (qdef.getIncludeDefaultRulesets() != null) { + params.add("default-rulesets", qdef.getIncludeDefaultRulesets() ? "include" : "exclude"); + } + + return postResource(reqlog, "/graphs/sparql", transaction, params, input, output); + } + + static private String getTransactionId(Transaction transaction) { + if (transaction == null) return null; + return transaction.getTransactionId(); + } + + static private String getReasonPhrase(Response response) { + if (response == null || response.message() == null) return ""; + // strip off the number part of the reason phrase + return response.message().replaceFirst("^\\d+ ", ""); + } + + static private T getEntity(BodyPart part, Class as) { + try { + String contentType = part.getContentType(); + return getEntity( + ResponseBody.create(Okio.buffer(Okio.source(part.getInputStream())), MediaType.parse(contentType), part.getSize()), + as); + } catch (IOException e) { + throw new MarkLogicIOException(e); + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } + } + + static private MediaType makeType(String mimetype) { + if (mimetype == null) return null; + MediaType type = MediaType.parse(mimetype); + if (type == null) throw new IllegalArgumentException("Invalid mime-type: " + mimetype); + return type; + } + + static private T getEntity(ResponseBody body, Class as) { + try { + if (as == InputStream.class) { + return (T) body.byteStream(); + } else if (as == byte[].class) { + return (T) body.bytes(); + } else if (as == Reader.class) { + return (T) body.charStream(); + } else if (as == String.class) { + return (T) body.string(); + } else if (as == MimeMultipart.class) { + MediaType mediaType = body.contentType(); + String contentType = (mediaType != null) ? mediaType.toString() : "application/x-unknown-content-type"; + ByteArrayDataSource dataSource = new ByteArrayDataSource(body.byteStream(), contentType); + return (T) new MimeMultipart(dataSource); + } else if (as == File.class) { + // write out the response body to a temp file in the system temp folder + // then return the path to that file as a File object + String suffix = ".unknown"; + boolean isBinary = true; + MediaType mediaType = body.contentType(); + if (mediaType != null) { + String subtype = mediaType.subtype(); + if (subtype != null) { + subtype = subtype.toLowerCase(); + if (subtype.endsWith("json")) { + suffix = ".json"; + isBinary = false; + } else if (subtype.endsWith("xml")) { + suffix = ".xml"; + isBinary = false; + } else if (subtype.equals("vnd.marklogic-js-module")) { + suffix = ".mjs"; + isBinary = false; + } else if (subtype.equals("vnd.marklogic-javascript")) { + suffix = ".sjs"; + isBinary = false; + } else if (subtype.equals("vnd.marklogic-xdmp") || subtype.endsWith("xquery")) { + suffix = ".xqy"; + isBinary = false; + } else if (subtype.endsWith("javascript")) { + suffix = ".js"; + isBinary = false; + } else if (subtype.endsWith("html")) { + suffix = ".html"; + isBinary = false; + } else if (mediaType.type().equalsIgnoreCase("text")) { + suffix = ".txt"; + isBinary = false; + } else { + suffix = "." + subtype; + } + } + } + Path path = Files.createTempFile("tmp", suffix); + if (isBinary == true) { + Files.copy(body.byteStream(), path, StandardCopyOption.REPLACE_EXISTING); + } else { + try (Writer out = Files.newBufferedWriter(path, Charset.forName("UTF-8"))) { + Utilities.write(body.charStream(), out); + } + } + return (T) path.toFile(); + } else { + throw new IllegalArgumentException( + "Handle recieveAs returned " + as + " which is not a supported type. " + + "Try InputStream, Reader, String, byte[], File."); + } + } catch (IOException e) { + throw new MarkLogicIOException(e); + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } + } + + static private List getPartList(MimeMultipart multipart) { + try { + if (multipart == null) return null; + List partList = new ArrayList(); + for (int i = 0; i < multipart.getCount(); i++) { + partList.add(multipart.getBodyPart(i)); + } + return partList; + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } + } + + static private class ObjectRequestBody extends RequestBody { + private Object obj; + private MediaType contentType; + + ObjectRequestBody(Object obj, MediaType contentType) { + super(); + this.obj = obj; + this.contentType = contentType; + } + + @Override + public MediaType contentType() { + return contentType; + } + + @Override + public void writeTo(BufferedSink sink) throws IOException { + if (obj instanceof InputStream) { + sink.writeAll(Okio.source((InputStream) obj)); + } else if (obj instanceof File) { + try (Source source = Okio.source((File) obj)) { + sink.writeAll(source); + } + } else if (obj instanceof byte[]) { + sink.write((byte[]) obj); + } else if (obj instanceof String) { + sink.write(((String) obj).getBytes(StandardCharsets.UTF_8)); + } else if (obj == null) { + } else { + throw new IllegalStateException("Cannot write object of type: " + obj.getClass()); + } + } + } + + // API First Changes + static private class EmptyRequestBody extends RequestBody { + @Override + public MediaType contentType() { + return null; + } + + @Override + public void writeTo(BufferedSink sink) { + } + } + + static class AtomicRequestBody extends RequestBody { + private MediaType contentType; + private String value; + + AtomicRequestBody(String value, MediaType contentType) { + super(); + this.value = value; + this.contentType = contentType; + } + + @Override + public MediaType contentType() { + return contentType; + } + + @Override + public void writeTo(BufferedSink sink) throws IOException { + sink.writeUtf8(value); + } + } + + public static RequestBody makeRequestBodyForContent(AbstractWriteHandle content) { + if (content == null) { + return new EmptyRequestBody(); + } + HandleImplementation handleBase = HandleAccessor.as(content); + Format format = handleBase.getFormat(); + String mimetype = (format == Format.BINARY) ? null : handleBase.getMimetype(); + MediaType mediaType = MediaType.parse( + (mimetype != null) ? mimetype : "application/x-unknown-content-type" + ); + return (content instanceof OutputStreamSender) ? + new StreamingOutputImpl((OutputStreamSender) content, null, mediaType) : + new ObjectRequestBody(HandleAccessor.sendContent(content), mediaType); + } + + class CallRequestImpl implements CallRequest { + private SessionStateImpl session; + private Request.Builder requestBldr; + private RequestBody requestBody; + private boolean hasStreamingPart; + private HttpMethod method; + private String endpoint; + private HttpUrl callBaseUri; + + CallRequestImpl(String endpoint, HttpMethod method, SessionState session) { + if (session != null && !(session instanceof SessionStateImpl)) { + throw new IllegalArgumentException("Session state must be implemented by internal class: " + session.getClass().getName()); + } + this.endpoint = endpoint; + this.method = method; + this.session = (SessionStateImpl) session; + this.hasStreamingPart = false; + this.callBaseUri = HttpUrlBuilder.newDataServicesBaseUri(baseUri); + } + + @Override + public CallResponse withEmptyResponse() { + prepareRequestBuilder(); + CallResponseImpl responseImpl = new CallResponseImpl(); + executeRequest(responseImpl); + return responseImpl; + } + + @Override + public SingleCallResponse withDocumentResponse(Format format) { + prepareRequestBuilder(); + SingleCallResponseImpl responseImpl = new SingleCallResponseImpl(format); + this.requestBldr = forDocumentResponse(requestBldr, format); + executeRequest(responseImpl); + return responseImpl; + } + + @Override + public MultipleCallResponse withMultipartMixedResponse(Format format) { + prepareRequestBuilder(); + MultipleCallResponseImpl responseImpl = new MultipleCallResponseImpl(format); + this.requestBldr = forMultipartMixedResponse(requestBldr); + executeRequest(responseImpl); + return responseImpl; + } + + @Override + public boolean hasStreamingPart() { + return this.hasStreamingPart; + } + + @Override + public SessionState getSession() { + return this.session; + } + + @Override + public String getEndpoint() { + return this.endpoint; + } + + @Override + public HttpMethod getHttpMethod() { + return this.method; + } + + private void prepareRequestBuilder() { + this.requestBldr = setupRequest(callBaseUri, endpoint, null); + if (session != null) { + this.requestBldr = addCookies(this.requestBldr, session.getCookies(), session.getCreatedTimestamp()); + // Add the Cookie header for SessionId if we have a session object passed + this.requestBldr.addHeader(HEADER_COOKIE, "SessionID=" + session.getSessionId()); + } + addHttpMethod(); + this.requestBldr.addHeader(HEADER_ERROR_FORMAT, MIMETYPE_APPLICATION_JSON); + } + + private void addHttpMethod() { + if (method != null && method == HttpMethod.POST) { + if (requestBody == null) { + throw new IllegalStateException("Request Body is null!"); + } + this.requestBldr.post(requestBody); + } else { + throw new IllegalStateException("HTTP method is null or invalid!"); + } + } + + private void executeRequest(CallResponseImpl responseImpl) { + SessionState session = getSession(); + //TODO: Add a telemetry agent if needed + // requestBuilder = addTelemetryAgentId(requestBuilder); + + boolean hasStreamingPart = hasStreamingPart(); + Consumer resendableConsumer = resendable -> { + if (hasStreamingPart) { + checkFirstRequest(); + throw new ResourceNotResendableException( + "Cannot retry request for " + getEndpoint()); + } + }; + + Function sendRequestFunction = requestBldr -> { + if (isFirstRequest() && hasStreamingPart) makeFirstRequest(callBaseUri, "", 0); + Response response = sendRequestOnce(requestBldr); + if (isFirstRequest()) setFirstRequest(false); + return response; + }; + + Response response = sendRequestWithRetry(requestBldr, sendRequestFunction, resendableConsumer); + + if (session != null) { + List cookies = new ArrayList<>(); + for (String setCookie : response.headers(HEADER_SET_COOKIE)) { + ClientCookie cookie = ClientCookie.parse(requestBldr.build().url(), setCookie); + cookies.add(cookie); + } + ((SessionStateImpl) session).setCookies(cookies); + } + checkStatus(response); + responseImpl.setResponse(response); + } + + private void checkStatus(Response response) { + int statusCode = response.code(); + if (statusCode >= 300) { + FailedRequest failure = null; + String contentType = response.header(HEADER_CONTENT_TYPE); + MediaType mediaType = MediaType.parse( + (contentType != null) ? contentType : "application/x-unknown-content-type" + ); + String subtype = (mediaType != null) ? mediaType.subtype() : null; + if (subtype != null) { + subtype = subtype.toLowerCase(); + if (subtype.endsWith("json") || subtype.endsWith("xml")) { + failure = extractErrorFields(response); + } + } + if (failure == null) { + closeResponse(response); + if (statusCode == STATUS_UNAUTHORIZED) { + failure = new FailedRequest(); + failure.setMessageString("Unauthorized"); + failure.setStatusString("Failed Auth"); + } else if (statusCode == STATUS_NOT_FOUND) { + throw new ResourceNotFoundException("Could not " + method + " at " + endpoint); + } else if (statusCode == STATUS_FORBIDDEN) { + throw new ForbiddenUserException("User is not allowed to " + method + " at " + endpoint); + } else { + failure = new FailedRequest(); + failure.setStatusCode(statusCode); + failure.setMessageCode("UNKNOWN"); + failure.setMessageString("Server did not respond with an expected Error message."); + failure.setStatusString("UNKNOWN"); + } + } + FailedRequestException ex = failure == null ? new FailedRequestException("failed to " + method + " at " + endpoint + ": " + + getReasonPhrase(response)) : new FailedRequestException("failed to " + method + " at " + endpoint + ": " + + getReasonPhrase(response), failure); + throw ex; + } + } + + public CallRequest withEmptyRequest() { + requestBody = new EmptyRequestBody(); + return this; + } + + public CallRequest withAtomicBodyRequest(CallField... params) { + String atomics = Stream.of(params) + .map(param -> encodeParamValue(param)) + .filter(param -> param != null) + .collect(Collectors.joining("&")); + requestBody = RequestBody.create((atomics == null) ? "" : atomics, URLENCODED_MIME_TYPE); + return this; + } + + public CallRequest withNodeBodyRequest(CallField... params) { + this.requestBody = makeRequestBody(params); + return this; + } + + private RequestBody makeRequestBody(String value) { + if (value == null) { + return new EmptyRequestBody(); + } + return new AtomicRequestBody(value, MediaType.parse("text/plain")); + } + + private RequestBody makeRequestBody(AbstractWriteHandle document) { + if (document == null) { + return new EmptyRequestBody(); + } + HandleImplementation handleBase = HandleAccessor.as(document); + Format format = handleBase.getFormat(); + String mimetype = (format == Format.BINARY) ? null : handleBase.getMimetype(); + MediaType mediaType = MediaType.parse( + (mimetype != null) ? mimetype : "application/x-unknown-content-type" + ); + return (document instanceof OutputStreamSender) ? + new StreamingOutputImpl((OutputStreamSender) document, null, mediaType) : + new ObjectRequestBody(HandleAccessor.sendContent(document), mediaType); + } + + private RequestBody makeRequestBody(CallField[] params) { + if (params == null || params.length == 0) { + return new EmptyRequestBody(); + } + + MultipartBody.Builder multiBldr = new MultipartBody.Builder(); + multiBldr.setType(MultipartBody.FORM); + + Condition hasValue = new Condition(); + Condition hasStreamingPartCondition = new Condition(); + for (CallField param : params) { + if (param == null) { + continue; + } + + final String paramName = param.getParamName(); + if (param instanceof SingleAtomicCallField) { + String paramValue = ((SingleAtomicCallField) param).getParamValue(); + if (paramValue != null) { + hasValue.set(); + multiBldr.addFormDataPart(paramName, null, makeRequestBody(paramValue)); + } + } else if (param instanceof MultipleAtomicCallField) { + Stream paramValues = ((MultipleAtomicCallField) param).getParamValues(); + if (paramValues != null) { + paramValues + .filter(paramValue -> paramValue != null) + .forEachOrdered(paramValue -> { + hasValue.set(); + multiBldr.addFormDataPart(paramName, null, makeRequestBody(paramValue)); + }); + } + } else if (param instanceof SingleNodeCallField) { + SingleNodeCallField singleNodeParam = (SingleNodeCallField) param; + BufferableHandle paramValue = singleNodeParam.getParamValue(); + if (paramValue != null) { + HandleImplementation handleBase = HandleAccessor.as(paramValue); + if (!handleBase.isResendable()) { + BytesHandle bytesHandle = new BytesHandle(paramValue); + singleNodeParam.setParamValue(bytesHandle); + paramValue = bytesHandle; + } + hasValue.set(); + multiBldr.addFormDataPart(paramName, null, makeRequestBodyForContent(paramValue)); + } + } else if (param instanceof UnbufferedMultipleNodeCallField) { + Stream paramValues = ((UnbufferedMultipleNodeCallField) param).getParamValues(); + if (paramValues != null) { + paramValues + .filter(paramValue -> paramValue != null) + .forEachOrdered(paramValue -> { + HandleImplementation handleBase = HandleAccessor.as(paramValue); + if (!handleBase.isResendable()) { + hasStreamingPartCondition.set(); + } + hasValue.set(); + multiBldr.addFormDataPart(paramName, null, makeRequestBody(paramValue)); + }); + } + } else if (param instanceof BufferedMultipleNodeCallField) { + BufferableHandle[] paramValues = ((BufferedMultipleNodeCallField) param).getParamValuesArray(); + if (paramValues != null) { + boolean checkedBuffer = false; + for (int i = 0; i < paramValues.length; i++) { + BufferableHandle paramValue = paramValues[i]; + if (paramValue != null) { + HandleImplementation handleBase = HandleAccessor.as(paramValue); + if (!handleBase.isResendable()) { + paramValue = new BytesHandle(paramValue); + if (!checkedBuffer) { + Class actualClass = paramValues.getClass().getComponentType(); + if (actualClass != BufferableHandle.class && actualClass != BytesHandle.class) { + paramValues = + Arrays.copyOf(paramValues, paramValues.length, BufferableHandle[].class); + } + checkedBuffer = true; + } + paramValues[i] = paramValue; + } + hasValue.set(); + multiBldr.addFormDataPart(paramName, null, makeRequestBody(paramValue)); + } + } + } + } else { + throw new IllegalStateException( + "unknown multipart " + paramName + " param of: " + param.getClass().getName() + ); + } + } + + if (!hasValue.get()) { + return new EmptyRequestBody(); + } + this.hasStreamingPart = hasStreamingPartCondition.get(); + return multiBldr.build(); + } + + } + + @Override + public CallRequest makeEmptyRequest(String endpoint, HttpMethod method, SessionState session) { + return new CallRequestImpl(endpoint, method, session).withEmptyRequest(); + } + + @Override + public CallRequest makeAtomicBodyRequest(String endpoint, HttpMethod method, SessionState session, CallField... params) { + if (params == null || params.length == 0) { + return makeEmptyRequest(endpoint, method, session); + } + return new CallRequestImpl(endpoint, method, session).withAtomicBodyRequest(params); + } + + @Override + public CallRequest makeNodeBodyRequest(String endpoint, HttpMethod method, SessionState session, CallField... params) { + if (params == null || params.length == 0) { + return makeEmptyRequest(endpoint, method, session); + } + return new CallRequestImpl(endpoint, method, session).withNodeBodyRequest(params); + } + + static private String encodeParamValue(String paramName, String value) { + if (value == null) { + return null; + } + try { + return paramName + "=" + URLEncoder.encode(value, UTF8_ID); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("UTF-8 is unsupported", e); + } + } + + static private String encodeParamValue(SingleAtomicCallField param) { + if (param == null) { + return null; + } + return encodeParamValue(param.getParamName(), param.getParamValue()); + } + + static private String encodeParamValue(MultipleAtomicCallField param) { + if (param == null) { + return null; + } + String paramName = param.getParamName(); + Stream paramValues = param.getParamValues(); + if (paramValues == null) { + return null; + } + String encodedParamValues = paramValues + .map(paramValue -> encodeParamValue(paramName, paramValue)) + .filter(paramValue -> (paramValue != null)) + .collect(Collectors.joining("&")); + if (encodedParamValues == null || encodedParamValues.length() == 0) { + return null; + } + return encodedParamValues; + } + + static private String encodeParamValue(CallField param) { + if (param == null) { + return null; + } else if (param instanceof SingleAtomicCallField) { + return encodeParamValue((SingleAtomicCallField) param); + } else if (param instanceof MultipleAtomicCallField) { + return encodeParamValue((MultipleAtomicCallField) param); + } + throw new IllegalStateException( + "could not encode parameter " + param.getParamName() + " of type: " + param.getClass().getName() + ); + } + + static class CallResponseImpl implements CallResponse { + private boolean isNull = true; + private Response response; + + Response getResponse() { + return response; + } + + void setResponse(Response response) { + this.response = response; + } + + @Override + public boolean isNull() { + return isNull; + } + + void setNull(boolean isNull) { + this.isNull = isNull; + } + + @Override + public int getStatusCode() { + return response.code(); + } + + @Override + public String getStatusMsg() { + return response.message(); + } + + //TODO: Check if this is needed since we are parsing it in the checkStatus(Respose). + //TODO: It might throw a closed exception since the response would be closed. Remove it after some testing + @Override + public String getErrorBody() { + try (ResponseBody errorBody = response.body()) { + if (errorBody.contentLength() > 0) { + MediaType errorType = errorBody.contentType(); + if (errorType != null) { + String subtype = errorType.subtype(); + if (subtype != null && subtype.toLowerCase().endsWith("json")) { + return errorBody.string(); + } + } + } + } catch (IOException e) { + throw new MarkLogicIOException(e); + } + return null; + } + + @Override + public void close() { + } + } + + static class SingleCallResponseImpl extends CallResponseImpl implements SingleCallResponse, AutoCloseable { + private Format format; + private ResponseBody responseBody; + + SingleCallResponseImpl(Format format) { + this.format = format; + } + + void setResponse(Response response) { + super.setResponse(response); + setResponseBody(response.body()); + } + + void setResponseBody(ResponseBody responseBody) { + if (!checkNull(responseBody, format)) { + this.responseBody = responseBody; + setNull(false); + } + } + + @Override + public byte[] asBytes() { + try { + if (responseBody == null) { + return null; + } + byte[] value = responseBody.bytes(); + closeImpl(); + return value; + } catch (IOException e) { + throw new MarkLogicIOException(e); + } + } + + @Override + public C asContent(BufferableContentHandle outputHandle) { + if (responseBody == null) return null; + HandleImplementation handleImpl = (HandleImplementation) outputHandle; + Class receiveClass = handleImpl.receiveAs(); + C content = outputHandle.toContent(getEntity(responseBody, receiveClass)); + return content; + } + + @Override + public > T asHandle(T outputHandle) { + if (responseBody == null) return null; + + return updateHandle(getResponse().headers(), responseBody, outputHandle); + } + + @Override + public InputStream asInputStream() { + return (responseBody == null) ? null : responseBody.byteStream(); + } + + @Override + public InputStreamHandle asInputStreamHandle() { + return (responseBody == null) ? null : + updateHandle(getResponse().headers(), responseBody, new InputStreamHandle()); + } + + @Override + public Reader asReader() { + return (responseBody == null) ? null : responseBody.charStream(); + } + + @Override + public ReaderHandle asReaderHandle() { + return (responseBody == null) ? null : + updateHandle(getResponse().headers(), responseBody, new ReaderHandle(asReader())); + } + + @Override + public String asString() { + try { + if (responseBody == null) { + return null; + } + String value = responseBody.string(); + closeImpl(); + return value; + } catch (IOException e) { + throw new MarkLogicIOException(e); + } + } + + @Override + public boolean asEndpointState(BytesHandle endpointStateHandle) { + try { + if (endpointStateHandle == null || responseBody == null) + return false; + byte[] value = responseBody.bytes(); + closeImpl(); + endpointStateHandle.set(value); + return true; + } catch (IOException e) { + throw new MarkLogicIOException(e); + } + } + + @Override + public void close() { + if (responseBody != null) { + closeImpl(); + } + } + + private void closeImpl() { + responseBody.close(); + responseBody = null; + } + } + + static class MultipleCallResponseImpl extends CallResponseImpl implements MultipleCallResponse { + private Format format; + private MimeMultipart multipart; + + MultipleCallResponseImpl(Format format) { + this.format = format; + } + + void setResponse(Response response) { + try { + super.setResponse(response); + ResponseBody responseBody = response.body(); + if (responseBody == null) { + setNull(true); + return; + } + MediaType contentType = responseBody.contentType(); + if (contentType == null) { + setNull(true); + return; + } + ByteArrayDataSource dataSource = new ByteArrayDataSource( + responseBody.byteStream(), contentType.toString() + ); + setMultipart(new MimeMultipart(dataSource)); + } catch (IOException e) { + throw new MarkLogicIOException(e); + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } + } + + void setMultipart(MimeMultipart multipart) { + if (!checkNull(multipart, format)) { + this.multipart = multipart; + setNull(false); + } + } + + @Override + public Stream asStreamOfBytes() { + try { + if (multipart == null) { + return Stream.empty(); + } + int partCount = multipart.getCount(); + + Stream.Builder builder = Stream.builder(); + for (int i = 0; i < partCount; i++) { + BodyPart bodyPart = multipart.getBodyPart(i); + builder.accept(NodeConverter.InputStreamToBytes(bodyPart.getInputStream())); + } + return builder.build(); + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } catch (IOException e) { + throw new MarkLogicIOException(e); + } + } + + @Override + public Stream asStreamOfContent( + BytesHandle endpointStateHandle, BufferableContentHandle outputHandle) { + try { + if (multipart == null) { + return Stream.empty(); + } + + boolean hasEndpointState = (endpointStateHandle != null); + + HandleImplementation handleImpl = (HandleImplementation) outputHandle; + Class receiveClass = handleImpl.receiveAs(); + + int partCount = multipart.getCount(); + Stream.Builder builder = Stream.builder(); + + for (int i = 0; i < partCount; i++) { + BodyPart bodyPart = multipart.getBodyPart(i); + if (hasEndpointState && i == 0) { + updateHandle(bodyPart, endpointStateHandle); + } else { + C value = responsePartToContent(outputHandle, bodyPart, receiveClass); + builder.accept(value); + } + } + + return builder.build(); + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } finally { + outputHandle.set(null); + } + } + + @Override + public > Stream asStreamOfHandles( + BytesHandle endpointStateHandle, T outputHandle + ) { + try { + if (multipart == null) { + return Stream.empty(); + } + + boolean hasEndpointState = (endpointStateHandle != null); + + Stream.Builder builder = Stream.builder(); + + int partCount = multipart.getCount(); + for (int i = 0; i < partCount; i++) { + BodyPart bodyPart = multipart.getBodyPart(i); + if (hasEndpointState && i == 0) { + updateHandle(bodyPart, endpointStateHandle); + } else { + builder.accept(updateHandle(bodyPart, (T) outputHandle.newHandle())); + } + } + return builder.build(); + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } finally { + outputHandle.set(null); + } + } + + @Override + public Stream asStreamOfInputStreamHandle() { + try { + if (multipart == null) { + return Stream.empty(); + } + int partCount = multipart.getCount(); + + Stream.Builder builder = Stream.builder(); + for (int i = 0; i < partCount; i++) { + BodyPart bodyPart = multipart.getBodyPart(i); + builder.accept(updateHandle(bodyPart, new InputStreamHandle())); + } + return builder.build(); + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } + } + + @Override + public Stream asStreamOfInputStream() { + try { + if (multipart == null) { + return Stream.empty(); + } + int partCount = multipart.getCount(); + + Stream.Builder builder = Stream.builder(); + for (int i = 0; i < partCount; i++) { + BodyPart bodyPart = multipart.getBodyPart(i); + builder.accept(bodyPart.getInputStream()); + } + return builder.build(); + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } catch (IOException e) { + throw new MarkLogicIOException(e); + } + } + + @Override + public Stream asStreamOfReader() { + try { + if (multipart == null) { + return Stream.empty(); + } + int partCount = multipart.getCount(); + + Stream.Builder builder = Stream.builder(); + for (int i = 0; i < partCount; i++) { + BodyPart bodyPart = multipart.getBodyPart(i); + builder.accept(NodeConverter.InputStreamToReader(bodyPart.getInputStream())); + } + return builder.build(); + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } catch (IOException e) { + throw new MarkLogicIOException(e); + } + } + + @Override + public Stream asStreamOfReaderHandle() { + try { + if (multipart == null) { + return Stream.empty(); + } + int partCount = multipart.getCount(); + + Stream.Builder builder = Stream.builder(); + for (int i = 0; i < partCount; i++) { + BodyPart bodyPart = multipart.getBodyPart(i); + builder.accept(updateHandle(bodyPart, new ReaderHandle())); + } + return builder.build(); + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } + } + + @Override + public Stream asStreamOfString() { + try { + if (multipart == null) { + return Stream.empty(); + } + int partCount = multipart.getCount(); + + Stream.Builder builder = Stream.builder(); + for (int i = 0; i < partCount; i++) { + BodyPart bodyPart = multipart.getBodyPart(i); + builder.accept(NodeConverter.InputStreamToString(bodyPart.getInputStream())); + } + return builder.build(); + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } catch (IOException e) { + throw new MarkLogicIOException(e); + } + } + + @Override + public byte[][] asArrayOfBytes() { + try { + if (multipart == null) { + return new byte[0][]; + } + int partCount = multipart.getCount(); + + byte[][] result = new byte[partCount][]; + for (int i = 0; i < partCount; i++) { + BodyPart bodyPart = multipart.getBodyPart(i); + result[i] = NodeConverter.InputStreamToBytes(bodyPart.getInputStream()); + } + return result; + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } catch (IOException e) { + throw new MarkLogicIOException(e); + } + } + + @Override + public C[] asArrayOfContent( + BytesHandle endpointStateHandle, BufferableContentHandle outputHandle + ) { + try { + if (multipart == null) { + return outputHandle.newArray(0); + } + + boolean hasEndpointState = (endpointStateHandle != null); + + HandleImplementation handleImpl = (HandleImplementation) outputHandle; + Class receiveClass = handleImpl.receiveAs(); + + int partCount = multipart.getCount(); + C[] result = outputHandle.newArray(hasEndpointState ? (partCount - 1) : partCount); + for (int i = 0; i < partCount; i++) { + BodyPart bodyPart = multipart.getBodyPart(i); + if (hasEndpointState && i == 0) { + updateHandle(bodyPart, endpointStateHandle); + } else { + C value = responsePartToContent(outputHandle, bodyPart, receiveClass); + result[hasEndpointState ? (i - 1) : i] = value; + } + } + + return result; + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } finally { + outputHandle.set(null); + } + } + + @Override + public BufferableContentHandle[] asArrayOfHandles( + BytesHandle endpointStateHandle, BufferableContentHandle outputHandle + ) { + try { + if (multipart == null) { + return outputHandle.newHandleArray(0); + } + + boolean hasEndpointState = (endpointStateHandle != null); + + int partCount = multipart.getCount(); + BufferableContentHandle[] result = outputHandle.newHandleArray(hasEndpointState ? (partCount - 1) : partCount); + for (int i = 0; i < partCount; i++) { + BodyPart bodyPart = multipart.getBodyPart(i); + if (hasEndpointState && i == 0) { + updateHandle(bodyPart, endpointStateHandle); + } else { + result[hasEndpointState ? (i - 1) : i] = updateHandle(bodyPart, outputHandle.newHandle()); + } + } + + return result; + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } finally { + outputHandle.set(null); + } + } + + private C responsePartToContent(BufferableContentHandle handle, BodyPart bodyPart, Class as) { + return handle.toContent(getEntity(bodyPart, as)); + } + + @Override + public InputStream[] asArrayOfInputStream() { + try { + if (multipart == null) { + return new InputStream[0]; + } + int partCount = multipart.getCount(); + + InputStream[] result = new InputStream[partCount]; + for (int i = 0; i < partCount; i++) { + BodyPart bodyPart = multipart.getBodyPart(i); + result[i] = bodyPart.getInputStream(); + } + return result; + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } catch (IOException e) { + throw new MarkLogicIOException(e); + } + } + + @Override + public InputStreamHandle[] asArrayOfInputStreamHandle() { + try { + if (multipart == null) { + return new InputStreamHandle[0]; + } + int partCount = multipart.getCount(); + + InputStreamHandle[] result = new InputStreamHandle[partCount]; + for (int i = 0; i < partCount; i++) { + BodyPart bodyPart = multipart.getBodyPart(i); + result[i] = updateHandle(bodyPart, new InputStreamHandle()); + } + return result; + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } + } + + @Override + public ReaderHandle[] asArrayOfReaderHandle() { + try { + if (multipart == null) { + return new ReaderHandle[0]; + } + int partCount = multipart.getCount(); + + ReaderHandle[] result = new ReaderHandle[partCount]; + for (int i = 0; i < partCount; i++) { + BodyPart bodyPart = multipart.getBodyPart(i); + result[i] = updateHandle(bodyPart, new ReaderHandle()); + } + return result; + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } + } + + @Override + public Reader[] asArrayOfReader() { + try { + if (multipart == null) { + return new Reader[0]; + } + int partCount = multipart.getCount(); + + Reader[] result = new Reader[partCount]; + for (int i = 0; i < partCount; i++) { + BodyPart bodyPart = multipart.getBodyPart(i); + result[i] = NodeConverter.InputStreamToReader(bodyPart.getInputStream()); + } + return result; + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } catch (IOException e) { + throw new MarkLogicIOException(e); + } + } + + @Override + public String[] asArrayOfString() { + try { + if (multipart == null) { + return new String[0]; + } + int partCount = multipart.getCount(); + + String[] result = new String[partCount]; + for (int i = 0; i < partCount; i++) { + BodyPart bodyPart = multipart.getBodyPart(i); + result[i] = NodeConverter.InputStreamToString(bodyPart.getInputStream()); + } + return result; + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } catch (IOException e) { + throw new MarkLogicIOException(e); + } + } + } + + static protected boolean checkNull(ResponseBody body, Format expectedFormat) { + if (body != null) { + if (body.contentLength() == 0) { + body.close(); + } else { + MediaType actualType = body.contentType(); + if (actualType == null) { + body.close(); + throw new RuntimeException( + "Returned document with unknown mime type instead of " + expectedFormat.getDefaultMimetype() + ); + } + if (expectedFormat != Format.UNKNOWN) { + Format actualFormat = Format.getFromMimetype(actualType.toString()); + if (expectedFormat != actualFormat) { + body.close(); + throw new RuntimeException( + "Mime type " + actualType.toString() + " for returned document not recognized for " + expectedFormat.name() + ); + } + } + return false; + } + } + return true; + } + + static protected boolean checkNull(MimeMultipart multipart, Format expectedFormat) { + if (multipart != null) { + try { + if (multipart.getCount() != 0) { + BodyPart firstPart = multipart.getBodyPart(0); + String actualType = (firstPart == null) ? null : firstPart.getContentType(); + if (actualType == null) { + throw new RuntimeException( + "Returned document with unknown mime type instead of " + expectedFormat.getDefaultMimetype() + ); + } + if (expectedFormat != Format.UNKNOWN) { + Format actualFormat = Format.getFromMimetype(actualType); + if (expectedFormat != actualFormat) { + throw new RuntimeException( + "Mime type " + actualType + " for returned document not recognized for " + expectedFormat.name() + ); + } + } + return false; + } + } catch (MessagingException e) { + throw new MarkLogicIOException(e); + } + } + return true; + } + + static private void closeResponse(Response response) { + if (response == null || response.body() == null) return; + response.close(); + } + + Request.Builder forDocumentResponse(Request.Builder requestBldr, Format format) { + return requestBldr.addHeader( + HEADER_ACCEPT, + (format == null || format == Format.BINARY || format == Format.UNKNOWN) ? + "application/x-unknown-content-type" : format.getDefaultMimetype()); + } + + Request.Builder forMultipartMixedResponse(Request.Builder requestBldr) { + return requestBldr.addHeader(HEADER_ACCEPT, multipartMixedWithBoundary()); + } + + static protected class Condition { + private boolean is = false; + + protected boolean get() { + return is; + } + + protected void set() { + if (!is) + is = true; + } + } + + static class ConnectionResultImpl implements ConnectionResult { + private boolean connected = false; + private int statusCode; + private String errorMessage; + + @Override + public boolean isConnected() { + return connected; + } + + private void setConnected(boolean connected) { + this.connected = connected; + } + + @Override + public Integer getStatusCode() { + return statusCode; + } + + private void setStatusCode(int statusCode) { + this.statusCode = statusCode; + } + + @Override + public String getErrorMessage() { + return errorMessage; + } + + private void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } + } + + @FunctionalInterface + private interface ResultIteratorConstructor { + T construct(RequestLogger logger, List list, Closeable closeable); + } } From 44450b785d6210b0d4925a83964a427c63356d20 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 3 Jan 2025 12:34:09 -0500 Subject: [PATCH 33/39] MLE-19222 Eval/invoke now stream results Finally makes use of the OkHttp MultipartReader to stream body parts instead of reading them all into memory. There's no test to be added here, as the only way to verify this is to try with a sufficient amount of data to cause an OutOfMemoryError. That will be verified manually instead. The existing regression tests will suffice to ensure that eval/invoke still work properly. --- .../com/marklogic/client/impl/IoUtil.java | 26 ++++++ .../marklogic/client/impl/OkHttpServices.java | 42 +++++++-- .../client/impl/okhttp/PartIterator.java | 86 +++++++++++++++++++ .../client/io/InputStreamHandle.java | 15 +--- 4 files changed, 151 insertions(+), 18 deletions(-) create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/impl/IoUtil.java create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/PartIterator.java diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/IoUtil.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/IoUtil.java new file mode 100644 index 000000000..a5dee1250 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/IoUtil.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2024 MarkLogic Corporation. All Rights Reserved. + */ +package com.marklogic.client.impl; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; + +public interface IoUtil { + + /** + * Tossing this commonly used logic here so that it can be reused. Can be removed when we drop Java 8 support, as + * Java 9+ has a "readAllBytes" method. + */ + static byte[] streamToBytes(InputStream stream) throws IOException { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + byte[] b = new byte[8192]; + int len = 0; + while ((len = stream.read(b)) != -1) { + buffer.write(b, 0, len); + } + buffer.flush(); + return buffer.toByteArray(); + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java index 120fbd600..52ab8a86d 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java @@ -19,6 +19,7 @@ import com.marklogic.client.eval.EvalResultIterator; import com.marklogic.client.impl.okhttp.HttpUrlBuilder; import com.marklogic.client.impl.okhttp.OkHttpUtil; +import com.marklogic.client.impl.okhttp.PartIterator; import com.marklogic.client.io.*; import com.marklogic.client.io.marker.*; import com.marklogic.client.query.*; @@ -3836,7 +3837,29 @@ private U postIteratedResourceImpl( Response response = sendRequestWithRetry(requestBldr, (transaction == null), doPostFunction, resendableConsumer); checkStatus(response, response.code(), "apply", "resource", path, ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); - return makeResults(constructor, reqlog, "apply", "resource", response); + + boolean shouldStreamResults = "eval".equalsIgnoreCase(path) || "invoke".equalsIgnoreCase(path); + boolean hasDataToStream = response.body().contentLength() != 0; + // If body is empty, we can use the "old" way of reading results as there's nothing to stream. + return shouldStreamResults && hasDataToStream ? + evalAndStreamResults(reqlog, response) : + makeResults(constructor, reqlog, "apply", "resource", response); + } + + /** + * Added to resolve MLE-19222, where the eval/invoke response was read into memory, leading to OutOfMemoryErrors. + * The one thing we are not able to do here though is check for errors in the trailers, as trailers cannot be + * read until the entire body has been read. But we don't want to read the entire body right away. + */ + private U evalAndStreamResults(RequestLogger reqlog, Response response) { + if (response == null) return null; + try { + MultipartReader reader = new MultipartReader(response.body()); + PartIterator partIterator = new PartIterator(reader); + return (U) new DefaultOkHttpResultIterator(reqlog, partIterator, response); + } catch (IOException e) { + throw new MarkLogicIOException(e); + } } @Override @@ -4587,6 +4610,12 @@ static abstract class OkHttpResultIterator { private long totalSize = -1; private Closeable closeable; + OkHttpResultIterator(RequestLogger reqlog, Iterator partIterator, Closeable closeable) { + this.reqlog = reqlog; + this.partQueue = partIterator; + this.closeable = closeable; + } + OkHttpResultIterator(RequestLogger reqlog, List partList, Closeable closeable) { this.reqlog = reqlog; if (partList != null && partList.size() > 0) { @@ -4685,14 +4714,15 @@ OkHttpServiceResult constructNext(RequestLogger logger, BodyPart part) { } } - static class DefaultOkHttpResultIterator - extends OkHttpResultIterator - implements Iterator { - DefaultOkHttpResultIterator(RequestLogger reqlog, - List partList, Closeable closeable) { + static class DefaultOkHttpResultIterator extends OkHttpResultIterator implements Iterator { + DefaultOkHttpResultIterator(RequestLogger reqlog, List partList, Closeable closeable) { super(reqlog, partList, closeable); } + DefaultOkHttpResultIterator(RequestLogger reqlog, Iterator partIterator, Closeable closeable) { + super(reqlog, partIterator, closeable); + } + OkHttpResult constructNext(RequestLogger logger, BodyPart part) { return new OkHttpResult(logger, part); } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/PartIterator.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/PartIterator.java new file mode 100644 index 000000000..d22a67d4b --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/PartIterator.java @@ -0,0 +1,86 @@ +/* + * Copyright © 2024 MarkLogic Corporation. All Rights Reserved. + */ +package com.marklogic.client.impl.okhttp; + +import com.marklogic.client.MarkLogicIOException; +import com.marklogic.client.impl.IoUtil; +import jakarta.activation.DataHandler; +import jakarta.mail.BodyPart; +import jakarta.mail.MessagingException; +import jakarta.mail.internet.MimeBodyPart; +import jakarta.mail.util.ByteArrayDataSource; +import okhttp3.Headers; +import okhttp3.MultipartReader; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Iterator; + +/** + * Adapts the iterator over the OkHttp MultipartReader to conform to the Iterator that is required by + * OkHttpEvalResultIterator. By converting each MultipartReader.Part into a jakarta.mail.BodyPart, we can reuse + * all the existing plumbing that depends on jakarta.mail.BodyPart. + *

+ * Added to resolve MLE-19222, where eval/invoke results are not being streamed but rather were all being read into + * memory, leading to OutOfMemoryErrors. + */ +public class PartIterator implements Iterator { + + private final MultipartReader reader; + private BodyPart nextBodyPart; + + public PartIterator(MultipartReader reader) { + this.reader = reader; + readNextPart(); + } + + @Override + public boolean hasNext() { + return nextBodyPart != null; + } + + @Override + public BodyPart next() { + BodyPart partToReturn = nextBodyPart; + readNextPart(); + return partToReturn; + } + + private void readNextPart() { + try { + // See http://okhttp.foofun.cn/4.x/okhttp/okhttp3/-multipart-reader/ for more info on the OkHttp + // MultipartReader. This was actually requested many moons ago by one of the original Java Client + // developers - https://github.com/square/okhttp/issues/3394. + MultipartReader.Part nextPart = reader.nextPart(); + this.nextBodyPart = nextPart != null ? convertPartToBodyPart(nextPart) : null; + } catch (Exception e) { + throw new MarkLogicIOException(e); + } + } + + private static BodyPart convertPartToBodyPart(MultipartReader.Part part) throws IOException, MessagingException { + MimeBodyPart bodyPart = new MimeBodyPart(); + + try { + try (InputStream inputStream = part.body().inputStream()) { + byte[] bytes = IoUtil.streamToBytes(inputStream); + bodyPart.setDataHandler(new DataHandler(new ByteArrayDataSource(bytes, part.headers().get("Content-Type")))); + } + + // part.headers.toMultimap() is lowercasing header names, which causes later issues. + Headers headers = part.headers(); + for (String headerName : headers.names()) { + for (String headerValue : headers.values(headerName)) { + bodyPart.addHeader(headerName, headerValue); + } + } + return bodyPart; + } finally { + // Looking at the OkHttp source code, this does not appear necessary, as closing the InputStream above should + // achieve the same effect. But there is no downside to doing this, as it may be required by a future version + // of OkHttp. + part.close(); + } + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/io/InputStreamHandle.java b/marklogic-client-api/src/main/java/com/marklogic/client/io/InputStreamHandle.java index efa3cbaf7..4c8bdd675 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/io/InputStreamHandle.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/io/InputStreamHandle.java @@ -10,6 +10,7 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; +import com.marklogic.client.impl.IoUtil; import com.marklogic.client.io.marker.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,8 +46,6 @@ public class InputStreamHandle private byte[] contentBytes; private InputStream content; - final static private int BUFFER_SIZE = 8192; - /** * Creates a factory to create an InputStreamHandle instance for an input stream. * @return the factory @@ -186,17 +185,9 @@ public InputStream bytesToContent(byte[] buffer) { public byte[] contentToBytes(InputStream content) { try { if (content == null) return null; - - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - - byte[] b = new byte[BUFFER_SIZE]; - int len = 0; - while ((len = content.read(b)) != -1) { - buffer.write(b, 0, len); - } + byte[] bytes = IoUtil.streamToBytes(content); content.close(); - - return buffer.toByteArray(); + return bytes; } catch (IOException e) { throw new MarkLogicIOException(e); } From dcabf41b283195ca7cb74299883423292a52882e Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 3 Jan 2025 14:59:05 -0500 Subject: [PATCH 34/39] Sanity check to ensure connection is closed Also added docs to ensure clients call `close()`, which they really ought to have been doing already given the name "iterator". --- .../client/eval/EvalResultIterator.java | 24 ++++++++++++------- .../marklogic/client/impl/OkHttpServices.java | 12 +++++++++- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/eval/EvalResultIterator.java b/marklogic-client-api/src/main/java/com/marklogic/client/eval/EvalResultIterator.java index 8278c11a8..a0c26fb77 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/eval/EvalResultIterator.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/eval/EvalResultIterator.java @@ -6,15 +6,23 @@ import java.io.Closeable; import java.util.Iterator; -/** An Iterator to walk through all results returned from calls to +/** + * An Iterator to walk through all results returned from calls to * {@link ServerEvaluationCall#eval()}. */ public interface EvalResultIterator extends Iterable, Iterator, Closeable { - @Override - Iterator iterator(); - @Override - boolean hasNext(); - @Override - EvalResult next(); - void close(); + @Override + Iterator iterator(); + + @Override + boolean hasNext(); + + @Override + EvalResult next(); + + /** + * As of 7.1.0, this must be called to ensure that the response is closed, as results are now + * streamed from MarkLogic instead of being read entirely into memory first. + */ + void close(); } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java index 52ab8a86d..ef8d51317 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java @@ -3856,7 +3856,17 @@ private U evalAndStreamResults(RequestLogger re try { MultipartReader reader = new MultipartReader(response.body()); PartIterator partIterator = new PartIterator(reader); - return (U) new DefaultOkHttpResultIterator(reqlog, partIterator, response); + return (U) new DefaultOkHttpResultIterator(reqlog, partIterator, () -> { + // Looking at OkHttp source code, it does not appear necessary to call close on the reader; it appears + // sufficient to only call it on the response. But doing both in case this behavior changes in a future + // OkHttp release. + try { + reader.close(); + } catch (IOException e) { + // Ignore, the next call should close everything properly. + } + response.close(); + }); } catch (IOException e) { throw new MarkLogicIOException(e); } From 9fba4be16a50d9cc0c970f71ab72999f6601c649 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 3 Jan 2025 15:42:33 -0500 Subject: [PATCH 35/39] MLE-18751 Added docs about descriptors and content versioning This is to provide some indication that a user can get null back for a read call when the document exists. --- .../client/document/DocumentDescriptor.java | 4 +++- .../client/document/DocumentManager.java | 24 ++++++++++++------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/document/DocumentDescriptor.java b/marklogic-client-api/src/main/java/com/marklogic/client/document/DocumentDescriptor.java index bc579e65d..c556cdde0 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/document/DocumentDescriptor.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/document/DocumentDescriptor.java @@ -6,7 +6,9 @@ import com.marklogic.client.io.Format; /** - * A Document Descriptor describes a database document. + * A Document Descriptor describes a database document. If content versioning is enabled on the app server used + * to retrieve a document via an instance of this class, note that you may receive a null return value if the + * corresponding document has not been modified. */ public interface DocumentDescriptor extends ContentDescriptor { /** diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/document/DocumentManager.java b/marklogic-client-api/src/main/java/com/marklogic/client/document/DocumentManager.java index 145860161..feaa392e6 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/document/DocumentManager.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/document/DocumentManager.java @@ -240,7 +240,8 @@ T read(String docId, T contentHandle, ServerTransform transform) * @param desc a descriptor for the URI identifier, format, and mimetype of the document * @param contentHandle a handle for reading the content of the document * @param the type of content handle to return - * @return the content handle populated with the content of the document in the database + * @return the content handle populated with the content of the document in the database, or null if content + * versioning is enabled and the document has not been modified. * @throws ResourceNotFoundException if the document is not found */ T read(DocumentDescriptor desc, T contentHandle) @@ -254,7 +255,8 @@ T read(DocumentDescriptor desc, T contentHandle) * @param contentHandle a handle for reading the content of the document * @param transform a server transform to modify the document content * @param the type of content handle to return - * @return the content handle populated with the content of the document in the database + * @return the content handle populated with the content of the document in the database, or null if content + * versioning is enabled and the document has not been modified. * @throws ResourceNotFoundException if the document is not found */ T read(DocumentDescriptor desc, T contentHandle, ServerTransform transform) @@ -298,7 +300,8 @@ T read(String docId, DocumentMetadataReadHandle metadataHandle, T * @param metadataHandle a handle for reading the metadata of the document * @param contentHandle a handle for reading the content of the document * @param the type of content handle to return - * @return the content handle populated with the content of the document in the database + * @return the content handle populated with the content of the document in the database, or null if content + * versioning is enabled and the document has not been modified. * @throws ResourceNotFoundException if the document is not found */ T read(DocumentDescriptor desc, DocumentMetadataReadHandle metadataHandle, T contentHandle) @@ -313,7 +316,8 @@ T read(DocumentDescriptor desc, DocumentMetadataReadHandle metadat * @param contentHandle a handle for reading the content of the document * @param transform a server transform to modify the document content * @param the type of content handle to return - * @return the content handle populated with the content of the document in the database + * @return the content handle populated with the content of the document in the database, or null if content + * versioning is enabled and the document has not been modified. * @throws ResourceNotFoundException if the document is not found */ T read(DocumentDescriptor desc, DocumentMetadataReadHandle metadataHandle, T contentHandle, ServerTransform transform) @@ -357,7 +361,8 @@ T read(String docId, T contentHandle, ServerTransform transform, T * @param contentHandle a handle for reading the content of the document * @param transaction a open transaction under which the document may have been created or deleted * @param the type of content handle to return - * @return the content handle populated with the content of the document in the database + * @return the content handle populated with the content of the document in the database, or null if content + * versioning is enabled and the document has not been modified. * @throws ResourceNotFoundException if the document is not found */ T read(DocumentDescriptor desc, T contentHandle, Transaction transaction) @@ -372,7 +377,8 @@ T read(DocumentDescriptor desc, T contentHandle, Transaction trans * @param transform a server transform to modify the document content * @param transaction a open transaction under which the document may have been created or deleted * @param the type of content handle to return - * @return the content handle populated with the content of the document in the database + * @return the content handle populated with the content of the document in the database, or null if content + * versioning is enabled and the document has not been modified. * @throws ResourceNotFoundException if the document is not found */ T read(DocumentDescriptor desc, T contentHandle, ServerTransform transform, Transaction transaction) @@ -419,7 +425,8 @@ T read(String docId, DocumentMetadataReadHandle metadataHandle, T * @param contentHandle a handle for reading the content of the document * @param transaction a open transaction under which the document may have been created or deleted * @param the type of content handle to return - * @return the content handle populated with the content of the document in the database + * @return the content handle populated with the content of the document in the database, or null if content + * versioning is enabled and the document has not been modified. * @throws ResourceNotFoundException if the document is not found */ T read(DocumentDescriptor desc, DocumentMetadataReadHandle metadataHandle, T contentHandle, Transaction transaction) @@ -435,7 +442,8 @@ T read(DocumentDescriptor desc, DocumentMetadataReadHandle metadat * @param transform a server transform to modify the document content * @param transaction a open transaction under which the document may have been created or deleted * @param the type of content handle to return - * @return the content handle populated with the content of the document in the database + * @return the content handle populated with the content of the document in the database, or null if content + * versioning is enabled and the document has not been modified. * @throws ResourceNotFoundException if the document is not found */ T read(DocumentDescriptor desc, DocumentMetadataReadHandle metadataHandle, T contentHandle, ServerTransform transform, Transaction transaction) From 919284af33d684929623c5c963798ef58f698d99 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 6 Jan 2025 10:00:51 -0500 Subject: [PATCH 36/39] MLE-19240 Added marklogic.client.connectionString Intent is to support this in places the upcoming query pipeline kit and other apps that want to create a client via properties. Going to remove this support in the Spark connector next and reuse this. --- .../client/DatabaseClientBuilder.java | 19 ++++ .../client/DatabaseClientFactory.java | 1 + .../client/impl/ConnectionString.java | 89 +++++++++++++++ .../impl/DatabaseClientPropertySource.java | 57 ++++++++-- .../DatabaseClientPropertySourceTest.java | 87 ++++++++++++++- .../test/DatabaseClientBuilderTest.java | 103 ++++++++++++++++-- 6 files changed, 334 insertions(+), 22 deletions(-) create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/impl/ConnectionString.java diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientBuilder.java b/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientBuilder.java index 523210972..1cfb5510a 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientBuilder.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientBuilder.java @@ -3,6 +3,7 @@ */ package com.marklogic.client; +import com.marklogic.client.impl.ConnectionString; import com.marklogic.client.impl.DatabaseClientPropertySource; import javax.net.ssl.SSLContext; @@ -84,6 +85,24 @@ public DatabaseClientBuilder withPort(int port) { return this; } + /** + * @param connectionString of the form "username:password@host:port/optionalDatabaseName". + * @since 7.1.0 + */ + public DatabaseClientBuilder withConnectionString(String connectionString) { + ConnectionString cs = new ConnectionString(connectionString, "connection string"); + if (!props.containsKey(PREFIX + "authType")) { + withAuthType("digest"); + } + if (cs.getDatabase() != null && cs.getDatabase().trim().length() > 0) { + withDatabase(cs.getDatabase()); + } + return withHost(cs.getHost()) + .withPort(cs.getPort()) + .withUsername(cs.getUsername()) + .withPassword(cs.getPassword()); + } + public DatabaseClientBuilder withBasePath(String basePath) { props.put(PREFIX + "basePath", basePath); return this; diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientFactory.java b/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientFactory.java index 2cef3ba31..f6ed16e99 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientFactory.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientFactory.java @@ -1088,6 +1088,7 @@ public String getCertificatePassword() { * "kerberos", "certificate", or "saml" *

  • marklogic.client.username = must be a String; required for basic and digest authentication
  • *
  • marklogic.client.password = must be a String; required for basic and digest authentication
  • + *
  • marklogic.client.connectionString = must be a String; must fit format of "username:password@host:port/optionalDatabaseName". Defaults the authentication type to "digest"; since 7.1.0.
  • *
  • marklogic.client.certificate.file = must be a String; optional for certificate authentication
  • *
  • marklogic.client.certificate.password = must be a String; optional for certificate authentication
  • *
  • marklogic.client.cloud.apiKey = must be a String; required for cloud authentication
  • diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/ConnectionString.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/ConnectionString.java new file mode 100644 index 000000000..6003cd149 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/ConnectionString.java @@ -0,0 +1,89 @@ +/* + * Copyright © 2024 MarkLogic Corporation. All Rights Reserved. + */ +package com.marklogic.client.impl; + +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; + +/** + * @since 7.1.0; copied from marklogic-spark-connector repository. + */ +public class ConnectionString { + + private final String host; + private final int port; + private final String username; + private final String password; + private final String database; + + public ConnectionString(String connectionString, String optionNameForErrorMessage) { + final String errorMessage = String.format( + "Invalid value for %s; must be username:password@host:port/optionalDatabaseName", + optionNameForErrorMessage + ); + + String[] parts = connectionString.split("@"); + if (parts.length != 2) { + throw new IllegalArgumentException(errorMessage); + } + String[] tokens = parts[0].split(":"); + if (tokens.length != 2) { + throw new IllegalArgumentException(errorMessage); + } + this.username = decodeValue(tokens[0], "username"); + this.password = decodeValue(tokens[1], "password"); + + tokens = parts[1].split(":"); + if (tokens.length != 2) { + throw new IllegalArgumentException(errorMessage); + } + this.host = tokens[0]; + if (tokens[1].contains("/")) { + tokens = tokens[1].split("/"); + this.port = parsePort(tokens[0], optionNameForErrorMessage); + this.database = tokens[1]; + } else { + this.port = parsePort(tokens[1], optionNameForErrorMessage); + this.database = null; + } + } + + private int parsePort(String value, String optionNameForErrorMessage) { + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(String.format( + "Invalid value for %s; port must be numeric, but was '%s'", optionNameForErrorMessage, value + )); + } + } + + private String decodeValue(String value, String label) { + try { + return URLDecoder.decode(value, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new IllegalArgumentException(String.format("Unable to decode '%s'; cause: %s", label, e.getMessage())); + } + } + + public String getHost() { + return host; + } + + public int getPort() { + return port; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + public String getDatabase() { + return database; + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientPropertySource.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientPropertySource.java index 601a293f7..18fa3182f 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientPropertySource.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientPropertySource.java @@ -59,6 +59,18 @@ public class DatabaseClientPropertySource { throw new IllegalArgumentException("Database must be of type String"); } }); + connectionPropertyHandlers.put(PREFIX + "connectionString", (bean, value) -> { + if (value instanceof String) { + ConnectionString cs = new ConnectionString((String) value, "connection string"); + bean.setHost(cs.getHost()); + bean.setPort(cs.getPort()); + if (cs.getDatabase() != null && cs.getDatabase().trim().length() > 0) { + bean.setDatabase(cs.getDatabase()); + } + } else { + throw new IllegalArgumentException("Connection string must be of type String"); + } + }); connectionPropertyHandlers.put(PREFIX + "basePath", (bean, value) -> { if (value instanceof String) { bean.setBasePath((String) value); @@ -125,6 +137,11 @@ public DatabaseClientFactory.Bean newClientBean() { return bean; } + private ConnectionString makeConnectionString() { + String value = (String) propertySource.apply(PREFIX + "connectionString"); + return value != null && value.trim().length() > 0 ? new ConnectionString(value, "connection string") : null; + } + private DatabaseClientFactory.SecurityContext newSecurityContext() { Object securityContextValue = propertySource.apply(PREFIX + "securityContext"); if (securityContextValue != null) { @@ -134,14 +151,11 @@ private DatabaseClientFactory.SecurityContext newSecurityContext() { throw new IllegalArgumentException("Security context must be of type " + DatabaseClientFactory.SecurityContext.class.getName()); } - Object typeValue = propertySource.apply(PREFIX + "authType"); - if (typeValue == null || !(typeValue instanceof String)) { - throw new IllegalArgumentException("Security context should be set, or auth type must be of type String"); - } - final String authType = (String) typeValue; + ConnectionString connectionString = makeConnectionString(); + final String authType = determineAuthType(connectionString); final SSLUtil.SSLInputs sslInputs = buildSSLInputs(authType); - DatabaseClientFactory.SecurityContext securityContext = newSecurityContext(authType, sslInputs); + DatabaseClientFactory.SecurityContext securityContext = newSecurityContext(authType, connectionString, sslInputs); if (sslInputs.getSslContext() != null) { securityContext.withSSLContext(sslInputs.getSslContext(), sslInputs.getTrustManager()); } @@ -149,12 +163,23 @@ private DatabaseClientFactory.SecurityContext newSecurityContext() { return securityContext; } - private DatabaseClientFactory.SecurityContext newSecurityContext(String type, SSLUtil.SSLInputs sslInputs) { + private String determineAuthType(ConnectionString connectionString) { + Object value = propertySource.apply(PREFIX + "authType"); + if (value == null && connectionString != null) { + return "digest"; + } + if (value == null || !(value instanceof String)) { + throw new IllegalArgumentException("Security context should be set, or auth type must be of type String"); + } + return (String) value; + } + + private DatabaseClientFactory.SecurityContext newSecurityContext(String type, ConnectionString connectionString, SSLUtil.SSLInputs sslInputs) { switch (type.toLowerCase()) { case DatabaseClientBuilder.AUTH_TYPE_BASIC: - return newBasicAuthContext(); + return newBasicAuthContext(connectionString); case DatabaseClientBuilder.AUTH_TYPE_DIGEST: - return newDigestAuthContext(); + return newDigestAuthContext(connectionString); case DatabaseClientBuilder.AUTH_TYPE_MARKLOGIC_CLOUD: return newCloudAuthContext(); case DatabaseClientBuilder.AUTH_TYPE_KERBEROS: @@ -194,14 +219,24 @@ private String getNullableStringValue(String propertyName, String defaultValue) return value != null ? (String) value : defaultValue; } - private DatabaseClientFactory.SecurityContext newBasicAuthContext() { + private DatabaseClientFactory.SecurityContext newBasicAuthContext(ConnectionString connectionString) { + if (connectionString != null) { + return new DatabaseClientFactory.BasicAuthContext( + connectionString.getUsername(), connectionString.getPassword() + ); + } return new DatabaseClientFactory.BasicAuthContext( getRequiredStringValue("username", "Must specify a username when using basic authentication."), getRequiredStringValue("password", "Must specify a password when using basic authentication.") ); } - private DatabaseClientFactory.SecurityContext newDigestAuthContext() { + private DatabaseClientFactory.SecurityContext newDigestAuthContext(ConnectionString connectionString) { + if (connectionString != null) { + return new DatabaseClientFactory.DigestAuthContext( + connectionString.getUsername(), connectionString.getPassword() + ); + } return new DatabaseClientFactory.DigestAuthContext( getRequiredStringValue("username", "Must specify a username when using digest authentication."), getRequiredStringValue("password", "Must specify a password when using digest authentication.") diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/impl/DatabaseClientPropertySourceTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/impl/DatabaseClientPropertySourceTest.java index 3c87efeb0..f68f04544 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/impl/DatabaseClientPropertySourceTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/impl/DatabaseClientPropertySourceTest.java @@ -5,14 +5,13 @@ import com.marklogic.client.DatabaseClientFactory; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.HashMap; import java.util.Map; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; /** * Intent of this test is to cover code that cannot be covered by DatabaseClientBuilderTest. @@ -123,6 +122,86 @@ void disableGzippedResponses() { buildBean(); } + @Test + void connectionString() { + useConnectionString("user:password@localhost:8000"); + DatabaseClientFactory.Bean bean = buildBean(); + + assertEquals("localhost", bean.getHost()); + assertEquals(8000, bean.getPort()); + assertNull(bean.getDatabase()); + DatabaseClientFactory.DigestAuthContext context = (DatabaseClientFactory.DigestAuthContext) bean.getSecurityContext(); + assertEquals("user", context.getUser()); + assertEquals("password", context.getPassword()); + } + + @Test + void connectionStringWithDatabase() { + useConnectionString("user:password@localhost:8000/Documents"); + DatabaseClientFactory.Bean bean = buildBean(); + + assertEquals("localhost", bean.getHost()); + assertEquals(8000, bean.getPort()); + assertEquals("Documents", bean.getDatabase()); + DatabaseClientFactory.DigestAuthContext context = (DatabaseClientFactory.DigestAuthContext) bean.getSecurityContext(); + assertEquals("user", context.getUser()); + assertEquals("password", context.getPassword()); + } + + @Test + void connectionStringWithSeparateDatabase() { + useConnectionString("user:password@localhost:8000"); + props.put(PREFIX + "database", "SomeDatabase"); + DatabaseClientFactory.Bean bean = buildBean(); + + assertEquals("localhost", bean.getHost()); + assertEquals(8000, bean.getPort()); + assertEquals("SomeDatabase", bean.getDatabase()); + } + + @Test + void usernameAndPasswordBothRequireDecoding() { + useConnectionString("test-user%40:sp%40r%3Ak@localhost:8000/Documents"); + DatabaseClientFactory.Bean bean = buildBean(); + + assertEquals("localhost", bean.getHost()); + assertEquals(8000, bean.getPort()); + assertEquals("Documents", bean.getDatabase()); + + DatabaseClientFactory.DigestAuthContext context = (DatabaseClientFactory.DigestAuthContext) bean.getSecurityContext(); + assertEquals("test-user@", context.getUser()); + assertEquals("sp@r:k", context.getPassword(), "Verifies that the user must encode username and password " + + "values that contain ':' or '@'. The builder is then expected to decode them into the correct values."); + } + + @ParameterizedTest + @ValueSource(strings = { + "user@host@port", + "user@host:port", + "user:password@host", + "user:password:something@host:port", + "user:password@host:port:something" + }) + void invalidConnectionString(String connectionString) { + useConnectionString(connectionString); + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> buildBean()); + assertEquals("Invalid value for connection string; must be username:password@host:port/optionalDatabaseName", + ex.getMessage()); + } + + @Test + void nonNumericPortInConnectionString() { + useConnectionString("user:password@host:nonNumericPort"); + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> buildBean()); + assertEquals("Invalid value for connection string; port must be numeric, but was 'nonNumericPort'", ex.getMessage()); + } + + private void useConnectionString(String connectionString) { + props = new HashMap() {{ + put(PREFIX + "connectionString", connectionString); + }}; + } + private DatabaseClientFactory.Bean buildBean() { DatabaseClientPropertySource source = new DatabaseClientPropertySource(propertyName -> props.get(propertyName)); return source.newClientBean(); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/DatabaseClientBuilderTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/DatabaseClientBuilderTest.java index 355b7b8fc..246d3afe4 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/DatabaseClientBuilderTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/DatabaseClientBuilderTest.java @@ -5,18 +5,14 @@ import com.marklogic.client.DatabaseClientFactory; import com.marklogic.client.ext.modulesloader.ssl.SimpleX509TrustManager; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import javax.net.ssl.SSLContext; import javax.net.ssl.X509TrustManager; - import java.security.NoSuchAlgorithmException; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; /** * These tests only verify that the Bean instance is built correctly, as in order to verify each connection type, we @@ -42,6 +38,99 @@ void minimumConnectionProperties() { assertTrue(bean.getSecurityContext() instanceof DatabaseClientFactory.BasicAuthContext); } + @Test + void validConnection() { + DatabaseClient client = DatabaseClientFactory.newClient(propertyName -> + "marklogic.client.connectionString".equals(propertyName) ? + String.format("%s:%s@%s:%d", Common.USER, Common.PASS, Common.HOST, Common.PORT) : null); + DatabaseClient.ConnectionResult result = client.checkConnection(); + assertNull(result.getErrorMessage()); + assertTrue(result.isConnected()); + } + + @Test + void connectionString() { + bean = new DatabaseClientBuilder() + .withConnectionString("user:password@localhost:8000") + .buildBean(); + + assertEquals("localhost", bean.getHost()); + assertEquals(8000, bean.getPort()); + assertNull(bean.getDatabase()); + + DatabaseClientFactory.DigestAuthContext context = (DatabaseClientFactory.DigestAuthContext) bean.getSecurityContext(); + assertEquals("user", context.getUser()); + assertEquals("password", context.getPassword()); + } + + @Test + void connectionStringWithDatabase() { + bean = new DatabaseClientBuilder() + .withConnectionString("user:password@localhost:8000/Documents") + .buildBean(); + + assertEquals("localhost", bean.getHost()); + assertEquals(8000, bean.getPort()); + assertEquals("Documents", bean.getDatabase()); + + DatabaseClientFactory.DigestAuthContext context = (DatabaseClientFactory.DigestAuthContext) bean.getSecurityContext(); + assertEquals("user", context.getUser()); + assertEquals("password", context.getPassword()); + } + + @Test + void connectionStringWithSeparateDatabase() { + bean = new DatabaseClientBuilder() + .withDatabase("SomeDatabase") + .withConnectionString("user:password@localhost:8000") + .buildBean(); + + assertEquals("localhost", bean.getHost()); + assertEquals(8000, bean.getPort()); + assertEquals("SomeDatabase", bean.getDatabase()); + } + + @Test + void usernameAndPasswordBothRequireDecoding() { + bean = new DatabaseClientBuilder() + .withConnectionString("test-user%40:sp%40r%3Ak@localhost:8000/Documents") + .buildBean(); + + assertEquals("localhost", bean.getHost()); + assertEquals(8000, bean.getPort()); + assertEquals("Documents", bean.getDatabase()); + + DatabaseClientFactory.DigestAuthContext context = (DatabaseClientFactory.DigestAuthContext) bean.getSecurityContext(); + assertEquals("test-user@", context.getUser()); + assertEquals("sp@r:k", context.getPassword(), "Verifies that the user must encode username and password " + + "values that contain ':' or '@'. The builder is then expected to decode them into the correct values."); + } + + @ParameterizedTest + @ValueSource(strings = { + "user@host@port", + "user@host:port", + "user:password@host", + "user:password:something@host:port", + "user:password@host:port:something" + }) + void invalidConnectionString(String value) { + DatabaseClientBuilder builder = new DatabaseClientBuilder(); + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> builder.withConnectionString(value)); + assertEquals("Invalid value for connection string; must be username:password@host:port/optionalDatabaseName", + ex.getMessage()); + } + + @Test + void nonNumericPortInConnectionString() { + DatabaseClientBuilder builder = new DatabaseClientBuilder(); + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> builder.withConnectionString("user:password@host:nonNumericPort")); + assertEquals("Invalid value for connection string; port must be numeric, but was 'nonNumericPort'", + ex.getMessage()); + } + @Test void allConnectionProperties() { bean = new DatabaseClientBuilder() From 24fc77fabe542723b2e505f3cd25a5efea221801 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 6 Jan 2025 10:58:14 -0500 Subject: [PATCH 37/39] MLE-19240 Allowing for host to override connection string --- .../impl/DatabaseClientPropertySource.java | 28 +++++++++++-------- .../DatabaseClientPropertySourceTest.java | 13 +++++++++ 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientPropertySource.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientPropertySource.java index 18fa3182f..c0e8b09bb 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientPropertySource.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientPropertySource.java @@ -36,6 +36,22 @@ public class DatabaseClientPropertySource { static { connectionPropertyHandlers = new LinkedHashMap<>(); + + // Connection string is handled first so that certain properties can override it as needed. This is useful + // for e.g. specifying a specific host in a cluster while reusing the rest of the connection string. + connectionPropertyHandlers.put(PREFIX + "connectionString", (bean, value) -> { + if (value instanceof String) { + ConnectionString cs = new ConnectionString((String) value, "connection string"); + bean.setHost(cs.getHost()); + bean.setPort(cs.getPort()); + if (cs.getDatabase() != null && cs.getDatabase().trim().length() > 0) { + bean.setDatabase(cs.getDatabase()); + } + } else { + throw new IllegalArgumentException("Connection string must be of type String"); + } + }); + connectionPropertyHandlers.put(PREFIX + "host", (bean, value) -> { if (value instanceof String) { bean.setHost((String) value); @@ -59,18 +75,6 @@ public class DatabaseClientPropertySource { throw new IllegalArgumentException("Database must be of type String"); } }); - connectionPropertyHandlers.put(PREFIX + "connectionString", (bean, value) -> { - if (value instanceof String) { - ConnectionString cs = new ConnectionString((String) value, "connection string"); - bean.setHost(cs.getHost()); - bean.setPort(cs.getPort()); - if (cs.getDatabase() != null && cs.getDatabase().trim().length() > 0) { - bean.setDatabase(cs.getDatabase()); - } - } else { - throw new IllegalArgumentException("Connection string must be of type String"); - } - }); connectionPropertyHandlers.put(PREFIX + "basePath", (bean, value) -> { if (value instanceof String) { bean.setBasePath((String) value); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/impl/DatabaseClientPropertySourceTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/impl/DatabaseClientPropertySourceTest.java index f68f04544..b6fb40a81 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/impl/DatabaseClientPropertySourceTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/impl/DatabaseClientPropertySourceTest.java @@ -196,6 +196,19 @@ void nonNumericPortInConnectionString() { assertEquals("Invalid value for connection string; port must be numeric, but was 'nonNumericPort'", ex.getMessage()); } + @Test + void hostTakesPrecedence() { + props = new HashMap() {{ + put(PREFIX + "host", "somehost"); + put(PREFIX + "connectionString", "user:password@localhost:8000/Documents"); + }}; + + DatabaseClientFactory.Bean bean = buildBean(); + assertEquals("somehost", bean.getHost(), "This allows a user to use a connection string as a starting " + + "point, and then override the host for a direct connection to a particular host in a cluster. This " + + "capability is used by our Spark connector to support direct connections to multiple hosts."); + } + private void useConnectionString(String connectionString) { props = new HashMap() {{ put(PREFIX + "connectionString", connectionString); From 6f174d6f2ed459ed7a6f26413cddb439e28a1265 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 6 Jan 2025 12:15:27 -0500 Subject: [PATCH 38/39] Updating NOTICE file This now fits the convention in our other repositories, only reporting on direct dependencies. Also removed a couple direct dependencies from build.gradle that are already brought in as transitive dependencies. --- NOTICE.txt | 329 +++++------------------------- marklogic-client-api/build.gradle | 3 - 2 files changed, 55 insertions(+), 277 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 18060a760..1e6aff3f2 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,291 +1,81 @@ -MarkLogic® for Java Client API +MarkLogic® Java Client Copyright © 2024 MarkLogic Corporation. All Rights Reserved. -This project is licensed under the Apache License, Version 2.0 (the "License"); you may not use this project except in compliance with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - -To the extent required by the applicable open-source license, a complete machine-readable copy of the source code corresponding to such code is available upon request. This offer is valid to anyone in receipt of this information and shall expire three years following the date of the final distribution of this product version by MarkLogic Corporation. To obtain such source code, send an email to Legal-thirdpartyreview@progress.com. Please specify the product and version for which you are requesting source code. - -The following software may be included in this project (last updated October 3, 2023): - -------------------------------------------------------------------------- -Third Party Components - commons-io 2.11.0 (Apache-2.0) - commons-lang3 3.14.0 (Apache-2.0) - dom4j 2.1.4 (BSD) - gson 2.10.1 (Apache-2.0) - htmlcleaner 2.29 (BSD) - httpclient 4.5.14 (Apache-2.0) - jackson-annotations 2.17.2 (Apache-2.0) - jackson-core 2.17.2 (Apache-2.0) - jackson-databind 2.17.2 (Apache-2.0) - jackson-dataformat-csv 2.17.2 (Apache-2.0) - jackson-module-kotlin 2.17.2 (Apache-2.0) - jakarta.mail 2.0.1 (CDDL-1.1) - javax.ws.rs-api 2.1.1 (CDDL-1.1) - jakarta.xml.bind-api 3.0.1 (CDDL-1.1) - jaxb-core 3.0.2 (CDDL-1.1) - jaxb-runtime 3.0.2 (CDDL-1.1) - jdom2 2.0.6.1 (BSD) - json-schema-validator 1.0.88 (Apache-2.0) - jsonassert 1.5.1 (Apache-2.0) - kotlin-stdlib 1.8.22 (Apache-2.0) - logback-classic 1.3.14 (EPL-1.0) - logging-interceptor 4.12.0 (Apache-2.0) - okhttp 4.12.0 (Apache-2.0) - okhttp-digest 2.7 (Apache-2.0) - okio 3.6.0 (Apache-2.0) - opencsv 4.6 (Apache-2.0) - slf4j-api 1.7.36 (MIT) - spring-jdbc 5.3.39 (Apache-2.0) - undertow-core 2.2.32.Final (Apache-2.0) - undertow-servlet 2.2.32.Final (Apache-2.0) +To the extent required by the applicable open-source license, a complete machine-readable copy of the source code +corresponding to such code is available upon request. This offer is valid to anyone in receipt of this information and +shall expire three years following the date of the final distribution of this product version by Progress Software +Corporation. To obtain such source code, send an email to Legal-thirdpartyreview@progress.com. Please specify the +product and version for which you are requesting source code. + +Third Party Notices + +jackson-databind 2.17.2 (Apache-2.0) +jackson-dataformat-csv 2.17.2 (Apache-2.0) +okhttp 4.12.0 (Apache-2.0) +logging-interceptor 4.12.0 (Apache-2.0) +jakarta.mail 2.0.1 (EPL-1.0) +okhttp-digest 2.7 (Apache-2.0) +jakarta.xml.bind-api 3.0.1 (EPL-1.0) +javax.ws.rs-api 2.1.1 (CDDL-1.1) +jaxb-runtime 3.0.2 (CDDL-1.1) +slf4j-api 2.0.16 (Apache-2.0) Common Licenses - Apache License 2.0 (Apache-2.0) - Common Development and Distribution License 1.1 (CDDL-1.1) - Eclipse Public License 1.0 (EPL-1.0) +Apache License 2.0 (Apache-2.0) +Common Development and Distribution License 1.1 (CDDL-1.1) +Eclipse Public License 1.0 (EPL-1.0) ------------------------------------------ Third-Party Components - The following is a list of the third-party components used by MarkLogic for Java Client API. - -commons-io 2.11.0 (Apache-2.0) - - https://repo1.maven.org/maven2/commons-io/commons-io - - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -commons-lang3 3.14.0 (Apache-2.0) - - https://repo1.maven.org/maven2/org/apache/commons/commons-lang3 - - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -dom4j 2.1.4 (BSD) - - https://repo1.maven.org/maven2/org/dom4j/dom4j - - Copyright 2001-2023 © MetaStuff, Ltd. and DOM4J contributors. All Rights Reserved. -Redistribution and use of this software and associated documentation -("Software"), with or without modification, are permitted provided -that the following conditions are met: -1. Redistributions of source code must retain copyright - statements and notices. Redistributions must also contain a - copy of this document. -2. Redistributions in binary form must reproduce the - above copyright notice, this list of conditions and the - following disclaimer in the documentation and/or other - materials provided with the distribution. -3. The name "DOM4J" must not be used to endorse or promote - products derived from this Software without prior written - permission of MetaStuff, Ltd. For written permission, - please contact dom4j-info@metastuff.com. -4. Products derived from this Software may not be called "DOM4J" - nor may "DOM4J" appear in their names without prior written - permission of MetaStuff, Ltd. DOM4J is a registered - trademark of MetaStuff, Ltd. -5. Due credit should be given to the DOM4J Project - https://dom4j.github.io/ -THIS SOFTWARE IS PROVIDED BY METASTUFF, LTD. AND CONTRIBUTORS -“AS IS” AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT -NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL -METASTUFF, LTD. OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, -INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED -OF THE POSSIBILITY OF SUCH DAMAGE. - -gson 2.10.1 (Apache-2.0) - - https://repo1.maven.org/maven2/com/google/code/gson/gson - - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -htmlcleaner 2.29 (BSD) - - https://repo1.maven.org/maven2/net/sourceforge/htmlcleaner/htmlcleaner - - Copyright (c) 2006-2023, the HTMLCleaner project All rights reserved. Redistribution and use of this software in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -* The name of HtmlCleaner may not be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Report any issues and contact the developers through Sourceforge at https://sourceforge.net/projects/htmlcleaner/ - -httpclient 4.5.14 (Apache-2.0) - - https://repo1.maven.org/maven2/org/apache/httpcomponents/httpclient - - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -jackson-annotations 2.17.2 (Apache-2.0) - - https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-annotations - - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -jackson-core 2.17.2 (Apache-2.0) - - https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core - - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -jackson-databind 2.17.2 (Apache-2.0) - - https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-databind - - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -jackson-dataformat-csv 2.17.2 (Apache-2.0) - - https://repo1.maven.org/maven2/com/fasterxml/jackson/dataformat/jackson-dataformat-csv - - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -jackson-module-kotlin 2.17.2 (Apache-2.0) - - https://repo1.maven.org/maven2/com/fasterxml/jackson/module/jackson-module-kotlin - - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -jakarta.mail 2.0.1 (CDDL-1.1) - - https://repo1.maven.org/maven2/com/sun/mail/jakarta.mail - - For the full text of the CDDL-1.1 license, see Common Development and Distribution License 1.1 (CDDL-1.1) - -javax.ws.rs-api 2.1.1 (CDDL-1.1) - - https://repo1.maven.org/maven2/javax/ws/rs/javax.ws.rs-api - - For the full text of the CDDL-1.1 license, see Common Development and Distribution License 1.1 (CDDL-1.1) - -jakarta.xml.bind-api 3.0.1 (CDDL-1.1) - - https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api - - - For the full text of the CDDL-1.1 license, see Common Development and Distribution License 1.1 (CDDL-1.1) -jaxb-core 3.0.2 (CDDL-1.1) +The following is a list of the third-party components used by the MarkLogic® Java Client 7.1.0 (last updated January 6, 2025): - https://repo1.maven.org/maven2/org/glassfish/jaxb/jaxb-core +jackson-databind 2.17.2 (Apache-2.0) +https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-databind/ +For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - For the full text of the CDDL-1.1 license, see Common Development and Distribution License 1.1 (CDDL-1.1) +jackson-dataformat-csv 2.17.2 (Apache-2.0) +https://repo1.maven.org/maven2/com/fasterxml/jackson/dataformat/jackson-dataformat-csv/ +For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) -jaxb-runtime 3.0.2 (CDDL-1.1) +okhttp 4.12.0 (Apache-2.0) +https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp/ +For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - https://repo1.maven.org/maven2/org/glassfish/jaxb/jaxb-runtime +logging-interceptor 4.12.0 (Apache-2.0) +https://repo1.maven.org/maven2/com/squareup/okhttp3/logging-interceptor/ +For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - For the full text of the CDDL-1.1 license, see Common Development and Distribution License 1.1 (CDDL-1.1) +jakarta.mail 2.0.1 (Apache-2.0) +https://repo1.maven.org/maven2/com/sun/mail/jakarta.mail/ +For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) -jdom2 2.0.6.1 (BSD) +okhttp-digest 2.7 (Apache-2.0) +https://repo1.maven.org/maven2/io/github/rburgst/okhttp-digest/ +For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - https://repo1.maven.org/maven2/org/jdom/jdom2 +jakarta.xml.bind-api 3.0.1 (Apache-2.0) +https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api/ +For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - Copyright (C) 2000-2012 Jason Hunter & Brett McLaughlin. All rights reserved. -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -1. Redistributions of source code must retain the above copyright notice, this list of conditions, and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions, and the disclaimer that follows these conditions in the documentation and/or other materials provided with the distribution. -3. The name "JDOM" must not be used to endorse or promote products derived from this software without prior written permission. For written permission, please contact request_AT_jdom_DOT_org. -4. Products derived from this software may not be called "JDOM", nor may "JDOM" appear in their name, without prior written permission from the JDOM Project Management, please contact request_AT_jdom_DOT_org. -In addition, we request (but do not require) that you include in the end-user documentation provided with the redistribution and/or in the software itself an acknowledgement equivalent to the following: "This product includes software developed by the JDOM Project (http://www.jdom.org/)." Alternatively, the acknowledgment may be graphical using the logos available at http://www.jdom.org/images/logos. -THIS SOFTWARE IS PROVIDED “AS IS” AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE JDOM AUTHORS OR THE PROJECT CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. This software consists of voluntary contributions made by many individuals on behalf of the JDOM Project and was originally created by Jason Hunter; jhunter_AT_jdom_DOT_org; and Brett McLaughlin brett_AT_jdom_DOT_org. For more information on the JDOM Project, please see http://www.jdom.org. +javax.ws.rs-api 2.1.1 (Apache-2.0) +https://repo1.maven.org/maven2/javax/ws/rs/javax.ws.rs-api/ +For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) -json-schema-validator 1.0.88 (Apache-2.0) +jaxb-runtime 3.0.2 (Apache-2.0) +https://repo1.maven.org/maven2/org/glassfish/jaxb/jaxb-runtime/ +For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - https://repo1.maven.org/maven2/com/networknt/json-schema-validator +slf4j-api 2.0.16 (Apache-2.0) +https://repo1.maven.org/maven2/org/slf4j/slf4j-api/ +For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -jsonassert 1.5.1 (Apache-2.0) - - https://repo1.maven.org/maven2/org/skyscreamer/jsonassert - - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -kotlin-stdlib 1.8.22 (Apache-2.0) - - https://repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-stdlib - - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -logback-classic 1.3.14 (EPL-1.0) - - https://repo1.maven.org/maven2/ch/qos/logback/logback-classic - - Logback LICENSE -Logback: the reliable, generic, fast and flexible logging framework. Copyright (C) 1999-2015, QOS.ch. All rights reserved. This program and the accompanying materials are dual-licensed under either the terms of the Eclipse Public License v1.0 as published by the Eclipse Foundation or (per the licensee's choosing) under the terms of the GNU Lesser General Public License version 2.1 as published by the Free Software Foundation. - -For the full text of the EPL-1.0 license, see Eclipse Public License 1.0 (EPL-1.0) - -logging-interceptor 4.12.0 (Apache-2.0) - - https://repo1.maven.org/maven2/com/squareup/okhttp3/logging-interceptor - - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -okhttp 4.12.0 (Apache-2.0) - - https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp - - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -okhttp-digest 2.7 (Apache-2.0) - - https://repo1.maven.org/maven2/io/github/rburgst/okhttp-digest - - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -okio 3.6.0 (Apache-2.0) - - https://repo1.maven.org/maven2/com/squareup/okio/okio - - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -opencsv 4.6 (Apache-2.0) - - https://repo1.maven.org/maven2/com/opencsv/opencsv - - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -slf4j-api 1.7.36 (MIT) - - https://repo1.maven.org/maven2/org/slf4j/slf4j-api - - Copyright (c) 2004-2023 QOS.ch All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED “AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -spring-jdbc 5.3.39 (Apache-2.0) - - https://repo1.maven.org/maven2/org/springframework/spring-jdbc - - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -undertow-core 2.2.32.Final (Apache-2.0) - - https://repo1.maven.org/maven2/io/undertow/undertow-core - - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - -undertow-servlet 2.2.32.Final (Apache-2.0) - - https://repo1.maven.org/maven2/io/undertow/undertow-servlet - - For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) - ------------------------------------------ Common Licenses -This section shows the text of common third-party licenses used by MarkLogic for Java Client API. +This section shows the text of common third-party licenses used by MarkLogic® Flux™ v1 (last updated July 2, 2024): -Apache License 2.0 (Apache-2.0) +Apache License 2.0 (Apache-2.0) https://spdx.org/licenses/Apache-2.0.html Apache License @@ -362,10 +152,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - ==================== - Common Development and Distribution License 1.1 (CDDL-1.1) https://spdx.org/licenses/CDDL-1.1.html @@ -524,10 +312,3 @@ Everyone is permitted to copy and distribute copies of this Agreement, but in or This Agreement is governed by the laws of the State of New York and the intellectual property laws of the United States of America. No party to this Agreement will bring a legal action under this Agreement more than one year after the cause of action arose. Each party waives its rights to a jury trial in any resulting litigation. - -==================== - - - - - diff --git a/marklogic-client-api/build.gradle b/marklogic-client-api/build.gradle index 9a3b6e2fb..8bfe32805 100644 --- a/marklogic-client-api/build.gradle +++ b/marklogic-client-api/build.gradle @@ -15,7 +15,6 @@ dependencies { // whereas the 4.x version requires Java 11 or higher. api "jakarta.xml.bind:jakarta.xml.bind-api:3.0.1" implementation "org.glassfish.jaxb:jaxb-runtime:3.0.2" - implementation "org.glassfish.jaxb:jaxb-core:3.0.2" implementation 'com.squareup.okhttp3:okhttp:4.12.0' implementation 'com.squareup.okhttp3:logging-interceptor:4.12.0' @@ -30,8 +29,6 @@ dependencies { implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' implementation 'org.slf4j:slf4j-api:2.0.16' - implementation "com.fasterxml.jackson.core:jackson-core:2.17.2" - implementation "com.fasterxml.jackson.core:jackson-annotations:2.17.2" implementation "com.fasterxml.jackson.core:jackson-databind:2.17.2" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.17.2" From 54ed64904a7e7f34d6d28e358b10da8f32a9e88b Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 6 Jan 2025 08:13:14 -0500 Subject: [PATCH 39/39] Bumped to 7.1.0 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index d2af75728..71f97e040 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ group=com.marklogic -version=7.1-SNAPSHOT +version=7.1.0 describedName=MarkLogic Java Client API publishUrl=file:../marklogic-java/releases