This is an automated email from the ASF dual-hosted git repository.
OpenPj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/manifoldcf.git
The following commit(s) were added to refs/heads/trunk by this push:
new 663645e17 Upgrade MongoDB and GridFS connectors to MongoDB Java sync
driver 5.x (CONNECTORS-1791)
663645e17 is described below
commit 663645e1799dbee0de79d45b728f617dae5e2f9b
Author: Piergiorgio Lucidi <[email protected]>
AuthorDate: Tue Oct 6 13:29:01 2026 +0200
Upgrade MongoDB and GridFS connectors to MongoDB Java sync driver 5.x
(CONNECTORS-1791)
---
.github/workflows/ci.yml | 9 +-
build.xml | 28 +-
connectors/gridfs/build.xml | 70 ++++-
.../crawler/connectors/gridfs/GridFSConstants.java | 8 +
.../gridfs/GridFSRepositoryConnector.java | 325 +++++++++++++--------
.../crawler/connectors/gridfs/Messages.java | 8 +-
.../tests/GridFSRepositoryConnectorHSQLDBIT.java | 318 ++++++++++++++++++++
connectors/gridfs/pom.xml | 41 ++-
.../{mongodb => gridfs}/test-materials/README.txt | 5 +-
connectors/mongodb/build.xml | 275 ++++++++---------
.../mongodboutput/MongodbOutputConnector.java | 183 +++++-------
.../mongodboutput/tests/APISanityHSQLDBIT.java | 63 ++--
.../output/mongodboutput/tests/BaseITHSQLDB.java | 39 +--
.../mongodboutput/tests/BaseITSanityTestUtils.java | 4 +-
connectors/mongodb/pom.xml | 9 +-
connectors/mongodb/test-materials/README.txt | 5 +-
16 files changed, 915 insertions(+), 475 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index aee4a5642..7be2134d3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -20,8 +20,8 @@ run-name: ${{ github.actor }} is executing CI build and tests
🚀
on:
push:
- branches:
- - trunk
+ tags:
+ - '*'
pull_request:
branches:
- trunk
@@ -43,11 +43,6 @@ jobs:
distribution: 'temurin'
architecture: x64
cache: maven
-
- - name: Install OpenSSL 1.1 runtime for embedded legacy MongoDB
- run: |
- sudo apt-get update
- sudo apt-get install -y libssl1.1
- name: Ant Build and Test
run: ant make-core-deps make-deps build test
diff --git a/build.xml b/build.xml
index e46b33c7a..27e2605fb 100644
--- a/build.xml
+++ b/build.xml
@@ -35,7 +35,7 @@
<property name="zookeeper.version" value="3.9.1"/>
<property name="metrics-core.version" value="4.2.21"/>
<property name="snappy-java.version" value="1.1.7.7"/>
- <property name="mongodb.version" value="2.14.3"/>
+ <property name="mongodb-driver-sync.version" value="5.6.5"/>
<property name="postgresql.version" value="42.7.4"/>
<property name="axis.version" value="1.4"/>
<property name="saaj-api.version" value="1.3"/>
@@ -1010,11 +1010,33 @@ Use Apache Forrest version forrest-0.9-dev or higher.
<target name="download-mongo-java-driver">
<mkdir dir="lib"/>
+ <!-- Modern MongoDB sync driver (MongoDB 9.x compliant), used by the
MongoDB output and GridFS repository connectors -->
<antcall target="download-via-maven">
<param name="project-path" value="org/mongodb"/>
- <param name="artifact-version" value="${mongodb.version}"/>
+ <param name="artifact-version"
value="${mongodb-driver-sync.version}"/>
<param name="target" value="lib"/>
- <param name="artifact-name" value="mongo-java-driver"/>
+ <param name="artifact-name" value="mongodb-driver-sync"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven">
+ <param name="project-path" value="org/mongodb"/>
+ <param name="artifact-version"
value="${mongodb-driver-sync.version}"/>
+ <param name="target" value="lib"/>
+ <param name="artifact-name" value="mongodb-driver-core"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven">
+ <param name="project-path" value="org/mongodb"/>
+ <param name="artifact-version"
value="${mongodb-driver-sync.version}"/>
+ <param name="target" value="lib"/>
+ <param name="artifact-name" value="bson"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven">
+ <param name="project-path" value="org/mongodb"/>
+ <param name="artifact-version"
value="${mongodb-driver-sync.version}"/>
+ <param name="target" value="lib"/>
+ <param name="artifact-name" value="bson-record-codec"/>
<param name="artifact-type" value="jar"/>
</antcall>
</target>
diff --git a/connectors/gridfs/build.xml b/connectors/gridfs/build.xml
index 02941f054..a231c1e0a 100644
--- a/connectors/gridfs/build.xml
+++ b/connectors/gridfs/build.xml
@@ -30,10 +30,23 @@
<import file="${mcf-dist}/connector-build.xml"/>
+ <target name="download-cleanup">
+ <delete>
+ <fileset dir="test-materials" excludes="README*.txt"
erroronmissingdir="false"/>
+ </delete>
+ </target>
+
+ <!-- MongoDB sync driver (MongoDB 9.x compliant) -->
+ <patternset id="mongodb-driver-jars">
+ <include name="mongodb-driver-sync-*.jar"/>
+ <include name="mongodb-driver-core-*.jar"/>
+ <include name="bson-*.jar"/>
+ </patternset>
+
<path id="connector-classpath">
<path refid="mcf-connector-build.connector-classpath"/>
<fileset dir="../../lib">
- <include name="mongo-java-driver*.jar"/>
+ <patternset refid="mongodb-driver-jars"/>
</fileset>
</path>
@@ -41,7 +54,7 @@
<mkdir dir="dist/lib"/>
<copy todir="dist/lib">
<fileset dir="../../lib">
- <include name="mongo-java-driver*.jar"/>
+ <patternset refid="mongodb-driver-jars"/>
</fileset>
</copy>
</target>
@@ -52,4 +65,57 @@
<param name="connector-class"
value="org.apache.manifoldcf.crawler.connectors.gridfs.GridFSRepositoryConnector"/>
</antcall>
</target>
+
+ <!-- Test materials: Testcontainers is used to run MongoDB 9.0.2 as a
Docker container during the tests -->
+ <property name="testcontainers.version" value="2.0.5"/>
+ <property name="docker-java.version" value="3.7.1"/>
+ <property name="duct-tape.version" value="1.0.8"/>
+ <property name="jna.version" value="5.10.0"/>
+ <property name="commons-compress.version" value="1.28.0"/>
+ <property name="jetbrains-annotations.version" value="17.0.0"/>
+ <property name="jackson-annotations.version" value="2.21"/>
+
+ <!-- Test materials come first so that the newer jars required by
Testcontainers
+ (e.g. jackson-annotations, commons-compress) take precedence over the
older shared ones -->
+ <path id="connector-test-classpath">
+ <fileset dir="test-materials" includes="*.jar"
erroronmissingdir="false"/>
+ <path refid="mcf-connector-build.connector-test-classpath"/>
+ </path>
+
+ <target name="calculate-testcode-condition">
+ <condition property="tests-present">
+ <and>
+ <available
file="test-materials/testcontainers-${testcontainers.version}.jar"/>
+ <available
file="test-materials/testcontainers-mongodb-${testcontainers.version}.jar"/>
+ <available
file="test-materials/docker-java-api-${docker-java.version}.jar"/>
+ <available
file="test-materials/docker-java-transport-${docker-java.version}.jar"/>
+ <available
file="test-materials/docker-java-transport-zerodep-${docker-java.version}.jar"/>
+ <available
file="test-materials/duct-tape-${duct-tape.version}.jar"/>
+ <available file="test-materials/jna-${jna.version}.jar"/>
+ <available
file="test-materials/commons-compress-${commons-compress.version}.jar"/>
+ <available
file="test-materials/annotations-${jetbrains-annotations.version}.jar"/>
+ <available
file="test-materials/jackson-annotations-${jackson-annotations.version}.jar"/>
+ </and>
+ </condition>
+ </target>
+
+ <target name="pretest-warn" depends="calculate-testcode-condition"
unless="tests-present">
+ <echo message="Need Testcontainers download in order to run tests -
run ant make-deps (Docker is required to run MongoDB 9.0.2)"/>
+ </target>
+
+ <target name="download-dependencies">
+ <property name="maven-repo" value="https://repo1.maven.org/maven2"/>
+ <mkdir dir="test-materials"/>
+ <get
src="${maven-repo}/org/testcontainers/testcontainers/${testcontainers.version}/testcontainers-${testcontainers.version}.jar"
dest="test-materials/testcontainers-${testcontainers.version}.jar"/>
+ <get
src="${maven-repo}/org/testcontainers/testcontainers-mongodb/${testcontainers.version}/testcontainers-mongodb-${testcontainers.version}.jar"
dest="test-materials/testcontainers-mongodb-${testcontainers.version}.jar"/>
+ <get
src="${maven-repo}/com/github/docker-java/docker-java-api/${docker-java.version}/docker-java-api-${docker-java.version}.jar"
dest="test-materials/docker-java-api-${docker-java.version}.jar"/>
+ <get
src="${maven-repo}/com/github/docker-java/docker-java-transport/${docker-java.version}/docker-java-transport-${docker-java.version}.jar"
dest="test-materials/docker-java-transport-${docker-java.version}.jar"/>
+ <get
src="${maven-repo}/com/github/docker-java/docker-java-transport-zerodep/${docker-java.version}/docker-java-transport-zerodep-${docker-java.version}.jar"
dest="test-materials/docker-java-transport-zerodep-${docker-java.version}.jar"/>
+ <get
src="${maven-repo}/org/rnorth/duct-tape/duct-tape/${duct-tape.version}/duct-tape-${duct-tape.version}.jar"
dest="test-materials/duct-tape-${duct-tape.version}.jar"/>
+ <get
src="${maven-repo}/net/java/dev/jna/jna/${jna.version}/jna-${jna.version}.jar"
dest="test-materials/jna-${jna.version}.jar"/>
+ <get
src="${maven-repo}/org/apache/commons/commons-compress/${commons-compress.version}/commons-compress-${commons-compress.version}.jar"
dest="test-materials/commons-compress-${commons-compress.version}.jar"/>
+ <get
src="${maven-repo}/org/jetbrains/annotations/${jetbrains-annotations.version}/annotations-${jetbrains-annotations.version}.jar"
dest="test-materials/annotations-${jetbrains-annotations.version}.jar"/>
+ <get
src="${maven-repo}/com/fasterxml/jackson/core/jackson-annotations/${jackson-annotations.version}/jackson-annotations-${jackson-annotations.version}.jar"
dest="test-materials/jackson-annotations-${jackson-annotations.version}.jar"/>
+ </target>
+
</project>
diff --git
a/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/GridFSConstants.java
b/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/GridFSConstants.java
index 70576c34d..92fd2cf89 100644
---
a/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/GridFSConstants.java
+++
b/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/GridFSConstants.java
@@ -81,6 +81,14 @@ public class GridFSConstants {
* MongoDB default unique key field name.
*/
protected static final String DEFAULT_ID_FIELD_NAME = "_id";
+ /**
+ * GridFS files document metadata field name.
+ */
+ protected static final String METADATA_FIELD_NAME = "metadata";
+ /**
+ * GridFS content type field name (top-level for legacy drivers, inside
metadata for modern ones).
+ */
+ protected static final String CONTENT_TYPE_FIELD_NAME = "contentType";
/**
* Job start point node type.
*/
diff --git
a/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/GridFSRepositoryConnector.java
b/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/GridFSRepositoryConnector.java
index 9a4afb472..b28a25743 100644
---
a/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/GridFSRepositoryConnector.java
+++
b/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/GridFSRepositoryConnector.java
@@ -15,26 +15,31 @@
*/
package org.apache.manifoldcf.crawler.connectors.gridfs;
-import com.mongodb.DB;
-import com.mongodb.DBCollection;
-import com.mongodb.DBCursor;
-import com.mongodb.DBObject;
-import com.mongodb.DBTCPConnector;
-import com.mongodb.Mongo;
-import com.mongodb.MongoClient;
-import com.mongodb.gridfs.GridFS;
-import com.mongodb.gridfs.GridFSDBFile;
+import com.mongodb.MongoClientSettings;
+import com.mongodb.MongoCredential;
+import com.mongodb.MongoException;
+import com.mongodb.ServerAddress;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoCursor;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.gridfs.GridFSBucket;
+import com.mongodb.client.gridfs.GridFSBuckets;
+import com.mongodb.client.model.Filters;
+import com.mongodb.client.model.Projections;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.InputStream;
-import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
-import org.apache.commons.lang.StringUtils;
+
+import org.apache.commons.lang3.StringUtils;
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
import org.apache.manifoldcf.core.interfaces.ConfigParams;
@@ -49,6 +54,8 @@ import
org.apache.manifoldcf.crawler.interfaces.IProcessActivity;
import org.apache.manifoldcf.crawler.interfaces.ISeedingActivity;
import org.apache.manifoldcf.crawler.interfaces.IExistingVersions;
import org.apache.manifoldcf.crawler.system.Logging;
+import org.bson.BsonValue;
+import org.bson.Document;
import org.bson.types.ObjectId;
/**
@@ -106,9 +113,13 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
*/
protected String denyAcl = null;
/**
- * MongoDB session.
+ * MongoDB client (owns the connection pool).
+ */
+ protected MongoClient client = null;
+ /**
+ * MongoDB session (database handle).
*/
- protected DB session = null;
+ protected MongoDatabase session = null;
/**
* Last session fetch time.
*/
@@ -214,20 +225,18 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
try {
getSession();
if (session != null) {
- Mongo currentMongoSession = session.getMongo();
- currentMongoSession.getConnector()
- .getDBPortPool(currentMongoSession.getAddress())
- .get()
- .ensureOpen();
- session.getMongo().close();
- session = null;
+ // The modern driver connects and authenticates lazily: a ping
forces a round trip
+ session.runCommand(new Document("ping", 1));
+ closeSession();
return super.check();
}
return "Not connected.";
} catch (ManifoldCFException e) {
+ closeSession();
return e.getMessage();
- } catch (IOException ex) {
- return ex.getMessage();
+ } catch (MongoException ex) {
+ closeSession();
+ return "GridFS: Connection check failed: " + ex.getMessage();
}
}
@@ -241,13 +250,11 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
public void disconnect() throws ManifoldCFException {
if (session != null) {
try {
- session.getMongo().close();
+ closeSession();
} catch (Exception e) {
Logging.connectors.error("GridFS: Error when trying to
disconnect: " + e.getMessage());
throw new ManifoldCFException("GridFS: Error when trying to
disconnect: " + e.getMessage(), e);
}
- session = null;
- lastSessionFetch = -1L;
username = null;
password = null;
host = null;
@@ -258,6 +265,7 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
acl = null;
denyAcl = null;
}
+ super.disconnect();
}
/**
@@ -274,11 +282,7 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
long currentTime = System.currentTimeMillis();
if (currentTime >= lastSessionFetch + SESSION_EXPIRATION_MILLISECONDS)
{
- if (session != null) {
- session.getMongo().close();
- session = null;
- }
- lastSessionFetch = -1L;
+ closeSession();
}
}
@@ -290,12 +294,25 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
*/
@Override
public boolean isConnected() {
- if (session == null) {
- return false;
+ // The client manages its own connection pool (reconnecting as
needed), so an open
+ // client is considered connected; no network round trip is done here.
+ return client != null && session != null;
+ }
+
+ /**
+ * Close the MongoDB client (and its connection pool), if any.
+ */
+ protected void closeSession() {
+ if (client != null) {
+ try {
+ client.close();
+ } catch (Exception e) {
+ Logging.connectors.warn("GridFS: Error closing MongoDB client:
" + e.getMessage(), e);
+ }
}
- Mongo currentMongoSession = session.getMongo();
- DBTCPConnector currentTCPConnection =
currentMongoSession.getConnector();
- return currentTCPConnection.isOpen();
+ client = null;
+ session = null;
+ lastSessionFetch = -1L;
}
/**
@@ -351,17 +368,20 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
String lastSeedVersion, long seedTime, int jobMode)
throws ManifoldCFException, ServiceInterruption {
getSession();
- DBCollection fsFiles = session.getCollection(
+ MongoCollection<Document> fsFiles = session.getCollection(
bucket + GridFSConstants.COLLECTION_SEPERATOR +
GridFSConstants.FILES_COLLECTION_NAME
);
- DBCursor dnc = fsFiles.find();
- while (dnc.hasNext()) {
- DBObject dbo = dnc.next();
- String _id = dbo.get("_id").toString();
- activities.addSeedDocument(_id);
- if (Logging.connectors.isDebugEnabled()) {
- Logging.connectors.debug("GridFS: Document _id = " + _id + "
added to queue");
+ try (MongoCursor<Document> dnc =
fsFiles.find().projection(Projections.include("_id")).iterator()) {
+ while (dnc.hasNext()) {
+ Document dbo = dnc.next();
+ String _id = dbo.get("_id").toString();
+ activities.addSeedDocument(_id);
+ if (Logging.connectors.isDebugEnabled()) {
+ Logging.connectors.debug("GridFS: Document _id = " + _id +
" added to queue");
+ }
}
+ } catch (MongoException e) {
+ handleMongoException(e);
}
return "";
}
@@ -387,22 +407,34 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
for (String documentIdentifier : documentIdentifiers) {
String versionString;
- GridFS gfs;
- GridFSDBFile document;
+ Document document;
+ Document metadata;
getSession();
String _id = documentIdentifier;
- gfs = new GridFS(session, bucket);
- document = gfs.findOne(new ObjectId(_id));
+ // Identifiers are normally ObjectIds, but GridFS allows any _id
type: fall back to the raw string
+ Object idValue = ObjectId.isValid(_id) ? new ObjectId(_id) : _id;
+ GridFSBucket gfs = GridFSBuckets.create(session, bucket);
+ MongoCollection<Document> fsFiles = session.getCollection(
+ bucket + GridFSConstants.COLLECTION_SEPERATOR +
GridFSConstants.FILES_COLLECTION_NAME
+ );
+ try {
+ // Read the raw files document so that legacy top-level fields
(e.g. contentType) are preserved
+ document = fsFiles.find(Filters.eq("_id", idValue)).first();
+ } catch (MongoException e) {
+ handleMongoException(e);
+ return;
+ }
if (document == null) {
activities.deleteDocument(documentIdentifier);
continue;
- } else {
- DBObject metadata = document.getMetaData();
- versionString = document.getMD5() + "+" + metadata != null
- ? Integer.toString(metadata.hashCode())
- : StringUtils.EMPTY;
}
+ metadata = document.get(GridFSConstants.METADATA_FIELD_NAME,
Document.class);
+ long fileLenght = getFileLength(document);
+ Date createdDate = document.getDate("uploadDate");
+ versionString = fileLenght + "+"
+ + (createdDate == null ? "" :
Long.toString(createdDate.getTime())) + "+"
+ + (metadata == null ? "" :
Integer.toString(metadata.toJson().hashCode()));
if (versionString.length() == 0 ||
activities.checkDocumentNeedsReindexing(documentIdentifier, versionString)) {
long startTime = System.currentTimeMillis();
@@ -415,7 +447,6 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
Logging.connectors.debug("GridFS: Processing document
_id = " + _id);
}
- DBObject metadata = document.getMetaData();
if (metadata == null) {
errorCode = "NULLMETADATA";
errorDesc = "Excluded because document had a null
Metadata";
@@ -424,9 +455,9 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
continue;
}
- String urlValue = document.getMetaData().get(this.url) ==
null
+ String urlValue = metadata.get(this.url) == null
? StringUtils.EMPTY
- : document.getMetaData().get(this.url).toString();
+ : metadata.get(this.url).toString();
if (!StringUtils.isEmpty(urlValue)) {
boolean validURL;
try {
@@ -436,14 +467,12 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
validURL = false;
}
if (validURL) {
- long fileLenght = document.getLength();
- Date createdDate = document.getUploadDate();
- String fileName = document.getFilename();
- String mimeType = document.getContentType();
+ String fileName = document.getString("filename");
+ String mimeType = getContentType(document,
metadata);
if (!activities.checkURLIndexable(urlValue)) {
Logging.connectors.warn("GridFS: Document " +
_id + " has a URL excluded by the output connector ('" + urlValue + "') -
skipping.");
- errorCode = activities.EXCLUDED_URL;
+ errorCode = IProcessActivity.EXCLUDED_URL;
errorDesc = "Excluded because of URL (" +
urlValue + ")";
activities.noDocument(_id, version);
continue;
@@ -451,7 +480,7 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
if (!activities.checkLengthIndexable(fileLenght)) {
Logging.connectors.warn("GridFS: Document " +
_id + " has a length excluded by the output connector (" + fileLenght + ") -
skipping.");
- errorCode = activities.EXCLUDED_LENGTH;
+ errorCode = IProcessActivity.EXCLUDED_LENGTH;
errorDesc = "Excluded because of length (" +
fileLenght + ")";
activities.noDocument(_id, version);
continue;
@@ -459,7 +488,7 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
if (!activities.checkMimeTypeIndexable(mimeType)) {
Logging.connectors.warn("GridFS: Document " +
_id + " has a mime type excluded by the output connector ('" + mimeType + "') -
skipping.");
- errorCode = activities.EXCLUDED_MIMETYPE;
+ errorCode = IProcessActivity.EXCLUDED_MIMETYPE;
errorDesc = "Excluded because of mime type ("
+ mimeType + ")";
activities.noDocument(_id, version);
continue;
@@ -467,7 +496,7 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
if (!activities.checkDateIndexable(createdDate)) {
Logging.connectors.warn("GridFS: Document " +
_id + " has a date excluded by the output connector (" + createdDate + ") -
skipping.");
- errorCode = activities.EXCLUDED_DATE;
+ errorCode = IProcessActivity.EXCLUDED_DATE;
errorDesc = "Excluded because of date (" +
createdDate + ")";
activities.noDocument(_id, version);
continue;
@@ -482,10 +511,10 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
String[] denyAclsArray = null;
if (acl != null) {
try {
- Object aclObject =
document.getMetaData().get(acl);
+ Object aclObject = metadata.get(acl);
if (aclObject != null) {
- List<String> acls = (List<String>)
aclObject;
- aclsArray = (String[]) acls.toArray();
+ List<String> acls =
toStringList(aclObject);
+ aclsArray = acls.toArray(new
String[0]);
}
} catch (ClassCastException e) {
// This is bad because security will fail
@@ -497,11 +526,12 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
}
if (denyAcl != null) {
try {
- Object denyAclObject =
document.getMetaData().get(denyAcl);
+ Object denyAclObject =
metadata.get(denyAcl);
if (denyAclObject != null) {
- List<String> denyAcls = (List<String>)
denyAclObject;
+ // Copy, so that the document's
metadata is not modified
+ List<String> denyAcls = new
ArrayList<String>(toStringList(denyAclObject));
denyAcls.add(GLOBAL_DENY_TOKEN);
- denyAclsArray = (String[])
denyAcls.toArray();
+ denyAclsArray = denyAcls.toArray(new
String[0]);
}
} catch (ClassCastException e) {
// This is bad because security will fail
@@ -513,45 +543,82 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
}
rd.setSecurity(RepositoryDocument.SECURITY_TYPE_DOCUMENT, aclsArray,
denyAclsArray);
- InputStream is = document.getInputStream();
- try {
+ try (InputStream is =
gfs.openDownloadStream(toBsonValue(document.get("_id")))) {
rd.setBinary(is, fileLenght);
- try {
-
activities.ingestDocumentWithException(_id, version, urlValue, rd);
- } catch (IOException e) {
- handleIOException(e);
- }
- } finally {
- try {
- is.close();
- } catch (IOException e) {
- handleIOException(e);
- }
+ activities.ingestDocumentWithException(_id,
version, urlValue, rd);
+ } catch (IOException e) {
+ handleIOException(e);
+ } catch (MongoException e) {
+ errorCode =
e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ errorDesc = "Error reading GridFS content: " +
e.getMessage();
+ handleMongoException(e);
}
- gfs.getDB().getMongo().close();
- session = null;
errorCode = "OK";
} else {
Logging.connectors.warn("GridFS: Document " + _id
+ " has a invalid URL: " + urlValue + " - skipping.");
- errorCode = activities.BAD_URL;
+ errorCode = IProcessActivity.BAD_URL;
errorDesc = "Excluded because document had illegal
URL ('" + urlValue + "')";
activities.noDocument(_id, version);
}
} else {
Logging.connectors.warn("GridFS: Document " + _id + "
has a null URL - skipping.");
- errorCode = activities.NULL_URL;
+ errorCode = IProcessActivity.NULL_URL;
errorDesc = "Excluded because document had a null
URL.";
activities.noDocument(_id, version);
}
} finally {
if (errorCode != null) {
- activities.recordActivity(startTime, ACTIVITY_FETCH,
document.getLength(), _id, errorCode, errorDesc, null);
+ activities.recordActivity(startTime, ACTIVITY_FETCH,
fileLenght, _id, errorCode, errorDesc, null);
}
}
}
}
}
+ /**
+ * Read the file length from a GridFS files document (stored as int32 or
int64 depending on the writer).
+ */
+ protected static long getFileLength(Document fileDocument) {
+ Object length = fileDocument.get("length");
+ return (length instanceof Number) ? ((Number) length).longValue() : 0L;
+ }
+
+ /**
+ * Resolve the content type of a GridFS file. Legacy drivers stored it as
a top-level
+ * <code>contentType</code> field (deprecated by the GridFS spec), modern
drivers expect it
+ * inside the metadata document.
+ */
+ protected static String getContentType(Document fileDocument, Document
metadata) {
+ Object contentType =
fileDocument.get(GridFSConstants.CONTENT_TYPE_FIELD_NAME);
+ if (contentType == null && metadata != null) {
+ contentType =
metadata.get(GridFSConstants.CONTENT_TYPE_FIELD_NAME);
+ }
+ return contentType == null ? null : contentType.toString();
+ }
+
+ /**
+ * Convert a metadata field value into a list of strings.
+ *
+ * @throws ClassCastException if the value is not a list of strings.
+ */
+ protected static List<String> toStringList(Object value) {
+ List<?> list = (List<?>) value;
+ List<String> result = new ArrayList<String>(list.size());
+ for (Object item : list) {
+ result.add((String) item);
+ }
+ return result;
+ }
+
+ /**
+ * Convert a Java _id value into the BsonValue expected by the GridFS
download API.
+ */
+ protected static BsonValue toBsonValue(Object idValue) {
+ return new Document("_id", idValue)
+ .toBsonDocument(Document.class,
MongoClientSettings.getDefaultCodecRegistry())
+ .get("_id");
+ }
+
protected static void handleIOException(IOException e) throws
ManifoldCFException, ServiceInterruption {
if (e instanceof InterruptedIOException) {
throw new ManifoldCFException(e.getMessage(), e,
ManifoldCFException.INTERRUPTED);
@@ -560,6 +627,23 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
}
}
+ /**
+ * Map driver exceptions: network/timeout problems are transient and lead
to a retry,
+ * anything else is reported as an error.
+ */
+ protected void handleMongoException(MongoException e) throws
ManifoldCFException, ServiceInterruption {
+ if (e instanceof com.mongodb.MongoSocketException || e instanceof
com.mongodb.MongoTimeoutException
+ || e instanceof com.mongodb.MongoExecutionTimeoutException) {
+ closeSession();
+ long currentTime = System.currentTimeMillis();
+ Logging.connectors.warn("GridFS: Transient MongoDB error, retrying
later: " + e.getMessage(), e);
+ throw new ServiceInterruption("GridFS: Transient MongoDB error: "
+ e.getMessage(), e,
+ currentTime + 300000L, currentTime + 3L * 60L * 60000L,
-1, false);
+ }
+ Logging.connectors.error("GridFS: MongoDB error: " + e.getMessage(),
e);
+ throw new ManifoldCFException("GridFS: MongoDB error: " +
e.getMessage(), e);
+ }
+
/**
* Output the configuration header section. This method is called in the
* head section of the connector's configuration page. Its purpose is to
add
@@ -721,43 +805,31 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
throw new ManifoldCFException("GridFS: Metadata URL field
cannot be empty.");
}
- if (StringUtils.isEmpty(host) && StringUtils.isEmpty(port)) {
- try {
- session = new MongoClient().getDB(db);
- } catch (UnknownHostException ex) {
- throw new ManifoldCFException("GridFS: Default host is not
found. Does mongod process run?" + ex.getMessage(), ex);
- }
- } else if (!StringUtils.isEmpty(host) &&
StringUtils.isEmpty(port)) {
- try {
- session = new MongoClient(host).getDB(db);
- } catch (UnknownHostException ex) {
- throw new ManifoldCFException("GridFS: Given host
information is not valid or mongod process doesn't run" + ex.getMessage(), ex);
- }
- } else if (!StringUtils.isEmpty(host) &&
!StringUtils.isEmpty(port)) {
- try {
- int integerPort = Integer.parseInt(port);
- session = new MongoClient(host, integerPort).getDB(db);
- } catch (UnknownHostException ex) {
- throw new ManifoldCFException("GridFS: Given information
is not valid or mongod process doesn't run" + ex.getMessage(), ex);
- } catch (NumberFormatException ex) {
- throw new ManifoldCFException("GridFS: Given port is not
valid number. " + ex.getMessage(), ex);
- }
- } else if (StringUtils.isEmpty(host) &&
!StringUtils.isEmpty(port)) {
+ String effectiveHost = StringUtils.isEmpty(host) ?
ServerAddress.defaultHost() : host;
+ int effectivePort = ServerAddress.defaultPort();
+ if (!StringUtils.isEmpty(port)) {
try {
- int integerPort = Integer.parseInt(port);
- session = new MongoClient(host, integerPort).getDB(db);
- } catch (UnknownHostException ex) {
- throw new ManifoldCFException("GridFS: Given information
is not valid or mongod process doesn't run" + ex.getMessage(), ex);
+ effectivePort = Integer.parseInt(port);
} catch (NumberFormatException ex) {
throw new ManifoldCFException("GridFS: Given port is not
valid number. " + ex.getMessage(), ex);
}
}
+ final ServerAddress serverAddress = new
ServerAddress(effectiveHost, effectivePort);
+ MongoClientSettings.Builder settingsBuilder =
MongoClientSettings.builder()
+ .applyToClusterSettings(builder ->
builder.hosts(Collections.singletonList(serverAddress)));
+
if (!StringUtils.isEmpty(username) &&
!StringUtils.isEmpty(password)) {
- boolean auth = session.authenticate(username,
password.toCharArray());
- if (!auth) {
- throw new ManifoldCFException("GridFS: Given database
username and password doesn't match.");
- }
+ // Credentials are validated lazily by the driver, on the
first operation (see check())
+
settingsBuilder.credential(MongoCredential.createCredential(username, db,
password.toCharArray()));
+ }
+
+ try {
+ client = MongoClients.create(settingsBuilder.build());
+ session = client.getDatabase(db);
+ } catch (MongoException | IllegalArgumentException ex) {
+ closeSession();
+ throw new ManifoldCFException("GridFS: Given information is
not valid or mongod process doesn't run: " + ex.getMessage(), ex);
}
lastSessionFetch = System.currentTimeMillis();
}
@@ -807,12 +879,12 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
/**
* Special column names, as far as document queries are concerned
*/
- protected static HashMap documentKnownColumns;
+ protected static HashMap<String, String> documentKnownColumns;
static {
- documentKnownColumns = new HashMap();
- documentKnownColumns.put(GridFSConstants.DEFAULT_ID_FIELD_NAME, "");
- documentKnownColumns.put(GridFSConstants.URL_RETURN_FIELD_NAME_PARAM,
"");
+ documentKnownColumns = new HashMap<String, String>();
+ documentKnownColumns.put(GridFSConstants.DEFAULT_ID_FIELD_NAME,
StringUtils.EMPTY);
+ documentKnownColumns.put(GridFSConstants.URL_RETURN_FIELD_NAME_PARAM,
StringUtils.EMPTY);
}
/**
@@ -822,16 +894,15 @@ public class GridFSRepositoryConnector extends
BaseRepositoryConnector {
* @param metadataMap is the resultset row to use to get the metadata. All
* non-special columns from this row will be considered to be metadata.
*/
- protected void applyMetadata(RepositoryDocument rd, DBObject metadataMap)
+ protected void applyMetadata(RepositoryDocument rd, Document metadataMap)
throws ManifoldCFException {
// Cycle through the document's fields
- Iterator iter = metadataMap.keySet().iterator();
- while (iter.hasNext()) {
- String fieldName = (String) iter.next();
+ for (Map.Entry<String, Object> entry : metadataMap.entrySet()) {
+ String fieldName = entry.getKey();
if (documentKnownColumns.get(fieldName) == null) {
// Consider this field to contain metadata.
// We can only accept non-binary metadata at this time.
- Object metadata = metadataMap.get(fieldName);
+ Object metadata = entry.getValue();
if (!(metadata instanceof String)) {
throw new ManifoldCFException("Metadata field '" +
fieldName + "' must be convertible to a string.");
}
diff --git
a/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/Messages.java
b/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/Messages.java
index 5cf4976a1..1d29b620f 100644
---
a/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/Messages.java
+++
b/connectors/gridfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/gridfs/Messages.java
@@ -17,15 +17,9 @@ package org.apache.manifoldcf.crawler.connectors.gridfs;
import java.util.Locale;
import java.util.Map;
-import static org.apache.manifoldcf.core.i18n.Messages.getString;
+
import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
-import static
org.apache.manifoldcf.ui.i18n.Messages.getAttributeJavascriptString;
-import static org.apache.manifoldcf.ui.i18n.Messages.getAttributeString;
-import static org.apache.manifoldcf.ui.i18n.Messages.getBodyJavascriptString;
-import static org.apache.manifoldcf.ui.i18n.Messages.getBodyString;
-import static org.apache.manifoldcf.ui.i18n.Messages.outputResource;
-import static
org.apache.manifoldcf.ui.i18n.Messages.outputResourceWithVelocity;
/**
*
diff --git
a/connectors/gridfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/gridfs/tests/GridFSRepositoryConnectorHSQLDBIT.java
b/connectors/gridfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/gridfs/tests/GridFSRepositoryConnectorHSQLDBIT.java
new file mode 100644
index 000000000..e50047248
--- /dev/null
+++
b/connectors/gridfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/gridfs/tests/GridFSRepositoryConnectorHSQLDBIT.java
@@ -0,0 +1,318 @@
+/* $Id$ */
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.manifoldcf.crawler.connectors.gridfs.tests;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
+import org.apache.manifoldcf.core.interfaces.ConfigParams;
+import
org.apache.manifoldcf.crawler.connectors.gridfs.GridFSRepositoryConnector;
+import org.apache.manifoldcf.crawler.interfaces.IProcessActivity;
+import org.apache.manifoldcf.crawler.interfaces.ISeedingActivity;
+import org.bson.Document;
+import org.bson.types.ObjectId;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.testcontainers.mongodb.MongoDBContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import com.mongodb.MongoClientSettings;
+import com.mongodb.ServerAddress;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.gridfs.GridFSBucket;
+import com.mongodb.client.gridfs.GridFSBuckets;
+import com.mongodb.client.gridfs.model.GridFSUploadOptions;
+import com.mongodb.client.model.Filters;
+import com.mongodb.client.model.Updates;
+
+/**
+ * Integration test for the GridFS repository connector against a real MongoDB
9.0.2
+ * instance started with Testcontainers (Docker is required).
+ */
+public class GridFSRepositoryConnectorHSQLDBIT extends BaseHSQLDB {
+
+ private static final String DATABASE = "gridfsTestDatabase";
+ private static final String BUCKET = "fs";
+ private static final String USERNAME = "gridfsuser";
+ private static final String PASSWORD = "gridfssecret";
+
+ private static MongoDBContainer mongoDBContainer;
+
+ private MongoClient testClient;
+ private MongoDatabase testDatabase;
+ private GridFSRepositoryConnector connector;
+
+ private ObjectId documentWithMetadataId;
+ private ObjectId legacyContentTypeId;
+ private ObjectId documentWithoutMetadataId;
+
+ @BeforeClass
+ public static void startMongoDB() {
+ mongoDBContainer = new
MongoDBContainer(DockerImageName.parse("mongo:9.0.2"));
+ mongoDBContainer.start();
+ }
+
+ @AfterClass
+ public static void stopMongoDB() {
+ if (mongoDBContainer != null) {
+ mongoDBContainer.stop();
+ mongoDBContainer = null;
+ }
+ }
+
+ @Before
+ public void setUpGridFS() throws Exception {
+ testClient = MongoClients.create(MongoClientSettings.builder()
+ .applyToClusterSettings(b -> b.hosts(Collections.singletonList(
+ new ServerAddress(mongoDBContainer.getHost(),
mongoDBContainer.getFirstMappedPort()))))
+ .build());
+ testDatabase = testClient.getDatabase(DATABASE);
+ GridFSBucket bucket = GridFSBuckets.create(testDatabase, BUCKET);
+
+ // Modern layout: content type stored in the metadata document
+ documentWithMetadataId = bucket.uploadFromStream("document.txt",
stream("Hello GridFS"),
+ new GridFSUploadOptions().metadata(new Document("url",
"http://www.example.com/document.txt";)
+ .append("contentType", "text/plain")
+ .append("acl", Arrays.asList("user1", "group1"))
+ .append("denyAcl",
Collections.singletonList("user2"))));
+
+ // Legacy layout: content type stored as a top-level field of the
files document
+ legacyContentTypeId = bucket.uploadFromStream("legacy.html",
stream("<html></html>"),
+ new GridFSUploadOptions().metadata(new Document("url",
"http://www.example.com/legacy.html";)));
+ testDatabase.getCollection(BUCKET + ".files")
+ .updateOne(Filters.eq("_id", legacyContentTypeId),
Updates.set("contentType", "text/html"));
+
+ // No metadata at all: must be skipped
+ documentWithoutMetadataId = bucket.uploadFromStream("nometadata.bin",
stream("binary"));
+
+ testDatabase.runCommand(new Document("createUser", USERNAME)
+ .append("pwd", PASSWORD)
+ .append("roles", Collections.singletonList(new
Document("role", "read").append("db", DATABASE))));
+
+ connector = new GridFSRepositoryConnector();
+ connector.connect(configParams(null, null));
+ }
+
+ @After
+ public void cleanUpGridFS() throws Exception {
+ if (connector != null) {
+ connector.disconnect();
+ connector = null;
+ }
+ if (testDatabase != null) {
+ try {
+ testDatabase.runCommand(new Document("dropUser", USERNAME));
+ } catch (Exception e) {
+ // ignore
+ }
+ testDatabase.drop();
+ testDatabase = null;
+ }
+ if (testClient != null) {
+ testClient.close();
+ testClient = null;
+ }
+ }
+
+ @Test
+ public void checkConnection() throws Exception {
+ assertEquals("Connection working", connector.check());
+ }
+
+ @Test
+ public void checkConnectionWithCredentials() throws Exception {
+ GridFSRepositoryConnector authenticated = new
GridFSRepositoryConnector();
+ authenticated.connect(configParams(USERNAME, PASSWORD));
+ try {
+ assertEquals("Connection working", authenticated.check());
+ } finally {
+ authenticated.disconnect();
+ }
+
+ GridFSRepositoryConnector wrongPassword = new
GridFSRepositoryConnector();
+ wrongPassword.connect(configParams(USERNAME, "wrong-password"));
+ try {
+ assertNotEquals("Connection working", wrongPassword.check());
+ } finally {
+ wrongPassword.disconnect();
+ }
+ }
+
+ @Test
+ public void seedsAllFiles() throws Exception {
+ ISeedingActivity seedingActivity = mock(ISeedingActivity.class);
+
+ connector.addSeedDocuments(seedingActivity, null, null,
System.currentTimeMillis(), 0);
+
+
verify(seedingActivity).addSeedDocument(documentWithMetadataId.toHexString());
+
verify(seedingActivity).addSeedDocument(legacyContentTypeId.toHexString());
+
verify(seedingActivity).addSeedDocument(documentWithoutMetadataId.toHexString());
+ verify(seedingActivity, times(3)).addSeedDocument(anyString());
+ }
+
+ @Test
+ public void processesDocuments() throws Exception {
+ IProcessActivity activities = indexEverythingActivity();
+ List<RepositoryDocument> ingested = new
ArrayList<RepositoryDocument>();
+ List<String> ingestedUris = new ArrayList<String>();
+ List<byte[]> ingestedContents = new ArrayList<byte[]>();
+ doAnswer(invocation -> {
+ RepositoryDocument rd = invocation.getArgument(3);
+ ingested.add(rd);
+ ingestedUris.add(invocation.getArgument(2));
+ ingestedContents.add(readAll(rd.getBinaryStream()));
+ return null;
+ }).when(activities).ingestDocumentWithException(anyString(),
anyString(), anyString(), any(RepositoryDocument.class));
+
+ String missingId = new ObjectId().toHexString();
+ connector.processDocuments(new String[]{
+ documentWithMetadataId.toHexString(),
+ legacyContentTypeId.toHexString(),
+ documentWithoutMetadataId.toHexString(),
+ missingId
+ }, null, null, activities, 0, true);
+
+ // Two documents ingested, in order
+ assertEquals(2, ingested.size());
+
+ RepositoryDocument modern = ingested.get(0);
+ assertEquals("http://www.example.com/document.txt";,
ingestedUris.get(0));
+ assertEquals("document.txt", modern.getFileName());
+ assertEquals("text/plain", modern.getMimeType());
+ assertEquals("Hello GridFS", new String(ingestedContents.get(0),
StandardCharsets.UTF_8));
+ assertArrayEquals(new String[]{"user1", "group1"},
+
modern.getSecurityACL(RepositoryDocument.SECURITY_TYPE_DOCUMENT));
+ String[] denyAcls =
modern.getSecurityDenyACL(RepositoryDocument.SECURITY_TYPE_DOCUMENT);
+ assertNotNull(denyAcls);
+ assertEquals(2, denyAcls.length);
+ assertEquals("user2", denyAcls[0]);
+
+ RepositoryDocument legacy = ingested.get(1);
+ assertEquals("http://www.example.com/legacy.html";,
ingestedUris.get(1));
+ assertEquals("text/html", legacy.getMimeType());
+ assertEquals("<html></html>", new String(ingestedContents.get(1),
StandardCharsets.UTF_8));
+
+ // No metadata: skipped; missing file: deleted
+
verify(activities).noDocument(eq(documentWithoutMetadataId.toHexString()),
anyString());
+ verify(activities).deleteDocument(missingId);
+ verify(activities,
never()).deleteDocument(documentWithMetadataId.toHexString());
+ }
+
+ @Test
+ public void versionStringChangesWhenMetadataChanges() throws Exception {
+ String id = documentWithMetadataId.toHexString();
+ List<String> versions = new ArrayList<String>();
+ IProcessActivity activities = mock(IProcessActivity.class);
+ when(activities.checkDocumentNeedsReindexing(anyString(),
anyString())).thenAnswer(invocation -> {
+ versions.add(invocation.getArgument(1));
+ return false;
+ });
+
+ connector.processDocuments(new String[]{id}, null, null, activities,
0, true);
+ connector.processDocuments(new String[]{id}, null, null, activities,
0, true);
+ testDatabase.getCollection(BUCKET + ".files")
+ .updateOne(Filters.eq("_id", documentWithMetadataId),
Updates.set("metadata.url", "http://www.example.com/moved.txt";));
+ connector.processDocuments(new String[]{id}, null, null, activities,
0, true);
+
+ assertEquals(3, versions.size());
+ assertFalse(versions.get(0).isEmpty());
+ assertEquals("Version must be stable", versions.get(0),
versions.get(1));
+ assertNotEquals("Version must change with metadata", versions.get(1),
versions.get(2));
+ verify(activities, never()).ingestDocumentWithException(anyString(),
anyString(), anyString(), any(RepositoryDocument.class));
+ }
+
+ @Test
+ public void sessionLifecycle() throws Exception {
+ assertFalse(connector.isConnected());
+ connector.addSeedDocuments(mock(ISeedingActivity.class), null, null,
System.currentTimeMillis(), 0);
+ assertTrue(connector.isConnected());
+ connector.disconnect();
+ assertFalse(connector.isConnected());
+ connector = null;
+ }
+
+ private IProcessActivity indexEverythingActivity() throws Exception {
+ IProcessActivity activities = mock(IProcessActivity.class);
+ when(activities.checkDocumentNeedsReindexing(anyString(),
anyString())).thenReturn(true);
+ when(activities.checkURLIndexable(any())).thenReturn(true);
+ when(activities.checkLengthIndexable(anyLong())).thenReturn(true);
+ when(activities.checkMimeTypeIndexable(any())).thenReturn(true);
+ when(activities.checkDateIndexable(any())).thenReturn(true);
+ return activities;
+ }
+
+ private ConfigParams configParams(String username, String password) {
+ ConfigParams params = new ConfigParams();
+ params.setParameter("host", mongoDBContainer.getHost());
+ params.setParameter("port",
String.valueOf(mongoDBContainer.getFirstMappedPort()));
+ params.setParameter("db", DATABASE);
+ params.setParameter("bucket", BUCKET);
+ params.setParameter("url", "url");
+ params.setParameter("acl", "acl");
+ params.setParameter("denyAcl", "denyAcl");
+ if (username != null) {
+ params.setParameter("username", username);
+ params.setObfuscatedParameter("password", password);
+ }
+ return params;
+ }
+
+ private static InputStream stream(String content) {
+ return new
ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
+ }
+
+ private static byte[] readAll(InputStream is) throws Exception {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ byte[] buffer = new byte[8192];
+ int read;
+ while ((read = is.read(buffer)) != -1) {
+ out.write(buffer, 0, read);
+ }
+ return out.toByteArray();
+ }
+}
diff --git a/connectors/gridfs/pom.xml b/connectors/gridfs/pom.xml
index 410ce86df..7acf8e8bd 100644
--- a/connectors/gridfs/pom.xml
+++ b/connectors/gridfs/pom.xml
@@ -59,6 +59,33 @@
<workingDirectory>target/test-output</workingDirectory>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>**/*Postgresql*.java</exclude>
+ <exclude>**/*MySQL*.java</exclude>
+ </excludes>
+ <forkCount>1</forkCount>
+ <reuseForks>false</reuseForks>
+ <workingDirectory>target/test-output</workingDirectory>
+ </configuration>
+ <executions>
+ <execution>
+ <id>integration-test</id>
+ <goals>
+ <goal>integration-test</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>verify</id>
+ <goals>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
@@ -135,8 +162,18 @@
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
- <artifactId>mongo-java-driver</artifactId>
- <version>2.11.3</version>
+ <artifactId>mongodb-driver-sync</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>testcontainers-mongodb</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>${mockito.version}</version>
+ <scope>test</scope>
</dependency>
</dependencies>
</project>
diff --git a/connectors/mongodb/test-materials/README.txt
b/connectors/gridfs/test-materials/README.txt
similarity index 80%
copy from connectors/mongodb/test-materials/README.txt
copy to connectors/gridfs/test-materials/README.txt
index 4be9d6c1b..bf08d6249 100644
--- a/connectors/mongodb/test-materials/README.txt
+++ b/connectors/gridfs/test-materials/README.txt
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-To test this connector, download the Embedded MongoDB dependencies using "ant
download-dependencies"
+To test this connector, download the Testcontainers dependencies using "ant
download-dependencies"
or the global "ant make-deps" target, and use the "ant test" target for the
standard ant build.
-You can read more about the process on the "how-to-build-and-deploy.html"
documentation page.
\ No newline at end of file
+The integration tests start MongoDB 9.0.2 as a Docker container, so a running
Docker daemon is required.
+You can read more about the process on the "how-to-build-and-deploy.html"
documentation page.
diff --git a/connectors/mongodb/build.xml b/connectors/mongodb/build.xml
index 58441ba1a..a88be2b2a 100644
--- a/connectors/mongodb/build.xml
+++ b/connectors/mongodb/build.xml
@@ -1,153 +1,122 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<project name="mongodb" default="all">
-
- <property environment="env"/>
- <condition property="mcf-dist" value="${env.MCFDISTPATH}">
- <isset property="env.MCFDISTPATH"/>
- </condition>
- <property name="abs-dist" location="../../dist"/>
- <condition property="mcf-dist" value="${abs-dist}">
- <not>
- <isset property="env.MCFDISTPATH"/>
- </not>
- </condition>
-
- <import file="${mcf-dist}/connector-build.xml"/>
-
- <target name="download-cleanup">
- <delete>
- <fileset dir="test-materials" excludes="README*.txt"/>
- </delete>
- </target>
-
- <path id="connector-classpath">
- <path refid="mcf-connector-build.connector-classpath"/>
- <fileset dir="../../lib">
- <include name="mongo-java-driver*.jar"/>
- </fileset>
- </path>
-
- <target name="lib" depends="mcf-connector-build.lib,precompile-check"
if="canBuild">
- <mkdir dir="dist/lib"/>
- <copy todir="dist/lib">
- <fileset dir="../../lib">
- <include name="mongo-java-driver*.jar"/>
- </fileset>
- </copy>
- </target>
-
- <target name="deliver-connector"
depends="mcf-connector-build.deliver-connector">
- <antcall target="general-add-output-connector">
- <param name="connector-label" value="MongoDB"/>
- <param name="connector-class"
-
value="org.apache.manifoldcf.agents.output.mongodboutput.MongodbOutputConnector"/>
- </antcall>
- </target>
-
- <property name="embedded-mongodb.version.label" value="4.33.0"/>
- <property name="embedded-mongodb.artifactId"
value="de.flapdoodle.embed.mongo"/>
- <property name="embedded-mongodb-package" value="de/flapdoodle/embed"/>
- <property name="embedded-mongodb.snapshot-jar"
value="${embedded-mongodb.artifactId}-${embedded-mongodb.version.label}.jar"/>
-
- <property name="embedded-mongodb.process.version.label" value="4.18.0"/>
- <property name="embedded-mongodb.process.artifactId"
value="de.flapdoodle.embed.process"/>
- <property name="embedded-mongodb.process-package"
value="de/flapdoodle/embed"/>
- <property name="embedded-mongodb.process.jar"
value="${embedded-mongodb.process.artifactId}-${embedded-mongodb.process.version.label}.jar"/>
-
- <property name="embedded-mongodb.packageresolver.version.label"
value="4.33.0"/>
- <property name="embedded-mongodb.packageresolver.artifactId"
value="de.flapdoodle.embed.mongo.packageresolver"/>
- <property name="embedded-mongodb-packageresolver-package"
value="de/flapdoodle/embed"/>
- <property name="embedded-mongodb.packageresolver.jar"
value="${embedded-mongodb.packageresolver.artifactId}-${embedded-mongodb.packageresolver.version.label}.jar"/>
-
- <property name="embedded-mongodb.os.version.label" value="1.11.0"/>
- <property name="embedded-mongodb.os.artifactId" value="de.flapdoodle.os"/>
- <property name="embedded-mongodb-os-package" value="de/flapdoodle"/>
- <property name="embedded-mongodb.os.jar"
value="${embedded-mongodb.os.artifactId}-${embedded-mongodb.os.version.label}.jar"/>
-
- <property name="embedded-mongodb.os-api.version.label" value="1.7.1"/>
- <property name="embedded-mongodb.os-api.artifactId"
value="de.flapdoodle.os-api"/>
- <property name="embedded-mongodb-os-api-package" value="de/flapdoodle"/>
- <property name="embedded-mongodb.os-api.jar"
value="${embedded-mongodb.os-api.artifactId}-${embedded-mongodb.os-api.version.label}.jar"/>
-
- <property name="embedded-mongodb.reverse.version.label" value="1.9.1"/>
- <property name="embedded-mongodb.reverse.artifactId"
value="de.flapdoodle.reverse"/>
- <property name="embedded-mongodb-reverse-package"
value="de/flapdoodle/reverse"/>
- <property name="embedded-mongodb.reverse.jar"
value="${embedded-mongodb.reverse.artifactId}-${embedded-mongodb.reverse.version.label}.jar"/>
-
- <property name="embedded-mongodb.graph.version.label" value="1.3.4"/>
- <property name="embedded-mongodb.graph.artifactId"
value="de.flapdoodle.graph"/>
- <property name="embedded-mongodb-graph-package"
value="de/flapdoodle/graph"/>
- <property name="embedded-mongodb.graph.jar"
value="${embedded-mongodb.graph.artifactId}-${embedded-mongodb.graph.version.label}.jar"/>
-
- <property name="embedded-mongodb.java8.version.label" value="1.7.2"/>
- <property name="embedded-mongodb.java8.artifactId"
value="de.flapdoodle.java8"/>
- <property name="embedded-mongodb-java8-package"
value="de/flapdoodle/java8"/>
- <property name="embedded-mongodb.java8.jar"
value="${embedded-mongodb.java8.artifactId}-${embedded-mongodb.java8.version.label}.jar"/>
-
- <property name="jgrapht-core.version.label" value="1.4.0"/>
- <property name="jgrapht-core.artifactId" value="jgrapht-core"/>
- <property name="jgrapht-core-package" value="org/jgrapht"/>
- <property name="jgrapht-core.jar"
value="${jgrapht-core.artifactId}-${jgrapht-core.version.label}.jar"/>
-
- <property name="jheaps.version.label" value="0.11"/>
- <property name="jheaps.artifactId" value="jheaps"/>
- <property name="jheaps-package" value="org/jheaps"/>
- <property name="jheaps.jar"
value="${jheaps.artifactId}-${jheaps.version.label}.jar"/>
-
- <target name="calculate-testcode-condition">
- <available file="test-materials/${embedded-mongodb.snapshot-jar}"
property="tests-present"/>
- <available file="test-materials/${embedded-mongodb.process.jar}"
property="tests-present"/>
- <available
file="test-materials/${embedded-mongodb.packageresolver.jar}"
property="tests-present"/>
- <available file="test-materials/${embedded-mongodb.os.jar}"
property="tests-present"/>
- <available file="test-materials/${embedded-mongodb.os-api.jar}"
property="tests-present"/>
- <available file="test-materials/${embedded-mongodb.reverse.jar}"
property="tests-present"/>
- <available file="test-materials/${embedded-mongodb.graph.jar}"
property="tests-present"/>
- <available file="test-materials/${embedded-mongodb.java8.jar}"
property="tests-present"/>
- <available file="test-materials/${jgrapht-core.jar}"
property="tests-present"/>
- <available file="test-materials/${jheaps.jar}"
property="tests-present"/>
-
- <copy todir="${mcf-dist}/test-lib">
- <fileset dir="test-materials">
- <include name="de.flapdoodle*.jar"/>
- <include name="jgrapht-core*.jar"/>
- <include name="jheaps*.jar"/>
- </fileset>
- </copy>
- </target>
-
- <target name="pretest-warn" depends="calculate-testcode-condition"
unless="tests-present">
- <echo message="Need Embedded MongoDB download in order to run tests -
run ant make-deps"/>
- </target>
-
- <target name="download-dependencies">
- <property name="sonatype-maven-repo-releases"
value="https://oss.sonatype.org/content/repositories/public"; />
- <property name="apache-maven-repo-releases"
value="https://repo1.maven.org/maven2"; />
- <get
src="${apache-maven-repo-releases}/${embedded-mongodb-package}/${embedded-mongodb.artifactId}/${embedded-mongodb.version.label}/${embedded-mongodb.snapshot-jar}"
dest="test-materials/${embedded-mongodb.snapshot-jar}"/>
- <get
src="${apache-maven-repo-releases}/${embedded-mongodb.process-package}/${embedded-mongodb.process.artifactId}/${embedded-mongodb.process.version.label}/${embedded-mongodb.process.jar}"
dest="test-materials/${embedded-mongodb.process.jar}"/>
- <get
src="${apache-maven-repo-releases}/${embedded-mongodb-packageresolver-package}/${embedded-mongodb.packageresolver.artifactId}/${embedded-mongodb.packageresolver.version.label}/${embedded-mongodb.packageresolver.jar}"
dest="test-materials/${embedded-mongodb.packageresolver.jar}"/>
- <get
src="${apache-maven-repo-releases}/${embedded-mongodb-os-package}/${embedded-mongodb.os.artifactId}/${embedded-mongodb.os.version.label}/${embedded-mongodb.os.jar}"
dest="test-materials/${embedded-mongodb.os.jar}"/>
- <get
src="${apache-maven-repo-releases}/${embedded-mongodb-os-api-package}/${embedded-mongodb.os-api.artifactId}/${embedded-mongodb.os-api.version.label}/${embedded-mongodb.os-api.jar}"
dest="test-materials/${embedded-mongodb.os-api.jar}"/>
- <get
src="${apache-maven-repo-releases}/${embedded-mongodb-reverse-package}/${embedded-mongodb.reverse.artifactId}/${embedded-mongodb.reverse.version.label}/${embedded-mongodb.reverse.jar}"
dest="test-materials/${embedded-mongodb.reverse.jar}"/>
- <get
src="${apache-maven-repo-releases}/${embedded-mongodb-graph-package}/${embedded-mongodb.graph.artifactId}/${embedded-mongodb.graph.version.label}/${embedded-mongodb.graph.jar}"
dest="test-materials/${embedded-mongodb.graph.jar}"/>
- <get
src="${apache-maven-repo-releases}/${embedded-mongodb-java8-package}/${embedded-mongodb.java8.artifactId}/${embedded-mongodb.java8.version.label}/${embedded-mongodb.java8.jar}"
dest="test-materials/${embedded-mongodb.java8.jar}"/>
- <get
src="${apache-maven-repo-releases}/${jgrapht-core-package}/${jgrapht-core.artifactId}/${jgrapht-core.version.label}/${jgrapht-core.jar}"
dest="test-materials/${jgrapht-core.jar}"/>
- <get
src="${apache-maven-repo-releases}/${jheaps-package}/${jheaps.artifactId}/${jheaps.version.label}/${jheaps.jar}"
dest="test-materials/${jheaps.jar}"/>
- </target>
-
-</project>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project name="mongodb" default="all">
+
+ <property environment="env"/>
+ <condition property="mcf-dist" value="${env.MCFDISTPATH}">
+ <isset property="env.MCFDISTPATH"/>
+ </condition>
+ <property name="abs-dist" location="../../dist"/>
+ <condition property="mcf-dist" value="${abs-dist}">
+ <not>
+ <isset property="env.MCFDISTPATH"/>
+ </not>
+ </condition>
+
+ <import file="${mcf-dist}/connector-build.xml"/>
+
+ <target name="download-cleanup">
+ <delete>
+ <fileset dir="test-materials" excludes="README*.txt"/>
+ </delete>
+ </target>
+
+ <!-- MongoDB sync driver (MongoDB 9.x compliant) -->
+ <patternset id="mongodb-driver-jars">
+ <include name="mongodb-driver-sync-*.jar"/>
+ <include name="mongodb-driver-core-*.jar"/>
+ <include name="bson-*.jar"/>
+ </patternset>
+
+ <path id="connector-classpath">
+ <path refid="mcf-connector-build.connector-classpath"/>
+ <fileset dir="../../lib">
+ <patternset refid="mongodb-driver-jars"/>
+ </fileset>
+ </path>
+
+ <target name="lib" depends="mcf-connector-build.lib,precompile-check"
if="canBuild">
+ <mkdir dir="dist/lib"/>
+ <copy todir="dist/lib">
+ <fileset dir="../../lib">
+ <patternset refid="mongodb-driver-jars"/>
+ </fileset>
+ </copy>
+ </target>
+
+ <target name="deliver-connector"
depends="mcf-connector-build.deliver-connector">
+ <antcall target="general-add-output-connector">
+ <param name="connector-label" value="MongoDB"/>
+ <param name="connector-class"
+
value="org.apache.manifoldcf.agents.output.mongodboutput.MongodbOutputConnector"/>
+ </antcall>
+ </target>
+
+ <!-- Test materials: Testcontainers is used to run MongoDB 9.0.2 as a
Docker container during the tests -->
+ <property name="testcontainers.version" value="2.0.5"/>
+ <property name="docker-java.version" value="3.7.1"/>
+ <property name="duct-tape.version" value="1.0.8"/>
+ <property name="jna.version" value="5.10.0"/>
+ <property name="commons-compress.version" value="1.28.0"/>
+ <property name="jetbrains-annotations.version" value="17.0.0"/>
+ <property name="jackson-annotations.version" value="2.21"/>
+
+ <!-- Test materials come first so that the newer jars required by
Testcontainers
+ (e.g. jackson-annotations, commons-compress) take precedence over the
older shared ones -->
+ <path id="connector-test-classpath">
+ <fileset dir="test-materials" includes="*.jar"
erroronmissingdir="false"/>
+ <path refid="mcf-connector-build.connector-test-classpath"/>
+ </path>
+
+ <target name="calculate-testcode-condition">
+ <condition property="tests-present">
+ <and>
+ <available
file="test-materials/testcontainers-${testcontainers.version}.jar"/>
+ <available
file="test-materials/testcontainers-mongodb-${testcontainers.version}.jar"/>
+ <available
file="test-materials/docker-java-api-${docker-java.version}.jar"/>
+ <available
file="test-materials/docker-java-transport-${docker-java.version}.jar"/>
+ <available
file="test-materials/docker-java-transport-zerodep-${docker-java.version}.jar"/>
+ <available
file="test-materials/duct-tape-${duct-tape.version}.jar"/>
+ <available file="test-materials/jna-${jna.version}.jar"/>
+ <available
file="test-materials/commons-compress-${commons-compress.version}.jar"/>
+ <available
file="test-materials/annotations-${jetbrains-annotations.version}.jar"/>
+ <available
file="test-materials/jackson-annotations-${jackson-annotations.version}.jar"/>
+ </and>
+ </condition>
+ </target>
+
+ <target name="pretest-warn" depends="calculate-testcode-condition"
unless="tests-present">
+ <echo message="Need Testcontainers download in order to run tests -
run ant make-deps (Docker is required to run MongoDB 9.0.2)"/>
+ </target>
+
+ <target name="download-dependencies">
+ <property name="maven-repo" value="https://repo1.maven.org/maven2"/>
+ <mkdir dir="test-materials"/>
+ <get
src="${maven-repo}/org/testcontainers/testcontainers/${testcontainers.version}/testcontainers-${testcontainers.version}.jar"
dest="test-materials/testcontainers-${testcontainers.version}.jar"/>
+ <get
src="${maven-repo}/org/testcontainers/testcontainers-mongodb/${testcontainers.version}/testcontainers-mongodb-${testcontainers.version}.jar"
dest="test-materials/testcontainers-mongodb-${testcontainers.version}.jar"/>
+ <get
src="${maven-repo}/com/github/docker-java/docker-java-api/${docker-java.version}/docker-java-api-${docker-java.version}.jar"
dest="test-materials/docker-java-api-${docker-java.version}.jar"/>
+ <get
src="${maven-repo}/com/github/docker-java/docker-java-transport/${docker-java.version}/docker-java-transport-${docker-java.version}.jar"
dest="test-materials/docker-java-transport-${docker-java.version}.jar"/>
+ <get
src="${maven-repo}/com/github/docker-java/docker-java-transport-zerodep/${docker-java.version}/docker-java-transport-zerodep-${docker-java.version}.jar"
dest="test-materials/docker-java-transport-zerodep-${docker-java.version}.jar"/>
+ <get
src="${maven-repo}/org/rnorth/duct-tape/duct-tape/${duct-tape.version}/duct-tape-${duct-tape.version}.jar"
dest="test-materials/duct-tape-${duct-tape.version}.jar"/>
+ <get
src="${maven-repo}/net/java/dev/jna/jna/${jna.version}/jna-${jna.version}.jar"
dest="test-materials/jna-${jna.version}.jar"/>
+ <get
src="${maven-repo}/org/apache/commons/commons-compress/${commons-compress.version}/commons-compress-${commons-compress.version}.jar"
dest="test-materials/commons-compress-${commons-compress.version}.jar"/>
+ <get
src="${maven-repo}/org/jetbrains/annotations/${jetbrains-annotations.version}/annotations-${jetbrains-annotations.version}.jar"
dest="test-materials/annotations-${jetbrains-annotations.version}.jar"/>
+ <get
src="${maven-repo}/com/fasterxml/jackson/core/jackson-annotations/${jackson-annotations.version}/jackson-annotations-${jackson-annotations.version}.jar"
dest="test-materials/jackson-annotations-${jackson-annotations.version}.jar"/>
+ </target>
+
+</project>
diff --git
a/connectors/mongodb/connector/src/main/java/org/apache/manifoldcf/agents/output/mongodboutput/MongodbOutputConnector.java
b/connectors/mongodb/connector/src/main/java/org/apache/manifoldcf/agents/output/mongodboutput/MongodbOutputConnector.java
index 3022700e9..861c23ff7 100644
---
a/connectors/mongodb/connector/src/main/java/org/apache/manifoldcf/agents/output/mongodboutput/MongodbOutputConnector.java
+++
b/connectors/mongodb/connector/src/main/java/org/apache/manifoldcf/agents/output/mongodboutput/MongodbOutputConnector.java
@@ -16,18 +16,19 @@
*/
package org.apache.manifoldcf.agents.output.mongodboutput;
-import com.mongodb.BasicDBObject;
-import com.mongodb.DB;
-import com.mongodb.DBCursor;
-import com.mongodb.DBCollection;
-import com.mongodb.MongoClient;
-import com.mongodb.DBTCPConnector;
-import com.mongodb.DBPort;
-import com.mongodb.DBPortPool;
-import com.mongodb.WriteConcern;
-import com.mongodb.WriteResult;
-import com.mongodb.MongoException;
-import org.bson.types.Binary;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InterruptedIOException;
+import java.net.ConnectException;
+import java.rmi.RemoteException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
@@ -36,27 +37,30 @@ import
org.apache.manifoldcf.agents.interfaces.IOutputRemoveActivity;
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
import org.apache.manifoldcf.agents.output.BaseOutputConnector;
-import org.apache.manifoldcf.core.interfaces.IThreadContext;
+import org.apache.manifoldcf.core.interfaces.ConfigParams;
import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
import org.apache.manifoldcf.core.interfaces.IPasswordMapperActivity;
import org.apache.manifoldcf.core.interfaces.IPostParameters;
-import org.apache.manifoldcf.core.interfaces.ConfigParams;
-import org.apache.manifoldcf.core.interfaces.VersionContext;
+import org.apache.manifoldcf.core.interfaces.IThreadContext;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.core.interfaces.VersionContext;
import org.apache.manifoldcf.crawler.system.Logging;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.bson.types.Binary;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InterruptedIOException;
-import java.net.ConnectException;
-import java.net.UnknownHostException;
-import java.rmi.RemoteException;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Date;
-import java.util.Iterator;
+import com.mongodb.MongoClientSettings;
+import com.mongodb.MongoCredential;
+import com.mongodb.MongoException;
+import com.mongodb.ServerAddress;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.Filters;
+import com.mongodb.client.model.ReplaceOptions;
+import com.mongodb.client.result.DeleteResult;
+import com.mongodb.client.result.UpdateResult;
/**
* This is the "output connector" for MongoDB.
@@ -152,7 +156,7 @@ public class MongodbOutputConnector extends
BaseOutputConnector {
/**
* MongoDB database handle instance
*/
- private DB mongoDatabase = null;
+ private MongoDatabase mongoDatabase = null;
/**
@@ -226,11 +230,15 @@ public class MongodbOutputConnector extends
BaseOutputConnector {
*/
@Override
public boolean isConnected() {
- if (client == null) {
+ if (client == null || mongoDatabase == null) {
+ return false;
+ }
+ try {
+ mongoDatabase.runCommand(new Document("ping", 1));
+ return true;
+ } catch (Exception e) {
return false;
}
- DBTCPConnector currentTCPConnection = client.getConnector();
- return currentTCPConnection.isOpen();
}
/**
@@ -642,55 +650,30 @@ public class MongodbOutputConnector extends
BaseOutputConnector {
try {
// Create a session
if (client == null) {
- if (StringUtils.isEmpty(host) &&
StringUtils.isEmpty(port)) {
- try {
- client = new MongoClient();
- mongoDatabase = client.getDB(database);
- } catch (UnknownHostException ex) {
- throw new ManifoldCFException("MongoDB: Default
host is not found. Is mongod process running?" + ex.getMessage(), ex);
- }
- } else if (!StringUtils.isEmpty(host) &&
StringUtils.isEmpty(port)) {
- try {
- client = new MongoClient(host);
- mongoDatabase = client.getDB(database);
- } catch (UnknownHostException ex) {
- throw new ManifoldCFException("MongoDB: Given host
information is not valid or mongod process doesn't run" + ex.getMessage(), ex);
- }
- } else if (!StringUtils.isEmpty(host) &&
!StringUtils.isEmpty(port)) {
+ String effectiveHost = StringUtils.isEmpty(host) ?
"localhost" : host;
+ int effectivePort = 27017;
+ if (!StringUtils.isEmpty(port)) {
try {
- int integerPort = Integer.parseInt(port);
- client = new MongoClient(host, integerPort);
- mongoDatabase = client.getDB(database);
- } catch (UnknownHostException ex) {
- throw new ManifoldCFException("MongoDB: Given
information is not valid or mongod process doesn't run" + ex.getMessage(), ex);
+ effectivePort = Integer.parseInt(port);
} catch (NumberFormatException ex) {
throw new ManifoldCFException("MongoDB: Given port
is not a valid number. " + ex.getMessage(), ex);
}
- } else if (StringUtils.isEmpty(host) &&
!StringUtils.isEmpty(port)) {
- try {
- int integerPort = Integer.parseInt(port);
- client = new MongoClient("localhost", integerPort);
- mongoDatabase = client.getDB(database);
- } catch (UnknownHostException e) {
- Logging.connectors.warn("MongoDB: Given
information is not valid or mongod process doesn't run" + e.getMessage(), e);
- throw new ManifoldCFException("MongoDB: Given
information is not valid or mongod process doesn't run" + e.getMessage(), e);
- } catch (NumberFormatException e) {
- Logging.connectors.warn("MongoDB: Given port is
not valid number. " + e.getMessage(), e);
- throw new ManifoldCFException("MongoDB: Given port
is not valid number. " + e.getMessage(), e);
- }
}
+
+ final String finalHost = effectiveHost;
+ final int finalPort = effectivePort;
+ MongoClientSettings.Builder settingsBuilder =
MongoClientSettings.builder()
+ .applyToClusterSettings(builder ->
+ builder.hosts(Collections.singletonList(new
ServerAddress(finalHost, finalPort)))
+ );
+
if (!StringUtils.isEmpty(username) &&
!StringUtils.isEmpty(password)) {
- boolean auth = mongoDatabase.authenticate(username,
password.toCharArray());
- if (!auth) {
- Logging.connectors.warn("MongoDB:Authentication
Error! Given database username and password doesn't match for the database
given.");
- throw new ManifoldCFException("MongoDB: Given
database username and password doesn't match for the database given.");
- } else {
- if (Logging.connectors.isDebugEnabled()) {
- Logging.connectors.debug("MongoDB: Username =
'" + username + "'");
- Logging.connectors.debug("MongoDB: Password
exists");
- }
- }
+ MongoCredential credential =
MongoCredential.createCredential(username, database, password.toCharArray());
+ settingsBuilder.credential(credential);
}
+
+ client = MongoClients.create(settingsBuilder.build());
+ mongoDatabase = client.getDatabase(database);
}
} catch (Throwable e) {
@@ -713,12 +696,8 @@ public class MongodbOutputConnector extends
BaseOutputConnector {
public void run() {
try {
- if (client != null) {
- DBTCPConnector dbtcpConnector = client.getConnector();
- DBPortPool dbPortPool =
dbtcpConnector.getDBPortPool(client.getAddress());
- DBPort dbPort = dbPortPool.get();
- dbPort.ensureOpen();
- client = null;
+ if (client != null && mongoDatabase != null) {
+ mongoDatabase.runCommand(new Document("ping", 1));
}
} catch (Throwable e) {
@@ -743,6 +722,9 @@ public class MongodbOutputConnector extends
BaseOutputConnector {
public void run() {
try {
+ if (client != null) {
+ client.close();
+ }
client = null;
mongoDatabase = null;
} catch (Throwable e) {
@@ -763,9 +745,6 @@ public class MongodbOutputConnector extends
BaseOutputConnector {
getSession();
- //to get an exception if write fails
- client.setWriteConcern(WriteConcern.SAFE);
-
long startTime = System.currentTimeMillis();
String resultDescription = StringUtils.EMPTY;
@@ -792,9 +771,9 @@ public class MongodbOutputConnector extends
BaseOutputConnector {
byte[] bytes = IOUtils.toByteArray(inputStream);
Binary content = new Binary(bytes);
- DBCollection mongoCollection =
mongoDatabase.getCollection(collection);
+ MongoCollection<Document> mongoCollection =
mongoDatabase.getCollection(collection);
- BasicDBObject newDocument = new BasicDBObject();
+ Document newDocument = new Document();
newDocument.append("fileName", fileName)
.append("creationDate", creationDate)
.append("lastModificationDate", lastModificationDate)
@@ -811,30 +790,21 @@ public class MongodbOutputConnector extends
BaseOutputConnector {
String fieldName = i.next();
Date[] dateFieldValues = document.getFieldAsDates(fieldName);
if (dateFieldValues != null) {
- newDocument.append(fieldName, dateFieldValues);
+ newDocument.append(fieldName,
Arrays.asList(dateFieldValues));
} else {
String[] fieldValues =
document.getFieldAsStrings(fieldName);
- newDocument.append(fieldName, fieldValues);
+ newDocument.append(fieldName, Arrays.asList(fieldValues));
}
}
- BasicDBObject searchQuery = new
BasicDBObject().append("documentURI", documentURI);
- DBCursor cursor = mongoCollection.find(searchQuery);
- Long numberOfDocumentsBeforeInsert = mongoCollection.count();
- WriteResult result;
- if (cursor.count() > 0) {
- result = mongoCollection.update(searchQuery, newDocument);
- } else {
- result = mongoCollection.insert(newDocument);
- }
- //result.getLastError().get("err") == null
- //To get the number of documents indexed i.e. tne number of
documents inserted or updated
- Long numberOfDocumentsAfterInsert = mongoCollection.count();
- Long numberOfDocumentsInserted = numberOfDocumentsAfterInsert -
numberOfDocumentsBeforeInsert;
- Long numberOfDocumentsIndexed = (numberOfDocumentsInserted != 0) ?
numberOfDocumentsInserted : result.getN();
+ Bson searchQuery = Filters.eq("documentURI", documentURI);
+ ReplaceOptions replaceOptions = new ReplaceOptions().upsert(true);
+ UpdateResult result = mongoCollection.replaceOne(searchQuery,
newDocument, replaceOptions);
+
+ long numberOfDocumentsIndexed = (result.getUpsertedId() != null) ?
1L : (result.getMatchedCount() > 0 ? 1L : 0L);
Logging.connectors.info("Number of documents indexed : " +
numberOfDocumentsIndexed);
- //check if a document is inserted or updated
(numberOfDocumentsInserted > 0) || (result.getN() > 0)
+ //check if a document is inserted or updated
if (numberOfDocumentsIndexed > 0) {
resultDescription = DOCUMENT_STATUS_ACCEPTED_DESC;
return DOCUMENT_STATUS_ACCEPTED;
@@ -874,15 +844,14 @@ public class MongodbOutputConnector extends
BaseOutputConnector {
try {
- DBCollection mongoCollection =
mongoDatabase.getCollection(collection);
+ MongoCollection<Document> mongoCollection =
mongoDatabase.getCollection(collection);
- BasicDBObject query = new BasicDBObject();
- query.append("documentURI", documentURI);
+ Bson query = Filters.eq("documentURI", documentURI);
- WriteResult result = mongoCollection.remove(query);
- Logging.connectors.info("Number of documents deleted : " +
result.getN());
+ DeleteResult result = mongoCollection.deleteOne(query);
+ Logging.connectors.info("Number of documents deleted : " +
result.getDeletedCount());
- if (result.getN() > 0) {
+ if (result.getDeletedCount() > 0) {
resultDescription = DOCUMENT_DELETION_STATUS_ACCEPTED;
} else {
resultDescription = DOCUMENT_DELETION_STATUS_REJECTED;
diff --git
a/connectors/mongodb/connector/src/test/java/org/apache/manifoldcf/agents/output/mongodboutput/tests/APISanityHSQLDBIT.java
b/connectors/mongodb/connector/src/test/java/org/apache/manifoldcf/agents/output/mongodboutput/tests/APISanityHSQLDBIT.java
index e759ea39f..a66ceeb49 100644
---
a/connectors/mongodb/connector/src/test/java/org/apache/manifoldcf/agents/output/mongodboutput/tests/APISanityHSQLDBIT.java
+++
b/connectors/mongodb/connector/src/test/java/org/apache/manifoldcf/agents/output/mongodboutput/tests/APISanityHSQLDBIT.java
@@ -18,17 +18,22 @@
*/
package org.apache.manifoldcf.agents.output.mongodboutput.tests;
-import com.mongodb.*;
+import com.mongodb.MongoClientSettings;
+import com.mongodb.ServerAddress;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
import org.apache.manifoldcf.agents.output.mongodboutput.MongodbOutputConfig;
import org.apache.manifoldcf.core.interfaces.Configuration;
import org.apache.manifoldcf.core.interfaces.ConfigurationNode;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
import org.apache.manifoldcf.crawler.system.ManifoldCF;
+import org.bson.Document;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import java.net.UnknownHostException;
import java.util.Collections;
/**
@@ -37,26 +42,20 @@ import java.util.Collections;
public class APISanityHSQLDBIT extends BaseITHSQLDB {
private MongoClient client = null;
- private DB mongoDatabase = null;
- private DBCollection testCollection = null;
+ private MongoDatabase mongoDatabase = null;
+ private MongoCollection<Document> testCollection = null;
private MongoClient getMongodbClientSession() throws Exception {
-
- MongoClient testClient = null;
-
- try {
- testClient = new
MongoClient(BaseITSanityTestUtils.TARGET_HOST_VALUE,
Integer.parseInt(BaseITSanityTestUtils.TARGET_PORT_VALUE));
- mongoDatabase =
testClient.getDB(BaseITSanityTestUtils.TARGET_DATABASE__VALUE);
- } catch (UnknownHostException ex) {
- throw new ManifoldCFException("Mongodb: Default host is not found.
Is mongod process running?" + ex.getMessage(), ex);
- }
-
- return testClient;
+ String host = BaseITSanityTestUtils.TARGET_HOST_VALUE;
+ int port = Integer.parseInt(BaseITSanityTestUtils.TARGET_PORT_VALUE);
+ return MongoClients.create(MongoClientSettings.builder()
+ .applyToClusterSettings(b -> b.hosts(Collections.singletonList(new
ServerAddress(host, port))))
+ .build());
}
- private long queryTestContents(DBCollection testCollection) {
+ private long queryTestContents(MongoCollection<Document> testCollection) {
// deduct 1 for the document we created at the test area
- return testCollection.count() - 1;
+ return testCollection.countDocuments() - 1;
}
@Before
@@ -67,20 +66,22 @@ public class APISanityHSQLDBIT extends BaseITHSQLDB {
client = getMongodbClientSession();
//creating a test database named testDatabase
- mongoDatabase =
client.getDB(BaseITSanityTestUtils.TARGET_DATABASE__VALUE);
+ mongoDatabase =
client.getDatabase(BaseITSanityTestUtils.TARGET_DATABASE__VALUE);
//creating a test collection named testCollection
testCollection =
mongoDatabase.getCollection(BaseITSanityTestUtils.TARGET_COLLECTION_VALUE);
//create credentials
- final BasicDBObject createUserCommand = new
BasicDBObject("createUser",
BaseITSanityTestUtils.TARGET_USERNAME_VALUE).append("pwd",
BaseITSanityTestUtils.TARGET_PASSWORD_VALUE).append("roles",
- Collections.singletonList(new BasicDBObject("role",
"readWrite").append("db", BaseITSanityTestUtils.TARGET_DATABASE__VALUE)));
+ final Document createUserCommand = new Document("createUser",
BaseITSanityTestUtils.TARGET_USERNAME_VALUE)
+ .append("pwd", BaseITSanityTestUtils.TARGET_PASSWORD_VALUE)
+ .append("roles",
+ Collections.singletonList(new Document("role",
"readWrite").append("db", BaseITSanityTestUtils.TARGET_DATABASE__VALUE)));
- CommandResult result = mongoDatabase.command(createUserCommand);
+ mongoDatabase.runCommand(createUserCommand);
//create a document to be inserted
- BasicDBObject newDocument = new BasicDBObject();
+ Document newDocument = new Document();
newDocument.append("fileName", "fileName")
.append("creationDate", "creationDate")
.append("lastModificationDate", "lastModificationDate")
@@ -92,7 +93,7 @@ public class APISanityHSQLDBIT extends BaseITHSQLDB {
.append("sourcePath", "sourcePath");
//insert the test document so that the test collection and test
Database are created
- testCollection.insert(newDocument);
+ testCollection.insertOne(newDocument);
} catch (Exception e) {
e.printStackTrace();
throw e;
@@ -105,9 +106,13 @@ public class APISanityHSQLDBIT extends BaseITHSQLDB {
throws Exception {
if (mongoDatabase != null) {
- // deleting the dummyuser
- final BasicDBObject deleteUserCommand = new
BasicDBObject("dropUser", BaseITSanityTestUtils.TARGET_USERNAME_VALUE);
- CommandResult result = mongoDatabase.command(deleteUserCommand);
+ try {
+ // deleting the dummyuser
+ final Document deleteUserCommand = new Document("dropUser",
BaseITSanityTestUtils.TARGET_USERNAME_VALUE);
+ mongoDatabase.runCommand(deleteUserCommand);
+ } catch (Exception e) {
+ // ignore
+ }
// dropping the test Collection
if (testCollection != null) {
@@ -115,7 +120,11 @@ public class APISanityHSQLDBIT extends BaseITHSQLDB {
}
// dropping the test Database
- mongoDatabase.dropDatabase();
+ mongoDatabase.drop();
+ }
+ if (client != null) {
+ client.close();
+ client = null;
}
}
diff --git
a/connectors/mongodb/connector/src/test/java/org/apache/manifoldcf/agents/output/mongodboutput/tests/BaseITHSQLDB.java
b/connectors/mongodb/connector/src/test/java/org/apache/manifoldcf/agents/output/mongodboutput/tests/BaseITHSQLDB.java
index 28e6c0f49..a2eb26666 100644
---
a/connectors/mongodb/connector/src/test/java/org/apache/manifoldcf/agents/output/mongodboutput/tests/BaseITHSQLDB.java
+++
b/connectors/mongodb/connector/src/test/java/org/apache/manifoldcf/agents/output/mongodboutput/tests/BaseITHSQLDB.java
@@ -21,17 +21,8 @@ package
org.apache.manifoldcf.agents.output.mongodboutput.tests;
import org.junit.After;
import org.junit.Before;
-import de.flapdoodle.embed.mongo.config.Net;
-import de.flapdoodle.embed.mongo.distribution.Version;
-import de.flapdoodle.embed.mongo.transitions.ImmutableMongod;
-import de.flapdoodle.embed.mongo.transitions.Mongod;
-import de.flapdoodle.embed.mongo.transitions.RunningMongodProcess;
-import de.flapdoodle.os.CommonArchitecture;
-import de.flapdoodle.os.CommonOS;
-import de.flapdoodle.os.ImmutablePlatform;
-import de.flapdoodle.os.Platform;
-import de.flapdoodle.reverse.TransitionWalker;
-import de.flapdoodle.reverse.transitions.Start;
+import org.testcontainers.mongodb.MongoDBContainer;
+import org.testcontainers.utility.DockerImageName;
/**
* Base integration tests class for MongoDB tested against a CMIS repository
@@ -40,7 +31,7 @@ import de.flapdoodle.reverse.transitions.Start;
*/
public class BaseITHSQLDB extends
org.apache.manifoldcf.crawler.tests.BaseITHSQLDB {
- private TransitionWalker.ReachedState<RunningMongodProcess>
mongodProcess;
+ protected static MongoDBContainer mongoDBContainer;
protected String[] getConnectorNames() {
return new String[] { "CMIS" };
@@ -62,27 +53,19 @@ public class BaseITHSQLDB extends
org.apache.manifoldcf.crawler.tests.BaseITHSQL
@Before
public void setUpMongoDB() throws Exception {
-
- String bindIp = "localhost";
- int port = 27017;
- ImmutableMongod.Builder mongodBuilder = Mongod.builder()
-
.net(Start.to(Net.class).initializedWith(Net.of(bindIp, port, false)));
-
- Platform detected = Platform.detect(CommonOS.list());
- if (detected.operatingSystem() == CommonOS.OS_X &&
detected.architecture() == CommonArchitecture.ARM_64) {
- Platform x86Platform =
ImmutablePlatform.builder().from(detected)
- .architecture(CommonArchitecture.X86_64)
- .build();
-
mongodBuilder.platform(Start.to(Platform.class).initializedWith(x86Platform));
+ if (mongoDBContainer == null) {
+ mongoDBContainer = new
MongoDBContainer(DockerImageName.parse("mongo:9.0.2"));
+ mongoDBContainer.start();
}
-
- mongodProcess = mongodBuilder.build().start(Version.Main.V4_0);
+ BaseITSanityTestUtils.TARGET_HOST_VALUE =
mongoDBContainer.getHost();
+ BaseITSanityTestUtils.TARGET_PORT_VALUE =
String.valueOf(mongoDBContainer.getFirstMappedPort());
}
@After
public void cleanUpMongoDB() throws Exception {
- if (mongodProcess != null) {
- mongodProcess.close();
+ if (mongoDBContainer != null) {
+ mongoDBContainer.stop();
+ mongoDBContainer = null;
}
}
diff --git
a/connectors/mongodb/connector/src/test/java/org/apache/manifoldcf/agents/output/mongodboutput/tests/BaseITSanityTestUtils.java
b/connectors/mongodb/connector/src/test/java/org/apache/manifoldcf/agents/output/mongodboutput/tests/BaseITSanityTestUtils.java
index b7af0d75b..dd6df3c81 100644
---
a/connectors/mongodb/connector/src/test/java/org/apache/manifoldcf/agents/output/mongodboutput/tests/BaseITSanityTestUtils.java
+++
b/connectors/mongodb/connector/src/test/java/org/apache/manifoldcf/agents/output/mongodboutput/tests/BaseITSanityTestUtils.java
@@ -30,8 +30,8 @@ public class BaseITSanityTestUtils {
//Test values for the source repository
public static final String TARGET_USERNAME_VALUE = "dummyuser";
public static final String TARGET_PASSWORD_VALUE = "dummysecrect";
- public static final String TARGET_HOST_VALUE = "localhost";
- public static final String TARGET_PORT_VALUE = "27017";
+ public static String TARGET_HOST_VALUE = "localhost";
+ public static String TARGET_PORT_VALUE = "27017";
public static final String TARGET_DATABASE__VALUE = "testDatabase";
public static final String TARGET_COLLECTION_VALUE = "testCollection";
diff --git a/connectors/mongodb/pom.xml b/connectors/mongodb/pom.xml
index 3a0591680..27674f7f2 100644
--- a/connectors/mongodb/pom.xml
+++ b/connectors/mongodb/pom.xml
@@ -29,7 +29,6 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <mongo.java.driver.version>2.14.3</mongo.java.driver.version>
</properties>
<developers>
@@ -74,8 +73,7 @@
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
- <artifactId>mongo-java-driver</artifactId>
- <version>${mongo.java.driver.version}</version>
+ <artifactId>mongodb-driver-sync</artifactId>
</dependency>
<!-- Test dependencies -->
@@ -87,9 +85,8 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>de.flapdoodle.embed</groupId>
- <artifactId>de.flapdoodle.embed.mongo</artifactId>
- <version>4.33.0</version>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>testcontainers-mongodb</artifactId>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/connectors/mongodb/test-materials/README.txt
b/connectors/mongodb/test-materials/README.txt
index 4be9d6c1b..bf08d6249 100644
--- a/connectors/mongodb/test-materials/README.txt
+++ b/connectors/mongodb/test-materials/README.txt
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-To test this connector, download the Embedded MongoDB dependencies using "ant
download-dependencies"
+To test this connector, download the Testcontainers dependencies using "ant
download-dependencies"
or the global "ant make-deps" target, and use the "ant test" target for the
standard ant build.
-You can read more about the process on the "how-to-build-and-deploy.html"
documentation page.
\ No newline at end of file
+The integration tests start MongoDB 9.0.2 as a Docker container, so a running
Docker daemon is required.
+You can read more about the process on the "how-to-build-and-deploy.html"
documentation page.
