Repository: incubator-gobblin Updated Branches: refs/heads/master 1cbd157fa -> ea1f6422a
[GOBBLIN-465] Add support for couchbase client cert auth The requisite version bump eliminates support for couchbase v3. Closes #2337 from wwest4/master Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/ea1f6422 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/ea1f6422 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/ea1f6422 Branch: refs/heads/master Commit: ea1f6422a28cfdb6d3a24cdd40459150b7889c5f Parents: 1cbd157 Author: William West <[email protected]> Authored: Wed May 2 07:53:12 2018 -0700 Committer: Abhishek Tiwari <[email protected]> Committed: Wed May 2 07:53:12 2018 -0700 ---------------------------------------------------------------------- gobblin-modules/gobblin-couchbase/build.gradle | 2 +- .../scripts/install_test_deps.sh | 5 +++-- .../writer/CouchbaseEnvironmentFactory.java | 19 ++++++++++++++++++- .../couchbase/writer/CouchbaseWriter.java | 14 +++++++++----- .../writer/CouchbaseWriterConfigurationKeys.java | 7 +++++++ .../gobblin/couchbase/CouchbaseTestServer.java | 4 ++-- 6 files changed, 40 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/ea1f6422/gobblin-modules/gobblin-couchbase/build.gradle ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-couchbase/build.gradle b/gobblin-modules/gobblin-couchbase/build.gradle index 7e158a9..310e4b6 100644 --- a/gobblin-modules/gobblin-couchbase/build.gradle +++ b/gobblin-modules/gobblin-couchbase/build.gradle @@ -23,7 +23,7 @@ dependencies { compile project(":gobblin-utility") compile project(":gobblin-metrics-libs:gobblin-metrics") - compile "com.couchbase.client:java-client:2.3.5" + compile "com.couchbase.client:java-client:2.5.4" compile externalDependency.avro compile externalDependency.jacksonCore compile externalDependency.jacksonMapper http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/ea1f6422/gobblin-modules/gobblin-couchbase/scripts/install_test_deps.sh ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-couchbase/scripts/install_test_deps.sh b/gobblin-modules/gobblin-couchbase/scripts/install_test_deps.sh index 8217884..92e24db 100755 --- a/gobblin-modules/gobblin-couchbase/scripts/install_test_deps.sh +++ b/gobblin-modules/gobblin-couchbase/scripts/install_test_deps.sh @@ -16,15 +16,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # +MOCK_COUCHBASE_VERSION=1.5.18 mkdir -p mock-couchbase if [ -d mock-couchbase/.git ]; then git -C mock-couchbase pull - git -C mock-couchbase checkout b929a9f99a31a233c43bd94b6a696ad966206e02 + git -C mock-couchbase checkout tags/${MOCK_COUCHBASE_VERSION} else git clone https://github.com/couchbase/CouchbaseMock.git mock-couchbase - git -C mock-couchbase checkout b929a9f99a31a233c43bd94b6a696ad966206e02 + git -C mock-couchbase checkout tags/${MOCK_COUCHBASE_VERSION} fi pushd mock-couchbase if ! mvn package -Dmaven.test.skip=true; then http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/ea1f6422/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseEnvironmentFactory.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseEnvironmentFactory.java b/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseEnvironmentFactory.java index 2edf4c1..102d6ed 100644 --- a/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseEnvironmentFactory.java +++ b/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseEnvironmentFactory.java @@ -20,6 +20,7 @@ package org.apache.gobblin.couchbase.writer; import com.couchbase.client.java.env.CouchbaseEnvironment; import com.couchbase.client.java.env.DefaultCouchbaseEnvironment; import com.typesafe.config.Config; +import org.apache.gobblin.util.ConfigUtils; /** @@ -38,9 +39,25 @@ public class CouchbaseEnvironmentFactory { */ public static synchronized CouchbaseEnvironment getInstance(Config config) { + Boolean sslEnabled = ConfigUtils.getBoolean(config, CouchbaseWriterConfigurationKeys.SSL_ENABLED, false); + String sslKeystoreFile = ConfigUtils.getString(config, CouchbaseWriterConfigurationKeys.SSL_KEYSTORE_FILE, ""); + String sslKeystorePassword = ConfigUtils.getString(config, CouchbaseWriterConfigurationKeys.SSL_KEYSTORE_PASSWORD, ""); + String sslTruststoreFile = ConfigUtils.getString(config, CouchbaseWriterConfigurationKeys.SSL_TRUSTSTORE_FILE, ""); + String sslTruststorePassword = ConfigUtils.getString(config, CouchbaseWriterConfigurationKeys.SSL_TRUSTSTORE_PASSWORD, ""); + Boolean certAuthEnabled = ConfigUtils.getBoolean(config, CouchbaseWriterConfigurationKeys.CERT_AUTH_ENABLED, false); + + DefaultCouchbaseEnvironment.Builder builder = DefaultCouchbaseEnvironment + .builder() + .sslEnabled(sslEnabled) + .sslKeystoreFile(sslKeystoreFile) + .sslKeystorePassword(sslKeystorePassword) + .sslTruststoreFile(sslTruststoreFile) + .sslTruststorePassword(sslTruststorePassword) + .certAuthEnabled(certAuthEnabled); + if (couchbaseEnvironment == null) { - return DefaultCouchbaseEnvironment.create(); + return builder.build(); } else { return couchbaseEnvironment; http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/ea1f6422/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriter.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriter.java b/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriter.java index f7001da..46c6821 100644 --- a/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriter.java +++ b/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriter.java @@ -109,16 +109,20 @@ public class CouchbaseWriter<D extends AbstractDocument> implements AsyncDataWri public CouchbaseWriter(CouchbaseEnvironment couchbaseEnvironment, Config config) { List<String> hosts = ConfigUtils.getStringList(config, CouchbaseWriterConfigurationKeys.BOOTSTRAP_SERVERS); - - _cluster = CouchbaseCluster.create(couchbaseEnvironment, hosts); + String password = ConfigUtils.getString(config, CouchbaseWriterConfigurationKeys.PASSWORD, ""); String bucketName = ConfigUtils.getString(config, CouchbaseWriterConfigurationKeys.BUCKET, CouchbaseWriterConfigurationKeys.BUCKET_DEFAULT); - String password = ConfigUtils.getString(config, CouchbaseWriterConfigurationKeys.PASSWORD, ""); + _cluster = CouchbaseCluster.create(couchbaseEnvironment, hosts); - _bucket = _cluster.openBucket(bucketName, password, - Collections.<Transcoder<? extends Document, ?>>singletonList(_tupleDocumentTranscoder)); + if(!password.isEmpty()) { + _bucket = _cluster.openBucket(bucketName, password, + Collections.<Transcoder<? extends Document, ?>>singletonList(_tupleDocumentTranscoder)); + } else { + _bucket = _cluster.openBucket(bucketName, + Collections.<Transcoder<? extends Document, ?>>singletonList(_tupleDocumentTranscoder)); + } _operationTimeout = ConfigUtils.getLong(config, CouchbaseWriterConfigurationKeys.OPERATION_TIMEOUT_MILLIS, CouchbaseWriterConfigurationKeys.OPERATION_TIMEOUT_DEFAULT); http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/ea1f6422/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterConfigurationKeys.java b/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterConfigurationKeys.java index 43e69b9..7e29ed8 100644 --- a/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterConfigurationKeys.java +++ b/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterConfigurationKeys.java @@ -34,6 +34,13 @@ public class CouchbaseWriterConfigurationKeys { public static final String BUCKET_DEFAULT = "default"; public static final String PASSWORD = prefix("password"); + public static final String SSL_ENABLED = prefix("sslEnabled"); + public static final String SSL_KEYSTORE_FILE = prefix("sslKeystoreFile"); + public static final String SSL_KEYSTORE_PASSWORD = prefix("sslKeystorePassword"); + public static final String SSL_TRUSTSTORE_FILE = prefix("sslTruststoreFile"); + public static final String SSL_TRUSTSTORE_PASSWORD = prefix("sslTruststorePassword"); + public static final String CERT_AUTH_ENABLED = prefix("certAuthEnabled"); + public static final String OPERATION_TIMEOUT_MILLIS = "operationTimeoutMillis"; public static final long OPERATION_TIMEOUT_DEFAULT = 10000; // 10 second default timeout http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/ea1f6422/gobblin-modules/gobblin-couchbase/src/test/java/org/apache/gobblin/couchbase/CouchbaseTestServer.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-couchbase/src/test/java/org/apache/gobblin/couchbase/CouchbaseTestServer.java b/gobblin-modules/gobblin-couchbase/src/test/java/org/apache/gobblin/couchbase/CouchbaseTestServer.java index 3189a55..63ae423 100644 --- a/gobblin-modules/gobblin-couchbase/src/test/java/org/apache/gobblin/couchbase/CouchbaseTestServer.java +++ b/gobblin-modules/gobblin-couchbase/src/test/java/org/apache/gobblin/couchbase/CouchbaseTestServer.java @@ -48,7 +48,7 @@ import org.apache.gobblin.test.TestUtils; public class CouchbaseTestServer { private static final String COUCHBASE_JAR_PATH="gobblin-modules/gobblin-couchbase/mock-couchbase/target/"; - private static final String COUCHBASE_MOCK_JAR=COUCHBASE_JAR_PATH + "CouchbaseMock-1.4.4.jar"; + private static final String COUCHBASE_MOCK_JAR=COUCHBASE_JAR_PATH + "CouchbaseMock-1.5.18.jar"; private Process couchbaseProcess; @@ -67,7 +67,7 @@ public class CouchbaseTestServer { String[] commands = {"/usr/bin/java", "-cp", COUCHBASE_MOCK_JAR, - "org.couchbase.mock.CouchbaseMock", + "com.couchbase.mock.CouchbaseMock", "--port", _port +"", "-n",
