Repository: incubator-gobblin Updated Branches: refs/heads/master 807eb46e1 -> 1fb2bb732
[GOBBLIN-549] Add configurability to Couchbase Writer (failure percentage, retries) Closes #2410 from shirshanka/couchbase-config Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/1fb2bb73 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/1fb2bb73 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/1fb2bb73 Branch: refs/heads/master Commit: 1fb2bb732a36994174aaab29455755c4ded15357 Parents: 807eb46 Author: Shirshanka Das <[email protected]> Authored: Tue Jul 31 20:30:43 2018 -0700 Committer: Abhishek Tiwari <[email protected]> Committed: Tue Jul 31 20:30:43 2018 -0700 ---------------------------------------------------------------------- .../couchbase/writer/CouchbaseWriterBuilder.java | 15 +++++++++++++-- .../writer/CouchbaseWriterConfigurationKeys.java | 11 ++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/1fb2bb73/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterBuilder.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterBuilder.java b/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterBuilder.java index 58f1a7a..7e8625b 100644 --- a/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterBuilder.java +++ b/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterBuilder.java @@ -42,12 +42,23 @@ public class CouchbaseWriterBuilder extends DataWriterBuilder { Config config = ConfigUtils.propertiesToConfig(taskProps); CouchbaseEnvironment couchbaseEnvironment = CouchbaseEnvironmentFactory.getInstance(config); //TODO: Read config to decide whether to build a blocking writer or an async writer + + double failureAllowance = ConfigUtils.getDouble(config, CouchbaseWriterConfigurationKeys.FAILURE_ALLOWANCE_PCT_CONFIG, + CouchbaseWriterConfigurationKeys.FAILURE_ALLOWANCE_PCT_DEFAULT) / 100.0; + + boolean retriesEnabled = ConfigUtils.getBoolean(config, CouchbaseWriterConfigurationKeys.RETRIES_ENABLED, + CouchbaseWriterConfigurationKeys.RETRIES_ENABLED_DEFAULT); + + int maxRetries = ConfigUtils.getInt(config, CouchbaseWriterConfigurationKeys.MAX_RETRIES, + CouchbaseWriterConfigurationKeys.MAX_RETRIES_DEFAULT); + // build an async couchbase writer AsyncDataWriter couchbaseWriter = new CouchbaseWriter(couchbaseEnvironment, config); return AsyncWriterManager.builder() .asyncDataWriter(couchbaseWriter) - .failureAllowanceRatio(0.0) - .retriesEnabled(false) + .failureAllowanceRatio(failureAllowance) + .retriesEnabled(retriesEnabled) + .numRetries(maxRetries) .config(config) .build(); } http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/1fb2bb73/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterConfigurationKeys.java b/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterConfigurationKeys.java index 7e29ed8..9d4400b 100644 --- a/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterConfigurationKeys.java +++ b/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterConfigurationKeys.java @@ -41,7 +41,16 @@ public class CouchbaseWriterConfigurationKeys { public static final String SSL_TRUSTSTORE_PASSWORD = prefix("sslTruststorePassword"); public static final String CERT_AUTH_ENABLED = prefix("certAuthEnabled"); - public static final String OPERATION_TIMEOUT_MILLIS = "operationTimeoutMillis"; + public static final String OPERATION_TIMEOUT_MILLIS = prefix("operationTimeoutMillis"); public static final long OPERATION_TIMEOUT_DEFAULT = 10000; // 10 second default timeout + public static final String RETRIES_ENABLED = prefix("retriesEnabled"); + public static final boolean RETRIES_ENABLED_DEFAULT = false; + + public static final String MAX_RETRIES = prefix("maxRetries"); + public static final int MAX_RETRIES_DEFAULT = 5; + + static final String FAILURE_ALLOWANCE_PCT_CONFIG = prefix("failureAllowancePercentage"); + static final double FAILURE_ALLOWANCE_PCT_DEFAULT = 0.0; + }
