Repository: incubator-gobblin Updated Branches: refs/heads/master 261819c63 -> 241a1b3b7
[GOBBLIN-453][GOBBLIN-452][GOBBLIN-190] Make the GAAS rest port configurable & Logging related Improvement in Gobblin Cluster Closes #2362 from dallaybatta/GOBBLIN-453 Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/241a1b3b Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/241a1b3b Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/241a1b3b Branch: refs/heads/master Commit: 241a1b3b74de8ffe19896e2abe20f7c8a76cf550 Parents: 261819c Author: [email protected] <[email protected]> Authored: Thu Jun 14 15:53:40 2018 -0700 Committer: Abhishek Tiwari <[email protected]> Committed: Thu Jun 14 15:53:40 2018 -0700 ---------------------------------------------------------------------- bin/gobblin-cluster-master.sh | 9 +-- bin/gobblin-cluster-worker.sh | 7 +- bin/gobblin-service.sh | 7 +- conf/standalone/master/application.conf | 71 ++++++++++++++++++++ conf/standalone/master/log4j-cluster.properties | 28 ++++++++ conf/standalone/master/quartz.properties | 3 + conf/standalone/master/reference.conf | 15 +++++ conf/standalone/worker/application.conf | 71 ++++++++++++++++++++ conf/standalone/worker/log4j-cluster.properties | 28 ++++++++ conf/standalone/worker/quartz.properties | 3 + conf/standalone/worker/reference.conf | 15 +++++ .../gobblin/service/ServiceConfigKeys.java | 3 + .../gobblin/kafka/writer/Kafka09DataWriter.java | 2 +- .../gobblin/kafka/KafkaClusterTestBase.java | 2 +- .../kafka/writer/Kafka09TopicProvisionTest.java | 2 +- .../writer/KafkaWriterConfigurationKeys.java | 2 +- .../gobblin/restli/EmbeddedRestliServer.java | 5 +- .../modules/core/GobblinServiceManager.java | 2 + 18 files changed, 259 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/241a1b3b/bin/gobblin-cluster-master.sh ---------------------------------------------------------------------- diff --git a/bin/gobblin-cluster-master.sh b/bin/gobblin-cluster-master.sh index 56cb179..cd196d9 100755 --- a/bin/gobblin-cluster-master.sh +++ b/bin/gobblin-cluster-master.sh @@ -29,12 +29,13 @@ function start() { fi LOG_ARGS="1>${FWDIR_LOGS}/GobblinCluster.master.stdout 2>${FWDIR_LOGS}/GobblinCluster.master.stderr" - - COMMAND="$JAVA_HOME/bin/java -cp $CLASSPATH $JVM_FLAGS org.apache.gobblin.cluster.GobblinClusterManager --standalone_cluster true --app_name $CLUSTER_NAME $LOG_ARGS" + + LOG4J_PATH=file://${FWDIR_CONF}/log4j-cluster.properties + COMMAND="$JAVA_HOME/bin/java -Dlog4j.configuration=$LOG4J_PATH -cp $CLASSPATH $JVM_FLAGS org.apache.gobblin.cluster.GobblinClusterManager --standalone_cluster true --app_name $CLUSTER_NAME $LOG_ARGS" echo "Running command:" echo "$COMMAND" - nohup $COMMAND >master.out 2>&1 & echo $! > $PID + nohup $COMMAND >clustermaster.out 2>&1 & echo $! > $PID } function stop() { @@ -52,7 +53,7 @@ function stop() { FWDIR="$(cd `dirname $0`/..; pwd)" FWDIR_LIB=${FWDIR}/lib -FWDIR_CONF=${FWDIR}/conf/standalone +FWDIR_CONF=${FWDIR}/conf/standalone/master FWDIR_BIN=${FWDIR}/bin FWDIR_LOGS=${FWDIR}/logs CLUSTER_NAME="standalone_cluster" http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/241a1b3b/bin/gobblin-cluster-worker.sh ---------------------------------------------------------------------- diff --git a/bin/gobblin-cluster-worker.sh b/bin/gobblin-cluster-worker.sh index a6459b9..000a67d 100755 --- a/bin/gobblin-cluster-worker.sh +++ b/bin/gobblin-cluster-worker.sh @@ -31,8 +31,9 @@ function start() { fi LOG_ARGS="1>${FWDIR_LOGS}/GobblinCluster.worker.$WORKER_ID.stdout 2>${FWDIR_LOGS}/GobblinCluster.worker.$WORKER_ID.stderr" - - COMMAND="$JAVA_HOME/bin/java -cp $CLASSPATH $JVM_FLAGS org.apache.gobblin.cluster.GobblinTaskRunner --app_name $CLUSTER_NAME --helix_instance_name worker.$WORKER_ID $LOG_ARGS" + + LOG4J_PATH=file://${FWDIR_CONF}/log4j-cluster.properties + COMMAND="$JAVA_HOME/bin/java -Dlog4j.configuration=$LOG4J_PATH -cp $CLASSPATH $JVM_FLAGS org.apache.gobblin.cluster.GobblinTaskRunner --app_name $CLUSTER_NAME --helix_instance_name worker.$WORKER_ID $LOG_ARGS" echo "Running command:" echo "$COMMAND" @@ -54,7 +55,7 @@ function stop() { FWDIR="$(cd `dirname $0`/..; pwd)" FWDIR_LIB=${FWDIR}/lib -FWDIR_CONF=${FWDIR}/conf/standalone +FWDIR_CONF=${FWDIR}/conf/standalone/worker FWDIR_BIN=${FWDIR}/bin FWDIR_LOGS=${FWDIR}/logs CLUSTER_NAME="standalone_cluster" http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/241a1b3b/bin/gobblin-service.sh ---------------------------------------------------------------------- diff --git a/bin/gobblin-service.sh b/bin/gobblin-service.sh index 762c9d7..67b6cee 100644 --- a/bin/gobblin-service.sh +++ b/bin/gobblin-service.sh @@ -30,14 +30,15 @@ function start() { LOG_ARGS="1>${FWDIR_LOGS}/GobblinService.stdout 2>${FWDIR_LOGS}/GobblinService.stderr" - LOG4J_ARGS="-Dlog4j.configuration=conf/log4j.xml" +# LOG4J_ARGS="-Dlog4j.configuration=conf/log4j.xml" DEBUGGER_ARGS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006" - COMMAND="$JAVA_HOME/bin/java -cp $CLASSPATH $JVM_FLAGS $LOG4J_ARGS $DEBUGGER_ARGS org.apache.gobblin.service.modules.core.GobblinServiceManager --service_name $SERVICE_NAME $LOG_ARGS" + LOG4J_PATH=file://${FWDIR_CONF}/log4j-service.properties + COMMAND="$JAVA_HOME/bin/java -Dlog4j.configuration=$LOG4J_PATH -cp $CLASSPATH $JVM_FLAGS $LOG4J_ARGS $DEBUGGER_ARGS org.apache.gobblin.service.modules.core.GobblinServiceManager --service_name $SERVICE_NAME $LOG_ARGS" echo "Running command:" echo "$COMMAND" - nohup $COMMAND >master.out 2>&1 & echo $! > $PID + nohup $COMMAND >service.out 2>&1 & echo $! > $PID } function stop() { http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/241a1b3b/conf/standalone/master/application.conf ---------------------------------------------------------------------- diff --git a/conf/standalone/master/application.conf b/conf/standalone/master/application.conf new file mode 100644 index 0000000..e9b8323 --- /dev/null +++ b/conf/standalone/master/application.conf @@ -0,0 +1,71 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Sample configuration properties for the Gobblin Standalone cluster +gobblin.cluster.workDir=${gobblin.cluster.work.dir}/GobblinStandaloneCluster + +# default is the JobConfigurationManager +# use this manager to accept jobs from Kafka. It requires some additional Kafka related parameters. +#gobblin.cluster.job.configuration.manager=org.apache.gobblin.cluster.StreamingJobConfigurationManager +#spec.kafka.topics=ruyang_test_kafka_gobblin +#kafka.brokers="hostname:12913/kafka-queuing" +#jobSpecMonitor.kafka.zookeeper.connect="hostname:12913/kafka-queuing" + +# Cluster configuration properties +gobblin.cluster.helix.cluster.name=GobblinStandaloneClusterCli + +# used by the JobConfigurationManager +gobblin.cluster.job.conf.path=${gobblin.cluster.work.dir}/jobs + +gobblin.cluster.jobconf.fullyQualifiedPath=${gobblin.cluster.work.dir}/jobs + +# File system URIs +writer.fs.uri=${fs.uri} +state.store.fs.uri=${fs.uri} + +# Writer related configuration properties +writer.destination.type=HDFS +writer.output.format=AVRO +writer.staging.dir=${gobblin.cluster.work.dir}/task-staging +writer.output.dir=${gobblin.cluster.work.dir}/task-output + +# Data publisher related configuration properties +data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher +data.publisher.final.dir=${gobblin.cluster.work.dir}/job-output +data.publisher.replace.final.dir=false + +# Directory where job/task state files are stored +state.store.dir=${gobblin.cluster.work.dir}/state-store + +# Directory where error files from the quality checkers are stored +qualitychecker.row.err.file=${gobblin.cluster.work.dir}/err + +# Disable job locking for now +job.lock.enabled=false + +# Directory where job locks are stored +job.lock.dir=${gobblin.cluster.work.dir}/locks + +# Directory where metrics log files are stored +metrics.log.dir=${gobblin.cluster.work.dir}/metrics + +# Interval of task state reporting in milliseconds +task.status.reportintervalinms=1000 + +# Enable metrics / events +metrics.enabled=true + http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/241a1b3b/conf/standalone/master/log4j-cluster.properties ---------------------------------------------------------------------- diff --git a/conf/standalone/master/log4j-cluster.properties b/conf/standalone/master/log4j-cluster.properties new file mode 100755 index 0000000..6effc68 --- /dev/null +++ b/conf/standalone/master/log4j-cluster.properties @@ -0,0 +1,28 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# log4j configuration used during build and unit tests + +log4j.rootLogger=info,stdout +log4j.threshhold=ALL +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss z} %-5p [%t] %C %X{tableName} - %m%n + +# Suppressed loggers +log4j.logger.org.apache.helix.controller.GenericHelixController=ERROR +log4j.logger.org.apache.helix.controller.stages=ERROR +log4j.logger.org.apache.helix.controller.strategy.AutoRebalanceStrategy=ERROR +log4j.logger.org.apache.helix.manager.zk=ERROR +log4j.logger.org.apache.helix.monitoring.mbeans.ClusterStatusMonitor=ERROR +log4j.logger.org.apache.helix.store.zk.AutoFallbackPropertyStore=ERROR + http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/241a1b3b/conf/standalone/master/quartz.properties ---------------------------------------------------------------------- diff --git a/conf/standalone/master/quartz.properties b/conf/standalone/master/quartz.properties new file mode 100644 index 0000000..71b2f25 --- /dev/null +++ b/conf/standalone/master/quartz.properties @@ -0,0 +1,3 @@ +org.quartz.scheduler.instanceName = LocalJobScheduler +org.quartz.threadPool.threadCount = 3 +org.quartz.jobStore.class = org.quartz.simpl.RAMJobStore http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/241a1b3b/conf/standalone/master/reference.conf ---------------------------------------------------------------------- diff --git a/conf/standalone/master/reference.conf b/conf/standalone/master/reference.conf new file mode 100644 index 0000000..936b8a1 --- /dev/null +++ b/conf/standalone/master/reference.conf @@ -0,0 +1,15 @@ +# Sample configuration properties with default values + +# Cluster configuration properties +gobblin.cluster.app.name=GobblinStandaloneCluster +gobblin.cluster.email.notification.on.shutdown=false +gobblin.cluster.helix.instance.max.retries=2 +gobblin.cluster.work.dir=/tmp/gobblin-cluster + +# Helix/Zookeeper configuration properties +gobblin.cluster.helix.cluster.name=GobblinStandaloneCluster +gobblin.cluster.zk.connection.string="localhost:2181" + +fs.uri="file:///" + +job.execinfo.server.enabled=false http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/241a1b3b/conf/standalone/worker/application.conf ---------------------------------------------------------------------- diff --git a/conf/standalone/worker/application.conf b/conf/standalone/worker/application.conf new file mode 100644 index 0000000..e9b8323 --- /dev/null +++ b/conf/standalone/worker/application.conf @@ -0,0 +1,71 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Sample configuration properties for the Gobblin Standalone cluster +gobblin.cluster.workDir=${gobblin.cluster.work.dir}/GobblinStandaloneCluster + +# default is the JobConfigurationManager +# use this manager to accept jobs from Kafka. It requires some additional Kafka related parameters. +#gobblin.cluster.job.configuration.manager=org.apache.gobblin.cluster.StreamingJobConfigurationManager +#spec.kafka.topics=ruyang_test_kafka_gobblin +#kafka.brokers="hostname:12913/kafka-queuing" +#jobSpecMonitor.kafka.zookeeper.connect="hostname:12913/kafka-queuing" + +# Cluster configuration properties +gobblin.cluster.helix.cluster.name=GobblinStandaloneClusterCli + +# used by the JobConfigurationManager +gobblin.cluster.job.conf.path=${gobblin.cluster.work.dir}/jobs + +gobblin.cluster.jobconf.fullyQualifiedPath=${gobblin.cluster.work.dir}/jobs + +# File system URIs +writer.fs.uri=${fs.uri} +state.store.fs.uri=${fs.uri} + +# Writer related configuration properties +writer.destination.type=HDFS +writer.output.format=AVRO +writer.staging.dir=${gobblin.cluster.work.dir}/task-staging +writer.output.dir=${gobblin.cluster.work.dir}/task-output + +# Data publisher related configuration properties +data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher +data.publisher.final.dir=${gobblin.cluster.work.dir}/job-output +data.publisher.replace.final.dir=false + +# Directory where job/task state files are stored +state.store.dir=${gobblin.cluster.work.dir}/state-store + +# Directory where error files from the quality checkers are stored +qualitychecker.row.err.file=${gobblin.cluster.work.dir}/err + +# Disable job locking for now +job.lock.enabled=false + +# Directory where job locks are stored +job.lock.dir=${gobblin.cluster.work.dir}/locks + +# Directory where metrics log files are stored +metrics.log.dir=${gobblin.cluster.work.dir}/metrics + +# Interval of task state reporting in milliseconds +task.status.reportintervalinms=1000 + +# Enable metrics / events +metrics.enabled=true + http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/241a1b3b/conf/standalone/worker/log4j-cluster.properties ---------------------------------------------------------------------- diff --git a/conf/standalone/worker/log4j-cluster.properties b/conf/standalone/worker/log4j-cluster.properties new file mode 100755 index 0000000..6effc68 --- /dev/null +++ b/conf/standalone/worker/log4j-cluster.properties @@ -0,0 +1,28 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# log4j configuration used during build and unit tests + +log4j.rootLogger=info,stdout +log4j.threshhold=ALL +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss z} %-5p [%t] %C %X{tableName} - %m%n + +# Suppressed loggers +log4j.logger.org.apache.helix.controller.GenericHelixController=ERROR +log4j.logger.org.apache.helix.controller.stages=ERROR +log4j.logger.org.apache.helix.controller.strategy.AutoRebalanceStrategy=ERROR +log4j.logger.org.apache.helix.manager.zk=ERROR +log4j.logger.org.apache.helix.monitoring.mbeans.ClusterStatusMonitor=ERROR +log4j.logger.org.apache.helix.store.zk.AutoFallbackPropertyStore=ERROR + http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/241a1b3b/conf/standalone/worker/quartz.properties ---------------------------------------------------------------------- diff --git a/conf/standalone/worker/quartz.properties b/conf/standalone/worker/quartz.properties new file mode 100644 index 0000000..71b2f25 --- /dev/null +++ b/conf/standalone/worker/quartz.properties @@ -0,0 +1,3 @@ +org.quartz.scheduler.instanceName = LocalJobScheduler +org.quartz.threadPool.threadCount = 3 +org.quartz.jobStore.class = org.quartz.simpl.RAMJobStore http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/241a1b3b/conf/standalone/worker/reference.conf ---------------------------------------------------------------------- diff --git a/conf/standalone/worker/reference.conf b/conf/standalone/worker/reference.conf new file mode 100644 index 0000000..936b8a1 --- /dev/null +++ b/conf/standalone/worker/reference.conf @@ -0,0 +1,15 @@ +# Sample configuration properties with default values + +# Cluster configuration properties +gobblin.cluster.app.name=GobblinStandaloneCluster +gobblin.cluster.email.notification.on.shutdown=false +gobblin.cluster.helix.instance.max.retries=2 +gobblin.cluster.work.dir=/tmp/gobblin-cluster + +# Helix/Zookeeper configuration properties +gobblin.cluster.helix.cluster.name=GobblinStandaloneCluster +gobblin.cluster.zk.connection.string="localhost:2181" + +fs.uri="file:///" + +job.execinfo.server.enabled=false http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/241a1b3b/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java ---------------------------------------------------------------------- diff --git a/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java b/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java index 72e06a0..c9b034c 100644 --- a/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java +++ b/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java @@ -111,4 +111,7 @@ public class ServiceConfigKeys { public static final String SERVICE_POLICY_NAME = GOBBLIN_SERVICE_PREFIX + "servicePolicy"; // Logging public static final String GOBBLIN_SERVICE_LOG4J_CONFIGURATION_FILE = "log4j-service.properties"; + // GAAS Listerning Port + public static final String SERVICE_PORT = GOBBLIN_SERVICE_PREFIX + "port"; + } http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/241a1b3b/gobblin-modules/gobblin-kafka-09/src/main/java/org/apache/gobblin/kafka/writer/Kafka09DataWriter.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-kafka-09/src/main/java/org/apache/gobblin/kafka/writer/Kafka09DataWriter.java b/gobblin-modules/gobblin-kafka-09/src/main/java/org/apache/gobblin/kafka/writer/Kafka09DataWriter.java index 89b637a..52e5df2 100644 --- a/gobblin-modules/gobblin-kafka-09/src/main/java/org/apache/gobblin/kafka/writer/Kafka09DataWriter.java +++ b/gobblin-modules/gobblin-kafka-09/src/main/java/org/apache/gobblin/kafka/writer/Kafka09DataWriter.java @@ -163,4 +163,4 @@ public class Kafka09DataWriter<D> implements AsyncDataWriter<D> { } log.info("Created Topic "+topicName+" with replication: "+replication+" and partitions :"+partitions); } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/241a1b3b/gobblin-modules/gobblin-kafka-09/src/test/java/org/apache/gobblin/kafka/KafkaClusterTestBase.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-kafka-09/src/test/java/org/apache/gobblin/kafka/KafkaClusterTestBase.java b/gobblin-modules/gobblin-kafka-09/src/test/java/org/apache/gobblin/kafka/KafkaClusterTestBase.java index 8cbe983..459aa64 100644 --- a/gobblin-modules/gobblin-kafka-09/src/test/java/org/apache/gobblin/kafka/KafkaClusterTestBase.java +++ b/gobblin-modules/gobblin-kafka-09/src/test/java/org/apache/gobblin/kafka/KafkaClusterTestBase.java @@ -125,4 +125,4 @@ public class KafkaClusterTestBase extends KafkaTestBase { bootServerString = bootServerString.substring(0,bootServerString.length()-1); return bootServerString; } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/241a1b3b/gobblin-modules/gobblin-kafka-09/src/test/java/org/apache/gobblin/kafka/writer/Kafka09TopicProvisionTest.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-kafka-09/src/test/java/org/apache/gobblin/kafka/writer/Kafka09TopicProvisionTest.java b/gobblin-modules/gobblin-kafka-09/src/test/java/org/apache/gobblin/kafka/writer/Kafka09TopicProvisionTest.java index 54593da..4765d1b 100644 --- a/gobblin-modules/gobblin-kafka-09/src/test/java/org/apache/gobblin/kafka/writer/Kafka09TopicProvisionTest.java +++ b/gobblin-modules/gobblin-kafka-09/src/test/java/org/apache/gobblin/kafka/writer/Kafka09TopicProvisionTest.java @@ -181,4 +181,4 @@ public class Kafka09TopicProvisionTest { } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/241a1b3b/gobblin-modules/gobblin-kafka-common/src/main/java/org/apache/gobblin/kafka/writer/KafkaWriterConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-kafka-common/src/main/java/org/apache/gobblin/kafka/writer/KafkaWriterConfigurationKeys.java b/gobblin-modules/gobblin-kafka-common/src/main/java/org/apache/gobblin/kafka/writer/KafkaWriterConfigurationKeys.java index 84255d4..d1522ab 100644 --- a/gobblin-modules/gobblin-kafka-common/src/main/java/org/apache/gobblin/kafka/writer/KafkaWriterConfigurationKeys.java +++ b/gobblin-modules/gobblin-kafka-common/src/main/java/org/apache/gobblin/kafka/writer/KafkaWriterConfigurationKeys.java @@ -62,4 +62,4 @@ public class KafkaWriterConfigurationKeys { static final int ZOOKEEPER_SESSION_TIMEOUT_DEFAULT = 10000; // 10 seconds public static final String ZOOKEEPER_CONNECTION_TIMEOUT = CLUSTER_ZOOKEEPER + ".cto"; static final int ZOOKEEPER_CONNECTION_TIMEOUT_DEFAULT = 8000; // 8 seconds -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/241a1b3b/gobblin-restli/gobblin-restli-utils/src/main/java/org/apache/gobblin/restli/EmbeddedRestliServer.java ---------------------------------------------------------------------- diff --git a/gobblin-restli/gobblin-restli-utils/src/main/java/org/apache/gobblin/restli/EmbeddedRestliServer.java b/gobblin-restli/gobblin-restli-utils/src/main/java/org/apache/gobblin/restli/EmbeddedRestliServer.java index ed9b21f..8d3452e 100644 --- a/gobblin-restli/gobblin-restli-utils/src/main/java/org/apache/gobblin/restli/EmbeddedRestliServer.java +++ b/gobblin-restli/gobblin-restli-utils/src/main/java/org/apache/gobblin/restli/EmbeddedRestliServer.java @@ -52,6 +52,7 @@ import com.linkedin.restli.server.validation.RestLiValidationFilter; import lombok.Builder; import lombok.Getter; +import lombok.Setter; /** @@ -76,8 +77,8 @@ public class EmbeddedRestliServer extends AbstractIdleService { @Getter private final URI serverUri; - @Getter - private final int port; + @Getter @Setter + private int port; @Getter private final Injector injector; private final Logger log; http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/241a1b3b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java ---------------------------------------------------------------------- diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java index 0daea79..9626bcb 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java @@ -233,6 +233,8 @@ public class GobblinServiceManager implements ApplicationLauncher, StandardMetri .resources(Lists.<Class<? extends BaseResource>>newArrayList(FlowConfigsResource.class)) .injector(injector) .build(); + this.restliServer.setPort(ConfigUtils.getInt(config, + ServiceConfigKeys.SERVICE_PORT, new Integer(0))); this.serviceLauncher.addService(restliServer); }
