http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-example/src/main/resources/streaming-kafka-console.pull ---------------------------------------------------------------------- diff --git a/gobblin-example/src/main/resources/streaming-kafka-console.pull b/gobblin-example/src/main/resources/streaming-kafka-console.pull index 6be541e..4cd03fa 100644 --- a/gobblin-example/src/main/resources/streaming-kafka-console.pull +++ b/gobblin-example/src/main/resources/streaming-kafka-console.pull @@ -33,11 +33,11 @@ task.executionMode=STREAMING gobblin.streaming.kafka.topic.key.deserializer=org.apache.kafka.common.serialization.StringDeserializer gobblin.streaming.kafka.topic.value.deserializer=org.apache.kafka.common.serialization.ByteArrayDeserializer -source.class=gobblin.source.extractor.extract.kafka.KafkaSimpleStreamingSource +source.class=org.apache.gobblin.source.extractor.extract.kafka.KafkaSimpleStreamingSource gobblin.streaming.kafka.topic.singleton=test kafka.brokers=localhost:9092 -writer.builder.class=gobblin.writer.ConsoleWriterBuilder +writer.builder.class=org.apache.gobblin.writer.ConsoleWriterBuilder -data.publisher.type=gobblin.publisher.NoopPublisher +data.publisher.type=org.apache.gobblin.publisher.NoopPublisher
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-example/src/main/resources/streaming-kafka-kafka.pull ---------------------------------------------------------------------- diff --git a/gobblin-example/src/main/resources/streaming-kafka-kafka.pull b/gobblin-example/src/main/resources/streaming-kafka-kafka.pull index 0fd31e1..1b2cbe6 100644 --- a/gobblin-example/src/main/resources/streaming-kafka-kafka.pull +++ b/gobblin-example/src/main/resources/streaming-kafka-kafka.pull @@ -34,18 +34,18 @@ task.executionMode=STREAMING gobblin.streaming.kafka.topic.key.deserializer=org.apache.kafka.common.serialization.StringDeserializer gobblin.streaming.kafka.topic.value.deserializer=org.apache.kafka.common.serialization.ByteArrayDeserializer -source.class=gobblin.source.extractor.extract.kafka.KafkaSimpleStreamingSource +source.class=org.apache.gobblin.source.extractor.extract.kafka.KafkaSimpleStreamingSource gobblin.streaming.kafka.topic.singleton=test kafka.brokers=localhost:9092 # Sample 10% of the records -converter.classes=gobblin.converter.SamplingConverter +converter.classes=org.apache.gobblin.converter.SamplingConverter converter.sample.ratio=0.10 -writer.builder.class=gobblin.kafka.writer.KafkaDataWriterBuilder +writer.builder.class=org.apache.gobblin.kafka.writer.KafkaDataWriterBuilder writer.kafka.topic=test_copied writer.kafka.producerConfig.bootstrap.servers=localhost:9092 -writer.kafka.producerConfig.value.serializer=org.apache.kafka.common.serialization.ByteArraySerializer +writer.kafka.producerConfig.value.serializer=org.apache.kafka.common.serialization.ByteArraySerializer -data.publisher.type=gobblin.publisher.NoopPublisher +data.publisher.type=org.apache.gobblin.publisher.NoopPublisher http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-example/src/main/resources/streaming-test-kafka.pull ---------------------------------------------------------------------- diff --git a/gobblin-example/src/main/resources/streaming-test-kafka.pull b/gobblin-example/src/main/resources/streaming-test-kafka.pull index 360d9e8..3db4db6 100644 --- a/gobblin-example/src/main/resources/streaming-test-kafka.pull +++ b/gobblin-example/src/main/resources/streaming-test-kafka.pull @@ -23,13 +23,13 @@ job.description=A getting started example for Gobblin streaming to Kafka job.lock.enabled=false task.executionMode=STREAMING -source.class=gobblin.test.SequentialTestSource +source.class=org.apache.gobblin.test.SequentialTestSource source.numParallelism=2 source.sleepTimePerRecordMillis=1000 -extract.namespace=gobblin.streaming.test +extract.namespace=org.apache.gobblin.streaming.test -#converter.classes=gobblin.test.AnyToJsonConverter +#converter.classes=org.apache.gobblin.test.AnyToJsonConverter # Configure watermark storage for streaming #streaming.watermarkStateStore.type=zk @@ -40,10 +40,10 @@ extract.namespace=gobblin.streaming.test #streaming.watermark.commitIntervalMillis=2000 #No partitioner as kafka writer doesn't work with a partitioner -writer.builder.class=gobblin.writer.ConsoleWriterBuilder -#writer.builder.class=gobblin.kafka.writer.KafkaDataWriterBuilder +writer.builder.class=org.apache.gobblin.writer.ConsoleWriterBuilder +#writer.builder.class=org.apache.gobblin.kafka.writer.KafkaDataWriterBuilder #writer.kafka.topic=StreamingTestTopic #writer.kafka.producerConfig.bootstrap.servers=localhost:9092 #writer.kafka.producerConfig.value.serializer=org.apache.kafka.common.serialization.StringSerializer -data.publisher.type=gobblin.publisher.NoopPublisher +data.publisher.type=org.apache.gobblin.publisher.NoopPublisher http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-example/src/main/resources/wikipedia-console.pull ---------------------------------------------------------------------- diff --git a/gobblin-example/src/main/resources/wikipedia-console.pull b/gobblin-example/src/main/resources/wikipedia-console.pull index 104a157..15c1e35 100644 --- a/gobblin-example/src/main/resources/wikipedia-console.pull +++ b/gobblin-example/src/main/resources/wikipedia-console.pull @@ -19,18 +19,18 @@ job.name=PullFromWikipedia job.group=Wikipedia job.description=A getting started example for Gobblin -source.class=gobblin.example.wikipedia.WikipediaSource +source.class=org.apache.gobblin.example.wikipedia.WikipediaSource source.page.titles=NASA,LinkedIn,Parris_Cues,Barbara_Corcoran source.revisions.cnt=5 wikipedia.api.rooturl=https://en.wikipedia.org/w/api.php wikipedia.avro.schema={"namespace": "example.wikipedia.avro","type": "record","name": "WikipediaArticle","fields": [{"name": "revid", "type": ["double", "null"]},{"name": "pageid", "type": ["double", "null"]},{"name": "title", "type": ["string", "null"]},{"name": "user", "type": ["string", "null"]},{"name": "anon", "type": ["string", "null"]},{"name": "userid", "type": ["double", "null"]},{"name": "timestamp", "type": ["string", "null"]},{"name": "size", "type": ["double", "null"]},{"name": "contentformat", "type": ["string", "null"]},{"name": "contentmodel", "type": ["string", "null"]},{"name": "content", "type": ["string", "null"]}]} -converter.classes=gobblin.example.wikipedia.WikipediaConverter +converter.classes=org.apache.gobblin.example.wikipedia.WikipediaConverter -extract.namespace=gobblin.example.wikipedia +extract.namespace=org.apache.gobblin.example.wikipedia #No partitioner as console writer doesn't work with a partitioner -writer.builder.class=gobblin.writer.ConsoleWriterBuilder +writer.builder.class=org.apache.gobblin.writer.ConsoleWriterBuilder -data.publisher.type=gobblin.publisher.NoopPublisher +data.publisher.type=org.apache.gobblin.publisher.NoopPublisher http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-example/src/main/resources/wikipedia-kafka.pull ---------------------------------------------------------------------- diff --git a/gobblin-example/src/main/resources/wikipedia-kafka.pull b/gobblin-example/src/main/resources/wikipedia-kafka.pull index f5aa98d..69c00b0 100644 --- a/gobblin-example/src/main/resources/wikipedia-kafka.pull +++ b/gobblin-example/src/main/resources/wikipedia-kafka.pull @@ -19,18 +19,18 @@ job.name=PullFromWikipediaToKafka job.group=Wikipedia job.description=Pull from Wikipedia and write to Kafka -source.class=gobblin.example.wikipedia.WikipediaSource +source.class=org.apache.gobblin.example.wikipedia.WikipediaSource source.page.titles=Wikipedia:Sandbox source.revisions.cnt=5 wikipedia.api.rooturl=https://en.wikipedia.org/w/api.php wikipedia.avro.schema={"namespace": "example.wikipedia.avro","type": "record","name": "WikipediaArticle","fields": [{"name": "revid", "type": ["double", "null"]},{"name": "pageid", "type": ["double", "null"]},{"name": "title", "type": ["string", "null"]},{"name": "user", "type": ["string", "null"]},{"name": "anon", "type": ["string", "null"]},{"name": "userid", "type": ["double", "null"]},{"name": "timestamp", "type": ["string", "null"]},{"name": "size", "type": ["double", "null"]},{"name": "contentformat", "type": ["string", "null"]},{"name": "contentmodel", "type": ["string", "null"]},{"name": "content", "type": ["string", "null"]}]} -converter.classes=gobblin.example.wikipedia.WikipediaConverter +converter.classes=org.apache.gobblin.example.wikipedia.WikipediaConverter -extract.namespace=gobblin.example.wikipedia +extract.namespace=org.apache.gobblin.example.wikipedia -writer.builder.class=gobblin.kafka.writer.KafkaDataWriterBuilder +writer.builder.class=org.apache.gobblin.kafka.writer.KafkaDataWriterBuilder writer.kafka.topic=WikipediaExample writer.kafka.producerConfig.bootstrap.servers=localhost:9092 @@ -40,10 +40,10 @@ writer.kafka.producerConfig.bootstrap.servers=localhost:9092 #writer.kafka.producerConfig.schema.registry.url=http://localhost:8081 #Use Local Schema Registry and serializers -writer.kafka.producerConfig.value.serializer=gobblin.kafka.serialize.LiAvroSerializer -writer.kafka.producerConfig.kafka.schemaRegistry.class=gobblin.kafka.schemareg.ConfigDrivenMd5SchemaRegistry +writer.kafka.producerConfig.value.serializer=org.apache.gobblin.kafka.serialize.LiAvroSerializer +writer.kafka.producerConfig.kafka.schemaRegistry.class=org.apache.gobblin.kafka.schemareg.ConfigDrivenMd5SchemaRegistry writer.kafka.producerConfig.schemaRegistry.schema.name=WikipediaExample writer.kafka.producerConfig.schemaRegistry.schema.value={"namespace": "example.wikipedia.avro","type": "record","name": "WikipediaArticle","fields": [{"name": "pageid", "type": ["double", "null"]},{"name": "title", "type": ["string", "null"]},{"name": "user", "type": ["string", "null"]},{"name": "anon", "type": ["string", "null"]},{"name": "userid", "type": ["double", "null"]},{"name": "timestamp", "type": ["string", "null"]},{"name": "size", "type": ["double", "null"]},{"name": "contentformat", "type": ["string", "null"]},{"name": "contentmodel", "type": ["string", "null"]},{"name": "content", "type": ["string", "null"]}]} -data.publisher.type=gobblin.publisher.NoopPublisher +data.publisher.type=org.apache.gobblin.publisher.NoopPublisher http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-example/src/main/resources/wikipedia-orc.pull ---------------------------------------------------------------------- diff --git a/gobblin-example/src/main/resources/wikipedia-orc.pull b/gobblin-example/src/main/resources/wikipedia-orc.pull index a8e2420..a6bad65 100644 --- a/gobblin-example/src/main/resources/wikipedia-orc.pull +++ b/gobblin-example/src/main/resources/wikipedia-orc.pull @@ -23,7 +23,7 @@ source.class=org.apache.gobblin.example.wikipedia.WikipediaSource source.page.titles=NASA,LinkedIn,Parris_Cues,Barbara_Corcoran source.revisions.cnt=5 -extract.namespace=gobblin.example.wikipedia +extract.namespace=org.apache.gobblin.example.wikipedia wikipedia.api.rooturl=https://en.wikipedia.org/w/api.php wikipedia.avro.schema={"namespace": "example.wikipedia.avro","type": "record","name": "WikipediaArticle","fields": [{"name": "revid", "type": ["double", "null"]},{"name": "pageid", "type": ["double", "null"]},{"name": "title", "type": ["string", "null"]},{"name": "user", "type": ["string", "null"]},{"name": "anon", "type": ["string", "null"]},{"name": "userid", "type": ["double", "null"]},{"name": "timestamp", "type": ["string", "null"]},{"name": "size", "type": ["double", "null"]},{"name": "contentformat", "type": ["string", "null"]},{"name": "contentmodel", "type": ["string", "null"]},{"name": "content", "type": ["string", "null"]}]} http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-example/src/main/resources/wikipedia.pull ---------------------------------------------------------------------- diff --git a/gobblin-example/src/main/resources/wikipedia.pull b/gobblin-example/src/main/resources/wikipedia.pull index b785501..68780a4 100644 --- a/gobblin-example/src/main/resources/wikipedia.pull +++ b/gobblin-example/src/main/resources/wikipedia.pull @@ -19,7 +19,7 @@ job.name=PullFromWikipedia job.group=Wikipedia job.description=A getting started example for Gobblin -source.class=gobblin.example.wikipedia.WikipediaSource +source.class=org.apache.gobblin.example.wikipedia.WikipediaSource source.page.titles=LinkedIn,Wikipedia:Sandbox source.revisions.cnt=5 @@ -27,12 +27,12 @@ wikipedia.api.rooturl=https://en.wikipedia.org/w/api.php wikipedia.avro.schema={"namespace": "example.wikipedia.avro","type": "record","name": "WikipediaArticle","fields": [{"name": "revid", "type": ["double", "null"]},{"name": "pageid", "type": ["double", "null"]},{"name": "title", "type": ["string", "null"]},{"name": "user", "type": ["string", "null"]},{"name": "anon", "type": ["string", "null"]},{"name": "userid", "type": ["double", "null"]},{"name": "timestamp", "type": ["string", "null"]},{"name": "size", "type": ["double", "null"]},{"name": "contentformat", "type": ["string", "null"]},{"name": "contentmodel", "type": ["string", "null"]},{"name": "content", "type": ["string", "null"]}]} gobblin.wikipediaSource.maxRevisionsPerPage=10 -converter.classes=gobblin.example.wikipedia.WikipediaConverter +converter.classes=org.apache.gobblin.example.wikipedia.WikipediaConverter -extract.namespace=gobblin.example.wikipedia +extract.namespace=org.apache.gobblin.example.wikipedia writer.destination.type=HDFS writer.output.format=AVRO -writer.partitioner.class=gobblin.example.wikipedia.WikipediaPartitioner +writer.partitioner.class=org.apache.gobblin.example.wikipedia.WikipediaPartitioner -data.publisher.type=gobblin.publisher.BaseDataPublisher +data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-example/src/main/resources/wikipedia.template ---------------------------------------------------------------------- diff --git a/gobblin-example/src/main/resources/wikipedia.template b/gobblin-example/src/main/resources/wikipedia.template index 2a494a9..b66310d 100644 --- a/gobblin-example/src/main/resources/wikipedia.template +++ b/gobblin-example/src/main/resources/wikipedia.template @@ -21,16 +21,16 @@ job.name=PullFromWikipedia job.group=Wikipedia job.description=A getting started example for Gobblin -source.class=gobblin.example.wikipedia.WikipediaSource +source.class=org.apache.gobblin.example.wikipedia.WikipediaSource source.page.titles=${titles} wikipedia.api.rooturl="https://en.wikipedia.org/w/api.php"; wikipedia.avro.schema="{\"namespace\": \"example.wikipedia.avro\",\"type\": \"record\",\"name\": \"WikipediaArticle\",\"fields\": [{\"name\": \"revid\", \"type\": [\"double\", \"null\"]},{\"name\": \"pageid\", \"type\": [\"double\", \"null\"]},{\"name\": \"title\", \"type\": [\"string\", \"null\"]},{\"name\": \"user\", \"type\": [\"string\", \"null\"]},{\"name\": \"anon\", \"type\": [\"string\", \"null\"]},{\"name\": \"userid\", \"type\": [\"double\", \"null\"]},{\"name\": \"timestamp\", \"type\": [\"string\", \"null\"]},{\"name\": \"size\", \"type\": [\"double\", \"null\"]},{\"name\": \"contentformat\", \"type\": [\"string\", \"null\"]},{\"name\": \"contentmodel\", \"type\": [\"string\", \"null\"]},{\"name\": \"content\", \"type\": [\"string\", \"null\"]}]}" gobblin.wikipediaSource.maxRevisionsPerPage=10 -extract.namespace=gobblin.example.wikipedia +extract.namespace=org.apache.gobblin.example.wikipedia #No partitioner as console writer doesn't work with a partitioner -writer.builder.class=gobblin.writer.ConsoleWriterBuilder +writer.builder.class=org.apache.gobblin.writer.ConsoleWriterBuilder -data.publisher.type=gobblin.publisher.NoopPublisher +data.publisher.type=org.apache.gobblin.publisher.NoopPublisher http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/ConsoleEventReporterFactory.java ---------------------------------------------------------------------- diff --git a/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/ConsoleEventReporterFactory.java b/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/ConsoleEventReporterFactory.java index dfc7eed..3eaecb8 100644 --- a/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/ConsoleEventReporterFactory.java +++ b/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/ConsoleEventReporterFactory.java @@ -29,7 +29,7 @@ import org.apache.gobblin.metrics.reporter.OutputStreamEventReporter; * A reporter factory to report event to console. * * <p> - * Set metrics.reporting.custom.builders=gobblin.metrics.ConsoleEventReporterFactory to report event to console + * Set metrics.reporting.custom.builders=org.apache.gobblin.metrics.ConsoleEventReporterFactory to report event to console * </p> */ public class ConsoleEventReporterFactory implements CustomCodahaleReporterFactory { http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/ConsoleReporterFactory.java ---------------------------------------------------------------------- diff --git a/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/ConsoleReporterFactory.java b/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/ConsoleReporterFactory.java index 6a86d98..42dbf55 100644 --- a/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/ConsoleReporterFactory.java +++ b/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/ConsoleReporterFactory.java @@ -27,7 +27,7 @@ import org.apache.gobblin.metrics.reporter.ScheduledReporter; * A reporter factory to report metrics to console. * * <p> - * Set metrics.reporting.custom.builders=gobblin.metrics.ConsoleReporterFactory to report event to console + * Set metrics.reporting.custom.builders=org.apache.gobblin.metrics.ConsoleReporterFactory to report event to console * </p> */ public class ConsoleReporterFactory implements CustomReporterFactory { http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-modules/gobblin-kafka-08/resource/job-props/testKafkaIngest.pull ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-kafka-08/resource/job-props/testKafkaIngest.pull b/gobblin-modules/gobblin-kafka-08/resource/job-props/testKafkaIngest.pull index b55efd9..7e5e433 100755 --- a/gobblin-modules/gobblin-kafka-08/resource/job-props/testKafkaIngest.pull +++ b/gobblin-modules/gobblin-kafka-08/resource/job-props/testKafkaIngest.pull @@ -19,23 +19,23 @@ job.name=GobblinKafkaWriterTest1 job.group=Test job.description=Test Gobblin job for kafka writer -extract.namespace=gobblin.kafka.writer.test1 +extract.namespace=org.apache.gobblin.kafka.writer.test1 -source.class=gobblin.test.SequentialTestSource +source.class=org.apache.gobblin.test.SequentialTestSource source.numRecordsPerExtract=5 -converter.classes=gobblin.test.AnyToStringConverter +converter.classes=org.apache.gobblin.test.AnyToStringConverter -writer.builder.class=gobblin.kafka.writer.KafkaDataWriterBuilder +writer.builder.class=org.apache.gobblin.kafka.writer.KafkaDataWriterBuilder writer.kafka.topic={$topic} -writer.kafka.producerClass=gobblin.kafka.FlakyKafkaProducer +writer.kafka.producerClass=org.apache.gobblin.kafka.FlakyKafkaProducer #writer.kafka.producerConfig.flaky.errorType=SET_BY_TEST #writer.kafka.producerConfig.flaky.errorEvery=SET_BY_TEST writer.kafka.producerConfig.bootstrap.servers=localhost:{$kafkaPort} writer.kafka.producerConfig.value.serializer=org.apache.kafka.common.serialization.StringSerializer writer.kafka.producerConfig.key.serializer=org.apache.kafka.common.serialization.StringSerializer -data.publisher.type=gobblin.publisher.NoopPublisher +data.publisher.type=org.apache.gobblin.publisher.NoopPublisher task.maxretries=0 workunit.retry.enabled=false http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-oozie/src/test/resources/local/gobblin-oozie-example-system.properties ---------------------------------------------------------------------- diff --git a/gobblin-oozie/src/test/resources/local/gobblin-oozie-example-system.properties b/gobblin-oozie/src/test/resources/local/gobblin-oozie-example-system.properties index 5982258..1dbb6cb 100644 --- a/gobblin-oozie/src/test/resources/local/gobblin-oozie-example-system.properties +++ b/gobblin-oozie/src/test/resources/local/gobblin-oozie-example-system.properties @@ -37,7 +37,7 @@ writer.staging.dir=${gobblin.work.dir}/task-staging writer.output.dir=${gobblin.work.dir}/task-output # Data publisher related configuration properties -data.publisher.type=gobblin.publisher.BaseDataPublisher +data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher data.publisher.final.dir=${gobblin.work.dir}/job-output data.publisher.replace.final.dir=false http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-oozie/src/test/resources/mapreduce/gobblin-oozie-mr-example-sysconfig.properties ---------------------------------------------------------------------- diff --git a/gobblin-oozie/src/test/resources/mapreduce/gobblin-oozie-mr-example-sysconfig.properties b/gobblin-oozie/src/test/resources/mapreduce/gobblin-oozie-mr-example-sysconfig.properties index a83e878..fda1412 100644 --- a/gobblin-oozie/src/test/resources/mapreduce/gobblin-oozie-mr-example-sysconfig.properties +++ b/gobblin-oozie/src/test/resources/mapreduce/gobblin-oozie-mr-example-sysconfig.properties @@ -37,7 +37,7 @@ writer.staging.dir=${gobblin.work.dir}/task-staging writer.output.dir=${gobblin.work.dir}/task-output # Data publisher related configuration properties -data.publisher.type=gobblin.publisher.BaseDataPublisher +data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher data.publisher.final.dir=${gobblin.work.dir}/job-output data.publisher.replace.final.dir=false http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-runtime/src/main/resources/templates/distcp.template ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/main/resources/templates/distcp.template b/gobblin-runtime/src/main/resources/templates/distcp.template index a8707d9..a210fe7 100644 --- a/gobblin-runtime/src/main/resources/templates/distcp.template +++ b/gobblin-runtime/src/main/resources/templates/distcp.template @@ -28,17 +28,17 @@ job.description="Distributed copy" data.publisher.final.dir=${to} gobblin.dataset.pattern=${from} -gobblin.dataset.profile.class=gobblin.data.management.copy.CopyableGlobDatasetFinder +gobblin.dataset.profile.class=org.apache.gobblin.data.management.copy.CopyableGlobDatasetFinder # ==================================================================== # Distcp configurations # ==================================================================== -extract.namespace=gobblin.copy -data.publisher.type=gobblin.data.management.copy.publisher.CopyDataPublisher -source.class=gobblin.data.management.copy.CopySource -writer.builder.class=gobblin.data.management.copy.writer.FileAwareInputStreamDataWriterBuilder -converter.classes=gobblin.converter.IdentityConverter +extract.namespace=org.apache.gobblin.copy +data.publisher.type=org.apache.gobblin.data.management.copy.publisher.CopyDataPublisher +source.class=org.apache.gobblin.data.management.copy.CopySource +writer.builder.class=org.apache.gobblin.data.management.copy.writer.FileAwareInputStreamDataWriterBuilder +converter.classes=org.apache.gobblin.converter.IdentityConverter task.maxretries=0 workunit.retry.enabled=false http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-runtime/src/main/resources/templates/gobblin-kafka.template ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/main/resources/templates/gobblin-kafka.template b/gobblin-runtime/src/main/resources/templates/gobblin-kafka.template index cc78665..5f5d91b 100644 --- a/gobblin-runtime/src/main/resources/templates/gobblin-kafka.template +++ b/gobblin-runtime/src/main/resources/templates/gobblin-kafka.template @@ -27,13 +27,13 @@ job.lock.enabled=true kafka.brokers="localhost:9092" -source.class=gobblin.source.extractor.extract.kafka.UniversalKafkaSource +source.class=org.apache.gobblin.source.extractor.extract.kafka.UniversalKafkaSource gobblin.source.kafka.extractorType=DESERIALIZER kafka.deserializer.type=BYTE_ARRAY -extract.namespace=gobblin.extract.kafka +extract.namespace=org.apache.gobblin.extract.kafka -writer.builder.class=gobblin.writer.SimpleDataWriterBuilder +writer.builder.class=org.apache.gobblin.writer.SimpleDataWriterBuilder simple.writer.delimiter="\n" simple.writer.prepend.size=false @@ -41,7 +41,7 @@ writer.file.path.type=tablename writer.destination.type=HDFS writer.output.format=txt -data.publisher.type=gobblin.publisher.TimePartitionedDataPublisher +data.publisher.type=org.apache.gobblin.publisher.TimePartitionedDataPublisher bootstrap.with.offset=latest http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-runtime/src/main/resources/templates/hello-world.template ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/main/resources/templates/hello-world.template b/gobblin-runtime/src/main/resources/templates/hello-world.template index 47e4b43..0572d7e 100644 --- a/gobblin-runtime/src/main/resources/templates/hello-world.template +++ b/gobblin-runtime/src/main/resources/templates/hello-world.template @@ -29,9 +29,9 @@ job.group=GobblinSamples job.description=The "Hello World" Gobblin job # Source, Converters, Writer, Publisher -source.class=gobblin.util.test.HelloWorldSource +source.class=org.apache.gobblin.util.test.HelloWorldSource writer.builder.class="org.apache.gobblin.util.test.StdoutWriter$Builder" -data.publisher.type=gobblin.publisher.NoopPublisher +data.publisher.type=org.apache.gobblin.publisher.NoopPublisher # Work paths http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-runtime/src/main/resources/templates/kafka-to-kafka.template ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/main/resources/templates/kafka-to-kafka.template b/gobblin-runtime/src/main/resources/templates/kafka-to-kafka.template index 3d69e03..34aa7e5 100644 --- a/gobblin-runtime/src/main/resources/templates/kafka-to-kafka.template +++ b/gobblin-runtime/src/main/resources/templates/kafka-to-kafka.template @@ -29,23 +29,23 @@ job.lock.enabled=true kafka.brokers=${inputBrokers} -source.class=gobblin.source.extractor.extract.kafka.UniversalKafkaSource +source.class=org.apache.gobblin.source.extractor.extract.kafka.UniversalKafkaSource gobblin.source.kafka.extractorType=DESERIALIZER kafka.deserializer.type=BYTE_ARRAY -extract.namespace=gobblin.extract.kafka +extract.namespace=org.apache.gobblin.extract.kafka bootstrap.with.offset=latest topic.whitelist=${inputTopics} -writer.builder.class=gobblin.kafka.writer.KafkaDataWriterBuilder +writer.builder.class=org.apache.gobblin.kafka.writer.KafkaDataWriterBuilder writer.kafka.topic=${outputTopic} writer.kafka.producerConfig.bootstrap.servers=${outputBrokers} writer.kafka.producerConfig.value.serializer=org.apache.kafka.common.serialization.ByteArraySerializer writer.output.format=TEXT -data.publisher.type=gobblin.publisher.NoopPublisher +data.publisher.type=org.apache.gobblin.publisher.NoopPublisher inputBrokers="localhost:9092" outputBrokers="localhost:9092" http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/TaskContextTest.java ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/TaskContextTest.java b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/TaskContextTest.java index 5dee390..41abaed 100644 --- a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/TaskContextTest.java +++ b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/TaskContextTest.java @@ -45,7 +45,7 @@ public class TaskContextTest { "job.group=Test\n" + "job.description=Test Gobblin job 1\n" + "job.schedule=0 0/1 * * * ?\n" + - "source.class=gobblin.test.TestSource\n" + + "source.class=org.apache.gobblin.test.TestSource\n" + "workunit.namespace=test\n" + "workunit.table=test\n" + "writer.destination.type=HDFS\n" + http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-runtime/src/test/resources/brokerTest/SimpleHelloWorldJob.jobconf ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/test/resources/brokerTest/SimpleHelloWorldJob.jobconf b/gobblin-runtime/src/test/resources/brokerTest/SimpleHelloWorldJob.jobconf index 373d5ff..4451bc8 100644 --- a/gobblin-runtime/src/test/resources/brokerTest/SimpleHelloWorldJob.jobconf +++ b/gobblin-runtime/src/test/resources/brokerTest/SimpleHelloWorldJob.jobconf @@ -27,11 +27,11 @@ job.group=GobblinSamples job.description=The "Hello World" Gobblin job # Source, Converters, Writer, Publisher -source.class=gobblin.util.test.HelloWorldSource +source.class=org.apache.gobblin.util.test.HelloWorldSource converter.classes="org.apache.gobblin.runtime.JobBrokerInjectionTest$JobBrokerConverter" writer.builder.class="org.apache.gobblin.writer.test.GobblinTestEventBusWriter$Builder" writer.GobblinTestEventBusWriter.eventBusId=${gobblin.workDir} -data.publisher.type=gobblin.publisher.NoopPublisher +data.publisher.type=org.apache.gobblin.publisher.NoopPublisher # Work paths http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-runtime/src/test/resources/gobblin/runtime/instance/SimpleHelloWorldJob.jobconf ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/test/resources/gobblin/runtime/instance/SimpleHelloWorldJob.jobconf b/gobblin-runtime/src/test/resources/gobblin/runtime/instance/SimpleHelloWorldJob.jobconf index 099960f..86335d7 100644 --- a/gobblin-runtime/src/test/resources/gobblin/runtime/instance/SimpleHelloWorldJob.jobconf +++ b/gobblin-runtime/src/test/resources/gobblin/runtime/instance/SimpleHelloWorldJob.jobconf @@ -30,10 +30,10 @@ job.group=GobblinSamples job.description=The "Hello World" Gobblin job # Source, Converters, Writer, Publisher -source.class=gobblin.util.test.HelloWorldSource +source.class=org.apache.gobblin.util.test.HelloWorldSource writer.builder.class="org.apache.gobblin.writer.test.GobblinTestEventBusWriter$Builder" writer.GobblinTestEventBusWriter.eventBusId=${gobblin.workDir} -data.publisher.type=gobblin.publisher.NoopPublisher +data.publisher.type=org.apache.gobblin.publisher.NoopPublisher # Work paths http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-runtime/src/test/resources/gobblin/runtime/instance/SimpleHelloWorldJob.template ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/test/resources/gobblin/runtime/instance/SimpleHelloWorldJob.template b/gobblin-runtime/src/test/resources/gobblin/runtime/instance/SimpleHelloWorldJob.template index 8a62da6..2cd12b0 100644 --- a/gobblin-runtime/src/test/resources/gobblin/runtime/instance/SimpleHelloWorldJob.template +++ b/gobblin-runtime/src/test/resources/gobblin/runtime/instance/SimpleHelloWorldJob.template @@ -32,10 +32,10 @@ job.group=GobblinSamples job.description=The "Hello World" Gobblin job # Source, Converters, Writer, Publisher -source.class=gobblin.util.test.HelloWorldSource +source.class=org.apache.gobblin.util.test.HelloWorldSource writer.builder.class="org.apache.gobblin.writer.test.GobblinTestEventBusWriter$Builder" writer.GobblinTestEventBusWriter.eventBusId=${gobblin.workDir} -data.publisher.type=gobblin.publisher.NoopPublisher +data.publisher.type=org.apache.gobblin.publisher.NoopPublisher # Work paths http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-runtime/src/test/resources/mr-job-conf/GobblinMRTest.pull ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/test/resources/mr-job-conf/GobblinMRTest.pull b/gobblin-runtime/src/test/resources/mr-job-conf/GobblinMRTest.pull index 9d68989..150f956 100644 --- a/gobblin-runtime/src/test/resources/mr-job-conf/GobblinMRTest.pull +++ b/gobblin-runtime/src/test/resources/mr-job-conf/GobblinMRTest.pull @@ -20,9 +20,9 @@ job.group=Test job.description=Test Gobblin job on Hadoop MR job.schedule=0 0/2 * * * ? -source.class=gobblin.test.TestSource -converter.classes=gobblin.test.TestConverter -extract.namespace=gobblin.MRTest +source.class=org.apache.gobblin.test.TestSource +converter.classes=org.apache.gobblin.test.TestConverter +extract.namespace=org.apache.gobblin.MRTest writer.destination.type=HDFS writer.output.format=AVRO http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-runtime/src/test/resources/templates/distcp-ng-hive.template ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/test/resources/templates/distcp-ng-hive.template b/gobblin-runtime/src/test/resources/templates/distcp-ng-hive.template index 2ca6f3d..da02dbb 100644 --- a/gobblin-runtime/src/test/resources/templates/distcp-ng-hive.template +++ b/gobblin-runtime/src/test/resources/templates/distcp-ng-hive.template @@ -16,11 +16,11 @@ # type=hadoopJava -job.class=gobblin.azkaban.AzkabanJobLauncher +job.class=org.apache.gobblin.azkaban.AzkabanJobLauncher -extract.namespace=gobblin.copy -data.publisher.type=gobblin.data.management.copy.publisher.CopyDataPublisher -source.class=gobblin.data.management.copy.CopySource +extract.namespace=org.apache.gobblin.copy +data.publisher.type=org.apache.gobblin.data.management.copy.publisher.CopyDataPublisher +source.class=org.apache.gobblin.data.management.copy.CopySource task.maxretries=0 workunit.retry.enabled=false http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-runtime/src/test/resources/templates/distcp-ng.template ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/test/resources/templates/distcp-ng.template b/gobblin-runtime/src/test/resources/templates/distcp-ng.template index f04c3e0..aead43e 100644 --- a/gobblin-runtime/src/test/resources/templates/distcp-ng.template +++ b/gobblin-runtime/src/test/resources/templates/distcp-ng.template @@ -18,14 +18,14 @@ job.name=GobblinDatabaseCopyTest job.description=Gobblin job for copy type=hadoopJava -job.class=gobblin.azkaban.AzkabanJobLauncher +job.class=org.apache.gobblin.azkaban.AzkabanJobLauncher -extract.namespace=gobblin.copy -data.publisher.type=gobblin.data.management.copy.publisher.CopyDataPublisher -source.class=gobblin.data.management.copy.CopySource -writer.builder.class=gobblin.data.management.copy.writer.FileAwareInputStreamDataWriterBuilder -converter.classes=gobblin.converter.IdentityConverter -gobblin.dataset.profile.class=gobblin.data.management.copy.hive.HiveDatasetFinder +extract.namespace=org.apache.gobblin.copy +data.publisher.type=org.apache.gobblin.data.management.copy.publisher.CopyDataPublisher +source.class=org.apache.gobblin.data.management.copy.CopySource +writer.builder.class=org.apache.gobblin.data.management.copy.writer.FileAwareInputStreamDataWriterBuilder +converter.classes=org.apache.gobblin.converter.IdentityConverter +gobblin.dataset.profile.class=org.apache.gobblin.data.management.copy.hive.HiveDatasetFinder task.maxretries=0 workunit.retry.enabled=false http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-runtime/src/test/resources/templates/performanceTest.template ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/test/resources/templates/performanceTest.template b/gobblin-runtime/src/test/resources/templates/performanceTest.template index 5726a25..fed1589 100644 --- a/gobblin-runtime/src/test/resources/templates/performanceTest.template +++ b/gobblin-runtime/src/test/resources/templates/performanceTest.template @@ -1,4 +1,4 @@ -source.class=gobblin.util.test.FastSequentialSource +source.class=org.apache.gobblin.util.test.FastSequentialSource FastSequentialSource.maxRecordsPerWorkUnit=1000000000 FastSequentialSource.maxSecondsPerWorkUnit=60 http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-runtime/src/test/resources/templates/textFileBasedSourceTest.template ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/test/resources/templates/textFileBasedSourceTest.template b/gobblin-runtime/src/test/resources/templates/textFileBasedSourceTest.template index 579541a..4cdf991 100644 --- a/gobblin-runtime/src/test/resources/templates/textFileBasedSourceTest.template +++ b/gobblin-runtime/src/test/resources/templates/textFileBasedSourceTest.template @@ -1,4 +1,4 @@ -source.class=gobblin.source.extractor.filebased.TextFileBasedSource +source.class=org.apache.gobblin.source.extractor.filebased.TextFileBasedSource writer.builder.class="org.apache.gobblin.writer.test.GobblinTestEventBusWriter$Builder" extract.table.type=APPEND_ONLY http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-test-harness/resource/dataManagement/copy/job-props/copy.pull ---------------------------------------------------------------------- diff --git a/gobblin-test-harness/resource/dataManagement/copy/job-props/copy.pull b/gobblin-test-harness/resource/dataManagement/copy/job-props/copy.pull index 2946744..2b99c16 100755 --- a/gobblin-test-harness/resource/dataManagement/copy/job-props/copy.pull +++ b/gobblin-test-harness/resource/dataManagement/copy/job-props/copy.pull @@ -20,20 +20,20 @@ job.group=Test job.description=Test Gobblin job for copy job.schedule=0 0/2 * * * ? -extract.namespace=gobblin.copy.test1 +extract.namespace=org.apache.gobblin.copy.test1 writer.fs.uri=file://localhost/ source.filebased.fs.uri=file://localhost/ -data.publisher.type=gobblin.data.management.copy.publisher.CopyDataPublisher +data.publisher.type=org.apache.gobblin.data.management.copy.publisher.CopyDataPublisher -source.class=gobblin.data.management.copy.CopySource +source.class=org.apache.gobblin.data.management.copy.CopySource -writer.builder.class=gobblin.data.management.copy.writer.TarArchiveInputStreamDataWriterBuilder +writer.builder.class=org.apache.gobblin.data.management.copy.writer.TarArchiveInputStreamDataWriterBuilder gobblin.dataset.pattern=gobblin-test-harness/resource/dataManagement/copy/data/ -converter.classes=gobblin.data.management.copy.converter.UnGzipConverter +converter.classes=org.apache.gobblin.data.management.copy.converter.UnGzipConverter task.maxretries=0 workunit.retry.enabled-false \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-test-harness/src/test/resources/runtime_test/skip_workunits_test.properties ---------------------------------------------------------------------- diff --git a/gobblin-test-harness/src/test/resources/runtime_test/skip_workunits_test.properties b/gobblin-test-harness/src/test/resources/runtime_test/skip_workunits_test.properties index 712b0ab..83a97de 100644 --- a/gobblin-test-harness/src/test/resources/runtime_test/skip_workunits_test.properties +++ b/gobblin-test-harness/src/test/resources/runtime_test/skip_workunits_test.properties @@ -26,4 +26,4 @@ writer.staging.dir=./gobblin-test-harness/src/test/resources/runtime_test/writer writer.output.dir=./gobblin-test-harness/src/test/resources/runtime_test/writer_output -source.class=gobblin.TestSkipWorkUnitsSource +source.class=org.apache.gobblin.TestSkipWorkUnitsSource http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-test/resource/job-conf/GobblinTest1.pull ---------------------------------------------------------------------- diff --git a/gobblin-test/resource/job-conf/GobblinTest1.pull b/gobblin-test/resource/job-conf/GobblinTest1.pull index 13f72ec..2e7f3a9 100644 --- a/gobblin-test/resource/job-conf/GobblinTest1.pull +++ b/gobblin-test/resource/job-conf/GobblinTest1.pull @@ -23,7 +23,7 @@ job.schedule=0 0/2 * * * ? source.class=org.apache.gobblin.test.TestSource converter.classes=org.apache.gobblin.test.TestConverter -extract.namespace=gobblin.test1 +extract.namespace=org.apache.gobblin.test1 writer.destination.type=HDFS writer.output.format=AVRO http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-test/resource/job-conf/GobblinTest2.pull ---------------------------------------------------------------------- diff --git a/gobblin-test/resource/job-conf/GobblinTest2.pull b/gobblin-test/resource/job-conf/GobblinTest2.pull index ee3064c..fe5929f 100644 --- a/gobblin-test/resource/job-conf/GobblinTest2.pull +++ b/gobblin-test/resource/job-conf/GobblinTest2.pull @@ -23,7 +23,7 @@ job.schedule=0 0/2 * * * ? source.class=org.apache.gobblin.test.TestSource converter.classes=org.apache.gobblin.test.TestConverter -extract.namespace=gobblin.test2 +extract.namespace=org.apache.gobblin.test2 writer.destination.type=HDFS writer.output.format=AVRO http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-test/resource/job-conf/GobblinTest3.pull ---------------------------------------------------------------------- diff --git a/gobblin-test/resource/job-conf/GobblinTest3.pull b/gobblin-test/resource/job-conf/GobblinTest3.pull index 2ae142b..5c6720c 100644 --- a/gobblin-test/resource/job-conf/GobblinTest3.pull +++ b/gobblin-test/resource/job-conf/GobblinTest3.pull @@ -23,7 +23,7 @@ job.schedule=0 0/2 * * * ? source.class=org.apache.gobblin.test.TestSource converter.classes=org.apache.gobblin.test.TestConverter -extract.namespace=gobblin.test3 +extract.namespace=org.apache.gobblin.test3 writer.destination.type=HDFS writer.output.format=AVRO http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-test/resource/mr-job-conf/GobblinMRTest.pull ---------------------------------------------------------------------- diff --git a/gobblin-test/resource/mr-job-conf/GobblinMRTest.pull b/gobblin-test/resource/mr-job-conf/GobblinMRTest.pull index 716558c..150f956 100644 --- a/gobblin-test/resource/mr-job-conf/GobblinMRTest.pull +++ b/gobblin-test/resource/mr-job-conf/GobblinMRTest.pull @@ -22,7 +22,7 @@ job.schedule=0 0/2 * * * ? source.class=org.apache.gobblin.test.TestSource converter.classes=org.apache.gobblin.test.TestConverter -extract.namespace=gobblin.MRTest +extract.namespace=org.apache.gobblin.MRTest writer.destination.type=HDFS writer.output.format=AVRO http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/fcfad779/gobblin-yarn/src/test/resources/reference.conf ---------------------------------------------------------------------- diff --git a/gobblin-yarn/src/test/resources/reference.conf b/gobblin-yarn/src/test/resources/reference.conf index 8a39310..b387249 100644 --- a/gobblin-yarn/src/test/resources/reference.conf +++ b/gobblin-yarn/src/test/resources/reference.conf @@ -48,7 +48,7 @@ writer.staging.dir=${gobblin.yarn.work.dir}/task-staging writer.output.dir=${gobblin.yarn.work.dir}/task-output # Data publisher related configuration properties -data.publisher.type=gobblin.publisher.BaseDataPublisher +data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher data.publisher.final.dir=${gobblin.yarn.work.dir}/job-output data.publisher.replace.final.dir=false
