Changed package from gobblin to org.apache.gobblin in docs and pull files
Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/2717a519 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/2717a519 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/2717a519 Branch: refs/heads/master Commit: 2717a519a052ea26a3070deb4228793d108903ce Parents: 177a42d Author: Abhishek Tiwari <[email protected]> Authored: Sun Jul 30 19:14:44 2017 -0700 Committer: Abhishek Tiwari <[email protected]> Committed: Sun Jul 30 19:14:44 2017 -0700 ---------------------------------------------------------------------- gobblin-core/src/main/resources/mysql/mysql.properties | 8 ++++---- gobblin-core/src/main/resources/sftp/sftp-example.job | 4 ++-- .../src/main/resources/sqlserver/sqlserver.properties | 8 ++++---- gobblin-docs/adaptors/Hive-Avro-To-ORC-Converter.md | 10 +++++----- gobblin-docs/user-guide/Working-with-the-ForkOperator.md | 10 +++++----- .../google-analytics-unsampledreport-to-avro.pull | 8 ++++---- gobblin-example/src/main/resources/google-web-master.pull | 8 ++++---- gobblin-example/src/main/resources/simplejson.pull | 10 +++++----- gobblin-example/src/main/resources/wikipedia-orc.pull | 6 +++--- .../src/test/resources/mr-job-conf/GobblinMRTest.pull | 6 +++--- .../src/main/resources/azkaban/common.properties | 10 +++++----- .../src/main/resources/stand-alone/common.properties | 8 ++++---- .../runtime_test/writer_output_format_test.properties | 6 +++--- gobblin-test/resource/job-conf/GobblinTest1.pull | 10 +++++----- gobblin-test/resource/job-conf/GobblinTest2.pull | 10 +++++----- gobblin-test/resource/job-conf/GobblinTest3.pull | 10 +++++----- gobblin-test/resource/mr-job-conf/GobblinMRTest.pull | 10 +++++----- 17 files changed, 71 insertions(+), 71 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/2717a519/gobblin-core/src/main/resources/mysql/mysql.properties ---------------------------------------------------------------------- diff --git a/gobblin-core/src/main/resources/mysql/mysql.properties b/gobblin-core/src/main/resources/mysql/mysql.properties index 4ebf497..62bdacc 100644 --- a/gobblin-core/src/main/resources/mysql/mysql.properties +++ b/gobblin-core/src/main/resources/mysql/mysql.properties @@ -16,7 +16,7 @@ # # Source properties - source class to extract data from Mysql Source -source.class=gobblin.source.extractor.extract.jdbc.MysqlSource +source.class=org.apache.gobblin.source.extractor.extract.jdbc.MysqlSource # Source properties source.max.number.of.partitions=<1> @@ -33,7 +33,7 @@ source.conn.port=<portnumber> source.conn.timeout=<timeout in seconds> # Converter properties - Record from mysql source will be processed by the below series of converters -converter.classes=gobblin.converter.avro.JsonIntermediateToAvroConverter +converter.classes=org.apache.gobblin.converter.avro.JsonIntermediateToAvroConverter # date columns format converter.avro.timestamp.format=yyyy-MM-dd HH:mm:ss'.0' @@ -41,8 +41,8 @@ converter.avro.date.format=yyyy-MM-dd converter.avro.time.format=HH:mm:ss # Qualitychecker properties -qualitychecker.task.policies=gobblin.policies.count.RowCountPolicy,gobblin.policies.schema.SchemaCompatibilityPolicy +qualitychecker.task.policies=org.apache.gobblin.policies.count.RowCountPolicy,org.apache.gobblin.policies.schema.SchemaCompatibilityPolicy qualitychecker.task.policy.types=OPTIONAL,OPTIONAL # Publisher properties -data.publisher.type=gobblin.publisher.BaseDataPublisher +data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/2717a519/gobblin-core/src/main/resources/sftp/sftp-example.job ---------------------------------------------------------------------- diff --git a/gobblin-core/src/main/resources/sftp/sftp-example.job b/gobblin-core/src/main/resources/sftp/sftp-example.job index bfcb59a..55d1600 100644 --- a/gobblin-core/src/main/resources/sftp/sftp-example.job +++ b/gobblin-core/src/main/resources/sftp/sftp-example.job @@ -21,7 +21,7 @@ job.group=ExampleJobs job.description=An example config file for SFTP jobs # Source properties -source.class=gobblin.source.extractor.extract.sftp.SftpSource +source.class=org.apache.gobblin.source.extractor.extract.sftp.SftpSource # The SftpSource class will look for data on the SFTP server under this directory source.filebased.data.directory=example/input @@ -39,7 +39,7 @@ extract.table.type=append_only # Converter properties # By specifying these converters, Gobblin will take each record returned by SftpSource, split each record by some delimiter, and then convert the record to an Avro record -converter.classes=gobblin.converter.csv.CsvToJsonConverter,gobblin.converter.avro.JsonIntermediateToAvroConverter +converter.classes=org.apache.gobblin.converter.csv.CsvToJsonConverter,org.apache.gobblin.converter.avro.JsonIntermediateToAvroConverter converter.csv.to.json.delimiter=, # Source connection properties http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/2717a519/gobblin-core/src/main/resources/sqlserver/sqlserver.properties ---------------------------------------------------------------------- diff --git a/gobblin-core/src/main/resources/sqlserver/sqlserver.properties b/gobblin-core/src/main/resources/sqlserver/sqlserver.properties index cb30c96..7f1853e 100644 --- a/gobblin-core/src/main/resources/sqlserver/sqlserver.properties +++ b/gobblin-core/src/main/resources/sqlserver/sqlserver.properties @@ -16,7 +16,7 @@ # # Source properties - source class to extract data from sqlserver Source -source.class=gobblin.source.extractor.extract.jdbc.SqlServerSource +source.class=org.apache.gobblin.source.extractor.extract.jdbc.SqlServerSource # Source properties source.max.number.of.partitions=<1> @@ -32,7 +32,7 @@ source.conn.port=<portnumber> source.conn.timeout=<timeout in seconds> # Converter properties - Record from mysql source will be processed by the below series of converters -converter.classes=gobblin.converter.avro.JsonIntermediateToAvroConverter +converter.classes=org.apache.gobblin.converter.avro.JsonIntermediateToAvroConverter # date columns format converter.avro.timestamp.format=yyyy-MM-dd HH:mm:ss'.0' @@ -40,8 +40,8 @@ converter.avro.date.format=yyyy-MM-dd converter.avro.time.format=HH:mm:ss # Qualitychecker properties -qualitychecker.task.policies=gobblin.policies.count.RowCountPolicy,gobblin.policies.schema.SchemaCompatibilityPolicy +qualitychecker.task.policies=org.apache.gobblin.policies.count.RowCountPolicy,org.apache.gobblin.policies.schema.SchemaCompatibilityPolicy qualitychecker.task.policy.types=OPTIONAL,OPTIONAL # Publisher properties -data.publisher.type=gobblin.publisher.BaseDataPublisher +data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/2717a519/gobblin-docs/adaptors/Hive-Avro-To-ORC-Converter.md ---------------------------------------------------------------------- diff --git a/gobblin-docs/adaptors/Hive-Avro-To-ORC-Converter.md b/gobblin-docs/adaptors/Hive-Avro-To-ORC-Converter.md index 117d506..e82b867 100644 --- a/gobblin-docs/adaptors/Hive-Avro-To-ORC-Converter.md +++ b/gobblin-docs/adaptors/Hive-Avro-To-ORC-Converter.md @@ -273,12 +273,12 @@ The diagram below describes timestamps captured in the SLA event. # Avro hive databases and tables to convert hive.dataset.whitelist=events.LogEvent|LoginEvent -data.publisher.type=gobblin.data.management.conversion.hive.publisher.HiveConvertPublisher -source.class=gobblin.data.management.conversion.hive.source.HiveAvroToOrcSource -writer.builder.class=gobblin.data.management.conversion.hive.writer.HiveQueryWriterBuilder -converter.classes=gobblin.data.management.conversion.hive.converter.HiveAvroToFlattenedOrcConverter,gobblin.data.management.conversion.hive.converter.HiveAvroToNestedOrcConverter +data.publisher.type=org.apache.gobblin.data.management.conversion.hive.publisher.HiveConvertPublisher +source.class=org.apache.gobblin.data.management.conversion.hive.source.HiveAvroToOrcSource +writer.builder.class=org.apache.gobblin.data.management.conversion.hive.writer.HiveQueryWriterBuilder +converter.classes=org.apache.gobblin.data.management.conversion.hive.converter.HiveAvroToFlattenedOrcConverter,org.apache.gobblin.data.management.conversion.hive.converter.HiveAvroToNestedOrcConverter -hive.dataset.finder.class=gobblin.data.management.conversion.hive.dataset.ConvertibleHiveDatasetFinder +hive.dataset.finder.class=org.apache.gobblin.data.management.conversion.hive.dataset.ConvertibleHiveDatasetFinder # Only flattened orc is enabled hive.conversion.avro.destinationFormats=flattenedOrc http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/2717a519/gobblin-docs/user-guide/Working-with-the-ForkOperator.md ---------------------------------------------------------------------- diff --git a/gobblin-docs/user-guide/Working-with-the-ForkOperator.md b/gobblin-docs/user-guide/Working-with-the-ForkOperator.md index 1eea8f4..54e3a59 100644 --- a/gobblin-docs/user-guide/Working-with-the-ForkOperator.md +++ b/gobblin-docs/user-guide/Working-with-the-ForkOperator.md @@ -142,17 +142,17 @@ public class SensitiveDataAwareForkOperator implements ForkOperator<Schema, Gene To make the example more concrete, let's assume that the job uses some converters and quality checkers before the schema and data records reach the `SensitiveDataAwareForkOperator`, and it also uses a converter to purge the sensitive fields and a quality checker that makes sure some mandatory fields exist for purged data records in branch 0. Both branches will be written to the same HDFS but into different locations. ``` -fork.operator.class=gobblin.example.fork.SensitiveDataAwareForkOperator +fork.operator.class=org.apache.gobblin.example.fork.SensitiveDataAwareForkOperator # Pre-fork or non-fork-specific configuration properties converter.classes=<Converter classes used in the task flow prior to OutlierAwareForkOperator> -qualitychecker.task.policies=gobblin.policies.count.RowCountPolicy,gobblin.policies.schema.SchemaCompatibilityPolicy +qualitychecker.task.policies=org.apache.gobblin.policies.count.RowCountPolicy,org.apache.gobblin.policies.schema.SchemaCompatibilityPolicy qualitychecker.task.policy.types=OPTIONAL,OPTIONAL -data.publisher.type=gobblin.publisher.BaseDataPublisher +data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher # Configuration properties for branch 0 -converter.classes.0=gobblin.example.converter.PurgingConverter -qualitychecker.task.policies.0=gobblin.example,policies.MandatoryFieldExistencePolicy +converter.classes.0=org.apache.gobblin.example.converter.PurgingConverter +qualitychecker.task.policies.0=org.apache.gobblin.example,policies.MandatoryFieldExistencePolicy qualitychecker.task.policy.types.0=FAILED writer.fs.uri.0=hdfs://<namenode host>:<namenode port>/ writer.destination.type.0=HDFS http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/2717a519/gobblin-example/src/main/resources/google-analytics-unsampledreport-to-avro.pull ---------------------------------------------------------------------- diff --git a/gobblin-example/src/main/resources/google-analytics-unsampledreport-to-avro.pull b/gobblin-example/src/main/resources/google-analytics-unsampledreport-to-avro.pull index df9cd8c..bdaf8df 100644 --- a/gobblin-example/src/main/resources/google-analytics-unsampledreport-to-avro.pull +++ b/gobblin-example/src/main/resources/google-analytics-unsampledreport-to-avro.pull @@ -21,7 +21,7 @@ job.name=test_ga_to_avro source.entity=GoogleAnayticsTest -source.class=gobblin.source.extractor.extract.google.GoogleAnalyticsUnsampledSource +source.class=org.apache.gobblin.source.extractor.extract.google.GoogleAnalyticsUnsampledSource source.conn.private.key=/path/to/private/key/file.json source.schema=[{"columnName":"Date","comment":"","isNullable":"true","dataType":{"type":"string"}}, {"columnName":"DeviceCategory","comment":"","isNullable":"false","dataType":{"type":"string"}}, {"columnName":"Sessions","comment":"","isNullable":"true","dataType":{"type":"int"}}, {"columnName":"BounceRate","comment":"","isNullable":"true","dataType":{"type":"double"}}, {"columnName":"AvgSessionDuration","comment":"","isNullable":"true","dataType":{"type":"double"}}, {"columnName":"Pageviews","comment":"","isNullable":"true","dataType":{"type":"int"}}, {"columnName":"PageviewsPerSession","comment":"","isNullable":"true","dataType":{"type":"double"}}, {"columnName":"UniquePageviews","comment":"","isNullable":"true","dataType":{"type":"int"}}, {"columnName":"AvgTimeOnPage","comment":"","isNullable":"true","dataType":{"type":"double"}}, {"columnName":"User_count","comment":"","isNullable":"true","dataType":{"type":"int"}}] source.csv_file.skip_top_rows_regex=\"?#.* @@ -52,14 +52,14 @@ extract.delta.fields=dummy extract.namespace=${source.entity} # Converter properties - Record from source will be processed by the below series of converters -converter.classes=gobblin.converter.csv.CsvToJsonConverterV2,gobblin.converter.avro.JsonIntermediateToAvroConverter +converter.classes=org.apache.gobblin.converter.csv.CsvToJsonConverterV2,org.apache.gobblin.converter.avro.JsonIntermediateToAvroConverter converter.csv_to_json.custom_order=0,1,3,4,5,6,7,8,9,10 data.publisher.final.dir=${env:GOBBLIN_WORK_DIR} writer.file.path=job-output/${source.entity} data.publisher.replace.final.dir=true -qualitychecker.task.policies=gobblin.policies.count.RowCountPolicy +qualitychecker.task.policies=org.apache.gobblin.policies.count.RowCountPolicy qualitychecker.task.policy.types=OPTIONAL metrics.reporting.file.enabled=true @@ -68,4 +68,4 @@ metrics.reporting.file.enabled=true writer.destination.type=HDFS writer.output.format=AVRO -data.publisher.type=gobblin.publisher.BaseDataPublisher \ No newline at end of file +data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/2717a519/gobblin-example/src/main/resources/google-web-master.pull ---------------------------------------------------------------------- diff --git a/gobblin-example/src/main/resources/google-web-master.pull b/gobblin-example/src/main/resources/google-web-master.pull index 9f215ef..0db95ae 100644 --- a/gobblin-example/src/main/resources/google-web-master.pull +++ b/gobblin-example/src/main/resources/google-web-master.pull @@ -54,7 +54,7 @@ source.querybased.append.max.watermark.limit=CURRENTDATE-0 extract.table.name=DailyPull source.querybased.watermark.type=hour source.querybased.partition.interval=23 -source.class=gobblin.ingestion.google.webmaster.GoogleWebMasterSourceDaily +source.class=org.apache.gobblin.ingestion.google.webmaster.GoogleWebMasterSourceDaily #Webmaster Specific source.google_webmasters.property_url=[Complete This] @@ -66,18 +66,18 @@ source.google_webmasters.request.query_limit=5000 source.google_webmasters.request.tuning.get_queries.apply_trie=true ## Converter properties -converter.classes=gobblin.converter.csv.CsvToJsonConverterV2,gobblin.converter.avro.JsonIntermediateToAvroConverter +converter.classes=org.apache.gobblin.converter.csv.CsvToJsonConverterV2,org.apache.gobblin.converter.avro.JsonIntermediateToAvroConverter ## Writer properties writer.output.format=AVRO writer.partitioner.google_webmasters.prefix=daily writer.partitioner.google_webmasters.column_names.include=false -writer.partitioner.class=gobblin.ingestion.google.webmaster.GoogleWebmasterDayPartitioner +writer.partitioner.class=org.apache.gobblin.ingestion.google.webmaster.GoogleWebmasterDayPartitioner writer.include.record.count.in.file.names=true ## Publisher properties -data.publisher.type=gobblin.publisher.BaseDataPublisher +data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher ## Configurations for saving to HDFS http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/2717a519/gobblin-example/src/main/resources/simplejson.pull ---------------------------------------------------------------------- diff --git a/gobblin-example/src/main/resources/simplejson.pull b/gobblin-example/src/main/resources/simplejson.pull index 735636d..22b6186 100644 --- a/gobblin-example/src/main/resources/simplejson.pull +++ b/gobblin-example/src/main/resources/simplejson.pull @@ -23,8 +23,8 @@ job.name=GobblinDemo job.group=demo job.description=A Gobblin job for demo purpose -source.class=gobblin.example.simplejson.SimpleJsonSource -converter.classes=gobblin.example.simplejson.SimpleJsonConverter +source.class=org.apache.gobblin.example.simplejson.SimpleJsonSource +converter.classes=org.apache.gobblin.example.simplejson.SimpleJsonConverter extract.namespace=gobblin.example.simplejson # source configuration properties @@ -40,14 +40,14 @@ source.conn.password= source.schema={"namespace":"example.avro", "type":"record", "name":"User", "fields":[{"name":"name", "type":"string"}, {"name":"favorite_number", "type":"int"}, {"name":"favorite_color", "type":"string"}]} # quality checker configuration properties -qualitychecker.task.policies=gobblin.policies.count.RowCountPolicy,gobblin.policies.schema.SchemaCompatibilityPolicy +qualitychecker.task.policies=org.apache.gobblin.policies.count.RowCountPolicy,org.apache.gobblin.policies.schema.SchemaCompatibilityPolicy qualitychecker.task.policy.types=OPTIONAL,OPTIONAL -qualitychecker.row.policies=gobblin.policies.schema.SchemaRowCheckPolicy +qualitychecker.row.policies=org.apache.gobblin.policies.schema.SchemaRowCheckPolicy qualitychecker.row.policy.types=OPTIONAL qualitychecker.row.err.file=test/jobOutput # data publisher class to be used -data.publisher.type=gobblin.publisher.BaseDataPublisher +data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher # writer configuration properties writer.destination.type=HDFS http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/2717a519/gobblin-example/src/main/resources/wikipedia-orc.pull ---------------------------------------------------------------------- diff --git a/gobblin-example/src/main/resources/wikipedia-orc.pull b/gobblin-example/src/main/resources/wikipedia-orc.pull index a980c97..a8e2420 100644 --- a/gobblin-example/src/main/resources/wikipedia-orc.pull +++ b/gobblin-example/src/main/resources/wikipedia-orc.pull @@ -19,7 +19,7 @@ job.name=PullFromWikipedia job.group=Wikipedia job.description=A getting started example for Gobblin -source.class=gobblin.example.wikipedia.WikipediaSource +source.class=org.apache.gobblin.example.wikipedia.WikipediaSource source.page.titles=NASA,LinkedIn,Parris_Cues,Barbara_Corcoran source.revisions.cnt=5 @@ -28,7 +28,7 @@ extract.namespace=gobblin.example.wikipedia wikipedia.api.rooturl=https://en.wikipedia.org/w/api.php wikipedia.avro.schema={"namespace": "example.wikipedia.avro","type": "record","name": "WikipediaArticle","fields": [{"name": "revid", "type": ["double", "null"]},{"name": "pageid", "type": ["double", "null"]},{"name": "title", "type": ["string", "null"]},{"name": "user", "type": ["string", "null"]},{"name": "anon", "type": ["string", "null"]},{"name": "userid", "type": ["double", "null"]},{"name": "timestamp", "type": ["string", "null"]},{"name": "size", "type": ["double", "null"]},{"name": "contentformat", "type": ["string", "null"]},{"name": "contentmodel", "type": ["string", "null"]},{"name": "content", "type": ["string", "null"]}]} -converter.classes=gobblin.example.wikipedia.WikipediaConverter,gobblin.converter.avro.AvroRecordToAvroWritableConverter,gobblin.converter.serde.HiveSerDeConverter +converter.classes=org.apache.gobblin.example.wikipedia.WikipediaConverter,org.apache.gobblin.converter.avro.AvroRecordToAvroWritableConverter,org.apache.gobblin.converter.serde.HiveSerDeConverter # Copy of wikipedia.avro.schema because Java Properties has problems with variable substitution for Avro schemas avro.schema.literal={"namespace": "example.wikipedia.avro","type": "record","name": "WikipediaArticle","fields": [{"name": "pageid", "type": ["double", "null"]},{"name": "title", "type": ["string", "null"]},{"name": "user", "type": ["string", "null"]},{"name": "anon", "type": ["string", "null"]},{"name": "userid", "type": ["double", "null"]},{"name": "timestamp", "type": ["string", "null"]},{"name": "size", "type": ["double", "null"]},{"name": "contentformat", "type": ["string", "null"]},{"name": "contentmodel", "type": ["string", "null"]},{"name": "content", "type": ["string", "null"]}]} @@ -36,7 +36,7 @@ avro.schema.literal={"namespace": "example.wikipedia.avro","type": "record","nam serde.deserializer.type=AVRO serde.serializer.type=ORC -writer.builder.class=gobblin.writer.HiveWritableHdfsDataWriterBuilder +writer.builder.class=org.apache.gobblin.writer.HiveWritableHdfsDataWriterBuilder writer.output.format=ORC fork.record.queue.capacity=1 http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/2717a519/gobblin-runtime/src/test/resources/mr-job-conf/GobblinMRTest.pull ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/test/resources/mr-job-conf/GobblinMRTest.pull b/gobblin-runtime/src/test/resources/mr-job-conf/GobblinMRTest.pull index 9036e29..9d68989 100644 --- a/gobblin-runtime/src/test/resources/mr-job-conf/GobblinMRTest.pull +++ b/gobblin-runtime/src/test/resources/mr-job-conf/GobblinMRTest.pull @@ -30,11 +30,11 @@ writer.fs.uri=file:/// source.files=gobblin-test/source/test.avro.0,gobblin-test/source/test.avro.1 -qualitychecker.task.policies=gobblin.policies.count.RowCountPolicy,gobblin.policies.schema.SchemaCompatibilityPolicy +qualitychecker.task.policies=org.apache.gobblin.policies.count.RowCountPolicy,org.apache.gobblin.policies.schema.SchemaCompatibilityPolicy qualitychecker.task.policy.types=OPTIONAL,OPTIONAL -qualitychecker.row.policies=gobblin.policies.schema.SchemaRowCheckPolicy +qualitychecker.row.policies=org.apache.gobblin.policies.schema.SchemaRowCheckPolicy qualitychecker.row.policy.types=OPTIONAL qualitychecker.row.err.file=gobblin-test/err -data.publisher.type=gobblin.publisher.BaseDataPublisher +data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/2717a519/gobblin-salesforce/src/main/resources/azkaban/common.properties ---------------------------------------------------------------------- diff --git a/gobblin-salesforce/src/main/resources/azkaban/common.properties b/gobblin-salesforce/src/main/resources/azkaban/common.properties index 9c00b1c..60abdda 100644 --- a/gobblin-salesforce/src/main/resources/azkaban/common.properties +++ b/gobblin-salesforce/src/main/resources/azkaban/common.properties @@ -27,7 +27,7 @@ mapred.job.queue.name=<queue name> # Azkaban config type=java -job.class=gobblin.azkaban.AzkabanJobLauncher +job.class=org.apache.gobblin.azkaban.AzkabanJobLauncher launcher.type=MAPREDUCE # Job properties @@ -39,11 +39,11 @@ writer.destination.type=HDFS writer.output.format=AVRO # Qualitychecker properties -qualitychecker.task.policies=gobblin.policies.count.RowCountPolicy,gobblin.policies.schema.SchemaCompatibilityPolicy +qualitychecker.task.policies=org.apache.gobblin.policies.count.RowCountPolicy,org.apache.gobblin.policies.schema.SchemaCompatibilityPolicy qualitychecker.task.policy.types=OPTIONAL,OPTIONAL # Publisher properties -data.publisher.type=gobblin.publisher.BaseDataPublisher +data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher data.publisher.replace.final.dir=false # Extract properties @@ -51,7 +51,7 @@ extract.delta.fields=SystemModstamp extract.primary.key.fields=Id # Source class properties -source.class=gobblin.salesforce.SalesforceSource +source.class=org.apache.gobblin.salesforce.SalesforceSource source.timezone=UTC source.querybased.schema=Core source.querybased.fetch.size=<records per batch ex: 5000> @@ -60,7 +60,7 @@ source.querybased.start.value=CURRENTDAY-1 workunit.retry.enabled=false # Converter properties -converter.classes=gobblin.converter.avro.JsonIntermediateToAvroConverter +converter.classes=org.apache.gobblin.converter.avro.JsonIntermediateToAvroConverter converter.avro.timestamp.format=yyyy-MM-dd'T'HH:mm:ss.SSS'Z',yyyy-MM-dd'T'HH:mm:ss.000+0000 converter.avro.date.format=yyyy-MM-dd converter.avro.time.format=HH:mm:ss,HH:mm:ss.000'Z' http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/2717a519/gobblin-salesforce/src/main/resources/stand-alone/common.properties ---------------------------------------------------------------------- diff --git a/gobblin-salesforce/src/main/resources/stand-alone/common.properties b/gobblin-salesforce/src/main/resources/stand-alone/common.properties index 040a48e..514b31f 100644 --- a/gobblin-salesforce/src/main/resources/stand-alone/common.properties +++ b/gobblin-salesforce/src/main/resources/stand-alone/common.properties @@ -16,7 +16,7 @@ # # Source properties - source class to extract data from Salesforce -source.class=gobblin.salesforce.SalesforceSource +source.class=org.apache.gobblin.salesforce.SalesforceSource # Source properties source.querybased.schema=Core @@ -45,17 +45,17 @@ extract.delta.fields=SystemModstamp extract.primary.key.fields=Id # Converter properties - Record from mysql source will be processed by the below series of converters -converter.classes=gobblin.converter.avro.JsonIntermediateToAvroConverter +converter.classes=org.apache.gobblin.converter.avro.JsonIntermediateToAvroConverter converter.avro.timestamp.format=yyyy-MM-dd'T'HH:mm:ss.SSS'Z',yyyy-MM-dd'T'HH:mm:ss.000+0000 converter.avro.date.format=yyyy-MM-dd converter.avro.time.format=HH:mm:ss,HH:mm:ss.000'Z' # Qualitychecker properties -qualitychecker.task.policies=gobblin.policies.count.RowCountPolicy,gobblin.policies.schema.SchemaCompatibilityPolicy +qualitychecker.task.policies=org.apache.gobblin.policies.count.RowCountPolicy,org.apache.gobblin.policies.schema.SchemaCompatibilityPolicy qualitychecker.task.policy.types=OPTIONAL,OPTIONAL # Publisher properties -data.publisher.type=gobblin.publisher.BaseDataPublisher +data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher # Proxy settings - optional source.conn.use.proxy.url=<Proxy host name - optional> http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/2717a519/gobblin-test-harness/src/test/resources/runtime_test/writer_output_format_test.properties ---------------------------------------------------------------------- diff --git a/gobblin-test-harness/src/test/resources/runtime_test/writer_output_format_test.properties b/gobblin-test-harness/src/test/resources/runtime_test/writer_output_format_test.properties index b669091..1c0ac20 100644 --- a/gobblin-test-harness/src/test/resources/runtime_test/writer_output_format_test.properties +++ b/gobblin-test-harness/src/test/resources/runtime_test/writer_output_format_test.properties @@ -33,10 +33,10 @@ writer.staging.dir=./gobblin-test-harness/src/test/resources/runtime_test/writer writer.output.dir=./gobblin-test-harness/src/test/resources/runtime_test/writer_output data.publisher.final.dir=./gobblin-test-harness/src/test/resources/runtime_test/final_dir -converter.classes=gobblin.converter.avro.AvroRecordToAvroWritableConverter,gobblin.converter.serde.HiveSerDeConverter +converter.classes=org.apache.gobblin.converter.avro.AvroRecordToAvroWritableConverter,org.apache.gobblin.converter.serde.HiveSerDeConverter serde.deserializer.type=AVRO -writer.builder.class=gobblin.writer.HiveWritableHdfsDataWriterBuilder +writer.builder.class=org.apache.gobblin.writer.HiveWritableHdfsDataWriterBuilder fork.record.queue.capacity=1 avro.schema.literal={"type":"record","name":"User","namespace":"example.avro","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]} -source.class=gobblin.TestAvroSource \ No newline at end of file +source.class=org.apache.gobblin.TestAvroSource \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/2717a519/gobblin-test/resource/job-conf/GobblinTest1.pull ---------------------------------------------------------------------- diff --git a/gobblin-test/resource/job-conf/GobblinTest1.pull b/gobblin-test/resource/job-conf/GobblinTest1.pull index bf95a6d..13f72ec 100644 --- a/gobblin-test/resource/job-conf/GobblinTest1.pull +++ b/gobblin-test/resource/job-conf/GobblinTest1.pull @@ -20,8 +20,8 @@ job.group=Test job.description=Test Gobblin job 1 job.schedule=0 0/2 * * * ? -source.class=gobblin.test.TestSource -converter.classes=gobblin.test.TestConverter +source.class=org.apache.gobblin.test.TestSource +converter.classes=org.apache.gobblin.test.TestConverter extract.namespace=gobblin.test1 @@ -31,11 +31,11 @@ writer.fs.uri=file://localhost/ source.files=gobblin-test/resource/source/test.avro.0,gobblin-test/resource/source/test.avro.1 -qualitychecker.task.policies=gobblin.policies.count.RowCountPolicy,gobblin.policies.schema.SchemaCompatibilityPolicy +qualitychecker.task.policies=org.apache.gobblin.policies.count.RowCountPolicy,org.apache.gobblin.policies.schema.SchemaCompatibilityPolicy qualitychecker.task.policy.types=OPTIONAL,OPTIONAL -qualitychecker.row.policies=gobblin.policies.schema.SchemaRowCheckPolicy +qualitychecker.row.policies=org.apache.gobblin.policies.schema.SchemaRowCheckPolicy qualitychecker.row.policy.types=OPTIONAL qualitychecker.row.err.file=gobblin-test/jobOutput -data.publisher.type=gobblin.publisher.BaseDataPublisher +data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/2717a519/gobblin-test/resource/job-conf/GobblinTest2.pull ---------------------------------------------------------------------- diff --git a/gobblin-test/resource/job-conf/GobblinTest2.pull b/gobblin-test/resource/job-conf/GobblinTest2.pull index 8f08a8c..ee3064c 100644 --- a/gobblin-test/resource/job-conf/GobblinTest2.pull +++ b/gobblin-test/resource/job-conf/GobblinTest2.pull @@ -20,8 +20,8 @@ job.group=Test job.description=Test Gobblin job 2 job.schedule=0 0/2 * * * ? -source.class=gobblin.test.TestSource -converter.classes=gobblin.test.TestConverter +source.class=org.apache.gobblin.test.TestSource +converter.classes=org.apache.gobblin.test.TestConverter extract.namespace=gobblin.test2 @@ -31,11 +31,11 @@ writer.fs.uri=file://localhost/ source.files=gobblin-test/resource/source/test.avro.0,gobblin-test/resource/source/test.avro.1 -qualitychecker.task.policies=gobblin.policies.count.RowCountPolicy,gobblin.policies.schema.SchemaCompatibilityPolicy +qualitychecker.task.policies=org.apache.gobblin.policies.count.RowCountPolicy,org.apache.gobblin.policies.schema.SchemaCompatibilityPolicy qualitychecker.task.policy.types=OPTIONAL,OPTIONAL -qualitychecker.row.policies=gobblin.policies.schema.SchemaRowCheckPolicy +qualitychecker.row.policies=org.apache.gobblin.policies.schema.SchemaRowCheckPolicy qualitychecker.row.policy.types=OPTIONAL qualitychecker.row.err.file=gobblin-test/jobOutput -data.publisher.type=gobblin.publisher.BaseDataPublisher +data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/2717a519/gobblin-test/resource/job-conf/GobblinTest3.pull ---------------------------------------------------------------------- diff --git a/gobblin-test/resource/job-conf/GobblinTest3.pull b/gobblin-test/resource/job-conf/GobblinTest3.pull index 2876b6d..2ae142b 100644 --- a/gobblin-test/resource/job-conf/GobblinTest3.pull +++ b/gobblin-test/resource/job-conf/GobblinTest3.pull @@ -20,8 +20,8 @@ job.group=Test job.description=Test Gobblin job 3 job.schedule=0 0/2 * * * ? -source.class=gobblin.test.TestSource -converter.classes=gobblin.test.TestConverter +source.class=org.apache.gobblin.test.TestSource +converter.classes=org.apache.gobblin.test.TestConverter extract.namespace=gobblin.test3 @@ -31,11 +31,11 @@ writer.fs.uri=file://localhost/ source.files=gobblin-test/resource/source/test.avro.0,gobblin-test/resource/source/test.avro.1 -qualitychecker.task.policies=gobblin.policies.count.RowCountPolicy,gobblin.policies.schema.SchemaCompatibilityPolicy +qualitychecker.task.policies=org.apache.gobblin.policies.count.RowCountPolicy,org.apache.gobblin.policies.schema.SchemaCompatibilityPolicy qualitychecker.task.policy.types=OPTIONAL,OPTIONAL -qualitychecker.row.policies=gobblin.policies.schema.SchemaRowCheckPolicy +qualitychecker.row.policies=org.apache.gobblin.policies.schema.SchemaRowCheckPolicy qualitychecker.row.policy.types=OPTIONAL qualitychecker.row.err.file=gobblin-test/jobOutput -data.publisher.type=gobblin.publisher.BaseDataPublisher +data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/2717a519/gobblin-test/resource/mr-job-conf/GobblinMRTest.pull ---------------------------------------------------------------------- diff --git a/gobblin-test/resource/mr-job-conf/GobblinMRTest.pull b/gobblin-test/resource/mr-job-conf/GobblinMRTest.pull index 9036e29..716558c 100644 --- a/gobblin-test/resource/mr-job-conf/GobblinMRTest.pull +++ b/gobblin-test/resource/mr-job-conf/GobblinMRTest.pull @@ -20,8 +20,8 @@ job.group=Test job.description=Test Gobblin job on Hadoop MR job.schedule=0 0/2 * * * ? -source.class=gobblin.test.TestSource -converter.classes=gobblin.test.TestConverter +source.class=org.apache.gobblin.test.TestSource +converter.classes=org.apache.gobblin.test.TestConverter extract.namespace=gobblin.MRTest writer.destination.type=HDFS @@ -30,11 +30,11 @@ writer.fs.uri=file:/// source.files=gobblin-test/source/test.avro.0,gobblin-test/source/test.avro.1 -qualitychecker.task.policies=gobblin.policies.count.RowCountPolicy,gobblin.policies.schema.SchemaCompatibilityPolicy +qualitychecker.task.policies=org.apache.gobblin.policies.count.RowCountPolicy,org.apache.gobblin.policies.schema.SchemaCompatibilityPolicy qualitychecker.task.policy.types=OPTIONAL,OPTIONAL -qualitychecker.row.policies=gobblin.policies.schema.SchemaRowCheckPolicy +qualitychecker.row.policies=org.apache.gobblin.policies.schema.SchemaRowCheckPolicy qualitychecker.row.policy.types=OPTIONAL qualitychecker.row.err.file=gobblin-test/err -data.publisher.type=gobblin.publisher.BaseDataPublisher +data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher
