Repository: incubator-gobblin Updated Branches: refs/heads/master f00266f06 -> cbb894327
[GOBBLIN-632] Template for Incremental Avro Ingestion Closes #2502 from cshen98/cashen_inc-avro_template Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/cbb89432 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/cbb89432 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/cbb89432 Branch: refs/heads/master Commit: cbb894327378165c90e8f1805f5d6e150d838639 Parents: f00266f Author: Carl Shen <[email protected]> Authored: Thu Nov 15 16:07:48 2018 -0800 Committer: Hung Tran <[email protected]> Committed: Thu Nov 15 16:07:48 2018 -0800 ---------------------------------------------------------------------- .../templates/incremental-avro.template | 48 ++++++++++++++++++++ 1 file changed, 48 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/cbb89432/gobblin-runtime/src/main/resources/templates/incremental-avro.template ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/main/resources/templates/incremental-avro.template b/gobblin-runtime/src/main/resources/templates/incremental-avro.template new file mode 100644 index 0000000..bc2dcc9 --- /dev/null +++ b/gobblin-runtime/src/main/resources/templates/incremental-avro.template @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +# These properties are required and depend on the job's intended behavior beyond the incremental avro ingestion + +gobblin.template.required_attributes="source.filebased.fs.uri,from,to,converter.classes,data.publisher.type,writer.builder.format" +data.publisher.final.dir=${to} +source.filebased.data.directory=${from} + + +# These properties can be configured and are unrelated to the actual behavior of the incremental avro ingestion + +extract.namespace=gobblin.incremental.avro +type=hadoopJava +job.class=org.apache.gobblin.azkaban.AzkabanJobLauncher +job.description="Incremental avro ingestion" +job.name=IncrementalAvroIngestion +task.maxretries=0 +workunit.retry.enabled=false + + +# These properties must be fixed to enable the incremental avro ingestion + +source.class=org.apache.gobblin.source.DatePartitionedAvroFileSource +source.entity=avro + + +# These properties can be configured and define the actual behavior of the incremental avro ingestion + +date.partitioned.source.min.watermark.value=2018-11-15_00_00_00 +date.partitioned.source.partition.lead_time.size=24 +date.partitioned.source.partition.pattern=yyyy-MM-dd_HH_mm_ss +date.partitioned.source.partition.prefix=hourly
