Repository: incubator-gobblin Updated Branches: refs/heads/master 602cee793 -> 49974214a
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/49974214/gobblin-service/src/test/resources/template_catalog/multihop/jobTemplates/distcp-push-hdfs-to-adl.template ---------------------------------------------------------------------- diff --git a/gobblin-service/src/test/resources/template_catalog/multihop/jobTemplates/distcp-push-hdfs-to-adl.template b/gobblin-service/src/test/resources/template_catalog/multihop/jobTemplates/distcp-push-hdfs-to-adl.template index b92a2bf..26b2ed1 100644 --- a/gobblin-service/src/test/resources/template_catalog/multihop/jobTemplates/distcp-push-hdfs-to-adl.template +++ b/gobblin-service/src/test/resources/template_catalog/multihop/jobTemplates/distcp-push-hdfs-to-adl.template @@ -1,7 +1,7 @@ # ==================================================================== # Job configurations # ==================================================================== -job.name=Distcp-HDFS-ADL +job.name=DistcpToADL #team.name and dataset.name to be supplied via flowConfig from=/data/encrypted/${team.name}/${dataset.name} http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/49974214/gobblin-service/src/test/resources/template_catalog/multihop/jobTemplates/distcp.template ---------------------------------------------------------------------- diff --git a/gobblin-service/src/test/resources/template_catalog/multihop/jobTemplates/distcp.template b/gobblin-service/src/test/resources/template_catalog/multihop/jobTemplates/distcp.template index c8a557e..d9fcf6d 100644 --- a/gobblin-service/src/test/resources/template_catalog/multihop/jobTemplates/distcp.template +++ b/gobblin-service/src/test/resources/template_catalog/multihop/jobTemplates/distcp.template @@ -1,7 +1,7 @@ # ==================================================================== # Job configurations # ==================================================================== -job.name=Distcp-HDFS-HDFS +job.name=Distcp # Source and destination paths to be obtained from flow config. from=${gobblin.flow.edge.input.dataset.descriptor.path} http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/49974214/gobblin-service/src/test/resources/template_catalog/multihop/jobTemplates/hdfs-convert-to-json-and-encrypt.template ---------------------------------------------------------------------- diff --git a/gobblin-service/src/test/resources/template_catalog/multihop/jobTemplates/hdfs-convert-to-json-and-encrypt.template b/gobblin-service/src/test/resources/template_catalog/multihop/jobTemplates/hdfs-convert-to-json-and-encrypt.template index fcc78cd..54d8f74 100644 --- a/gobblin-service/src/test/resources/template_catalog/multihop/jobTemplates/hdfs-convert-to-json-and-encrypt.template +++ b/gobblin-service/src/test/resources/template_catalog/multihop/jobTemplates/hdfs-convert-to-json-and-encrypt.template @@ -1,7 +1,7 @@ # ==================================================================== # Job configurations (can be changed) # ==================================================================== -job.name=convert-to-json-and-encrypt +job.name=ConvertToJsonAndEncrypt job.description="Convert date partitioned avro files to json and encrypt" from=/data/out/${team.name}/${dataset.name} to=/data/encrypted/${team.name}/${dataset.name} http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/49974214/gobblin-service/src/test/resources/template_catalog/multihop/jobTemplates/hdfs-retention.template ---------------------------------------------------------------------- diff --git a/gobblin-service/src/test/resources/template_catalog/multihop/jobTemplates/hdfs-retention.template b/gobblin-service/src/test/resources/template_catalog/multihop/jobTemplates/hdfs-retention.template new file mode 100644 index 0000000..23c8b36 --- /dev/null +++ b/gobblin-service/src/test/resources/template_catalog/multihop/jobTemplates/hdfs-retention.template @@ -0,0 +1,13 @@ +# ==================================================================== +# Retention template +# ==================================================================== +job.name=SnapshotRetention +job.class=gobblin.data.management.retention.DatasetCleanerJob +#Dataset pattern +gobblin.dataset.pattern=${gobblin.flow.edge.input.dataset.descriptor.path} +#Dataset finder class +gobblin.dataset.profile.class="org.apache.gobblin.data.management.retention.profile.SnapshotDatasetProfile" +# Dataset version finder +gobblin.retention.version.finder.class=${retention.version.finder.class} +## Setting the trash location, which will contain the files deleted during the retention +gobblin.trash.location="/tmp/${user.to.proxy}" \ No newline at end of file
