This is an automated email from the ASF dual-hosted git repository.
suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 4dad872 [GOBBLIN-1261] Migrate .pdsc schemas to .pdl
4dad872 is described below
commit 4dad87236bcf6c904a5c2b6e3afc65d44cc42dc8
Author: Jack Moseley <[email protected]>
AuthorDate: Tue Sep 15 15:50:21 2020 -0700
[GOBBLIN-1261] Migrate .pdsc schemas to .pdl
Closes #3101 from jack-moseley/pdl-migration
---
gobblin-rest-service/gobblin-rest-api/build.gradle | 3 +
.../org/apache/gobblin/rest/JobExecutionInfo.pdl | 84 ++++++++++++++++++
.../org/apache/gobblin/rest/JobExecutionInfo.pdsc | 99 ----------------------
.../org/apache/gobblin/rest/JobExecutionQuery.pdl | 69 +++++++++++++++
.../org/apache/gobblin/rest/JobExecutionQuery.pdsc | 84 ------------------
.../gobblin/rest/JobExecutionQueryResult.pdl | 12 +++
.../gobblin/rest/JobExecutionQueryResult.pdsc | 16 ----
.../pegasus/org/apache/gobblin/rest/Metric.pdl | 31 +++++++
.../pegasus/org/apache/gobblin/rest/Metric.pdsc | 32 -------
.../main/pegasus/org/apache/gobblin/rest/Table.pdl | 26 ++++++
.../pegasus/org/apache/gobblin/rest/Table.pdsc | 29 -------
.../org/apache/gobblin/rest/TaskExecutionInfo.pdl | 74 ++++++++++++++++
.../org/apache/gobblin/rest/TaskExecutionInfo.pdsc | 87 -------------------
.../pegasus/org/apache/gobblin/rest/TimeRange.pdl | 22 +++++
.../pegasus/org/apache/gobblin/rest/TimeRange.pdsc | 25 ------
.../gobblin-rest-server/build.gradle | 3 +
.../org/apache/gobblin/service/ExecutionStatus.pdl | 47 ++++++++++
.../apache/gobblin/service/ExecutionStatus.pdsc | 17 ----
.../org/apache/gobblin/service/FlowConfig.pdl | 33 ++++++++
.../org/apache/gobblin/service/FlowConfig.pdsc | 37 --------
.../org/apache/gobblin/service/FlowExecution.pdl | 32 +++++++
.../org/apache/gobblin/service/FlowExecution.pdsc | 33 --------
.../pegasus/org/apache/gobblin/service/FlowId.pdl | 25 ++++++
.../pegasus/org/apache/gobblin/service/FlowId.pdsc | 29 -------
.../org/apache/gobblin/service/FlowStatistics.pdl | 17 ++++
.../org/apache/gobblin/service/FlowStatistics.pdsc | 18 ----
.../org/apache/gobblin/service/FlowStatus.pdl | 33 ++++++++
.../org/apache/gobblin/service/FlowStatus.pdsc | 34 --------
.../org/apache/gobblin/service/FlowStatusId.pdl | 22 +++++
.../org/apache/gobblin/service/FlowStatusId.pdsc | 23 -----
.../pegasus/org/apache/gobblin/service/JobId.pdl | 25 ++++++
.../pegasus/org/apache/gobblin/service/JobId.pdsc | 29 -------
.../org/apache/gobblin/service/JobState.pdl | 17 ++++
.../org/apache/gobblin/service/JobState.pdsc | 18 ----
.../org/apache/gobblin/service/JobStatistics.pdl | 22 +++++
.../org/apache/gobblin/service/JobStatistics.pdsc | 23 -----
.../org/apache/gobblin/service/JobStatus.pdl | 42 +++++++++
.../org/apache/gobblin/service/JobStatus.pdsc | 44 ----------
.../org/apache/gobblin/service/Schedule.pdl | 18 ++++
.../org/apache/gobblin/service/Schedule.pdsc | 22 -----
.../gobblin/restli/throttling/PermitAllocation.pdl | 32 +++++++
.../restli/throttling/PermitAllocation.pdsc | 13 ---
.../gobblin/restli/throttling/PermitRequest.pdl | 32 +++++++
.../gobblin/restli/throttling/PermitRequest.pdsc | 13 ---
.../apache/gobblin/restli/throttling/Policy.pdl | 32 +++++++
.../apache/gobblin/restli/throttling/Policy.pdsc | 13 ---
gradle/scripts/defaultBuildProperties.gradle | 2 +-
gradle/scripts/repositories.gradle | 3 +
48 files changed, 757 insertions(+), 739 deletions(-)
diff --git a/gobblin-rest-service/gobblin-rest-api/build.gradle
b/gobblin-rest-service/gobblin-rest-api/build.gradle
index e792a26..02f26a6 100644
--- a/gobblin-rest-service/gobblin-rest-api/build.gradle
+++ b/gobblin-rest-service/gobblin-rest-api/build.gradle
@@ -133,6 +133,9 @@ buildscript {
repositories {
mavenCentral()
mavenLocal()
+ maven {
+ url "https://linkedin.bintray.com/maven";
+ }
}
dependencies {
diff --git
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/JobExecutionInfo.pdl
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/JobExecutionInfo.pdl
new file mode 100644
index 0000000..3c4a8f6
--- /dev/null
+++
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/JobExecutionInfo.pdl
@@ -0,0 +1,84 @@
+namespace org.apache.gobblin.rest
+
+/**
+ * Gobblin job execution information
+ */
+record JobExecutionInfo {
+
+ /**
+ * Job name
+ */
+ jobName: string
+
+ /**
+ * Job ID
+ */
+ jobId: string
+
+ /**
+ * Job start time (epoch time)
+ */
+ startTime: optional long
+
+ /**
+ * Job end time (epoch time)
+ */
+ endTime: optional long
+
+ /**
+ * Job duration in milliseconds
+ */
+ duration: optional long
+
+ /**
+ * Job state
+ */
+ state: optional enum JobStateEnum {
+ PENDING
+ RUNNING
+ SUCCESSFUL
+ COMMITTED
+ FAILED
+ CANCELLED
+ }
+
+ /**
+ * Number of launched tasks
+ */
+ launchedTasks: optional int
+
+ /**
+ * Number of completed tasks
+ */
+ completedTasks: optional int
+
+ /**
+ * Job launcher type
+ */
+ launcherType: optional enum LauncherTypeEnum {
+ LOCAL
+ MAPREDUCE
+ CLUSTER
+ YARN
+ }
+
+ /**
+ * Job execution tracking URL
+ */
+ trackingUrl: optional string
+
+ /**
+ * An array of task execution information records
+ */
+ taskExecutions: array[TaskExecutionInfo]
+
+ /**
+ * Job properties
+ */
+ jobProperties: optional map[string, string]
+
+ /**
+ * Job metrics
+ */
+ metrics: array[Metric]
+}
\ No newline at end of file
diff --git
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/JobExecutionInfo.pdsc
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/JobExecutionInfo.pdsc
deleted file mode 100644
index 24c34a8..0000000
---
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/JobExecutionInfo.pdsc
+++ /dev/null
@@ -1,99 +0,0 @@
-{
- "type": "record",
- "name": "JobExecutionInfo",
- "namespace" : "org.apache.gobblin.rest",
- "doc": "Gobblin job execution information",
- "fields": [
- {
- "name": "jobName",
- "type": "string",
- "doc": "Job name"
- },
- {
- "name": "jobId",
- "type": "string",
- "doc": "Job ID"
- },
- {
- "name": "startTime",
- "type": "long",
- "optional": true,
- "doc": "Job start time (epoch time)"
- },
- {
- "name": "endTime",
- "type": "long",
- "optional": true,
- "doc": "Job end time (epoch time)"
- },
- {
- "name": "duration",
- "type": "long",
- "optional": true,
- "doc": "Job duration in milliseconds"
- },
- {
- "name": "state",
- "type": {
- "name": "JobStateEnum",
- "type": "enum",
- "symbols" : [ "PENDING", "RUNNING", "SUCCESSFUL", "COMMITTED",
"FAILED", "CANCELLED" ]
- },
- "optional": true,
- "doc": "Job state"
- },
- {
- "name": "launchedTasks",
- "type": "int",
- "optional": true,
- "doc": "Number of launched tasks"
- },
- {
- "name": "completedTasks",
- "type": "int",
- "optional": true,
- "doc": "Number of completed tasks"
- },
- {
- "name": "launcherType",
- "type": {
- "name": "LauncherTypeEnum",
- "type": "enum",
- "symbols" : [ "LOCAL", "MAPREDUCE", "CLUSTER", "YARN" ]
- },
- "optional": true,
- "doc": "Job launcher type"
- },
- {
- "name": "trackingUrl",
- "type": "string",
- "optional": true,
- "doc": "Job execution tracking URL"
- },
- {
- "name": "taskExecutions",
- "type": {
- "type": "array",
- "items": "TaskExecutionInfo"
- },
- "doc": "An array of task execution information records"
- },
- {
- "name": "jobProperties",
- "type": {
- "type": "map",
- "values": "string"
- },
- "optional": true,
- "doc": "Job properties"
- },
- {
- "name": "metrics",
- "type": {
- "type": "array",
- "items": "Metric"
- },
- "doc": "Job metrics"
- }
- ]
-}
diff --git
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/JobExecutionQuery.pdl
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/JobExecutionQuery.pdl
new file mode 100644
index 0000000..d4c0098
--- /dev/null
+++
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/JobExecutionQuery.pdl
@@ -0,0 +1,69 @@
+namespace org.apache.gobblin.rest
+
+/**
+ * Gobblin job execution query
+ */
+record JobExecutionQuery {
+
+ /**
+ * Query ID (a job ID, job name, table definition, or list type)
+ */
+ id: union[string, Table,
+ /**
+ * Listing distinct jobs fetches the most recent execution info for each
job name in the database. Listing recent jobs fetches the most recent job
executions, regardless of job name.
+ */
+ enum QueryListType {
+ DISTINCT
+ RECENT
+ }]
+
+ /**
+ * Query ID type
+ */
+ idType: enum QueryIdTypeEnum {
+ JOB_NAME
+ JOB_ID
+ TABLE
+ LIST_TYPE
+ }
+
+ /**
+ * Query time range
+ */
+ timeRange: optional TimeRange
+
+ /**
+ * Query limit
+ */
+ limit: optional int = 1
+
+ /**
+ * Comma-separated list of job properties to include in the query result
+ */
+ jobProperties: optional string
+
+ /**
+ * Comma-separated list of task properties to include in the query result
+ */
+ taskProperties: optional string
+
+ /**
+ * true/false if the response should include job metrics (default: true)
+ */
+ includeJobMetrics: optional boolean = true
+
+ /**
+ * true/false if the response should include task metrics (default: true)
+ */
+ includeTaskMetrics: optional boolean = true
+
+ /**
+ * true/false if the response should include task executions (default: true)
+ */
+ includeTaskExecutions: optional boolean = true
+
+ /**
+ * true/false if the response should include jobs that did not launch tasks
(default: true)
+ */
+ includeJobsWithoutTasks: optional boolean = true
+}
\ No newline at end of file
diff --git
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/JobExecutionQuery.pdsc
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/JobExecutionQuery.pdsc
deleted file mode 100644
index baf7278..0000000
---
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/JobExecutionQuery.pdsc
+++ /dev/null
@@ -1,84 +0,0 @@
-{
- "type": "record",
- "name": "JobExecutionQuery",
- "namespace" : "org.apache.gobblin.rest",
- "doc": "Gobblin job execution query",
- "fields": [
- {
- "name": "id",
- "type": [
- "string",
- "Table",
- {
- "name": "QueryListType",
- "type": "enum",
- "symbols" : [ "DISTINCT", "RECENT" ],
- "doc": "Listing distinct jobs fetches the most recent
execution info for each job name in the database. Listing recent jobs fetches
the most recent job executions, regardless of job name."
- }
- ],
- "doc": "Query ID (a job ID, job name, table definition, or list
type)"
- },
- {
- "name": "idType",
- "type": {
- "name": "QueryIdTypeEnum",
- "type": "enum",
- "symbols" : [ "JOB_NAME", "JOB_ID", "TABLE", "LIST_TYPE" ]
- },
- "doc": "Query ID type"
- },
- {
- "name": "timeRange",
- "type": "TimeRange",
- "optional": true,
- "doc": "Query time range"
- },
- {
- "name": "limit",
- "type": "int",
- "optional": true,
- "default": 1,
- "doc": "Query limit"
- },
- {
- "name": "jobProperties",
- "type": "string",
- "optional": true,
- "doc": "Comma-separated list of job properties to include in the
query result"
- },
- {
- "name": "taskProperties",
- "type": "string",
- "optional": true,
- "doc": "Comma-separated list of task properties to include in the
query result"
- },
- {
- "name": "includeJobMetrics",
- "type": "boolean",
- "optional": true,
- "default": true,
- "doc": "true/false if the response should include job metrics
(default: true)"
- },
- {
- "name": "includeTaskMetrics",
- "type": "boolean",
- "optional": true,
- "default": true,
- "doc": "true/false if the response should include task metrics
(default: true)"
- },
- {
- "name": "includeTaskExecutions",
- "type": "boolean",
- "optional": true,
- "default": true,
- "doc": "true/false if the response should include task executions
(default: true)"
- },
- {
- "name": "includeJobsWithoutTasks",
- "type": "boolean",
- "optional": true,
- "default": true,
- "doc": "true/false if the response should include jobs that did not
launch tasks (default: true)"
- }
- ]
-}
\ No newline at end of file
diff --git
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/JobExecutionQueryResult.pdl
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/JobExecutionQueryResult.pdl
new file mode 100644
index 0000000..72a8d94
--- /dev/null
+++
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/JobExecutionQueryResult.pdl
@@ -0,0 +1,12 @@
+namespace org.apache.gobblin.rest
+
+/**
+ * Gobblin job execution query result
+ */
+record JobExecutionQueryResult {
+
+ /**
+ * An array of job execution information records
+ */
+ jobExecutions: array[JobExecutionInfo]
+}
\ No newline at end of file
diff --git
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/JobExecutionQueryResult.pdsc
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/JobExecutionQueryResult.pdsc
deleted file mode 100644
index b1921b9..0000000
---
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/JobExecutionQueryResult.pdsc
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "type": "record",
- "name": "JobExecutionQueryResult",
- "namespace" : "org.apache.gobblin.rest",
- "doc": "Gobblin job execution query result",
- "fields": [
- {
- "name": "jobExecutions",
- "type": {
- "type": "array",
- "items": "JobExecutionInfo"
- },
- "doc": "An array of job execution information records"
- }
- ]
-}
\ No newline at end of file
diff --git
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/Metric.pdl
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/Metric.pdl
new file mode 100644
index 0000000..58afc8f
--- /dev/null
+++
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/Metric.pdl
@@ -0,0 +1,31 @@
+namespace org.apache.gobblin.rest
+
+/**
+ * Gobblin metric
+ */
+record Metric {
+
+ /**
+ * Metric group
+ */
+ group: string
+
+ /**
+ * Metric name
+ */
+ name: string
+
+ /**
+ * Metric type
+ */
+ type: enum MetricTypeEnum {
+ COUNTER
+ METER
+ GAUGE
+ }
+
+ /**
+ * Metric value
+ */
+ value: string
+}
\ No newline at end of file
diff --git
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/Metric.pdsc
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/Metric.pdsc
deleted file mode 100644
index 6510f3d..0000000
---
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/Metric.pdsc
+++ /dev/null
@@ -1,32 +0,0 @@
-{
- "type": "record",
- "name": "Metric",
- "namespace" : "org.apache.gobblin.rest",
- "doc": "Gobblin metric",
- "fields": [
- {
- "name": "group",
- "type": "string",
- "doc": "Metric group"
- },
- {
- "name": "name",
- "type": "string",
- "doc": "Metric name"
- },
- {
- "name": "type",
- "type": {
- "name": "MetricTypeEnum",
- "type": "enum",
- "symbols" : [ "COUNTER", "METER", "GAUGE" ]
- },
- "doc": "Metric type"
- },
- {
- "name": "value",
- "type": "string",
- "doc": "Metric value"
- }
- ]
-}
\ No newline at end of file
diff --git
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/Table.pdl
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/Table.pdl
new file mode 100644
index 0000000..567e0a0
--- /dev/null
+++
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/Table.pdl
@@ -0,0 +1,26 @@
+namespace org.apache.gobblin.rest
+
+/**
+ * Gobblin table definition
+ */
+record Table {
+
+ /**
+ * Table namespace
+ */
+ `namespace`: optional string
+
+ /**
+ * Table name
+ */
+ name: string
+
+ /**
+ * Table type
+ */
+ type: optional enum TableTypeEnum {
+ SNAPSHOT_ONLY
+ SNAPSHOT_APPEND
+ APPEND_ONLY
+ }
+}
\ No newline at end of file
diff --git
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/Table.pdsc
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/Table.pdsc
deleted file mode 100644
index 58f04ec..0000000
---
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/Table.pdsc
+++ /dev/null
@@ -1,29 +0,0 @@
-{
- "type": "record",
- "name": "Table",
- "namespace" : "org.apache.gobblin.rest",
- "doc": "Gobblin table definition",
- "fields": [
- {
- "name": "namespace",
- "type": "string",
- "optional": true,
- "doc": "Table namespace"
- },
- {
- "name": "name",
- "type": "string",
- "doc": "Table name"
- },
- {
- "name": "type",
- "type": {
- "name": "TableTypeEnum",
- "type": "enum",
- "symbols" : [ "SNAPSHOT_ONLY", "SNAPSHOT_APPEND", "APPEND_ONLY" ]
- },
- "optional": true,
- "doc": "Table type"
- }
- ]
-}
\ No newline at end of file
diff --git
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/TaskExecutionInfo.pdl
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/TaskExecutionInfo.pdl
new file mode 100644
index 0000000..adbd80b
--- /dev/null
+++
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/TaskExecutionInfo.pdl
@@ -0,0 +1,74 @@
+namespace org.apache.gobblin.rest
+
+/**
+ * Gobblin task execution information
+ */
+record TaskExecutionInfo {
+
+ /**
+ * Job ID
+ */
+ jobId: string
+
+ /**
+ * Task ID
+ */
+ taskId: string
+
+ /**
+ * Task start time (epoch time)
+ */
+ startTime: optional long
+
+ /**
+ * Task end time (epoch time)
+ */
+ endTime: optional long
+
+ /**
+ * Task duration in milliseconds
+ */
+ duration: optional long
+
+ /**
+ * Task state
+ */
+ state: optional enum TaskStateEnum {
+ PENDING
+ RUNNING
+ SUCCESSFUL
+ COMMITTED
+ FAILED
+ CANCELLED
+ }
+
+ /**
+ * Task failure exception message
+ */
+ failureException: optional string
+
+ /**
+ * Low watermark
+ */
+ lowWatermark: optional long
+
+ /**
+ * High watermark
+ */
+ highWatermark: optional long
+
+ /**
+ * Table definition
+ */
+ table: optional Table
+
+ /**
+ * Task properties
+ */
+ taskProperties: optional map[string, string]
+
+ /**
+ * Job metrics
+ */
+ metrics: array[Metric]
+}
\ No newline at end of file
diff --git
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/TaskExecutionInfo.pdsc
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/TaskExecutionInfo.pdsc
deleted file mode 100644
index afdd188..0000000
---
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/TaskExecutionInfo.pdsc
+++ /dev/null
@@ -1,87 +0,0 @@
-{
- "type": "record",
- "name": "TaskExecutionInfo",
- "namespace" : "org.apache.gobblin.rest",
- "doc": "Gobblin task execution information",
- "fields": [
- {
- "name": "jobId",
- "type": "string",
- "doc": "Job ID"
- },
- {
- "name": "taskId",
- "type": "string",
- "doc": "Task ID"
- },
- {
- "name": "startTime",
- "type": "long",
- "optional": true,
- "doc": "Task start time (epoch time)"
- },
- {
- "name": "endTime",
- "type": "long",
- "optional": true,
- "doc": "Task end time (epoch time)"
- },
- {
- "name": "duration",
- "type": "long",
- "optional": true,
- "doc": "Task duration in milliseconds"
- },
- {
- "name": "state",
- "type": {
- "name": "TaskStateEnum",
- "type": "enum",
- "symbols" : [ "PENDING", "RUNNING", "SUCCESSFUL", "COMMITTED",
"FAILED", "CANCELLED" ]
- },
- "optional": true,
- "doc": "Task state"
- },
- {
- "name": "failureException",
- "type": "string",
- "optional": true,
- "doc": "Task failure exception message"
- },
- {
- "name": "lowWatermark",
- "type": "long",
- "optional": true,
- "doc": "Low watermark"
- },
- {
- "name": "highWatermark",
- "type": "long",
- "optional": true,
- "doc": "High watermark"
- },
- {
- "name": "table",
- "type": "Table",
- "optional": true,
- "doc": "Table definition"
- },
- {
- "name": "taskProperties",
- "type": {
- "type": "map",
- "values": "string"
- },
- "optional": true,
- "doc": "Task properties"
- },
- {
- "name": "metrics",
- "type": {
- "type": "array",
- "items": "Metric"
- },
- "doc": "Job metrics"
- }
- ]
-}
\ No newline at end of file
diff --git
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/TimeRange.pdl
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/TimeRange.pdl
new file mode 100644
index 0000000..c343701
--- /dev/null
+++
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/TimeRange.pdl
@@ -0,0 +1,22 @@
+namespace org.apache.gobblin.rest
+
+/**
+ * Query time range
+ */
+record TimeRange {
+
+ /**
+ * Start time of the query range
+ */
+ startTime: optional string
+
+ /**
+ * End time of the query range
+ */
+ endTime: optional string
+
+ /**
+ * Date/time format used to parse the start time and end time
+ */
+ timeFormat: string
+}
\ No newline at end of file
diff --git
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/TimeRange.pdsc
b/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/TimeRange.pdsc
deleted file mode 100644
index 86ad3e0..0000000
---
a/gobblin-rest-service/gobblin-rest-api/src/main/pegasus/org/apache/gobblin/rest/TimeRange.pdsc
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "type": "record",
- "name": "TimeRange",
- "namespace" : "org.apache.gobblin.rest",
- "doc": "Query time range",
- "fields": [
- {
- "name": "startTime",
- "type": "string",
- "optional": true,
- "doc": "Start time of the query range"
- },
- {
- "name": "endTime",
- "type": "string",
- "optional": true,
- "doc": "End time of the query range"
- },
- {
- "name": "timeFormat",
- "type": "string",
- "doc": "Date/time format used to parse the start time and end time"
- }
- ]
-}
\ No newline at end of file
diff --git a/gobblin-rest-service/gobblin-rest-server/build.gradle
b/gobblin-rest-service/gobblin-rest-server/build.gradle
index 9b99d29..66899d4 100644
--- a/gobblin-rest-service/gobblin-rest-server/build.gradle
+++ b/gobblin-rest-service/gobblin-rest-server/build.gradle
@@ -55,6 +55,9 @@ buildscript {
repositories {
mavenCentral()
mavenLocal()
+ maven {
+ url "https://linkedin.bintray.com/maven";
+ }
}
dependencies {
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/ExecutionStatus.pdl
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/ExecutionStatus.pdl
new file mode 100644
index 0000000..c0e63a9
--- /dev/null
+++
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/ExecutionStatus.pdl
@@ -0,0 +1,47 @@
+namespace org.apache.gobblin.service
+
+/**
+ * Execution status for a flow or job
+ */
+enum ExecutionStatus {
+
+ /**
+ * Flow compiled to jobs.
+ */
+ COMPILED
+
+ /**
+ * Flow or job is in pending state.
+ */
+ PENDING
+
+ /**
+ * Flow or job is pending retry.
+ */
+ PENDING_RETRY
+
+ /**
+ * Job(s) orchestrated to spec executors.
+ */
+ ORCHESTRATED
+
+ /**
+ * Flow or job is currently executing
+ */
+ RUNNING
+
+ /**
+ * Flow or job completed execution
+ */
+ COMPLETE
+
+ /**
+ * Flow or job failed
+ */
+ FAILED
+
+ /**
+ * Flow cancelled.
+ */
+ CANCELLED
+}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/ExecutionStatus.pdsc
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/ExecutionStatus.pdsc
deleted file mode 100644
index 3040eab..0000000
---
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/ExecutionStatus.pdsc
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "type" : "enum",
- "name" : "ExecutionStatus",
- "namespace" : "org.apache.gobblin.service",
- "doc" : "Execution status for a flow or job",
- "symbols" : ["COMPILED", "PENDING", "PENDING_RETRY", "ORCHESTRATED",
"RUNNING", "COMPLETE", "FAILED", "CANCELLED" ],
- "symbolDocs" : {
- "COMPILED":"Flow compiled to jobs.",
- "PENDING":"Flow or job is in pending state.",
- "PENDING_RETRY":"Flow or job is pending retry.",
- "ORCHESTRATED":"Job(s) orchestrated to spec executors.",
- "RUNNING": "Flow or job is currently executing",
- "COMPLETE":"Flow or job completed execution",
- "FAILED":"Flow or job failed",
- "CANCELLED":"Flow cancelled."
- }
-}
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowConfig.pdl
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowConfig.pdl
new file mode 100644
index 0000000..8459b32
--- /dev/null
+++
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowConfig.pdl
@@ -0,0 +1,33 @@
+namespace org.apache.gobblin.service
+
+/**
+ * Defines a flow configuration that can be compiled into Gobblin jobs
+ */
+record FlowConfig {
+
+ /**
+ * Identifier for the flow
+ */
+ id: FlowId
+
+ /**
+ * Optional schedule for when to execution the flow. If a schedule is not
specified then the flow is executed immediately.
+ */
+ schedule: optional Schedule
+
+ /**
+ * Comma separated list of URIs for templates used in the flow. The template
location is defined by the multiproduct that packages the template.
+ */
+ @validate.`org.apache.gobblin.service.validator.TemplateUriValidator` = { }
+ templateUris: string
+
+ /**
+ * Return the compiled flow as a string. If enabled, the flow is not added.
+ */
+ explain: boolean = false
+
+ /**
+ * Properties for the flow. These properties are passed to the compiled
Gobblin jobs.
+ */
+ properties: map[string, string]
+}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowConfig.pdsc
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowConfig.pdsc
deleted file mode 100644
index 09af4f1..0000000
---
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowConfig.pdsc
+++ /dev/null
@@ -1,37 +0,0 @@
-{
- "type" : "record",
- "name" : "FlowConfig",
- "namespace" : "org.apache.gobblin.service",
- "doc" : "Defines a flow configuration that can be compiled into Gobblin
jobs",
- "fields" : [
- {
- "name" : "id",
- "type" : "FlowId",
- "doc" : "Identifier for the flow"
- },
- {
- "name" : "schedule",
- "type" : "Schedule",
- "optional" : true,
- "doc" : "Optional schedule for when to execution the flow. If a schedule
is not specified then the flow is executed immediately."
- },
- {
- "name" : "templateUris",
- "type" : "string",
- "doc" : "Comma separated list of URIs for templates used in the flow.
The template location is defined by the multiproduct that packages the
template.",
- "validate": {
- "org.apache.gobblin.service.validator.TemplateUriValidator": {}
- }
- },
- {
- "name" : "explain",
- "type" : "boolean",
- "default" : false,
- "doc" : "Return the compiled flow as a string. If enabled, the flow is
not added."
- },
- {
- "name" : "properties",
- "type" : { "type" : "map", "values" : "string" },
- "doc" : "Properties for the flow. These properties are passed to the
compiled Gobblin jobs."
- } ]
-}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowExecution.pdl
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowExecution.pdl
new file mode 100644
index 0000000..791c6d4
--- /dev/null
+++
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowExecution.pdl
@@ -0,0 +1,32 @@
+namespace org.apache.gobblin.service
+
+/**
+ * Represents an execution of a flow
+ */
+record FlowExecution {
+
+ /**
+ * Flow status identifier
+ */
+ id: FlowStatusId
+
+ /**
+ * Statistics for this execution of the flow
+ */
+ executionStatistics: FlowStatistics
+
+ /**
+ * Flow execution status
+ */
+ executionStatus: ExecutionStatus
+
+ /**
+ * Error or status message
+ */
+ message: string
+
+ /**
+ * Status of jobs belonging to the flow
+ */
+ jobStatuses: array[JobStatus]
+}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowExecution.pdsc
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowExecution.pdsc
deleted file mode 100644
index 2736366..0000000
---
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowExecution.pdsc
+++ /dev/null
@@ -1,33 +0,0 @@
-{
- "type" : "record",
- "name" : "FlowExecution",
- "namespace" : "org.apache.gobblin.service",
- "doc" : "Represents an execution of a flow",
- "fields" : [
- {
- "name" : "id",
- "type" : "FlowStatusId",
- "doc" : "Flow status identifier"
- },
- {
- "name" : "executionStatistics",
- "type" : "FlowStatistics",
- "doc" : "Statistics for this execution of the flow"
- },
- {
- "name" : "executionStatus",
- "type" : "ExecutionStatus",
- "doc" : "Flow execution status"
- },
- {
- "name" : "message",
- "type" : "string",
- "doc" : "Error or status message"
- },
- {
- "name" : "jobStatuses",
- "type" : {"type" : "array", "items" : "JobStatus"},
- "doc" : "Status of jobs belonging to the flow"
- }
- ]
-}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowId.pdl
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowId.pdl
new file mode 100644
index 0000000..414515a
--- /dev/null
+++
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowId.pdl
@@ -0,0 +1,25 @@
+namespace org.apache.gobblin.service
+
+/**
+ * Identifier for a Gobblin as a Service flow
+ */
+record FlowId {
+
+ /**
+ * Name of the flow
+ */
+ @validate.strlen = {
+ "max" : 128,
+ "min" : 1
+ }
+ flowName: string
+
+ /**
+ * Group of the flow. This defines the namespace for the flow.
+ */
+ @validate.strlen = {
+ "max" : 128,
+ "min" : 1
+ }
+ flowGroup: string
+}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowId.pdsc
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowId.pdsc
deleted file mode 100644
index e079598..0000000
---
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowId.pdsc
+++ /dev/null
@@ -1,29 +0,0 @@
-{
- "type" : "record",
- "name" : "FlowId",
- "namespace" : "org.apache.gobblin.service",
- "doc" : "Identifier for a Gobblin as a Service flow",
- "fields" : [
- {
- "name" : "flowName",
- "type" : "string",
- "doc" : "Name of the flow",
- "validate": {
- "strlen": {
- "min": 1,
- "max": 128
- }
- }
- },
- {
- "name" : "flowGroup",
- "type" : "string",
- "doc" : "Group of the flow. This defines the namespace for the flow.",
- "validate": {
- "strlen": {
- "min": 1,
- "max": 128
- }
- }
- } ]
-}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowStatistics.pdl
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowStatistics.pdl
new file mode 100644
index 0000000..c910c76
--- /dev/null
+++
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowStatistics.pdl
@@ -0,0 +1,17 @@
+namespace org.apache.gobblin.service
+
+/**
+ * Statistics from a flow execution
+ */
+record FlowStatistics {
+
+ /**
+ * Epoch time of when the execution began
+ */
+ executionStartTime: long
+
+ /**
+ * Epoch time of when the execution ended
+ */
+ executionEndTime: long
+}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowStatistics.pdsc
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowStatistics.pdsc
deleted file mode 100644
index a900200..0000000
---
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowStatistics.pdsc
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "type" : "record",
- "name" : "FlowStatistics",
- "namespace" : "org.apache.gobblin.service",
- "doc" : "Statistics from a flow execution",
- "fields" : [
- {
- "name" : "executionStartTime",
- "type" : "long",
- "doc" : "Epoch time of when the execution began"
- },
- {
- "name" : "executionEndTime",
- "type" : "long",
- "doc" : "Epoch time of when the execution ended"
- }
- ]
-}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowStatus.pdl
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowStatus.pdl
new file mode 100644
index 0000000..bf411f4
--- /dev/null
+++
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowStatus.pdl
@@ -0,0 +1,33 @@
+namespace org.apache.gobblin.service
+
+/**
+ * Status of a flow
+ */
+@deprecated = "Use FlowExecution instead"
+record FlowStatus {
+
+ /**
+ * Flow status identifier
+ */
+ id: FlowStatusId
+
+ /**
+ * Statistics for this execution of the flow
+ */
+ executionStatistics: FlowStatistics
+
+ /**
+ * Flow execution status
+ */
+ executionStatus: ExecutionStatus
+
+ /**
+ * Error or status message
+ */
+ message: string
+
+ /**
+ * Status of jobs belonging to the flow
+ */
+ jobStatuses: array[JobStatus]
+}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowStatus.pdsc
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowStatus.pdsc
deleted file mode 100644
index 165100f..0000000
---
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowStatus.pdsc
+++ /dev/null
@@ -1,34 +0,0 @@
-{
- "type" : "record",
- "name" : "FlowStatus",
- "namespace" : "org.apache.gobblin.service",
- "doc" : "Status of a flow",
- "deprecated" : "Use FlowExecution instead",
- "fields" : [
- {
- "name" : "id",
- "type" : "FlowStatusId",
- "doc" : "Flow status identifier"
- },
- {
- "name" : "executionStatistics",
- "type" : "FlowStatistics",
- "doc" : "Statistics for this execution of the flow"
- },
- {
- "name" : "executionStatus",
- "type" : "ExecutionStatus",
- "doc" : "Flow execution status"
- },
- {
- "name" : "message",
- "type" : "string",
- "doc" : "Error or status message"
- },
- {
- "name" : "jobStatuses",
- "type" : {"type" : "array", "items" : "JobStatus"},
- "doc" : "Status of jobs belonging to the flow"
- }
- ]
-}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowStatusId.pdl
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowStatusId.pdl
new file mode 100644
index 0000000..9f65de7
--- /dev/null
+++
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowStatusId.pdl
@@ -0,0 +1,22 @@
+namespace org.apache.gobblin.service
+
+/**
+ * Identifier for a specific execution of a flow
+ */
+record FlowStatusId {
+
+ /**
+ * Name of the flow
+ */
+ flowName: string
+
+ /**
+ * Group of the flow. This defines the namespace for the flow.
+ */
+ flowGroup: string
+
+ /**
+ * Execution id for the flow
+ */
+ flowExecutionId: long
+}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowStatusId.pdsc
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowStatusId.pdsc
deleted file mode 100644
index f5a0344..0000000
---
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/FlowStatusId.pdsc
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "type" : "record",
- "name" : "FlowStatusId",
- "namespace" : "org.apache.gobblin.service",
- "doc" : "Identifier for a specific execution of a flow",
- "fields" : [
- {
- "name" : "flowName",
- "type" : "string",
- "doc" : "Name of the flow"
- },
- {
- "name" : "flowGroup",
- "type" : "string",
- "doc" : "Group of the flow. This defines the namespace for the flow."
- },
- {
- "name" : "flowExecutionId",
- "type" : "long",
- "doc" : "Execution id for the flow"
- }
- ]
-}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobId.pdl
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobId.pdl
new file mode 100644
index 0000000..1647f2a
--- /dev/null
+++
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobId.pdl
@@ -0,0 +1,25 @@
+namespace org.apache.gobblin.service
+
+/**
+ * Identifier for a Gobblin job
+ */
+record JobId {
+
+ /**
+ * Name of the job
+ */
+ @validate.strlen = {
+ "max" : 600,
+ "min" : 1
+ }
+ jobName: string
+
+ /**
+ * Group of the job. This defines the namespace for the job.
+ */
+ @validate.strlen = {
+ "max" : 128,
+ "min" : 1
+ }
+ jobGroup: string
+}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobId.pdsc
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobId.pdsc
deleted file mode 100644
index b1aeedd..0000000
---
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobId.pdsc
+++ /dev/null
@@ -1,29 +0,0 @@
-{
- "type" : "record",
- "name" : "JobId",
- "namespace" : "org.apache.gobblin.service",
- "doc" : "Identifier for a Gobblin job",
- "fields" : [
- {
- "name" : "jobName",
- "type" : "string",
- "doc" : "Name of the job",
- "validate": {
- "strlen": {
- "min": 1,
- "max": 600
- }
- }
- },
- {
- "name" : "jobGroup",
- "type" : "string",
- "doc" : "Group of the job. This defines the namespace for the job.",
- "validate": {
- "strlen": {
- "min": 1,
- "max": 128
- }
- }
- } ]
-}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobState.pdl
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobState.pdl
new file mode 100644
index 0000000..1606ab0
--- /dev/null
+++
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobState.pdl
@@ -0,0 +1,17 @@
+namespace org.apache.gobblin.service
+
+/**
+ * Job state that is updated at the end of an execution
+ */
+record JobState {
+
+ /**
+ * Low watermark after last execution
+ */
+ lowWatermark: string
+
+ /**
+ * High watermark after last execution
+ */
+ highWatermark: string
+}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobState.pdsc
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobState.pdsc
deleted file mode 100644
index 6a225b1..0000000
---
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobState.pdsc
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "type" : "record",
- "name" : "JobState",
- "namespace" : "org.apache.gobblin.service",
- "doc" : "Job state that is updated at the end of an execution",
- "fields" : [
- {
- "name" : "lowWatermark",
- "type" : "string",
- "doc" : "Low watermark after last execution"
- },
- {
- "name" : "highWatermark",
- "type" : "string",
- "doc" : "High watermark after last execution"
- }
- ]
-}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobStatistics.pdl
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobStatistics.pdl
new file mode 100644
index 0000000..4869f50
--- /dev/null
+++
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobStatistics.pdl
@@ -0,0 +1,22 @@
+namespace org.apache.gobblin.service
+
+/**
+ * Statistics from a job execution
+ */
+record JobStatistics {
+
+ /**
+ * Epoch time of when the execution began
+ */
+ executionStartTime: long
+
+ /**
+ * Epoch time of when the execution ended
+ */
+ executionEndTime: long
+
+ /**
+ * number of records processed in the last job execution
+ */
+ processedCount: long
+}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobStatistics.pdsc
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobStatistics.pdsc
deleted file mode 100644
index e48c83c..0000000
---
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobStatistics.pdsc
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "type" : "record",
- "name" : "JobStatistics",
- "namespace" : "org.apache.gobblin.service",
- "doc" : "Statistics from a job execution",
- "fields" : [
- {
- "name" : "executionStartTime",
- "type" : "long",
- "doc" : "Epoch time of when the execution began"
- },
- {
- "name" : "executionEndTime",
- "type" : "long",
- "doc" : "Epoch time of when the execution ended"
- },
- {
- "name" : "processedCount",
- "type" : "long",
- "doc" : "number of records processed in the last job execution"
- }
- ]
-}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobStatus.pdl
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobStatus.pdl
new file mode 100644
index 0000000..f063dae
--- /dev/null
+++
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobStatus.pdl
@@ -0,0 +1,42 @@
+namespace org.apache.gobblin.service
+
+/**
+ * Execution status for a job
+ */
+record JobStatus {
+
+ /**
+ * Identifier of the flow the job belongs to
+ */
+ flowId: FlowId
+
+ /**
+ * Identifier of the job
+ */
+ jobId: JobId
+
+ /**
+ * Tag of the job
+ */
+ jobTag: optional string
+
+ /**
+ * Job execution status
+ */
+ executionStatus: ExecutionStatus
+
+ /**
+ * Error or status message
+ */
+ message: string
+
+ /**
+ * Statistics from the job execution. The values may be updated during the
run of a job.
+ */
+ executionStatistics: JobStatistics
+
+ /**
+ * Job state that is updated only at the start and end of a job execution.
+ */
+ jobState: JobState
+}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobStatus.pdsc
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobStatus.pdsc
deleted file mode 100644
index f13d366..0000000
---
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/JobStatus.pdsc
+++ /dev/null
@@ -1,44 +0,0 @@
-{
- "type" : "record",
- "name" : "JobStatus",
- "namespace" : "org.apache.gobblin.service",
- "doc" : "Execution status for a job",
- "fields" : [
- {
- "name" : "flowId",
- "type" : "FlowId",
- "doc" : "Identifier of the flow the job belongs to"
- },
- {
- "name" : "jobId",
- "type" : "JobId",
- "doc" : "Identifier of the job"
- },
- {
- "name" : "jobTag",
- "type" : "string",
- "optional" : true,
- "doc" : "Tag of the job"
- },
- {
- "name" : "executionStatus",
- "type" : "ExecutionStatus",
- "doc" : "Job execution status"
- },
- {
- "name" : "message",
- "type" : "string",
- "doc" : "Error or status message"
- },
- {
- "name" : "executionStatistics",
- "type" : "JobStatistics",
- "doc" : "Statistics from the job execution. The values may be updated
during the run of a job."
- },
- {
- "name" : "jobState",
- "type" : "JobState",
- "doc" : "Job state that is updated only at the start and end of a job
execution."
- }
- ]
-}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/Schedule.pdl
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/Schedule.pdl
new file mode 100644
index 0000000..e84d35e
--- /dev/null
+++
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/Schedule.pdl
@@ -0,0 +1,18 @@
+namespace org.apache.gobblin.service
+
+/**
+ * Attributes for defining a job schedule
+ */
+record Schedule {
+
+ /**
+ * Schedule for flow in cron format
+ */
+ @validate.`org.apache.gobblin.service.validator.CronValidator` = { }
+ cronSchedule: string
+
+ /**
+ * Set to true to request that a job with a schedule be run immediately in
addition to being scheduled
+ */
+ runImmediately: boolean = false
+}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/Schedule.pdsc
b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/Schedule.pdsc
deleted file mode 100644
index 86f78b7..0000000
---
a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-api/src/main/pegasus/org/apache/gobblin/service/Schedule.pdsc
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "type" : "record",
- "name" : "Schedule",
- "namespace" : "org.apache.gobblin.service",
- "doc" : "Attributes for defining a job schedule",
- "fields" : [
- {
- "name" : "cronSchedule",
- "type" : "string",
- "doc" : "Schedule for flow in cron format",
- "validate": {
- "org.apache.gobblin.service.validator.CronValidator": {}
- }
- },
- {
- "name" : "runImmediately",
- "type" : "boolean",
- "default" : false,
- "doc" : "Set to true to request that a job with a schedule be run
immediately in addition to being scheduled"
- }
- ]
-}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-api/src/main/pegasus/org/apache/gobblin/restli/throttling/PermitAllocation.pdl
b/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-api/src/main/pegasus/org/apache/gobblin/restli/throttling/PermitAllocation.pdl
new file mode 100644
index 0000000..7af103c
--- /dev/null
+++
b/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-api/src/main/pegasus/org/apache/gobblin/restli/throttling/PermitAllocation.pdl
@@ -0,0 +1,32 @@
+namespace org.apache.gobblin.restli.throttling
+
+/**
+ * Used by gobblin-throttling-service to allocate permits to a client.
+ */
+record PermitAllocation {
+
+ /**
+ * Number of permits allocated. This may be 0 if no permits are allocated,
or the number of requested permits.
+ */
+ permits: long
+
+ /**
+ * Expiration time in Unix timestamp of the allocated permits.
+ */
+ expiration: long
+
+ /**
+ * Client should not try to acquire permits before this delay has passed.
+ */
+ minRetryDelayMillis: optional long
+
+ /**
+ * Client must wait this many millis before allocating provided permits.
+ */
+ waitForPermitUseMillis: optional long = 0
+
+ /**
+ * If larger than 0, specifies request larger than this number are
impossible to satisfy by the policy.
+ */
+ unsatisfiablePermits: optional long = 0
+}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-api/src/main/pegasus/org/apache/gobblin/restli/throttling/PermitAllocation.pdsc
b/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-api/src/main/pegasus/org/apache/gobblin/restli/throttling/PermitAllocation.pdsc
deleted file mode 100644
index 8601d29..0000000
---
a/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-api/src/main/pegasus/org/apache/gobblin/restli/throttling/PermitAllocation.pdsc
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "name": "PermitAllocation",
- "type": "record",
- "namespace" : "org.apache.gobblin.restli.throttling",
- "doc": "Used by gobblin-throttling-service to allocate permits to a client.",
- "fields": [
- {"name": "permits", "type": "long", "doc": "Number of permits allocated.
This may be 0 if no permits are allocated, or the number of requested
permits."},
- {"name": "expiration", "type": "long", "doc": "Expiration time in Unix
timestamp of the allocated permits."},
- {"name": "minRetryDelayMillis", "type": "long", "doc": "Client should not
try to acquire permits before this delay has passed.", "optional" : true},
- {"name": "waitForPermitUseMillis", "type": "long", "doc": "Client must
wait this many millis before allocating provided permits.", "optional": true,
"default": 0},
- {"name": "unsatisfiablePermits", "type": "long", "doc": "If larger than 0,
specifies request larger than this number are impossible to satisfy by the
policy.", "optional": true, "default": 0}
- ]
-}
diff --git
a/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-api/src/main/pegasus/org/apache/gobblin/restli/throttling/PermitRequest.pdl
b/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-api/src/main/pegasus/org/apache/gobblin/restli/throttling/PermitRequest.pdl
new file mode 100644
index 0000000..315c0e2
--- /dev/null
+++
b/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-api/src/main/pegasus/org/apache/gobblin/restli/throttling/PermitRequest.pdl
@@ -0,0 +1,32 @@
+namespace org.apache.gobblin.restli.throttling
+
+/**
+ * Used to request permits from a gobblin-throttling-service.
+ */
+record PermitRequest {
+
+ /**
+ * Resource for which permits are needed.
+ */
+ resource: string
+
+ /**
+ * Number of permits needed.
+ */
+ permits: long
+
+ /**
+ * Minimum number of useful permits.
+ */
+ minPermits: optional long = 0
+
+ /**
+ * Identifier of the service requesting the permits.
+ */
+ requestorIdentifier: string
+
+ /**
+ * Protocol version, see ThrottlingProtocolVersion.java. Allows the server
to avoid asking the client for unsupported operations.
+ */
+ version: optional int = 0
+}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-api/src/main/pegasus/org/apache/gobblin/restli/throttling/PermitRequest.pdsc
b/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-api/src/main/pegasus/org/apache/gobblin/restli/throttling/PermitRequest.pdsc
deleted file mode 100644
index 1cee3df..0000000
---
a/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-api/src/main/pegasus/org/apache/gobblin/restli/throttling/PermitRequest.pdsc
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "name": "PermitRequest",
- "type": "record",
- "namespace" : "org.apache.gobblin.restli.throttling",
- "doc": "Used to request permits from a gobblin-throttling-service.",
- "fields": [
- {"name": "resource", "type": "string", "doc": "Resource for which permits
are needed."},
- {"name": "permits", "type": "long", "doc": "Number of permits needed."},
- {"name": "minPermits", "type": "long", "doc": "Minimum number of useful
permits.", "optional" : true, "default": 0},
- {"name": "requestorIdentifier", "type": "string", "doc" : "Identifier of
the service requesting the permits."},
- {"name": "version", "type": "int", "doc": "Protocol version, see
ThrottlingProtocolVersion.java. Allows the server to avoid asking the client
for unsupported operations.", "optional": true, "default": 0}
- ]
-}
diff --git
a/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-api/src/main/pegasus/org/apache/gobblin/restli/throttling/Policy.pdl
b/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-api/src/main/pegasus/org/apache/gobblin/restli/throttling/Policy.pdl
new file mode 100644
index 0000000..5d4c0f6
--- /dev/null
+++
b/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-api/src/main/pegasus/org/apache/gobblin/restli/throttling/Policy.pdl
@@ -0,0 +1,32 @@
+namespace org.apache.gobblin.restli.throttling
+
+/**
+ * A throttling policy for Gobblin throttling service.
+ */
+record Policy {
+
+ /**
+ * Resource for which this policy applied.
+ */
+ resource: string
+
+ /**
+ * The name of the policy.
+ */
+ policyName: string
+
+ /**
+ * Parameters for this policy.
+ */
+ parameters: map[string, string]
+
+ /**
+ * Metrics for this policy.
+ */
+ metrics: map[string, string]
+
+ /**
+ * Human readable policy details.
+ */
+ policyDetails: string
+}
\ No newline at end of file
diff --git
a/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-api/src/main/pegasus/org/apache/gobblin/restli/throttling/Policy.pdsc
b/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-api/src/main/pegasus/org/apache/gobblin/restli/throttling/Policy.pdsc
deleted file mode 100644
index 87298e8..0000000
---
a/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-api/src/main/pegasus/org/apache/gobblin/restli/throttling/Policy.pdsc
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "name": "Policy",
- "type": "record",
- "namespace" : "org.apache.gobblin.restli.throttling",
- "doc": "A throttling policy for Gobblin throttling service.",
- "fields": [
- {"name": "resource", "type": "string", "doc": "Resource for which this
policy applied."},
- {"name": "policyName", "type": "string", "doc": "The name of the policy."},
- {"name": "parameters", "type": {"type": "map", "values": "string"}, "doc":
"Parameters for this policy."},
- {"name": "metrics", "type": {"type": "map", "values": "string"}, "doc":
"Metrics for this policy."},
- {"name": "policyDetails", "type": "string", "doc": "Human readable policy
details."}
- ]
-}
diff --git a/gradle/scripts/defaultBuildProperties.gradle
b/gradle/scripts/defaultBuildProperties.gradle
index e772acd..f12a1f4 100644
--- a/gradle/scripts/defaultBuildProperties.gradle
+++ b/gradle/scripts/defaultBuildProperties.gradle
@@ -35,7 +35,7 @@ def BuildProperties BUILD_PROPERTIES = new
BuildProperties(project)
"Java languange compatibility; supported versions: " +
JavaVersion.VERSION_1_8))
.register(new BuildProperty("kafka08Version", "0.8.2.2", "Kafka 0.8
dependencies version"))
.register(new BuildProperty("kafka09Version", "0.9.0.1", "Kafka 0.9
dependencies version"))
- .register(new BuildProperty("pegasusVersion", "11.0.0", "Pegasus
dependencies version"))
+ .register(new BuildProperty("pegasusVersion", "29.6.4", "Pegasus
dependencies version"))
.register(new BuildProperty("publishToMaven", false, "Enable publishing of
artifacts to a central Maven repository"))
.register(new BuildProperty("publishToNexus", false, "Enable publishing of
artifacts to Nexus"))
.register(new BuildProperty("salesforceVersion", "42.0.0", "Salesforce
dependencies version"))
diff --git a/gradle/scripts/repositories.gradle
b/gradle/scripts/repositories.gradle
index 82d8c08..b35b050 100644
--- a/gradle/scripts/repositories.gradle
+++ b/gradle/scripts/repositories.gradle
@@ -23,6 +23,9 @@ repositories {
maven {
url "http://packages.confluent.io/maven/";
}
+ maven {
+ url "https://linkedin.bintray.com/maven";
+ }
jcenter()
}
