[S2GRAPH-185]: Support Spark Structured Streaming to work with data in
streaming and batch
JIRA:
[S2GRAPH-185] https://issues.apache.org/jira/browse/S2GRAPH-185
Pull Request:
Closes #144
Author
Chul Kang <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/245335ad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/245335ad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/245335ad
Branch: refs/heads/master
Commit: 245335ad3be3b907faea0ea0609390bd687dcbf1
Parents: 36e6896
Author: DO YUNG YOON <[email protected]>
Authored: Thu Mar 29 14:34:22 2018 +0900
Committer: DO YUNG YOON <[email protected]>
Committed: Thu Mar 29 14:34:22 2018 +0900
----------------------------------------------------------------------
CHANGES | 1 +
.../scala/org/apache/s2graph/s2jobs/Job.scala | 19 ++++++++++++++++++
.../apache/s2graph/s2jobs/JobDescription.scala | 19 ++++++++++++++++++
.../org/apache/s2graph/s2jobs/JobLauncher.scala | 19 ++++++++++++++++++
.../org/apache/s2graph/s2jobs/Logger.scala | 19 ++++++++++++++++++
.../org/apache/s2graph/s2jobs/Schema.scala | 19 ++++++++++++++++++
.../apache/s2graph/s2jobs/task/Process.scala | 19 ++++++++++++++++++
.../org/apache/s2graph/s2jobs/task/Sink.scala | 19 ++++++++++++++++++
.../org/apache/s2graph/s2jobs/task/Source.scala | 19 ++++++++++++++++++
.../org/apache/s2graph/s2jobs/task/Task.scala | 19 ++++++++++++++++++
.../s2graph/spark/sql/streaming/Logger.scala | 19 ++++++++++++++++++
.../spark/sql/streaming/S2CommitProtocol.scala | 19 ++++++++++++++++++
.../spark/sql/streaming/S2SinkConfigs.scala | 19 ++++++++++++++++++
.../spark/sql/streaming/S2SinkContext.scala | 19 ++++++++++++++++++
.../spark/sql/streaming/S2SinkMetadataLog.scala | 21 +++++++++++++++++++-
.../spark/sql/streaming/S2SinkProvider.scala | 19 ++++++++++++++++++
.../spark/sql/streaming/S2SinkStatus.scala | 19 ++++++++++++++++++
.../sql/streaming/S2SparkSqlStreamingSink.scala | 19 ++++++++++++++++++
.../sql/streaming/S2StreamQueryWriter.scala | 19 ++++++++++++++++++
.../s2graph/s2jobs/task/ProcessTest.scala | 19 ++++++++++++++++++
20 files changed, 363 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 45d49ec..5abb879 100644
--- a/CHANGES
+++ b/CHANGES
@@ -67,6 +67,7 @@ Release Notes - S2Graph - Version 0.2.0
* [S2GRAPH-175] - Provide Elastic Search Index Provider.
* [S2GRAPH-172] - Suggest to implement GraphQL as standard web interface
for S2Graph.
* [S2GRAPH-177] - Add support for createServiceColumn/addVertex APIs on
GraphQL.
+ * [S2GRAPH-185] - Support Spark Structured Streaming to work with data in
streaming and batch.
** Task
* [S2GRAPH-162] - Update year in the NOTICE file.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/Job.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/Job.scala
b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/Job.scala
index beef135..717d087 100644
--- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/Job.scala
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/Job.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.s2jobs
import org.apache.spark.sql.{DataFrame, SparkSession}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/JobDescription.scala
----------------------------------------------------------------------
diff --git
a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/JobDescription.scala
b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/JobDescription.scala
index 0a9194f..2eaf6aa 100644
--- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/JobDescription.scala
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/JobDescription.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.s2jobs
import play.api.libs.json.{JsValue, Json}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/JobLauncher.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/JobLauncher.scala
b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/JobLauncher.scala
index d0b355c..a64a399 100644
--- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/JobLauncher.scala
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/JobLauncher.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.s2jobs
import org.apache.spark.sql.SparkSession
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/Logger.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/Logger.scala
b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/Logger.scala
index 1df55f0..05f82a2 100644
--- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/Logger.scala
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/Logger.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.s2jobs
import org.apache.commons.logging.{Log, LogFactory}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/Schema.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/Schema.scala
b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/Schema.scala
index f214218..7c9c393 100644
--- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/Schema.scala
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/Schema.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.s2jobs
import org.apache.spark.sql.types.{LongType, StringType, StructField,
StructType}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Process.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Process.scala
b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Process.scala
index 73d2fab..6bc100f 100644
--- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Process.scala
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Process.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.s2jobs.task
import org.apache.spark.sql.{DataFrame, SparkSession}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Sink.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Sink.scala
b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Sink.scala
index c15ed52..3d5beb6 100644
--- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Sink.scala
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Sink.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.s2jobs.task
import org.apache.spark.sql._
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala
b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala
index c81ac9f..d2ca8ef 100644
--- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.s2jobs.task
import org.apache.spark.sql.{DataFrame, SparkSession}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Task.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Task.scala
b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Task.scala
index bd9259f..e8f11e3 100644
--- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Task.scala
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Task.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.s2jobs.task
import org.apache.s2graph.s2jobs.Logger
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/Logger.scala
----------------------------------------------------------------------
diff --git
a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/Logger.scala
b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/Logger.scala
index 06232c5..ba4ed8e 100644
--- a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/Logger.scala
+++ b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/Logger.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.spark.sql.streaming
import org.apache.commons.logging.{Log, LogFactory}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2CommitProtocol.scala
----------------------------------------------------------------------
diff --git
a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2CommitProtocol.scala
b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2CommitProtocol.scala
index 162b9ae..d440cc2 100644
---
a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2CommitProtocol.scala
+++
b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2CommitProtocol.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.spark.sql.streaming
import org.apache.spark.sql.execution.streaming.MetadataLog
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkConfigs.scala
----------------------------------------------------------------------
diff --git
a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkConfigs.scala
b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkConfigs.scala
index c24df5c..6ef0111 100644
---
a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkConfigs.scala
+++
b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkConfigs.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.spark.sql.streaming
import com.typesafe.config.Config
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkContext.scala
----------------------------------------------------------------------
diff --git
a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkContext.scala
b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkContext.scala
index 74b3308..951d9ae 100644
---
a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkContext.scala
+++
b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkContext.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.spark.sql.streaming
import com.typesafe.config.Config
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkMetadataLog.scala
----------------------------------------------------------------------
diff --git
a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkMetadataLog.scala
b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkMetadataLog.scala
index eb985f1..de34e46 100644
---
a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkMetadataLog.scala
+++
b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkMetadataLog.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.spark.sql.streaming
import com.typesafe.config.Config
@@ -22,4 +41,4 @@ class S2SinkMetadataLog(sparkSession: SparkSession,
config:Config, logPath:Strin
object S2SinkMetadataLog {
private val VERSION = 1
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkProvider.scala
----------------------------------------------------------------------
diff --git
a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkProvider.scala
b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkProvider.scala
index 6198c81..e8c7939 100644
---
a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkProvider.scala
+++
b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkProvider.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.spark.sql.streaming
import com.typesafe.config.{Config, ConfigFactory, ConfigRenderOptions}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkStatus.scala
----------------------------------------------------------------------
diff --git
a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkStatus.scala
b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkStatus.scala
index 9b5ab8e..36f8145 100644
---
a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkStatus.scala
+++
b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SinkStatus.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.spark.sql.streaming
case class S2SinkStatus(
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SparkSqlStreamingSink.scala
----------------------------------------------------------------------
diff --git
a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SparkSqlStreamingSink.scala
b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SparkSqlStreamingSink.scala
index c41b231..42bbc3d 100644
---
a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SparkSqlStreamingSink.scala
+++
b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2SparkSqlStreamingSink.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.spark.sql.streaming
import java.util.UUID
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2StreamQueryWriter.scala
----------------------------------------------------------------------
diff --git
a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2StreamQueryWriter.scala
b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2StreamQueryWriter.scala
index de5aea0..e15efe8 100644
---
a/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2StreamQueryWriter.scala
+++
b/s2jobs/src/main/scala/org/apache/s2graph/spark/sql/streaming/S2StreamQueryWriter.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.spark.sql.streaming
import com.typesafe.config.ConfigFactory
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/245335ad/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/task/ProcessTest.scala
----------------------------------------------------------------------
diff --git
a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/task/ProcessTest.scala
b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/task/ProcessTest.scala
index adbebac..d9bbb5b 100644
--- a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/task/ProcessTest.scala
+++ b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/task/ProcessTest.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.s2jobs.task
import com.holdenkarau.spark.testing.DataFrameSuiteBase
