Merge branch 'master' into S2GRAPH-191 * master: [S2GRAPH-192] could not find service column when creating the label [S2GRAPH-190] Remove unnecessary TC JsonBenchmarkSpec fix name conflict remove broken import remove unused benchmark [S2GRAPH-185]: Support Spark Structured Streaming to work with data in streaming and batch remove count operation wait termination for multiple streaming query shade protobuf to resolve confilct with spark lib add s2graph sink for spark structured streaming add simple etl job interface
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/5c81ac8b Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/5c81ac8b Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/5c81ac8b Branch: refs/heads/master Commit: 5c81ac8bf6195646efdc7e3abdea529791db8c4d Parents: d137bb3 e0b2dc0 Author: daewon <[email protected]> Authored: Fri Mar 30 15:24:01 2018 +0900 Committer: daewon <[email protected]> Committed: Fri Mar 30 15:24:01 2018 +0900 ---------------------------------------------------------------------- CHANGES | 3 + project/Common.scala | 2 +- .../core/benchmark/BenchmarkCommon.scala | 1 - .../core/benchmark/JsonBenchmarkSpec.scala | 78 -------- .../core/benchmark/SamplingBenchmarkSpec.scala | 105 ---------- .../loader/core/CounterEtlFunctionsSpec.scala | 2 +- .../graphql/types/S2ManagementType.scala | 4 +- .../org/apache/s2graph/graphql/SchemaTest.scala | 22 ++- s2jobs/build.sbt | 12 +- .../scala/org/apache/s2graph/s2jobs/Job.scala | 70 +++++++ .../apache/s2graph/s2jobs/JobDescription.scala | 88 +++++++++ .../org/apache/s2graph/s2jobs/JobLauncher.scala | 88 +++++++++ .../org/apache/s2graph/s2jobs/Logger.scala | 26 +++ .../org/apache/s2graph/s2jobs/Schema.scala | 35 ++++ .../apache/s2graph/s2jobs/task/Process.scala | 53 +++++ .../org/apache/s2graph/s2jobs/task/Sink.scala | 195 +++++++++++++++++++ .../org/apache/s2graph/s2jobs/task/Source.scala | 104 ++++++++++ .../org/apache/s2graph/s2jobs/task/Task.scala | 38 ++++ .../s2graph/spark/sql/streaming/Logger.scala | 26 +++ .../spark/sql/streaming/S2CommitProtocol.scala | 67 +++++++ .../spark/sql/streaming/S2SinkConfigs.scala | 47 +++++ .../spark/sql/streaming/S2SinkContext.scala | 44 +++++ .../spark/sql/streaming/S2SinkMetadataLog.scala | 44 +++++ .../spark/sql/streaming/S2SinkProvider.scala | 45 +++++ .../spark/sql/streaming/S2SinkStatus.scala | 28 +++ .../sql/streaming/S2SparkSqlStreamingSink.scala | 89 +++++++++ .../sql/streaming/S2StreamQueryWriter.scala | 133 +++++++++++++ .../s2graph/s2jobs/task/ProcessTest.scala | 48 +++++ 28 files changed, 1297 insertions(+), 200 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5c81ac8b/project/Common.scala ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5c81ac8b/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctionsSpec.scala ---------------------------------------------------------------------- diff --cc s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctionsSpec.scala index 6985758,6985758..35575fb --- a/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctionsSpec.scala +++ b/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctionsSpec.scala @@@ -22,7 -22,7 +22,7 @@@ package org.apache.s2graph.counter.load import com.typesafe.config.ConfigFactory import org.apache.s2graph.core.mysqls.{Label, Service} import org.apache.s2graph.core.types.HBaseType --import org.apache.s2graph.core.{S2Graph$, Management} ++import org.apache.s2graph.core.{S2Graph, Management} import org.apache.s2graph.counter.models.DBModel import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
