[S2GRAPH-221]: Unify configurations for bulk and mutate in S2GraphSink. JIRA: [S2GRAPH-221] https://issues.apache.org/jira/browse/S2GRAPH-221
Pull Request: Closes #173 Author DO YUNG YOON <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/c30531bc Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/c30531bc Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/c30531bc Branch: refs/heads/master Commit: c30531bcda92980d0c3d56b2f8d9673eca67218d Parents: 5582644 e1a4fdb Author: DO YUNG YOON <[email protected]> Authored: Thu Jun 21 10:21:54 2018 +0900 Committer: DO YUNG YOON <[email protected]> Committed: Thu Jun 21 10:23:19 2018 +0900 ---------------------------------------------------------------------- CHANGES | 2 + .../org/apache/s2graph/core/GraphUtil.scala | 9 + .../scala/org/apache/s2graph/core/S2Graph.scala | 45 +---- .../apache/s2graph/core/S2GraphConfigs.scala | 179 +++++++++++++++++++ .../s2graph/s2jobs/loader/HFileGenerator.scala | 47 +++-- .../s2jobs/loader/HFileMRGenerator.scala | 2 +- .../loader/LocalBulkLoaderTransformer.scala | 7 +- .../loader/SparkBulkLoaderTransformer.scala | 4 +- .../org/apache/s2graph/s2jobs/task/Sink.scala | 58 ++++-- .../org/apache/s2graph/s2jobs/task/Source.scala | 25 +-- .../org/apache/s2graph/s2jobs/task/Task.scala | 21 +-- .../spark/sql/streaming/S2SinkConfigs.scala | 21 ++- .../spark/sql/streaming/S2SourceConfigs.scala | 40 +++++ .../apache/s2graph/s2jobs/BaseSparkTest.scala | 26 ++- .../s2graph/s2jobs/S2GraphHelperTest.scala | 20 +-- .../s2jobs/loader/GraphFileGeneratorTest.scala | 6 +- .../apache/s2graph/s2jobs/task/SinkTest.scala | 7 +- .../apache/s2graph/s2jobs/task/SourceTest.scala | 25 ++- 18 files changed, 413 insertions(+), 131 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c30531bc/CHANGES ---------------------------------------------------------------------- diff --cc CHANGES index b6bc0cd,b9bf81b..13dfef9 --- a/CHANGES +++ b/CHANGES @@@ -80,7 -78,6 +80,8 @@@ Release Notes - S2Graph - Version 0.2. * [S2GRAPH-210] - Rename package `mysqls` to `schema` * [S2GRAPH-213] - Abstract Query/Mutation from Storage. * [S2GRAPH-214] - Add REAME for movielens examples + * [S2GRAPH-216] - Provide a transform directive in the GraphQL query result. ++ * [S2GRAPH-221] - Unify configurations for bulk and mutate in S2GraphSink. ** New Feature * [S2GRAPH-123] - Support different index on out/in direction. @@@ -91,11 -88,8 +92,12 @@@ * [S2GRAPH-177] - Add support for createServiceColumn/addVertex APIs on GraphQL. * [S2GRAPH-185] - Support Spark Structured Streaming to work with data in streaming and batch. * [S2GRAPH-183] - Provide batch job to dump data stored in HBase into file. + * [S2GRAPH-203] - Support "application/graphql" Content-Type header. * [S2GRAPH-206] - Generalize machine learning model serving. - * [S2GRAPH-215] - Implement a Storage Backend for JDBC driver, such as H2, MySql using the Mutator and Fetcher interfaces + * [S2GRAPH-215] - Implement a Storage Backend for JDBC driver, such as H2, MySql using the Mutator and Fetcher interfaces. + * [S2GRAPH-219] - Added query that includes all vertices and associated edges for GraphVisualize. + * [S2GRAPH-222] - Support Not logical operator in WhereParser. ++ * [S2GRAPH-223] - Support WhereParser on Vertex. ** Task * [S2GRAPH-162] - Update year in the NOTICE file. http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c30531bc/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c30531bc/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Sink.scala ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c30531bc/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Source.scala ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c30531bc/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Task.scala ---------------------------------------------------------------------- diff --cc s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Task.scala index 1210132,51e13f0..ab02900 --- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Task.scala +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/Task.scala @@@ -39,10 -40,11 +40,10 @@@ object TaskConf } def parseLocalCacheConfigs(taskConf: TaskConf): Map[String, Any] = { - taskConf.options.filterKeys(_.startsWith("cache.")).mapValues(_.toInt) + taskConf.options.filterKeys(S2GraphConfigs.CacheConfigs.DEFAULTS.keySet).mapValues(_.toInt) } } - -case class TaskConf(name: String, `type`: String, inputs: Seq[String] = Nil, options: Map[String, String] = Map.empty) +case class TaskConf(name:String, `type`:String, inputs:Seq[String] = Nil, options:Map[String, String] = Map.empty, cache:Option[Boolean]=None) trait Task extends Serializable with Logger { val conf: TaskConf
