bug fix on vertex property parsing.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/1111275a Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/1111275a Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/1111275a Branch: refs/heads/master Commit: 1111275ad8134143b576bd323211115dd4cb24f6 Parents: cd41b8f Author: DO YUNG YOON <[email protected]> Authored: Wed Feb 28 14:56:13 2018 +0900 Committer: DO YUNG YOON <[email protected]> Committed: Wed Feb 28 14:56:13 2018 +0900 ---------------------------------------------------------------------- .../loader/subscriber/TransferToHFileTest.scala | 51 +++++++++++++++++++- .../org/apache/s2graph/core/JSONParser.scala | 3 +- .../s2graph/core/mysqls/ServiceColumn.scala | 4 +- 3 files changed, 54 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/1111275a/loader/src/test/scala/org/apache/s2graph/loader/subscriber/TransferToHFileTest.scala ---------------------------------------------------------------------- diff --git a/loader/src/test/scala/org/apache/s2graph/loader/subscriber/TransferToHFileTest.scala b/loader/src/test/scala/org/apache/s2graph/loader/subscriber/TransferToHFileTest.scala index 36ee530..77c147e 100644 --- a/loader/src/test/scala/org/apache/s2graph/loader/subscriber/TransferToHFileTest.scala +++ b/loader/src/test/scala/org/apache/s2graph/loader/subscriber/TransferToHFileTest.scala @@ -20,13 +20,14 @@ package org.apache.s2graph.loader.subscriber import java.util -import org.apache.s2graph.core.Management +import org.apache.s2graph.core.{Management, PostProcess} import org.apache.s2graph.core.Management.JsonModel.{Index, Prop} import org.apache.s2graph.core.mysqls.Label import org.apache.s2graph.core.storage.CanSKeyValue import org.apache.s2graph.core.types.HBaseType import org.apache.spark.{SparkConf, SparkContext} import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers} +import play.api.libs.json.Json import scala.util.Try @@ -76,7 +77,7 @@ class TransferToHFileTest extends FunSuite with Matchers with BeforeAndAfterAll } - test("test TransferToHFile Local.") { + test("test edge only.") { import scala.collection.JavaConverters._ import org.apache.s2graph.core.storage.CanSKeyValue._ @@ -120,4 +121,50 @@ class TransferToHFileTest extends FunSuite with Matchers with BeforeAndAfterAll bulkEdge shouldBe(indexEdge) } + + test("test vertex only.") { + import scala.collection.JavaConverters._ + /* initialize model for test */ + val management = GraphSubscriberHelper.management + + val service = management.createService(serviceName = "s2graph", cluster = "localhost", + hTableName = "s2graph", preSplitSize = -1, hTableTTL = -1, compressionAlgorithm = "gz") + + val serviceColumn = management.createServiceColumn(service.serviceName, "imei", "string", + Seq( + Prop(name = "first_time", defaultValue = "''", dataType = "string"), + Prop(name = "last_time", defaultValue = "''", dataType = "string"), + Prop(name = "total_active_days", defaultValue = "-1", dataType = "integer"), + Prop(name = "query_amount", defaultValue = "-1", dataType = "integer"), + Prop(name = "active_months", defaultValue = "-1", dataType = "integer"), + Prop(name = "fua", defaultValue = "''", dataType = "string"), + Prop(name = "location_often_province", defaultValue = "''", dataType = "string"), + Prop(name = "location_often_city", defaultValue = "''", dataType = "string"), + Prop(name = "location_often_days", defaultValue = "-1", dataType = "integer"), + Prop(name = "location_last_province", defaultValue = "''", dataType = "string"), + Prop(name = "location_last_city", defaultValue = "''", dataType = "string"), + Prop(name = "fimei_legality", defaultValue = "-1", dataType = "integer") + ).asJava) + + val bulkVertexString = "20171201\tinsert\tvertex\t800188448586078\ts2graph\timei\t{\"first_time\":\"20171025\",\"last_time\":\"20171112\",\"total_active_days\":14,\"query_amount\":1526.0,\"active_months\":2,\"fua\":\"M5+Note\",\"location_often_province\":\"广ä¸ç\",\"location_often_city\":\"æ·±å³å¸\",\"location_often_days\":6,\"location_last_province\":\"广ä¸ç\",\"location_last_city\":\"æ·±å³å¸\",\"fimei_legality\":3}" + val bulkVertex = GraphSubscriberHelper.g.elementBuilder.toGraphElement(bulkVertexString, options.labelMapping).get + + val input = sc.parallelize(Seq(bulkVertexString)) + + val kvs = TransferToHFile.generateKeyValues(sc, s2Config, input, options) + + val ls = kvs.map(kv => CanSKeyValue.hbaseKeyValue.toSKeyValue(kv)).collect().toList + + val serDe = GraphSubscriberHelper.g.defaultStorage.serDe + + + + val vertex = serDe.vertexDeserializer(serviceColumn.schemaVersion).fromKeyValues(ls, None).get + + PostProcess.s2VertexToJson(vertex).foreach { jsValue => + println(Json.prettyPrint(jsValue)) + } + + bulkVertex shouldBe(vertex) + } } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/1111275a/s2core/src/main/scala/org/apache/s2graph/core/JSONParser.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/JSONParser.scala b/s2core/src/main/scala/org/apache/s2graph/core/JSONParser.scala index 9de3d9d..b92e47b 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/JSONParser.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/JSONParser.scala @@ -344,8 +344,9 @@ object JSONParser { def fromJsonToProperties(jsObject: JsObject): Map[String, Any] = { val kvs = for { (k, v) <- jsObject.fieldSet + anyVal <- jsValueToAny(v) } yield { - k -> jsValueToString(v) + k -> anyVal } kvs.toMap } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/1111275a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ServiceColumn.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ServiceColumn.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ServiceColumn.scala index be1ae9a..90b9e06 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ServiceColumn.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ServiceColumn.scala @@ -109,11 +109,13 @@ case class ServiceColumn(id: Option[Int], lazy val toJson = Json.obj("serviceName" -> service.serviceName, "columnName" -> columnName, "columnType" -> columnType) def propsToInnerVals(props: Map[String, Any]): Map[ColumnMeta, InnerValLike] = { - for { + val ret = for { (k, v) <- props labelMeta <- metasInvMap.get(k) innerVal = toInnerVal(v, labelMeta.dataType, schemaVersion) } yield labelMeta -> innerVal + + ret } def innerValsToProps(props: Map[Int, InnerValLike]): Map[String, Any] = {
