Repository: incubator-griffin Updated Branches: refs/heads/master 87fd749ab -> f783cb78e
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f783cb78/griffin-streaming-model/src/test/scala/org/apache/griffin/config/ConfigFileReaderTest.scala ---------------------------------------------------------------------- diff --git a/griffin-streaming-model/src/test/scala/org/apache/griffin/config/ConfigFileReaderTest.scala b/griffin-streaming-model/src/test/scala/org/apache/griffin/config/ConfigFileReaderTest.scala new file mode 100644 index 0000000..6090264 --- /dev/null +++ b/griffin-streaming-model/src/test/scala/org/apache/griffin/config/ConfigFileReaderTest.scala @@ -0,0 +1,21 @@ +package org.apache.griffin.config + +import org.junit.runner.RunWith +import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} +import org.scalatest.junit.JUnitRunner + +import org.apache.griffin.config.params._ + + +@RunWith(classOf[JUnitRunner]) +class ConfigFileReaderTest extends FunSuite with Matchers with BeforeAndAfter { + + test("test file reader") { + val reader: ConfigReader = ConfigFileReader("src/main/resources/config.json") +// val param = reader.readConfig[AllParam] + val param = reader.readConfig + + println(param) + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f783cb78/griffin-streaming-model/src/test/scala/org/apache/griffin/config/ConfigReadTest.scala ---------------------------------------------------------------------- diff --git a/griffin-streaming-model/src/test/scala/org/apache/griffin/config/ConfigReadTest.scala b/griffin-streaming-model/src/test/scala/org/apache/griffin/config/ConfigReadTest.scala new file mode 100644 index 0000000..9818662 --- /dev/null +++ b/griffin-streaming-model/src/test/scala/org/apache/griffin/config/ConfigReadTest.scala @@ -0,0 +1,45 @@ +package org.apache.griffin.config + +import com.fasterxml.jackson.annotation.JsonInclude.Include +import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty} +import org.apache.griffin.utils.JsonUtil + +object ConfigReadTest { + + def main(args: Array[String]): Unit = { +// val originalMap = Map("a" -> List(1,2), "b" -> List(3,4,5), "c" -> List()) +// val json = JsonUtil.toJson(originalMap) +// val map = JsonUtil.toMap[Seq[Int]](json) +// +// println(json) +// println(map) +// +// val mutableSymbolMap = JsonUtil.fromJson[collection.mutable.Map[Symbol,Seq[Int]]](json) +// println(mutableSymbolMap) +// +// +// +// val jeroen = Person("Jeroen", 26) +// val martin = Person("Martin", 54) +// +// val originalGroup = Group(null, Seq(jeroen,martin), martin) +// +// val groupJson = JsonUtil.toJson(originalGroup) +// +// val group = JsonUtil.fromJson[Group](groupJson) +// +// println(originalGroup) +// println(groupJson) +// println(group) +// +// val groupJson1 = """{"prs":[{"name":"Jeroen","age":26}]}""" +// val group1 = JsonUtil.fromJson[Group](groupJson1) +// println(group1) + } + +} + +case class Person(name: String, age: Int) + +@JsonInclude(Include.NON_NULL) +case class Group(name: String, @JsonProperty("prs") persons: Seq[Person], leader: Person) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f783cb78/griffin-streaming-model/src/test/scala/org/apache/griffin/prep/parse/Json2MapParserTest.scala ---------------------------------------------------------------------- diff --git a/griffin-streaming-model/src/test/scala/org/apache/griffin/prep/parse/Json2MapParserTest.scala b/griffin-streaming-model/src/test/scala/org/apache/griffin/prep/parse/Json2MapParserTest.scala new file mode 100644 index 0000000..54b4b55 --- /dev/null +++ b/griffin-streaming-model/src/test/scala/org/apache/griffin/prep/parse/Json2MapParserTest.scala @@ -0,0 +1,49 @@ +package org.apache.griffin.prep.parse + +import org.apache.griffin.config.{ConfigFileReader, ConfigReader} +import org.junit.runner.RunWith +import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} +import org.scalatest.junit.JUnitRunner +import org.apache.griffin.config.params._ +import org.apache.griffin.utils.{ExtractJsonUtil, JsonUtil} + +import scala.io.Source +import scala.util.{Failure, Try} + + +@RunWith(classOf[JUnitRunner]) +class Json2MapParserTest extends FunSuite with Matchers with BeforeAndAfter { + +// test("test json 2 map parser") { +// +// val reader: ConfigReader = ConfigFileReader("src/main/resources/config.json") +// val param = reader.readConfig[AllParam] +// +// val lines = Source.fromFile("src/test/resources/input.msg").getLines +// +// param.dataAssetParamMap.get("source") match { +// case Some(dataAssetParam: DataAssetParam) => { +// val configParam = dataAssetParam.prepParam.parseParam.configParam +// println(configParam) +// val parser = Json2MapParser(configParam) +// +// lines.foreach { line => +// val res = parser.parse(line) +// +// val record: String = res.map { r => +// r.values.mkString(",") +// }.mkString("\n") +// +// println(record) +// +//// for (mp <- res) { +//// +//// println(mp) +//// } +// } +// } +// } +// +// } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f783cb78/griffin-streaming-model/src/test/scala/org/apache/griffin/test/DataParseTest.scala ---------------------------------------------------------------------- diff --git a/griffin-streaming-model/src/test/scala/org/apache/griffin/test/DataParseTest.scala b/griffin-streaming-model/src/test/scala/org/apache/griffin/test/DataParseTest.scala new file mode 100644 index 0000000..eb47378 --- /dev/null +++ b/griffin-streaming-model/src/test/scala/org/apache/griffin/test/DataParseTest.scala @@ -0,0 +1,101 @@ +package org.apache.griffin.test + +import org.junit.runner.RunWith +import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} +import org.scalatest.junit.JUnitRunner +import org.apache.griffin.config.params._ +import org.apache.griffin.utils.{ExtractJsonUtil, JsonUtil} + +import scala.io.Source + + +@RunWith(classOf[JUnitRunner]) +class DataParseTest extends FunSuite with Matchers with BeforeAndAfter { + +// test ("test extract list") { +// val urlSteps = List("json", ".seeds", "[*]", "json", ".url") +// val createdTsSteps = List("json", ".seeds", "[*]", "json", ".metadata", "json", ".tracker", ".crawlRequestCreateTS") +// +// val lines = Source.fromFile("src/test/resources/input.msg").getLines +// +// lines.foreach { line => +// +// val datas = Some(line) :: Nil +// val schema = Map(("url" -> urlSteps), ("createdTs" -> createdTsSteps)) +// val result = ExtractJsonUtil.extractDataListWithSchemaMap(datas, schema) +// +// result.foreach { res => +// val r = res.mapValues(_.getOrElse("none")) +// println(r) +// } +// +// } +// } + + test ("test extract list 1111") { + val urlSteps = List("json", ".seeds", "[*]", "json", ".url") + val createdTsSteps = List("json", ".seeds", "[*]", "json", ".metadata", "json", ".tracker", ".crawlRequestCreateTS") + + val lines = Source.fromFile("src/test/resources/input.msg").getLines + + lines.foreach { line => + + val datas = Some(line) :: Nil + val schema = Map(("url" -> urlSteps), ("createdTs" -> createdTsSteps)) + val result = ExtractJsonUtil.extractDataListWithSchemaMap(datas, schema) + + result.foreach { res => + val r = res.mapValues(_.getOrElse("none")) + println(r) + } + + } + } + + test ("test extract list 2222") { + val urlSteps = List("json", ".groups", "[0]", ".attrsList", "[.name=URL]", ".values", "[0]") + val createdTsSteps = List("json", ".groups", "[0]", ".attrsList", "[.name=CRAWLMETADATA]", ".values", "[0]", "json", ".tracker", ".crawlRequestCreateTS") + +// val lines = Source.fromFile("src/test/resources/out1.msg").getLines + val lines = Source.fromFile("src/test/resources/output.msg").getLines + + lines.foreach { line => + + val datas = Some(line) :: Nil + val schema = Map(("url" -> urlSteps), ("createdTs" -> createdTsSteps)) + val result = ExtractJsonUtil.extractDataListWithSchemaMap(datas, schema) + + result.foreach { res => + val r = res.mapValues(_.getOrElse("none")) + println(r) + } + + } + } + +// test ("test cret") { +// val nameSteps = List("json", ".spark", ".app.name") +// val logLevelSteps = List("json", ".spark", ".log.level") +// val sourceOutSchemaNameSteps = List("json", ".dataAsset", ".source", ".pre.process", ".parse", ".config", ".out.schema", "[*]", ".name") +// val sourceOutSchemaNameSteps1 = List("json", ".dataAsset", ".source", ".pre.process", ".parse", ".config", ".out.schema", "[0]", ".name") +// val sourceOutSchemaNameSteps2 = List("json", ".dataAsset", ".source", ".pre.process", ".parse", ".config", ".out.schema", "[*]", +// ".extract.steps", "[*]") +// val sourceOutSchemaNameSteps3 = List("json", ".dataAsset", ".source", ".pre.process", ".parse", ".config", ".out.schema", "[*]", +// ".extract.steps", "[4]") +// +// val file = "src/main/resources/config.json" +// val line = scala.io.Source.fromFile(file).mkString +// +// val datas = Some(line) :: Nil +// val schema = Map( +// ("name" -> nameSteps), +// ("loglevel" -> logLevelSteps), +// ("srcschemaname" -> sourceOutSchemaNameSteps), +// ("test1" -> sourceOutSchemaNameSteps1), +// ("test2" -> sourceOutSchemaNameSteps2), +// ("test3" -> sourceOutSchemaNameSteps3) +// ) +// ExtractJsonUtil.extractDataListWithSchemaMapTest(datas, schema) +// } + +}
