http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-models/src/main/scala/org/apache/griffin/util/DataTypeUtils.scala ---------------------------------------------------------------------- diff --git a/griffin-models/src/main/scala/org/apache/griffin/util/DataTypeUtils.scala b/griffin-models/src/main/scala/org/apache/griffin/util/DataTypeUtils.scala new file mode 100644 index 0000000..64f7fb5 --- /dev/null +++ b/griffin-models/src/main/scala/org/apache/griffin/util/DataTypeUtils.scala @@ -0,0 +1,146 @@ +package org.apache.griffin.util + +import java.sql.Timestamp +import java.util.Date + +import org.apache.spark.sql.Row +import org.apache.spark.sql.types._ + +object DataTypeUtils { + + final val intType = ("int", """^[Ii]nt(?:eger)?(?:Type)?$""".r, IntegerType, RowGetFunc.getInt _) + final val shortType = ("short", """^[Ss]hort(?:Type)?[Ss]mall(?:[Ii]nt)?$""".r, ShortType, RowGetFunc.getShort _) + final val longType = ("long", """^[Ll]ong(?:Type)?|[Bb]ig(?:[Ii]nt)?$""".r, LongType, RowGetFunc.getLong _) + final val byteType = ("byte", """^[Bb]yte(?:Type)?|[Tt]iny(?:[Ii]nt)?$""".r, ByteType, RowGetFunc.getByte _) + final val floatType = ("float", """^[Ff]loat(?:Type)?$""".r, FloatType, RowGetFunc.getFloat _) + final val doubleType = ("double", """^[Dd]ouble(?:Type)?$""".r, DoubleType, RowGetFunc.getDouble _) + final val dateType = ("date", """^[Dd]ate(?:Type)?$""".r, DateType, RowGetFunc.getDate _) + final val timestampType = ("timestamp", """^[Tt]ime(?:[Ss]tamp)?(?:Type)?$""".r, TimestampType, RowGetFunc.getTimestamp _) + final val stringType = ("string", """^[Ss]tr(?:ing)?(?:Type)?|[Vv]ar(?:[Cc]har)?|[Cc]har$""".r, StringType, RowGetFunc.getString _) + final val booleanType = ("boolean", """^[Bb]ool(?:ean)?(?:Type)?|[Bb]inary$""".r, BooleanType, RowGetFunc.getBoolean _) + + def str2DataType(tp: String): DataType = { + tp match { + case intType._2() => intType._3 + case shortType._2() => shortType._3 + case longType._2() => longType._3 + case byteType._2() => byteType._3 + case floatType._2() => floatType._3 + case doubleType._2() => doubleType._3 + case dateType._2() => dateType._3 + case timestampType._2() => timestampType._3 + case stringType._2() => stringType._3 + case booleanType._2() => booleanType._3 + case _ => stringType._3 + } + } + + def str2RowGetFunc(tp: String): (Row, Int) => Any = { + tp match { + case intType._2() => intType._4 + case shortType._2() => shortType._4 + case longType._2() => longType._4 + case byteType._2() => byteType._4 + case floatType._2() => floatType._4 + case doubleType._2() => doubleType._4 + case dateType._2() => dateType._4 + case timestampType._2() => timestampType._4 + case stringType._2() => stringType._4 + case booleanType._2() => booleanType._4 + case _ => stringType._4 + } + } + + def dataType2Str(dt: DataType): String = { + dt match { + case intType._3 => intType._1 + case shortType._3 => shortType._1 + case longType._3 => longType._1 + case byteType._3 => byteType._1 + case floatType._3 => floatType._1 + case doubleType._3 => doubleType._1 + case dateType._3 => dateType._1 + case timestampType._3 => timestampType._1 + case stringType._3 => stringType._1 + case booleanType._3 => booleanType._1 + case _ => stringType._1 + } + } + + def dataType2RowGetFunc(dt: DataType): (Row, Int) => Any = { + dt match { + case intType._3 => intType._4 + case shortType._3 => shortType._4 + case longType._3 => longType._4 + case byteType._3 => byteType._4 + case floatType._3 => floatType._4 + case doubleType._3 => doubleType._4 + case dateType._3 => dateType._4 + case timestampType._3 => timestampType._4 + case stringType._3 => stringType._4 + case booleanType._3 => booleanType._4 + case _ => stringType._4 + } + } + + def isNum(tp: String): Boolean = { + tp match { + case intType._2() => true + case shortType._2() => true + case longType._2() => true + case byteType._2() => true + case floatType._2() => true + case doubleType._2() => true + case _ => false + } + } + +} + + +object RowGetFunc { + def getInt(r: Row, col: Int) = { r.getInt(col) } + def getShort(r: Row, col: Int) = { r.getShort(col) } + def getLong(r: Row, col: Int) = { r.getLong(col) } + def getByte(r: Row, col: Int) = { r.getByte(col) } + def getFloat(r: Row, col: Int) = { r.getFloat(col) } + def getDouble(r: Row, col: Int) = { r.getDouble(col) } + def getDate(r: Row, col: Int) = { r.getDate(col) } + def getTimestamp(r: Row, col: Int) = { r.getTimestamp(col) } + def getString(r: Row, col: Int) = { r.getString(col) } + def getBoolean(r: Row, col: Int) = { r.getBoolean(col) } +} + +object DataConverter { + def getDouble(data: Any): Double = { + data match { + case x: Double => x + case x: Int => x.toDouble + case x: Short => x.toDouble + case x: Long => x.toDouble + case x: Byte => x.toDouble + case x: Float => x.toDouble + case x: Date => x.getTime +// case x: Timestamp => x.getTime + case x: String => x.toDouble + case x: Boolean => if (x) 1 else 0 + case _ => 0 + } + } + + def getString(data: Any): String = { + data match { + case x: String => x + case x: Int => x.toString + case x: Short => x.toString + case x: Long => x.toString + case x: Byte => x.toString + case x: Float => x.toString + case x: Double => x.toString + case x: Date => x.toString +// case x: Timestamp => x.toString + case x: Boolean => x.toString + case _ => "" + } + } +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-models/src/main/scala/org/apache/griffin/util/HdfsUtils.scala ---------------------------------------------------------------------- diff --git a/griffin-models/src/main/scala/org/apache/griffin/util/HdfsUtils.scala b/griffin-models/src/main/scala/org/apache/griffin/util/HdfsUtils.scala new file mode 100644 index 0000000..a3d23ec --- /dev/null +++ b/griffin-models/src/main/scala/org/apache/griffin/util/HdfsUtils.scala @@ -0,0 +1,27 @@ +package org.apache.griffin.util + +import java.io.File + +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FSDataInputStream, FSDataOutputStream, FileSystem, Path} + +object HdfsUtils { + + private val conf = new Configuration() + + private val dfs = FileSystem.get(conf) + + def createFile(filePath: String): FSDataOutputStream = { + return dfs.create(new Path(filePath)) + } + + def openFile(filePath: String): FSDataInputStream = { + return dfs.open(new Path(filePath)) + } + + def writeFile(filePath: String, message: String): Unit = { + val out = createFile(filePath) + out.write(message.getBytes("utf-8")) + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-models/src/main/scala/org/apache/griffin/util/PartitionUtils.scala ---------------------------------------------------------------------- diff --git a/griffin-models/src/main/scala/org/apache/griffin/util/PartitionUtils.scala b/griffin-models/src/main/scala/org/apache/griffin/util/PartitionUtils.scala new file mode 100644 index 0000000..a92d6d8 --- /dev/null +++ b/griffin-models/src/main/scala/org/apache/griffin/util/PartitionUtils.scala @@ -0,0 +1,31 @@ +package org.apache.griffin.util + +import org.apache.griffin.common.PartitionPair + +object PartitionUtils { + def generateWhereClause(partition: List[PartitionPair]): String = { + var first = true + partition.foldLeft("") { (clause, pair) => + if (first) { + first = false + s"where ${pair.colName} = ${pair.colValue}" + } + else s"$clause AND ${pair.colName} = ${pair.colValue}" + } + } + + def generateSourceSQLClause(sourceTable: String, partition: List[PartitionPair]): String = { + s"SELECT * FROM $sourceTable ${generateWhereClause(partition)}" + } + + def generateTargetSQLClause(targetTable: String, partitions: List[List[PartitionPair]]): String = { + var first = true + partitions.foldLeft(s"SELECT * FROM $targetTable") { (clause, partition) => + if (first) { + first = false + s"$clause ${generateWhereClause(partition)}" + } + else s"$clause UNION ALL SELECT * FROM $targetTable ${generateWhereClause(partition)}" + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-models/src/main/scala/org/apache/griffin/validility/MetricsType.scala ---------------------------------------------------------------------- diff --git a/griffin-models/src/main/scala/org/apache/griffin/validility/MetricsType.scala b/griffin-models/src/main/scala/org/apache/griffin/validility/MetricsType.scala new file mode 100644 index 0000000..acc831f --- /dev/null +++ b/griffin-models/src/main/scala/org/apache/griffin/validility/MetricsType.scala @@ -0,0 +1,16 @@ +package org.apache.griffin.validility + +object MetricsType extends Enumeration{ + type MetricsType = Value + val DefaultCount = Value(0, "defaultCount") + val TotalCount = Value(1, "totalCount") + val NullCount = Value(2, "nullCount") + val UniqueCount = Value(3, "uniqueCount") + val DuplicateCount = Value(4, "duplicateCount") + val Maximum = Value(5, "maximum") + val Minimum = Value(6, "minimum") + val Mean = Value(7, "mean") + val Median = Value(8, "median") + val RegularExp = Value(9, "regularExp") + val PatternFreq = Value(10, "patternFreq") +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-models/src/main/scala/org/apache/griffin/validility/Vali.scala ---------------------------------------------------------------------- diff --git a/griffin-models/src/main/scala/org/apache/griffin/validility/Vali.scala b/griffin-models/src/main/scala/org/apache/griffin/validility/Vali.scala new file mode 100644 index 0000000..4c5b9f4 --- /dev/null +++ b/griffin-models/src/main/scala/org/apache/griffin/validility/Vali.scala @@ -0,0 +1,211 @@ +package org.apache.griffin.validility + +import org.apache.griffin.dataLoaderUtils.DataLoaderFactory +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import org.apache.griffin.util.{DataConverter, DataTypeUtils, HdfsUtils, PartitionUtils} +import org.apache.spark.{Logging, SparkConf, SparkContext} +import org.apache.spark.mllib.linalg.Vectors +import org.apache.spark.mllib.stat.{MultivariateStatisticalSummary, Statistics} +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.hive.HiveContext + +object Vali extends Logging { + + def main(args: Array[String]): Unit ={ + if (args.length < 2) { + logError("Usage: class <input-conf-file> <outputPath>") + logError("For input-conf-file, please use vali_config.json as an template to reflect test dataset accordingly.") + sys.exit(-1) + } + val input = HdfsUtils.openFile(args(0)) + + val outputPath = args(1) + System.getProperty("file.separator") + + //add files for job scheduling + val startFile = outputPath + "_START" + val resultFile = outputPath + "_RESULT" + val doneFile = outputPath + "_FINISHED" + + val mapper = new ObjectMapper() + mapper.registerModule(DefaultScalaModule) + + //read the config info of comparison + val configure = mapper.readValue(input, classOf[ValidityConfEntity]) + + val conf = new SparkConf().setAppName("Vali") + val sc: SparkContext = new SparkContext(conf) + val sqlContext = new HiveContext(sc) + + //add spark applicationId for debugging + val applicationId = sc.applicationId + + //for spark monitoring + HdfsUtils.writeFile(startFile, applicationId) + + //get data + val dataLoader = DataLoaderFactory.getDataLoader(sqlContext, DataLoaderFactory.hive) + val sojdf = dataLoader.getValiDataFrame(configure) + + //-- algorithm -- + calcVali(configure, sojdf) + + //--output metrics data-- + val out = HdfsUtils.createFile(resultFile) + mapper.writeValue(out, configure) + + //for spark monitoring + HdfsUtils.createFile(doneFile) + + sc.stop() + } + + def calcVali(configure: ValidityConfEntity, sojdf: DataFrame) : Unit = { + val dfCount = sojdf.count() + + //--1. get all cols name, and types-- + val fnts = sojdf.schema.fields.map(x => (x.name, x.dataType.simpleString)).toMap + + //get col type + val req: List[ValidityReq] = configure.validityReq.map { r => + val fv = fnts.getOrElse(r.colName, None) + if (fv != None) { + r.colType = fv.toString + r.isNum = DataTypeUtils.isNum(r.colType) + } + r + } + + //--2. calc num cols metrics-- + val numcols = req.filter(r => r.isNum) + + val numIdx = numcols.map(c => c.colId).toArray + val numIdxZip = numIdx.zipWithIndex + val numColsCount = numcols.length + + //median number function + def funcMedian(df: DataFrame, col: Int): Double = { + val dt = sojdf.schema(col).dataType + val getFunc = DataTypeUtils.dataType2RowGetFunc(dt) + + val mp = df.map { v => + if (v.isNullAt(col)) (0.0, 0L) + else (DataConverter.getDouble(getFunc(v, col)), 1L) + }.reduceByKey(_+_) + val allCnt = mp.aggregate(0L)((c, m) => c + m._2, _+_) + val cnt = mp.sortByKey().collect() + var tmp, tmp1 = 0L + var median, median1 = cnt(0)._1 + if (allCnt % 2 != 0) { + for (i <- 0 until cnt.length if (tmp < allCnt / 2 + 1)) { + tmp += cnt(i)._2 + median = cnt(i)._1 + } + median + } else { + for (i <- 0 until cnt.length if (tmp1 < allCnt / 2 + 1)) { + tmp1 += cnt(i)._2 + median1 = cnt(i)._1 + if (tmp < allCnt / 2) { + tmp = tmp1 + median = median1 + } + } + (median + median1) / 2 + } + } + + //match num metrics request + def getNumStats(smry: MultivariateStatisticalSummary, df: DataFrame, op: Int, col: Int): Any = { + val i = numIdx.indexWhere(_ == col) + if (i >= 0) { + MetricsType(op) match { + case MetricsType.TotalCount => smry.count + case MetricsType.Maximum => smry.max(i) + case MetricsType.Minimum => smry.min(i) + case MetricsType.Mean => smry.mean(i) + case MetricsType.Median => funcMedian(df, col) +// case MetricsType.Variance => smry.variance(i) +// case MetricsType.NumNonZeros => smry.numNonzeros(i) + case _ => None + } + } + } + + if (numColsCount > 0) { + val idxType = numIdxZip.map(i => (i._2, i._1, sojdf.schema(i._1).dataType)) + + //calc metrics of all numeric cols once + val numcolVals = sojdf.map { row => + val vals = idxType.foldLeft((List[Int](), List[Double]())) { (arr, i) => + if (row.isNullAt(i._2)) arr + else { + val v = DataTypeUtils.dataType2RowGetFunc(i._3)(row, i._2) + (i._1 :: arr._1, DataConverter.getDouble(v) :: arr._2) + } + } + Vectors.sparse(numColsCount, vals._1.toArray, vals._2.toArray) + } + + val summary = Statistics.colStats(numcolVals) + + //get numeric metrics from summary + numcols.foreach(vr => vr.metrics.foreach(mc => mc.result = getNumStats(summary, sojdf, mc.name, vr.colId))) + } + + //--3. calc str/other cols metrics-- + val strcols = req.filter(r => !r.isNum) + + //count function + def funcCount(df: DataFrame, col: Int): Long = { + dfCount + } + //null count function + def funcNullCount(df: DataFrame, col: Int): Long = { + val nullRow = df.map(row => if (row.isNullAt(col)) 1L else 0) + nullRow.fold(0)((a,b)=>a+b) + } + //unique count function + def funcUniqueCount(df: DataFrame, col: Int): Long = { + val dt = sojdf.schema(col).dataType + val getFunc = DataTypeUtils.dataType2RowGetFunc(dt) + + val mp = df.map(v=>(DataConverter.getString(getFunc(v, col))->1L)) + val rs = mp.reduceByKey(_+_) + rs.count() + } + //duplicate count function + def funcDuplicateCount(df: DataFrame, col: Int): Long = { + val dt = sojdf.schema(col).dataType + val getFunc = DataTypeUtils.dataType2RowGetFunc(dt) + + val mp = df.map(v=>(DataConverter.getString(getFunc(v, col))->1L)) + val rs = mp.reduceByKey(_+_) + rs.aggregate(0)((s, v) => if (v._2 == 1) s else s + 1, (s1, s2) => s1 + s2) + } + + //regex and match str metrics request + def getStrResult(df: DataFrame, op: Int, col: Int): Any = { + MetricsType(op) match { + case MetricsType.TotalCount => funcCount(df, col) + case MetricsType.NullCount => funcNullCount(df, col) + case MetricsType.UniqueCount => funcUniqueCount(df, col) + case MetricsType.DuplicateCount => funcDuplicateCount(df, col) + case _ => None + } + } + + if (strcols.length > 0) { + //calc str metrics one by one + strcols.foreach(vr => vr.metrics.foreach(mc => mc.result = getStrResult(sojdf, mc.name, vr.colId))) + } + + //union the num cols and str cols metrics, and put the result into configure object + val rsltCols = numcols.union(strcols) + configure.validityReq = rsltCols + + //output: need to change + logInfo("== result ==\n" + rsltCols) + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-models/src/main/scala/org/apache/griffin/validility/ValidityConfEntity.scala ---------------------------------------------------------------------- diff --git a/griffin-models/src/main/scala/org/apache/griffin/validility/ValidityConfEntity.scala b/griffin-models/src/main/scala/org/apache/griffin/validility/ValidityConfEntity.scala new file mode 100644 index 0000000..b488a15 --- /dev/null +++ b/griffin-models/src/main/scala/org/apache/griffin/validility/ValidityConfEntity.scala @@ -0,0 +1,17 @@ +package org.apache.griffin.validility + +import org.apache.griffin.common.PartitionPair + +class ValidityConfEntity { + var dataSet: String = _ + + var validityReq: List[ValidityReq] = List() + + var timePartitions: List[PartitionPair] = List() + + override def toString = "dataSet: " +dataSet+", validityReq: " +validityReq + +// { +// s"dataSet: $dataSet, validityReq: $validityReq" +// } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-models/src/main/scala/org/apache/griffin/validility/ValidityReq.scala ---------------------------------------------------------------------- diff --git a/griffin-models/src/main/scala/org/apache/griffin/validility/ValidityReq.scala b/griffin-models/src/main/scala/org/apache/griffin/validility/ValidityReq.scala new file mode 100644 index 0000000..5345269 --- /dev/null +++ b/griffin-models/src/main/scala/org/apache/griffin/validility/ValidityReq.scala @@ -0,0 +1,22 @@ +package org.apache.griffin.validility + +class ValidityReq { + var colId: Int = _ + var colName: String = _ + + var colType: String = _ + var isNum: Boolean = _ + + var metrics: List[MetricsReq] = List() + + override def toString = "colId: "+colId+", colName: "+colName+", colType: "+colType+", isNum: "+isNum+", metrics: "+metrics + +} + +class MetricsReq { + var name: Int = _ + var result: Any = _ + + override def toString = "name: "+name+", result: "+result + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-models/src/test/scala/modelTest/AccuTest.scala ---------------------------------------------------------------------- diff --git a/griffin-models/src/test/scala/modelTest/AccuTest.scala b/griffin-models/src/test/scala/modelTest/AccuTest.scala new file mode 100644 index 0000000..7f0699f --- /dev/null +++ b/griffin-models/src/test/scala/modelTest/AccuTest.scala @@ -0,0 +1,86 @@ +package modelTest + +import org.apache.griffin._ +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import org.apache.spark.{SparkConf, SparkContext} +import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} +import org.apache.spark.sql.{DataFrame, SQLContext} +import java.io.{FileInputStream, FileOutputStream} + +import org.apache.griffin.dataLoaderUtils.{DataLoaderFactory, FileLoaderUtil} + +import scala.collection.mutable.MutableList +import org.junit.runner.RunWith +import org.scalatest.junit.JUnitRunner +@RunWith(classOf[JUnitRunner]) +class AccuTest extends FunSuite with Matchers with BeforeAndAfter { + + val dataFilePath = FileLoaderUtil.convertPath("data/test/dataFile/") + val reqJsonPath = FileLoaderUtil.convertPath("data/test/reqJson/") + val recordFilePath = FileLoaderUtil.convertPath("data/test/recordFile/") + val recordFileName = "_RESULT_ACCU" + + case class AccuData() { + var cnt: Int = _ + var reqJson: String = _ + var configure: AccuracyConfEntity = _ + var dataFrameSrc: DataFrame = _ + var dataFrameTgt: DataFrame = _ + var result: ((Long, Long), List[String]) = _ + } + val accuDatas = MutableList[AccuData]() + + var sc: SparkContext = _ + + before { + val conf = new SparkConf().setMaster("local[*]").setAppName("AccTest") + sc = new SparkContext(conf) + val sqlContext = new SQLContext(sc) + val mapper = new ObjectMapper() + mapper.registerModule(DefaultScalaModule) + + var cnt = 1; + val accTests = List("accuAvroTest.json") + for (tf <- accTests) { + val reqJson = reqJsonPath + tf + val accuData = new AccuData() + accuData.cnt = cnt + accuData.reqJson = reqJson + val input = new FileInputStream(reqJson) + accuData.configure = mapper.readValue(input, classOf[AccuracyConfEntity]) + val dataLoader = DataLoaderFactory.getDataLoader(sqlContext, DataLoaderFactory.avro, dataFilePath) + val dfs = dataLoader.getAccuDataFrame(accuData.configure) + accuData.dataFrameSrc = dfs._1 + accuData.dataFrameTgt = dfs._2 + accuDatas += accuData + cnt += 1 + } + } + + test("test accuracy requests") { + for (accuData <- accuDatas) { + //-- algorithm -- + accuData.result = Accu.calcAccu(accuData.configure, accuData.dataFrameSrc, accuData.dataFrameTgt) + } + } + + after { + val out = new FileOutputStream(recordFilePath + recordFileName) + for (accuData <- accuDatas) { + //output + out.write(("//" + "=" * 10).getBytes("utf-8")) + out.write((s" ${accuData.cnt}. Test Accuracy model result with request file: ${accuData.reqJson} ").getBytes("utf-8")) + out.write(("=" * 10 + "\n").getBytes("utf-8")) + + val ((missCount, srcCount), missedList) = accuData.result + val rslt = s"match percentage: ${((1 - missCount.toDouble / srcCount) * 100)} %" + val rcds = missedList.mkString("\n") + val rcd = rslt + "\n\n" + rcds + "\n\n"; + + out.write(rcd.getBytes("utf-8")) + } + out.close() + sc.stop() + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-models/src/test/scala/modelTest/ValiTest.scala ---------------------------------------------------------------------- diff --git a/griffin-models/src/test/scala/modelTest/ValiTest.scala b/griffin-models/src/test/scala/modelTest/ValiTest.scala new file mode 100644 index 0000000..b487f66 --- /dev/null +++ b/griffin-models/src/test/scala/modelTest/ValiTest.scala @@ -0,0 +1,80 @@ +package modelTest + +import org.apache.griffin._ +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import org.apache.spark.{SparkConf, SparkContext} +import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} +import org.apache.spark.sql.{DataFrame, SQLContext} +import java.io.{FileInputStream, FileOutputStream} + +import org.apache.griffin.dataLoaderUtils.{DataLoaderFactory, FileLoaderUtil} + +import scala.collection.mutable.MutableList +import org.junit.runner.RunWith +import org.scalatest.junit.JUnitRunner + + +@RunWith(classOf[JUnitRunner]) +class ValiTest extends FunSuite with Matchers with BeforeAndAfter { + + val dataFilePath = FileLoaderUtil.convertPath("data/test/dataFile/") + val reqJsonPath = FileLoaderUtil.convertPath("data/test/reqJson/") + val recordFilePath = FileLoaderUtil.convertPath("data/test/recordFile/") + val recordFileName = "_RESULT_VALI" + + case class ValiData() { + var cnt: Int = _ + var reqJson: String = _ + var configure: ValidityConfEntity = _ + var dataFrame: DataFrame = _ + } + val valiDatas = MutableList[ValiData]() + + var sc: SparkContext = _ + + before { + val conf = new SparkConf().setMaster("local[*]").setAppName("AccTest") + sc = new SparkContext(conf) + val sqlContext = new SQLContext(sc) + val mapper = new ObjectMapper() + mapper.registerModule(DefaultScalaModule) + + var cnt = 1; + val valiTests = List("valiAvroTest.json") + for (tf <- valiTests) { + val reqJson = reqJsonPath + tf + val valiData = new ValiData() + valiData.cnt = cnt + valiData.reqJson = reqJson + val input = new FileInputStream(reqJson) + valiData.configure = mapper.readValue(input, classOf[ValidityConfEntity]) + val dataLoader = DataLoaderFactory.getDataLoader(sqlContext, DataLoaderFactory.avro, dataFilePath) + valiData.dataFrame = dataLoader.getValiDataFrame(valiData.configure) + valiDatas += valiData + cnt += 1 + } + } + + test("test validity requests") { + for (valiData <- valiDatas) { + //-- algorithm -- + Vali.calcVali(valiData.configure, valiData.dataFrame) + } + } + + after { + val out = new FileOutputStream(recordFilePath + recordFileName) + for (valiData <- valiDatas) { + //output + out.write(("//" + "=" * 10).getBytes("utf-8")) + out.write((s" ${valiData.cnt}. Test Validity model result with request file: ${valiData.reqJson} ").getBytes("utf-8")) + out.write(("=" * 10 + "\n").getBytes("utf-8")) + + val rcd = valiData.configure.toString + "\n\n" + out.write(rcd.getBytes("utf-8")) + } + out.close() + sc.stop() + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-models/src/test/scala/samples/junit.scala ---------------------------------------------------------------------- diff --git a/griffin-models/src/test/scala/samples/junit.scala b/griffin-models/src/test/scala/samples/junit.scala new file mode 100644 index 0000000..89513d5 --- /dev/null +++ b/griffin-models/src/test/scala/samples/junit.scala @@ -0,0 +1,17 @@ +package samples + +import org.junit._ +import Assert._ + +@Test +class AppTest { + + @Test + def testOK() = assertTrue(true) + +// @Test +// def testKO() = assertTrue(false) + +} + + http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-models/src/test/scala/samples/scalatest.scala ---------------------------------------------------------------------- diff --git a/griffin-models/src/test/scala/samples/scalatest.scala b/griffin-models/src/test/scala/samples/scalatest.scala new file mode 100644 index 0000000..d326656 --- /dev/null +++ b/griffin-models/src/test/scala/samples/scalatest.scala @@ -0,0 +1,109 @@ +/* + * Copyright 2001-2009 Artima, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package samples + +/* +ScalaTest facilitates different styles of testing by providing traits you can mix +together to get the behavior and syntax you prefer. A few examples are +included here. For more information, visit: + +http://www.scalatest.org/ + +One way to use ScalaTest is to help make JUnit or TestNG tests more +clear and concise. Here's an example: +*/ +import scala.collection.mutable.Stack +import org.scalatest.Assertions +import org.junit.Test + +class StackSuite extends Assertions { + +// @Test def stackShouldPopValuesIinLastInFirstOutOrder() { +// val stack = new Stack[Int] +// stack.push(1) +// stack.push(2) +// assert(stack.pop() === 2) +// assert(stack.pop() === 1) +// } + + @Test def stackShouldThrowNoSuchElementExceptionIfAnEmptyStackIsPopped() { + val emptyStack = new Stack[String] + intercept[NoSuchElementException] { + emptyStack.pop() + } + } +} + +/* +Here's an example of a FunSuite with ShouldMatchers mixed in: +*/ +import org.scalatest.FunSuite +import org.scalatest.matchers.ShouldMatchers + +import org.junit.runner.RunWith +import org.scalatest.junit.JUnitRunner +@RunWith(classOf[JUnitRunner]) +class ListSuite extends FunSuite with ShouldMatchers { + + test("An empty list should be empty") { + List() should be ('empty) + Nil should be ('empty) + } + + test("A non-empty list should not be empty") { + List(1, 2, 3) should not be ('empty) + List("fee", "fie", "foe", "fum") should not be ('empty) + } + + test("A list's length should equal the number of elements it contains") { + List() should have length (0) + List(1, 2) should have length (2) + List("fee", "fie", "foe", "fum") should have length (4) + } +} + +/* +ScalaTest also supports the behavior-driven development style, in which you +combine tests with text that specifies the behavior being tested. Here's +an example whose text output when run looks like: + +A Map +- should only contain keys and values that were added to it +- should report its size as the number of key/value pairs it contains +*/ +import org.scalatest.FunSpec +import scala.collection.mutable.Stack + +class ExampleSpec extends FunSpec { + +// describe("A Stack") { +// +// it("should pop values in last-in-first-out order") { +// val stack = new Stack[Int] +// stack.push(1) +// stack.push(2) +// assert(stack.pop() === 2) +// assert(stack.pop() === 1) +// } +// +// it("should throw NoSuchElementException if an empty stack is popped") { +// val emptyStack = new Stack[Int] +// intercept[NoSuchElementException] { +// emptyStack.pop() +// } +// } +// } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-models/src/test/scala/samples/specs.scala ---------------------------------------------------------------------- diff --git a/griffin-models/src/test/scala/samples/specs.scala b/griffin-models/src/test/scala/samples/specs.scala new file mode 100644 index 0000000..9e4dfe9 --- /dev/null +++ b/griffin-models/src/test/scala/samples/specs.scala @@ -0,0 +1,31 @@ +package samples + +import org.junit.runner.RunWith +import org.specs2.mutable._ +import org.specs2.runner._ + + +/** + * Sample specification. + * + * This specification can be executed with: scala -cp <your classpath=""> ${package}.SpecsTest + * Or using maven: mvn test + * + * For more information on how to write or run specifications, please visit: + * http://etorreborre.github.com/specs2/guide/org.specs2.guide.Runners.html + * + */ +@RunWith(classOf[JUnitRunner]) +class MySpecTest extends Specification { + "The 'Hello world' string" should { + "contain 11 characters" in { + "Hello world" must have size(11) + } + "start with 'Hello'" in { + "Hello world" must startWith("Hello") + } + "end with 'world'" in { + "Hello world" must endWith("world") + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-models/vali_config.json ---------------------------------------------------------------------- diff --git a/griffin-models/vali_config.json b/griffin-models/vali_config.json new file mode 100644 index 0000000..a07c4cd --- /dev/null +++ b/griffin-models/vali_config.json @@ -0,0 +1,131 @@ +{ + "dataSet": "users_info_src", + "validityReq": [ + { + "colId": 0, + "colName": "user_id", + "metrics": [ + { + "name": 1 + }, + { + "name": 2 + }, + { + "name": 3 + }, + { + "name": 4 + } + ] + }, + { + "colId": 1, + "colName": "first_name", + "metrics": [ + { + "name": 1 + }, + { + "name": 2 + }, + { + "name": 3 + }, + { + "name": 4 + } + ] + }, + { + "colId": 2, + "colName": "last_name", + "metrics": [ + { + "name": 1 + }, + { + "name": 2 + }, + { + "name": 3 + }, + { + "name": 4 + } + ] + }, + { + "colId": 3, + "colName": "address", + "metrics": [ + { + "name": 1 + }, + { + "name": 2 + }, + { + "name": 3 + }, + { + "name": 4 + } + ] + }, + { + "colId": 4, + "colName": "email", + "metrics": [ + { + "name": 1 + }, + { + "name": 2 + }, + { + "name": 3 + }, + { + "name": 4 + } + ] + }, + { + "colId": 5, + "colName": "phone", + "metrics": [ + { + "name": 1 + }, + { + "name": 2 + }, + { + "name": 3 + }, + { + "name": 4 + } + ] + }, + { + "colId": 6, + "colName": "post_code", + "metrics": [ + { + "name": 1 + }, + { + "name": 2 + }, + { + "name": 3 + }, + { + "name": 4 + } + ] + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-scheduler/pom.xml ---------------------------------------------------------------------- diff --git a/griffin-scheduler/pom.xml b/griffin-scheduler/pom.xml new file mode 100644 index 0000000..0231c6e --- /dev/null +++ b/griffin-scheduler/pom.xml @@ -0,0 +1,83 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright (c) 2016 eBay Software Foundation. Licensed under the Apache + License, Version 2.0 (the "License"); you may not use this file except in + compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <parent> + <groupId>com.ebay.oss</groupId> + <artifactId>griffin-parent</artifactId> + <version>0.1.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + + <modelVersion>4.0.0</modelVersion> + <artifactId>griffin-scheduler</artifactId> + <name>griffin-scheduler</name> + <packaging>jar</packaging> + + + <dependencies> + +<!-- <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <version>2.4</version> + </dependency> + + --> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + + + + + <dependency> + <groupId>org.mongodb</groupId> + <artifactId>mongo-java-driver</artifactId> + <version>${mongo.version}</version> + </dependency> + + +<!-- <dependency> + <groupId>com.google.code.morphia</groupId> + <artifactId>morphia</artifactId> + <version>0.104</version> + </dependency> + + <dependency> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + </dependency> --> + + + + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </dependency> + </dependencies> + + <build> + <finalName>griffin-scheduler</finalName> + <plugins> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <testFailureIgnore>true</testFailureIgnore> + </configuration> + </plugin> + </plugins> + </build> + +</project> http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-scheduler/src/main/java/org/apache/bark/scheduler/BarkScheduler.java ---------------------------------------------------------------------- diff --git a/griffin-scheduler/src/main/java/org/apache/bark/scheduler/BarkScheduler.java b/griffin-scheduler/src/main/java/org/apache/bark/scheduler/BarkScheduler.java new file mode 100644 index 0000000..f1cd084 --- /dev/null +++ b/griffin-scheduler/src/main/java/org/apache/bark/scheduler/BarkScheduler.java @@ -0,0 +1,11 @@ +package org.apache.griffin.scheduler; +/** + * TODO::This is for the next version design, currently scheduler is in bark-core + * @author lzhixing + * + */ +public class BarkScheduler { + // public String sayHello(){ + // return "Hello"; + // } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-scheduler/src/test/java/org/apache/bark/scheduler/BarkSchedulerTest.java ---------------------------------------------------------------------- diff --git a/griffin-scheduler/src/test/java/org/apache/bark/scheduler/BarkSchedulerTest.java b/griffin-scheduler/src/test/java/org/apache/bark/scheduler/BarkSchedulerTest.java new file mode 100644 index 0000000..70b75e2 --- /dev/null +++ b/griffin-scheduler/src/test/java/org/apache/bark/scheduler/BarkSchedulerTest.java @@ -0,0 +1,13 @@ +package org.apache.griffin.scheduler; + + +public class BarkSchedulerTest { + // @Test + // public void testSayHello(){ + // BarkScheduler sch = new BarkScheduler(); + // sch.sayHello(); + // assertEquals("Hello", sch.sayHello()); + // + // } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-ui/.gitignore ---------------------------------------------------------------------- diff --git a/griffin-ui/.gitignore b/griffin-ui/.gitignore new file mode 100644 index 0000000..90ad874 --- /dev/null +++ b/griffin-ui/.gitignore @@ -0,0 +1,6 @@ +node_modules +test-coverage +build +/.idea/ +*.iml +target \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-ui/apidocs/bark.json ---------------------------------------------------------------------- diff --git a/griffin-ui/apidocs/bark.json b/griffin-ui/apidocs/bark.json new file mode 100644 index 0000000..9a01d6e --- /dev/null +++ b/griffin-ui/apidocs/bark.json @@ -0,0 +1,737 @@ +{ + "swagger": "2.0", + "info": { + "title": "Griffin API", + "description": "Move your app forward with the Griffin API", + "version": "1.0.0" + }, + "host": "localhost:8080", + "schemes": [ + "http" + ], + "produces": [ + "application/json" + ], + "paths": { + "/api/v1/dq/metrics/heatmap": { + "get": { + "summary": "User Profile", + "description": "heatmap", + "tags": [ + "health" + ], + "responses": { + "200": { + "description": "success" + }, + "default": { + "description": "Unexpected error" + } + } + } + }, + "/js/mock_data/statistics.json": { + "get": { + "summary": "User Profile", + "description": "statistics", + "tags": [ + "health" + ], + "responses": { + "200": { + "description": "success" + }, + "default": { + "description": "Unexpected error" + } + } + } + }, + "/js/mock_data/rulemetric.json": { + "get": { + "summary": "User Profile", + "description": "rulemetric", + "responses": { + "200": { + "description": "success" + }, + "default": { + "description": "Unexpected error" + } + } + } + }, + "/api/v1/dq/metrics/briefmetrics": { + "get": { + "summary": "User Profile", + "description": "Metrics on side bar", + "tags": [ + "health" + ], + "responses": { + "200": { + "description": "success" + }, + "default": { + "description": "Unexpected error" + } + } + } + }, + "/api/v1/dataassets/srctree": { + "get": { + "summary": "User Profile", + "description": "Metrics on side bar", + "tags": [ + "createrule-ac" + ], + "responses": { + "200": { + "description": "success" + }, + "default": { + "description": "Unexpected error" + } + } + } + }, + "/api/v1/dataassets/{id}": { + "get": { + "summary": "User Profile", + "description": "Get specfied data in datasets", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "description": "ID like 1,2,3...", + "type": "integer" + } + ], + "tags": [ + "createrule-ac" + ], + "responses": { + "200": { + "description": "success" + }, + "default": { + "description": "Unexpected error" + } + } + } + }, + "/api/v1/dq/metrics/complete/{id}": { + "get": { + "summary": "User Profile", + "description": "metrics/complete", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "description": "ID like accuracy_viewitem_queue, accuracy_bid_new_queue,...", + "type": "string" + } + ], + "tags": [ + "metrics/Bullseye" + ], + "responses": { + "200": { + "description": "success" + }, + "default": { + "description": "Unexpected error" + } + } + } + }, + "/api/v1/dq/metrics/dashboard/Bullseye": { + "get": { + "summary": "User Profile", + "description": "dashboard", + "tags": [ + "metrics/Bullseye" + ], + "responses": { + "200": { + "description": "success" + }, + "default": { + "description": "Unexpected error" + } + } + } + }, + "/api/v1/dq/metrics/dashboard": { + "get": { + "summary": "User Profile", + "description": "dashboard", + "tags": [ + "metrics" + ], + "responses": { + "200": { + "description": "success" + }, + "default": { + "description": "Unexpected error" + } + } + } + }, + "/api/v1/model/allModels": { + "get": { + "summary": "User Profile", + "description": "allModels", + "tags": [ + "rules" + ], + "responses": { + "200": { + "description": "success" + }, + "default": { + "description": "Unexpected error" + } + } + } + }, + "/api/v1/model/getAnomalyModel/{id}": { + "get": { + "summary": "User Profile", + "description": "GetAnomalyModel by id", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "description": "ID like vad1, ...", + "type": "string" + } + ], + "tags": [ + "viewrule" + ], + "responses": { + "200": { + "description": "success" + }, + "default": { + "description": "Unexpected error" + } + } + } + }, + "/api/v1/model/getPublishModel/{id}": { + "get": { + "summary": "User Profile", + "description": "GetPublishModel by id", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "description": "ID like vad1, ...", + "type": "string" + } + ], + "tags": [ + "viewrule" + ], + "responses": { + "200": { + "description": "success" + }, + "default": { + "description": "Unexpected error" + } + } + } + }, + "/api/v1/model/getAccuracyModel/{id}": { + "get": { + "summary": "User Profile", + "description": "GetAccuracyModel by id", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "description": "ID like test_accuracy, ...", + "type": "string" + } + ], + "tags": [ + "viewrule" + ], + "responses": { + "200": { + "description": "success" + }, + "default": { + "description": "Unexpected error" + } + } + } + }, + "/api/v1/model/getValidityModel/{id}": { + "get": { + "summary": "User Profile", + "description": "GetValidityModel by id", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "description": "ID like vad1, ...", + "type": "string" + } + ], + "tags": [ + "viewrule" + ], + "responses": { + "200": { + "description": "success" + }, + "default": { + "description": "Unexpected error" + } + } + } + }, + "/api/v1/model/newAccuracyModel": { + "post": { + "summary": "User Profile", + "description": "newAccuracyModel", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "description": "Please only change the name! ", + "schema": { + "$ref": "#/definitions/newAccuracyModel" + } + } + ], + "tags": [ + "viewrule" + ], + "responses": { + "200": { + "description": "Profile information for a user" + }, + "default": { + "description": "Unexpected error" + } + } + } + }, + "/api/v1/model/newValidityModel": { + "post": { + "summary": "User Profile", + "description": "newValidityModel", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "description": "Please only change the name! ", + "schema": { + "$ref": "#/definitions/newValidityModel" + } + } + ], + "tags": [ + "viewrule" + ], + "responses": { + "200": { + "description": "Profile information for a user" + }, + "default": { + "description": "Unexpected error" + } + } + } + }, + "/api/v1/model/newAnomalyModel": { + "post": { + "summary": "User Profile", + "description": "newAnomalyModel", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "description": "Please only change the name! ", + "schema": { + "$ref": "#/definitions/newAnomalyModel" + } + } + ], + "tags": [ + "viewrule" + ], + "responses": { + "200": { + "description": "Profile information for a user" + }, + "default": { + "description": "Unexpected error" + } + } + } + }, + "/api/v1/model/newPublishModel": { + "post": { + "summary": "User Profile", + "description": "newPublishModel", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "description": "Please only change the name! ", + "schema": { + "$ref": "#/definitions/newPublishModel" + } + } + ], + "tags": [ + "viewrule" + ], + "responses": { + "200": { + "description": "Profile information for a user" + }, + "default": { + "description": "Unexpected error" + } + } + } + }, + "/api/v1/model/deleteModel/{id}": { + "delete": { + "summary": "User Profile", + "description": "Get specfied data in datasets", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "description": "ID like wx_publish, ...", + "type": "string" + } + ], + "tags": [ + "viewrule" + ], + "responses": { + "200": { + "description": "success" + }, + "default": { + "description": "Unexpected error" + } + } + } + } + }, + "definitions": { + "mappingsItem":{ + "type": "object", + "properties": { + "target": { + "type": "string", + "enum": ["sitespeed.key"] + }, + "src": { + "type": "string", + "enum": ["dw_bid.uid"] + }, + "matchMethod": { + "type": "string", + "enum": ["EXACT"] + }, + "isPk": { + "type": "boolean", + "enum": [true] + } + } + }, + "newAccuracyModel": { + "type": "object", + "properties": { + "basic": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["0"] + }, + "system": { + "type": "string", + "enum": ["2"] + }, + "threshold": { + "type": "integer", + "enum": [90] + }, + "scheduleType": { + "type": "string", + "enum": ["0"] + }, + "owner": { + "type": "string", + "enum": ["xwang21"] + }, + "name": { + "type": "string", + "enum": ["wx_ac"] + }, + "desc": { + "type": "string", + "enum": ["ggggg"] + }, + "email": { + "type": "string", + "enum": ["g@g"] + }, + "dataaset": { + "type": "string", + "enum": ["sitespeed"] + }, + "dataasetId": { + "type": "integer", + "enum": [21] + } + } + }, + "extra": { + "type": "object", + "properties": { + "srcDb": { + "type": "string", + "enum": ["Apollo"] + }, + "srcDataSet": { + "type": "string", + "enum": ["Bullseye"] + }, + "targetDb": { + "type": "string", + "enum": ["Apollo"] + }, + "targetDataSet": { + "type": "string", + "enum": ["SiteSpeed"] + } + } + }, + "mappings": { + "type": "array", + "items":{ + "$ref": "mappingsItem" + } + } + } + }, + "newValidityModel": { + "type": "object", + "properties": { + "basic": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["1"] + }, + "system": { + "type": "string", + "enum": ["3"] + }, + "scheduleType": { + "type": "string", + "enum": ["1"] + }, + "owner": { + "type": "string", + "enum": ["xwang21"] + }, + "name": { + "type": "string", + "enum": ["wx_valid"] + }, + "desc": { + "type": "string", + "enum": ["wwwwww"] + }, + "threshold": { + "type": "integer", + "enum": [60] + }, + "email": { + "type": "string", + "enum": ["w@f"] + }, + "dataaset": { + "type": "string", + "enum": ["ubi_event"] + }, + "dataasetId": { + "type": "integer", + "enum": [22] + } + } + }, + "extra": { + "type": "object", + "properties": { + "srcDb": { + "type": "string", + "enum": ["Apollo"] + }, + "srcDataSet": { + "type": "string", + "enum": ["Sojourner"] + } + } + }, + "vaType": { + "type": "string", + "enum": ["5"] + }, + "column": { + "type": "string", + "enum": ["guid"] + } + } + }, + "newAnomalyModel": { + "type": "object", + "properties": { + "basic": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["2"] + }, + "system": { + "type": "string", + "enum": ["4"] + }, + "scheduleType": { + "type": "string", + "enum": ["0"] + }, + "owner": { + "type": "string", + "enum": ["xwang21"] + }, + "name": { + "type": "string", + "enum": ["wx_detec"] + }, + "desc": { + "type": "string", + "enum": ["wwwwww"] + }, + "threshold": { + "type": "integer", + "enum": [30] + }, + "email": { + "type": "string", + "enum": ["w@f"] + }, + "dataaset": { + "type": "string", + "enum": ["ubi_event"] + }, + "dataasetId": { + "type": "integer", + "enum": [22] + } + } + }, + "extra": { + "type": "object", + "properties": { + "srcDb": { + "type": "string", + "enum": ["Apollo"] + }, + "srcDataSet": { + "type": "string", + "enum": ["Sojourner"] + } + } + }, + "anType": { + "type": "string", + "enum": ["3"] + } + } + }, + "newPublishModel": { + "type": "object", + "properties": { + "basic": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["3"] + }, + "system": { + "type": "string", + "enum": ["3"] + }, + "scheduleType": { + "type": "string", + "enum": ["0"] + }, + "owner": { + "type": "string", + "enum": ["xwang21"] + }, + "name": { + "type": "string", + "enum": ["wx_publish"] + }, + "desc": { + "type": "string", + "enum": ["wwwwww"] + }, + "dataaset": { + "type": "string", + "enum": ["sssss"] + }, + "threshold": { + "type": "integer", + "enum": [80] + }, + "email": { + "type": "string", + "enum": ["w@f"] + } + } + }, + "extra": { + "type": "object", + "properties": { + "publishUrl": { + "type": "string", + "enum": ["http://dq.vip.ebay.com/api/v1/publishmetric/wx_publish";] + } + } + } + } + } + } +}
