http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/process/temp/TimeRange.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/process/temp/TimeRange.scala b/measure/src/main/scala/org/apache/griffin/measure/process/temp/TimeRange.scala deleted file mode 100644 index 4073753..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/process/temp/TimeRange.scala +++ /dev/null @@ -1,48 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.process.temp - -import scala.math.{min, max} - -case class TimeRange(begin: Long, end: Long, tmsts: Set[Long]) extends Serializable { - def merge(tr: TimeRange): TimeRange = { - TimeRange(min(begin, tr.begin), max(end, tr.end), tmsts ++ tr.tmsts) - } - def minTmstOpt: Option[Long] = { - try { - if (tmsts.nonEmpty) Some(tmsts.min) else None - } catch { - case _: Throwable => None - } - } -} - -object TimeRange { - val emptyTimeRange = TimeRange(0, 0, Set[Long]()) - def apply(range: (Long, Long), tmsts: Set[Long]): TimeRange = TimeRange(range._1, range._2, tmsts) - def apply(ts: Long, tmsts: Set[Long]): TimeRange = TimeRange(ts, ts, tmsts) - def apply(ts: Long): TimeRange = TimeRange(ts, ts, Set[Long](ts)) - def apply(tmsts: Set[Long]): TimeRange = { - try { - TimeRange(tmsts.min, tmsts.max, tmsts) - } catch { - case _: Throwable => emptyTimeRange - } - } -} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/result/AccuracyResult.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/result/AccuracyResult.scala b/measure/src/main/scala/org/apache/griffin/measure/result/AccuracyResult.scala deleted file mode 100644 index 7b75043..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/result/AccuracyResult.scala +++ /dev/null @@ -1,50 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.result - -// result for accuracy: miss count, total count -case class AccuracyResult(miss: Long, total: Long) extends Result { - - type T = AccuracyResult - - override def isLegal(): Boolean = getTotal > 0 - - def update(delta: T): T = { - AccuracyResult(delta.miss, total) - } - - def initial(): Boolean = { - getMatch <= 0 - } - - def eventual(): Boolean = { - this.miss <= 0 - } - - def differsFrom(other: T): Boolean = { - (this.miss != other.miss) || (this.total != other.total) - } - - def getMiss = miss - def getTotal = total - def getMatch = total - miss - - def matchPercentage: Double = if (getTotal <= 0) 0 else getMatch.toDouble / getTotal * 100 - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/result/DataInfo.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/result/DataInfo.scala b/measure/src/main/scala/org/apache/griffin/measure/result/DataInfo.scala deleted file mode 100644 index 7ec0783..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/result/DataInfo.scala +++ /dev/null @@ -1,50 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.result - - -sealed trait DataInfo { - type T - val key: String - def wrap(value: T) = (key -> value) - def defWrap() = wrap(dfv) - val dfv: T -} - -final case object TimeStampInfo extends DataInfo { - type T = Long - val key = "_tmst_" - val dfv = 0L -} - -final case object MismatchInfo extends DataInfo { - type T = String - val key = "_mismatch_" - val dfv = "" -} - -final case object ErrorInfo extends DataInfo { - type T = String - val key = "_error_" - val dfv = "" -} - -object DataInfo { - val cacheInfoList = List(TimeStampInfo, MismatchInfo, ErrorInfo) -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/result/ProfileResult.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/result/ProfileResult.scala b/measure/src/main/scala/org/apache/griffin/measure/result/ProfileResult.scala deleted file mode 100644 index c90e095..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/result/ProfileResult.scala +++ /dev/null @@ -1,48 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.result - -// result for profile: match count, total count -case class ProfileResult(matchCount: Long, totalCount: Long) extends Result { - - type T = ProfileResult - - def update(delta: T): T = { - ProfileResult(matchCount + delta.matchCount, totalCount) - } - - def initial(): Boolean = { - this.matchCount <= 0 - } - - def eventual(): Boolean = { - this.matchCount >= totalCount - } - - def differsFrom(other: T): Boolean = { - (this.matchCount != other.matchCount) || (this.totalCount != other.totalCount) - } - - def getMiss = totalCount - matchCount - def getTotal = totalCount - def getMatch = matchCount - - def matchPercentage: Double = if (getTotal <= 0) 0 else getMatch.toDouble / getTotal * 100 - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/result/Result.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/result/Result.scala b/measure/src/main/scala/org/apache/griffin/measure/result/Result.scala deleted file mode 100644 index caf6d96..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/result/Result.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.result - - -trait Result extends Serializable { - - type T <: Result - - def isLegal(): Boolean = true - - def update(delta: T): T - - def initial(): Boolean - - def eventual(): Boolean - - def differsFrom(other: T): Boolean - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/AdaptPhase.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/AdaptPhase.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/AdaptPhase.scala deleted file mode 100644 index 26db78d..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/AdaptPhase.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.adaptor - -sealed trait AdaptPhase {} - -final case object PreProcPhase extends AdaptPhase {} - -final case object RunPhase extends AdaptPhase {} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/DataFrameOprAdaptor.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/DataFrameOprAdaptor.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/DataFrameOprAdaptor.scala deleted file mode 100644 index 0b0b461..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/DataFrameOprAdaptor.scala +++ /dev/null @@ -1,62 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.adaptor - -import org.apache.griffin.measure.process.{ExportMode, ProcessType} -import org.apache.griffin.measure.process.temp.TimeRange -import org.apache.griffin.measure.rule.plan.{TimeInfo, _} -import org.apache.griffin.measure.utils.ParamUtil._ - -case class DataFrameOprAdaptor() extends RuleAdaptor { - -// def genRuleStep(timeInfo: TimeInfo, param: Map[String, Any]): Seq[RuleStep] = { -// val ruleInfo = RuleInfoGen(param, timeInfo) -// DfOprStep(timeInfo, ruleInfo) :: Nil -//// DfOprStep(getName(param), getRule(param), getDetails(param), -//// getPersistType(param), getUpdateDataSource(param)) :: Nil -// } -// def adaptConcreteRuleStep(ruleStep: RuleStep): Seq[ConcreteRuleStep] = { -// ruleStep match { -// case rs @ DfOprStep(_, _) => rs :: Nil -// case _ => Nil -// } -// } - -// def getTempSourceNames(param: Map[String, Any]): Seq[String] = { -// param.get(_name) match { -// case Some(name) => name.toString :: Nil -// case _ => Nil -// } -// } - - import RuleParamKeys._ - - def genRulePlan(timeInfo: TimeInfo, param: Map[String, Any], - procType: ProcessType, dsTimeRanges: Map[String, TimeRange]): RulePlan = { - val name = getRuleName(param) - val step = DfOprStep(name, getRule(param), getDetails(param), getCache(param), getGlobal(param)) - val mode = ExportMode.defaultMode(procType) - RulePlan( - step :: Nil, - genRuleExports(param, name, name, timeInfo.calcTime, mode), - genDsUpdates(param, "", name) - ) - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/GlobalKeys.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/GlobalKeys.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/GlobalKeys.scala deleted file mode 100644 index f6f35da..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/GlobalKeys.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.adaptor - -object GlobalKeys { - val _initRule = "init.rule" -} - -object ProcessDetailsKeys { - val _baselineDataSource = "baseline.data.source" -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/GriffinDslAdaptor.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/GriffinDslAdaptor.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/GriffinDslAdaptor.scala deleted file mode 100644 index d07aa02..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/GriffinDslAdaptor.scala +++ /dev/null @@ -1,73 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.adaptor - -import org.apache.griffin.measure.process.temp._ -import org.apache.griffin.measure.process._ -import org.apache.griffin.measure.rule.dsl.parser.GriffinDslParser -import org.apache.griffin.measure.rule.plan.{TimeInfo, _} -import org.apache.griffin.measure.rule.trans._ - -import scala.util.{Failure, Success} - -case class GriffinDslAdaptor(dataSourceNames: Seq[String], - functionNames: Seq[String] - ) extends RuleAdaptor { - - import RuleParamKeys._ - - val filteredFunctionNames = functionNames.filter { fn => - fn.matches("""^[a-zA-Z_]\w*$""") - } - val parser = GriffinDslParser(dataSourceNames, filteredFunctionNames) - - private val emptyRulePlan = RulePlan(Nil, Nil) - - override def genRulePlan(timeInfo: TimeInfo, param: Map[String, Any], - processType: ProcessType, dsTimeRanges: Map[String, TimeRange] - ): RulePlan = { - val name = getRuleName(param) - val rule = getRule(param) - val dqType = getDqType(param) - try { - val result = parser.parseRule(rule, dqType) - if (result.successful) { - val expr = result.get - val rulePlanTrans = RulePlanTrans(dqType, dataSourceNames, timeInfo, - name, expr, param, processType, dsTimeRanges) - rulePlanTrans.trans match { - case Success(rp) => rp - case Failure(ex) => { - warn(s"translate rule [ ${rule} ] fails: \n${ex.getMessage}") - emptyRulePlan - } - } - } else { - warn(s"parse rule [ ${rule} ] fails: \n${result}") - emptyRulePlan - } - } catch { - case e: Throwable => { - error(s"generate rule plan ${name} fails: ${e.getMessage}") - emptyRulePlan - } - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/InternalColumns.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/InternalColumns.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/InternalColumns.scala deleted file mode 100644 index fa04288..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/InternalColumns.scala +++ /dev/null @@ -1,35 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.adaptor - -object InternalColumns { - val tmst = "__tmst" - val metric = "__metric" - val record = "__record" - val empty = "__empty" - - val beginTs = "__begin_ts" - val endTs = "__end_ts" - - val distinct = "__distinct" - - val rowNumber = "__rn" - - val columns = List[String](tmst, metric, record, empty, beginTs, endTs, distinct, rowNumber) -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/RuleAdaptor.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/RuleAdaptor.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/RuleAdaptor.scala deleted file mode 100644 index e85575f..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/RuleAdaptor.scala +++ /dev/null @@ -1,100 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.adaptor - -import java.util.concurrent.atomic.AtomicLong - -import org.apache.griffin.measure.log.Loggable -import org.apache.griffin.measure.process.{ExportMode, ProcessType} -import org.apache.griffin.measure.process.temp.TimeRange -import org.apache.griffin.measure.rule.dsl._ -import org.apache.griffin.measure.rule.plan.{TimeInfo, _} -import org.apache.griffin.measure.rule.trans.{DsUpdateFactory, RuleExportFactory} -import org.apache.griffin.measure.utils.ParamUtil._ - -object RuleParamKeys { - val _name = "name" - val _rule = "rule" - val _dslType = "dsl.type" - val _dqType = "dq.type" - val _cache = "cache" - val _global = "global" - val _details = "details" - - val _metric = "metric" - val _record = "record" - val _dsUpdate = "ds.update" - - def getName(param: Map[String, Any], defName: String): String = param.getString(_name, defName) - def getRule(param: Map[String, Any]): String = param.getString(_rule, "") - def getDqType(param: Map[String, Any]): DqType = DqType(param.getString(_dqType, "")) - def getCache(param: Map[String, Any]): Boolean = param.getBoolean(_cache, false) - def getGlobal(param: Map[String, Any]): Boolean = param.getBoolean(_global, false) - def getDetails(param: Map[String, Any]): Map[String, Any] = param.getParamMap(_details) - - def getMetricOpt(param: Map[String, Any]): Option[Map[String, Any]] = param.getParamMapOpt(_metric) - def getRecordOpt(param: Map[String, Any]): Option[Map[String, Any]] = param.getParamMapOpt(_record) - def getDsUpdateOpt(param: Map[String, Any]): Option[Map[String, Any]] = param.getParamMapOpt(_dsUpdate) -} - -trait RuleAdaptor extends Loggable with Serializable { - - protected def getRuleName(param: Map[String, Any]): String = { - RuleParamKeys.getName(param, RuleStepNameGenerator.genName) - } - - def genRulePlan(timeInfo: TimeInfo, param: Map[String, Any], - procType: ProcessType, dsTimeRanges: Map[String, TimeRange]): RulePlan - - protected def genRuleExports(param: Map[String, Any], defName: String, - stepName: String, defTimestamp: Long, - mode: ExportMode - ): Seq[RuleExport] = { - val metricOpt = RuleParamKeys.getMetricOpt(param) - val metricExportSeq = metricOpt.map( - RuleExportFactory.genMetricExport(_, defName, stepName, defTimestamp, mode) - ).toSeq - val recordOpt = RuleParamKeys.getRecordOpt(param) - val recordExportSeq = recordOpt.map( - RuleExportFactory.genRecordExport(_, defName, stepName, defTimestamp, mode) - ).toSeq - metricExportSeq ++ recordExportSeq - } - - protected def genDsUpdates(param: Map[String, Any], defDsName: String, - stepName: String - ): Seq[DsUpdate] = { - val dsUpdateOpt = RuleParamKeys.getDsUpdateOpt(param) - dsUpdateOpt.map(DsUpdateFactory.genDsUpdate(_, defDsName, stepName)).toSeq - } - -} - -object RuleStepNameGenerator { - private val counter: AtomicLong = new AtomicLong(0L) - private val head: String = "rs" - - def genName: String = { - s"${head}${increment}" - } - - private def increment: Long = { - counter.incrementAndGet() - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/RuleAdaptorGroup.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/RuleAdaptorGroup.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/RuleAdaptorGroup.scala deleted file mode 100644 index 5b5f419..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/RuleAdaptorGroup.scala +++ /dev/null @@ -1,311 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.adaptor - -import org.apache.griffin.measure.cache.tmst.TempName -import org.apache.griffin.measure.config.params.user._ -import org.apache.griffin.measure.process.ProcessType -import org.apache.griffin.measure.process.temp.{TableRegisters, TimeRange} -import org.apache.griffin.measure.rule.dsl._ -import org.apache.griffin.measure.rule.plan._ -import org.apache.spark.sql.{Encoders, SQLContext, SparkSession} - -import scala.collection.mutable.{Map => MutableMap} - -object RuleAdaptorGroup { - - val _dslType = "dsl.type" -// import RuleInfoKeys._ - - var dataSourceNames: Seq[String] = Nil - var functionNames: Seq[String] = Nil - - var baselineDsName: String = "" - - private val emptyRulePlan = RulePlan(Nil, Nil) - - def init(dsNames: Seq[String], blDsName: String, funcNames: Seq[String]): Unit = { - dataSourceNames = dsNames - baselineDsName = blDsName - functionNames = funcNames - } - - def init(sparkSession: SparkSession, dsNames: Seq[String], blDsName: String): Unit = { - implicit val encoder = Encoders.STRING - val functions = sparkSession.catalog.listFunctions - functionNames = functions.map(_.name).collect.toSeq - dataSourceNames = dsNames - - baselineDsName = blDsName - } - -// def init(sqlContext: SQLContext, dsNames: Seq[String], blDsName: String): Unit = { - // val functions = sqlContext.sql("show functions") - // functionNames = functions.map(_.getString(0)).collect.toSeq - // dataSourceNames = dsNames - // - // baselineDsName = blDsName - // } - - private def getDslType(param: Map[String, Any], defDslType: DslType) = { - DslType(param.getOrElse(_dslType, defDslType.desc).toString) - } - - private def genRuleAdaptor(dslType: DslType, dsNames: Seq[String] - ): Option[RuleAdaptor] = { - dslType match { - case SparkSqlType => Some(SparkSqlAdaptor()) - case DfOprType => Some(DataFrameOprAdaptor()) - case GriffinDslType => Some(GriffinDslAdaptor(dsNames, functionNames)) - case _ => None - } - } - -// def genRuleSteps(evaluateRuleParam: EvaluateRuleParam): Seq[RuleStep] = { -// val dslTypeStr = if (evaluateRuleParam.dslType == null) "" else evaluateRuleParam.dslType -// val defaultDslType = DslType(dslTypeStr) -// val rules = evaluateRuleParam.rules -// var dsNames = dataSourceNames -// val steps = rules.flatMap { param => -// val dslType = getDslType(param) -// genRuleAdaptor(dslType) match { -// case Some(ruleAdaptor) => ruleAdaptor.genRuleStep(param) -// case _ => Nil -// } -// } -// steps.foreach(println) -// steps -// } - -// def genConcreteRuleSteps(timeInfo: TimeInfo, evaluateRuleParam: EvaluateRuleParam, -// dsTmsts: Map[String, Set[Long]], procType: ProcessType, -// adaptPhase: AdaptPhase -// ): Seq[ConcreteRuleStep] = { -// val dslTypeStr = if (evaluateRuleParam.dslType == null) "" else evaluateRuleParam.dslType -// val defaultDslType = DslType(dslTypeStr) -// val ruleParams = evaluateRuleParam.rules -// genConcreteRuleSteps(timeInfo, ruleParams, dsTmsts, defaultDslType, procType, adaptPhase) -// } -// -// def genConcreteRuleSteps(timeInfo: TimeInfo, ruleParams: Seq[Map[String, Any]], -// dsTmsts: Map[String, Set[Long]], defDslType: DslType, -// procType: ProcessType, adaptPhase: AdaptPhase -// ): Seq[ConcreteRuleStep] = { -// val (steps, dsNames) = ruleParams.foldLeft((Seq[ConcreteRuleStep](), dataSourceNames)) { (res, param) => -// val (preSteps, preNames) = res -// val dslType = getDslType(param, defDslType) -// val (curSteps, curNames) = genRuleAdaptor(dslType, preNames, procType, adaptPhase) match { -// case Some(ruleAdaptor) => { -// val concreteSteps = ruleAdaptor.genConcreteRuleStep(timeInfo, param, dsTmsts) -// (concreteSteps, preNames ++ ruleAdaptor.getPersistNames(concreteSteps)) -// } -// case _ => (Nil, preNames) -// } -// (preSteps ++ curSteps, curNames) -// } -// steps -// } - - // -- gen rule plan -- - def genRulePlan(timeInfo: TimeInfo, evaluateRuleParam: EvaluateRuleParam, - procType: ProcessType, dsTimeRanges: Map[String, TimeRange] - ): RulePlan = { - val dslTypeStr = if (evaluateRuleParam.dslType == null) "" else evaluateRuleParam.dslType - val defaultDslType = DslType(dslTypeStr) - val ruleParams = evaluateRuleParam.rules - genRulePlan(timeInfo, ruleParams, defaultDslType, procType, dsTimeRanges) - } - - def genRulePlan(timeInfo: TimeInfo, ruleParams: Seq[Map[String, Any]], - defaultDslType: DslType, procType: ProcessType, - dsTimeRanges: Map[String, TimeRange] - ): RulePlan = { - val (rulePlan, dsNames) = ruleParams.foldLeft((emptyRulePlan, dataSourceNames)) { (res, param) => - val (plan, names) = res - val dslType = getDslType(param, defaultDslType) - val curPlan: RulePlan = genRuleAdaptor(dslType, names) match { - case Some(adaptor) => adaptor.genRulePlan(timeInfo, param, procType, dsTimeRanges) - case _ => emptyRulePlan - } - val globalNames = curPlan.globalRuleSteps.map(_.name) - globalNames.foreach(TableRegisters.registerCompileGlobalTable(_)) - val curNames = curPlan.normalRuleSteps.map(_.name) - curNames.foreach(TableRegisters.registerCompileTempTable(timeInfo.key, _)) - - val retPlan = plan.merge(curPlan) - (retPlan, names ++ globalNames ++ curNames) - } - - rulePlan - } - - - // -- gen steps -- -// def genRuleSteps(timeInfo: TimeInfo, evaluateRuleParam: EvaluateRuleParam, dsTmsts: Map[String, Set[Long]] -// ): Seq[ConcreteRuleStep] = { -// val dslTypeStr = if (evaluateRuleParam.dslType == null) "" else evaluateRuleParam.dslType -// val defaultDslType = DslType(dslTypeStr) -// val ruleParams = evaluateRuleParam.rules -// val tmsts = dsTmsts.getOrElse(baselineDsName, Set[Long]()).toSeq -// genRuleSteps(timeInfo, ruleParams, tmsts, defaultDslType) -// } -// -// def genRuleSteps(timeInfo: TimeInfo, ruleParams: Seq[Map[String, Any]], -// tmsts: Seq[Long], defaultDslType: DslType, -// adapthase: AdaptPhase = RunPhase -// ): Seq[ConcreteRuleStep] = { -// val calcTime = timeInfo.calcTime -// val (ruleInfos, dsNames) = ruleParams.foldLeft((Seq[RuleInfo](), dataSourceNames)) { (res, param) => -// val (preRuleInfos, preNames) = res -// val dslType = getDslType(param, defaultDslType) -// val (curRuleInfos, curNames) = genRuleAdaptor(dslType, preNames) match { -// case Some(adaptor) => { -// val ris = adaptor.genRuleInfos(param, timeInfo) -// val rins = ris.filter(!_.global).map(_.name) -// (ris, rins) -// } -// case _ => (Nil, Nil) -// } -// if (adapthase == RunPhase) { -// curNames.foreach(TempTables.registerTempTableNameOnly(timeInfo.key, _)) -// } -// (preRuleInfos ++ curRuleInfos, preNames ++ curNames) -// } -// -// adapthase match { -// case PreProcPhase => { -// ruleInfos.flatMap { ri => -// genConcRuleSteps(timeInfo, ri) -// } -// } -// case RunPhase => { -// val riGroups = ruleInfos.foldRight(List[(List[RuleInfo], Boolean)]()) { (ri, groups) => -// groups match { -// case head :: tail if (ri.gather == head._2) => (ri :: head._1, head._2) :: tail -// case _ => (ri :: Nil, ri.gather) :: groups -// } -// }.foldLeft(List[(List[RuleInfo], Boolean, List[String], List[RuleInfo])]()) { (groups, rigs) => -// val preGatherNames = groups.lastOption match { -// case Some(t) => if (t._2) t._3 ::: t._1.map(_.name) else t._3 -// case _ => baselineDsName :: Nil -// } -// val persistRuleInfos = groups.lastOption match { -// case Some(t) if (t._2) => t._1.filter(_.persistType.needPersist) -// case _ => Nil -// } -// groups :+ (rigs._1, rigs._2, preGatherNames, persistRuleInfos) -// } -// -// riGroups.flatMap { group => -// val (ris, gather, srcNames, persistRis) = group -// if (gather) { -// ris.flatMap { ri => -// genConcRuleSteps(timeInfo, ri) -// } -// } else { -// tmsts.flatMap { tmst => -// val concTimeInfo = TmstTimeInfo(calcTime, tmst) -// val tmstInitRuleInfos = genTmstInitRuleInfo(concTimeInfo, srcNames, persistRis) -// (tmstInitRuleInfos ++ ris).flatMap { ri => -// genConcRuleSteps(concTimeInfo, ri) -// } -// } -// } -// } -// } -// } -// -// -// } -// -// private def genConcRuleSteps(timeInfo: TimeInfo, ruleInfo: RuleInfo): Seq[ConcreteRuleStep] = { -// val nri = if (ruleInfo.persistType.needPersist && ruleInfo.tmstNameOpt.isEmpty) { -// val tmstName = if (ruleInfo.gather) { -// TempName.tmstName(ruleInfo.name, timeInfo.calcTime) -// } else { -// TempName.tmstName(ruleInfo.name, timeInfo) -// } -// ruleInfo.setTmstNameOpt(Some(tmstName)) -// } else ruleInfo -// ruleInfo.dslType match { -// case SparkSqlType => SparkSqlStep(timeInfo, nri) :: Nil -// case DfOprType => DfOprStep(timeInfo, nri) :: Nil -// case _ => Nil -// } -// } -// -// private def genTmstInitRuleInfo(timeInfo: TmstTimeInfo, srcNames: Seq[String], -// persistRis: Seq[RuleInfo]): Seq[RuleInfo] = { -// val TmstTimeInfo(calcTime, tmst, _) = timeInfo -// srcNames.map { srcName => -// val srcTmstName = TempName.tmstName(srcName, calcTime) -// val filterSql = { -// s"SELECT * FROM `${srcTmstName}` WHERE `${InternalColumns.tmst}` = ${tmst}" -// } -// val params = persistRis.filter(_.name == srcName).headOption match { -// case Some(ri) => ri.details -// case _ => Map[String, Any]() -// } -// RuleInfo(srcName, None, SparkSqlType, filterSql, params, false) -// } -// } - -// def genRuleSteps(timeInfo: TimeInfo, ruleParams: Seq[Map[String, Any]], -// tmsts: Seq[Long], defaultDslType: DslType, -// adapthase: AdaptPhase = RunPhase -// ): Seq[ConcreteRuleStep] = { -// tmsts.flatMap { tmst => -// val newTimeInfo = TimeInfo(timeInfo.calcTime, tmst) -// val initSteps: Seq[ConcreteRuleStep] = adapthase match { -// case RunPhase => genTmstInitStep(newTimeInfo) -// case PreProcPhase => Nil -// } -// val (steps, dsNames) = ruleParams.foldLeft((initSteps, dataSourceNames)) { (res, param) => -// val (preSteps, preNames) = res -// val dslType = getDslType(param, defaultDslType) -// val (curSteps, curNames) = genRuleAdaptor(dslType, preNames) match { -// case Some(ruleAdaptor) => { -// val concreteSteps = ruleAdaptor.genConcreteRuleStep(newTimeInfo, param) -// val persistNames = ruleAdaptor.getPersistNames(concreteSteps) -// (concreteSteps, persistNames) -// } -// case _ => (Nil, Nil) -// } -// (preSteps ++ curSteps, preNames ++ curNames) -// } -// steps -// } -// } - - - -// private def genTmstInitStep(timeInfo: TimeInfo): Seq[ConcreteRuleStep] = { -// val TimeInfo(calcTime, tmst) = timeInfo -// val tmstDsName = TempName.tmstName(baselineDsName, calcTime) -// val filterSql = { -// s"SELECT * FROM `${tmstDsName}` WHERE `${InternalColumns.tmst}` = ${tmst}" -// } -// SparkSqlStep( -// timeInfo, -// RuleInfo(baselineDsName, None, filterSql, Map[String, Any]()) -// ) :: Nil -// } - - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/SparkSqlAdaptor.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/SparkSqlAdaptor.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/SparkSqlAdaptor.scala deleted file mode 100644 index b7c68b5..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/adaptor/SparkSqlAdaptor.scala +++ /dev/null @@ -1,55 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.adaptor - -import org.apache.griffin.measure.cache.tmst.TempName -import org.apache.griffin.measure.process.{ExportMode, ProcessType} -import org.apache.griffin.measure.process.temp.TimeRange -import org.apache.griffin.measure.rule.dsl.MetricPersistType -import org.apache.griffin.measure.rule.plan.{TimeInfo, _} -import org.apache.griffin.measure.utils.ParamUtil._ - -case class SparkSqlAdaptor() extends RuleAdaptor { - -// def genRuleStep(timeInfo: TimeInfo, param: Map[String, Any]): Seq[RuleStep] = { -// val ruleInfo = RuleInfoGen(param, timeInfo) -// SparkSqlStep(timeInfo, ruleInfo) :: Nil -// } -// def adaptConcreteRuleStep(ruleStep: RuleStep): Seq[ConcreteRuleStep] = { -// ruleStep match { -// case rs @ SparkSqlStep(ti, ri) => rs :: Nil -// case _ => Nil -// } -// } - - import RuleParamKeys._ - - def genRulePlan(timeInfo: TimeInfo, param: Map[String, Any], - procType: ProcessType, dsTimeRanges: Map[String, TimeRange]): RulePlan = { - val name = getRuleName(param) - val step = SparkSqlStep(name, getRule(param), getDetails(param), getCache(param), getGlobal(param)) - val mode = ExportMode.defaultMode(procType) - RulePlan( - step :: Nil, - genRuleExports(param, name, name, timeInfo.calcTime, mode), - genDsUpdates(param, "", name) - ) - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/CollectType.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/CollectType.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/CollectType.scala deleted file mode 100644 index 313c486..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/CollectType.scala +++ /dev/null @@ -1,57 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.dsl - -import scala.util.matching.Regex - -sealed trait CollectType { - val regex: Regex - val desc: String -} - -object CollectType { - private val collectTypes: List[CollectType] = List(DefaultCollectType, EntriesCollectType, ArrayCollectType, MapCollectType) - def apply(ptn: String): CollectType = { - collectTypes.find(tp => ptn match { - case tp.regex() => true - case _ => false - }).getOrElse(DefaultCollectType) - } - def unapply(pt: CollectType): Option[String] = Some(pt.desc) -} - -final case object DefaultCollectType extends CollectType { - val regex: Regex = "".r - val desc: String = "default" -} - -final case object EntriesCollectType extends CollectType { - val regex: Regex = "^(?i)entries$".r - val desc: String = "entries" -} - -final case object ArrayCollectType extends CollectType { - val regex: Regex = "^(?i)array|list$".r - val desc: String = "array" -} - -final case object MapCollectType extends CollectType { - val regex: Regex = "^(?i)map$".r - val desc: String = "map" -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/DqType.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/DqType.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/DqType.scala deleted file mode 100644 index 830e319..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/DqType.scala +++ /dev/null @@ -1,75 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.dsl - -import scala.util.matching.Regex - - -sealed trait DqType { - val regex: Regex - val desc: String -} - -object DqType { - private val dqTypes: List[DqType] = List( - AccuracyType, ProfilingType, UniquenessType, DistinctnessType, TimelinessType, CompletenessType, UnknownType - ) - def apply(ptn: String): DqType = { - dqTypes.find(tp => ptn match { - case tp.regex() => true - case _ => false - }).getOrElse(UnknownType) - } - def unapply(pt: DqType): Option[String] = Some(pt.desc) -} - -final case object AccuracyType extends DqType { - val regex = "^(?i)accuracy$".r - val desc = "accuracy" -} - -final case object ProfilingType extends DqType { - val regex = "^(?i)profiling$".r - val desc = "profiling" -} - -final case object UniquenessType extends DqType { - val regex = "^(?i)uniqueness|duplicate$".r - val desc = "uniqueness" -} - -final case object DistinctnessType extends DqType { - val regex = "^(?i)distinct$".r - val desc = "distinct" -} - -final case object TimelinessType extends DqType { - val regex = "^(?i)timeliness$".r - val desc = "timeliness" -} - -final case object CompletenessType extends DqType { - val regex = "^(?i)completeness$".r - val desc = "completeness" -} - -final case object UnknownType extends DqType { - val regex = "".r - val desc = "unknown" -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/DslType.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/DslType.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/DslType.scala deleted file mode 100644 index dd2b1c7..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/DslType.scala +++ /dev/null @@ -1,53 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.dsl - -import scala.util.matching.Regex - - -sealed trait DslType { - val regex: Regex - val desc: String -} - -object DslType { - private val dslTypes: List[DslType] = List(SparkSqlType, GriffinDslType, DfOprType) - def apply(ptn: String): DslType = { - dslTypes.find(tp => ptn match { - case tp.regex() => true - case _ => false - }).getOrElse(GriffinDslType) - } - def unapply(pt: DslType): Option[String] = Some(pt.desc) -} - -final case object SparkSqlType extends DslType { - val regex = "^(?i)spark-?sql$".r - val desc = "spark-sql" -} - -final case object DfOprType extends DslType { - val regex = "^(?i)df-?opr$".r - val desc = "df-opr" -} - -final case object GriffinDslType extends DslType { - val regex = "^(?i)griffin-?dsl$".r - val desc = "griffin-dsl" -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/PersistType.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/PersistType.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/PersistType.scala deleted file mode 100644 index a16c571..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/PersistType.scala +++ /dev/null @@ -1,60 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.dsl - -import scala.util.matching.Regex - -sealed trait PersistType { - val regex: Regex - val desc: String -// def temp: Boolean = false -// def persist: Boolean = false -// def collect: Boolean = false - def needPersist: Boolean = true -} - -object PersistType { - private val persistTypes: List[PersistType] = List(RecordPersistType, MetricPersistType, NonePersistType) - def apply(ptn: String): PersistType = { - persistTypes.find(tp => ptn match { - case tp.regex() => true - case _ => false - }).getOrElse(NonePersistType) - } - def unapply(pt: PersistType): Option[String] = Some(pt.desc) -} - -final case object NonePersistType extends PersistType { - val regex: Regex = "".r - val desc: String = "none" - override def needPersist: Boolean = false -} - -final case object RecordPersistType extends PersistType { - val regex: Regex = "^(?i)record$".r - val desc: String = "record" -// override def temp: Boolean = true -} - -final case object MetricPersistType extends PersistType { - val regex: Regex = "^(?i)metric$".r - val desc: String = "metric" -// override def temp: Boolean = true -// override def collect: Boolean = true -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/AccuracyAnalyzer.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/AccuracyAnalyzer.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/AccuracyAnalyzer.scala deleted file mode 100644 index 7efb32e..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/AccuracyAnalyzer.scala +++ /dev/null @@ -1,41 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.dsl.analyzer - -import org.apache.griffin.measure.rule.dsl.expr._ - - -case class AccuracyAnalyzer(expr: LogicalExpr, sourceName: String, targetName: String) extends BasicAnalyzer { - - val dataSourceNames = expr.preOrderTraverseDepthFirst(Set[String]())(seqDataSourceNames, combDataSourceNames) - - val sourceSelectionExprs = { - val seq = seqSelectionExprs(sourceName) - expr.preOrderTraverseDepthFirst(Seq[SelectionExpr]())(seq, combSelectionExprs) - } - val targetSelectionExprs = { - val seq = seqSelectionExprs(targetName) - expr.preOrderTraverseDepthFirst(Seq[SelectionExpr]())(seq, combSelectionExprs) - } - - val selectionExprs = sourceSelectionExprs ++ { - expr.preOrderTraverseDepthFirst(Seq[AliasableExpr]())(seqWithAliasExprs, combWithAliasExprs) - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/BasicAnalyzer.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/BasicAnalyzer.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/BasicAnalyzer.scala deleted file mode 100644 index e14e0da..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/BasicAnalyzer.scala +++ /dev/null @@ -1,53 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.dsl.analyzer - -import org.apache.griffin.measure.rule.dsl.expr.{MathExpr, _} - - -trait BasicAnalyzer extends Serializable { - - val expr: Expr - - val seqDataSourceNames = (expr: Expr, v: Set[String]) => { - expr match { - case DataSourceHeadExpr(name) => v + name - case _ => v - } - } - val combDataSourceNames = (a: Set[String], b: Set[String]) => a ++ b - - val seqSelectionExprs = (dsName: String) => (expr: Expr, v: Seq[SelectionExpr]) => { - expr match { - case se @ SelectionExpr(head: DataSourceHeadExpr, _, _) if (head.name == dsName) => v :+ se - case _ => v - } - } - val combSelectionExprs = (a: Seq[SelectionExpr], b: Seq[SelectionExpr]) => a ++ b - - val seqWithAliasExprs = (expr: Expr, v: Seq[AliasableExpr]) => { - expr match { - case se: SelectExpr => v - case a: AliasableExpr if (a.alias.nonEmpty) => v :+ a - case _ => v - } - } - val combWithAliasExprs = (a: Seq[AliasableExpr], b: Seq[AliasableExpr]) => a ++ b - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/CompletenessAnalyzer.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/CompletenessAnalyzer.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/CompletenessAnalyzer.scala deleted file mode 100644 index ad56e1a..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/CompletenessAnalyzer.scala +++ /dev/null @@ -1,46 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.dsl.analyzer - -import org.apache.griffin.measure.rule.dsl.expr._ - - -case class CompletenessAnalyzer(expr: CompletenessClause, sourceName: String) extends BasicAnalyzer { - - val seqAlias = (expr: Expr, v: Seq[String]) => { - expr match { - case apr: AliasableExpr => v ++ apr.alias - case _ => v - } - } - val combAlias = (a: Seq[String], b: Seq[String]) => a ++ b - - private val exprs = expr.exprs - private def genAlias(idx: Int): String = s"alias_${idx}" - val selectionPairs = exprs.zipWithIndex.map { pair => - val (pr, idx) = pair - val res = pr.preOrderTraverseDepthFirst(Seq[String]())(seqAlias, combAlias) - (pr, res.headOption.getOrElse(genAlias(idx))) - } - - if (selectionPairs.isEmpty) { - throw new Exception(s"completeness analyzer error: empty selection") - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/DistinctnessAnalyzer.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/DistinctnessAnalyzer.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/DistinctnessAnalyzer.scala deleted file mode 100644 index af59eb4..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/DistinctnessAnalyzer.scala +++ /dev/null @@ -1,47 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.dsl.analyzer - -import org.apache.griffin.measure.rule.dsl.expr._ - - -//case class DistinctnessAnalyzer(expr: DistinctnessClause, sourceName: String, targetName: String) extends BasicAnalyzer { -case class DistinctnessAnalyzer(expr: DistinctnessClause, sourceName: String) extends BasicAnalyzer { - - val seqAlias = (expr: Expr, v: Seq[String]) => { - expr match { - case apr: AliasableExpr => v ++ apr.alias - case _ => v - } - } - val combAlias = (a: Seq[String], b: Seq[String]) => a ++ b - - private val exprs = expr.exprs - private def genAlias(idx: Int): String = s"alias_${idx}" - val selectionPairs = exprs.zipWithIndex.map { pair => - val (pr, idx) = pair - val res = pr.preOrderTraverseDepthFirst(Seq[String]())(seqAlias, combAlias) - (pr, res.headOption.getOrElse(genAlias(idx)), pr.tag.isEmpty) - } - - if (selectionPairs.isEmpty) { - throw new Exception(s"uniqueness analyzer error: empty selection") - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/ProfilingAnalyzer.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/ProfilingAnalyzer.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/ProfilingAnalyzer.scala deleted file mode 100644 index 6872977..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/ProfilingAnalyzer.scala +++ /dev/null @@ -1,42 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.dsl.analyzer - -import org.apache.griffin.measure.rule.dsl.expr._ - - -case class ProfilingAnalyzer(expr: ProfilingClause, sourceName: String) extends BasicAnalyzer { - - val dataSourceNames = expr.preOrderTraverseDepthFirst(Set[String]())(seqDataSourceNames, combDataSourceNames) - - val selectionExprs: Seq[Expr] = { - expr.selectClause.exprs.map(_.extractSelf).flatMap { expr => - expr match { - case e: SelectionExpr => Some(e) - case e: FunctionExpr => Some(e) - case _ => None - } - } - } - - val groupbyExprOpt = expr.groupbyClauseOpt - val preGroupbyExprs = expr.preGroupbyClauses.map(_.extractSelf) - val postGroupbyExprs = expr.postGroupbyClauses.map(_.extractSelf) - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/TimelinessAnalyzer.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/TimelinessAnalyzer.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/TimelinessAnalyzer.scala deleted file mode 100644 index 37d4651..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/TimelinessAnalyzer.scala +++ /dev/null @@ -1,65 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.dsl.analyzer - -import org.apache.griffin.measure.rule.dsl.expr._ - - -case class TimelinessAnalyzer(expr: TimelinessClause, sourceName: String) extends BasicAnalyzer { - -// val tsExpr = expr.desc - -// val seqAlias = (expr: Expr, v: Seq[String]) => { -// expr match { -// case apr: AliasableExpr => v ++ apr.alias -// case _ => v -// } -// } -// val combAlias = (a: Seq[String], b: Seq[String]) => a ++ b -// -// private val exprs = expr.exprs.toList -// val selectionPairs = exprs.map { pr => -// val res = pr.preOrderTraverseDepthFirst(Seq[String]())(seqAlias, combAlias) -// println(res) -// println(pr) -// (pr, res.headOption) -// } -// -// val (tsExprPair, endTsPairOpt) = selectionPairs match { -// case Nil => throw new Exception(s"timeliness analyzer error: ts column not set") -// case tsPair :: Nil => (tsPair, None) -// case tsPair :: endTsPair :: _ => (tsPair, Some(endTsPair)) -// } -// -// def getSelAlias(pair: (Expr, Option[String]), defAlias: String): (String, String) = { -// val (pr, aliasOpt) = pair -// val alias = aliasOpt.getOrElse(defAlias) -// (pr.desc, alias) -// } - - - private val exprs = expr.exprs.map(_.desc).toList - - val (btsExpr, etsExprOpt) = exprs match { - case Nil => throw new Exception(s"timeliness analyzer error: ts column not set") - case btsExpr :: Nil => (btsExpr, None) - case btsExpr :: etsExpr :: _ => (btsExpr, Some(etsExpr)) - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/UniquenessAnalyzer.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/UniquenessAnalyzer.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/UniquenessAnalyzer.scala deleted file mode 100644 index 9fe65c2..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/analyzer/UniquenessAnalyzer.scala +++ /dev/null @@ -1,46 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.dsl.analyzer - -import org.apache.griffin.measure.rule.dsl.expr.{AliasableExpr, _} - - -case class UniquenessAnalyzer(expr: UniquenessClause, sourceName: String, targetName: String) extends BasicAnalyzer { - - val seqAlias = (expr: Expr, v: Seq[String]) => { - expr match { - case apr: AliasableExpr => v ++ apr.alias - case _ => v - } - } - val combAlias = (a: Seq[String], b: Seq[String]) => a ++ b - - private val exprs = expr.exprs - private def genAlias(idx: Int): String = s"alias_${idx}" - val selectionPairs = exprs.zipWithIndex.map { pair => - val (pr, idx) = pair - val res = pr.preOrderTraverseDepthFirst(Seq[String]())(seqAlias, combAlias) - (pr, res.headOption.getOrElse(genAlias(idx))) - } - - if (selectionPairs.isEmpty) { - throw new Exception(s"uniqueness analyzer error: empty selection") - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/AliasableExpr.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/AliasableExpr.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/AliasableExpr.scala deleted file mode 100644 index 33a12e0..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/AliasableExpr.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.dsl.expr - -trait AliasableExpr extends Expr { - - def alias: Option[String] - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/ClauseExpression.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/ClauseExpression.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/ClauseExpression.scala deleted file mode 100644 index ecc5d67..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/ClauseExpression.scala +++ /dev/null @@ -1,270 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.dsl.expr - -trait ClauseExpression extends Expr { -} - -case class SelectClause(exprs: Seq[Expr], extraConditionOpt: Option[ExtraConditionExpr] - ) extends ClauseExpression { - - addChildren(exprs) - - def desc: String = { - extraConditionOpt match { - case Some(cdtn) => s"${cdtn.desc} ${exprs.map(_.desc).mkString(", ")}" - case _ => s"${exprs.map(_.desc).mkString(", ")}" - } - } - def coalesceDesc: String = desc - - override def map(func: (Expr) => Expr): SelectClause = { - SelectClause(exprs.map(func(_)), extraConditionOpt.map(func(_).asInstanceOf[ExtraConditionExpr])) - } - -} - -case class FromClause(dataSource: String) extends ClauseExpression { - - def desc: String = s"FROM `${dataSource}`" - def coalesceDesc: String = desc - -} - -case class WhereClause(expr: Expr) extends ClauseExpression { - - addChild(expr) - - def desc: String = s"WHERE ${expr.desc}" - def coalesceDesc: String = s"WHERE ${expr.coalesceDesc}" - - override def map(func: (Expr) => Expr): WhereClause = { - WhereClause(func(expr)) - } - -} - -case class GroupbyClause(exprs: Seq[Expr], havingClauseOpt: Option[Expr]) extends ClauseExpression { - - addChildren(exprs ++ havingClauseOpt.toSeq) - - def desc: String = { - val gbs = exprs.map(_.desc).mkString(", ") - havingClauseOpt match { - case Some(having) => s"GROUP BY ${gbs} HAVING ${having.desc}" - case _ => s"GROUP BY ${gbs}" - } - } - def coalesceDesc: String = { - val gbs = exprs.map(_.desc).mkString(", ") - havingClauseOpt match { - case Some(having) => s"GROUP BY ${gbs} HAVING ${having.coalesceDesc}" - case _ => s"GROUP BY ${gbs}" - } - } - - def merge(other: GroupbyClause): GroupbyClause = { - val newHavingClauseOpt = (havingClauseOpt, other.havingClauseOpt) match { - case (Some(hc), Some(ohc)) => { - val logical1 = LogicalFactorExpr(hc, false, None) - val logical2 = LogicalFactorExpr(ohc, false, None) - Some(BinaryLogicalExpr(logical1, ("AND", logical2) :: Nil)) - } - case (a @ Some(_), _) => a - case (_, b @ Some(_)) => b - case (_, _) => None - } - GroupbyClause(exprs ++ other.exprs, newHavingClauseOpt) - } - - override def map(func: (Expr) => Expr): GroupbyClause = { - GroupbyClause(exprs.map(func(_)), havingClauseOpt.map(func(_))) - } - -} - -case class OrderItem(expr: Expr, orderOpt: Option[String]) extends Expr { - addChild(expr) - def desc: String = { - orderOpt match { - case Some(os) => s"${expr.desc} ${os.toUpperCase}" - case _ => s"${expr.desc}" - } - } - def coalesceDesc: String = desc - - override def map(func: (Expr) => Expr): OrderItem = { - OrderItem(func(expr), orderOpt) - } -} - -case class OrderbyClause(items: Seq[OrderItem]) extends ClauseExpression { - - addChildren(items.map(_.expr)) - - def desc: String = { - val obs = items.map(_.desc).mkString(", ") - s"ORDER BY ${obs}" - } - def coalesceDesc: String = { - val obs = items.map(_.desc).mkString(", ") - s"ORDER BY ${obs}" - } - - override def map(func: (Expr) => Expr): OrderbyClause = { - OrderbyClause(items.map(func(_).asInstanceOf[OrderItem])) - } -} - -case class SortbyClause(items: Seq[OrderItem]) extends ClauseExpression { - - addChildren(items.map(_.expr)) - - def desc: String = { - val obs = items.map(_.desc).mkString(", ") - s"SORT BY ${obs}" - } - def coalesceDesc: String = { - val obs = items.map(_.desc).mkString(", ") - s"SORT BY ${obs}" - } - - override def map(func: (Expr) => Expr): SortbyClause = { - SortbyClause(items.map(func(_).asInstanceOf[OrderItem])) - } -} - -case class LimitClause(expr: Expr) extends ClauseExpression { - - addChild(expr) - - def desc: String = s"LIMIT ${expr.desc}" - def coalesceDesc: String = s"LIMIT ${expr.coalesceDesc}" - - override def map(func: (Expr) => Expr): LimitClause = { - LimitClause(func(expr)) - } -} - -case class CombinedClause(selectClause: SelectClause, fromClauseOpt: Option[FromClause], - tails: Seq[ClauseExpression] - ) extends ClauseExpression { - - addChildren({ - val headClauses: Seq[ClauseExpression] = selectClause +: (fromClauseOpt.toSeq) - headClauses ++ tails - }) - - def desc: String = { - val selectDesc = s"SELECT ${selectClause.desc}" - val fromDesc = fromClauseOpt.map(_.desc).mkString(" ") - val headDesc = s"${selectDesc} ${fromDesc}" - tails.foldLeft(headDesc) { (head, tail) => - s"${head} ${tail.desc}" - } - } - def coalesceDesc: String = { - val selectDesc = s"SELECT ${selectClause.coalesceDesc}" - val fromDesc = fromClauseOpt.map(_.coalesceDesc).mkString(" ") - val headDesc = s"${selectDesc} ${fromDesc}" - tails.foldLeft(headDesc) { (head, tail) => - s"${head} ${tail.coalesceDesc}" - } - } - - override def map(func: (Expr) => Expr): CombinedClause = { - CombinedClause(func(selectClause).asInstanceOf[SelectClause], - fromClauseOpt.map(func(_).asInstanceOf[FromClause]), - tails.map(func(_).asInstanceOf[ClauseExpression]) - ) - } -} - -case class ProfilingClause(selectClause: SelectClause, - fromClauseOpt: Option[FromClause], - groupbyClauseOpt: Option[GroupbyClause], - preGroupbyClauses: Seq[ClauseExpression], - postGroupbyClauses: Seq[ClauseExpression] - ) extends ClauseExpression { - addChildren({ - val headClauses: Seq[ClauseExpression] = selectClause +: (fromClauseOpt.toSeq) - groupbyClauseOpt match { - case Some(gc) => (headClauses ++ preGroupbyClauses) ++ (gc +: postGroupbyClauses) - case _ => (headClauses ++ preGroupbyClauses) ++ postGroupbyClauses - } - }) - - def desc: String = { - val selectDesc = selectClause.desc - val fromDesc = fromClauseOpt.map(_.desc).mkString(" ") - val groupbyDesc = groupbyClauseOpt.map(_.desc).mkString(" ") - val preDesc = preGroupbyClauses.map(_.desc).mkString(" ") - val postDesc = postGroupbyClauses.map(_.desc).mkString(" ") - s"${selectDesc} ${fromDesc} ${preDesc} ${groupbyDesc} ${postDesc}" - } - def coalesceDesc: String = { - val selectDesc = selectClause.coalesceDesc - val fromDesc = fromClauseOpt.map(_.coalesceDesc).mkString(" ") - val groupbyDesc = groupbyClauseOpt.map(_.coalesceDesc).mkString(" ") - val preDesc = preGroupbyClauses.map(_.coalesceDesc).mkString(" ") - val postDesc = postGroupbyClauses.map(_.coalesceDesc).mkString(" ") - s"${selectDesc} ${fromDesc} ${preDesc} ${groupbyDesc} ${postDesc}" - } - - override def map(func: (Expr) => Expr): ProfilingClause = { - ProfilingClause(func(selectClause).asInstanceOf[SelectClause], - fromClauseOpt.map(func(_).asInstanceOf[FromClause]), - groupbyClauseOpt.map(func(_).asInstanceOf[GroupbyClause]), - preGroupbyClauses.map(func(_).asInstanceOf[ClauseExpression]), - postGroupbyClauses.map(func(_).asInstanceOf[ClauseExpression]) - ) - } -} - -case class UniquenessClause(exprs: Seq[Expr]) extends ClauseExpression { - addChildren(exprs) - - def desc: String = exprs.map(_.desc).mkString(", ") - def coalesceDesc: String = exprs.map(_.coalesceDesc).mkString(", ") - override def map(func: (Expr) => Expr): UniquenessClause = UniquenessClause(exprs.map(func(_))) -} - -case class DistinctnessClause(exprs: Seq[Expr]) extends ClauseExpression { - addChildren(exprs) - - def desc: String = exprs.map(_.desc).mkString(", ") - def coalesceDesc: String = exprs.map(_.coalesceDesc).mkString(", ") - override def map(func: (Expr) => Expr): DistinctnessClause = DistinctnessClause(exprs.map(func(_))) -} - -case class TimelinessClause(exprs: Seq[Expr]) extends ClauseExpression { - addChildren(exprs) - - def desc: String = exprs.map(_.desc).mkString(", ") - def coalesceDesc: String = exprs.map(_.coalesceDesc).mkString(", ") - override def map(func: (Expr) => Expr): TimelinessClause = TimelinessClause(exprs.map(func(_))) -} - -case class CompletenessClause(exprs: Seq[Expr]) extends ClauseExpression { - addChildren(exprs) - - def desc: String = exprs.map(_.desc).mkString(", ") - def coalesceDesc: String = exprs.map(_.coalesceDesc).mkString(", ") - override def map(func: (Expr) => Expr): CompletenessClause = CompletenessClause(exprs.map(func(_))) -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/Expr.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/Expr.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/Expr.scala deleted file mode 100644 index 0b653b1..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/Expr.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.dsl.expr - -trait Expr extends TreeNode with ExprTag with Serializable { - - def desc: String - - def coalesceDesc: String - - def extractSelf: Expr = this - - // execution - def map(func: (Expr) => Expr): Expr = func(this) - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/ExprTag.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/ExprTag.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/ExprTag.scala deleted file mode 100644 index 2e31bbe..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/ExprTag.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.dsl.expr - -trait ExprTag { this: Expr => - var tag: String = "" -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/ExtraConditionExpr.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/ExtraConditionExpr.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/ExtraConditionExpr.scala deleted file mode 100644 index eb7ba48..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/ExtraConditionExpr.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.dsl.expr - -case class ExtraConditionExpr(cdtn: String) extends Expr { - - def desc: String = cdtn.toUpperCase - - def coalesceDesc: String = desc - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/FunctionExpr.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/FunctionExpr.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/FunctionExpr.scala deleted file mode 100644 index 1bbed83..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/FunctionExpr.scala +++ /dev/null @@ -1,45 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.dsl.expr - -case class FunctionExpr(functionName: String, args: Seq[Expr], - extraConditionOpt: Option[ExtraConditionExpr], - aliasOpt: Option[String] - ) extends Expr with AliasableExpr { - - addChildren(args) - - def desc: String = { - extraConditionOpt match { - case Some(cdtn) => s"${functionName}(${cdtn.desc} ${args.map(_.desc).mkString(", ")})" - case _ => s"${functionName}(${args.map(_.desc).mkString(", ")})" - } - } - def coalesceDesc: String = desc - def alias: Option[String] = { - if (aliasOpt.isEmpty) { - Some(functionName) - } else aliasOpt - } - - override def map(func: (Expr) => Expr): FunctionExpr = { - FunctionExpr(functionName, args.map(func(_)), - extraConditionOpt.map(func(_).asInstanceOf[ExtraConditionExpr]), aliasOpt) - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/LiteralExpr.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/LiteralExpr.scala b/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/LiteralExpr.scala deleted file mode 100644 index 60290bc..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/rule/dsl/expr/LiteralExpr.scala +++ /dev/null @@ -1,72 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.dsl.expr - -import org.apache.griffin.measure.utils.TimeUtil - -trait LiteralExpr extends Expr { - def coalesceDesc: String = desc -} - -case class LiteralNullExpr(str: String) extends LiteralExpr { - def desc: String = "NULL" -} - -case class LiteralNanExpr(str: String) extends LiteralExpr { - def desc: String = "NaN" -} - -case class LiteralStringExpr(str: String) extends LiteralExpr { - def desc: String = str -} - -case class LiteralNumberExpr(str: String) extends LiteralExpr { - def desc: String = { - try { - if (str.contains(".")) { - str.toDouble.toString - } else { - str.toLong.toString - } - } catch { - case e: Throwable => throw new Exception(s"${str} is invalid number") - } - } -} - -case class LiteralTimeExpr(str: String) extends LiteralExpr { - def desc: String = { - TimeUtil.milliseconds(str) match { - case Some(t) => t.toString - case _ => throw new Exception(s"${str} is invalid time") - } - } -} - -case class LiteralBooleanExpr(str: String) extends LiteralExpr { - final val TrueRegex = """(?i)true""".r - final val FalseRegex = """(?i)false""".r - def desc: String = { - str match { - case TrueRegex() => true.toString - case FalseRegex() => false.toString - case _ => throw new Exception(s"${str} is invalid boolean") - } - } -} \ No newline at end of file
