Repository: incubator-griffin Updated Branches: refs/heads/master 5b0a58fd1 -> 77aad4955
2018 06 11 refactor measure Author: William Guo <gu...@apache.org> Closes #296 from guoyuepeng/2018_06_11_refactor_measure. Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/77aad495 Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/77aad495 Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/77aad495 Branch: refs/heads/master Commit: 77aad4955a57fe3e87f58a58ae7c20468c07dcf1 Parents: 5b0a58f Author: William Guo <gu...@apache.org> Authored: Wed Jun 13 10:06:01 2018 +0800 Committer: Lionel Liu <bhlx3l...@163.com> Committed: Wed Jun 13 10:06:01 2018 +0800 ---------------------------------------------------------------------- .../apache/griffin/measure/Application.scala | 2 +- .../configuration/json/ParamFileReader.scala | 43 ------------------ .../configuration/json/ParamJsonReader.scala | 39 ---------------- .../configuration/json/ParamReader.scala | 35 -------------- .../configuration/json/ParamReaderFactory.scala | 48 -------------------- .../params/reader/ParamFileReader.scala | 43 ++++++++++++++++++ .../params/reader/ParamJsonReader.scala | 39 ++++++++++++++++ .../params/reader/ParamReader.scala | 35 ++++++++++++++ .../params/reader/ParamReaderFactory.scala | 48 ++++++++++++++++++++ .../streaming/info/InfoCacheInstance.scala | 2 +- 10 files changed, 167 insertions(+), 167 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/77aad495/measure/src/main/scala/org/apache/griffin/measure/Application.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/Application.scala b/measure/src/main/scala/org/apache/griffin/measure/Application.scala index 6580ffc..dbbe970 100644 --- a/measure/src/main/scala/org/apache/griffin/measure/Application.scala +++ b/measure/src/main/scala/org/apache/griffin/measure/Application.scala @@ -19,7 +19,7 @@ under the License. package org.apache.griffin.measure import org.apache.griffin.measure.configuration.enums._ -import org.apache.griffin.measure.configuration.json.ParamReaderFactory +import org.apache.griffin.measure.configuration.params.reader.ParamReaderFactory import org.apache.griffin.measure.configuration.params.{AllParam, DQParam, EnvParam, Param} import org.apache.griffin.measure.configuration.validator.ParamValidator import org.apache.griffin.measure.context.writer.PersistTaskRunner http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/77aad495/measure/src/main/scala/org/apache/griffin/measure/configuration/json/ParamFileReader.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/configuration/json/ParamFileReader.scala b/measure/src/main/scala/org/apache/griffin/measure/configuration/json/ParamFileReader.scala deleted file mode 100644 index 5091d49..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/configuration/json/ParamFileReader.scala +++ /dev/null @@ -1,43 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.configuration.json - -import org.apache.griffin.measure.configuration.params.Param -import org.apache.griffin.measure.utils.{HdfsUtil, JsonUtil} - -import scala.util.Try - -/** - * read params from config file path - * @param filePath: hdfs path ("hdfs://cluster-name/path") - * local file path ("file:///path") - * relative file path ("relative/path") - */ -case class ParamFileReader(filePath: String) extends ParamReader { - - def readConfig[T <: Param](implicit m : Manifest[T]): Try[T] = { - Try { - val source = HdfsUtil.openFile(filePath) - val param = JsonUtil.fromJson[T](source) - source.close - param - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/77aad495/measure/src/main/scala/org/apache/griffin/measure/configuration/json/ParamJsonReader.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/configuration/json/ParamJsonReader.scala b/measure/src/main/scala/org/apache/griffin/measure/configuration/json/ParamJsonReader.scala deleted file mode 100644 index 2f4a5e7..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/configuration/json/ParamJsonReader.scala +++ /dev/null @@ -1,39 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.configuration.json - -import org.apache.griffin.measure.configuration.params.Param -import org.apache.griffin.measure.utils.JsonUtil - -import scala.util.Try - -/** - * read params from json string directly - * @param jsonString - */ -case class ParamJsonReader(jsonString: String) extends ParamReader { - - def readConfig[T <: Param](implicit m : Manifest[T]): Try[T] = { - Try { - val param = JsonUtil.fromJson[T](jsonString) - param - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/77aad495/measure/src/main/scala/org/apache/griffin/measure/configuration/json/ParamReader.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/configuration/json/ParamReader.scala b/measure/src/main/scala/org/apache/griffin/measure/configuration/json/ParamReader.scala deleted file mode 100644 index d923cd5..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/configuration/json/ParamReader.scala +++ /dev/null @@ -1,35 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.configuration.json - -import org.apache.griffin.measure.Loggable -import org.apache.griffin.measure.configuration.params.Param - -import scala.util.Try - -trait ParamReader extends Loggable with Serializable { - - /** - * read config param - * @tparam T param type expected - * @return parsed param - */ - def readConfig[T <: Param](implicit m : Manifest[T]): Try[T] - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/77aad495/measure/src/main/scala/org/apache/griffin/measure/configuration/json/ParamReaderFactory.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/configuration/json/ParamReaderFactory.scala b/measure/src/main/scala/org/apache/griffin/measure/configuration/json/ParamReaderFactory.scala deleted file mode 100644 index 14fce02..0000000 --- a/measure/src/main/scala/org/apache/griffin/measure/configuration/json/ParamReaderFactory.scala +++ /dev/null @@ -1,48 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.configuration.json - -import org.apache.griffin.measure.utils.JsonUtil - -object ParamReaderFactory { - - val json = "json" - val file = "file" - - /** - * parse string content to get param reader - * @param pathOrJson - * @return - */ - def getParamReader(pathOrJson: String): ParamReader = { - val strType = paramStrType(pathOrJson) - if (json.equals(strType)) ParamJsonReader(pathOrJson) - else ParamFileReader(pathOrJson) - } - - private def paramStrType(str: String): String = { - try { - JsonUtil.toAnyMap(str) - json - } catch { - case e: Throwable => file - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/77aad495/measure/src/main/scala/org/apache/griffin/measure/configuration/params/reader/ParamFileReader.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/configuration/params/reader/ParamFileReader.scala b/measure/src/main/scala/org/apache/griffin/measure/configuration/params/reader/ParamFileReader.scala new file mode 100644 index 0000000..0ff5b06 --- /dev/null +++ b/measure/src/main/scala/org/apache/griffin/measure/configuration/params/reader/ParamFileReader.scala @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ +package org.apache.griffin.measure.configuration.params.reader + +import org.apache.griffin.measure.configuration.params.Param +import org.apache.griffin.measure.utils.{HdfsUtil, JsonUtil} + +import scala.util.Try + +/** + * read params from config file path + * @param filePath: hdfs path ("hdfs://cluster-name/path") + * local file path ("file:///path") + * relative file path ("relative/path") + */ +case class ParamFileReader(filePath: String) extends ParamReader { + + def readConfig[T <: Param](implicit m : Manifest[T]): Try[T] = { + Try { + val source = HdfsUtil.openFile(filePath) + val param = JsonUtil.fromJson[T](source) + source.close + param + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/77aad495/measure/src/main/scala/org/apache/griffin/measure/configuration/params/reader/ParamJsonReader.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/configuration/params/reader/ParamJsonReader.scala b/measure/src/main/scala/org/apache/griffin/measure/configuration/params/reader/ParamJsonReader.scala new file mode 100644 index 0000000..7a7eaf6 --- /dev/null +++ b/measure/src/main/scala/org/apache/griffin/measure/configuration/params/reader/ParamJsonReader.scala @@ -0,0 +1,39 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ +package org.apache.griffin.measure.configuration.params.reader + +import org.apache.griffin.measure.configuration.params.Param +import org.apache.griffin.measure.utils.JsonUtil + +import scala.util.Try + +/** + * read params from json string directly + * @param jsonString + */ +case class ParamJsonReader(jsonString: String) extends ParamReader { + + def readConfig[T <: Param](implicit m : Manifest[T]): Try[T] = { + Try { + val param = JsonUtil.fromJson[T](jsonString) + param + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/77aad495/measure/src/main/scala/org/apache/griffin/measure/configuration/params/reader/ParamReader.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/configuration/params/reader/ParamReader.scala b/measure/src/main/scala/org/apache/griffin/measure/configuration/params/reader/ParamReader.scala new file mode 100644 index 0000000..77134ea --- /dev/null +++ b/measure/src/main/scala/org/apache/griffin/measure/configuration/params/reader/ParamReader.scala @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ +package org.apache.griffin.measure.configuration.params.reader + +import org.apache.griffin.measure.Loggable +import org.apache.griffin.measure.configuration.params.Param + +import scala.util.Try + +trait ParamReader extends Loggable with Serializable { + + /** + * read config param + * @tparam T param type expected + * @return parsed param + */ + def readConfig[T <: Param](implicit m : Manifest[T]): Try[T] + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/77aad495/measure/src/main/scala/org/apache/griffin/measure/configuration/params/reader/ParamReaderFactory.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/configuration/params/reader/ParamReaderFactory.scala b/measure/src/main/scala/org/apache/griffin/measure/configuration/params/reader/ParamReaderFactory.scala new file mode 100644 index 0000000..363b865 --- /dev/null +++ b/measure/src/main/scala/org/apache/griffin/measure/configuration/params/reader/ParamReaderFactory.scala @@ -0,0 +1,48 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ +package org.apache.griffin.measure.configuration.params.reader + +import org.apache.griffin.measure.utils.JsonUtil + +object ParamReaderFactory { + + val json = "json" + val file = "file" + + /** + * parse string content to get param reader + * @param pathOrJson + * @return + */ + def getParamReader(pathOrJson: String): ParamReader = { + val strType = paramStrType(pathOrJson) + if (json.equals(strType)) ParamJsonReader(pathOrJson) + else ParamFileReader(pathOrJson) + } + + private def paramStrType(str: String): String = { + try { + JsonUtil.toAnyMap(str) + json + } catch { + case e: Throwable => file + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/77aad495/measure/src/main/scala/org/apache/griffin/measure/context/streaming/info/InfoCacheInstance.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/context/streaming/info/InfoCacheInstance.scala b/measure/src/main/scala/org/apache/griffin/measure/context/streaming/info/InfoCacheInstance.scala index 9e8a9f6..198f157 100644 --- a/measure/src/main/scala/org/apache/griffin/measure/context/streaming/info/InfoCacheInstance.scala +++ b/measure/src/main/scala/org/apache/griffin/measure/context/streaming/info/InfoCacheInstance.scala @@ -50,4 +50,4 @@ object InfoCacheInstance extends InfoCache { } def genLock(s: String): CacheLock = MultiCacheLock(infoCaches.map(_.genLock(s))) -} \ No newline at end of file +}
