Repository: incubator-carbondata Updated Branches: refs/heads/master 20186ca1e -> 1cf43a156
Fix scalastyle in ExampleUtils.scala Modify scalastyle in comment to use java doc convention corrent ExampleUtils object name Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/fc67daf3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/fc67daf3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/fc67daf3 Branch: refs/heads/master Commit: fc67daf33771327f7c97be99542b7c38a989b3a8 Parents: 20186ca Author: Jacky Li <jacky.li...@huawei.com> Authored: Tue Oct 4 16:15:44 2016 +0800 Committer: jackylk <jacky.li...@huawei.com> Committed: Tue Oct 4 21:11:28 2016 +0800 ---------------------------------------------------------------------- .../examples/AllDictionaryExample.scala | 8 +- .../carbondata/examples/CarbonExample.scala | 6 +- .../examples/ComplexTypeExample.scala | 6 +- .../examples/DataFrameAPIExample.scala | 6 +- .../carbondata/examples/DatasourceExample.scala | 6 +- .../examples/GenerateDictionaryExample.scala | 8 +- .../carbondata/examples/HadoopFileExample.scala | 6 +- .../apache/carbondata/examples/PerfTest.scala | 6 +- .../carbondata/examples/util/ExampleUitls.scala | 79 -------------------- .../carbondata/examples/util/ExampleUtils.scala | 79 ++++++++++++++++++++ 10 files changed, 105 insertions(+), 105 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fc67daf3/examples/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala ---------------------------------------------------------------------- diff --git a/examples/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala b/examples/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala index dfba300..cbd866d 100644 --- a/examples/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala +++ b/examples/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala @@ -18,15 +18,15 @@ package org.apache.carbondata.examples import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.util.CarbonProperties -import org.apache.carbondata.examples.util.{AllDictionaryUtil, ExampleUitls} +import org.apache.carbondata.examples.util.{AllDictionaryUtil, ExampleUtils} object AllDictionaryExample { def main(args: Array[String]) { - val cc = ExampleUitls.createCarbonContext("CarbonExample") - val testData = ExampleUitls.currentPath + "/src/main/resources/data.csv" + val cc = ExampleUtils.createCarbonContext("CarbonExample") + val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv" val csvHeader = "ID,date,country,name,phonetype,serialname,salary" val dictCol = "|date|country|name|phonetype|serialname|" - val allDictFile = ExampleUitls.currentPath + "/src/main/resources/data.dictionary" + val allDictFile = ExampleUtils.currentPath + "/src/main/resources/data.dictionary" // extract all dictionary files from source data AllDictionaryUtil.extractDictionary(cc.sparkContext, testData, allDictFile, csvHeader, dictCol) http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fc67daf3/examples/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala ---------------------------------------------------------------------- diff --git a/examples/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala b/examples/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala index 444bc0d..a1e5f49 100644 --- a/examples/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala +++ b/examples/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala @@ -19,12 +19,12 @@ package org.apache.carbondata.examples import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.util.CarbonProperties -import org.apache.carbondata.examples.util.ExampleUitls +import org.apache.carbondata.examples.util.ExampleUtils object CarbonExample { def main(args: Array[String]) { - val cc = ExampleUitls.createCarbonContext("CarbonExample") - val testData = ExampleUitls.currentPath + "/src/main/resources/data.csv" + val cc = ExampleUtils.createCarbonContext("CarbonExample") + val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv" // Specify timestamp format based on raw data CarbonProperties.getInstance() http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fc67daf3/examples/src/main/scala/org/apache/carbondata/examples/ComplexTypeExample.scala ---------------------------------------------------------------------- diff --git a/examples/src/main/scala/org/apache/carbondata/examples/ComplexTypeExample.scala b/examples/src/main/scala/org/apache/carbondata/examples/ComplexTypeExample.scala index bbdae68..992c3f9 100644 --- a/examples/src/main/scala/org/apache/carbondata/examples/ComplexTypeExample.scala +++ b/examples/src/main/scala/org/apache/carbondata/examples/ComplexTypeExample.scala @@ -17,7 +17,7 @@ package org.apache.carbondata.examples -import org.apache.carbondata.examples.util.ExampleUitls +import org.apache.carbondata.examples.util.ExampleUtils /** * Carbon supports the complex types ARRAY and STRUCT. @@ -26,8 +26,8 @@ import org.apache.carbondata.examples.util.ExampleUitls object ComplexTypeExample { def main(args: Array[String]) { - val cc = ExampleUitls.createCarbonContext("ComplexTypeExample") - val dataPath = ExampleUitls.currentPath + "/src/main/resources/complexdata.csv" + val cc = ExampleUtils.createCarbonContext("ComplexTypeExample") + val dataPath = ExampleUtils.currentPath + "/src/main/resources/complexdata.csv" val tableName = "complexTypeTable" cc.sql(s"DROP TABLE IF EXISTS $tableName") http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fc67daf3/examples/src/main/scala/org/apache/carbondata/examples/DataFrameAPIExample.scala ---------------------------------------------------------------------- diff --git a/examples/src/main/scala/org/apache/carbondata/examples/DataFrameAPIExample.scala b/examples/src/main/scala/org/apache/carbondata/examples/DataFrameAPIExample.scala index f3fcadc..88bab3c 100644 --- a/examples/src/main/scala/org/apache/carbondata/examples/DataFrameAPIExample.scala +++ b/examples/src/main/scala/org/apache/carbondata/examples/DataFrameAPIExample.scala @@ -19,14 +19,14 @@ package org.apache.carbondata.examples import org.apache.spark.sql.SaveMode -import org.apache.carbondata.examples.util.ExampleUitls +import org.apache.carbondata.examples.util.ExampleUtils // scalastyle:off println object DataFrameAPIExample { def main(args: Array[String]) { - val cc = ExampleUitls.createCarbonContext("DataFrameAPIExample") - ExampleUitls.writeSampleCarbonFile(cc, "carbon1") + val cc = ExampleUtils.createCarbonContext("DataFrameAPIExample") + ExampleUtils.writeSampleCarbonFile(cc, "carbon1") // use datasource api to read val in = cc.read http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fc67daf3/examples/src/main/scala/org/apache/carbondata/examples/DatasourceExample.scala ---------------------------------------------------------------------- diff --git a/examples/src/main/scala/org/apache/carbondata/examples/DatasourceExample.scala b/examples/src/main/scala/org/apache/carbondata/examples/DatasourceExample.scala index 112c274..791a126 100644 --- a/examples/src/main/scala/org/apache/carbondata/examples/DatasourceExample.scala +++ b/examples/src/main/scala/org/apache/carbondata/examples/DatasourceExample.scala @@ -19,14 +19,14 @@ package org.apache.carbondata.examples import org.apache.spark.sql.{SaveMode, SQLContext} -import org.apache.carbondata.examples.util.ExampleUitls +import org.apache.carbondata.examples.util.ExampleUtils object DatasourceExample { def main(args: Array[String]) { // use CarbonContext to write CarbonData files - val cc = ExampleUitls.createCarbonContext("DatasourceExample") - ExampleUitls.writeSampleCarbonFile(cc, "table1") + val cc = ExampleUtils.createCarbonContext("DatasourceExample") + ExampleUtils.writeSampleCarbonFile(cc, "table1") // Use SQLContext to read CarbonData files val sqlContext = new SQLContext(cc.sparkContext) http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fc67daf3/examples/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala ---------------------------------------------------------------------- diff --git a/examples/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala b/examples/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala index 20ebc8c..728746b 100644 --- a/examples/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala +++ b/examples/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala @@ -23,7 +23,7 @@ import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentif import org.apache.carbondata.core.carbon.{CarbonTableIdentifier, ColumnIdentifier} import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension import org.apache.carbondata.core.carbon.path.CarbonStorePath -import org.apache.carbondata.examples.util.ExampleUitls +import org.apache.carbondata.examples.util.ExampleUtils import org.apache.carbondata.spark.load.CarbonLoaderUtil /** @@ -34,9 +34,9 @@ import org.apache.carbondata.spark.load.CarbonLoaderUtil object GenerateDictionaryExample { def main(args: Array[String]) { - val cc = ExampleUitls.createCarbonContext("GenerateDictionaryExample") - val factFilePath = ExampleUitls.currentPath + "/src/main/resources/factSample.csv" - val carbonTablePath = CarbonStorePath.getCarbonTablePath(ExampleUitls.storeLocation, + val cc = ExampleUtils.createCarbonContext("GenerateDictionaryExample") + val factFilePath = ExampleUtils.currentPath + "/src/main/resources/factSample.csv" + val carbonTablePath = CarbonStorePath.getCarbonTablePath(ExampleUtils.storeLocation, new CarbonTableIdentifier("default", "dictSample", "1")) val dictFolderPath = carbonTablePath.getMetadataDirectoryPath http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fc67daf3/examples/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala ---------------------------------------------------------------------- diff --git a/examples/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala b/examples/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala index 2099906..329b3c9 100644 --- a/examples/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala +++ b/examples/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala @@ -17,15 +17,15 @@ package org.apache.carbondata.examples -import org.apache.carbondata.examples.util.ExampleUitls +import org.apache.carbondata.examples.util.ExampleUtils import org.apache.carbondata.hadoop.CarbonInputFormat // scalastyle:off println object HadoopFileExample { def main(args: Array[String]): Unit = { - val cc = ExampleUitls.createCarbonContext("DataFrameAPIExample") - ExampleUitls.writeSampleCarbonFile(cc, "carbon1") + val cc = ExampleUtils.createCarbonContext("DataFrameAPIExample") + ExampleUtils.writeSampleCarbonFile(cc, "carbon1") val sc = cc.sparkContext val input = sc.newAPIHadoopFile(s"${cc.storePath}/default/carbon1", http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fc67daf3/examples/src/main/scala/org/apache/carbondata/examples/PerfTest.scala ---------------------------------------------------------------------- diff --git a/examples/src/main/scala/org/apache/carbondata/examples/PerfTest.scala b/examples/src/main/scala/org/apache/carbondata/examples/PerfTest.scala index 945a75e..b1f6b24 100644 --- a/examples/src/main/scala/org/apache/carbondata/examples/PerfTest.scala +++ b/examples/src/main/scala/org/apache/carbondata/examples/PerfTest.scala @@ -25,7 +25,7 @@ import org.apache.spark.sql.{CarbonContext, DataFrame, Row, SaveMode, SQLContext import org.apache.spark.sql.types.{DataTypes, StructType} import org.apache.carbondata.examples.PerfTest._ -import org.apache.carbondata.examples.util.ExampleUitls +import org.apache.carbondata.examples.util.ExampleUtils // scalastyle:off println @@ -268,7 +268,7 @@ object PerfTest { ) def main(args: Array[String]) { - val cc = ExampleUitls.createCarbonContext("PerfTest") + val cc = ExampleUtils.createCarbonContext("PerfTest") // prepare performance queries var workload = Seq[Query]() @@ -318,7 +318,7 @@ object PerfTest { } def savePath(datasource: String): String = - s"${ExampleUitls.currentPath}/target/perftest/${datasource}" + s"${ExampleUtils.currentPath}/target/perftest/${datasource}" def withTime(body: => Unit): Long = { val start = System.nanoTime() http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fc67daf3/examples/src/main/scala/org/apache/carbondata/examples/util/ExampleUitls.scala ---------------------------------------------------------------------- diff --git a/examples/src/main/scala/org/apache/carbondata/examples/util/ExampleUitls.scala b/examples/src/main/scala/org/apache/carbondata/examples/util/ExampleUitls.scala deleted file mode 100644 index f2fe8d6..0000000 --- a/examples/src/main/scala/org/apache/carbondata/examples/util/ExampleUitls.scala +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.examples.util - -import java.io.File - -import org.apache.spark.{SparkConf, SparkContext} -import org.apache.spark.sql.{CarbonContext, SaveMode} - -import org.apache.carbondata.core.util.CarbonProperties - -// scalastyle:off println - -object ExampleUitls { - - def currentPath: String = new File(this.getClass.getResource("/").getPath + "/../../") - .getCanonicalPath - val storeLocation = currentPath + "/target/store" - val kettleHome = new File(currentPath + "/../processing/carbonplugins").getCanonicalPath - - def createCarbonContext(appName: String): CarbonContext = { - val sc = new SparkContext(new SparkConf() - .setAppName(appName) - .setMaster("local[2]")) - sc.setLogLevel("ERROR") - - println(s"Starting $appName using spark version ${sc.version}") - - val cc = new CarbonContext(sc, storeLocation, currentPath + "/target/carbonmetastore") - cc.setConf("carbon.kettle.home", kettleHome) - - // whether use table split partition - // true -> use table split partition, support multiple partition loading - // false -> use node split partition, support data load by host partition - CarbonProperties.getInstance().addProperty("carbon.table.split.partition.enable", "false") - cc - } - - /** - * This func will write a sample CarbonData file containing following schema: - * c1: String, c2: String, c3: Double - */ - def writeSampleCarbonFile(cc: CarbonContext, tableName: String): Unit = { - // use CarbonContext to write CarbonData files - import cc.implicits._ - val sc = cc.sparkContext - // create a dataframe, it can be from parquet or hive table - val df = sc.parallelize(1 to 1000, 2) - .map(x => ("a", "b", x)) - .toDF("c1", "c2", "c3") - - cc.sql(s"DROP TABLE IF EXISTS $tableName") - - // save dataframe to carbon file - df.write - .format("carbondata") - .option("tableName", tableName) - .option("compress", "true") - .mode(SaveMode.Overwrite) - .save() - } -} -// scalastyle:on println - http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fc67daf3/examples/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala ---------------------------------------------------------------------- diff --git a/examples/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala b/examples/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala new file mode 100644 index 0000000..0126c66 --- /dev/null +++ b/examples/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.examples.util + +import java.io.File + +import org.apache.spark.{SparkConf, SparkContext} +import org.apache.spark.sql.{CarbonContext, SaveMode} + +import org.apache.carbondata.core.util.CarbonProperties + +// scalastyle:off println + +object ExampleUtils { + + def currentPath: String = new File(this.getClass.getResource("/").getPath + "/../../") + .getCanonicalPath + val storeLocation = currentPath + "/target/store" + val kettleHome = new File(currentPath + "/../processing/carbonplugins").getCanonicalPath + + def createCarbonContext(appName: String): CarbonContext = { + val sc = new SparkContext(new SparkConf() + .setAppName(appName) + .setMaster("local[2]")) + sc.setLogLevel("ERROR") + + println(s"Starting $appName using spark version ${sc.version}") + + val cc = new CarbonContext(sc, storeLocation, currentPath + "/target/carbonmetastore") + cc.setConf("carbon.kettle.home", kettleHome) + + // whether use table split partition + // true -> use table split partition, support multiple partition loading + // false -> use node split partition, support data load by host partition + CarbonProperties.getInstance().addProperty("carbon.table.split.partition.enable", "false") + cc + } + + /** + * This func will write a sample CarbonData file containing following schema: + * c1: String, c2: String, c3: Double + */ + def writeSampleCarbonFile(cc: CarbonContext, tableName: String): Unit = { + // use CarbonContext to write CarbonData files + import cc.implicits._ + val sc = cc.sparkContext + // create a dataframe, it can be from parquet or hive table + val df = sc.parallelize(1 to 1000, 2) + .map(x => ("a", "b", x)) + .toDF("c1", "c2", "c3") + + cc.sql(s"DROP TABLE IF EXISTS $tableName") + + // save dataframe to carbon file + df.write + .format("carbondata") + .option("tableName", tableName) + .option("compress", "true") + .mode(SaveMode.Overwrite) + .save() + } +} +// scalastyle:on println +
