Repository: carbondata Updated Branches: refs/heads/master a470a76da -> 8b9c65540
[CARBONDATA-2839] Add custom compaction example [CARBONDATA-2839] Add custom compaction example This closes #2620 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8b9c6554 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8b9c6554 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8b9c6554 Branch: refs/heads/master Commit: 8b9c655406ca2e0b2c026cf152e13afd240e0600 Parents: a470a76 Author: Jin Zhou <xapr...@yeah.net> Authored: Wed Aug 8 17:01:43 2018 +0800 Committer: chenliang613 <chenliang...@huawei.com> Committed: Sun Aug 12 11:35:39 2018 +0530 ---------------------------------------------------------------------- .../examples/CustomCompactionExample.scala | 100 +++++++++++++++++++ 1 file changed, 100 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/8b9c6554/examples/spark2/src/main/scala/org/apache/carbondata/examples/CustomCompactionExample.scala ---------------------------------------------------------------------- diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CustomCompactionExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CustomCompactionExample.scala new file mode 100644 index 0000000..fcd5949 --- /dev/null +++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CustomCompactionExample.scala @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.examples + +import java.io.File + +import org.apache.spark.sql.SparkSession + +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties +import org.apache.carbondata.examples.util.ExampleUtils + + +/** + * This example is for showing how to use custom compaction to merge specified segments. + */ + +object CustomCompactionExample { + + def main(args: Array[String]): Unit = { + val spark = ExampleUtils.createCarbonSession("CustomCompactionExample") + exampleBody(spark) + spark.close() + } + + def exampleBody(spark : SparkSession): Unit = { + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd") + + spark.sql("DROP TABLE IF EXISTS custom_compaction_table") + + spark.sql( + s""" + | CREATE TABLE IF NOT EXISTS custom_compaction_table( + | ID Int, + | date Date, + | country String, + | name String, + | phonetype String, + | serialname String, + | salary Int, + | floatField float + | ) STORED BY 'carbondata' + """.stripMargin) + + val rootPath = new File(this.getClass.getResource("/").getPath + + "../../../..").getCanonicalPath + val path = s"$rootPath/examples/spark2/src/main/resources/dataSample.csv" + + // load 4 segments + // scalastyle:off + (1 to 4).foreach(_ => spark.sql( + s""" + | LOAD DATA LOCAL INPATH '$path' + | INTO TABLE custom_compaction_table + | OPTIONS('HEADER'='true') + """.stripMargin)) + // scalastyle:on + + // show all segment ids: 0,1,2,3 + spark.sql("SHOW SEGMENTS FOR TABLE custom_compaction_table").show() + + // query data + spark.sql("SELECT count(*) FROM custom_compaction_table").show() + spark.sql("SELECT * FROM custom_compaction_table WHERE ID=5").show() + + // do custom compaction, segments specified will be merged + spark.sql("ALTER TABLE custom_compaction_table COMPACT 'CUSTOM' WHERE SEGMENT.ID IN (1,2)") + + // show all segment ids after custom compaction + spark.sql("SHOW SEGMENTS FOR TABLE custom_compaction_table").show() + + // query data again, results should be same + spark.sql("SELECT count(*) FROM custom_compaction_table").show() + spark.sql("SELECT * FROM custom_compaction_table WHERE ID=5").show() + + CarbonProperties.getInstance().addProperty( + CarbonCommonConstants.CARBON_DATE_FORMAT, + CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT) + + // Drop table + spark.sql("DROP TABLE IF EXISTS custom_compaction_table") + } + +}