[2/2] carbondata git commit: [CARBONDATA-2613] Support csv based carbon table
[CARBONDATA-2613] Support csv based carbon table
1. create csv based carbon table using
CREATE TABLE fact_table (col1 bigint, col2 string, ..., col100 string)
STORED BY 'CarbonData'
TBLPROPERTIES(
'foramt'='csv',
'csv.delimiter'=',',
'csv.header'='col1,col2,col100')
2. Load data to this table using
ALTER TABLE fact_table ADD SEGMENT LOCATION 'path/to/data1'
This closes #2374
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/0d2769f7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/0d2769f7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/0d2769f7
Branch: refs/heads/external-format
Commit: 0d2769f75dec5f8b4b340ebc43e813f06dfc99d7
Parents: a37a2ff
Author: xuchuanyin
Authored: Wed Jun 13 09:03:28 2018 +0800
Committer: Jacky Li
Committed: Wed Jul 25 16:22:55 2018 +0800
--
.../carbondata/common/annotations/Since.java| 38 ++
.../core/constants/CarbonCommonConstants.java | 4 +
.../core/metadata/schema/table/TableInfo.java | 52 ++
.../core/statusmanager/FileFormat.java | 10 +-
.../statusmanager/FileFormatProperties.java | 32 ++
.../core/statusmanager/LoadMetadataDetails.java | 16 +-
.../hadoop/CarbonMultiBlockSplit.java | 6 +
.../carbondata/hadoop/CsvRecordReader.java | 506 +++
.../hadoop/api/CarbonFileInputFormat.java | 11 +-
.../hadoop/api/CarbonInputFormat.java | 120 -
.../hadoop/api/CarbonTableInputFormat.java | 12 +-
.../datawithoutheader_delimiter_separator.csv | 10 +
.../CsvBasedCarbonTableSuite.scala | 244 +
.../carbondata/spark/format/CsvReadSupport.java | 107
.../spark/format/VectorCsvReadSupport.java | 130 +
.../carbondata/spark/rdd/CarbonScanRDD.scala| 77 ++-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 1 +
.../command/carbonTableSchemaCommon.scala | 10 +
.../management/CarbonAddSegmentCommand.scala| 135 +
.../sql/parser/CarbonSpark2SqlParser.scala | 13 +-
20 files changed, 1499 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0d2769f7/common/src/main/java/org/apache/carbondata/common/annotations/Since.java
--
diff --git
a/common/src/main/java/org/apache/carbondata/common/annotations/Since.java
b/common/src/main/java/org/apache/carbondata/common/annotations/Since.java
new file mode 100644
index 000..b7e4391
--- /dev/null
+++ b/common/src/main/java/org/apache/carbondata/common/annotations/Since.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.common.annotations;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * The annotation indicates that the version number since a member or a type
has been present.
+ */
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.FIELD, ElementType.TYPE, ElementType.METHOD})
+public @interface Since {
+ /**
+ * the value indicating a version number since this member
+ * or type has been present.
+ */
+ String value();
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0d2769f7/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 8bf22c9..be34917 100644
---
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -966,6 +966,8 @@ public final class CarbonCommonConstants {
*/
public static final String
[2/2] carbondata git commit: [CARBONDATA-2613] Support csv based carbon table
[CARBONDATA-2613] Support csv based carbon table
1. create csv based carbon table using
CREATE TABLE fact_table (col1 bigint, col2 string, ..., col100 string)
STORED BY 'CarbonData'
TBLPROPERTIES(
'foramt'='csv',
'csv.delimiter'=',',
'csv.header'='col1,col2,col100')
2. Load data to this table using
ALTER TABLE fact_table ADD SEGMENT LOCATION 'path/to/data1'
This closes #2374
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e4bfb570
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e4bfb570
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e4bfb570
Branch: refs/heads/carbonstore
Commit: e4bfb5701b018d742b62e08dedc9760489d7d241
Parents: d0fa523
Author: xuchuanyin
Authored: Wed Jun 13 09:03:28 2018 +0800
Committer: QiangCai
Committed: Wed Jul 11 16:00:43 2018 +0800
--
.../carbondata/common/annotations/Since.java| 38 ++
.../core/constants/CarbonCommonConstants.java | 4 +
.../core/metadata/schema/table/TableInfo.java | 52 ++
.../core/statusmanager/FileFormat.java | 10 +-
.../statusmanager/FileFormatProperties.java | 32 ++
.../core/statusmanager/LoadMetadataDetails.java | 16 +-
.../hadoop/CarbonMultiBlockSplit.java | 6 +
.../carbondata/hadoop/CsvRecordReader.java | 506 +++
.../hadoop/api/CarbonFileInputFormat.java | 11 +-
.../hadoop/api/CarbonInputFormat.java | 120 -
.../hadoop/api/CarbonTableInputFormat.java | 12 +-
.../datawithoutheader_delimiter_separator.csv | 10 +
.../CsvBasedCarbonTableSuite.scala | 244 +
.../carbondata/spark/format/CsvReadSupport.java | 107
.../spark/format/VectorCsvReadSupport.java | 130 +
.../carbondata/spark/rdd/CarbonScanRDD.scala| 77 ++-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 1 +
.../command/carbonTableSchemaCommon.scala | 10 +
.../management/CarbonAddSegmentCommand.scala| 135 +
.../sql/parser/CarbonSpark2SqlParser.scala | 13 +-
20 files changed, 1499 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e4bfb570/common/src/main/java/org/apache/carbondata/common/annotations/Since.java
--
diff --git
a/common/src/main/java/org/apache/carbondata/common/annotations/Since.java
b/common/src/main/java/org/apache/carbondata/common/annotations/Since.java
new file mode 100644
index 000..b7e4391
--- /dev/null
+++ b/common/src/main/java/org/apache/carbondata/common/annotations/Since.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.common.annotations;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * The annotation indicates that the version number since a member or a type
has been present.
+ */
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.FIELD, ElementType.TYPE, ElementType.METHOD})
+public @interface Since {
+ /**
+ * the value indicating a version number since this member
+ * or type has been present.
+ */
+ String value();
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e4bfb570/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
--
diff --git
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index ff6b358..ffa2e35 100644
---
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -947,6 +947,8 @@ public final class CarbonCommonConstants {
*/
public static final String DICTIONARY_PATH =
