[carbondata] branch master updated: [CARBONDATA-4329] Fix multiple issues with External table
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 46b62cf [CARBONDATA-4329] Fix multiple issues with External table
46b62cf is described below
commit 46b62cf6f79d1d826b498609435337b2ed342bbe
Author: Indhumathi27
AuthorDate: Wed Mar 23 11:16:01 2022 +0530
[CARBONDATA-4329] Fix multiple issues with External table
Why is this PR needed?
Issue 1:
When we create external table on transactional table location,
schema file will be present. While creating external table,
which is also transactional, the schema file is overwritten
Issue 2:
If external table is created on a location, where the source table
already exists, on drop external table, it is deleting the table data.
Query on the source table fails
What changes were proposed in this PR?
Avoid writing schema file if table type is external and transactional
Dont drop external table location data, if table_type is external
This closes #4255
---
.../core/metadata/schema/table/CarbonTable.java| 3 +-
.../core/metadata/schema/table/TableInfo.java | 4 +
.../org/apache/spark/sql/CarbonCountStar.scala | 35 ---
.../scala/org/apache/spark/sql/CarbonSource.scala | 4 +-
.../command/table/CarbonDropTableCommand.scala | 2 +-
.../spark/sql/hive/CarbonFileMetastore.scala | 6 ++
.../sql/parser/CarbonSparkSqlParserUtil.scala | 2 +-
.../createTable/TestCreateExternalTable.scala | 107 +
8 files changed, 147 insertions(+), 16 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 2c22142..1e7dfee 100644
---
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -880,8 +880,7 @@ public class CarbonTable implements Serializable, Writable {
* an internal table property set during table creation)
*/
public boolean isExternalTable() {
-String external =
tableInfo.getFactTable().getTableProperties().get("_external");
-return external != null && external.equalsIgnoreCase("true");
+return tableInfo.isExternal();
}
public boolean isFileLevelFormat() {
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
index 1ce7230..84fea45 100644
---
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
+++
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
@@ -289,6 +289,10 @@ public class TableInfo implements Serializable, Writable {
return isTransactionalTable;
}
+ public boolean isExternal() {
+return
Boolean.parseBoolean(factTable.getTableProperties().getOrDefault("_external",
"false"));
+ }
+
public void setTransactionalTable(boolean transactionalTable) {
isTransactionalTable = transactionalTable;
}
diff --git
a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCountStar.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCountStar.scala
index 678a0be..4d1ed11 100644
---
a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCountStar.scala
+++
b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCountStar.scala
@@ -24,6 +24,7 @@ import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.execution.LeafExecNode
import org.apache.spark.sql.optimizer.CarbonFilters
@@ -54,17 +55,29 @@ case class CarbonCountStar(
CarbonInputFormat.setQuerySegment(job.getConfiguration, carbonTable)
// get row count
-var rowCount = CarbonUpdateUtil.getRowCount(
- tableInputFormat.getBlockRowCount(
-job,
-carbonTable,
-CarbonFilters.getPartitions(
- Seq.empty,
- sparkSession,
- TableIdentifier(
-carbonTable.getTableName,
-Some(carbonTable.getDatabaseName))).map(_.toList.asJava).orNull,
false),
- carbonTable)
+var rowCount = try {
+ CarbonUpdateUtil.getRowCount(
+tableInputFormat.getBlockRowCount(
+ job,
+ carbonTable,
+ CarbonFilters.getPartitions(
+Seq.empty,
+
[carbondata-site] 02/02: removed 1.6.0 download links
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/carbondata-site.git
commit 098e57f2e445bb5904df18932c29544c107a2cb7
Author: kunal642
AuthorDate: Mon Mar 21 22:34:46 2022 +0530
removed 1.6.0 download links
---
content/js/custom.js | 1 -
src/main/webapp/js/custom.js | 1 -
2 files changed, 2 deletions(-)
diff --git a/content/js/custom.js b/content/js/custom.js
index e4ebb01..e5e8708 100644
--- a/content/js/custom.js
+++ b/content/js/custom.js
@@ -25,7 +25,6 @@ releaseMap.set("2.3.0", ["3.1.1", "2.4.5", "2.3.4"])
releaseMap.set("2.2.0", ["3.1.1", "2.4.5", "2.3.4"])
releaseMap.set("2.1.1", ["2.4.5", "2.3.4"])
releaseMap.set("2.0.1", ["2.4.5", "2.3.4"])
-releaseMap.set("1.6.0", ["2.1.0", "2.2.1", "2.3.2"])
var selectedRelease = "2.3.0"
var selectedPackage = ""
diff --git a/src/main/webapp/js/custom.js b/src/main/webapp/js/custom.js
index e4ebb01..e5e8708 100644
--- a/src/main/webapp/js/custom.js
+++ b/src/main/webapp/js/custom.js
@@ -25,7 +25,6 @@ releaseMap.set("2.3.0", ["3.1.1", "2.4.5", "2.3.4"])
releaseMap.set("2.2.0", ["3.1.1", "2.4.5", "2.3.4"])
releaseMap.set("2.1.1", ["2.4.5", "2.3.4"])
releaseMap.set("2.0.1", ["2.4.5", "2.3.4"])
-releaseMap.set("1.6.0", ["2.1.0", "2.2.1", "2.3.2"])
var selectedRelease = "2.3.0"
var selectedPackage = ""
[carbondata-site] branch asf-site updated (0c81575 -> 098e57f)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/carbondata-site.git. from 0c81575 removed download link from index.html new 58b4f2e removed 1.6.0 download links new 098e57f removed 1.6.0 download links The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: content/js/custom.js | 1 - src/main/webapp/js/custom.js | 1 - 2 files changed, 2 deletions(-)
[carbondata-site] 01/02: removed 1.6.0 download links
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/carbondata-site.git
commit 58b4f2e105ea0ed180311970e82e0ab14829a4cb
Author: kunal642
AuthorDate: Mon Mar 21 22:34:10 2022 +0530
removed 1.6.0 download links
---
content/js/custom.js | 2 +-
src/main/webapp/js/custom.js | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/content/js/custom.js b/content/js/custom.js
index 5b165bf..e4ebb01 100644
--- a/content/js/custom.js
+++ b/content/js/custom.js
@@ -25,7 +25,7 @@ releaseMap.set("2.3.0", ["3.1.1", "2.4.5", "2.3.4"])
releaseMap.set("2.2.0", ["3.1.1", "2.4.5", "2.3.4"])
releaseMap.set("2.1.1", ["2.4.5", "2.3.4"])
releaseMap.set("2.0.1", ["2.4.5", "2.3.4"])
-releaseMap.set("1.6.0", ["2.4.5", "2.3.4"])
+releaseMap.set("1.6.0", ["2.1.0", "2.2.1", "2.3.2"])
var selectedRelease = "2.3.0"
var selectedPackage = ""
diff --git a/src/main/webapp/js/custom.js b/src/main/webapp/js/custom.js
index 5b165bf..e4ebb01 100644
--- a/src/main/webapp/js/custom.js
+++ b/src/main/webapp/js/custom.js
@@ -25,7 +25,7 @@ releaseMap.set("2.3.0", ["3.1.1", "2.4.5", "2.3.4"])
releaseMap.set("2.2.0", ["3.1.1", "2.4.5", "2.3.4"])
releaseMap.set("2.1.1", ["2.4.5", "2.3.4"])
releaseMap.set("2.0.1", ["2.4.5", "2.3.4"])
-releaseMap.set("1.6.0", ["2.4.5", "2.3.4"])
+releaseMap.set("1.6.0", ["2.1.0", "2.2.1", "2.3.2"])
var selectedRelease = "2.3.0"
var selectedPackage = ""
[carbondata-site] branch asf-site updated: removed download link from index.html
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/carbondata-site.git The following commit(s) were added to refs/heads/asf-site by this push: new 0c81575 removed download link from index.html 0c81575 is described below commit 0c815753b51c7af2c52ebf1e0d3f716ba0c5ac69 Author: kunal642 AuthorDate: Mon Mar 21 22:05:13 2022 +0530 removed download link from index.html --- content/index.html | 18 ++ src/main/webapp/index.html | 18 ++ 2 files changed, 12 insertions(+), 24 deletions(-) diff --git a/content/index.html b/content/index.html index fefeae8..92f123b 100644 --- a/content/index.html +++ b/content/index.html @@ -292,43 +292,37 @@ -https://dlcdn.apache.org/carbondata/2.3.0/; - target="_blank">Apache CarbonData 2.3.0 +Apache CarbonData 2.3.0 Mar 2022 https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+2.3.0+Release; class="whatsnew" target="_blank">what's new -https://dlcdn.apache.org/carbondata/2.2.0/; - target="_blank">Apache CarbonData 2.2.0 +Apache CarbonData 2.2.0 Aug 2021 https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+2.2.0+Release; class="whatsnew" target="_blank">what's new -https://dlcdn.apache.org/carbondata/2.1.1/; - target="_blank">Apache CarbonData 2.1.1 +Apache CarbonData 2.1.1 May 2021 https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+2.1.1+Release; class="whatsnew" target="_blank">what's new -https://dlcdn.apache.org/carbondata/2.0.1/; - target="_blank">Apache CarbonData 2.0.1 +Apache CarbonData 2.0.1 Jun 2020 https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+2.0.1+Release; class="whatsnew" target="_blank">what's new -https://dlcdn.apache.org/carbondata/1.6.1/; - target="_blank">Apache CarbonData 1.6.1 +Apache CarbonData 1.6.1 Oct 2019 https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+1.6.1+Release; class="whatsnew" target="_blank">what's new -https://archive.apache.org/dist/carbondata/1.5.4/; - target="_blank">Apache CarbonData 1.5.4 +Apache CarbonData 1.5.4 May 2019 https://cwiki.apache.org/confluence/display/CARBONDATA/Apache+CarbonData+1.5.4+Release; class="whatsnew" target="_blank">what's new diff --git a/src/main/webapp/index.html b/src/main/webapp/index.html index fefeae8..92f123b 100644 --- a/src/main/webapp/index.html +++ b/src/main/webapp/index.html @@ -292,43 +292,37 @@ -https://dlcdn.apache.org/carbondata/2.3.0/; - target="_blank"&
[carbondata-site] branch asf-site updated (f7470ac -> a1d71d0)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/carbondata-site.git. discard f7470ac Added a sperate page to download latest releases This update removed existing revisions from the reference, leaving the reference pointing at a previous point in the repository history. * -- * -- N refs/heads/asf-site (a1d71d0) \ O -- O -- O (f7470ac) Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: content/alluxio-guide.html | 44 - content/bloomfilter-datamap-guide.html | 26 ++- content/bloomfilter-index-guide.html | 44 - content/carbon-as-spark-datasource-guide.html | 44 - content/clean-files.html | 44 - content/configuration-parameters.html | 44 - content/csdk-guide.html| 44 - content/datamap-developer-guide.html | 26 ++- content/datamap-management.html| 26 ++- content/ddl-of-carbondata.html | 44 - content/dml-of-carbondata.html | 44 - content/documentation.html | 44 - content/downloads.html | 203 - content/faq.html | 44 - content/file-structure-of-carbondata.html | 44 - content/flink-integration-guide.html | 44 - content/hive-guide.html| 44 - .../how-to-contribute-to-apache-carbondata.html| 44 - content/index-developer-guide.html | 44 - content/index-management.html | 44 - content/index-server.html | 44 - content/index.html | 30 ++- content/introduction.html | 44 - content/js/custom.js | 70 --- content/language-manual.html | 44 - content/lucene-datamap-guide.html | 26 ++- content/lucene-index-guide.html| 44 - content/mv-guide.html | 44 - content/performance-tuning.html| 44 - content/preaggregate-datamap-guide.html| 26 ++- content/prestodb-guide.html| 44 - content/prestosql-guide.html | 44 - content/quick-start-guide.html | 44 - content/release-guide.html | 44 - content/s3-guide.html | 44 - content/scd-and-cdc-guide.html | 44 - content/sdk-guide.html | 44 - content/secondary-index-guide.html | 44 - content/security.html | 44 - content/segment-management-on-carbondata.html | 44 - content/spatial-index-guide.html | 44 - content/streaming-guide.html | 44 - content/supported-data-types-in-carbondata.html| 44 - content/timeseries-datamap-guide.html | 26 ++- content/usecases.html | 44 - content/videogallery.html | 44 - src/main/webapp/alluxio-guide.html | 44 - src/main/webapp/bloomfilter-datamap-guide.html | 26 ++- src/main/webapp/bloomfilter-index-guide.html | 44 - .../webapp/carbon-as-spark-datasource-guide.html | 44 - src/main/webapp/clean-files.html | 44 - src/main/webapp/configuration-parameters.html | 44 - src/main/webapp/csdk-guide.html| 44 - src/main/webapp/datamap-developer-guide.html | 26 ++- src/main/webapp/datamap-management.html| 26 ++- src/main/webapp/ddl-of-carbondata.html | 44 - src/main/webapp/dml-of-carbondata.html | 44 - src/main/webapp/documentation.html | 44 - src/main/webapp/downloads.html | 203 - src/main/webapp/faq.html | 44 - src/main/webapp/file-structure-of-carbondata.html | 44 - src/main/webapp/flink-integration-guide.html | 44 - src/main/webapp/hive-guide.html| 44 - .../how-to-contribute-to-apache-carbondata.html| 44 - src/main/webapp/index-developer-guide.html | 44 - src/main/webapp/index-management.html | 44 - src/main/webapp/index-server.html
[carbondata-site] branch asf-site updated (98d8e76 -> a1d71d0)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/carbondata-site.git. discard 98d8e76 Added a sperate page to download latest releases This update removed existing revisions from the reference, leaving the reference pointing at a previous point in the repository history. * -- * -- N refs/heads/asf-site (a1d71d0) \ O -- O -- O (98d8e76) Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: content/alluxio-guide.html | 44 - content/bloomfilter-datamap-guide.html | 26 ++- content/bloomfilter-index-guide.html | 44 - content/carbon-as-spark-datasource-guide.html | 44 - content/clean-files.html | 44 - content/configuration-parameters.html | 44 - content/csdk-guide.html| 44 - content/datamap-developer-guide.html | 26 ++- content/datamap-management.html| 26 ++- content/ddl-of-carbondata.html | 44 - content/dml-of-carbondata.html | 44 - content/documentation.html | 44 - content/downloads.html | 203 - content/faq.html | 44 - content/file-structure-of-carbondata.html | 44 - content/flink-integration-guide.html | 44 - content/hive-guide.html| 44 - .../how-to-contribute-to-apache-carbondata.html| 44 - content/index-developer-guide.html | 44 - content/index-management.html | 44 - content/index-server.html | 44 - content/index.html | 30 ++- content/introduction.html | 44 - content/js/custom.js | 70 --- content/language-manual.html | 44 - content/lucene-datamap-guide.html | 26 ++- content/lucene-index-guide.html| 44 - content/mv-guide.html | 44 - content/performance-tuning.html| 44 - content/preaggregate-datamap-guide.html| 26 ++- content/prestodb-guide.html| 44 - content/prestosql-guide.html | 44 - content/quick-start-guide.html | 44 - content/release-guide.html | 44 - content/s3-guide.html | 44 - content/scd-and-cdc-guide.html | 44 - content/sdk-guide.html | 44 - content/secondary-index-guide.html | 44 - content/security.html | 44 - content/segment-management-on-carbondata.html | 44 - content/spatial-index-guide.html | 44 - content/streaming-guide.html | 44 - content/supported-data-types-in-carbondata.html| 44 - content/timeseries-datamap-guide.html | 26 ++- content/usecases.html | 44 - content/videogallery.html | 44 - src/main/webapp/alluxio-guide.html | 44 - src/main/webapp/bloomfilter-datamap-guide.html | 26 ++- src/main/webapp/bloomfilter-index-guide.html | 44 - .../webapp/carbon-as-spark-datasource-guide.html | 44 - src/main/webapp/clean-files.html | 44 - src/main/webapp/configuration-parameters.html | 44 - src/main/webapp/csdk-guide.html| 44 - src/main/webapp/datamap-developer-guide.html | 26 ++- src/main/webapp/datamap-management.html| 26 ++- src/main/webapp/ddl-of-carbondata.html | 44 - src/main/webapp/dml-of-carbondata.html | 44 - src/main/webapp/documentation.html | 44 - src/main/webapp/downloads.html | 203 - src/main/webapp/faq.html | 44 - src/main/webapp/file-structure-of-carbondata.html | 44 - src/main/webapp/flink-integration-guide.html | 44 - src/main/webapp/hive-guide.html| 44 - .../how-to-contribute-to-apache-carbondata.html| 44 - src/main/webapp/index-developer-guide.html | 44 - src/main/webapp/index-management.html | 44 - src/main/webapp/index-server.html
svn commit: r53231 - in /release/carbondata: 2.0.1/ 2.1.1/ 2.2.0/ 2.3.0/
Author: kunalkapoor Date: Mon Mar 21 10:32:29 2022 New Revision: 53231 Log: deleted md5 files Removed: release/carbondata/2.0.1/apache-carbondata-2.0.1-source-release.zip.md5 release/carbondata/2.1.1/apache-carbondata-2.1.1-source-release.zip.md5 release/carbondata/2.2.0/apache-carbondata-2.2.0-source-release.zip.md5 release/carbondata/2.3.0/apache-carbondata-2.3.0-source-release.zip.md5
[carbondata] branch master updated: [CARBONDATA-4327] Update documentation related to partition
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 41831ce [CARBONDATA-4327] Update documentation related to partition
41831ce is described below
commit 41831cefa60d9df8f33f1603531b18b8def1b8fa
Author: ShreelekhyaG
AuthorDate: Thu Mar 17 19:41:05 2022 +0530
[CARBONDATA-4327] Update documentation related to partition
Why is this PR needed?
Drop partition with data is not supported and a few of the links are not
working.
What changes were proposed in this PR?
Removed unsupported syntax , duplicate headings and updated the header with
proper linkage.
This closes #4254
---
docs/ddl-of-carbondata.md | 63 ---
1 file changed, 16 insertions(+), 47 deletions(-)
diff --git a/docs/ddl-of-carbondata.md b/docs/ddl-of-carbondata.md
index dbf616b..3d04684 100644
--- a/docs/ddl-of-carbondata.md
+++ b/docs/ddl-of-carbondata.md
@@ -60,12 +60,11 @@ CarbonData DDL statements are documented here,which
includes:
* [REFRESH TABLE](#refresh-table)
* [COMMENTS](#table-and-column-comment)
* [PARTITION](#partition)
- * [STANDARD PARTITION(HIVE)](#standard-partition)
-* [INSERT OVERWRITE PARTITION](#insert-overwrite)
+ * [CREATE PARTITION](#create-partition-table)
* [SHOW PARTITIONS](#show-partitions)
- * [ADD PARTITION](#add-a-new-partition)
- * [SPLIT PARTITION](#split-a-partition)
- * [DROP PARTITION](#drop-a-partition)
+ * [ADD PARTITION](#add-partition)
+ * [DROP PARTITION](#drop-partition)
+ * [INSERT OVERWRITE PARTITION](#insert-overwrite)
* [BUCKETING](#bucketing)
* [CACHE](#cache)
@@ -1085,9 +1084,20 @@ Users can specify which columns to include and exclude
for local dictionary gene
SHOW PARTITIONS [db_name.]table_name
```
+### Add Partition
+
+ This command adds the specified Hive partition.
+ ```
+ ALTER TABLE [db_name].table_name ADD PARTITION(part_spec, ...)
+ ```
+ Example:
+ ```
+ ALTER TABLE locationTable ADD PARTITION (country = 'US');
+ ```
+
Drop Partition
- This command drops the specified Hive partition only.
+ This command drops the specified Hive partition.
```
ALTER TABLE table_name DROP [IF EXISTS] PARTITION (part_spec, ...)
```
@@ -1115,47 +1125,6 @@ Users can specify which columns to include and exclude
for local dictionary gene
WHERE au.country = 'US';
```
-### Show Partitions
-
- The following command is executed to get the partition information of the
table
-
- ```
- SHOW PARTITIONS [db_name.]table_name
- ```
-
-### Add a new partition
-
- ```
- ALTER TABLE [db_name].table_name ADD PARTITION('new_partition')
- ```
-
-### Drop a partition
-
- Only drop partition definition, but keep data
- ```
- ALTER TABLE [db_name].table_name DROP PARTITION(partition_id)
- ```
-
- Drop both partition definition and data
- ```
- ALTER TABLE [db_name].table_name DROP PARTITION(partition_id) WITH DATA
- ```
-
- **NOTE:**
- * Hash partition table is not supported for ADD, SPLIT and DROP commands.
- * Partition Id: in CarbonData like the hive, folders are not used to divide
partitions instead partition id is used to replace the task id. It could make
use of the characteristic and meanwhile reduce some metadata.
-
- ```
- SegmentDir/0_batchno0-0-1502703086921.carbonindex
-^
- SegmentDir/part-0-0_batchno0-0-1502703086921.carbondata
- ^
- ```
-
- Here are some useful tips to improve query performance of carbonData
partition table:
- * The partitioned column can be excluded from SORT_COLUMNS, this will let
other columns to do the efficient sorting.
- * When writing SQL on a partition table, try to use filters on the partition
column.
-
## BUCKETING
Bucketing feature can be used to distribute/organize the table/partition
data into multiple files such
svn commit: r52992 - /release/carbondata/2.3.0/
Author: kunalkapoor Date: Sat Mar 12 05:17:27 2022 New Revision: 52992 Log: Uploading 2.3.0 signature files Added: release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.3.4-hadoop2.7.2.jar.asc release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.3.4-hadoop2.7.2.jar.sha512 release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.4.5-hadoop2.7.2.jar.asc release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.4.5-hadoop2.7.2.jar.sha512 release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark3.1.1-hadoop2.7.2.jar.asc release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark3.1.1-hadoop2.7.2.jar.sha512 release/carbondata/2.3.0/apache-carbondata-2.3.0-source-release.zip (with props) release/carbondata/2.3.0/apache-carbondata-2.3.0-source-release.zip.asc release/carbondata/2.3.0/apache-carbondata-2.3.0-source-release.zip.md5 release/carbondata/2.3.0/apache-carbondata-2.3.0-source-release.zip.sha512 Added: release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.3.4-hadoop2.7.2.jar.asc == --- release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.3.4-hadoop2.7.2.jar.asc (added) +++ release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.3.4-hadoop2.7.2.jar.asc Sat Mar 12 05:17:27 2022 @@ -0,0 +1,14 @@ +-BEGIN PGP SIGNATURE- + +iQGzBAABCgAdFiEER4oGoEjiZ19CtKBtzP3MO+Fg6doFAmIsK9UACgkQzP3MO+Fg +6dppUgv8D3OXKxzXOXfZ2NBhRxtZHIbfoCC5SAaHopsa6IDPI80MKxaKoriqhTQO +SF/PWFFSiNPMejjXXk4Vcg4dgO+SLc/PxwPc4w71eFsAvdqsmqHFhty+/xUtZyjt +rJx6J3cepF1ChNeRK/NqNMp00+1slHOZ5cG3vYMrGwfmfCCMic3zCIsljsJYI6uw +YeHV5yjLNdWi0m4OFHvkSWCQ/8OeDeot4lVz/2xTmLTWD+Xvz8pbv0MRjy6fO72D +TjwuzCYraZInp2lAOVbNcREdxyge/MML5Cwzuk6Hz5vwRolLRp6g91HvI4oQSm8J +LluYmHgiTvdzq5GjuG84V2ABfWOMVl/oICJQUk1YfRS7YguSYw052fZb3q4z9NBQ +1jCY+u1jpv2VfHwRIrJ77V3RJ88zUTOd3fLjsHNRYOyN6XmcdVBiz9gQ6LNYuLuI +Agw6YUd/b1bAAc5sLVBSZjJ2U50M3Xm7160Yap0FtAvp04jvUdh6qOzhiUi+ev0r +og+eRhMG +=XiRF +-END PGP SIGNATURE- Added: release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.3.4-hadoop2.7.2.jar.sha512 == --- release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.3.4-hadoop2.7.2.jar.sha512 (added) +++ release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.3.4-hadoop2.7.2.jar.sha512 Sat Mar 12 05:17:27 2022 @@ -0,0 +1 @@ +470c12affdb547c79ad12e4848fce7bea3ceae2ecbd6dc1a2efbd03cce58cbe3cdaab973e583421385d39c3ba3116c12dc4ceb92c1e501c60d4dd3653ef2026a apache-carbondata-2.3.0-bin-spark2.3.4-hadoop2.7.2.jar Added: release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.4.5-hadoop2.7.2.jar.asc == --- release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.4.5-hadoop2.7.2.jar.asc (added) +++ release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.4.5-hadoop2.7.2.jar.asc Sat Mar 12 05:17:27 2022 @@ -0,0 +1,14 @@ +-BEGIN PGP SIGNATURE- + +iQGzBAABCgAdFiEER4oGoEjiZ19CtKBtzP3MO+Fg6doFAmIsK+QACgkQzP3MO+Fg +6dq5vgwA12CPcKYVfHqReujLkV9FkxZtFT4MHgtUHfTf+JT7P43twIATM8uPVoeb +icvGyUYAPtOw718v4DEUrH/CfMZ0/FHF5p2/6sj0Ma/c9JTtyO3fzWO0NXo2IbAb +NwRNmJ/sDZJhNNgo2wdSvG2fLAmEBb/b05TCFnXMlCz9rI1A8yqV45Ir3RjFgsB8 +tmxQSctTE+LSpMjNFPMfIydx/WrxbuBriwuZ7wiNJne8uIzSP2kpftrXF4vTsttw +Qkt9k+ppHkR0VU5uz7+vvhzFLpGBA/uhv3KsT6D8AwECxrsozYyP9k/W7nZDSe4D +mlr69SARprRFVq37Px9gi1awGXBlEagdr3vW6agLCzcuZuLZ+Se4VNcTy8wj9kNS +S7oXcfEvkoxp5lZdW5LItsPbW/XDYKbww5tsY1W+yk6I0MLuI/6JQiX4d9bw47qw +RLUyFg+GkZoWfRiNWNfg1teFC42fzUQVz7GFeMdeumh/XhM5ffmtiaY1qJ/TJiPI +Vxdr0osH +=vG4V +-END PGP SIGNATURE- Added: release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.4.5-hadoop2.7.2.jar.sha512 == --- release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.4.5-hadoop2.7.2.jar.sha512 (added) +++ release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.4.5-hadoop2.7.2.jar.sha512 Sat Mar 12 05:17:27 2022 @@ -0,0 +1 @@ +7abc0a30d641746388ddac13258003c4d486956d8955cd5389dde40e423f259efea337c270bccdf6f4ae469d17269ff6e9e771e28a36afe18cf0b73730690f7f apache-carbondata-2.3.0-bin-spark2.4.5-hadoop2.7.2.jar Added: release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark3.1.1-hadoop2.7.2.jar.asc == --- release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark3.1.1-hadoop2.7.2.jar.asc (added) +++ release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark3.1.1-hadoop2.7.2.jar.asc Sat Mar 12 05:17:27 2022 @@ -0,0 +1,14 @@ +-BEGIN PGP SIGNATURE- + +iQGzBAABCgAdFiEER4oGoEjiZ19CtKBtzP3MO+Fg6doFAmIsK+kACgkQzP3MO+Fg +6dqPXQwAs2HkgrNgUBJ6p76uAlrUFUpzfEbE0Vc1NlSEe7LsPiWVC7MPpAWI3zhI +H0rgf0wQPSVohxe2nckmLjCn4p1/vsVdHnO4GGxD/cgwo9Me1979r9lJ6bcegQuQ +ALmoxHNP7wfEyWI74c2D
[carbondata] 02/03: [maven-release-plugin] prepare for next development iteration
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git commit e25d5b6fa292ba00cc0ed50b8f4610c2b968c0c5 Author: kunal642 AuthorDate: Wed Jan 19 16:50:11 2022 +0530 [maven-release-plugin] prepare for next development iteration --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 1107569..d8d03b7 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 88dc1b8..6205781 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 0e74e7a..a5f2e75 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index da34787..360fcff 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index ca666db..64f0d9f 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index a8c5b60..e992b2e 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index 83bbbd9..46f6695 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index 298337b..c38b0f9 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index 1096951..7ec45b4 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 3522097..bccbae1 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index 88d8afb..e23b94d 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index f4ce62e..b1a2414 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index ba37c2e..a4b320e 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/integration/flink-proxy/pom.xml b/integration/flink-proxy/pom.xml index 832d804..b7aee50 100644 --- a/integration/flink-proxy/pom.xml +++ b/integration/flink-proxy
[carbondata] 03/03: [CARBONDATA-4306] Fix Query Performance issue for Spark 3.1
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
commit a838531f243f4b826778b51d1f238461d3414fe4
Author: Indhumathi27
AuthorDate: Fri Mar 4 13:25:36 2022 +0530
[CARBONDATA-4306] Fix Query Performance issue for Spark 3.1
Why is this PR needed?
Some non-partition filters, which cannot be handled by carbon, is not
pushed down to spark.
What changes were proposed in this PR?
If partition filters is non empty, then the filter column is not partition
column, then push the filter to spark
This closes #4252
---
.../apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala
index 824c7fb..85044a2 100644
---
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala
+++
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala
@@ -158,10 +158,10 @@ private[sql] object CarbonSourceStrategy extends
SparkStrategy {
SparkSession.getActiveSession.get,
relation.catalogTable.get.identifier
)
+ // remove dynamic partition filter from predicates
+ filterPredicates = CarbonToSparkAdapter.getDataFilter(partitionSet,
+allPredicates, partitionsFilter)
}
-// remove dynamic partition filter from predicates
-filterPredicates = CarbonToSparkAdapter.getDataFilter(partitionSet,
- allPredicates, partitionsFilter)
val table = relation.relation.asInstanceOf[CarbonDatasourceHadoopRelation]
val projects = rawProjects.map {p =>
p.transform {
[carbondata] 01/03: [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc2
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git commit 9b749511153c653e15bfcfbd2757dd4d8f337530 Author: kunal642 AuthorDate: Wed Jan 19 16:48:56 2022 +0530 [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc2 --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index d4c7aba..1107569 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 68fea9c..88dc1b8 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index f4d2c16..0e74e7a 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index cd4b691..da34787 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index 0f804f4..ca666db 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index c547d37..a8c5b60 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index fcbd80c..83bbbd9 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index e8be41e..298337b 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index c4071ba..1096951 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 3873996..3522097 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index b10583d..88d8afb 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index b998143..f4ce62e 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index c443497..ba37c2e 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/integration/flink-proxy/pom.xml b/integration/flink-proxy/pom.xml index 23f5169..832d804 100644 --- a/integration/flink-proxy/pom.xml +++ b/integration/flink-proxy
[carbondata] branch master updated (19343a7 -> a838531)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git. from 19343a7 [CARBONDATA-4326] MV not hitting with multiple sessions issue fix new 9b74951 [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc2 new e25d5b6 [maven-release-plugin] prepare for next development iteration new a838531 [CARBONDATA-4306] Fix Query Performance issue for Spark 3.1 The 3 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- .../apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala | 6 +++--- mv/plan/pom.xml | 2 +- pom.xml | 2 +- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 25 files changed, 27 insertions(+), 27 deletions(-)
svn commit: r52235 - in /release/carbondata/2.3.0: ./ apache-carbondata-2.3.0-bin-spark2.3.4-hadoop2.7.2.jar apache-carbondata-2.3.0-bin-spark2.4.5-hadoop2.7.2.jar apache-carbondata-2.3.0-bin-spark3.1
Author: kunalkapoor Date: Mon Jan 24 06:05:09 2022 New Revision: 52235 Log: uploading 2.2.0 release Added: release/carbondata/2.3.0/ release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.3.4-hadoop2.7.2.jar (with props) release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.4.5-hadoop2.7.2.jar (with props) release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark3.1.1-hadoop2.7.2.jar (with props) Added: release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.3.4-hadoop2.7.2.jar == Binary file - no diff available. Propchange: release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.3.4-hadoop2.7.2.jar -- svn:mime-type = application/octet-stream Added: release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.4.5-hadoop2.7.2.jar == Binary file - no diff available. Propchange: release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark2.4.5-hadoop2.7.2.jar -- svn:mime-type = application/octet-stream Added: release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark3.1.1-hadoop2.7.2.jar == Binary file - no diff available. Propchange: release/carbondata/2.3.0/apache-carbondata-2.3.0-bin-spark3.1.1-hadoop2.7.2.jar -- svn:mime-type = application/octet-stream
svn commit: r52151 - in /dev/carbondata/2.3.0-rc2: ./ apache-carbondata-2.3.0-source-release.zip apache-carbondata-2.3.0-source-release.zip.asc apache-carbondata-2.3.0-source-release.zip.md5 apache-ca
Author: kunalkapoor Date: Wed Jan 19 15:52:29 2022 New Revision: 52151 Log: Upload 2.3.0-rc2 Added: dev/carbondata/2.3.0-rc2/ dev/carbondata/2.3.0-rc2/apache-carbondata-2.3.0-source-release.zip (with props) dev/carbondata/2.3.0-rc2/apache-carbondata-2.3.0-source-release.zip.asc dev/carbondata/2.3.0-rc2/apache-carbondata-2.3.0-source-release.zip.md5 dev/carbondata/2.3.0-rc2/apache-carbondata-2.3.0-source-release.zip.sha512 Added: dev/carbondata/2.3.0-rc2/apache-carbondata-2.3.0-source-release.zip == Binary file - no diff available. Propchange: dev/carbondata/2.3.0-rc2/apache-carbondata-2.3.0-source-release.zip -- svn:mime-type = application/octet-stream Added: dev/carbondata/2.3.0-rc2/apache-carbondata-2.3.0-source-release.zip.asc == --- dev/carbondata/2.3.0-rc2/apache-carbondata-2.3.0-source-release.zip.asc (added) +++ dev/carbondata/2.3.0-rc2/apache-carbondata-2.3.0-source-release.zip.asc Wed Jan 19 15:52:29 2022 @@ -0,0 +1,14 @@ +-BEGIN PGP SIGNATURE- + +iQGzBAABCgAdFiEER4oGoEjiZ19CtKBtzP3MO+Fg6doFAmHmUKQACgkQzP3MO+Fg +6dp1QAv/R1fo7WKqxOqxN7Nw2Efk0VZ/ylNjwW0Cw0aQpcMw4SDMWkGYY6iZPXPm +Y2aUfolFB7QRtFVPpl6EjT2O4zHBgYOy+4PXbyKa663YuBetiFp510wFPK6Wz/It +p0EEbMM2Ec1xvWJaX/dZEhPX1GTeoxNDNaN+UAjkMCLCA0i9d4pLtZo0A22GV5bx +4C0HPXUZQpUx6gavcM6vRNwAtJo6Hnw/rhayXSZmNmdADZEUhT/Zry+k1MAIvWV4 +IRHPpX0qZ65CrHbaK8QUWRzrfzIRZ4qv+ZEJ6S7KNPIT+J6fIEgdqtZ6S/DD5UO5 +mldGz4/69/TJyQPDDqTcl8EF62gVMHNemIzJjbpleYRHmfXtS1jd5bf9X9+SjWS4 +cDY15jHgKFPRrUie+MRsD1MuaaV4vXQaaFK6BIcv1NW2i1hJ2bhbVBhRVLXm+Hq2 +xX072nTY/v9vGQxRnw5Q93KTD+m00cgEPfdh56sm1MZKZ4teNwcBpKnILSrzwLld +xT1c4XI8 +=Pi5J +-END PGP SIGNATURE- Added: dev/carbondata/2.3.0-rc2/apache-carbondata-2.3.0-source-release.zip.md5 == --- dev/carbondata/2.3.0-rc2/apache-carbondata-2.3.0-source-release.zip.md5 (added) +++ dev/carbondata/2.3.0-rc2/apache-carbondata-2.3.0-source-release.zip.md5 Wed Jan 19 15:52:29 2022 @@ -0,0 +1 @@ +5240f4ff0a32a845a76038d7dd335ab3 apache-carbondata-2.3.0-source-release.zip Added: dev/carbondata/2.3.0-rc2/apache-carbondata-2.3.0-source-release.zip.sha512 == --- dev/carbondata/2.3.0-rc2/apache-carbondata-2.3.0-source-release.zip.sha512 (added) +++ dev/carbondata/2.3.0-rc2/apache-carbondata-2.3.0-source-release.zip.sha512 Wed Jan 19 15:52:29 2022 @@ -0,0 +1 @@ +720addf7c3926812533102349c86d8030581aa5bdc7a93ee7d8e0ea4cbc42b7128762af816c60a44c43a983c988370496578fccc4889f830096db79e93dba6b1 apache-carbondata-2.3.0-source-release.zip
[carbondata] branch branch-2.3 created (now 6db604a)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/carbondata.git. at 6db604a [maven-release-plugin] prepare for next development iteration This branch includes the following new commits: new 6db604a [maven-release-plugin] prepare for next development iteration The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[carbondata] 01/01: [maven-release-plugin] prepare for next development iteration
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/carbondata.git commit 6db604a6389673194b30e3c45e7252af6400d54b Author: kunal642 AuthorDate: Wed Jan 19 16:50:11 2022 +0530 [maven-release-plugin] prepare for next development iteration --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 1107569..d8d03b7 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 88dc1b8..6205781 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 0e74e7a..a5f2e75 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index da34787..360fcff 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index ca666db..64f0d9f 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index a8c5b60..e992b2e 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index 83bbbd9..46f6695 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index 298337b..c38b0f9 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index 1096951..7ec45b4 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 3522097..bccbae1 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index 88d8afb..e23b94d 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index f4ce62e..b1a2414 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index ba37c2e..a4b320e 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/integration/flink-proxy/pom.xml b/integration/flink-proxy/pom.xml index 832d804..b7aee50 100644 --- a/integration/flink-proxy/pom.xml +++ b/integration/flink-proxy
[carbondata] 01/01: [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc2
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to annotated tag apache-carbondata-2.3.0-rc2 in repository https://gitbox.apache.org/repos/asf/carbondata.git commit fee16db32bd29007d0de06e60fa74a1ee929cd90 Author: kunal642 AuthorDate: Wed Jan 19 16:48:56 2022 +0530 [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc2 --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index d4c7aba..1107569 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 68fea9c..88dc1b8 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index f4d2c16..0e74e7a 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index cd4b691..da34787 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index 0f804f4..ca666db 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index c547d37..a8c5b60 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index fcbd80c..83bbbd9 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index e8be41e..298337b 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index c4071ba..1096951 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 3873996..3522097 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index b10583d..88d8afb 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index b998143..f4ce62e 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index c443497..ba37c2e 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/integration/flink-proxy/pom.xml b/integration/flink-proxy/pom.xml index 23f5169..832d804 100644 --- a/integration/flink-proxy/pom.xml
[carbondata] annotated tag apache-carbondata-2.3.0-rc2 created (now eca13c5)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to annotated tag apache-carbondata-2.3.0-rc2 in repository https://gitbox.apache.org/repos/asf/carbondata.git. at eca13c5 (tag) tagging fee16db32bd29007d0de06e60fa74a1ee929cd90 (commit) by kunal642 on Wed Jan 19 16:49:38 2022 +0530 - Log - [maven-release-plugin] copy for tag apache-carbondata-2.3.0-rc2 --- This annotated tag includes the following new commits: new fee16db [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc2 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[carbondata] branch branch-2.3 deleted (was db0896a)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/carbondata.git. was db0896a [maven-release-plugin] prepare for next development iteration This change permanently discards the following revisions: discard db0896a [maven-release-plugin] prepare for next development iteration discard 253e9cc [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc2
[carbondata] annotated tag apache-carbondata-2.3.0-rc2 deleted (was 65454a1)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to annotated tag apache-carbondata-2.3.0-rc2 in repository https://gitbox.apache.org/repos/asf/carbondata.git. *** WARNING: tag apache-carbondata-2.3.0-rc2 was deleted! *** tag was 65454a1 The revisions that were on this annotated tag are still contained in other references; therefore, this change does not discard any commits from the repository.
[carbondata] 01/01: [maven-release-plugin] prepare for next development iteration
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/carbondata.git commit db0896a8bdd195ba6b8623d3e939e75d3b6433f0 Author: kunal642 AuthorDate: Wed Jan 19 16:14:55 2022 +0530 [maven-release-plugin] prepare for next development iteration --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 1107569..d8d03b7 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 88dc1b8..6205781 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 0e74e7a..a5f2e75 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index da34787..360fcff 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index ca666db..64f0d9f 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index a8c5b60..e992b2e 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index 83bbbd9..46f6695 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index 298337b..c38b0f9 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index 1096951..7ec45b4 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 3522097..bccbae1 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index 88d8afb..e23b94d 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index f4ce62e..b1a2414 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index ba37c2e..a4b320e 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/integration/flink-proxy/pom.xml b/integration/flink-proxy/pom.xml index 832d804..b7aee50 100644 --- a/integration/flink-proxy/pom.xml +++ b/integration/flink-proxy
[carbondata] branch branch-2.3 created (now db0896a)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/carbondata.git. at db0896a [maven-release-plugin] prepare for next development iteration This branch includes the following new commits: new db0896a [maven-release-plugin] prepare for next development iteration The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[carbondata] annotated tag apache-carbondata-2.3.0-rc2 created (now 65454a1)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to annotated tag apache-carbondata-2.3.0-rc2 in repository https://gitbox.apache.org/repos/asf/carbondata.git. at 65454a1 (tag) tagging 253e9cc7ce6a5dbc7787880738fec6582c34b685 (commit) by kunal642 on Wed Jan 19 16:14:40 2022 +0530 - Log - [maven-release-plugin] copy for tag apache-carbondata-2.3.0-rc2 --- This annotated tag includes the following new commits: new 253e9cc [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc2 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[carbondata] 01/01: [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc2
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to annotated tag apache-carbondata-2.3.0-rc2 in repository https://gitbox.apache.org/repos/asf/carbondata.git commit 253e9cc7ce6a5dbc7787880738fec6582c34b685 Author: kunal642 AuthorDate: Wed Jan 19 11:36:34 2022 +0530 [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc2 --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index d4c7aba..1107569 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 68fea9c..88dc1b8 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index f4d2c16..0e74e7a 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index cd4b691..da34787 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index 0f804f4..ca666db 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index c547d37..a8c5b60 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index fcbd80c..83bbbd9 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index e8be41e..298337b 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index c4071ba..1096951 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 3873996..3522097 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index b10583d..88d8afb 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index b998143..f4ce62e 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index c443497..ba37c2e 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/integration/flink-proxy/pom.xml b/integration/flink-proxy/pom.xml index 23f5169..832d804 100644 --- a/integration/flink-proxy/pom.xml
[carbondata] annotated tag apache-carbondata-2.3.0-rc2 deleted (was d53c1e7)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to annotated tag apache-carbondata-2.3.0-rc2 in repository https://gitbox.apache.org/repos/asf/carbondata.git. *** WARNING: tag apache-carbondata-2.3.0-rc2 was deleted! *** tag was d53c1e7 This change permanently discards the following revisions: discard 56a71ca [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc2
[carbondata] branch branch-2.3 deleted (was dbc1157)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/carbondata.git. was dbc1157 [maven-release-plugin] prepare for next development iteration This change permanently discards the following revisions: discard dbc1157 [maven-release-plugin] prepare for next development iteration
[carbondata] branch branch-2.3 created (now dbc1157)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/carbondata.git. at dbc1157 [maven-release-plugin] prepare for next development iteration This branch includes the following new commits: new dbc1157 [maven-release-plugin] prepare for next development iteration The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[carbondata] 01/01: [maven-release-plugin] prepare for next development iteration
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/carbondata.git commit dbc11579397801b6ac31c1d7a41832e70f2cd905 Author: kunal642 AuthorDate: Wed Jan 19 10:54:32 2022 +0530 [maven-release-plugin] prepare for next development iteration --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 1107569..d8d03b7 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 88dc1b8..6205781 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 0e74e7a..a5f2e75 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index da34787..360fcff 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index ca666db..64f0d9f 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index a8c5b60..e992b2e 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index 83bbbd9..46f6695 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index 298337b..c38b0f9 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index 1096951..7ec45b4 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 3522097..bccbae1 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index 88d8afb..e23b94d 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index f4ce62e..b1a2414 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index ba37c2e..a4b320e 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/integration/flink-proxy/pom.xml b/integration/flink-proxy/pom.xml index 832d804..b7aee50 100644 --- a/integration/flink-proxy/pom.xml +++ b/integration/flink-proxy
[carbondata] annotated tag apache-carbondata-2.3.0-rc2 created (now d53c1e7)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to annotated tag apache-carbondata-2.3.0-rc2 in repository https://gitbox.apache.org/repos/asf/carbondata.git. at d53c1e7 (tag) tagging 56a71caea37b51c6d045c5a0b386de842be799a3 (commit) by kunal642 on Wed Jan 19 10:54:16 2022 +0530 - Log - [maven-release-plugin] copy for tag apache-carbondata-2.3.0-rc2 --- This annotated tag includes the following new commits: new 56a71ca [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc2 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[carbondata] 01/01: [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc2
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to annotated tag apache-carbondata-2.3.0-rc2 in repository https://gitbox.apache.org/repos/asf/carbondata.git commit 56a71caea37b51c6d045c5a0b386de842be799a3 Author: kunal642 AuthorDate: Wed Jan 19 10:48:52 2022 +0530 [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc2 --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index d4c7aba..1107569 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 68fea9c..88dc1b8 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index f4d2c16..0e74e7a 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index cd4b691..da34787 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index 0f804f4..ca666db 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index c547d37..a8c5b60 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index fcbd80c..83bbbd9 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index e8be41e..298337b 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index c4071ba..1096951 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 3873996..3522097 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index b10583d..88d8afb 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index b998143..f4ce62e 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index c443497..ba37c2e 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/integration/flink-proxy/pom.xml b/integration/flink-proxy/pom.xml index 23f5169..832d804 100644 --- a/integration/flink-proxy/pom.xml
[carbondata] branch branch-2.3 deleted (was 494ad6c)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/carbondata.git. was 494ad6c [maven-release-plugin] prepare for next development iteration This change permanently discards the following revisions: discard 494ad6c [maven-release-plugin] prepare for next development iteration
[carbondata] branch branch-2.3 updated: [maven-release-plugin] prepare for next development iteration
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/carbondata.git The following commit(s) were added to refs/heads/branch-2.3 by this push: new 494ad6c [maven-release-plugin] prepare for next development iteration 494ad6c is described below commit 494ad6ce257354ba143c39c6c336bb02638d206a Author: kunal642 AuthorDate: Tue Jan 18 11:20:31 2022 +0530 [maven-release-plugin] prepare for next development iteration --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 1107569..d8d03b7 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 88dc1b8..6205781 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 0e74e7a..a5f2e75 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index da34787..360fcff 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index ca666db..64f0d9f 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index a8c5b60..e992b2e 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index 83bbbd9..46f6695 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index 298337b..c38b0f9 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index 1096951..7ec45b4 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 3522097..bccbae1 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index 88d8afb..e23b94d 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index f4ce62e..b1a2414 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index ba37c2e..a4b320e 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml
[carbondata] 01/01: [maven-release-plugin] prepare release apache-carbondata-1.1.0-rc3
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/carbondata.git commit 72178a479887e58d035e1d0869001f7704af3729 Author: kunal642 AuthorDate: Tue Jan 18 11:13:31 2022 +0530 [maven-release-plugin] prepare release apache-carbondata-1.1.0-rc3 --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index d4c7aba..1107569 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 68fea9c..88dc1b8 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index f4d2c16..0e74e7a 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index cd4b691..da34787 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index 0f804f4..ca666db 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index c547d37..a8c5b60 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index fcbd80c..83bbbd9 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index e8be41e..298337b 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index c4071ba..1096951 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 3873996..3522097 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index b10583d..88d8afb 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index b998143..f4ce62e 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index c443497..ba37c2e 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/integration/flink-proxy/pom.xml b/integration/flink-proxy/pom.xml index 23f5169..832d804 100644 --- a/integration/flink-proxy/pom.xml +++ b/integration/flink
[carbondata] branch master updated: [CARBONDATA-4318]Improve load overwrite performance for partition tables
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 308906e [CARBONDATA-4318]Improve load overwrite performance for
partition tables
308906e is described below
commit 308906e459607383eedf20c18236c7d17509959e
Author: akashrn5
AuthorDate: Wed Dec 8 15:49:47 2021 +0530
[CARBONDATA-4318]Improve load overwrite performance for partition tables
Why is this PR needed?
With the increase in the number of overwrite loads for the partition table,
the time takes for each load keeps on increasing over time. This is because,
1. whenever a load overwrite for partition table is fired, it basically
means
that we need to overwrite or drop the partitions if anything overlaps with
current partitions getting loaded. Since carbondata stores the partition
information in the segments file, to identify and drop partitions, it's
reading all the previous segment files to identify and drop the overwriting
partitions, which leads to a decrease in performance.
2. After partition load is completed, a cleanSegments method is called which
again reads segment file and table status file to identify MArked for delete
segments to clean. But Since the force clean is false and timeout also is
more than a day by default, it's not necessary to call this method.
Clean files should handle this part.
What changes were proposed in this PR?
1. we already have the information about current partitions, so with that
first
identify if there are any partitions to overwrite, if yes then only we read
segment
files to call dropParitition, else we don't read the segment files
unnecessarily.
It also contains other refactoring to avoid reading table status file also.
2. no need to call clean segments after every load. Clean files will take
care
to delete the expired ones.
This closes #4242
---
.../carbondata/core/metadata/SegmentFileStore.java | 15 +++--
.../hadoop/api/CarbonOutputCommitter.java | 73 +-
.../spark/rdd/CarbonDropPartitionRDD.scala | 6 +-
.../command/management/CommonLoadUtils.scala | 5 --
.../CarbonAlterTableDropHivePartitionCommand.scala | 15 ++---
.../allqueries/InsertIntoCarbonTableTestCase.scala | 24 ---
6 files changed, 76 insertions(+), 62 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
index 67c061b..eda819b 100644
---
a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
+++
b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
@@ -1005,10 +1005,9 @@ public class SegmentFileStore {
* @param uniqueId
* @throws IOException
*/
- public void dropPartitions(Segment segment, List
partitionSpecs,
+ public void dropPartitions(String segmentNo, List partitionLocations,
String uniqueId, List toBeDeletedSegments, List
toBeUpdatedSegments)
throws IOException {
-readSegment(tablePath, segment.getSegmentFileName());
boolean updateSegment = false;
for (Map.Entry entry :
segmentFile.getLocationMap().entrySet()) {
String location = entry.getKey();
@@ -1017,9 +1016,9 @@ public class SegmentFileStore {
}
Path path = new Path(location);
// Update the status to delete if path equals
- if (null != partitionSpecs) {
-for (PartitionSpec spec : partitionSpecs) {
- if (path.equals(spec.getLocation())) {
+ if (null != partitionLocations) {
+for (String partitionLocation : partitionLocations) {
+ if (path.toString().equals(partitionLocation)) {
entry.getValue().setStatus(SegmentStatus.MARKED_FOR_DELETE.getMessage());
updateSegment = true;
break;
@@ -1031,7 +1030,7 @@ public class SegmentFileStore {
String writePath = CarbonTablePath.getSegmentFilesLocation(tablePath);
writePath =
writePath + CarbonCommonConstants.FILE_SEPARATOR +
- SegmentFileStore.genSegmentFileName(segment.getSegmentNo(),
String.valueOf(uniqueId))
+ SegmentFileStore.genSegmentFileName(segmentNo,
String.valueOf(uniqueId))
+ CarbonTablePath.SEGMENT_EXT;
writeSegmentFile(segmentFile, writePath);
}
@@ -1044,10 +1043,10 @@ public class SegmentFileStore {
}
}
if (deleteSegment) {
- toBeDeletedSegments.add(segment.getSegmentNo());
+ toBeDeletedSegments.add(segmentNo);
}
if (updateSegment) {
- toBeUpdatedSegments.add(segment.getSegmentNo());
+ toBeUpdatedSegments.add(segmentNo);
}
}
diff --git
a/hadoop/src/main/java/org/apache/carbondata/hadoop
[carbondata] branch master updated: [CARBONDATA-4319] Fixed clean files not deleteting stale delete delta files after horizontal compaction
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new a072e7a [CARBONDATA-4319] Fixed clean files not deleteting stale
delete delta files after horizontal compaction
a072e7a is described below
commit a072e7a9e8af7491ee009c3f4005cd028801f02e
Author: Vikram Ahuja
AuthorDate: Wed Dec 22 13:21:06 2021 +0530
[CARBONDATA-4319] Fixed clean files not deleteting stale delete delta files
after horizontal compaction
Why is this PR needed?
After horizontal compaction was performed on partition and non partition
tables, the clean files
operation was not deleting the stale delete delta files. the code was
removed as the part of clean
files refactoring done previously.
What changes were proposed in this PR?
Clean files with force option now handles removal of these stale delta
files as well as the stale
tableupdatestatus file for both partition and non partition table.
This closes #4245
---
.../carbondata/core/mutate/CarbonUpdateUtil.java | 122 +
.../core/statusmanager/SegmentStatusManager.java | 20
docs/clean-files.md| 8 +-
.../apache/carbondata/trash/DataTrashManager.scala | 20 +++-
.../cleanfiles/TestCleanFileCommand.scala | 34 ++
.../TestCleanFilesCommandPartitionTable.scala | 68
6 files changed, 266 insertions(+), 6 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index b814152..da8d329 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -25,6 +25,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -688,4 +689,125 @@ public class CarbonUpdateUtil {
}
return latestTimestamp;
}
+
+
+ /**
+ * Handling of the clean up of old carbondata files, index files , delete
delta,
+ * update status files.
+ *
+ * @param table clean up will be handled on this table.
+ * @param isDryRun if clean files dryRun selected, then only size will be
shown,
+ *files will not be deleted.
+ */
+ public static long cleanUpDeltaFiles(CarbonTable table, boolean isDryRun)
throws IOException {
+
+SegmentStatusManager ssm = new
SegmentStatusManager(table.getAbsoluteTableIdentifier());
+LoadMetadataDetails[] details =
+SegmentStatusManager.readLoadMetadata(table.getMetadataPath());
+long totalSizeDeleted = 0;
+ArrayList filesToBeDeleted = new ArrayList<>();
+SegmentUpdateStatusManager updateStatusManager = new
SegmentUpdateStatusManager(table);
+SegmentUpdateDetails[] segmentUpdateDetails =
updateStatusManager.getUpdateStatusDetails();
+// hold all the segments updated so that wen can check the delta files in
them, ne need to
+// check the others.
+Set updatedSegments = new HashSet<>();
+for (SegmentUpdateDetails updateDetails : segmentUpdateDetails) {
+ updatedSegments.add(updateDetails.getSegmentName());
+}
+boolean isInvalidFile = false;
+// take the update status file name from 0th segment.
+String validUpdateStatusFile = ssm.getUpdateStatusFileName(details);
+// scan through each segment.
+
+if (table.isHivePartitionTable()) {
+ List partitionList =
Arrays.stream(FileFactory.getCarbonFile(table
+ .getTablePath()).listFiles()).filter(partitionName ->
partitionName.getName()
+ .contains("=")).collect(Collectors.toList());
+
+ List totalDeltaFiles = new ArrayList<>();
+
+ for (CarbonFile carbonFile : partitionList) {
+
totalDeltaFiles.addAll(carbonFile.listFiles(true).stream().filter(fileName ->
fileName
+
.getName().endsWith(CarbonCommonConstants.DELETE_DELTA_FILE_EXT))
+.collect(Collectors.toList()));
+ }
+ SegmentUpdateDetails[] updateDetails =
updateStatusManager.readLoadMetadata();
+ for (SegmentUpdateDetails block : updateDetails) {
+totalDeltaFiles.stream().filter(fileName ->
fileName.getName().endsWith(block
+.getDeleteDeltaStartTimestamp() +
CarbonCommonConstants.DELETE_DELTA_FILE_EXT))
+.collect(Collectors.toList()).forEach(fileName ->
totalDeltaFiles.remove(fileName));
+ }
+ for (CarbonFile invalidFile: totalDeltaFiles) {
+totalSizeDeleted += invalidFile.getSize();
+
[carbondata] branch carbon3.0 created (now 0f1d2a4)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to branch carbon3.0 in repository https://gitbox.apache.org/repos/asf/carbondata.git. at 0f1d2a4 [CARBONDATA-4317] Fix TPCDS performance issues No new revisions were added by this update.
[carbondata] branch master updated: [CARBONDATA-4317] Fix TPCDS performance issues
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 0f1d2a4 [CARBONDATA-4317] Fix TPCDS performance issues
0f1d2a4 is described below
commit 0f1d2a45e5f614fd123bd734ab37d7e453c21344
Author: Indhumathi27
AuthorDate: Tue Dec 7 20:32:05 2021 +0530
[CARBONDATA-4317] Fix TPCDS performance issues
Why is this PR needed?
The following issues has degraded the TPCDS query performance
1. If dynamic filters is not present in partitionFilters Set, then that
filter is skipped, to pushdown to spark.
2. In some cases, some nodes like Exchange / Shuffle is not reused, because
the CarbonDataSourceSCan plan is not mached
3. While accessing the metadata on the canonicalized plan throws NPE
What changes were proposed in this PR?
1. Check if dynamic filters is present in PartitionFilters set. If not,
pushdown the filter
2. Match the plans, by converting them to canonicalized and by normalising
the expressions
3. Move variables used in metadata(), to avoid NPE while comparing plans
This closes #4241
---
.../execution/strategy/CarbonDataSourceScan.scala | 49 ++
.../execution/strategy/CarbonSourceStrategy.scala | 6 +--
.../apache/spark/sql/CarbonToSparkAdapter.scala| 4 +-
.../apache/spark/sql/CarbonToSparkAdapter.scala| 4 +-
.../apache/spark/sql/CarbonToSparkAdapter.scala| 7 +++-
5 files changed, 54 insertions(+), 16 deletions(-)
diff --git
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonDataSourceScan.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonDataSourceScan.scala
index 2e1bb96..31685b0 100644
---
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonDataSourceScan.scala
+++
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonDataSourceScan.scala
@@ -31,7 +31,6 @@ import
org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partition
import org.apache.spark.sql.execution.WholeStageCodegenExec
import org.apache.carbondata.core.metadata.schema.BucketingInfo
-import org.apache.carbondata.core.readcommitter.ReadCommittedScope
import org.apache.carbondata.core.scan.expression.Expression
import org.apache.carbondata.hadoop.CarbonProjection
@@ -44,7 +43,6 @@ case class CarbonDataSourceScan(
output: Seq[Attribute],
partitionFiltersWithoutDpp: Seq[SparkExpression],
dataFilters: Seq[SparkExpression],
-@transient readCommittedScope: ReadCommittedScope,
@transient pushedDownProjection: CarbonProjection,
@transient pushedDownFilters: Seq[Expression],
directScanSupport: Boolean,
@@ -64,6 +62,10 @@ case class CarbonDataSourceScan(
partitionFiltersWithDpp,
segmentIds) {
+ val pushDownFiltersStr: String =
seqToString(pushedDownFilters.map(_.getStatement))
+
+ val projectionColStr: String =
seqToString(pushedDownProjection.getAllColumns)
+
override lazy val (outputPartitioning, outputOrdering): (Partitioning,
Seq[SortOrder]) = {
val info: BucketingInfo = relation.carbonTable.getBucketingInfo
if (info != null) {
@@ -91,15 +93,18 @@ case class CarbonDataSourceScan(
}
}
+ def seqToString(seq: Seq[Any]): String = seq.mkString("[", ", ", "]")
+
override lazy val metadata: Map[String, String] = {
-def seqToString(seq: Seq[Any]) = seq.mkString("[", ", ", "]")
val metadata =
Map(
-"ReadSchema" -> seqToString(pushedDownProjection.getAllColumns),
+"ReadSchema" -> projectionColStr,
"Batched" -> supportsBatchOrColumnar.toString,
"DirectScan" -> (supportsBatchOrColumnar &&
directScanSupport).toString,
-"PushedFilters" -> seqToString(pushedDownFilters.map(_.getStatement)))
-if (relation.carbonTable.isHivePartitionTable) {
+"PushedFilters" -> pushDownFiltersStr)
+// if plan is canonicalized, then filter expressions will be normalized.
In that case,
+// skip adding selected partitions to metadata
+if (!this.isCanonicalizedPlan &&
relation.carbonTable.isHivePartitionTable) {
metadata + ("PartitionFilters" -> seqToString(partitionFiltersWithDpp)) +
("PartitionCount" -> selectedPartitions.size.toString)
} else {
@@ -142,14 +147,40 @@ case class CarbonDataSourceScan(
outputAttibutesAfterNormalizingExpressionIds,
QueryPlan.normalizePredicates(partitionFiltersWithoutDpp, output),
QueryPlan.normalizePredicates(dataFilters, output),
- null,
- null,
+ pushedDownProjection,
Seq.empty,
[carbondata] branch master updated: [CARBONDATA-4316]Fix horizontal compaction failure for partition tables
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new d629dc0 [CARBONDATA-4316]Fix horizontal compaction failure for
partition tables
d629dc0 is described below
commit d629dc0b894a64bfbef762736775a182e40827fe
Author: akashrn5
AuthorDate: Tue Dec 7 18:55:13 2021 +0530
[CARBONDATA-4316]Fix horizontal compaction failure for partition tables
Why is this PR needed?
Horizontal compaction fails for partition table leading to many delete
delta files for a single block, leading to slower query performance.
This is happening because during horizontal compaction the delta file
path prepared for the partition table is wrong which fails to identify
the path and fails the operation.
What changes were proposed in this PR?
If it is a partition table, read the segment file and identity the
partition where the block is present to prepare a proper partition path.
This closes #4240
---
.../statusmanager/SegmentUpdateStatusManager.java | 24 ++-
.../mutation/CarbonProjectForDeleteCommand.scala | 9 --
.../command/mutation/HorizontalCompaction.scala| 4 +--
.../testsuite/iud/DeleteCarbonTableTestCase.scala | 34 ++
.../processing/merger/CarbonDataMergerUtil.java| 9 +++---
5 files changed, 69 insertions(+), 11 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index fe40494..8572ae5 100644
---
a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++
b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.core.statusmanager;
import java.io.*;
import java.util.*;
+import java.util.stream.Collectors;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -33,6 +34,7 @@ import org.apache.carbondata.core.locks.CarbonLockFactory;
import org.apache.carbondata.core.locks.ICarbonLock;
import org.apache.carbondata.core.locks.LockUsage;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.mutate.SegmentUpdateDetails;
@@ -364,7 +366,8 @@ public class SegmentUpdateStatusManager {
* @param blockName the specified block of the segment
* @return delete delta file list of the block
*/
- public List getDeleteDeltaFilesList(final Segment segment, final
String blockName) {
+ public List getDeleteDeltaFilesList(final Segment segment, final
String blockName)
+ throws IOException {
List deleteDeltaFileList = new ArrayList<>();
String segmentPath = null;
if (segment.isExternalSegment()) {
@@ -374,6 +377,25 @@ public class SegmentUpdateStatusManager {
break;
}
}
+} else if (isPartitionTable) {
+ String segmentFileName = Arrays.stream(segmentDetails).filter(
+ loadMetaDataDetail -> loadMetaDataDetail.getLoadName()
+
.equalsIgnoreCase(segment.getSegmentNo())).collect(Collectors.toList()).get(0)
+ .getSegmentFile();
+ SegmentFileStore segmentFileStore =
+ new SegmentFileStore(identifier.getTablePath(), segmentFileName);
+ segmentFileStore.readIndexFiles(SegmentStatus.SUCCESS, false,
FileFactory.getConfiguration());
+ for (Map.Entry> entry :
segmentFileStore.getIndexFilesMap().entrySet()) {
+List matchedBlocksInPartition =
entry.getValue().stream().filter(blockFile -> {
+ String blockFileName =
blockFile.substring(blockFile.lastIndexOf(File.separator) + 1);
+ return
blockName.equalsIgnoreCase(CarbonUpdateUtil.getBlockName(blockFileName));
+}).collect(Collectors.toList());
+if (matchedBlocksInPartition.size() > 0) {
+ segmentPath = matchedBlocksInPartition.get(0)
+ .substring(0,
matchedBlocksInPartition.get(0).lastIndexOf(File.separator));
+ break;
+}
+ }
} else {
segmentPath = CarbonTablePath.getSegmentPath(
identifier.getTablePath(), segment.getSegmentNo());
diff --git
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForDeleteCommand.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForDeleteCommand.scala
index 87b1e41..7d54186 100644
---
a/integration/spark/
svn commit: r51639 - in /dev/carbondata/2.3.0-rc1: ./ apache-carbondata-2.3.0-source-release.zip apache-carbondata-2.3.0-source-release.zip.asc apache-carbondata-2.3.0-source-release.zip.md5 apache-ca
Author: kunalkapoor Date: Mon Dec 20 13:36:25 2021 New Revision: 51639 Log: Upload 2.3.0 rc1 Added: dev/carbondata/2.3.0-rc1/ dev/carbondata/2.3.0-rc1/apache-carbondata-2.3.0-source-release.zip (with props) dev/carbondata/2.3.0-rc1/apache-carbondata-2.3.0-source-release.zip.asc dev/carbondata/2.3.0-rc1/apache-carbondata-2.3.0-source-release.zip.md5 dev/carbondata/2.3.0-rc1/apache-carbondata-2.3.0-source-release.zip.sha512 Added: dev/carbondata/2.3.0-rc1/apache-carbondata-2.3.0-source-release.zip == Binary file - no diff available. Propchange: dev/carbondata/2.3.0-rc1/apache-carbondata-2.3.0-source-release.zip -- svn:mime-type = application/octet-stream Added: dev/carbondata/2.3.0-rc1/apache-carbondata-2.3.0-source-release.zip.asc == --- dev/carbondata/2.3.0-rc1/apache-carbondata-2.3.0-source-release.zip.asc (added) +++ dev/carbondata/2.3.0-rc1/apache-carbondata-2.3.0-source-release.zip.asc Mon Dec 20 13:36:25 2021 @@ -0,0 +1,14 @@ +-BEGIN PGP SIGNATURE- + +iQGzBAABCgAdFiEER4oGoEjiZ19CtKBtzP3MO+Fg6doFAmHAQJAACgkQzP3MO+Fg +6drCRQv+Prlt6h4D9j7yDEnrbYtY1ZCEp+MeVvcPV4ovhsNLD8W79V9+W23e6gXD +mwfwoDA8nJEyhtE4v3ku6OAMaQQK5FXn2e072qn2+XwPt8yq39vejIs/U4/72y9W +2/z78076ThsUeO1CbnzCB5fy4A1RTFE6kKW6SWEXkjz6waG+Yrgze3Mti+z0Jre9 +TYk+xMQYZ4hGJ3ed62kv1ZDn/umOfN+fXxsAXlOpZi/Zb3K0QcskWFTjaC4as/gp +Ddn3GNKoN1h84Scg5B/INrqobGZnpjZ9XNl/ZPx82mV7PPUYVwqL/FifNA7Zypse +5XO/Jilja84VtYABDwV19F6xToOpX85CxRchUn2FF/XObjE7UKaJvNgXCBF22bem +77aL3vCCycQMdRKWMp0wQU035WXtyWt12B8FzvFxKosI5yz1JjyiNhgCTc1pAIOU ++/T373fInbax15mIZbcNVOBRZvZFFu+ysPWcTnbNfFvD4ElM0/m8+v2XB3f56D56 +6hLbuHdo +=Ohii +-END PGP SIGNATURE- Added: dev/carbondata/2.3.0-rc1/apache-carbondata-2.3.0-source-release.zip.md5 == --- dev/carbondata/2.3.0-rc1/apache-carbondata-2.3.0-source-release.zip.md5 (added) +++ dev/carbondata/2.3.0-rc1/apache-carbondata-2.3.0-source-release.zip.md5 Mon Dec 20 13:36:25 2021 @@ -0,0 +1 @@ +8fd9a7e0246da32e73f8b936f7d88633 apache-carbondata-2.3.0-source-release.zip Added: dev/carbondata/2.3.0-rc1/apache-carbondata-2.3.0-source-release.zip.sha512 == --- dev/carbondata/2.3.0-rc1/apache-carbondata-2.3.0-source-release.zip.sha512 (added) +++ dev/carbondata/2.3.0-rc1/apache-carbondata-2.3.0-source-release.zip.sha512 Mon Dec 20 13:36:25 2021 @@ -0,0 +1 @@ +7bd191c5a070c5380af89cef71052fbf9d4b161080fd92443b3457fe5890d053c99ad47dbe2d0002a39af5906f3f4aad3405fb9304817916c0ea40ce35e15003 apache-carbondata-2.3.0-source-release.zip
[carbondata] branch branch-2.3 created (now 7006589)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/carbondata.git. at 7006589 [maven-release-plugin] prepare for next development iteration This branch includes the following new commits: new 7006589 [maven-release-plugin] prepare for next development iteration The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[carbondata] 01/01: [maven-release-plugin] prepare for next development iteration
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/carbondata.git commit 70065894d02ce2e898b1ed3cd7b0b10f6305db44 Author: kunal642 AuthorDate: Mon Dec 20 18:04:52 2021 +0530 [maven-release-plugin] prepare for next development iteration --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 1107569..d8d03b7 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 88dc1b8..6205781 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 0e74e7a..a5f2e75 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index da34787..360fcff 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index ca666db..64f0d9f 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index a8c5b60..e992b2e 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index 83bbbd9..46f6695 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index 298337b..c38b0f9 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index 1096951..7ec45b4 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 3522097..bccbae1 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index 88d8afb..e23b94d 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index f4ce62e..b1a2414 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index ba37c2e..a4b320e 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/integration/flink-proxy/pom.xml b/integration/flink-proxy/pom.xml index 832d804..b7aee50 100644 --- a/integration/flink-proxy/pom.xml +++ b/integration/flink-proxy
[carbondata] annotated tag apache-carbondata-2.3.0-rc1 created (now 6bb1227)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to annotated tag apache-carbondata-2.3.0-rc1 in repository https://gitbox.apache.org/repos/asf/carbondata.git. at 6bb1227 (tag) tagging 7fe1824733131785f78093567b34b6f94f1dc25a (commit) by kunal642 on Mon Dec 20 18:04:23 2021 +0530 - Log - [maven-release-plugin] copy for tag apache-carbondata-2.3.0-rc1 --- This annotated tag includes the following new commits: new 7fe1824 [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc1 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[carbondata] 01/01: [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc1
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to annotated tag apache-carbondata-2.3.0-rc1 in repository https://gitbox.apache.org/repos/asf/carbondata.git commit 7fe1824733131785f78093567b34b6f94f1dc25a Author: kunal642 AuthorDate: Mon Dec 20 17:59:41 2021 +0530 [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc1 --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index d4c7aba..1107569 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 68fea9c..88dc1b8 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index f4d2c16..0e74e7a 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index cd4b691..da34787 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index 0f804f4..ca666db 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index c547d37..a8c5b60 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index fcbd80c..83bbbd9 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index e8be41e..298337b 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index c4071ba..1096951 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 3873996..3522097 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index b10583d..88d8afb 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index b998143..f4ce62e 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index c443497..ba37c2e 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/integration/flink-proxy/pom.xml b/integration/flink-proxy/pom.xml index 23f5169..832d804 100644 --- a/integration/flink-proxy/pom.xml
[carbondata] branch master updated: [maven-release-plugin] Reverted the pom changes to 2.2.0-SNAPSHOT
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git The following commit(s) were added to refs/heads/master by this push: new 0ced3c8 [maven-release-plugin] Reverted the pom changes to 2.2.0-SNAPSHOT 0ced3c8 is described below commit 0ced3c864257d92780409088c77bba0a1652f1cb Author: kunal642 AuthorDate: Mon Dec 20 16:05:26 2021 +0530 [maven-release-plugin] Reverted the pom changes to 2.2.0-SNAPSHOT --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 2 +- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 24 insertions(+), 24 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index d8d03b7..d4c7aba 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 6205781..68fea9c 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index a5f2e75..f4d2c16 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index 360fcff..cd4b691 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index 64f0d9f..0f804f4 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index e992b2e..c547d37 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index 46f6695..fcbd80c 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index c38b0f9..e8be41e 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index 7ec45b4..c4071ba 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index bccbae1..3873996 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index e23b94d..b10583d 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index b1a2414..b998143 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index a4b320e..c443497 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7
[carbondata] annotated tag apache-carbondata-2.3.0-rc1 deleted (was 189252b)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to annotated tag apache-carbondata-2.3.0-rc1 in repository https://gitbox.apache.org/repos/asf/carbondata.git. *** WARNING: tag apache-carbondata-2.3.0-rc1 was deleted! *** tag was 189252b The revisions that were on this annotated tag are still contained in other references; therefore, this change does not discard any commits from the repository.
[carbondata] branch master updated: [maven-release-plugin] prepare for next development iteration
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git The following commit(s) were added to refs/heads/master by this push: new c0211fc [maven-release-plugin] prepare for next development iteration c0211fc is described below commit c0211fcff16a79b8e9b4ce73196125526dfbc10b Author: kunal642 AuthorDate: Mon Dec 20 16:25:30 2021 +0530 [maven-release-plugin] prepare for next development iteration --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 1107569..d8d03b7 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 88dc1b8..6205781 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 0e74e7a..a5f2e75 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index da34787..360fcff 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index ca666db..64f0d9f 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index a8c5b60..e992b2e 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index 83bbbd9..46f6695 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index 298337b..c38b0f9 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index 1096951..7ec45b4 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 3522097..bccbae1 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index 88d8afb..e23b94d 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index f4ce62e..b1a2414 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index ba37c2e..a4b320e 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml
[carbondata] annotated tag apache-carbondata-2.3.0-rc1 created (now 189252b)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to annotated tag apache-carbondata-2.3.0-rc1 in repository https://gitbox.apache.org/repos/asf/carbondata.git. at 189252b (tag) tagging 053d080cf54ec84be5a547eb62c948300b55a929 (commit) by kunal642 on Mon Dec 20 16:25:16 2021 +0530 - Log - [maven-release-plugin] copy for tag apache-carbondata-2.3.0-rc1 --- No new revisions were added by this update.
[carbondata] annotated tag apache-carbondata-2.3.0-rc1 deleted (was 9038617)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to annotated tag apache-carbondata-2.3.0-rc1 in repository https://gitbox.apache.org/repos/asf/carbondata.git. *** WARNING: tag apache-carbondata-2.3.0-rc1 was deleted! *** tag was 9038617 The revisions that were on this annotated tag are still contained in other references; therefore, this change does not discard any commits from the repository.
[carbondata] branch master updated: [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc1
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git The following commit(s) were added to refs/heads/master by this push: new 053d080 [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc1 053d080 is described below commit 053d080cf54ec84be5a547eb62c948300b55a929 Author: kunal642 AuthorDate: Mon Dec 20 16:20:15 2021 +0530 [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc1 --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index d4c7aba..1107569 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 68fea9c..88dc1b8 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index f4d2c16..0e74e7a 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index cd4b691..da34787 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index 0f804f4..ca666db 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index c547d37..a8c5b60 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index fcbd80c..83bbbd9 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index e8be41e..298337b 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index c4071ba..1096951 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 3873996..3522097 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index b10583d..88d8afb 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index b998143..f4ce62e 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index c443497..ba37c2e 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml
[carbondata] branch master updated: [maven-release-plugin] Reverted the pom changes to 2.2.0-SNAPSHOT
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git The following commit(s) were added to refs/heads/master by this push: new 01f8e1a [maven-release-plugin] Reverted the pom changes to 2.2.0-SNAPSHOT 01f8e1a is described below commit 01f8e1afb1bf5e25202b1fe7f264a9abaf0ceb75 Author: kunal642 AuthorDate: Mon Dec 20 16:05:26 2021 +0530 [maven-release-plugin] Reverted the pom changes to 2.2.0-SNAPSHOT --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 2 +- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 24 insertions(+), 24 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index d8d03b7..d4c7aba 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 6205781..68fea9c 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index a5f2e75..f4d2c16 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index 360fcff..cd4b691 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index 64f0d9f..0f804f4 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index e992b2e..c547d37 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index 46f6695..fcbd80c 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index c38b0f9..e8be41e 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index 7ec45b4..c4071ba 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index bccbae1..3873996 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index e23b94d..b10583d 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index b1a2414..b998143 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.1-SNAPSHOT +2.3.0-SNAPSHOT ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index a4b320e..c443497 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7
[carbondata] 01/01: [maven-release-plugin] prepare for next development iteration
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git commit 861fc679bb5c3176da9c8c2677cbb342b540e819 Author: kunal642 AuthorDate: Mon Dec 20 15:15:19 2021 +0530 [maven-release-plugin] prepare for next development iteration --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 1107569..d8d03b7 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 88dc1b8..6205781 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 0e74e7a..a5f2e75 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index da34787..360fcff 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index ca666db..64f0d9f 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index a8c5b60..e992b2e 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index 83bbbd9..46f6695 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index 298337b..c38b0f9 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index 1096951..7ec45b4 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 3522097..bccbae1 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index 88d8afb..e23b94d 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index f4ce62e..b1a2414 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index ba37c2e..a4b320e 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/integration/flink-proxy/pom.xml b/integration/flink-proxy/pom.xml index 832d804..b7aee50 100644 --- a/integration/flink-proxy/pom.xml +++ b/integration/flink-proxy
[carbondata] branch master updated (18840af -> 861fc67)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git. from 18840af [CARBONDATA-4305] Support Carbondata Streamer tool for incremental fetch and merge from kafka and DFS Sources add fc3914f [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc1 new 861fc67 [maven-release-plugin] prepare for next development iteration The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 2 +- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 24 insertions(+), 24 deletions(-)
[carbondata] annotated tag apache-carbondata-2.3.0-rc1 created (now 9038617)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to annotated tag apache-carbondata-2.3.0-rc1 in repository https://gitbox.apache.org/repos/asf/carbondata.git. at 9038617 (tag) tagging fc3914f3646177e1f0bccb6688be4688b5aaffd9 (commit) by kunal642 on Mon Dec 20 15:15:03 2021 +0530 - Log - [maven-release-plugin] copy for tag apache-carbondata-2.3.0-rc1 --- This annotated tag includes the following new commits: new fc3914f [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc1 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[carbondata] 01/01: [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc1
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to annotated tag apache-carbondata-2.3.0-rc1 in repository https://gitbox.apache.org/repos/asf/carbondata.git commit fc3914f3646177e1f0bccb6688be4688b5aaffd9 Author: kunal642 AuthorDate: Mon Dec 20 15:11:10 2021 +0530 [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc1 --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index d4c7aba..1107569 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 68fea9c..88dc1b8 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index f4d2c16..0e74e7a 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index cd4b691..da34787 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index 0f804f4..ca666db 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index c547d37..a8c5b60 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index fcbd80c..83bbbd9 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index e8be41e..298337b 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index c4071ba..1096951 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 3873996..3522097 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index b10583d..88d8afb 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index b998143..f4ce62e 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index c443497..ba37c2e 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/integration/flink-proxy/pom.xml b/integration/flink-proxy/pom.xml index 23f5169..832d804 100644 --- a/integration/flink-proxy/pom.xml
[carbondata] annotated tag apache-carbondata-2.3.0-rc1 deleted (was a4a61a8)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to annotated tag apache-carbondata-2.3.0-rc1 in repository https://gitbox.apache.org/repos/asf/carbondata.git. *** WARNING: tag apache-carbondata-2.3.0-rc1 was deleted! *** tag was a4a61a8 This change permanently discards the following revisions: discard 232cbaa [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc1
[carbondata] branch branch-2.3 deleted (was b572b91)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/carbondata.git. was b572b91 [maven-release-plugin] prepare for next development iteration This change permanently discards the following revisions: discard b572b91 [maven-release-plugin] prepare for next development iteration
[carbondata] 01/01: [maven-release-plugin] prepare for next development iteration
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/carbondata.git commit b572b919ef6eef30e5ca367deb714cdd66438721 Author: kunal642 AuthorDate: Mon Dec 20 14:17:28 2021 +0530 [maven-release-plugin] prepare for next development iteration --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 1107569..d8d03b7 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 88dc1b8..6205781 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 0e74e7a..a5f2e75 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index da34787..360fcff 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index ca666db..64f0d9f 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index a8c5b60..e992b2e 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index 83bbbd9..46f6695 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index 298337b..c38b0f9 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index 1096951..7ec45b4 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 3522097..bccbae1 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index 88d8afb..e23b94d 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index f4ce62e..b1a2414 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index ba37c2e..a4b320e 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0 +2.3.1-SNAPSHOT ../../pom.xml diff --git a/integration/flink-proxy/pom.xml b/integration/flink-proxy/pom.xml index 832d804..b7aee50 100644 --- a/integration/flink-proxy/pom.xml +++ b/integration/flink-proxy
[carbondata] branch branch-2.3 created (now b572b91)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/carbondata.git. at b572b91 [maven-release-plugin] prepare for next development iteration This branch includes the following new commits: new b572b91 [maven-release-plugin] prepare for next development iteration The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[carbondata] 01/01: [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc1
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to annotated tag apache-carbondata-2.3.0-rc1 in repository https://gitbox.apache.org/repos/asf/carbondata.git commit 232cbaae2b3d50450478e458b580195e9f830a9c Author: kunal642 AuthorDate: Mon Dec 20 14:15:36 2021 +0530 [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc1 --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index d4c7aba..1107569 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index 68fea9c..88dc1b8 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index f4d2c16..0e74e7a 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index cd4b691..da34787 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index 0f804f4..ca666db 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index c547d37..a8c5b60 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index fcbd80c..83bbbd9 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index e8be41e..298337b 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index c4071ba..1096951 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 3873996..3522097 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index b10583d..88d8afb 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index b998143..f4ce62e 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml index c443497..ba37c2e 100644 --- a/integration/flink-build/pom.xml +++ b/integration/flink-build/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.3.0-SNAPSHOT +2.3.0 ../../pom.xml diff --git a/integration/flink-proxy/pom.xml b/integration/flink-proxy/pom.xml index 23f5169..832d804 100644 --- a/integration/flink-proxy/pom.xml
[carbondata] annotated tag apache-carbondata-2.3.0-rc1 created (now a4a61a8)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to annotated tag apache-carbondata-2.3.0-rc1 in repository https://gitbox.apache.org/repos/asf/carbondata.git. at a4a61a8 (tag) tagging 232cbaae2b3d50450478e458b580195e9f830a9c (commit) by kunal642 on Mon Dec 20 14:17:05 2021 +0530 - Log - [maven-release-plugin] copy for tag apache-carbondata-2.3.0-rc1 --- This annotated tag includes the following new commits: new 232cbaa [maven-release-plugin] prepare release apache-carbondata-2.3.0-rc1 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[carbondata] branch master updated: [CARBONDATA-4305] Support Carbondata Streamer tool for incremental fetch and merge from kafka and DFS Sources
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 18840af [CARBONDATA-4305] Support Carbondata Streamer tool for
incremental fetch and merge from kafka and DFS Sources
18840af is described below
commit 18840af9c1f7154b58e3c397dfc5a4440674bcee
Author: akashrn5
AuthorDate: Wed Sep 1 19:24:24 2021 +0530
[CARBONDATA-4305] Support Carbondata Streamer tool for incremental fetch
and merge from kafka and DFS Sources
Why is this PR needed?
In the current Carbondata CDC solution, if any user wants to integrate it
with a streaming source then he
need to write a separate spark application to capture changes which is an
overhead. We should be able to
incrementally capture the data changes from primary databases and should be
able to incrementally ingest
the same in the data lake so that the overall latency decreases. The former
is taken care of using
log-based CDC systems like Maxwell and Debezium. Here is a solution for the
second aspect using Apache Carbondata.
What changes were proposed in this PR?
Carbondata streamer tool is a spark streaming application which enables
users to incrementally ingest data
from various sources, like Kafka(standard pipeline would be like MYSQL =>
debezium => (kafka + Schema registry) => Carbondata Streamer tool)
and DFS into their data lakes. The tool comes with out-of-the-box support
for almost all types of schema
evolution use cases. With the streamer tool only add column support is
given with drop column and
other schema changes capability in line in the upcoming days. Please refer
to design document for
more details about usage and working of the tool.
This closes #4235
---
.../core/constants/CarbonCommonConstants.java | 162 +++-
.../carbondata/core/util/CarbonProperties.java | 4 +
integration/spark/pom.xml | 47
.../apache/carbondata/streamer/AvroDFSSource.scala | 58 +
.../carbondata/streamer/AvroKafkaSource.scala | 64 +
.../apache/carbondata/streamer/CarbonDStream.scala | 56
.../carbondata/streamer/CarbonDataStreamer.scala | 135 ++
.../streamer/CarbonDataStreamerException.scala | 33 +++
.../carbondata/streamer/CarbonStreamerConfig.scala | 282 +
.../apache/carbondata/streamer/SchemaSource.scala | 106
.../org/apache/carbondata/streamer/Source.scala| 235 +
.../apache/carbondata/streamer/SourceFactory.scala | 54
.../mutation/merge/CarbonMergeDataSetCommand.scala | 5 +
.../spark/testsuite/merge/MergeTestCase.scala | 68 +
pom.xml| 4 +
15 files changed, 1312 insertions(+), 1 deletion(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index f24108a..919fc90 100644
---
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -2684,6 +2684,150 @@ public final class CarbonCommonConstants {
//
// CDC streamer configs start here
//
+ /**
+ * The database name where the target table is present to merge the incoming
data. If not given by
+ * user, system will take the current database in the spark session.
+ */
+ @CarbonProperty
+ public static final String CARBON_STREAMER_DATABASE_NAME =
"carbon.streamer.target.database";
+
+ /**
+ * The target carbondata table where the data has to be merged. If this is
not configured by user,
+ * the operation will fail.
+ */
+ @CarbonProperty
+ public static final String CARBON_STREAMER_TABLE_NAME =
"carbon.streamer.target.table";
+
+ /**
+ * Source type to ingest data from. It can be kafka or DFS
+ */
+ @CarbonProperty
+ public static final String CARBON_STREAMER_SOURCE_TYPE =
"carbon.streamer.source.type";
+
+ public static final String CARBON_STREAMER_SOURCE_TYPE_DEFAULT = "kafka";
+
+ /**
+ * An absolute path on a given file system from where data needs to be read
to ingest into the
+ * target carbondata table. Mandatory if the ingestion source type is DFS.
+ */
+ @CarbonProperty
+ public static final String CARBON_STREAMER_DFS_INPUT_PATH =
"carbon.streamer.dfs.input.path";
+
+ /**
+ * Schema registry url in case schema registry is selected as schema
provider.
+ */
+ @CarbonProperty
+ pub
[carbondata] branch master updated: [CARBONDATA-4296]: schema evolution, enforcement and deduplication utilities added
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 3be05d2 [CARBONDATA-4296]: schema evolution, enforcement and
deduplication utilities added
3be05d2 is described below
commit 3be05d2a44d805cf763df05cbeacce2d90a44da0
Author: pratyakshsharma
AuthorDate: Wed Oct 27 13:54:37 2021 +0530
[CARBONDATA-4296]: schema evolution, enforcement and deduplication
utilities added
Why is this PR needed?
This PR adds schema enforcement, schema evolution and deduplication
capabilities for
carbondata streamer tool specifically. For the existing IUD scenarios, some
work
needs to be done to handle it completely, for example -
1. passing default values and storing them in table properties.
Changes proposed for the phase 2 -
1. Handling delete use cases with upsert operation/command itself. Right
now we
consider update as delete + insert. With the new streamer tool, it is
possible that
user sets upsert as the operation type and incoming stream has delete
records as well.
What changes were proposed in this PR?
Configs and utility methods are added for the following use cases -
1. Schema enforcement
2. Schema evolution - add column, delete column, data type change scenario
3. Deduplicate the incoming dataset against incoming dataset itself. This
is useful
in scenarios where incoming stream of data has multiple updates for the
same record
and we want to pick the latest.
4. Deduplicate the incoming dataset against existing target dataset. This
is useful
when operation type is set as INSERT and user does not want to insert
duplicate records.
This closes #4227
---
.../exceptions/sql/CarbonSchemaException.java | 39 +++
.../core/constants/CarbonCommonConstants.java | 54 +++
.../mutation/merge/CarbonMergeDataSetCommand.scala | 80 -
.../mutation/merge/CarbonMergeDataSetUtil.scala| 390 -
.../spark/sql/execution/strategy/DDLHelper.scala | 97 +++--
.../spark/testsuite/merge/MergeTestCase.scala | 264 +++---
6 files changed, 824 insertions(+), 100 deletions(-)
diff --git
a/common/src/main/java/org/apache/carbondata/common/exceptions/sql/CarbonSchemaException.java
b/common/src/main/java/org/apache/carbondata/common/exceptions/sql/CarbonSchemaException.java
new file mode 100644
index 000..522f132
--- /dev/null
+++
b/common/src/main/java/org/apache/carbondata/common/exceptions/sql/CarbonSchemaException.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.common.exceptions.sql;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+@InterfaceAudience.User
+@InterfaceStability.Stable
+public class CarbonSchemaException extends Exception {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String message;
+
+ public CarbonSchemaException(String message) {
+super(message);
+this.message = message;
+ }
+
+ public String getMessage() {
+return this.message;
+ }
+}
diff --git
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index d72d6c1..f24108a 100644
---
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -2681,4 +2681,58 @@ public final class CarbonCommonConstants {
public static final String CARBON_CDC_MINMAX_PRUNING_ENABLED_DEFAULT =
"false";
+
//
+ // CDC streamer configs start here
+
//
+
+ /**
+ * Name of the field from source schema whose value can be used f
[carbondata] branch master updated: [CARBONDATA-4306] Fix Query Performance issue for Spark 3.1
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 8953cde [CARBONDATA-4306] Fix Query Performance issue for Spark 3.1
8953cde is described below
commit 8953cde6ad93ddd3622c60cd48117acfbbee56a2
Author: Indhumathi27
AuthorDate: Thu Sep 30 14:41:35 2021 +0530
[CARBONDATA-4306] Fix Query Performance issue for Spark 3.1
Why is this PR needed?
Currently, with Spark 3.1, some rules are applied many times resulting in
performance degrade.
What changes were proposed in this PR?
Changed Rules apply strategy from Fixed to Once and CarbonOptimizer can
directly extend SparkOptimizer avoiding applying same rules many times
This Closes #4229
---
.../execution/strategy/CarbonSourceStrategy.scala | 2 +-
.../apache/spark/sql/CarbonToSparkAdapter.scala| 23 ++
2 files changed, 7 insertions(+), 18 deletions(-)
diff --git
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala
index 009d8c7..8a0779f 100644
---
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala
+++
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala
@@ -152,7 +152,7 @@ private[sql] object CarbonSourceStrategy extends
SparkStrategy {
var partitions : (Seq[CatalogTablePartition], Seq[PartitionSpec],
Seq[Expression]) =
(null, null, Seq.empty)
var filterPredicates = allPredicates
-if(names.nonEmpty) {
+if(names.nonEmpty && partitionsFilter.nonEmpty) {
partitions = CarbonFilters.getCatalogTablePartitions(
partitionsFilter.filterNot(e =>
e.find(_.isInstanceOf[PlanExpression[_]]).isDefined),
SparkSession.getActiveSession.get,
diff --git
a/integration/spark/src/main/spark3.1/org/apache/spark/sql/CarbonToSparkAdapter.scala
b/integration/spark/src/main/spark3.1/org/apache/spark/sql/CarbonToSparkAdapter.scala
index 25a27ad..eaceb85 100644
---
a/integration/spark/src/main/spark3.1/org/apache/spark/sql/CarbonToSparkAdapter.scala
+++
b/integration/spark/src/main/spark3.1/org/apache/spark/sql/CarbonToSparkAdapter.scala
@@ -33,6 +33,7 @@ import
org.apache.spark.sql.catalyst.expressions.codegen.Block._
import org.apache.spark.sql.catalyst.optimizer.Optimizer
import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias}
import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.execution.SparkOptimizer
import org.apache.spark.sql.execution.datasources.{FilePartition,
PartitionedFile}
import org.apache.spark.sql.hive.HiveExternalCatalog
import org.apache.spark.sql.optimizer.{CarbonIUDRule, CarbonUDFTransformRule,
MVRewriteRule}
@@ -238,33 +239,21 @@ object CarbonToSparkAdapter extends SparkVersionAdapter {
}
class CarbonOptimizer(session: SparkSession, optimizer: Optimizer) extends
- Optimizer(session.sessionState.catalogManager) {
+ SparkOptimizer(session.sessionState.catalogManager,
+session.sessionState.catalog,
+session.sessionState.experimentalMethods) {
private lazy val mvRules = Seq(Batch("Materialized View Optimizers", Once,
Seq(new MVRewriteRule(session)): _*))
- private lazy val iudRule = Batch("IUD Optimizers", fixedPoint,
+ private lazy val iudRule = Batch("IUD Optimizers", Once,
Seq(new CarbonIUDRule(), new CarbonUDFTransformRule(), new
CarbonFileIndexReplaceRule()): _*)
private lazy val secondaryIndexRule = Batch("SI Optimizers", Once,
Seq(new CarbonSITransformationRule(session)): _*)
override def defaultBatches: Seq[Batch] = {
-mvRules ++ convertedBatch() :+ iudRule :+ secondaryIndexRule
+mvRules ++ super.defaultBatches :+ iudRule :+ secondaryIndexRule
}
- def convertedBatch(): Seq[Batch] = {
-optimizer.batches.map { batch =>
- Batch(
-batch.name,
-batch.strategy match {
- case optimizer.Once =>
-Once
- case _: optimizer.FixedPoint =>
-fixedPoint
-},
-batch.rules: _*
- )
-}
- }
}
[carbondata] branch master updated: [CARBONDATA-4228] [CARBONDATA-4203] Fixed update/delete after alter add segment
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new bca62cd [CARBONDATA-4228] [CARBONDATA-4203] Fixed update/delete after
alter add segment
bca62cd is described below
commit bca62cda806302fe10076c8d2f50e6fad95b147a
Author: nihal0107
AuthorDate: Tue Sep 21 18:14:21 2021 +0530
[CARBONDATA-4228] [CARBONDATA-4203] Fixed update/delete after alter add
segment
Why is this PR needed?
Deleted records are reappearing or updated records are showing old values
in select
queries. It is because after horizontal compaction delete delta file for
the external
segment is written to the default path which is Fact\part0\segment_x\ while
if the
segment is an external segment then delete delta file should be written to
the path
where the segment is present.
What changes were proposed in this PR?
After delete/update operation on the segment, horizontal compaction will be
triggered.
Now after horizontal compaction for external segments, the delete delta
file will be
written to the segment path at the place of the default path.
This closes #4220
---
.../statusmanager/SegmentUpdateStatusManager.java | 17 +--
.../testsuite/addsegment/AddSegmentTestCase.scala | 33 ++
2 files changed, 47 insertions(+), 3 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index 31e253b..fe40494 100644
---
a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++
b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -366,8 +366,18 @@ public class SegmentUpdateStatusManager {
*/
public List getDeleteDeltaFilesList(final Segment segment, final
String blockName) {
List deleteDeltaFileList = new ArrayList<>();
-String segmentPath = CarbonTablePath.getSegmentPath(
-identifier.getTablePath(), segment.getSegmentNo());
+String segmentPath = null;
+if (segment.isExternalSegment()) {
+ for (LoadMetadataDetails details : segmentDetails) {
+if (details.getLoadName().equals(segment.getSegmentNo())) {
+ segmentPath = details.getPath();
+ break;
+}
+ }
+} else {
+ segmentPath = CarbonTablePath.getSegmentPath(
+ identifier.getTablePath(), segment.getSegmentNo());
+}
for (SegmentUpdateDetails block : updateDetails) {
if ((block.getBlockName().equalsIgnoreCase(blockName)) &&
@@ -375,8 +385,9 @@ public class SegmentUpdateStatusManager {
!CarbonUpdateUtil.isBlockInvalid(block.getSegmentStatus())) {
Set deltaFileTimestamps = block.getDeltaFileStamps();
if (deltaFileTimestamps != null && deltaFileTimestamps.size() > 0) {
+ String finalSegmentPath = segmentPath;
deltaFileTimestamps.forEach(timestamp -> deleteDeltaFileList.add(
- CarbonUpdateUtil.getDeleteDeltaFilePath(segmentPath, blockName,
timestamp)));
+ CarbonUpdateUtil.getDeleteDeltaFilePath(finalSegmentPath,
blockName, timestamp)));
} else {
// when the deltaFileTimestamps is null, then there is only one
delta file
// and the SegmentUpdateDetails will have same start and end
timestamp,
diff --git
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
index 72b8134..5d2134f 100644
---
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
+++
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
@@ -86,6 +86,39 @@ class AddSegmentTestCase extends QueryTest with
BeforeAndAfterAll {
FileFactory.deleteAllFilesOfDir(new File(newPath))
}
+ test("test update/delete operation on added segment which required
horizontal compaction") {
+sql("drop table if exists uniqdata")
+sql("""CREATE TABLE uniqdata(empname String, designation String, doj
Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate
Date,attendance int,
+ | utilization int,salary int, empno int)
+ | STORED AS carbondata""".stripMargin)
+
+sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE
uniqdata
+ | OPTIONS('DELIMITER'= ',', '
[carbondata] branch master updated: [CARBONDATA-4288][CARBONDATA-4289] Fix various issues with Index Server caching mechanism.
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new ce860d0 [CARBONDATA-4288][CARBONDATA-4289] Fix various issues with
Index Server caching mechanism.
ce860d0 is described below
commit ce860d0431f41e0594cee848149cdc344f728dd3
Author: Vikram Ahuja
AuthorDate: Fri Sep 17 12:54:01 2021 +0530
[CARBONDATA-4288][CARBONDATA-4289] Fix various issues with Index Server
caching mechanism.
Why is this PR needed?
There are 2 issues in the Index Server flow:
In case when there is a main table with a SI table with prepriming disabled
and index serve
enabled, new load to main table and SI table put the cache for the main
table in the index
server. Cache is also getting again when a select query is fired. This
issue happens because
during load to SI table, getSplits is called on the main table segment
which is in Insert In
Progress state. Index server considers this segment as a legacy segment
because it's index
size = 0 and does not put it's entry in the tableToExecutor mapping. In the
getsplits method
isRefreshneeded is false the first time getSplits is called. During the
select query, in
getSplits method isRefreshNeeded is true and the previous loaded entry is
removed from the
driver but since there is no entry for that table in tableToExecutor
mapping, the previous
cache value becomes dead cache and always stays in the index server. The
newly loaded cache
is loaded to a new executor and 2 copies of cache for the same segment is
being mantained.
Concurrent select queries to the index server shows wrong cache values in
the Index server.
What changes were proposed in this PR?
The following changes are proposed to the index server code:
Removing cache object from the index server in case the segment is INSERT
IN PROGRESS and
in the case of legacy segment adding the value in tabeToExecutor mappping
so that the cache
is also removed from the executor side.
Concurrent queries were able adding duplicate cache values to other
executors. Changed logic
of assign executors method so that concurrent queries are not able to add
cache for same segment
in other executors
This closes #4219
---
.../indexserver/DistributedPruneRDD.scala | 11
.../indexserver/DistributedRDDUtils.scala | 76 +++---
.../indexserver/DistributedRDDUtilsTest.scala | 26 +++-
3 files changed, 88 insertions(+), 25 deletions(-)
diff --git
a/integration/spark/src/main/scala/org/apache/carbondata/indexserver/DistributedPruneRDD.scala
b/integration/spark/src/main/scala/org/apache/carbondata/indexserver/DistributedPruneRDD.scala
index dac47ff..bd0b5c9 100644
---
a/integration/spark/src/main/scala/org/apache/carbondata/indexserver/DistributedPruneRDD.scala
+++
b/integration/spark/src/main/scala/org/apache/carbondata/indexserver/DistributedPruneRDD.scala
@@ -115,6 +115,17 @@ private[indexserver] class DistributedPruneRDD(@transient
private val ss: SparkS
// scalastyle:on
service.shutdownNow()
val LOGGER =
LogServiceFactory.getLogService(classOf[DistributedPruneRDD].getName)
+ // remove the cache of Inprogress segments, this case is required during
the case of
+ // loading to SI. We do get splits of main table in case of Inprogress
segment. No
+ // need to load it to the cache.
+ val inProgressSegments =
indexInputFormat.getValidSegments.asScala.collect {
+case seg if DistributedRDDUtils
+ .isSegmentInProgress(indexInputFormat, seg.getSegmentNo) =>
seg.getSegmentNo
+ }
+ if (inProgressSegments.nonEmpty) {
+
IndexStoreManager.getInstance().clearInvalidSegments(indexInputFormat.getCarbonTable,
+ inProgressSegments.asJava)
+ }
LOGGER.info(s"Time taken to collect ${ inputSplits.size } blocklets : " +
(System.currentTimeMillis() - startTime))
val cacheSize = if (CacheProvider.getInstance().getCarbonCache != null) {
diff --git
a/integration/spark/src/main/scala/org/apache/carbondata/indexserver/DistributedRDDUtils.scala
b/integration/spark/src/main/scala/org/apache/carbondata/indexserver/DistributedRDDUtils.scala
index 1994e15..41db8f1 100644
---
a/integration/spark/src/main/scala/org/apache/carbondata/indexserver/DistributedRDDUtils.scala
+++
b/integration/spark/src/main/scala/org/apache/carbondata/indexserver/DistributedRDDUtils.scala
@@ -32,7 +32,7 @@ import
org.apache.carbondata.core.indexstore.ExtendedBlockletWrapper
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import
org.apache.carbondata.core.readcommitter.{LatestFilesReadC
[carbondata] branch master updated: [CARBONDATA-4285] Fix alter add complex columns with global sort compaction failure
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 22342f8 [CARBONDATA-4285] Fix alter add complex columns with global
sort compaction failure
22342f8 is described below
commit 22342f847d7db515e5f8c17525522085f49bd2a5
Author: Mahesh Raju Somalaraju
AuthorDate: Thu Sep 16 22:41:54 2021 +0530
[CARBONDATA-4285] Fix alter add complex columns with global sort compaction
failure
Why is this PR needed?
Alter add complex columns with global sort compaction is failing due to
AOI exception : Currently creating default complex delimiter list in global
sort compaction
with size of 3. For map case need extra complex delimiter for handling the
key-value
bad record handling: When we add complex columns after insert the data,
complex columns
has null data for previously loaded segments. this null value is going to
treat as bad
record and compaction is failed.
What changes were proposed in this PR?
In Global sort compaction flow create default complex delimiter with 4, as
already
doing in load flow.
Bad records handling pruned for compaction case. No need to check bad
records for
compaction as they are already checked while loading. previously loaded
segments data
we are inserting again in compaction case
This closes #4218
---
.../hadoop/api/CarbonTableOutputFormat.java| 9 ++-
.../spark/load/DataLoadProcessBuilderOnSpark.scala | 12 +++-
.../spark/load/DataLoadProcessorStepOnSpark.scala | 5 +-
.../spark/rdd/CarbonTableCompactor.scala | 3 +-
.../alterTable/TestAlterTableAddColumns.scala | 80 ++
.../processing/loading/BadRecordsLogger.java | 9 ++-
.../loading/BadRecordsLoggerProvider.java | 12 +++-
.../loading/converter/impl/RowConverterImpl.java | 14 ++--
8 files changed, 128 insertions(+), 16 deletions(-)
diff --git
a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
index ed447a5..12f68d8 100644
---
a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
+++
b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
@@ -365,11 +365,16 @@ public class CarbonTableOutputFormat extends
FileOutputFormat 2) {
+if (split.length > 3) {
+ model.setComplexDelimiter(split[1]);
+ model.setComplexDelimiter(split[2]);
+ model.setComplexDelimiter(split[3]);
+} else if (split.length > 2) {
model.setComplexDelimiter(split[1]);
model.setComplexDelimiter(split[2]);
} else if (split.length > 1) {
diff --git
a/integration/spark/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
b/integration/spark/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
index 83d1890..fe1de5c 100644
---
a/integration/spark/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
+++
b/integration/spark/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
@@ -73,7 +73,8 @@ object DataLoadProcessBuilderOnSpark {
dataFrame: Option[DataFrame],
model: CarbonLoadModel,
hadoopConf: Configuration,
- segmentMetaDataAccumulator: CollectionAccumulator[Map[String,
SegmentMetaDataInfo]])
+ segmentMetaDataAccumulator: CollectionAccumulator[Map[String,
SegmentMetaDataInfo]],
+ isCompactionFlow: Boolean = false)
: Array[(String, (LoadMetadataDetails, ExecutionErrors))] = {
var isLoadFromCSV = false
val originRDD = if (dataFrame.isDefined) {
@@ -121,8 +122,13 @@ object DataLoadProcessBuilderOnSpark {
// 2. Convert
val convertRDD = inputRDD.mapPartitionsWithIndex { case (index, rows) =>
ThreadLocalSessionInfo.setConfigurationToCurrentThread(conf.value.value)
- DataLoadProcessorStepOnSpark.convertFunc(rows, index, modelBroadcast,
partialSuccessAccum,
-convertStepRowCounter)
+ DataLoadProcessorStepOnSpark.convertFunc(rows,
+index,
+modelBroadcast,
+partialSuccessAccum,
+convertStepRowCounter,
+false,
+isCompactionFlow)
}.filter(_ != null) // Filter the bad record
// 3. Sort
diff --git
a/integration/spark/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
b/integration/spark/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
index 1694579..8f59200 100644
---
a/integration/spark/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
+++
b/integration/spark/src/main/scala/org/apache/carbondat
[carbondata] branch master updated: [CARBONDATA-4284] Load/insert after alter add column on partition table with complex column fails
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 3b29bcb [CARBONDATA-4284] Load/insert after alter add column on
partition table with complex column fails
3b29bcb is described below
commit 3b29bcb1057166c85e05b67a061ec616e809831d
Author: ShreelekhyaG
AuthorDate: Mon Sep 13 18:44:19 2021 +0530
[CARBONDATA-4284] Load/insert after alter add column on partition table
with complex column fails
Why is this PR needed?
Insert after alter add column on partition table with complex column fails
with bufferUnderFlowException
List of columns order in TableSchema is different after alter add column.
Ex: If partition is of dimension type, when table is created the schema
columns order is as
dimension columns(partition column also) + complex column
After alter add, we are changing the order of columns in schema by moving
the partition column to last.
complex column + partition column
Due to this change in order, while fillDimensionAndMeasureDetails, the
indexing is wrong as it
expects complex column to be last always which causes
bufferUnderFlowException while flattening complex row.
What changes were proposed in this PR?
After alter add, removed changes to add partition column at last.
This closes #4215
---
.../spark/sql/execution/command/carbonTableSchemaCommon.scala | 5 -
.../spark/testsuite/alterTable/TestAlterTableAddColumns.scala | 11 +++
2 files changed, 11 insertions(+), 5 deletions(-)
diff --git
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
index 2096e93..dff6ac1 100644
---
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
+++
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
@@ -387,11 +387,6 @@ class AlterTableColumnSchemaGenerator(
allColumns = CarbonScalaUtil.reArrangeColumnSchema(allColumns)
-if (tableInfo.getFactTable.getPartitionInfo != null) {
- val par = tableInfo.getFactTable.getPartitionInfo.getColumnSchemaList
- allColumns = allColumns.filterNot(b => par.contains(b)) ++= par.asScala
-}
-
def getLocalDictColumnList(tableProperties: mutable.Map[String, String],
columns: mutable.ListBuffer[ColumnSchema]):
(mutable.ListBuffer[ColumnSchema],
mutable.ListBuffer[ColumnSchema]) = {
diff --git
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/alterTable/TestAlterTableAddColumns.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/alterTable/TestAlterTableAddColumns.scala
index f5bfb32..bd4a112 100644
---
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/alterTable/TestAlterTableAddColumns.scala
+++
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/alterTable/TestAlterTableAddColumns.scala
@@ -481,6 +481,17 @@ class TestAlterTableAddColumns extends QueryTest with
BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS alter_com")
}
+ test("test add column to partition table with complex column") {
+sql("drop table if exists alter_com")
+sql("create table alter_com(id int, map1 map) " +
+"partitioned by(name string) stored as carbondata")
+sql("insert into alter_com values( 1,map(1,2),'sh')")
+sql("ALTER TABLE alter_com ADD COLUMNS(intF int)")
+sql("insert into alter_com values(1,map(1,2),1,'df')")
+checkAnswer(sql("select * from alter_com"),
+ Seq(Row(1, Map(1 -> 2), null, "sh"), Row(1, Map(1 -> 2), 1, "df")))
+ }
+
test("Validate default values of complex columns added by alter command") {
sql("DROP TABLE IF EXISTS alter_com")
sql("CREATE TABLE alter_com(doubleField double, arr1 array ) STORED
AS carbondata")
[carbondata] branch master updated: [CARBONDATA-4271] Support DPP for carbon
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new bdc9484 [CARBONDATA-4271] Support DPP for carbon
bdc9484 is described below
commit bdc9484ac8455e8f53e86367c0e5104364799068
Author: Indhumathi27
AuthorDate: Tue Jul 13 17:07:39 2021 +0530
[CARBONDATA-4271] Support DPP for carbon
Why is this PR needed?
This PR enables Dynamic Partition Pruning for carbon.
What changes were proposed in this PR?
CarbonDatasourceHadoopRelation has to extend HadoopFsRelation,
because spark has added a check to use DPP only for relation matching
HadoopFsRelation
Apply Dynamic filter and get runtimePartitions and set this to
CarbonScanRDD for pruning
This closes #4199
---
.../spark/sql/CarbonDataSourceScanHelper.scala | 3 +
.../spark/rdd/CarbonDeltaRowScanRDD.scala | 4 +-
.../carbondata/spark/rdd/CarbonScanRDD.scala | 2 +-
.../spark/sql/CarbonDatasourceHadoopRelation.scala | 49 -
.../scala/org/apache/spark/sql/CarbonSource.scala | 24 ++-
.../CarbonInsertIntoHadoopFsRelationCommand.scala | 2 -
.../table/CarbonCreateTableAsSelectCommand.scala | 2 +-
.../execution/strategy/CarbonDataSourceScan.scala | 26 ++-
.../sql/execution/strategy/CarbonPlanHelper.scala | 44 ++--
.../execution/strategy/CarbonSourceStrategy.scala | 44 ++--
.../spark/sql/execution/strategy/DMLStrategy.scala | 5 +-
.../spark/sql/hive/CarbonFileMetastore.scala | 2 +-
.../apache/spark/sql/optimizer/CarbonFilters.scala | 6 +-
.../spark/sql/CarbonDataSourceScanHelper.scala | 43 +++-
.../apache/spark/sql/CarbonToSparkAdapter.scala| 8 +-
.../dblocation/DBLocationCarbonTableTestCase.scala | 2 +-
.../sql/DynamicPartitionPruningTestCase.scala | 235 +
17 files changed, 426 insertions(+), 75 deletions(-)
diff --git
a/integration/spark/src/main/common2.3and2.4/org/apache/spark/sql/CarbonDataSourceScanHelper.scala
b/integration/spark/src/main/common2.3and2.4/org/apache/spark/sql/CarbonDataSourceScanHelper.scala
index 32a2e34..22183de 100644
---
a/integration/spark/src/main/common2.3and2.4/org/apache/spark/sql/CarbonDataSourceScanHelper.scala
+++
b/integration/spark/src/main/common2.3and2.4/org/apache/spark/sql/CarbonDataSourceScanHelper.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql
import org.apache.spark.CarbonInputMetrics
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.catalog.CatalogTablePartition
import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression =>
SparkExpression}
import org.apache.spark.sql.catalyst.plans.QueryPlan
import org.apache.spark.sql.execution.{ColumnarBatchScan, DataSourceScanExec}
@@ -40,6 +41,8 @@ abstract class CarbonDataSourceScanHelper(relation:
CarbonDatasourceHadoopRelat
pushedDownProjection: CarbonProjection,
directScanSupport: Boolean,
extraRDD: Option[(RDD[InternalRow], Boolean)],
+selectedCatalogPartitions: Seq[CatalogTablePartition],
+partitionFilterWithDpp: Seq[SparkExpression],
segmentIds: Option[String])
extends DataSourceScanExec with ColumnarBatchScan {
diff --git
a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeltaRowScanRDD.scala
b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeltaRowScanRDD.scala
index de6aeb5..949216c 100644
---
a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeltaRowScanRDD.scala
+++
b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeltaRowScanRDD.scala
@@ -44,7 +44,7 @@ class CarbonDeltaRowScanRDD[T: ClassTag](
@transient private val spark: SparkSession,
@transient private val serializedTableInfo: Array[Byte],
@transient private val tableInfo: TableInfo,
-@transient override val partitionNames: Seq[PartitionSpec],
+@transient private val newPartitionNames: Seq[PartitionSpec],
override val columnProjection: CarbonProjection,
var filter: IndexFilter,
identifier: AbsoluteTableIdentifier,
@@ -62,7 +62,7 @@ class CarbonDeltaRowScanRDD[T: ClassTag](
serializedTableInfo,
tableInfo,
inputMetricsStats,
-partitionNames,
+newPartitionNames,
dataTypeConverterClz,
readSupportClz) {
override def internalGetPartitions: Array[Partition] = {
diff --git
a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index fcb379c..90f1f8b 100644
---
a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++
b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -8
[carbondata] branch master updated: [CARBONDATA-4274] Fix create partition table error with spark 3.1
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new ca659b5 [CARBONDATA-4274] Fix create partition table error with spark
3.1
ca659b5 is described below
commit ca659b5cd3219de7c8a9603784559e20fdff1fda
Author: ShreelekhyaG
AuthorDate: Thu Aug 19 20:26:00 2021 +0530
[CARBONDATA-4274] Fix create partition table error with spark 3.1
Why is this PR needed?
With spark 3.1, we can create a partition table by giving partition
columns from schema.
Like below example:
create table partitionTable(c1 int, c2 int, v1 string, v2 string)
stored as carbondata partitioned by (v2,c2)
When the table is created by SparkSession with CarbonExtension,
catalog table is created with the specified partitions.
But in cluster/ with carbon session, when we create partition
table with above syntax it is creating normal table with no partitions.
What changes were proposed in this PR?
partitionByStructFields is empty when we directly give partition
column names. So it was not creating a partition table. Made
changes to identify the partition column names and get the struct
field and datatype info from table columns.
This closes #4208
---
.../org/apache/spark/sql/SparkVersionAdapter.scala | 16 -
.../spark/sql/parser/CarbonSparkSqlParser.scala| 27 --
.../StandardPartitionTableQueryTestCase.scala | 21 +
3 files changed, 50 insertions(+), 14 deletions(-)
diff --git
a/integration/spark/src/main/spark3.1/org/apache/spark/sql/SparkVersionAdapter.scala
b/integration/spark/src/main/spark3.1/org/apache/spark/sql/SparkVersionAdapter.scala
index 5cf5c15..21d7586 100644
---
a/integration/spark/src/main/spark3.1/org/apache/spark/sql/SparkVersionAdapter.scala
+++
b/integration/spark/src/main/spark3.1/org/apache/spark/sql/SparkVersionAdapter.scala
@@ -233,17 +233,6 @@ trait SparkVersionAdapter {
val partitionerFields = partitionByStructFields.map { structField =>
PartitionerField(structField.name, Some(structField.dataType.toString),
null)
}
-// validate partition clause
-if (partitionerFields.nonEmpty) {
- // partition columns should not be part of the schema
- val badPartCols =
partitionerFields.map(_.partitionColumn.toLowerCase).toSet
-.intersect(colNames.map(_.toLowerCase).toSet)
- if (badPartCols.nonEmpty) {
-operationNotAllowed(s"Partition columns should not be specified in the
schema: " +
- badPartCols.map("\"" + _ + "\"").mkString("[", ",", "]")
- , partitionColumns: PartitionFieldListContext)
- }
-}
partitionerFields
}
@@ -280,7 +269,10 @@ trait SparkVersionAdapter {
val options = new CarbonOption(properties)
// validate streaming property
validateStreamingProperty(options)
-var fields = parser.getFields(cols ++ partitionByStructFields)
+// with Spark 3.1, partitioned columns can be already present in schema.
+// Check and remove from fields and add partition columns at last
+val updatedCols = cols.filterNot(x => partitionByStructFields.contains(x))
+var fields = parser.getFields(updatedCols ++ partitionByStructFields)
// validate for create table as select
selectQuery match {
case Some(q) =>
diff --git
a/integration/spark/src/main/spark3.1/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
b/integration/spark/src/main/spark3.1/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
index ac0be49..49978d4 100644
---
a/integration/spark/src/main/spark3.1/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
+++
b/integration/spark/src/main/spark3.1/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
@@ -21,6 +21,7 @@ import scala.collection.mutable
import org.antlr.v4.runtime.tree.TerminalNode
import org.apache.spark.sql.{CarbonThreadUtil, CarbonToSparkAdapter,
SparkSession}
import org.apache.spark.sql.catalyst.parser.{AbstractSqlParser, SqlBaseParser}
+import org.apache.spark.sql.catalyst.parser.ParserUtils.operationNotAllowed
import org.apache.spark.sql.catalyst.parser.SqlBaseParser._
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.{SparkSqlAstBuilder, SparkSqlParser}
@@ -130,8 +131,30 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf,
val tableProperties = convertPropertiesToLowercase(properties)
// validate partition clause
-val partitionByStructFields = Option(partitionColumns).toSeq
-.flatMap(x => visitPartitionFieldList(x)._2)
+// There can be two scenarios for creating partition table with spark 3.1.
+// Scenario 1: create partition columns wit
[carbondata] branch master updated: [CARBONDATA-4204][CARBONDATA-4231] Fix add segment error message, index server failed testcases and dataload fail error on update
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new c9a5231 [CARBONDATA-4204][CARBONDATA-4231] Fix add segment error
message, index server failed testcases and dataload fail error on update
c9a5231 is described below
commit c9a5231ab73d64a0966a45a755ff7e9031c0201e
Author: ShreelekhyaG
AuthorDate: Mon Jun 21 20:44:31 2021 +0530
[CARBONDATA-4204][CARBONDATA-4231] Fix add segment error message,
index server failed testcases and dataload fail error on update
Why is this PR needed?
1. When the path is empty in Carbon add segments then
StringIndexOutOfBoundsException is thrown.
2. Index server UT failures fix.
3. Update fails with dataload fail error if set bad
records action is specified to force with spark 3.1v.
What changes were proposed in this PR?
1. Added check to see if the path is empty and then throw
a valid error message.
2. Used checkAnswer instead of assert in test cases so
that the order of rows returned would be same with or
without index server. Excluded 2 test cases where explain
with query statistics is used, as we are not setting any
pruning info from index server.
3. On update command, dataframe.persist is called and with
latest 3.1 spark changes, spark returns a cloned
SparkSession from cacheManager with all specified
configurations disabled. As now it's using a different
sparkSession for 3.1 which is not initialized in CarbonEnv.
So CarbonEnv.init is called where new CarbonSessionInfo is
created with no sessionParams. So, the properties set were
not accessible. When a new carbonSessionInfo object is
getting created, made changes to set existing sessionparams
from currentThreadSessionInfo.
This closes #4157
---
.../main/scala/org/apache/spark/sql/CarbonEnv.scala | 1 +
.../command/management/CarbonAddLoadCommand.scala | 3 +++
.../testsuite/addsegment/AddSegmentTestCase.scala | 9 +
.../alterTable/TestAlterTableAddColumns.scala | 21 ++---
.../createTable/TestRenameTableWithIndex.scala | 8 ++--
.../AlterTableColumnRenameTestCase.scala| 19 ---
6 files changed, 37 insertions(+), 24 deletions(-)
diff --git
a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
index bb6e2ef..98c18ac 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
@@ -119,6 +119,7 @@ class CarbonEnv {
val threadLevelCarbonSessionInfo = new CarbonSessionInfo()
if (currentThreadSessionInfo != null) {
threadLevelCarbonSessionInfo.setThreadParams(currentThreadSessionInfo.getThreadParams)
+
threadLevelCarbonSessionInfo.setSessionParams(currentThreadSessionInfo.getSessionParams)
}
ThreadLocalSessionInfo.setCarbonSessionInfo(threadLevelCarbonSessionInfo)
ThreadLocalSessionInfo.setConfigurationToCurrentThread(
diff --git
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAddLoadCommand.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAddLoadCommand.scala
index e319185..ed40d9a 100644
---
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAddLoadCommand.scala
+++
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAddLoadCommand.scala
@@ -89,6 +89,9 @@ case class CarbonAddLoadCommand(
var givenPath = options.getOrElse(
"path", throw new UnsupportedOperationException("PATH is mandatory"))
+if (givenPath.length == 0) {
+ throw new UnsupportedOperationException("PATH cannot be empty")
+}
// remove file separator if already present
if (givenPath.charAt(givenPath.length - 1) == '/') {
givenPath = givenPath.substring(0, givenPath.length - 1)
diff --git
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
index cc5a373..72b8134 100644
---
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
+++
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
@@ -1120,6 +1120,15 @@ class AddSegmentTestCase extends QueryTest with
BeforeAndAfterAll {
assert(ex.getMessage.contains("can not add same segment path repeatedly"))
}
+ test("Test add segment with empty
[carbondata] branch master updated: [CARBONDATA-4210] Handle 3.1 parsing failures related to alter complex types
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 02e7723 [CARBONDATA-4210] Handle 3.1 parsing failures related to
alter complex types
02e7723 is described below
commit 02e77234ebb9c1e18e194c1844aaf2d4a2473dcc
Author: akkio-97
AuthorDate: Wed Jun 23 13:19:48 2021 +0530
[CARBONDATA-4210] Handle 3.1 parsing failures related to alter complex types
Why is this PR needed?
For 2.3 and 2.4 parsing of alter commands are done by spark. Which is not
in the case of 3.1.
What changes were proposed in this PR?
So carbon is responsible for the parsing here.
Previously ignored test cases due to this issue are now enabled.
This closes #4162
---
.../TestSIWithComplexArrayType.scala | 254 +++---
.../spark/sql/catalyst/CarbonParserUtil.scala | 47 +++
.../spark/sql/parser/CarbonSpark2SqlParser.scala | 32 +-
.../sql/parser/CarbonSparkSqlParserUtil.scala | 29 +-
.../alterTable/TestAlterTableAddColumns.scala | 8 +-
.../AlterTableColumnRenameTestCase.scala | 387 ++---
6 files changed, 390 insertions(+), 367 deletions(-)
diff --git
a/index/secondary-index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSIWithComplexArrayType.scala
b/index/secondary-index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSIWithComplexArrayType.scala
index 46908b7..3fe1443 100644
---
a/index/secondary-index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSIWithComplexArrayType.scala
+++
b/index/secondary-index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSIWithComplexArrayType.scala
@@ -46,158 +46,142 @@ class TestSIWithComplexArrayType extends QueryTest with
BeforeAndAfterEach {
}
test("Test restructured array and existing string column as index
columns on SI with compaction") {
-// TODO: Support alter chnage column for complex type for SPARK 3.1.1
-// REFER: https://issues.apache.org/jira/browse/CARBONDATA-4210
-if (!sqlContext.sparkContext.version.startsWith("3.1")) {
- sql("drop table if exists complextable")
- sql("create table complextable (id string, country array,
columnName string) stored as carbondata")
- sql("insert into complextable select 1,array('china', 'us'), 'b'")
- sql("insert into complextable select 2,array('pak'), 'v'")
+sql("drop table if exists complextable")
+sql("create table complextable (id string, country array,
columnName string) stored as carbondata")
+sql("insert into complextable select 1,array('china', 'us'), 'b'")
+sql("insert into complextable select 2,array('pak'), 'v'")
- sql("drop index if exists index_11 on complextable")
- sql("ALTER TABLE complextable ADD COLUMNS(newArray array)")
- sql("alter table complextable change newArray arr2 array")
- sql("alter table complextable change columnName name string")
- sql("insert into complextable select 3,array('china'),
'f',array('hello','world')")
- sql("insert into complextable select
4,array('India'),'g',array('iron','man','jarvis')")
+sql("drop index if exists index_11 on complextable")
+sql("ALTER TABLE complextable ADD COLUMNS(newArray array)")
+sql("alter table complextable change newArray arr2 array")
+sql("alter table complextable change columnName name string")
+sql("insert into complextable select 3,array('china'),
'f',array('hello','world')")
+sql("insert into complextable select
4,array('India'),'g',array('iron','man','jarvis')")
- checkAnswer(sql("select * from complextable where
array_contains(arr2,'iron')"),
-Seq(Row("4", mutable.WrappedArray.make(Array("India")), "g",
- mutable.WrappedArray.make(Array("iron", "man", "jarvis")
- val result1 = sql("select * from complextable where
array_contains(arr2,'iron') and name='g'")
- val result2 = sql("select * from complextable where arr2[0]='iron' and
name='f'")
- sql("create index index_11 on table complextable(arr2, name) as
'carbondata'")
- sql("alter table complextable compact 'minor'")
- val df1 = sql(" select * from complextable where
array_contains(arr2,'iron') and name='g'")
- val df2 = sql(" select * from complextable where arr2[0]='iron' and
name='f'")
- if (!isFilterPushedDownToSI(df1.queryExecution.sparkPlan)) {
-assert(fa
[carbondata] branch master updated: [HOTFIX]Revert wrong pom changes commit during prepare release process.
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git The following commit(s) were added to refs/heads/master by this push: new c7a3d6d [HOTFIX]Revert wrong pom changes commit during prepare release process. c7a3d6d is described below commit c7a3d6d13084c2a399f8a2a7356108c5161e51f3 Author: akashrn5 AuthorDate: Mon Jul 5 22:00:52 2021 +0530 [HOTFIX]Revert wrong pom changes commit during prepare release process. Why is this PR needed? Due to wrong branch release, wrong pom changes are present. What changes were proposed in this PR? revert the pom changes. This closes #4167 --- assembly/pom.xml| 2 +- common/pom.xml | 2 +- core/pom.xml| 2 +- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 2 +- format/pom.xml | 2 +- geo/pom.xml | 2 +- hadoop/pom.xml | 2 +- index/bloom/pom.xml | 2 +- index/examples/pom.xml | 2 +- index/lucene/pom.xml| 2 +- index/secondary-index/pom.xml | 2 +- integration/flink-build/pom.xml | 2 +- integration/flink-proxy/pom.xml | 2 +- integration/flink/pom.xml | 2 +- integration/hive/pom.xml| 2 +- integration/presto/pom.xml | 2 +- integration/spark/pom.xml | 2 +- mv/plan/pom.xml | 2 +- pom.xml | 2 +- processing/pom.xml | 2 +- sdk/sdk/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/cli/pom.xml | 2 +- 24 files changed, 24 insertions(+), 24 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index b4054a2..5d96c9b 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.2.1-SNAPSHOT +2.2.0-SNAPSHOT ../pom.xml diff --git a/common/pom.xml b/common/pom.xml index c533990..327708e 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.2.1-SNAPSHOT +2.2.0-SNAPSHOT ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index e559376..c144b11 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.2.1-SNAPSHOT +2.2.0-SNAPSHOT ../pom.xml diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index 1a224eb..bce06c3 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.2.1-SNAPSHOT +2.2.0-SNAPSHOT ../../pom.xml diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index cbdb49e..3a95d84 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.2.1-SNAPSHOT +2.2.0-SNAPSHOT ../../pom.xml diff --git a/format/pom.xml b/format/pom.xml index 709f46f..ccfb03f 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.2.1-SNAPSHOT +2.2.0-SNAPSHOT ../pom.xml diff --git a/geo/pom.xml b/geo/pom.xml index 5d623c1..f2c882a 100644 --- a/geo/pom.xml +++ b/geo/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.2.1-SNAPSHOT +2.2.0-SNAPSHOT ../pom.xml diff --git a/hadoop/pom.xml b/hadoop/pom.xml index 799472f..6b04990 100644 --- a/hadoop/pom.xml +++ b/hadoop/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.2.1-SNAPSHOT +2.2.0-SNAPSHOT ../pom.xml diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml index 70673f7..57007b9 100644 --- a/index/bloom/pom.xml +++ b/index/bloom/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.2.1-SNAPSHOT +2.2.0-SNAPSHOT ../../pom.xml diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 4cad0b7..aedaade 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.2.1-SNAPSHOT +2.2.0-SNAPSHOT ../../pom.xml diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml index 48ebbb3..0e0baf5 100644 --- a/index/lucene/pom.xml +++ b/index/lucene/pom.xml @@ -4,7 +4,7 @@ org.apache.carbondata carbondata-parent -2.2.1-SNAPSHOT +2.2.0-SNAPSHOT ../../pom.xml diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index 77f12fe..9754bd0 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -22,7 +22,7 @@ org.apache.carbondata carbondata-parent -2.2.1-SNAPSHOT +2.2.0-SNAPSHOT ../../pom.xml diff --git
[carbondata] branch branch-2.2 deleted (was 952cf82)
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a change to branch branch-2.2 in repository https://gitbox.apache.org/repos/asf/carbondata.git. was 952cf82 [maven-release-plugin] prepare for next development iteration This change permanently discards the following revisions: discard 952cf82 [maven-release-plugin] prepare for next development iteration
[carbondata] branch master updated: [CARBONDATA-4191] update table for primitive column not working when complex child column name and primitive column name match
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 4c04f7c [CARBONDATA-4191] update table for primitive column not
working when complex child column name and primitive column name match
4c04f7c is described below
commit 4c04f7c0e49efe5e3203ff2ce04bd7bf1b7fe6aa
Author: Mahesh Raju Somalaraju
AuthorDate: Mon May 24 18:01:34 2021 +0530
[CARBONDATA-4191] update table for primitive column not working when
complex child
column name and primitive column name match
Why is this PR needed?
Update primitive column not working when complex column child name and
primitive
data type name same.
When an update for primitive is received, we are checking in complex child
columns
if column name matches then returning UnsupportedOperationbException.
What changes were proposed in this PR?
Currently, we are ignoring the prefix of all columns and passing only
columns/child
column info to the update command.
New Changes: Passing full column(alias name/table name.columnName) name
which is given
by the user and added checks for handling the unsupported update operation
of complex columns.
This closes #4139
---
.../apache/spark/sql/optimizer/CarbonIUDRule.scala | 8 ---
.../spark/sql/parser/CarbonSpark2SqlParser.scala | 34 --
.../complexType/TestComplexDataType.scala | 72 +-
3 files changed, 100 insertions(+), 14 deletions(-)
diff --git
a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonIUDRule.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonIUDRule.scala
index da1ca55..77d85a8 100644
---
a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonIUDRule.scala
+++
b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonIUDRule.scala
@@ -41,14 +41,6 @@ class CarbonIUDRule extends Rule[LogicalPlan] with
PredicateHelper {
case Project(pList, child) if !isTransformed =>
var (dest: Seq[NamedExpression], source: Seq[NamedExpression]) =
pList
.splitAt(pList.size - cols.size)
-// check complex column
-cols.foreach { col =>
- val complexExists = "\"name\":\"" + col + "\""
- if (dest.exists(m => m.dataType.json.contains(complexExists))) {
-throw new UnsupportedOperationException(
- "Unsupported operation on Complex data type")
- }
-}
// check updated columns exists in table
val diff = cols.diff(dest.map(_.name.toLowerCase))
if (diff.nonEmpty) {
diff --git
a/integration/spark/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
index b77f16f..945a420 100644
---
a/integration/spark/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
+++
b/integration/spark/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
@@ -17,8 +17,9 @@
package org.apache.spark.sql.parser
-import scala.collection.JavaConverters.seqAsJavaListConverter
+import scala.collection.JavaConverters._
import scala.collection.mutable
+import scala.collection.mutable.ListBuffer
import scala.language.implicitConversions
import org.apache.commons.lang3.StringUtils
@@ -257,6 +258,25 @@ class CarbonSpark2SqlParser extends CarbonDDLSqlParser {
(SET ~> "(" ~> repsep(element, ",") <~ ")") ~
("=" ~> restInput) <~ opt(";") ^^ {
case tab ~ columns ~ rest =>
+// If update is received for complex data types then throw exception
+var finalColumns = List.empty[String]
+var updateColumns = new ListBuffer[String]()
+columns.foreach { column =>
+ if (column.contains('.')) {
+val columnFullName = column.split('.')
+if (columnFullName.size >= 3) {
+ throw new UnsupportedOperationException("Unsupported operation
on Complex data types")
+} else if ((tab._3.isDefined &&
tab._3.get.equals(columnFullName(0)))
+|| tab._4.table.equals(columnFullName(0))) {
+ updateColumns += columnFullName(1)
+} else {
+ throw new UnsupportedOperationException("Unsupported operation
on Complex data types")
+}
+ } else {
+updateColumns += column
+ }
+}
+finalColumns = updateColumns.toList
val (sel, where) = splitQuery(rest)
val selectPattern = """^\s*select\s+"
[carbondata] branch master updated: [CARBONDATA-4186] Fixed insert failure when partition column present in local sort scope
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 01fd120 [CARBONDATA-4186] Fixed insert failure when partition column
present in local sort scope
01fd120 is described below
commit 01fd12042e3300a5d18eef15a634df66b29580ee
Author: Nihal ojha
AuthorDate: Wed May 12 12:26:42 2021 +0530
[CARBONDATA-4186] Fixed insert failure when partition column present in
local sort scope
Why is this PR needed?
Currently when we create table with partition column and put the same
column as part of
local sort scope then Insert query fails with ArrayIndexOutOfBounds
exception.
What changes were proposed in this PR?
Handle ArrayIndexOutOfBound exception, earlier array size was not
increasing because data
was inconsistence and in the wrong order for sortcolumn and
isDimNoDictFlags.
This closes #4132
---
.../carbondata/core/datastore/TableSpec.java | 21 +++-
.../StandardPartitionTableLoadingTestCase.scala| 18
.../loading/sort/unsafe/UnsafeCarbonRowPage.java | 17 ++-
.../processing/sort/sortdata/SortParameters.java | 119 ++---
.../processing/sort/sortdata/TableFieldStat.java | 28 +++--
.../processing/util/CarbonDataProcessorUtil.java | 2 +-
6 files changed, 174 insertions(+), 31 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
b/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
index 6cbf07c..68363aa 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
@@ -62,17 +62,28 @@ public class TableSpec {
List dimensions = carbonTable.getVisibleDimensions();
List measures = carbonTable.getVisibleMeasures();
if (keepPartitionColumnsToEnd && carbonTable.getPartitionInfo() != null) {
- // keep the partition columns in the end
+ // keep the partition columns in the end if that is not present in sort
columns
List reArrangedDimensions = new ArrayList<>();
List reArrangedMeasures = new ArrayList<>();
List partitionDimensions = new ArrayList<>();
List partitionMeasures = new ArrayList<>();
List columnSchemaList =
carbonTable.getPartitionInfo().getColumnSchemaList();
+ String[] sortColumns = carbonTable.getTableInfo()
+ .getFactTable().getTableProperties().getOrDefault("sort_columns",
"").split(",");
+ for (String col : sortColumns) {
+for (CarbonDimension dim : dimensions) {
+ if (dim.getColName().equalsIgnoreCase(col)) {
+reArrangedDimensions.add(dim);
+ }
+}
+ }
for (CarbonDimension dim : dimensions) {
-if (columnSchemaList.contains(dim.getColumnSchema())) {
- partitionDimensions.add(dim);
-} else {
- reArrangedDimensions.add(dim);
+if (!dim.isSortColumn()) {
+ if (columnSchemaList.contains(dim.getColumnSchema())) {
+partitionDimensions.add(dim);
+ } else {
+reArrangedDimensions.add(dim);
+ }
}
}
if (partitionDimensions.size() != 0) {
diff --git
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
index 3a143b6..bdeacc1 100644
---
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
+++
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
@@ -517,6 +517,24 @@ class StandardPartitionTableLoadingTestCase extends
QueryTest with BeforeAndAfte
assert(result.get(0).get(6).equals(dataAndIndexSize._2))
}
+ test("test partition column with different sort scope") {
+verifySortWithPartition("global_sort")
+verifySortWithPartition("no_sort")
+verifySortWithPartition("local_sort")
+ }
+
+ def verifySortWithPartition(scope: String): Unit = {
+sql("drop table if exists carbon_partition")
+sql(s"create table carbon_partition(id int, name string, salary double) " +
+"partitioned by(country string, id1 int)" +
+s"stored as carbondata
tblproperties('sort_scope'='$scope','sort_columns'='country, id')")
+sql("insert into carbon_partition select 1, 'Ram',3500,'India', 20")
+checkAnswer(
+ sql("SELECT * FROM carbon_partit
[carbondata] branch master updated: [CARBONDATA-4185] Doc Changes for Heterogeneous format segments in carbondata
This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git The following commit(s) were added to refs/heads/master by this push: new 861ba2e [CARBONDATA-4185] Doc Changes for Heterogeneous format segments in carbondata 861ba2e is described below commit 861ba2e44f8087e15fbdba278e21e229355b7d99 Author: Mahesh Raju Somalaraju AuthorDate: Wed May 12 23:11:57 2021 +0530 [CARBONDATA-4185] Doc Changes for Heterogeneous format segments in carbondata Why is this PR needed? Heterogeneous format segments in carbondata documenation. What changes were proposed in this PR? Add segment feature background and impact on existed carbondata features This closes #4134 --- README.md| 1 + docs/addsegment-guide.md | 78 2 files changed, 79 insertions(+) diff --git a/README.md b/README.md index 4f8da5c..d1b47af 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ CarbonData is built using Apache Maven, to [build CarbonData](https://github.com * [CarbonData Lucene Index](https://github.com/apache/carbondata/blob/master/docs/index/lucene-index-guide.md) * [CarbonData MV](https://github.com/apache/carbondata/blob/master/docs/mv-guide.md) * [Carbondata Secondary Index](https://github.com/apache/carbondata/blob/master/docs/index/secondary-index-guide.md) +* [Heterogeneous format segments in carbondata](https://github.com/apache/carbondata/blob/master/docs/addsegment-guide.md) * [SDK Guide](https://github.com/apache/carbondata/blob/master/docs/sdk-guide.md) * [C++ SDK Guide](https://github.com/apache/carbondata/blob/master/docs/csdk-guide.md) * [Performance Tuning](https://github.com/apache/carbondata/blob/master/docs/performance-tuning.md) diff --git a/docs/addsegment-guide.md b/docs/addsegment-guide.md new file mode 100644 index 000..a45e6d6 --- /dev/null +++ b/docs/addsegment-guide.md @@ -0,0 +1,78 @@ + + +# Heterogeneous format segments in carbondata + +###Background +In the industry, many users already adopted to data with different formats like ORC, Parquet, JSON, CSV etc., +If users want to migrate to Carbondata for better performance or for better features then there is no direct way. +All the existing data needs to be converted to Carbondata to migrate. +This solution works out if the existing data is less, what if the existing data is more? +Heterogeneous format segments aims to solve this problem by avoiding data conversion. + +###Add segment with path and format +Users can add the existing data as a segment to the carbon table provided the schema of the data + and the carbon table should be the same. + +``` +Alter table table_name add segment options (‘path’= 'hdfs://usr/oldtable,'format'=parquet) +``` +In the above command user can add the existing data to the carbon table as a new segment and also + can provide the data format. + +During add segment, it will infer the schema from data and validates the schema against the carbon table. +If the schema doesn’t match it throws an exception. + +###Changes to tablestatus file +Carbon adds the new segment by adding segment information to tablestatus file. In order to add the path and format information to tablestatus, we are going to add `segmentPath` and ‘format’ to the tablestatus file. +And any extra `options` will be added to the segment file. + + +###Changes to Spark Integration +During select query carbon reads data through RDD which is created by + CarbonDatasourceHadoopRelation.buildScan, This RDD reads data from physical carbondata files and provides data to spark query plan. +To support multiple formats per segment basis we can create multiple RDD using the existing Spark + file format scan class FileSourceScanExec . This class can generate scan RDD for all spark supported formats. We can union all these multi-format RDD and create a single RDD and provide it to spark query plan. + +Note: This integration will be clean as we use the sparks optimized reading, pruning and it + involves whole codegen and vector processing with unsafe support. + +###Changes to Presto Integration +CarbondataSplitManager can create the splits for carbon and as well as for other formats and + choose the page source as per the split. + +### Impact on existed feature +**Count(\*) query:** In case if the segments are mixed with different formats then driver side + optimization for count(*) query will not work so it will be executed on executor side. + +**Index DataMaps:** Datamaps like block/blocklet datamap will only work for carbondata format + segments so there would not be any driver side pruning for other formats. + +**Update/Delete:** Update & Delete operations cannot be allowed on the table which has mixed formats +But it can be allowed if the external segments are added with carbon
[carbondata] branch master updated: [CARBONDATA-4188] Fixed select query with small table page size after alter add column
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 41a756f [CARBONDATA-4188] Fixed select query with small table page
size after alter add column
41a756f is described below
commit 41a756fff7d8774fd919540561d1c39cfc24b9b4
Author: Nihal ojha
AuthorDate: Mon May 17 15:30:24 2021 +0530
[CARBONDATA-4188] Fixed select query with small table page size after alter
add column
Why is this PR needed?
Select query on table with long string data type and small page size throws
ArrayIndexOutOfBoudException after alter add columns.
Query fails because after changing the schema, the number of rows set in
bitsetGroup(RestructureIncludeFilterExecutorImpl.applyFilter()) for pages
is not correct.
What changes were proposed in this PR?
Set the correct number of rows inside every page of bitsetGroup.
This closes #4137
---
.../carbondata/core/scan/filter/FilterUtil.java| 29 ++-
.../executer/RangeValueFilterExecutorImpl.java | 4 +-
.../RestructureExcludeFilterExecutorImpl.java | 6 +--
.../RestructureIncludeFilterExecutorImpl.java | 6 +--
...velRangeGreaterThanEqualFilterExecutorImpl.java | 4 +-
...RowLevelRangeGreaterThanFilterExecutorImpl.java | 4 +-
...wLevelRangeLessThanEqualFilterExecutorImpl.java | 4 +-
.../RowLevelRangeLessThanFilterExecutorImpl.java | 4 +-
.../core/scan/filter/FilterUtilTest.java | 43 --
9 files changed, 52 insertions(+), 52 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 9dc6b5b..1acae54 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -30,7 +30,6 @@ import java.util.Set;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
@@ -81,6 +80,7 @@ import
org.apache.carbondata.core.scan.filter.resolver.RowLevelRangeFilterResolv
import
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.ColumnResolvedFilterInfo;
import
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import
org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.scan.result.vector.CarbonDictionary;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
@@ -898,30 +898,23 @@ public final class FilterUtil {
}
/**
- * This method will create default bitset group. Applicable for restructure
scenarios.
+ * This method will create bit set group for particular raw blocklet column
chunk.
+ * Applicable for restructure scenarios.
*
- * @param pageCount
- * @param totalRowCount
+ * @param rawBlockletColumnChunks
* @param defaultValue
* @return
*/
- public static BitSetGroup createBitSetGroupWithDefaultValue(int pageCount,
int totalRowCount,
- boolean defaultValue) {
+ public static BitSetGroup
createBitSetGroupWithColumnChunk(RawBlockletColumnChunks
+ rawBlockletColumnChunks, boolean defaultValue) {
+int pageCount = rawBlockletColumnChunks.getDataBlock().numberOfPages();
BitSetGroup bitSetGroup = new BitSetGroup(pageCount);
-int numberOfRows =
CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
-int pagesTobeFullFilled = totalRowCount / numberOfRows;
-int rowCountForLastPage = totalRowCount % numberOfRows;
-for (int i = 0; i < pagesTobeFullFilled; i++) {
- BitSet bitSet = new BitSet(numberOfRows);
- bitSet.set(0, numberOfRows, defaultValue);
+for (int i = 0; i < pageCount; i++) {
+ int pageRowCount =
rawBlockletColumnChunks.getDataBlock().getPageRowCount(i);
+ BitSet bitSet = new BitSet(pageRowCount);
+ bitSet.set(0, pageRowCount, defaultValue);
bitSetGroup.setBitSet(bitSet, i);
}
-// create and fill bitset for the last page if any records are left
-if (rowCountForLastPage > 0) {
- BitSet bitSet = new BitSet(rowCountForLastPage);
- bitSet.set(0, rowCountForLastPage, defaultValue);
- bitSetGroup.setBitSet(bitSet, pagesTobeFullFilled);
-}
return bitSetGroup;
[carbondata] branch master updated: [CARBONDATA-4175] [CARBONDATA-4162] Leverage Secondary Index till segment level
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 8996369 [CARBONDATA-4175] [CARBONDATA-4162] Leverage Secondary Index
till segment level
8996369 is described below
commit 899636910d7ebf914d7ec4e2443ce6a4e4ae2ef0
Author: Nihal ojha
AuthorDate: Wed Mar 24 15:17:44 2021 +0530
[CARBONDATA-4175] [CARBONDATA-4162] Leverage Secondary Index till segment
level
Why is this PR needed?
In the existing architecture, if the parent(main) table and SI table don’t
have
the same valid segments then we disable the SI table. And then from the
next query
onwards, we scan and prune only the parent table until we trigger the next
load or
REINDEX command (as these commands will make the parent and SI table
segments in sync).
Because of this, queries take more time to give the result when SI is
disabled.
What changes were proposed in this PR?
Instead of disabling the SI table(when parent and child table segments are
not in sync)
we will do pruning on SI tables for all the valid segments(segments with
status success,
marked for update and load partial success) and the rest of the segments
will be pruned by the parent table.
As of now, query on the SI table can be pruned in two ways:
a) With SI as data map.
b) WIth spark plan rewrite.
This PR contains changes to support both methods of SI to leverage till
segment level.
This closes #4116
---
.../apache/carbondata/core/index/IndexFilter.java | 10 +++
.../org/apache/carbondata/core/index/Segment.java | 11 +++
.../apache/carbondata/core/index/TableIndex.java | 24 ++---
.../carbondata/core/index/dev/IndexFactory.java| 1 +
.../executer/RowLevelFilterExecutorImpl.java | 4 +-
.../carbondata/hadoop/api/CarbonInputFormat.java | 15
.../TestCreateIndexWithLoadAndCompaction.scala | 32 ++-
.../secondaryindex/TestIndexModelWithIUD.scala | 30 +++
.../testsuite/secondaryindex/TestIndexRepair.scala | 20 ++---
.../TestSIWithComplexArrayType.scala | 9 +-
.../secondaryindex/TestSIWithSecondaryIndex.scala | 100 +++--
.../carbondata/index/secondary/SecondaryIndex.java | 46 --
.../index/secondary/SecondaryIndexFactory.java | 3 +
.../command/index/ShowIndexesCommand.scala | 26 +-
.../apache/spark/sql/index/CarbonIndexUtil.scala | 28 --
.../secondaryindex/command/SICreationCommand.scala | 26 +-
.../joins/BroadCastSIFilterPushJoin.scala | 18 +++-
.../spark/sql/secondaryindex/load/Compactor.scala | 14 ---
.../optimizer/CarbonSecondaryIndexOptimizer.scala | 30 ++-
.../secondaryindex/rdd/SecondaryIndexCreator.scala | 18
.../testsuite/addsegment/AddSegmentTestCase.scala | 36
21 files changed, 326 insertions(+), 175 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/index/IndexFilter.java
b/core/src/main/java/org/apache/carbondata/core/index/IndexFilter.java
index 1579a5e..af752f3 100644
--- a/core/src/main/java/org/apache/carbondata/core/index/IndexFilter.java
+++ b/core/src/main/java/org/apache/carbondata/core/index/IndexFilter.java
@@ -66,6 +66,8 @@ public class IndexFilter implements Serializable {
// limit value used for row scanning, collected when carbon.mapOrderPushDown
is enabled
private int limit = -1;
+ private Set missingSISegments;
+
public IndexFilter(CarbonTable table, Expression expression) {
this(table, expression, false);
}
@@ -283,4 +285,12 @@ public class IndexFilter implements Serializable {
throw new RuntimeException("Error while resolving filter expression", e);
}
}
+
+ public void setMissingSISegments(Set missingSISegments) {
+this.missingSISegments = missingSISegments;
+ }
+
+ public Set getMissingSISegments() {
+return missingSISegments;
+ }
}
diff --git a/core/src/main/java/org/apache/carbondata/core/index/Segment.java
b/core/src/main/java/org/apache/carbondata/core/index/Segment.java
index 0a32984..4bbe543 100644
--- a/core/src/main/java/org/apache/carbondata/core/index/Segment.java
+++ b/core/src/main/java/org/apache/carbondata/core/index/Segment.java
@@ -28,6 +28,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.metadata.schema.table.Writable;
import org.apache.carbondata.core.mutate.UpdateVO;
import org.apache.carbondata.core.readcommitter.ReadCommittedScope;
@@ -96,6 +97,8 @@ public class Segment implements Serializable, Writable {
*/
private SegmentMetaDataInfo segmentMetaDataInfo;
+ private List defaultIndexPrunedBlocklets;
+
public Segment() {
}
@@ -417,4 +420,12 @@ public cla
[carbondata] branch master updated: [CARBONDATA-4172] Select query having parent and child struct column in projection returns incorrect results
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 3b411bb [CARBONDATA-4172] Select query having parent and child struct
column in projection returns incorrect results
3b411bb is described below
commit 3b411bbea81ed9c70e88a4933dfff2742d686b6e
Author: Indhumathi27
AuthorDate: Thu Apr 22 13:50:39 2021 +0530
[CARBONDATA-4172] Select query having parent and child struct column in
projection returns incorrect results
Why is this PR needed?
After PR-3574, a scenario has been missed while code refactor.
Currently, if select query has both Parent and its child struct column in
projection,
only child column is pushed down to carbon for filling result. For other
columns in parent Struct, data output is null.
What changes were proposed in this PR?
If parent struct column is also present in projection, then push down only
parent column to carbon.
This closes #4123
---
.../sql/execution/strategy/PushDownHelper.scala| 26 ---
.../complexType/TestComplexDataType.scala | 51 +++---
2 files changed, 26 insertions(+), 51 deletions(-)
diff --git
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/PushDownHelper.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/PushDownHelper.scala
index e532cd4..c8a8501 100644
---
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/PushDownHelper.scala
+++
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/PushDownHelper.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.execution.strategy
-import scala.collection.mutable.ArrayBuffer
+import scala.collection.mutable.{ArrayBuffer, ListBuffer}
import org.apache.spark.sql.catalyst.expressions.{AttributeReference,
GetArrayItem, GetMapValue, GetStructField, Literal, NamedExpression}
import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType}
@@ -35,15 +35,24 @@ object PushDownHelper {
// parent/child field and push down the corresponding project column. In
case of Array, Map,
// ArrayOfStruct, StructOfArray, MapOfStruct or StructOfMap, push down
parent column
val output = ArrayBuffer[String]()
-projects.foreach(PushDownHelper.collectColumns(_, output))
+var parentColumns = ArrayBuffer[String]()
+projects.foreach(PushDownHelper.collectColumns(_, output, parentColumns))
+parentColumns = parentColumns.distinct
if (output.isEmpty) {
requiredColumns.foreach(projection.addColumn)
} else {
requiredColumns.map(_.toLowerCase).foreach { requiredColumn =>
val childOption = output.filter(_.startsWith(requiredColumn + "."))
-childOption.isEmpty match {
- case true => projection.addColumn(requiredColumn)
- case false => childOption.foreach(projection.addColumn)
+if (childOption.isEmpty) {
+ projection.addColumn(requiredColumn)
+} else {
+ // If projection contains both parent and its child, then push down
parent column
+ // itself instead of its child column
+ if (parentColumns.contains(requiredColumn)) {
+projection.addColumn(requiredColumn)
+ } else {
+childOption.foreach(projection.addColumn)
+ }
}
}
}
@@ -51,7 +60,8 @@ object PushDownHelper {
private def collectColumns(
exp: NamedExpression,
- pushDownColumns: ArrayBuffer[String]
+ pushDownColumns: ArrayBuffer[String],
+ parentColumns: ArrayBuffer[String]
): Unit = {
exp transform {
case struct: GetStructField =>
@@ -69,7 +79,9 @@ object PushDownHelper {
pushDownColumns += getParentName(map)
Literal.TrueLiteral
case attr: AttributeReference =>
-pushDownColumns += attr.name.toLowerCase
+val attrName = attr.name.toLowerCase
+pushDownColumns += attrName
+parentColumns += attrName
Literal.TrueLiteral
}
}
diff --git
a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
index 88f5613..6c457dd 100644
---
a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
+++
b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
@@ -611,51 +611,14 @@ class TestComplexDataType extends QueryTest with
BeforeAndAfterAll {
checkAnswer(sql("select a.d,a.f from table1"),
Seq(Row(2, Row(3, "mno"
[carbondata] branch master updated: [CARBONDATA-4158]Add Secondary Index as a coarse-grain index and use secondary indexes for Presto queries
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 09ad509 [CARBONDATA-4158]Add Secondary Index as a coarse-grain index
and use secondary indexes for Presto queries
09ad509 is described below
commit 09ad509d67fbe752892c7180fb8a10cd62fef465
Author: Venu Reddy
AuthorDate: Wed Mar 10 00:23:07 2021 +0530
[CARBONDATA-4158]Add Secondary Index as a coarse-grain index and use
secondary indexes for Presto queries
Why is this PR needed?
At present, secondary indexes are leveraged for query pruning via spark
plan modification.
This approach is tightly coupled with spark because the plan modification
is specific to
spark engine. In order to use secondary indexes for Presto or Hive queries,
it is not
feasible to modify the query plans as we desire in the current approach.
Thus need arises
for an engine agnostic approach to use secondary indexes in query pruning.
What changes were proposed in this PR?
1. Add Secondary Index as a coarse grain index.
2. Add a new insegment() UDF to support query within the particular segments
3. Control the use of Secondary Index as a coarse grain index pruning with
property('carbon.coarse.grain.secondary.index')
4. Use Index Server driver for Secondary Index pruning
5. Use Secondary Indexes with Presto Queries
This closes #4110
---
README.md | 1 +
.../core/constants/CarbonCommonConstants.java | 28
.../carbondata/core/index/AbstractIndexJob.java| 10 ++
.../apache/carbondata/core/index/IndexChooser.java | 17 +-
.../apache/carbondata/core/index/IndexFilter.java | 2 +-
.../carbondata/core/index/IndexInputFormat.java| 14 +-
.../org/apache/carbondata/core/index/IndexJob.java | 2 +
.../carbondata/core/index/IndexStoreManager.java | 12 +-
.../apache/carbondata/core/index/IndexUtil.java| 12 +-
.../secondaryindex}/CarbonCostBasedOptimizer.java | 50 +-
.../carbondata/core/metadata/index/IndexType.java | 4 +-
.../metadata/schema/indextable/IndexTableInfo.java | 13 +-
.../core/scan/expression/ExpressionResult.java | 24 +--
.../core/scan/expression/LiteralExpression.java| 17 +-
.../conditional/NotEqualsExpression.java | 2 +-
.../carbondata/core/util/CarbonProperties.java | 28
.../apache/carbondata/core/util/CarbonUtil.java| 34
.../apache/carbondata/core/util/SessionParams.java | 6 +-
docs/index/secondary-index-guide.md| 29
.../carbondata/hadoop/api/CarbonInputFormat.java | 46 +-
.../org/apache/carbon/flink/TestCarbonWriter.scala | 2 +-
.../presto/CarbondataPageSourceProvider.java | 4 +
.../carbondata/presto/CarbondataSplitManager.java | 3 +
.../carbondata/presto/impl/CarbonTableReader.java | 56 +++
.../presto/CarbondataPageSourceProvider.java | 4 +
.../carbondata/presto/CarbondataSplitManager.java | 3 +
.../carbondata/presto/impl/CarbonTableReader.java | 56 +++
.../carbondata/index/secondary/SecondaryIndex.java | 125 +++
.../index/secondary/SecondaryIndexFactory.java | 172 +
.../index/secondary/SecondaryIndexModel.java | 103
.../indexserver/DistributedRDDUtils.scala | 37 -
.../indexserver/DistributedShowCacheRDD.scala | 2 +-
.../carbondata/indexserver/IndexServer.scala | 23 ++-
.../carbondata/spark/rdd/CarbonScanRDD.scala | 13 +-
.../scala/org/apache/spark/sql/CarbonEnv.scala | 3 +
.../command/index/CarbonCreateIndexCommand.scala | 4 +-
.../command/index/CarbonRefreshIndexCommand.scala | 4 +-
.../execution/strategy/CarbonDataSourceScan.scala | 6 +-
.../execution/strategy/CarbonSourceStrategy.scala | 13 +-
.../execution/command/CarbonHiveCommands.scala | 3 +-
.../apache/spark/sql/index/CarbonIndexUtil.scala | 92 ++-
.../secondaryindex/command/SICreationCommand.scala | 8 +
.../jobs/StringProjectionQueryJob.scala| 34 ++--
.../spark/sql/secondaryindex/load/Compactor.scala | 6 +
.../optimizer/CarbonSITransformationRule.scala | 13 +-
.../optimizer/CarbonSecondaryIndexOptimizer.scala | 1 +
.../secondaryindex/rdd/SecondaryIndexCreator.scala | 8 +
.../index/bloom/BloomCoarseGrainIndexSuite.scala | 4 +-
.../complexType/TestArrayContainsPushDown.scala| 12 +-
.../processing/index/IndexWriterListener.java | 6 +-
50 files changed, 1067 insertions(+), 104 deletions(-)
diff --git a/README.md b/README.md
index f706e1c..4f47546 100644
--- a/README.md
+++ b/README.md
@@ -76,6 +76,7 @@ Some features are marked as experimental because the
syntax/implementation might
2. Accelerating performance using MV on parquet/orc.
3
[carbondata] branch master updated: [CARBONDATA-4156] Fix Writing Segment Min max with all blocks of a segment
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 865ec9b [CARBONDATA-4156] Fix Writing Segment Min max with all blocks
of a segment
865ec9b is described below
commit 865ec9b908783a6169d380415aa6598c41b275d5
Author: Indhumathi27
AuthorDate: Tue Mar 9 16:09:37 2021 +0530
[CARBONDATA-4156] Fix Writing Segment Min max with all blocks of a segment
Why is this PR needed?
PR-3999 has removed some code related to getting segment min max from all
blocks.
Because of this, if segment has more than one block, currently, it is
writing
min max considering one block only.
What changes were proposed in this PR?
Reverted specific code from above PR. Removed unwanted synchronization for
some methods
This closes #4101
---
.../core/segmentmeta/SegmentMetaDataInfoStats.java | 35 --
.../carbondata/hadoop/testutil/StoreCreator.java | 3 ++
.../allqueries/TestPruneUsingSegmentMinMax.scala | 1 +
3 files changed, 30 insertions(+), 9 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/segmentmeta/SegmentMetaDataInfoStats.java
b/core/src/main/java/org/apache/carbondata/core/segmentmeta/SegmentMetaDataInfoStats.java
index 4f11eb0..9c4ce63 100644
---
a/core/src/main/java/org/apache/carbondata/core/segmentmeta/SegmentMetaDataInfoStats.java
+++
b/core/src/main/java/org/apache/carbondata/core/segmentmeta/SegmentMetaDataInfoStats.java
@@ -22,6 +22,7 @@ import java.util.LinkedHashMap;
import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.indexstore.blockletindex.BlockIndex;
import org.apache.carbondata.core.util.ByteUtil;
/**
@@ -53,7 +54,7 @@ public class SegmentMetaDataInfoStats {
* @param segmentId get corresponding segment Id from map
* @return segmentMetaDataInfo for the corresponding segment
*/
- public synchronized SegmentMetaDataInfo getTableSegmentMetaDataInfo(String
tableName,
+ public SegmentMetaDataInfo getTableSegmentMetaDataInfo(String tableName,
String segmentId) {
Map segmentColumnMetaDataInfoMap = new
LinkedHashMap<>();
Map segmentMetaDataInfoMap =
@@ -85,14 +86,30 @@ public class SegmentMetaDataInfoStats {
public synchronized void setBlockMetaDataInfo(String tableName, String
segmentId,
BlockColumnMetaDataInfo currentBlockColumnMetaInfo) {
// check if tableName is present in tableSegmentMetaDataInfoMap
-Map segmentMinMaxMap = new HashMap<>();
-if (!this.tableSegmentMetaDataInfoMap.isEmpty()
-&& null != this.tableSegmentMetaDataInfoMap.get(tableName)
-&& !this.tableSegmentMetaDataInfoMap.get(tableName).isEmpty()) {
- segmentMinMaxMap = this.tableSegmentMetaDataInfoMap.get(tableName);
+if (!this.tableSegmentMetaDataInfoMap.isEmpty() && null !=
this.tableSegmentMetaDataInfoMap
+.get(tableName) &&
!this.tableSegmentMetaDataInfoMap.get(tableName).isEmpty()
+&& null !=
this.tableSegmentMetaDataInfoMap.get(tableName).get(segmentId)) {
+ // get previous blockColumn metadata information
+ BlockColumnMetaDataInfo previousBlockColumnMetaInfo =
+ this.tableSegmentMetaDataInfoMap.get(tableName).get(segmentId);
+ // compare and get updated min and max values
+ byte[][] updatedMin =
BlockIndex.compareAndUpdateMinMax(previousBlockColumnMetaInfo.getMin(),
+ currentBlockColumnMetaInfo.getMin(), true);
+ byte[][] updatedMax =
BlockIndex.compareAndUpdateMinMax(previousBlockColumnMetaInfo.getMax(),
+ currentBlockColumnMetaInfo.getMax(), false);
+ // update the segment
+ this.tableSegmentMetaDataInfoMap.get(tableName).get(segmentId)
+ .setMinMax(updatedMin, updatedMax);
+} else {
+ Map segmentMinMaxMap = new HashMap<>();
+ if (!this.tableSegmentMetaDataInfoMap.isEmpty()
+ && null != this.tableSegmentMetaDataInfoMap.get(tableName)
+ && !this.tableSegmentMetaDataInfoMap.get(tableName).isEmpty()) {
+segmentMinMaxMap = this.tableSegmentMetaDataInfoMap.get(tableName);
+ }
+ segmentMinMaxMap.put(segmentId, currentBlockColumnMetaInfo);
+ this.tableSegmentMetaDataInfoMap.put(tableName, segmentMinMaxMap);
}
-segmentMinMaxMap.put(segmentId, currentBlockColumnMetaInfo);
-this.tableSegmentMetaDataInfoMap.put(tableName, segmentMinMaxMap);
}
/**
@@ -112,7 +129,7 @@ public class SegmentMetaDataInfoStats {
/**
* This method will do min/max comparison of values and update if required
*/
- public synchronized byte[] compareAndUpdateMinMax(byte[] minMaxValueCompare1,
+ public byte[] compareAndUpdateMinMax(byte[] minMaxValu
[carbondata] branch master updated: [CARBONDATA-4155] Fix Create table like table with MV
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 0f53bdb [CARBONDATA-4155] Fix Create table like table with MV
0f53bdb is described below
commit 0f53bdb0de7a501d6ee4bb07321599a05a0de1ee
Author: Indhumathi27
AuthorDate: Mon Mar 22 20:42:35 2021 +0530
[CARBONDATA-4155] Fix Create table like table with MV
Why is this PR needed?
PR-4076 has added a new table property to fact table.
While executing create table like command, this property
is not excluded, which leads to parsing exception.
What changes were proposed in this PR?
Remove MV related info from destination table properties
This closes #4111
---
.../command/table/CarbonCreateTableLikeCommand.scala | 4
.../carbondata/view/rewrite/TestAllOperationsOnMV.scala | 16
2 files changed, 20 insertions(+)
diff --git
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableLikeCommand.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableLikeCommand.scala
index 926b951..8921601 100644
---
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableLikeCommand.scala
+++
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableLikeCommand.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.execution.command.MetadataCommand
import
org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.schema.{SchemaEvolution,
SchemaEvolutionEntry}
import org.apache.carbondata.core.metadata.schema.table.{TableInfo,
TableSchema}
@@ -59,6 +60,9 @@ case class CarbonCreateTableLikeCommand(
dstTableSchema.setTableName(targetTable.table)
dstTableSchema.setTableId(UUID.randomUUID().toString)
+// remove mv related info from source table tblProperties
+
dstTableSchema.getTableProperties.remove(CarbonCommonConstants.RELATED_MV_TABLES_MAP)
+
val schemaEvol: SchemaEvolution = new SchemaEvolution
val schEntryList: util.List[SchemaEvolutionEntry] = new
util.ArrayList[SchemaEvolutionEntry]
schemaEvol.setSchemaEvolutionEntryList(schEntryList)
diff --git
a/integration/spark/src/test/scala/org/apache/carbondata/view/rewrite/TestAllOperationsOnMV.scala
b/integration/spark/src/test/scala/org/apache/carbondata/view/rewrite/TestAllOperationsOnMV.scala
index 167d628..251e460 100644
---
a/integration/spark/src/test/scala/org/apache/carbondata/view/rewrite/TestAllOperationsOnMV.scala
+++
b/integration/spark/src/test/scala/org/apache/carbondata/view/rewrite/TestAllOperationsOnMV.scala
@@ -708,6 +708,22 @@ class TestAllOperationsOnMV extends QueryTest with
BeforeAndAfterEach {
}.getMessage.contains("Materialized view default.does_not_exist does not
exist")
}
+ test("test create table like maintable having mv") {
+sql("drop table IF EXISTS maintable")
+sql("create table maintable(name string, c_code int, price int) STORED AS
carbondata")
+sql("drop materialized view if exists mv_table ")
+sql("create materialized view mv_table as select name, sum(price) from
maintable group by name")
+sql("drop table if exists new_Table")
+sql("create table new_Table like maintable")
+sql("insert into table new_Table select 'abc',21,2000")
+checkAnswer(sql("select * from new_Table"), Seq(Row("abc", 21, 2000)))
+intercept[MalformedCarbonCommandException] {
+ sql("create table new_Table1 like mv_table")
+}.getMessage.contains("Unsupported operation on SI table or MV.")
+sql("drop table if exists new_Table")
+sql("drop table IF EXISTS maintable")
+ }
+
test("drop meta cache on mv materialized view table") {
defaultConfig()
sql("drop table IF EXISTS maintable")
[carbondata] branch master updated: [CARBONDATA-4153] Fix DoNot Push down not equal to filter with Cast on SI
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new fd0ff22 [CARBONDATA-4153] Fix DoNot Push down not equal to filter
with Cast on SI
fd0ff22 is described below
commit fd0ff22a9d5834d34b39cfd7a781e7332802e4ad
Author: Indhumathi27
AuthorDate: Tue Mar 16 15:04:25 2021 +0530
[CARBONDATA-4153] Fix DoNot Push down not equal to filter with Cast on SI
Why is this PR needed?
NOT EQUAL TO filter on SI index column, should not be pushed down to SI
table.
Currently, where x!='2' is not pushing down to SI, but where x!=2 is pushed
down to SI.
This is because "x != 2" will be wrapped in a CAST expression like NOT
EQUAL TO(cast(x as int) = 2).
What changes were proposed in this PR?
Handle CAST case while checking DONOT PUSH DOWN to SI
This closes #4108
---
.../secondaryindex/TestSIWithSecondaryIndex.scala| 16
.../optimizer/CarbonSecondaryIndexOptimizer.scala| 2 ++
2 files changed, 18 insertions(+)
diff --git
a/index/secondary-index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSIWithSecondaryIndex.scala
b/index/secondary-index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSIWithSecondaryIndex.scala
index e0dc1ce..a5c1d34 100644
---
a/index/secondary-index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSIWithSecondaryIndex.scala
+++
b/index/secondary-index/src/test/scala/org/apache/carbondata/spark/testsuite/secondaryindex/TestSIWithSecondaryIndex.scala
@@ -653,6 +653,22 @@ class TestSIWithSecondaryIndex extends QueryTest with
BeforeAndAfterAll {
sql("drop table if exists maintable")
}
+ test("test SI with donot push down not equal to filter with Cast") {
+sql("drop table if exists maintable")
+sql("create table maintable (a string,b string,c string) STORED AS
carbondata ")
+sql("insert into maintable values ('aa', '3', 'cc')")
+sql("create index indextable on table maintable(b) AS 'carbondata'")
+val df1 = sql("select * from maintable where b!=2")
+val df2 = sql("select * from maintable where b!='2'")
+if (isFilterPushedDownToSI(df1.queryExecution.sparkPlan) &&
+isFilterPushedDownToSI(df2.queryExecution.sparkPlan)) {
+ assert(false)
+} else {
+ assert(true)
+}
+sql("drop table if exists maintable")
+ }
+
def createAndInsertDataIntoTable(): Unit = {
sql("drop table if exists maintable2")
sql("create table maintable2 (a string,b string,c int) STORED AS
carbondata ")
diff --git
a/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/optimizer/CarbonSecondaryIndexOptimizer.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/optimizer/CarbonSecondaryIndexOptimizer.scala
index df56b24..23d17d2 100644
---
a/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/optimizer/CarbonSecondaryIndexOptimizer.scala
+++
b/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/optimizer/CarbonSecondaryIndexOptimizer.scala
@@ -513,6 +513,8 @@ class CarbonSecondaryIndexOptimizer(sparkSession:
SparkSession) {
val doNotPushToSI = condition match {
case IsNotNull(child: AttributeReference) => !pushDownNotNullFilter
case Not(EqualTo(left: AttributeReference, right: Literal)) => true
+ case Not(EqualTo(left: Cast, right: Literal))
+if left.child.isInstanceOf[AttributeReference] => true
case Not(Like(left: AttributeReference, right: Literal)) => true
case Not(In(left: AttributeReference, right: Seq[Expression])) => true
case Not(Contains(left: AttributeReference, right: Literal)) => true
[carbondata] branch master updated: [CARBONDATA-4141] Index Server is not caching indexes for external tables with sdk segments
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 25c5687 [CARBONDATA-4141] Index Server is not caching indexes for
external tables with sdk segments
25c5687 is described below
commit 25c5687e2c8514f54aae4617410d3909ad8840e8
Author: Karan980
AuthorDate: Tue Mar 2 17:13:49 2021 +0530
[CARBONDATA-4141] Index Server is not caching indexes for external tables
with sdk segments
Why is this PR needed?
Indexes cached in Executor cache are not dropped when drop table is called
for external table
with SDK segments. Because, external tables with sdk segments will not have
metadata like table
status file. So in drop table command we send zero segments to indexServer
clearIndexes job,
which clears nothing from executor side. So when we drop this type of
table, executor side
indexes are not dropped. Now when we again create external table with same
location and do
select * or select count(*), it will not cache the indexes for this table,
because indexes with
same loaction are already present. Now show metacache on this newly created
table will use new tableId ,
but indexes present have the old tableId, whose table is already dropped.
So show metacache will return
nothing, because of tableId mismatch.
What changes were proposed in this PR?
Prepared the validSegments from indexFiles present at external table
location and send it to IndexServer clearIndexes job through IndexInputFormat.
This closes #4099
---
.../apache/carbondata/core/index/IndexUtil.java| 36 +-
.../blockletindex/BlockletIndexFactory.java| 8 +++--
2 files changed, 34 insertions(+), 10 deletions(-)
diff --git a/core/src/main/java/org/apache/carbondata/core/index/IndexUtil.java
b/core/src/main/java/org/apache/carbondata/core/index/IndexUtil.java
index c663b78..87d2a40 100644
--- a/core/src/main/java/org/apache/carbondata/core/index/IndexUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/index/IndexUtil.java
@@ -20,6 +20,7 @@ package org.apache.carbondata.core.index;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -36,7 +37,10 @@ import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.RelationIdentifier;
+import org.apache.carbondata.core.readcommitter.LatestFilesReadCommittedScope;
+import org.apache.carbondata.core.readcommitter.ReadCommittedScope;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.util.BlockletIndexUtil;
import org.apache.carbondata.core.util.CarbonProperties;
@@ -112,15 +116,31 @@ public class IndexUtil {
*/
private static void executeClearIndexJob(IndexJob indexJob,
CarbonTable carbonTable, String indexToClear) throws IOException {
-SegmentStatusManager.ValidAndInvalidSegmentsInfo
validAndInvalidSegmentsInfo =
-getValidAndInvalidSegments(carbonTable,
FileFactory.getConfiguration());
-List invalidSegment = new ArrayList<>();
-for (Segment segment : validAndInvalidSegmentsInfo.getInvalidSegments()) {
- invalidSegment.add(segment.getSegmentNo());
+IndexInputFormat indexInputFormat;
+if (!carbonTable.isTransactionalTable()) {
+ ReadCommittedScope readCommittedScope =
+ new LatestFilesReadCommittedScope(carbonTable.getTablePath(),
+ FileFactory.getConfiguration());
+ LoadMetadataDetails[] loadMetadataDetails =
readCommittedScope.getSegmentList();
+ List listOfValidSegments = new
ArrayList<>(loadMetadataDetails.length);
+ Arrays.stream(loadMetadataDetails).forEach(segment -> {
+Segment seg = new Segment(segment.getLoadName(),
segment.getSegmentFile());
+seg.setLoadMetadataDetails(segment);
+listOfValidSegments.add(seg);
+ });
+ indexInputFormat =
+ new IndexInputFormat(carbonTable, listOfValidSegments, new
ArrayList<>(0), true,
+ indexToClear);
+} else {
+ SegmentStatusManager.ValidAndInvalidSegmentsInfo
validAndInvalidSegmentsInfo =
+ getValidAndInvalidSegments(carbonTable,
FileFactory.getConfiguration());
+ List invalidSegment = new ArrayList<>();
+ validAndInvalidSegmentsInfo.getInvalidSegments()
+ .forEach(segment -> invalidSe
[carbondata] branch master updated: [CARBONDATA-4121] Prepriming is not working in Index Server
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 59ad77a [CARBONDATA-4121] Prepriming is not working in Index Server
59ad77a is described below
commit 59ad77a96bcbada2c62bc31cbda2acf24bfa7498
Author: Karan980
AuthorDate: Fri Feb 5 16:56:23 2021 +0530
[CARBONDATA-4121] Prepriming is not working in Index Server
Why is this PR needed?
Prepriming is not working in Index Server. Server.getRemoteUser
returns null value in async call of prepriming which results in
NPE and crashes the indexServer application. Issue Induced after PR #3952
What changes were proposed in this PR?
Computed the Server.getRemoteUser value before making the async prepriming
call and then used the same value during async call. Code reset to code
before PR #3952
This closes #4088
---
.../main/scala/org/apache/carbondata/indexserver/IndexServer.scala | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git
a/integration/spark/src/main/scala/org/apache/carbondata/indexserver/IndexServer.scala
b/integration/spark/src/main/scala/org/apache/carbondata/indexserver/IndexServer.scala
index 207fb85..089b857 100644
---
a/integration/spark/src/main/scala/org/apache/carbondata/indexserver/IndexServer.scala
+++
b/integration/spark/src/main/scala/org/apache/carbondata/indexserver/IndexServer.scala
@@ -128,6 +128,10 @@ object IndexServer extends ServerInterface {
def getCount(request: IndexInputFormat): LongWritable = {
doAs {
val sparkSession = SparkSQLUtil.getSparkSession
+ var currentUser: String = null
+ if (!request.isFallbackJob) {
+currentUser = Server.getRemoteUser.getShortUserName
+ }
lazy val getCountTask = {
if (!request.isFallbackJob) {
sparkSession.sparkContext.setLocalProperty("spark.jobGroup.id",
request.getTaskGroupId)
@@ -139,8 +143,7 @@ object IndexServer extends ServerInterface {
}
sparkSession.sparkContext.setLocalProperty("spark.job.description",
taskGroupDesc)
// Fire Generic Event like ACLCheck..etc
- val indexServerEvent = IndexServerEvent(sparkSession,
request.getCarbonTable,
-Server.getRemoteUser.getShortUserName)
+ val indexServerEvent = IndexServerEvent(sparkSession,
request.getCarbonTable, currentUser)
OperationListenerBus.getInstance().fireEvent(indexServerEvent,
operationContext)
}
val splits = new DistributedCountRDD(sparkSession, request).collect()
[carbondata] branch master updated: [CARBONDATA-4126] Concurrent compaction failed with load on table
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 5ec3536 [CARBONDATA-4126] Concurrent compaction failed with load on
table
5ec3536 is described below
commit 5ec35367684a9b8e7024d52442eae138d166440e
Author: Karan980
AuthorDate: Thu Feb 11 13:14:07 2021 +0530
[CARBONDATA-4126] Concurrent compaction failed with load on table
Why is this PR needed?
Concurrent compaction was failing when run in parallel with load.
During load we acquire SegmentLock for a particular segment, and
when this same lock we try to acquire during compaction, we were
not able to acquire this lock and compaction fails.
What changes were proposed in this PR?
Skipped compaction for segments for which we are not able to acquire
the SegmentLock instead of throwing the exception.
This closes #4093
---
.../apache/carbondata/spark/rdd/CarbonTableCompactor.scala | 13 -
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git
a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
index a381089..5db344b 100644
---
a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
+++
b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
@@ -92,19 +92,22 @@ class CarbonTableCompactor(
val lastSegment = sortedSegments.get(sortedSegments.size() - 1)
val compactedLoad = CarbonDataMergerUtil.getMergedLoadName(loadsToMerge)
var segmentLocks: ListBuffer[ICarbonLock] = ListBuffer.empty
+ val validSegments = new java.util.ArrayList[LoadMetadataDetails]
loadsToMerge.asScala.foreach { segmentId =>
val segmentLock = CarbonLockFactory
.getCarbonLockObj(carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
.getAbsoluteTableIdentifier,
CarbonTablePath.addSegmentPrefix(segmentId.getLoadName) +
LockUsage.LOCK)
-if (!segmentLock.lockWithRetries()) {
- throw new Exception(s"Failed to acquire lock on segment
${segmentId.getLoadName}," +
-s" during compaction of table
${compactionModel.carbonTable.getQualifiedName}")
+if (segmentLock.lockWithRetries()) {
+ validSegments.add(segmentId)
+ segmentLocks += segmentLock
+} else {
+ LOGGER.warn(s"Failed to acquire lock on segment
${segmentId.getLoadName}, " +
+ s"during compaction of table
${compactionModel.carbonTable.getQualifiedName}")
}
-segmentLocks += segmentLock
}
try {
-scanSegmentsAndSubmitJob(loadsToMerge, compactedSegments,
compactedLoad)
+scanSegmentsAndSubmitJob(validSegments, compactedSegments,
compactedLoad)
} catch {
case e: Exception =>
LOGGER.error(s"Exception in compaction thread ${ e.getMessage }", e)
[carbondata] branch master updated: [CARBONDATA-4082] Fix alter table add segment query on adding a segment having delete delta files
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new aa7efda [CARBONDATA-4082] Fix alter table add segment query on adding
a segment having delete delta files
aa7efda is described below
commit aa7efda9eec8485723be81455d96ba17851571e8
Author: Karan980
AuthorDate: Mon Jan 4 12:10:05 2021 +0530
[CARBONDATA-4082] Fix alter table add segment query on adding a segment
having delete delta files
Why is this PR needed?
When a segment is added to a carbon table by alter table add segment query
and that segment also have a deleteDelta file present in it, then on
querying
the carbon table the deleted rows are coming in the result.
What changes were proposed in this PR?
Updating the tableStatus and tableUpdateStatus files in correct way for the
segments having delta delta files.
This closes #4070
---
.../carbondata/core/mutate/CarbonUpdateUtil.java | 21 +++-
.../carbondata/core/util/path/CarbonTablePath.java | 2 +-
.../hadoop/api/CarbonFileInputFormat.java | 5 +-
.../hadoop/api/CarbonOutputCommitter.java | 6 +-
.../hadoop/api/CarbonTableOutputFormat.java| 7 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 2 +-
.../command/management/CarbonAddLoadCommand.scala | 126 -
.../command/mutation/DeleteExecution.scala | 2 +-
.../mutation/merge/CarbonMergeDataSetCommand.scala | 2 +-
.../testsuite/addsegment/AddSegmentTestCase.scala | 103 -
.../processing/merger/CarbonDataMergerUtil.java| 2 +-
.../processing/util/CarbonLoaderUtil.java | 23 ++--
12 files changed, 275 insertions(+), 26 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index c5296dc..e78b630 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -47,6 +47,7 @@ import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
@@ -145,9 +146,17 @@ public class CarbonUpdateUtil {
* @return
*/
public static boolean updateSegmentStatus(List
updateDetailsList,
- CarbonTable table, String updateStatusFileIdentifier, boolean
isCompaction) {
+ CarbonTable table, String updateStatusFileIdentifier, boolean
isCompaction,
+ boolean isForceWrite) {
boolean status = false;
SegmentUpdateStatusManager segmentUpdateStatusManager = new
SegmentUpdateStatusManager(table);
+if (isForceWrite && !CollectionUtils.isEmpty(updateDetailsList)) {
+ String segId = String.valueOf(SegmentStatusManager
+
.createNewSegmentId(segmentUpdateStatusManager.getLoadMetadataDetails()));
+ for (SegmentUpdateDetails detail : updateDetailsList) {
+detail.setSegmentName(segId);
+ }
+}
ICarbonLock updateLock =
segmentUpdateStatusManager.getTableUpdateStatusLock();
boolean lockStatus = false;
@@ -178,8 +187,14 @@ public class CarbonUpdateUtil {
updateDetailsValidSeg.add(updateDetails);
}
}
-segmentUpdateStatusManager
-.writeLoadDetailsIntoFile(updateDetailsValidSeg,
updateStatusFileIdentifier);
+// In case of ForceWrite, write the segmentUpdateDetails to the
tableUpdateStatus file
+// without any validation of segments.
+if (isForceWrite) {
+ segmentUpdateStatusManager.writeLoadDetailsIntoFile(oldList,
updateStatusFileIdentifier);
+} else {
+ segmentUpdateStatusManager
+ .writeLoadDetailsIntoFile(updateDetailsValidSeg,
updateStatusFileIdentifier);
+}
status = true;
} else {
LOGGER.error("Not able to acquire the segment update lock.");
diff --git
a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 458ebdf..9967723 100644
---
a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++
b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -43,7 +43,7 @@ public class CarbonTablePath {
private static final String FACT_DIR = "Fact";
public static final String SEGMENT_PREFIX = "Segment_";
private static final String PARTITION_PREFIX = "Part";
- private static final String DATA_PART_PREFIX = "p
[carbondata] branch master updated: [CARBONDATA-4113] Partition prune and cache fix when carbon.read.partition.hive.direct is disabled
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 440ab03 [CARBONDATA-4113] Partition prune and cache fix when
carbon.read.partition.hive.direct is disabled
440ab03 is described below
commit 440ab03822008509bc6547a6499049137eed5a72
Author: ShreelekhyaG
AuthorDate: Thu Jan 28 15:40:27 2021 +0530
[CARBONDATA-4113] Partition prune and cache fix when
carbon.read.partition.hive.direct is disabled
Why is this PR needed?
When carbon.read.partition.hive.direct is false then select queries on
partition table result is invalid . For a single partition, partition
values are appended to form the wrong path when loaded by the same segment.
Ex: For partition on column b, path: /tablepath/b=1/b=2
What changes were proposed in this PR?
In PartitionCacheManager, changes made to handle single and multiple
partitions.
Encoded the URI path to handle space values in the string.
This closes #4084
---
.../apache/spark/util/PartitionCacheManager.scala | 27 --
.../StandardPartitionTableLoadingTestCase.scala| 17 ++
2 files changed, 32 insertions(+), 12 deletions(-)
diff --git
a/integration/spark/src/main/scala/org/apache/spark/util/PartitionCacheManager.scala
b/integration/spark/src/main/scala/org/apache/spark/util/PartitionCacheManager.scala
index 39f33e5..2945dfa 100644
---
a/integration/spark/src/main/scala/org/apache/spark/util/PartitionCacheManager.scala
+++
b/integration/spark/src/main/scala/org/apache/spark/util/PartitionCacheManager.scala
@@ -23,6 +23,7 @@ import java.util
import scala.collection.JavaConverters._
import scala.collection.mutable
+import org.apache.commons.httpclient.util.URIUtil
import org.apache.log4j.Logger
import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat,
CatalogTablePartition}
@@ -96,7 +97,7 @@ object PartitionCacheManager extends Cache[PartitionCacheKey,
} else if (invalidSegmentMap != null && invalidSegmentMap.nonEmpty) {
CACHE.remove(identifier.tableId)
}
-finalCache.values.flatMap(_._1).toList.asJava
+finalCache.values.flatMap(_._1).toSet.toList.asJava
}
override def getAll(keys: util.List[PartitionCacheKey]):
@@ -115,18 +116,20 @@ object PartitionCacheManager extends
Cache[PartitionCacheKey,
private def readPartition(identifier: PartitionCacheKey, segmentFilePath:
String) = {
val segmentFile = SegmentFileStore.readSegmentFile(segmentFilePath)
-val partitionPath = new mutable.StringBuilder()
var partitionSpec: Map[String, String] = Map()
-segmentFile.getLocationMap.values().asScala
- .flatMap(_.getPartitions.asScala).toSet.foreach { uniquePartition:
String =>
-
partitionPath.append(CarbonCommonConstants.FILE_SEPARATOR).append(uniquePartition)
- val partitionSplit = uniquePartition.split("=")
- partitionSpec = partitionSpec. +(partitionSplit(0) -> partitionSplit(1))
-}
-Seq(CatalogTablePartition(partitionSpec,
- CatalogStorageFormat(
-Some(new URI(identifier.tablePath + partitionPath)),
-None, None, None, compressed = false, Map(
+segmentFile.getLocationMap.keySet().asScala
+ .map { uniquePartition: String =>
+val partitionSplit = uniquePartition.substring(1)
+ .split(CarbonCommonConstants.FILE_SEPARATOR)
+val storageFormat = CatalogStorageFormat(
+ Some(new URI(URIUtil.encodeQuery(identifier.tablePath +
uniquePartition))),
+ None, None, None, compressed = false, Map())
+partitionSplit.foreach(partition => {
+ val partitionArray = partition.split("=")
+ partitionSpec = partitionSpec. + (partitionArray(0) ->
partitionArray(1))
+})
+CatalogTablePartition(partitionSpec, storageFormat)
+ }.toSeq
}
override def put(key: PartitionCacheKey,
diff --git
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
index 6ab5e51..c8a0926 100644
---
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
+++
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
@@ -633,6 +633,23 @@ class StandardPartitionTableLoadingTestCase extends
QueryTest with BeforeAndAfte
CarbonProperties.getInstance().addProperty("carbon.read.partition.hive.direct",
"true")
}
+ test("test read hive partitions alternatively a
[carbondata] branch master updated: [CARBONDATA-4097] ColumnVectors should not be initialized as ColumnVectorWrapperDirect for alter tables
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new f5e35cd [CARBONDATA-4097] ColumnVectors should not be initialized as
ColumnVectorWrapperDirect for alter tables
f5e35cd is described below
commit f5e35cd039109af3bbadd2375205e3c608052027
Author: Karan980
AuthorDate: Tue Dec 22 18:51:23 2020 +0530
[CARBONDATA-4097] ColumnVectors should not be initialized as
ColumnVectorWrapperDirect for alter tables
Why is this PR needed?
Direct filling of column vectors is not allowed for alter tables,
But its column vectors were getting initialized as
ColumnVectorWrapperDirect.
What changes were proposed in this PR?
Changed the initialization of column vectors to ColumnVectorWrapper
for alter tables.
This closes #4062
---
.../scan/executor/impl/AbstractQueryExecutor.java | 4 ++--
.../core/scan/executor/util/RestructureUtil.java | 7 +--
.../scan/executor/util/RestructureUtilTest.java| 8 ++--
.../vectorreader/AddColumnTestCases.scala | 23 ++
4 files changed, 36 insertions(+), 6 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 3822f3e..72e3b31 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -445,7 +445,7 @@ public abstract class AbstractQueryExecutor implements
QueryExecutor {
blockExecutionInfo.getActualQueryDimensions(),
segmentProperties.getDimensions(),
segmentProperties.getComplexDimensions(),
blockExecutionInfo.getActualQueryMeasures().length,
-queryModel.getTable().getTableInfo().isTransactionalTable());
+queryModel.getTable().getTableInfo().isTransactionalTable(),
queryModel);
boolean isStandardTable =
CarbonUtil.isStandardCarbonTable(queryModel.getTable());
String blockId = CarbonUtil
.getBlockId(queryModel.getAbsoluteTableIdentifier(), filePath,
segment.getSegmentNo(),
@@ -461,7 +461,7 @@ public abstract class AbstractQueryExecutor implements
QueryExecutor {
List projectionMeasures = RestructureUtil
.createMeasureInfoAndGetCurrentBlockQueryMeasures(blockExecutionInfo,
blockExecutionInfo.getActualQueryMeasures(),
segmentProperties.getMeasures(),
-queryModel.getTable().getTableInfo().isTransactionalTable());
+queryModel.getTable().getTableInfo().isTransactionalTable(),
queryModel);
blockExecutionInfo.setProjectionMeasures(
projectionMeasures.toArray(new
ProjectionMeasure[projectionMeasures.size()]));
blockExecutionInfo.setDataBlock(blockIndex);
diff --git
a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
index d08ff1b..2f2cdfe 100644
---
a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
+++
b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
@@ -67,7 +67,7 @@ public class RestructureUtil {
public static List
createDimensionInfoAndGetCurrentBlockQueryDimension(
BlockExecutionInfo blockExecutionInfo, ProjectionDimension[]
queryDimensions,
List tableBlockDimensions, List
tableComplexDimension,
- int measureCount, boolean isTransactionalTable) {
+ int measureCount, boolean isTransactionalTable, QueryModel queryModel) {
List presentDimension =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
boolean[] isDimensionExists = new boolean[queryDimensions.length];
@@ -133,6 +133,7 @@ public class RestructureUtil {
if (!isDimensionExists[dimIndex]) {
defaultValues[dimIndex] =
validateAndGetDefaultValue(queryDimension.getDimension());
blockExecutionInfo.setRestructuredBlock(true);
+ queryModel.setDirectVectorFill(false);
// set the flag to say whether a new dictionary column or no
dictionary column
// has been added. This will be useful after restructure for
compaction scenarios where
// newly added columns data need to be filled
@@ -401,7 +402,8 @@ public class RestructureUtil {
*/
public static List
createMeasureInfoAndGetCurrentBlockQueryMeasures(
BlockExecutionInfo blockExecutionInfo, ProjectionMeasure[] queryMeasures,
- List currentBlockMeasures, boolean isTransactionalTable) {
+ List currentBlockMeasures, boolean isTransactionalTable,
+ QueryModel quer
[carbondata] branch master updated: [CARBONDATA-4096] SDK read fails from cluster and sdk read filter query on sort column giving wrong result with IndexServer
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 7585656 [CARBONDATA-4096] SDK read fails from cluster and sdk read
filter query on sort column giving wrong result with IndexServer
7585656 is described below
commit 7585656b565d7eb76802fafcc84a28daba89b25e
Author: ShreelekhyaG
AuthorDate: Tue Dec 22 18:44:43 2020 +0530
[CARBONDATA-4096] SDK read fails from cluster and sdk read filter query on
sort column giving wrong result with IndexServer
Why is this PR needed?
1. Create a table and read from sdk written files fails in cluster with
java.nio.file.NoSuchFileException:
hdfs:/hacluster/user/hive/warehouse/carbon.store/default/sdk.
2. After fixing the above path issue, filter query on sort column gives
the wrong result with IndexServer.
What changes were proposed in this PR?
1. In getAllDeleteDeltaFiles , used CarbonFiles.listFiles instead of
Files.walk
to handle custom file types.
2. In PruneWithFilter , isResolvedOnSegment is used in filterResolver step.
Have set table and expression on executor side, so indexserver can use this
in filterResolver step.
This closes #4064
---
.../carbondata/core/index/IndexInputFormat.java| 8 ++--
.../hadoop/api/CarbonFileInputFormat.java | 23 --
2 files changed, 19 insertions(+), 12 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/index/IndexInputFormat.java
b/core/src/main/java/org/apache/carbondata/core/index/IndexInputFormat.java
index dbb5b4f..072dbbc 100644
--- a/core/src/main/java/org/apache/carbondata/core/index/IndexInputFormat.java
+++ b/core/src/main/java/org/apache/carbondata/core/index/IndexInputFormat.java
@@ -154,8 +154,12 @@ public class IndexInputFormat extends
FileInputFormat
if (indexLevel == null) {
TableIndex defaultIndex = IndexStoreManager.getInstance()
.getIndex(table,
distributable.getDistributable().getIndexSchema());
- blocklets = defaultIndex
- .prune(segmentsToLoad, new IndexFilter(filterResolverIntf),
partitions);
+ IndexFilter filter = new IndexFilter(filterResolverIntf);
+ filter.setTable(table);
+ if (filterResolverIntf != null) {
+filter.setExpression(filterResolverIntf.getFilterExpression());
+ }
+ blocklets = defaultIndex.prune(segmentsToLoad, filter, partitions);
blocklets = IndexUtil
.pruneIndexes(table, filterResolverIntf, segmentsToLoad,
partitions, blocklets,
indexChooser);
diff --git
a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonFileInputFormat.java
b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonFileInputFormat.java
index 2a655e8..91116b4 100644
---
a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonFileInputFormat.java
+++
b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonFileInputFormat.java
@@ -20,21 +20,17 @@ package org.apache.carbondata.hadoop.api;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.annotations.InterfaceStability;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.index.IndexFilter;
import org.apache.carbondata.core.index.Segment;
@@ -265,11 +261,18 @@ public class CarbonFileInputFormat extends
CarbonInputFormat implements Se
}
private List getAllDeleteDeltaFiles(String path) {
-List deltaFiles = null;
-try (Stream walk = Files.walk(Paths.get(path))) {
- deltaFiles = walk.map(x -> x.toString())
- .filter(f -> f.endsWith(CarbonCommonConstants.DELETE_DELTA_FILE_EXT))
- .collect(Collectors.toList());
+List deltaFiles = new ArrayList<>();
+try {
+ FileFactory.getCarbonFile(path).listFiles(true, new CarbonFileFilter() {
+@Override
+public boolean accept(CarbonFile file) {
+ if
(file.getName().endsWith(CarbonCommonConstants.DELETE_DELTA_FILE_EXT)) {
+deltaFiles.add(file.getAbsolutePath());
+return true;
+ }
+
[carbondata] branch master updated: [CARBONDATA-4055]Fix creation of empty segment directory and meta entry when there is no update/insert data
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new aa2121e [CARBONDATA-4055]Fix creation of empty segment directory and
meta entry when there is no update/insert data
aa2121e is described below
commit aa2121e761f620e60470f5f856bc29141777f97a
Author: akashrn5
AuthorDate: Mon Nov 23 19:18:06 2020 +0530
[CARBONDATA-4055]Fix creation of empty segment directory and meta
entry when there is no update/insert data
Why is this PR needed?
1. After #3999 when an update happens on the table, a new segment
is created for updated data. But when there is no data to update,
still the segments are created and the table status has in progress
entries for those empty segments. This leads to unnecessary segment
dirs and an increase in table status entries.
2. after this, clean files don't clean these empty segments.
3. when the source table do not have data, CTAS will result in same
problem mentioned.
What changes were proposed in this PR?
when the data is not present during update, make the segment as marked
for delete so that the clean files take care to delete the segment,
for cats already handled, added test cases.
This closes #4018
---
.../org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala | 3 ++-
.../spark/testsuite/createTable/TestCreateTableAsSelect.scala | 10 ++
.../spark/testsuite/iud/UpdateCarbonTableTestCase.scala| 8 +++-
3 files changed, 19 insertions(+), 2 deletions(-)
diff --git
a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 09059b1..f62aa64 100644
---
a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++
b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -315,7 +315,8 @@ object CarbonDataRDDFactory {
try {
if (!carbonLoadModel.isCarbonTransactionalTable ||
segmentLock.lockWithRetries()) {
if (updateModel.isDefined && dataFrame.get.rdd.isEmpty()) {
- // if the rowToBeUpdated is empty, do nothing
+ // if the rowToBeUpdated is empty, mark created segment as marked
for delete and return
+ CarbonLoaderUtil.updateTableStatusForFailure(carbonLoadModel, "")
} else {
status = if (scanResultRdd.isDefined) {
val colSchema = carbonLoadModel
diff --git
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableAsSelect.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableAsSelect.scala
index 8384939..0bd8f73 100644
---
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableAsSelect.scala
+++
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableAsSelect.scala
@@ -27,7 +27,9 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.filesystem.CarbonFile
import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
/**
* test functionality for create table as select command
@@ -209,6 +211,7 @@ class TestCreateTableAsSelect extends QueryTest with
BeforeAndAfterAll {
test("test create table as select with " +
"where clause in select from hive/orc table that does not return data")
{
+
CarbonProperties.getInstance().addProperty("carbon.clean.file.force.allowed",
"true")
sql("DROP TABLE IF EXISTS ctas_select_where_orc")
sql(
"""
@@ -216,8 +219,15 @@ class TestCreateTableAsSelect extends QueryTest with
BeforeAndAfterAll {
| STORED AS carbondata
| AS SELECT * FROM orc_ctas_test
| where key=300""".stripMargin)
+val carbonTable = CarbonEnv.getCarbonTable(Some("default"),
"ctas_select_where_orc")(sqlContext
+ .sparkSession)
+val segmentPath =
FileFactory.getCarbonFile(CarbonTablePath.getSegmentPath(carbonTable
+ .getTablePath, "0"))
+sql("clean files for table ctas_select_where_orc options('force' =
'true')")
+assert(!segmentPath.exists())
checkAnswer(sql("SELECT * FROM ctas_select_where_orc"),
sql("SELECT * FROM orc_ctas_test where key=300"))
+
CarbonProperti
[carbondata] branch master updated: [CARBONDATA-4080] Wrong results for select count on invalid segments
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 35c73e6 [CARBONDATA-4080] Wrong results for select count on invalid
segments
35c73e6 is described below
commit 35c73e61c55f39f1bf38105e6bf32ff3673396fb
Author: akkio-97
AuthorDate: Wed Dec 9 15:15:51 2020 +0530
[CARBONDATA-4080] Wrong results for select count on invalid segments
Why is this PR needed?
Wrong results for
1. select count on marked for delete segment
2. select count on compacted segment
Issue comes only when the user explicitly sets deleted/compacted
segments using the property carbon.input.segments.
As select * on such segments gives 0 rows as output, in order to
maintain consistency, select count should also give 0 rows.
What changes were proposed in this PR?
So this PR focuses on considering only valid segments to be accessed
during select count query.
This closes #4050
---
.../hadoop/api/CarbonTableInputFormat.java | 15 ++-
.../segmentreading/TestSegmentReading.scala| 104 +
2 files changed, 117 insertions(+), 2 deletions(-)
diff --git
a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index a2c162e..f24a2f1 100644
---
a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++
b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -55,6 +55,7 @@ import
org.apache.carbondata.core.readcommitter.TableStatusReadCommittedScope;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.core.statusmanager.FileFormat;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
import org.apache.carbondata.core.statusmanager.StageInputCollector;
@@ -220,7 +221,7 @@ public class CarbonTableInputFormat extends
CarbonInputFormat {
* `INPUT_SEGMENT_NUMBERS` in job configuration
*/
private List getFilteredSegment(JobContext job, List
validSegments,
- boolean validationRequired, ReadCommittedScope readCommittedScope) {
+ boolean validationRequired, ReadCommittedScope readCommittedScope)
throws IOException {
Segment[] segmentsToAccess = getSegmentsToAccess(job, readCommittedScope);
if (segmentsToAccess.length == 0 ||
segmentsToAccess[0].getSegmentNo().equalsIgnoreCase("*")) {
return validSegments;
@@ -241,7 +242,17 @@ public class CarbonTableInputFormat extends
CarbonInputFormat {
}
if (!validationRequired && filteredSegmentToAccess.size() !=
segmentToAccessMap.size()) {
for (Segment segment : segmentToAccessMap.values()) {
-if (!filteredSegmentToAccess.containsKey(segment.getSegmentNo())) {
+boolean isSegmentValid = true;
+LoadMetadataDetails[] segmentList =
readCommittedScope.getSegmentList();
+for (LoadMetadataDetails validSegment : segmentList) {
+ if (validSegment.getLoadName().equals(segment.getSegmentNo()) && (
+
validSegment.getSegmentStatus().equals(SegmentStatus.MARKED_FOR_DELETE)
+ ||
validSegment.getSegmentStatus().equals(SegmentStatus.COMPACTED))) {
+isSegmentValid = false;
+break;
+ }
+}
+if (isSegmentValid &&
!filteredSegmentToAccess.containsKey(segment.getSegmentNo())) {
filteredSegmentToAccess.put(segment.getSegmentNo(), segment);
}
}
diff --git
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/segmentreading/TestSegmentReading.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/segmentreading/TestSegmentReading.scala
index 8ce8cc5..5c0336c 100644
---
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/segmentreading/TestSegmentReading.scala
+++
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/segmentreading/TestSegmentReading.scala
@@ -22,6 +22,8 @@ import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
import
org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.view.rewrite.TestUtil
/**
* Created by rahul on 19/9/17.
@@ -420,4 +422,106 @@ class TestSegmentReading extends QueryTest with
BeforeAndAfterAll {
sql("set spark.sql.adaptive.enabled=false")
}
+
+ test(&
[carbondata] branch master updated: [CARBONDATA-4077] Refactor and Fix Insert into partition issue with FileMergeSortComparator
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 29ecd5f [CARBONDATA-4077] Refactor and Fix Insert into partition
issue with FileMergeSortComparator
29ecd5f is described below
commit 29ecd5fee7af97633b756bfd944251a18dc76c18
Author: Indhumathi27
AuthorDate: Sat Dec 5 19:35:58 2020 +0530
[CARBONDATA-4077] Refactor and Fix Insert into partition issue with
FileMergeSortComparator
Why is this PR needed?
From PR-3995 changes, insert into partition flow scenario is missed. Using
Map for getting
Dict/No-Dict sort column info during final sort task will affect load
performance,
if number of sort columns is more.
What changes were proposed in this PR?
Handled the insert into partition flow
Refactored the code, to use list of only Dict/No-Dict sort column indexes
instead of
Map to fix performance issue.
This closes #4039
---
.../query/SecondaryIndexQueryResultProcessor.java | 2 -
.../unsafe/holder/UnsafeFinalMergePageHolder.java | 6 +-
.../sort/unsafe/holder/UnsafeInmemoryHolder.java | 6 +-
.../holder/UnsafeSortTempFileChunkHolder.java | 6 +-
.../merger/CompactionResultSortProcessor.java | 11 +-
.../sort/sortdata/FileMergeSortComparator.java | 100
.../processing/sort/sortdata/SortParameters.java | 69 +++
.../sort/sortdata/SortTempFileChunkHolder.java | 12 +-
.../processing/sort/sortdata/TableFieldStat.java | 27 +++--
.../processing/util/CarbonDataProcessorUtil.java | 127 ++---
.../sort/sortdata/FileMergeSortComparatorTest.java | 27 ++---
11 files changed, 239 insertions(+), 154 deletions(-)
diff --git
a/integration/spark/src/main/java/org/apache/spark/sql/secondaryindex/query/SecondaryIndexQueryResultProcessor.java
b/integration/spark/src/main/java/org/apache/spark/sql/secondaryindex/query/SecondaryIndexQueryResultProcessor.java
index 4d045f0..41a5c43 100644
---
a/integration/spark/src/main/java/org/apache/spark/sql/secondaryindex/query/SecondaryIndexQueryResultProcessor.java
+++
b/integration/spark/src/main/java/org/apache/spark/sql/secondaryindex/query/SecondaryIndexQueryResultProcessor.java
@@ -528,8 +528,6 @@ public class SecondaryIndexQueryResultProcessor {
CarbonCommonConstants.FILE_SEPARATOR,
CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
sortParameters.setNoDictionarySortColumn(
CarbonDataProcessorUtil.getNoDictSortColMapping(indexTable));
-sortParameters.setSortColumnSchemaOrderMap(
-CarbonDataProcessorUtil.getSortColSchemaOrderMapping(indexTable));
finalMerger = new SingleThreadFinalSortFilesMerger(sortTempFileLocation,
indexTable.getTableName(), sortParameters);
}
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
index 717bb91..7abd3a9 100644
---
a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
+++
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
@@ -61,10 +61,10 @@ public class UnsafeFinalMergePageHolder implements
SortTempChunkHolder {
}
this.noDictDataType = rowPages[0].getTableFieldStat().getNoDictDataType();
LOGGER.info("Processing unsafe inmemory rows page with size : " +
actualSize);
-this.comparator = new
FileMergeSortComparator(tableFieldStat.getIsSortColNoDictFlags(),
-tableFieldStat.getNoDictSchemaDataType(),
+this.comparator = new
FileMergeSortComparator(tableFieldStat.getNoDictSchemaDataType(),
tableFieldStat.getNoDictSortColumnSchemaOrderMapping(),
-tableFieldStat.getSortColSchemaOrderMap());
+tableFieldStat.getNoDictSortColIdxSchemaOrderMapping(),
+tableFieldStat.getDictSortColIdxSchemaOrderMapping());
}
public boolean hasNext() {
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
index a46811f..a21e802 100644
---
a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
+++
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
@@ -48,10 +48,10 @@ public class UnsafeInmemoryHolder implements
SortTempChunkHolder {
this.rowPage = rowPage;
LOGGER.info("Processing unsafe inmemory rows page with size : " +
actualSize);
[carbondata] branch master updated: [CARBONDATA-4050]Avoid redundant RPC calls to get file status when CarbonFile is instantiated with fileStatus constructor
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new c5f464f [CARBONDATA-4050]Avoid redundant RPC calls to get file status
when CarbonFile is instantiated with fileStatus constructor
c5f464f is described below
commit c5f464fab7117000aa76150c30a616ca02b5d443
Author: Venu Reddy
AuthorDate: Thu Nov 19 00:33:20 2020 +0530
[CARBONDATA-4050]Avoid redundant RPC calls to get file status when
CarbonFile is
instantiated with fileStatus constructor
Why is this PR needed?
In createCarbonDataFileBlockMetaInfoMapping method, we get list of
carbondata
files in the segment, loop through all the carbon files and make a map of
fileNameToMetaInfoMapping
In that carbon files loop, if the file is of AbstractDFSCarbonFile type, we
get
the org.apache.hadoop.fs.FileStatus thrice for each file. And the method to
get
file status is an RPC call(fileSystem.getFileStatus(path)). It takes ~2ms
in the
cluster for each call. Thus, incur an overhead of ~6ms per file. So overall
driver
side query processing time has increased significantly when there are more
carbon
files. Hence caused TPC-DS queries performance degradation.
What changes were proposed in this PR?
Avoided redundant RPC calls to get file status in getAbsolutePath(),
getSize() and
getLocations() methods when CarbonFile is instantiated with FileStatus
constructor
This closes #4010
---
.../core/datastore/filesystem/AbstractDFSCarbonFile.java | 11 ++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
index 338076e..25337c7 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
@@ -107,6 +107,9 @@ public abstract class AbstractDFSCarbonFile implements
CarbonFile {
@Override
public String getAbsolutePath() {
try {
+ if (fileStatus != null) {
+return fileStatus.getPath().toString();
+ }
return fileSystem.getFileStatus(path).getPath().toString();
} catch (IOException e) {
throw new CarbonFileException("Unable to get file status: ", e);
@@ -155,6 +158,9 @@ public abstract class AbstractDFSCarbonFile implements
CarbonFile {
@Override
public long getSize() {
try {
+ if (fileStatus != null) {
+return fileStatus.getLen();
+ }
return fileSystem.getFileStatus(path).getLen();
} catch (IOException e) {
throw new CarbonFileException("Unable to get file status for " +
path.toString(), e);
@@ -541,7 +547,10 @@ public abstract class AbstractDFSCarbonFile implements
CarbonFile {
@Override
public String[] getLocations() throws IOException {
BlockLocation[] blkLocations;
-FileStatus fileStatus = fileSystem.getFileStatus(path);
+FileStatus fileStatus = this.fileStatus;
+if (fileStatus == null) {
+ fileStatus = fileSystem.getFileStatus(path);
+}
if (fileStatus instanceof LocatedFileStatus) {
blkLocations = ((LocatedFileStatus) fileStatus).getBlockLocations();
} else {
[carbondata] branch master updated: [CARBONDATA-4046] Handled multiple partition columns for partition cache
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new b379ebe [CARBONDATA-4046] Handled multiple partition columns for
partition cache
b379ebe is described below
commit b379ebec3cc4f251f784f704cefff345ef17c032
Author: Nihal ojha
AuthorDate: Tue Nov 3 10:47:02 2020 +0530
[CARBONDATA-4046] Handled multiple partition columns for partition cache
Why is this PR needed?
1. Currently when property carbon.read.partition.hive.direct is false then
select count * fails on table which contains multiple partition columns.
2. Subtraction of the different data types.
3. If the final cache is empty and the invalid segment list is non-empty
then clear the cache.
What changes were proposed in this PR?
1. Handled multiple partition columns.
2. Handled subtraction of the different data types.
3. If the final cache is empty and the invalid segment list is non-empty
then clear the cache.
This closes #4002
---
.../apache/spark/util/PartitionCacheManager.scala | 23 ++
.../StandardPartitionTableLoadingTestCase.scala| 15 ++
2 files changed, 30 insertions(+), 8 deletions(-)
diff --git
a/integration/spark/src/main/scala/org/apache/spark/util/PartitionCacheManager.scala
b/integration/spark/src/main/scala/org/apache/spark/util/PartitionCacheManager.scala
index 411cbe2..39f33e5 100644
---
a/integration/spark/src/main/scala/org/apache/spark/util/PartitionCacheManager.scala
+++
b/integration/spark/src/main/scala/org/apache/spark/util/PartitionCacheManager.scala
@@ -79,9 +79,10 @@ object PartitionCacheManager extends Cache[PartitionCacheKey,
segmentFilePath.getAbsolutePath), segmentFileModifiedTime))
}
}.toMap
+val invalidSegmentMap = validInvalidSegments.getInvalidSegments.asScala
+ .map(seg => (seg.getSegmentNo, seg)).toMap
// remove all invalid segment entries from cache
-val finalCache = cacheablePartitionSpecs --
-
validInvalidSegments.getInvalidSegments.asScala.map(_.getSegmentNo)
+val finalCache = cacheablePartitionSpecs -- invalidSegmentMap.keySet
val cacheObject = CacheablePartitionSpec(finalCache)
if (finalCache.nonEmpty) {
// remove the existing cache as new cache values may be added.
@@ -92,6 +93,8 @@ object PartitionCacheManager extends Cache[PartitionCacheKey,
cacheObject,
cacheObject.getMemorySize,
identifier.expirationTime)
+} else if (invalidSegmentMap != null && invalidSegmentMap.nonEmpty) {
+ CACHE.remove(identifier.tableId)
}
finalCache.values.flatMap(_._1).toList.asJava
}
@@ -112,14 +115,18 @@ object PartitionCacheManager extends
Cache[PartitionCacheKey,
private def readPartition(identifier: PartitionCacheKey, segmentFilePath:
String) = {
val segmentFile = SegmentFileStore.readSegmentFile(segmentFilePath)
+val partitionPath = new mutable.StringBuilder()
+var partitionSpec: Map[String, String] = Map()
segmentFile.getLocationMap.values().asScala
- .flatMap(_.getPartitions.asScala).toSet.map { uniquePartition: String =>
+ .flatMap(_.getPartitions.asScala).toSet.foreach { uniquePartition:
String =>
+
partitionPath.append(CarbonCommonConstants.FILE_SEPARATOR).append(uniquePartition)
val partitionSplit = uniquePartition.split("=")
- val storageFormat = CatalogStorageFormat(
-Some(new URI(identifier.tablePath + "/" + uniquePartition)),
-None, None, None, compressed = false, Map())
- CatalogTablePartition(Map(partitionSplit(0) -> partitionSplit(1)),
storageFormat)
-}.toSeq
+ partitionSpec = partitionSpec. +(partitionSplit(0) -> partitionSplit(1))
+}
+Seq(CatalogTablePartition(partitionSpec,
+ CatalogStorageFormat(
+Some(new URI(identifier.tablePath + partitionPath)),
+None, None, None, compressed = false, Map(
}
override def put(key: PartitionCacheKey,
diff --git
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
index 128274e..6ab5e51 100644
---
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
+++
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
@@ -618,6 +618,21 @@ class StandardPartitionTableLoadingTestCase extends
QueryTest with BeforeAndAfte
CarbonProperties.getInstance().addProperty("carbon.read.parti
[carbondata] branch master updated: [CARBONDATA-4022] Fix invalid path issue for segment added through alter add segment query
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new e91a645 [CARBONDATA-4022] Fix invalid path issue for segment added
through alter add segment query
e91a645 is described below
commit e91a645f53793f54a8f8b6e1a7256dc7c29c7a09
Author: Karan980
AuthorDate: Fri Nov 20 17:11:16 2020 +0530
[CARBONDATA-4022] Fix invalid path issue for segment added through alter
add segment query
Why is this PR needed?
Segment Added through alter table add segment query doesn't require
tablepath in its
filepath, because the segment is present at some external location which is
not inside
the table.
What changes were proposed in this PR?
Filepath for normal segments (which are not added through alter table add
segment query)
is tablePath + filePath. But for segments added through alter table add
segment query
doesn't require tablePath, so we directly use the filePath of the segment.
This closes #4017
---
.../org/apache/carbondata/core/indexstore/ExtendedBlocklet.java | 8 +++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
b/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
index edb921b..e0b21d5 100644
---
a/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
+++
b/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
@@ -20,6 +20,7 @@ package org.apache.carbondata.core.indexstore;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.DataOutputStream;
+import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
@@ -219,7 +220,12 @@ public class ExtendedBlocklet extends Blocklet {
if (in.readBoolean()) {
indexUniqueId = in.readUTF();
}
-setFilePath(tablePath + getPath());
+String filePath = getPath();
+if (filePath.startsWith(File.separator)) {
+ setFilePath(tablePath + filePath);
+} else {
+ setFilePath(filePath);
+}
boolean isSplitPresent = in.readBoolean();
if (isSplitPresent) {
// getting the length of the data
svn commit: r44383 - in /release/carbondata: ./ 2.1.0/
Author: kunalkapoor Date: Sun Nov 8 16:35:05 2020 New Revision: 44383 Log: Upload 2.1.0 release Added: release/carbondata/2.1.0/ release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.3.4-hadoop2.7.2.jar (with props) release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.3.4-hadoop2.7.2.jar.asc (with props) release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.3.4-hadoop2.7.2.jar.sha512 release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.4.5-hadoop2.7.2.jar (with props) release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.4.5-hadoop2.7.2.jar.asc (with props) release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.4.5-hadoop2.7.2.jar.sha512 release/carbondata/2.1.0/apache-carbondata-2.1.0-source-release.zip (with props) release/carbondata/2.1.0/apache-carbondata-2.1.0-source-release.zip.asc (with props) release/carbondata/2.1.0/apache-carbondata-2.1.0-source-release.zip.md5 release/carbondata/2.1.0/apache-carbondata-2.1.0-source-release.zip.sha512 Modified: release/carbondata/KEYS Added: release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.3.4-hadoop2.7.2.jar == Binary file - no diff available. Propchange: release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.3.4-hadoop2.7.2.jar -- svn:mime-type = application/java-archive Added: release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.3.4-hadoop2.7.2.jar.asc == Binary file - no diff available. Propchange: release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.3.4-hadoop2.7.2.jar.asc -- svn:mime-type = application/pgp-signature Added: release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.3.4-hadoop2.7.2.jar.sha512 == --- release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.3.4-hadoop2.7.2.jar.sha512 (added) +++ release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.3.4-hadoop2.7.2.jar.sha512 Sun Nov 8 16:35:05 2020 @@ -0,0 +1 @@ +1dee1960178f947156cae8cb66c6e66b2ae594906b16668a6532ca8913af6fc104898d344359b7e3f2d6cad445b2d49cb244499c9b1810c4be7f6a6043a1022c apache-carbondata-2.1.0-bin-spark2.3.4-hadoop2.7.2.jar Added: release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.4.5-hadoop2.7.2.jar == Binary file - no diff available. Propchange: release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.4.5-hadoop2.7.2.jar -- svn:mime-type = application/java-archive Added: release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.4.5-hadoop2.7.2.jar.asc == Binary file - no diff available. Propchange: release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.4.5-hadoop2.7.2.jar.asc -- svn:mime-type = application/pgp-signature Added: release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.4.5-hadoop2.7.2.jar.sha512 == --- release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.4.5-hadoop2.7.2.jar.sha512 (added) +++ release/carbondata/2.1.0/apache-carbondata-2.1.0-bin-spark2.4.5-hadoop2.7.2.jar.sha512 Sun Nov 8 16:35:05 2020 @@ -0,0 +1 @@ +7ca85fb6bfd4572f65cd3cba0cb99ed9f80dc11ec04ba6d9c2a345adc8c69d1521b77a207b265df2a443d0ccb94d411ca9b2aefb54d651e8fad5aa5f80f5f2f2 apache-carbondata-2.1.0-bin-spark2.4.5-hadoop2.7.2.jar Added: release/carbondata/2.1.0/apache-carbondata-2.1.0-source-release.zip == Binary file - no diff available. Propchange: release/carbondata/2.1.0/apache-carbondata-2.1.0-source-release.zip -- svn:mime-type = application/zip Added: release/carbondata/2.1.0/apache-carbondata-2.1.0-source-release.zip.asc == Binary file - no diff available. Propchange: release/carbondata/2.1.0/apache-carbondata-2.1.0-source-release.zip.asc -- svn:mime-type = application/pgp-signature Added: release/carbondata/2.1.0/apache-carbondata-2.1.0-source-release.zip.md5 == --- release/carbondata/2.1.0/apache-carbondata-2.1.0-source
svn commit: r42277 - in /dev/carbondata/2.1.0-rc2: ./ apache-carbondata-2.1.0-source-release.zip apache-carbondata-2.1.0-source-release.zip.asc apache-carbondata-2.1.0-source-release.zip.md5 apache-ca
Author: kunalkapoor Date: Tue Nov 3 21:37:14 2020 New Revision: 42277 Log: Upload 2.1.0-rc2 Added: dev/carbondata/2.1.0-rc2/ dev/carbondata/2.1.0-rc2/apache-carbondata-2.1.0-source-release.zip (with props) dev/carbondata/2.1.0-rc2/apache-carbondata-2.1.0-source-release.zip.asc (with props) dev/carbondata/2.1.0-rc2/apache-carbondata-2.1.0-source-release.zip.md5 dev/carbondata/2.1.0-rc2/apache-carbondata-2.1.0-source-release.zip.sha512 Added: dev/carbondata/2.1.0-rc2/apache-carbondata-2.1.0-source-release.zip == Binary file - no diff available. Propchange: dev/carbondata/2.1.0-rc2/apache-carbondata-2.1.0-source-release.zip -- svn:mime-type = application/zip Added: dev/carbondata/2.1.0-rc2/apache-carbondata-2.1.0-source-release.zip.asc == Binary file - no diff available. Propchange: dev/carbondata/2.1.0-rc2/apache-carbondata-2.1.0-source-release.zip.asc -- svn:mime-type = application/pgp-signature Added: dev/carbondata/2.1.0-rc2/apache-carbondata-2.1.0-source-release.zip.md5 == --- dev/carbondata/2.1.0-rc2/apache-carbondata-2.1.0-source-release.zip.md5 (added) +++ dev/carbondata/2.1.0-rc2/apache-carbondata-2.1.0-source-release.zip.md5 Tue Nov 3 21:37:14 2020 @@ -0,0 +1 @@ +e3f64d72b204360ab753c2e961624f35 apache-carbondata-2.1.0-source-release.zip Added: dev/carbondata/2.1.0-rc2/apache-carbondata-2.1.0-source-release.zip.sha512 == --- dev/carbondata/2.1.0-rc2/apache-carbondata-2.1.0-source-release.zip.sha512 (added) +++ dev/carbondata/2.1.0-rc2/apache-carbondata-2.1.0-source-release.zip.sha512 Tue Nov 3 21:37:14 2020 @@ -0,0 +1 @@ +a1f23b309059c3dc75c8b562a4d27011d65ebda9094a9f0f5a91eb4153feacb6ad36536ff6ab9840f93a4af1d5d8d92936edf9a377b52f34c5fe130110c6da10 apache-carbondata-2.1.0-source-release.zip
