This is an automated email from the ASF dual-hosted git repository.
indhumuthumurugesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 1cab165 [CARBONDATA-3962] Fixed concurrent load failure with flat
folder structure.
1cab165 is described below
commit 1cab165deb94a8ea1b2009442de92fb7b6bf05f3
Author: Nihal ojha <[email protected]>
AuthorDate: Tue Feb 16 13:01:34 2021 +0530
[CARBONDATA-3962] Fixed concurrent load failure with flat folder structure.
Why is this PR needed?
PR #3904 has added the code to remove fact directory and because of this
concurrent
load fails with file not found exception.
What changes were proposed in this PR?
Reverted PR 3904.
This closes #4905
---
.../org/apache/carbondata/core/metadata/SegmentFileStore.java | 10 +---------
.../testsuite/flatfolder/FlatFolderTableLoadingTestCase.scala | 8 +++-----
2 files changed, 4 insertions(+), 14 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
index 9e8be0a..0a26b5e 100644
---
a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
+++
b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
@@ -443,22 +443,14 @@ public class SegmentFileStore {
/**
* Move the loaded data from source folder to destination folder.
*/
- private static void moveFromTempFolder(String source, String dest) throws
IOException {
+ private static void moveFromTempFolder(String source, String dest) {
CarbonFile oldFolder = FileFactory.getCarbonFile(source);
CarbonFile[] oldFiles = oldFolder.listFiles();
for (CarbonFile file : oldFiles) {
file.renameForce(dest + CarbonCommonConstants.FILE_SEPARATOR +
file.getName());
}
- // delete the segment path at any cost at first, we we dont want to delete
fact directory in
- // case of multiple load scenario or update, delete scenario
oldFolder.delete();
- CarbonFile partDir =
FileFactory.getCarbonFile(CarbonTablePath.getPartitionDir(dest));
- // once last segment is processed(in case of update delete), delete the
main fact directory
- if (partDir.listFiles(false).size() == 0) {
- CarbonFile oldFactDirPath =
FileFactory.getCarbonFile(CarbonTablePath.getFactDir(dest));
- oldFactDirPath.delete();
- }
}
/**
diff --git
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/flatfolder/FlatFolderTableLoadingTestCase.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/flatfolder/FlatFolderTableLoadingTestCase.scala
index c6306c0..cde2f88 100644
---
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/flatfolder/FlatFolderTableLoadingTestCase.scala
+++
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/flatfolder/FlatFolderTableLoadingTestCase.scala
@@ -46,12 +46,10 @@ class FlatFolderTableLoadingTestCase extends QueryTest with
BeforeAndAfterAll {
}
- def validateDataFiles(tableUniqueName: String): Unit = {
+ def validateDataFiles(tableUniqueName: String, segmentId: String): Unit = {
val carbonTable =
CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
val files = FileFactory.getCarbonFile(carbonTable.getTablePath).listFiles()
- val factPath =
FileFactory.getCarbonFile(CarbonTablePath.getFactDir(carbonTable.getTablePath))
assert(files.exists(_.getName.endsWith(CarbonTablePath.CARBON_DATA_EXT)))
- assert(!factPath.exists())
}
test("data loading for flat folder with global sort") {
@@ -65,7 +63,7 @@ class FlatFolderTableLoadingTestCase extends QueryTest with
BeforeAndAfterAll {
""".stripMargin)
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE
flatfolder_gs OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
- validateDataFiles("default_flatfolder_gs")
+ validateDataFiles("default_flatfolder_gs", "0")
checkAnswer(sql("select empno, empname, designation, doj,
workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode,
projectjoindate, projectenddate, attendance, utilization, salary from
flatfolder_gs order by empno"),
sql("select empno, empname, designation, doj, workgroupcategory,
workgroupcategoryname, deptno, deptname, projectcode, projectjoindate,
projectenddate, attendance, utilization, salary from originTable order by
empno"))
@@ -83,7 +81,7 @@ class FlatFolderTableLoadingTestCase extends QueryTest with
BeforeAndAfterAll {
""".stripMargin)
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE
flatfolder OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
- validateDataFiles("default_flatfolder")
+ validateDataFiles("default_flatfolder", "0")
checkAnswer(sql("select empno, empname, designation, doj,
workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode,
projectjoindate, projectenddate, attendance, utilization, salary from
flatfolder order by empno"),
sql("select empno, empname, designation, doj, workgroupcategory,
workgroupcategoryname, deptno, deptname, projectcode, projectjoindate,
projectenddate, attendance, utilization, salary from originTable order by
empno"))
