Repository: carbondata Updated Branches: refs/heads/master b853963b2 -> bea277f83
[CARBONDATA-2621][BloomDataMap] Lock problem in index datamap Problem The locking for the index Datamap is not correct. The HDFS lock is not working properly, because the lock is getting created the the local filesystem instead of HDFS. Solution Corrected the system folder path. This closes #2387 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/bea277f8 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/bea277f8 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/bea277f8 Branch: refs/heads/master Commit: bea277f83001c6d3f0cc3702bd35fa90435dd400 Parents: b853963 Author: mohammadshahidkhan <[email protected]> Authored: Wed Jun 20 15:48:17 2018 +0530 Committer: manishgupta88 <[email protected]> Committed: Tue Jul 24 15:34:56 2018 +0530 ---------------------------------------------------------------------- .../apache/carbondata/core/util/CarbonProperties.java | 3 +++ .../table/DiskBasedDMSchemaStoraheProviderSuite.java | 2 +- .../carbondata/hadoop/ft/CarbonTableInputFormatTest.java | 3 +++ .../hadoop/ft/CarbonTableOutputFormatTest.java | 2 ++ .../presto/integrationtest/PrestoAllDataTypeTest.scala | 5 +++++ .../datamap/lucene/LuceneFineGrainDataMapSuite.scala | 2 -- .../LuceneFineGrainDataMapWithSearchModeSuite.scala | 6 ------ .../org/apache/spark/sql/test/TestQueryExecutor.scala | 6 ++++++ .../apache/carbondata/sdk/file/AvroCarbonWriterTest.java | 11 +++++++++++ .../apache/carbondata/sdk/file/CSVCarbonWriterTest.java | 11 +++++++++++ .../sdk/file/CSVNonTransactionalCarbonWriterTest.java | 11 +++++++++++ .../org/apache/carbondata/sdk/file/CarbonReaderTest.java | 10 ++++++++++ .../apache/carbondata/store/LocalCarbonStoreTest.java | 11 +++++++++++ 13 files changed, 74 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/bea277f8/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java index f7ace5e..004eb74 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java @@ -34,6 +34,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.constants.CarbonLoadOptionConstants; import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.ColumnarFormatVersion; import static org.apache.carbondata.core.constants.CarbonCommonConstants.BLOCKLET_SIZE; import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION; @@ -1531,6 +1532,8 @@ public final class CarbonProperties { if (systemLocation == null) { systemLocation = getStorePath(); } + systemLocation = CarbonUtil.checkAndAppendFileSystemURIScheme(systemLocation); + systemLocation = FileFactory.getUpdatedFilePath(systemLocation); return systemLocation + CarbonCommonConstants.FILE_SEPARATOR + "_system"; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/bea277f8/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStoraheProviderSuite.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStoraheProviderSuite.java b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStoraheProviderSuite.java index 709215a..9af7f8f 100644 --- a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStoraheProviderSuite.java +++ b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStoraheProviderSuite.java @@ -37,7 +37,7 @@ public class DiskBasedDMSchemaStoraheProviderSuite { @BeforeClass public static void setUp() throws IOException { String path = - new File(DiskBasedDMSchemaStorageProvider.class.getResource("/").getPath() + "../") + new File(DiskBasedDMSchemaStoraheProviderSuite.class.getResource("/").getPath() + "../") .getCanonicalPath().replaceAll("\\\\", "/"); CarbonProperties.getInstance() http://git-wip-us.apache.org/repos/asf/carbondata/blob/bea277f8/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableInputFormatTest.java ---------------------------------------------------------------------- diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableInputFormatTest.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableInputFormatTest.java index ea242d1..fb9ff9c 100644 --- a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableInputFormatTest.java +++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableInputFormatTest.java @@ -40,6 +40,7 @@ import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.carbondata.hadoop.CarbonProjection; import org.apache.carbondata.hadoop.api.CarbonTableInputFormat; import org.apache.carbondata.hadoop.testutil.StoreCreator; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; @@ -58,6 +59,8 @@ public class CarbonTableInputFormatTest { static { CarbonProperties.getInstance(). addProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC, "/tmp/carbon/badrecords"); + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, "/tmp/carbon/"); try { StoreCreator.createCarbonStore(); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/bea277f8/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableOutputFormatTest.java ---------------------------------------------------------------------- diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableOutputFormatTest.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableOutputFormatTest.java index aa2a500..0d8c38b 100644 --- a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableOutputFormatTest.java +++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableOutputFormatTest.java @@ -51,6 +51,8 @@ public class CarbonTableOutputFormatTest { static { CarbonProperties.getInstance(). addProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC, "/tmp/carbon/badrecords"); + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, "/tmp/carbon/"); try { carbonLoadModel = StoreCreator.createTableAndLoadModel(); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/bea277f8/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala ---------------------------------------------------------------------- diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala index 8793765..6995117 100644 --- a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala +++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala @@ -24,8 +24,10 @@ import org.apache.hadoop.fs.permission.{FsAction, FsPermission} import org.scalatest.{BeforeAndAfterAll, FunSuiteLike} import org.apache.carbondata.common.logging.LogServiceFactory +import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.datastore.impl.FileFactory import org.apache.carbondata.core.datastore.impl.FileFactory.FileType +import org.apache.carbondata.core.util.CarbonProperties import org.apache.carbondata.presto.server.PrestoServer @@ -37,6 +39,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll { private val rootPath = new File(this.getClass.getResource("/").getPath + "../../../..").getCanonicalPath private val storePath = s"$rootPath/integration/presto/target/store" + private val systemPath = s"$rootPath/integration/presto/target/system" // Table schema: @@ -68,6 +71,8 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll { override def beforeAll: Unit = { import org.apache.carbondata.presto.util.CarbonDataStoreCreator + CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, + systemPath) CarbonDataStoreCreator .createCarbonStore(storePath, s"$rootPath/integration/presto/src/test/resources/alldatatype.csv") http://git-wip-us.apache.org/repos/asf/carbondata/blob/bea277f8/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala index b56701b..08e44f7 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala @@ -45,8 +45,6 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll { LuceneFineGrainDataMapSuite.createFile(file2) sql("create database if not exists lucene") CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, - CarbonEnv.getDatabaseLocation("lucene", sqlContext.sparkSession)) .addProperty(CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP, "true") sql("use lucene") sql("DROP TABLE IF EXISTS normal_test") http://git-wip-us.apache.org/repos/asf/carbondata/blob/bea277f8/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapWithSearchModeSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapWithSearchModeSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapWithSearchModeSuite.scala index 0ceead8..e6a2a36 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapWithSearchModeSuite.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapWithSearchModeSuite.scala @@ -45,9 +45,6 @@ class LuceneFineGrainDataMapWithSearchModeSuite extends QueryTest with BeforeAnd .addProperty(CarbonCommonConstants.CARBON_SEARCH_QUERY_TIMEOUT, "100s") LuceneFineGrainDataMapSuite.createFile(file2, n) sql("create database if not exists lucene") - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, - CarbonEnv.getDatabaseLocation("lucene", sqlContext.sparkSession)) sql("use lucene") sql("DROP TABLE IF EXISTS datamap_test") sql( @@ -305,9 +302,6 @@ class LuceneFineGrainDataMapWithSearchModeSuite extends QueryTest with BeforeAnd sql("DROP TABLE IF EXISTS datamap_test") sql("DROP TABLE IF EXISTS datamap_test5") sql("use default") - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, - CarbonProperties.getStorePath) sqlContext.sparkSession.asInstanceOf[CarbonSession].stopSearchMode() } http://git-wip-us.apache.org/repos/asf/carbondata/blob/bea277f8/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala index 464cca7..d3a20c3 100644 --- a/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala +++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala @@ -128,6 +128,11 @@ object TestQueryExecutor { } else { s"$target/bad_store" } + val systemFolderPath = if (hdfsUrl.startsWith("hdfs://")) { + s"$hdfsUrl/systemfolder" + System.nanoTime() + } else { + s"$target/systemfolder" + } createDirectory(badStoreLocation) val hiveresultpath = if (hdfsUrl.startsWith("hdfs://")) { @@ -175,6 +180,7 @@ object TestQueryExecutor { (CarbonCommonConstants.DICTIONARY_SERVER_PORT_DEFAULT.toInt + Random.nextInt(100)) + "") .addProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE, "1024") .addProperty(CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE, "1024") + .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, systemFolderPath) private def lookupQueryExecutor: Class[_] = { ServiceLoader.load(classOf[TestQueryExecutorRegister], Utils.getContextOrSparkClassLoader) http://git-wip-us.apache.org/repos/asf/carbondata/blob/bea277f8/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java index d0128fc..4320edc 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java @@ -27,6 +27,8 @@ import java.util.Map; import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.schema.table.DiskBasedDMSchemaStorageProvider; +import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.avro.generic.GenericData; @@ -44,6 +46,15 @@ public class AvroCarbonWriterTest { @Before public void cleanFile() { + String path = null; + try { + path = new File(AvroCarbonWriterTest.class.getResource("/").getPath() + "../") + .getCanonicalPath().replaceAll("\\\\", "/"); + } catch (IOException e) { + assert (false); + } + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, path); assert (TestUtil.cleanMdtFile()); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/bea277f8/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java index 865097b..78d0240 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java @@ -24,6 +24,8 @@ import java.io.IOException; import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.schema.table.DiskBasedDMSchemaStorageProvider; +import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.commons.io.FileUtils; @@ -39,6 +41,15 @@ public class CSVCarbonWriterTest { @Before public void cleanFile() { + String path = null; + try { + path = new File(CSVCarbonWriterTest.class.getResource("/").getPath() + "../") + .getCanonicalPath().replaceAll("\\\\", "/"); + } catch (IOException e) { + assert (false); + } + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, path); assert (TestUtil.cleanMdtFile()); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/bea277f8/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVNonTransactionalCarbonWriterTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVNonTransactionalCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVNonTransactionalCarbonWriterTest.java index 0393077..c1d5f88 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVNonTransactionalCarbonWriterTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVNonTransactionalCarbonWriterTest.java @@ -24,6 +24,8 @@ import java.io.IOException; import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.schema.table.DiskBasedDMSchemaStorageProvider; +import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.commons.io.FileUtils; @@ -38,6 +40,15 @@ import org.junit.Test; public class CSVNonTransactionalCarbonWriterTest { @Before public void cleanFile() { + String path = null; + try { + path = new File(CSVNonTransactionalCarbonWriterTest.class.getResource("/").getPath() + "../") + .getCanonicalPath().replaceAll("\\\\", "/"); + } catch (IOException e) { + assert (false); + } + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, path); assert (TestUtil.cleanMdtFile()); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/bea277f8/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java index 19bf194..279b3b6 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java @@ -28,6 +28,7 @@ import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.schema.table.DiskBasedDMSchemaStorageProvider; import org.apache.carbondata.core.scan.expression.ColumnExpression; import org.apache.carbondata.core.scan.expression.LiteralExpression; import org.apache.carbondata.core.scan.expression.conditional.*; @@ -937,6 +938,15 @@ public class CarbonReaderTest extends TestCase { @Override public void setUp() { carbonProperties = CarbonProperties.getInstance(); + String path = null; + try { + path = new File(CarbonReaderTest.class.getResource("/").getPath() + "../") + .getCanonicalPath().replaceAll("\\\\", "/"); + } catch (IOException e) { + assert (false); + } + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, path); } private static final LogService LOGGER = http://git-wip-us.apache.org/repos/asf/carbondata/blob/bea277f8/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java b/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java index 2b8c708..0319df9 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java @@ -21,9 +21,11 @@ import java.io.File; import java.io.IOException; import java.util.Iterator; +import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datastore.row.CarbonRow; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.sdk.file.Field; import org.apache.carbondata.sdk.file.Schema; import org.apache.carbondata.sdk.file.TestUtil; @@ -36,6 +38,15 @@ import org.junit.Test; public class LocalCarbonStoreTest { @Before public void cleanFile() { + String path = null; + try { + path = new File(LocalCarbonStoreTest.class.getResource("/").getPath() + "../") + .getCanonicalPath().replaceAll("\\\\", "/"); + } catch (IOException e) { + assert (false); + } + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, path); assert (TestUtil.cleanMdtFile()); }
