Repository: carbondata Updated Branches: refs/heads/master fa9a4eeeb -> 8af737204
[CARBONDATA-3030] Remove no use parameter in test case Remove no use parameter in test case This closes #2839 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8af73720 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8af73720 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8af73720 Branch: refs/heads/master Commit: 8af7372042469f467d74c5b90047e401c0dc778e Parents: fa9a4ee Author: xubo245 <[email protected]> Authored: Mon Oct 22 12:06:08 2018 +0800 Committer: Jacky Li <[email protected]> Committed: Wed Oct 24 10:21:24 2018 +0800 ---------------------------------------------------------------------- docs/sdk-guide.md | 2 +- .../carbondata/examples/S3UsingSDkExample.scala | 3 +- .../TestNonTransactionalCarbonTable.scala | 37 ++++++++++---------- .../apache/carbondata/sdk/file/TestUtil.java | 15 +++----- .../carbondata/sdk/file/CarbonReaderTest.java | 24 ++++++------- 5 files changed, 36 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/8af73720/docs/sdk-guide.md ---------------------------------------------------------------------- diff --git a/docs/sdk-guide.md b/docs/sdk-guide.md index be42b3f..3a9be71 100644 --- a/docs/sdk-guide.md +++ b/docs/sdk-guide.md @@ -28,7 +28,7 @@ In the carbon jars package, there exist a carbondata-store-sdk-x.x.x-SNAPSHOT.ja This SDK writer, writes carbondata file and carbonindex file at a given path. External client can make use of this writer to convert other format data or live data to create carbondata and index files. -These SDK writer output contains just a carbondata and carbonindex files. No metadata folder will be present. +These SDK writer output contains just carbondata and carbonindex files. No metadata folder will be present. ## Quick example http://git-wip-us.apache.org/repos/asf/carbondata/blob/8af73720/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3UsingSDkExample.scala ---------------------------------------------------------------------- diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3UsingSDkExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3UsingSDkExample.scala index 014bec5..666db1f 100644 --- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3UsingSDkExample.scala +++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3UsingSDkExample.scala @@ -34,8 +34,7 @@ object S3UsingSDKExample { // prepare SDK writer output def buildTestData( path: String, - num: Int = 3, - persistSchema: Boolean = false): Any = { + num: Int = 3): Any = { // getCanonicalPath gives path with \, but the code expects /. val writerPath = path.replace("\\", "/"); http://git-wip-us.apache.org/repos/asf/carbondata/blob/8af73720/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala index 0c55d1b..38cf201 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala @@ -68,46 +68,46 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { def buildTestDataSingleFile(): Any = { FileUtils.deleteDirectory(new File(writerPath)) - buildTestData(3, false, null) + buildTestData(3, null) } def buildTestDataMultipleFiles(): Any = { FileUtils.deleteDirectory(new File(writerPath)) - buildTestData(1000000, false, null) + buildTestData(1000000, null) } def buildTestDataTwice(): Any = { FileUtils.deleteDirectory(new File(writerPath)) - buildTestData(3, false, null) - buildTestData(3, false, null) + buildTestData(3, null) + buildTestData(3, null) } def buildTestDataSameDirectory(): Any = { - buildTestData(3, false, null) + buildTestData(3, null) } def buildTestDataWithBadRecordForce(): Any = { FileUtils.deleteDirectory(new File(writerPath)) var options = Map("bAd_RECords_action" -> "FORCE").asJava - buildTestData(3, false, options) + buildTestData(3, options) } def buildTestDataWithBadRecordFail(): Any = { FileUtils.deleteDirectory(new File(writerPath)) var options = Map("bAd_RECords_action" -> "FAIL").asJava - buildTestData(15001, false, options) + buildTestData(15001, options) } def buildTestDataWithBadRecordIgnore(): Any = { FileUtils.deleteDirectory(new File(writerPath)) var options = Map("bAd_RECords_action" -> "IGNORE").asJava - buildTestData(3, false, options) + buildTestData(3, options) } def buildTestDataWithBadRecordRedirect(): Any = { FileUtils.deleteDirectory(new File(writerPath)) var options = Map("bAd_RECords_action" -> "REDIRECT").asJava - buildTestData(3, false, options) + buildTestData(3, options) } def buildTestDataWithSortColumns(sortColumns: List[String]): Any = { @@ -115,7 +115,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { buildTestData(3, null, sortColumns) } - def buildTestData(rows: Int, persistSchema: Boolean, options: util.Map[String, String]): Any = { + def buildTestData(rows: Int, options: util.Map[String, String]): Any = { buildTestData(rows, options, List("name")) } @@ -194,7 +194,6 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { // prepare sdk writer output def buildTestDataWithSameUUID(rows: Int, - persistSchema: Boolean, options: util.Map[String, String], sortColumns: List[String]): Any = { val schema = new StringBuilder() @@ -297,7 +296,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { "Read two sdk writer outputs before and after deleting the existing files and creating new " + "files with same schema and UUID") { FileUtils.deleteDirectory(new File(writerPath)) - buildTestDataWithSameUUID(3, false, null, List("name")) + buildTestDataWithSameUUID(3, null, List("name")) assert(new File(writerPath).exists()) sql("DROP TABLE IF EXISTS sdkOutputTable") @@ -319,7 +318,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { // and creation of new file can happen at same timestamp. Thread.sleep(1000) assert(!new File(writerPath).exists()) - buildTestDataWithSameUUID(4, false, null, List("name")) + buildTestDataWithSameUUID(4, null, List("name")) checkAnswer(sql("select * from sdkOutputTable"), Seq( Row("robot0", 0, 0.0), Row("robot1", 1, 0.5), @@ -334,7 +333,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { test(" test csv fileheader for transactional table") { FileUtils.deleteDirectory(new File(writerPath)) - buildTestDataWithSameUUID(3, false, null, List("Name")) + buildTestDataWithSameUUID(3, null, List("Name")) assert(new File(writerPath).exists()) sql("DROP TABLE IF EXISTS sdkOutputTable") @@ -363,14 +362,14 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { test("test count star with multiple loads files with same schema and UUID") { FileUtils.deleteDirectory(new File(writerPath)) - buildTestDataWithSameUUID(3, false, null, List("namE")) + buildTestDataWithSameUUID(3, null, List("namE")) assert(new File(writerPath).exists()) sql("DROP TABLE IF EXISTS sdkOutputTable") sql( s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY 'carbondata' LOCATION |'$writerPath' """.stripMargin) checkAnswer(sql(s"""select count(*) from sdkOutputTable """), Seq(Row(3))) - buildTestDataWithSameUUID(3, false, null, List("name")) + buildTestDataWithSameUUID(3, null, List("name")) // should reflect new count checkAnswer(sql(s"""select count(*) from sdkOutputTable """), Seq(Row(6))) sql("DROP TABLE sdkOutputTable") @@ -841,7 +840,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { Row("robot1", 1, 0.5), Row("robot2", 2, 1.0))) - buildTestDataWithSameUUID(3, false, null, List("name")) + buildTestDataWithSameUUID(3, null, List("name")) checkAnswer(sql("select * from sdkOutputTable"), Seq( Row("robot0", 0, 0.0), @@ -854,7 +853,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { Row("robot1", 1, 0.5), Row("robot2", 2, 1.0))) - buildTestDataWithSameUUID(3, false, null, List("name")) + buildTestDataWithSameUUID(3, null, List("name")) checkAnswer(sql("select * from sdkOutputTable"), Seq( Row("robot0", 0, 0.0), @@ -906,7 +905,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll { Row("robot2", 2, 1.0))) FileUtils.deleteDirectory(new File(writerPath)) - buildTestData(4, false, null) + buildTestData(4, null) checkAnswer(sql("select * from sdkOutputTable"), Seq( Row("robot0", 0, 0.0), http://git-wip-us.apache.org/repos/asf/carbondata/blob/8af73720/store/sdk/src/main/java/org/apache/carbondata/sdk/file/TestUtil.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/TestUtil.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/TestUtil.java index dc6e012..df31f9f 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/TestUtil.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/TestUtil.java @@ -72,11 +72,6 @@ public class TestUtil { 100, schema, path, sortColumns, -1, -1); } - public static void writeFilesAndVerify(int rows, Schema schema, String path) { - writeFilesAndVerify( - rows, schema, path, null, -1, -1); - } - public static void writeFilesAndVerify(Schema schema, String path) { writeFilesAndVerify( 100, schema, path, null, -1, -1); @@ -85,14 +80,12 @@ public class TestUtil { /** * write file and verify * - * @param rows number of rows - * @param schema schema - * @param path table store path - * @param persistSchema whether persist schema - * @param isTransactionalTable whether is transactional table + * @param rows number of rows + * @param schema schema + * @param path table store path */ public static void writeFilesAndVerify( - int rows, Schema schema, String path, boolean persistSchema, boolean isTransactionalTable) { + int rows, Schema schema, String path) { writeFilesAndVerify(rows, schema, path, null, -1, -1); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/8af73720/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java index 58f3f61..f37b832 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java @@ -132,7 +132,7 @@ public class CarbonReaderTest extends TestCase { fields[0] = new Field("name", DataTypes.STRING); fields[1] = new Field("age", DataTypes.INT); - TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false); + TestUtil.writeFilesAndVerify(200, new Schema(fields), path); ColumnExpression columnExpression = new ColumnExpression("name", DataTypes.STRING); EqualToExpression equalToExpression = new EqualToExpression(columnExpression, @@ -167,7 +167,7 @@ public class CarbonReaderTest extends TestCase { fields[0] = new Field("name", DataTypes.STRING); fields[1] = new Field("age", DataTypes.INT); - TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false); + TestUtil.writeFilesAndVerify(200, new Schema(fields), path); ColumnExpression columnExpression = new ColumnExpression("age", DataTypes.INT); @@ -204,7 +204,7 @@ public class CarbonReaderTest extends TestCase { fields[1] = new Field("age", DataTypes.INT); fields[2] = new Field("doubleField", DataTypes.DOUBLE); - TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false); + TestUtil.writeFilesAndVerify(200, new Schema(fields), path); ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE); EqualToExpression equalToExpression = new EqualToExpression(columnExpression, @@ -246,7 +246,7 @@ public class CarbonReaderTest extends TestCase { fields[1] = new Field("age", DataTypes.INT); fields[2] = new Field("doubleField", DataTypes.DOUBLE); - TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false); + TestUtil.writeFilesAndVerify(200, new Schema(fields), path); ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE); EqualToExpression equalToExpression = new EqualToExpression(columnExpression, @@ -288,7 +288,7 @@ public class CarbonReaderTest extends TestCase { fields[1] = new Field("age", DataTypes.INT); fields[2] = new Field("doubleField", DataTypes.DOUBLE); - TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false); + TestUtil.writeFilesAndVerify(200, new Schema(fields), path); ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE); GreaterThanExpression greaterThanExpression = new GreaterThanExpression(columnExpression, @@ -330,7 +330,7 @@ public class CarbonReaderTest extends TestCase { fields[1] = new Field("age", DataTypes.INT); fields[2] = new Field("doubleField", DataTypes.DOUBLE); - TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false); + TestUtil.writeFilesAndVerify(200, new Schema(fields), path); ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE); LessThanExpression lessThanExpression = new LessThanExpression(columnExpression, @@ -372,7 +372,7 @@ public class CarbonReaderTest extends TestCase { fields[1] = new Field("age", DataTypes.INT); fields[2] = new Field("doubleField", DataTypes.DOUBLE); - TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false); + TestUtil.writeFilesAndVerify(200, new Schema(fields), path); ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE); LessThanExpression lessThanExpression = new LessThanExpression(columnExpression, @@ -414,7 +414,7 @@ public class CarbonReaderTest extends TestCase { fields[1] = new Field("age", DataTypes.INT); fields[2] = new Field("doubleField", DataTypes.DOUBLE); - TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false); + TestUtil.writeFilesAndVerify(200, new Schema(fields), path); ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE); LessThanExpression lessThanExpression = new LessThanExpression(columnExpression, @@ -456,7 +456,7 @@ public class CarbonReaderTest extends TestCase { fields[1] = new Field("age", DataTypes.INT); fields[2] = new Field("doubleField", DataTypes.DOUBLE); - TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false); + TestUtil.writeFilesAndVerify(200, new Schema(fields), path); ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE); LessThanExpression lessThanExpression = new LessThanExpression(columnExpression, @@ -1408,9 +1408,9 @@ public class CarbonReaderTest extends TestCase { fields[0] = new Field("name", DataTypes.STRING); fields[1] = new Field("age", DataTypes.INT); - TestUtil.writeFilesAndVerify(200, new Schema(fields), path1, false, false); - TestUtil.writeFilesAndVerify(200, new Schema(fields), path2, false, false); - TestUtil.writeFilesAndVerify(200, new Schema(fields), path3, false, false); + TestUtil.writeFilesAndVerify(200, new Schema(fields), path1); + TestUtil.writeFilesAndVerify(200, new Schema(fields), path2); + TestUtil.writeFilesAndVerify(200, new Schema(fields), path3); EqualToExpression equalToExpression = new EqualToExpression( new ColumnExpression("name", DataTypes.STRING),
