Repository: tajo Updated Branches: refs/heads/master 8b2fe1145 -> b3d4a3402
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3d4a340/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java index 951cc91..6ff291c 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java @@ -18,10 +18,8 @@ package org.apache.tajo.storage; -import com.google.common.base.Preconditions; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -33,10 +31,8 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.storage.fragment.FileFragment; -import org.apache.tajo.util.FileUtil; import org.junit.Test; -import java.io.File; import java.io.IOException; import java.net.URL; @@ -91,8 +87,8 @@ public class TestDelimitedTextFile { @Test public void testStripQuote() throws IOException, CloneNotSupportedException { TableMeta meta = CatalogUtil.newTableMeta("TEXT"); - meta.putOption(StorageUtil.TEXT_DELIMITER, ","); - meta.putOption(StorageUtil.QUOTE_CHAR, "\""); + meta.putProperty(StorageUtil.TEXT_DELIMITER, ","); + meta.putProperty(StorageUtil.QUOTE_CHAR, "\""); FileFragment fragment = getFileFragment("testStripQuote.txt"); Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null); scanner.init(); @@ -110,8 +106,8 @@ public class TestDelimitedTextFile { @Test public void testIncompleteQuote() throws IOException, CloneNotSupportedException { TableMeta meta = CatalogUtil.newTableMeta("TEXT"); - meta.putOption(StorageUtil.TEXT_DELIMITER, ","); - meta.putOption(StorageUtil.QUOTE_CHAR, "\""); + meta.putProperty(StorageUtil.TEXT_DELIMITER, ","); + meta.putProperty(StorageUtil.QUOTE_CHAR, "\""); FileFragment fragment = getFileFragment("testIncompleteQuote.txt"); Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null); scanner.init(); @@ -129,7 +125,7 @@ public class TestDelimitedTextFile { @Test public void testIgnoreAllErrors() throws IOException { TableMeta meta = CatalogUtil.newTableMeta("JSON"); - meta.putOption(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "-1"); + meta.putProperty(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "-1"); FileFragment fragment = getFileFragment("testErrorTolerance1.json"); Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null); scanner.init(); @@ -147,7 +143,7 @@ public class TestDelimitedTextFile { @Test public void testIgnoreOneErrorTolerance() throws IOException { TableMeta meta = CatalogUtil.newTableMeta("JSON"); - meta.putOption(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "1"); + meta.putProperty(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "1"); FileFragment fragment = getFileFragment("testErrorTolerance1.json"); Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null); scanner.init(); @@ -168,7 +164,7 @@ public class TestDelimitedTextFile { @Test public void testNoErrorTolerance() throws IOException { TableMeta meta = CatalogUtil.newTableMeta("JSON"); - meta.putOption(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "0"); + meta.putProperty(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "0"); FileFragment fragment = getFileFragment("testErrorTolerance2.json"); Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null); scanner.init(); @@ -187,7 +183,7 @@ public class TestDelimitedTextFile { public void testIgnoreTruncatedValueErrorTolerance() throws IOException { TajoConf conf = new TajoConf(); TableMeta meta = CatalogUtil.newTableMeta("JSON"); - meta.putOption(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "1"); + meta.putProperty(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "1"); FileFragment fragment = getFileFragment("testErrorTolerance3.json"); Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null); scanner.init(); @@ -203,7 +199,7 @@ public class TestDelimitedTextFile { @Test public void testSkippingHeaderWithJson() throws IOException { TableMeta meta = CatalogUtil.newTableMeta("JSON"); - meta.putOption(StorageConstants.TEXT_SKIP_HEADER_LINE, "2"); + meta.putProperty(StorageConstants.TEXT_SKIP_HEADER_LINE, "2"); FileFragment fragment = getFileFragment("testNormal.json"); Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null); @@ -229,8 +225,8 @@ public class TestDelimitedTextFile { @Test public void testSkippingHeaderWithText() throws IOException { TableMeta meta = CatalogUtil.newTableMeta("TEXT"); - meta.putOption(StorageConstants.TEXT_SKIP_HEADER_LINE, "1"); - meta.putOption(StorageConstants.TEXT_DELIMITER, ","); + meta.putProperty(StorageConstants.TEXT_SKIP_HEADER_LINE, "1"); + meta.putProperty(StorageConstants.TEXT_DELIMITER, ","); FileFragment fragment = getFileFragment("testSkip.txt"); Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null); http://git-wip-us.apache.org/repos/asf/tajo/blob/b3d4a340/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java index b800ed2..a478538 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java @@ -117,7 +117,7 @@ public class TestLineReader { schema.addColumn("comment2", Type.TEXT); TableMeta meta = CatalogUtil.newTableMeta("TEXT"); - meta.putOption("compression.codec", DeflateCodec.class.getCanonicalName()); + meta.putProperty("compression.codec", DeflateCodec.class.getCanonicalName()); Path tablePath = new Path(testDir, "testLineDelimitedReaderWithCompression." + DeflateCodec.class.getSimpleName()); FileAppender appender = (FileAppender) (TablespaceManager.getLocalFs()).getAppender( http://git-wip-us.apache.org/repos/asf/tajo/blob/b3d4a340/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java index 78fbd6b..1598b50 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java @@ -107,9 +107,9 @@ public class TestMergeScanner { KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); - meta.setOptions(CatalogUtil.newDefaultProperty(dataFormat)); + meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat)); if (dataFormat.equalsIgnoreCase("AVRO")) { - meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_MULTIPLE_FILES_AVRO_SCHEMA); + meta.putProperty(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_MULTIPLE_FILES_AVRO_SCHEMA); } Path table1Path = new Path(testDir, dataFormat + "_1.data"); http://git-wip-us.apache.org/repos/asf/tajo/blob/b3d4a340/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java index 1f6168a..c70e07c 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java @@ -217,9 +217,9 @@ public class TestStorages { schema.addColumn("score", Type.FLOAT4); TableMeta meta = CatalogUtil.newTableMeta(dataFormat); - meta.setOptions(CatalogUtil.newDefaultProperty(dataFormat)); + meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat)); if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) { - meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, + meta.putProperty(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_PROJECTION_AVRO_SCHEMA); } @@ -315,9 +315,9 @@ public class TestStorages { schema.addColumn("score", Type.FLOAT4); TableMeta meta = CatalogUtil.newTableMeta(dataFormat); - meta.setOptions(CatalogUtil.newDefaultProperty(dataFormat)); + meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat)); if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) { - meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, + meta.putProperty(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_PROJECTION_AVRO_SCHEMA); } @@ -392,10 +392,10 @@ public class TestStorages { KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); - meta.setOptions(CatalogUtil.newDefaultProperty(dataFormat)); + meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat)); if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) { String path = JavaResourceUtil.getResourceURL("dataset/testVariousTypes.avsc").toString(); - meta.putOption(StorageConstants.AVRO_SCHEMA_URL, path); + meta.putProperty(StorageConstants.AVRO_SCHEMA_URL, path); } FileTablespace sm = TablespaceManager.getLocalFs(); @@ -466,13 +466,13 @@ public class TestStorages { KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); - meta.setOptions(CatalogUtil.newDefaultProperty(dataFormat)); - meta.putOption(StorageConstants.TEXT_NULL, "\\\\N"); - meta.putOption(StorageConstants.RCFILE_NULL, "\\\\N"); - meta.putOption(StorageConstants.RCFILE_SERDE, TextSerializerDeserializer.class.getName()); - meta.putOption(StorageConstants.SEQUENCEFILE_NULL, "\\"); + meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat)); + meta.putProperty(StorageConstants.TEXT_NULL, "\\\\N"); + meta.putProperty(StorageConstants.RCFILE_NULL, "\\\\N"); + meta.putProperty(StorageConstants.RCFILE_SERDE, TextSerializerDeserializer.class.getName()); + meta.putProperty(StorageConstants.SEQUENCEFILE_NULL, "\\"); if (dataFormat.equalsIgnoreCase("AVRO")) { - meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_NULL_HANDLING_TYPES_AVRO_SCHEMA); + meta.putProperty(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_NULL_HANDLING_TYPES_AVRO_SCHEMA); } Path tablePath = new Path(testDir, "testVariousTypes.data"); @@ -561,7 +561,7 @@ public class TestStorages { KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); - meta.putOption(StorageConstants.CSVFILE_SERDE, TextSerializerDeserializer.class.getName()); + meta.putProperty(StorageConstants.CSVFILE_SERDE, TextSerializerDeserializer.class.getName()); Path tablePath = new Path(testDir, "testVariousTypes.data"); FileTablespace sm = TablespaceManager.getLocalFs(); @@ -630,7 +630,7 @@ public class TestStorages { KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); - meta.putOption(StorageConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName()); + meta.putProperty(StorageConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName()); Path tablePath = new Path(testDir, "testVariousTypes.data"); FileTablespace sm = TablespaceManager.getLocalFs(); @@ -699,7 +699,7 @@ public class TestStorages { KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); - meta.putOption(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName()); + meta.putProperty(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName()); Path tablePath = new Path(testDir, "testVariousTypes.data"); FileTablespace sm = TablespaceManager.getLocalFs(); @@ -771,7 +771,7 @@ public class TestStorages { KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); - meta.putOption(StorageConstants.SEQUENCEFILE_SERDE, BinarySerializerDeserializer.class.getName()); + meta.putProperty(StorageConstants.SEQUENCEFILE_SERDE, BinarySerializerDeserializer.class.getName()); Path tablePath = new Path(testDir, "testVariousTypes.data"); FileTablespace sm = TablespaceManager.getLocalFs(); @@ -954,7 +954,7 @@ public class TestStorages { KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) { - meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_MAX_VALUE_AVRO_SCHEMA); + meta.putProperty(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_MAX_VALUE_AVRO_SCHEMA); } if (dataFormat.equalsIgnoreCase(BuiltinStorages.RAW)) { @@ -1019,7 +1019,7 @@ public class TestStorages { KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); - meta.setOptions(CatalogUtil.newDefaultProperty(dataFormat)); + meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat)); Path tablePath = new Path(testDir, "testLessThanSchemaSize.data"); FileTablespace sm = TablespaceManager.getLocalFs(); @@ -1083,7 +1083,7 @@ public class TestStorages { KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); - meta.setOptions(CatalogUtil.newDefaultProperty(dataFormat)); + meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat)); Path tablePath = new Path(testDir, "test_dataformat_oversize.data"); FileTablespace sm = TablespaceManager.getLocalFs(); @@ -1168,9 +1168,9 @@ public class TestStorages { schema.addColumn("score", Type.FLOAT4); TableMeta meta = CatalogUtil.newTableMeta(dataFormat); - meta.setOptions(CatalogUtil.newDefaultProperty(dataFormat)); + meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat)); if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) { - meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, + meta.putProperty(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_PROJECTION_AVRO_SCHEMA); } http://git-wip-us.apache.org/repos/asf/tajo/blob/b3d4a340/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java index 960448c..ddeb340 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java @@ -59,12 +59,12 @@ public class TestAvroUtil { @Test public void testGetSchema() throws IOException, URISyntaxException { TableMeta meta = CatalogUtil.newTableMeta("AVRO"); - meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, FileUtil.readTextFile(new File(schemaUrl.getPath()))); + meta.putProperty(StorageConstants.AVRO_SCHEMA_LITERAL, FileUtil.readTextFile(new File(schemaUrl.getPath()))); Schema schema = AvroUtil.getAvroSchema(meta, new TajoConf()); assertEquals(expected, schema); meta = CatalogUtil.newTableMeta("AVRO"); - meta.putOption(StorageConstants.AVRO_SCHEMA_URL, schemaUrl.getPath()); + meta.putProperty(StorageConstants.AVRO_SCHEMA_URL, schemaUrl.getPath()); schema = AvroUtil.getAvroSchema(meta, new TajoConf()); assertEquals(expected, schema); @@ -75,7 +75,7 @@ public class TestAvroUtil { String url = "http://127.0.0.1:"; + addr.getPort() + schemaUrl.getPath(); meta = CatalogUtil.newTableMeta("AVRO"); - meta.putOption(StorageConstants.AVRO_SCHEMA_URL, url); + meta.putProperty(StorageConstants.AVRO_SCHEMA_URL, url); schema = AvroUtil.getAvroSchema(meta, new TajoConf()); } finally { server.stop();
