Repository: tajo Updated Branches: refs/heads/branch-0.11.0 9b4c8abb8 -> b418c4991
http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index 98ad292..4e7ee83 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -775,14 +775,14 @@ public class Stage implements EventHandler<StageEvent> { DataChannel channel = masterPlan.getOutgoingChannels(getId()).get(0); // if store plan (i.e., CREATE or INSERT OVERWRITE) - String storeType = PlannerUtil.getStoreType(masterPlan.getLogicalPlan()); - if (storeType == null) { + String dataFormat = PlannerUtil.getDataFormat(masterPlan.getLogicalPlan()); + if (dataFormat == null) { // get final output store type (i.e., SELECT) - storeType = channel.getStoreType(); + dataFormat = channel.getDataFormat(); } schema = channel.getSchema(); - meta = CatalogUtil.newTableMeta(storeType, new KeyValueSet()); + meta = CatalogUtil.newTableMeta(dataFormat, new KeyValueSet()); inputStatistics = statsArray[0]; resultStatistics = statsArray[1]; } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java index a586e4b..db0a982 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java @@ -276,7 +276,7 @@ public class Task implements EventHandler<TaskEvent> { fragmentList.add(fragment.toString()); } catch (Exception e) { LOG.error(e.getMessage(), e); - fragmentList.add("ERROR: " + eachFragment.getStoreType() + "," + eachFragment.getId() + ": " + e.getMessage()); + fragmentList.add("ERROR: " + eachFragment.getDataFormat() + "," + eachFragment.getId() + ": " + e.getMessage()); } } taskHistory.setFragments(fragmentList.toArray(new String[]{})); http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core/src/main/resources/webapps/admin/catalogview.jsp ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/resources/webapps/admin/catalogview.jsp b/tajo-core/src/main/resources/webapps/admin/catalogview.jsp index 6771912..2388f71 100644 --- a/tajo-core/src/main/resources/webapps/admin/catalogview.jsp +++ b/tajo-core/src/main/resources/webapps/admin/catalogview.jsp @@ -186,7 +186,7 @@ <div style=''>Detail</div> <table border="1" class='border_table'> <tr><td width='100'>Table path</td><td width='410'><%=tableDesc.getUri()%></td></tr> - <tr><td>Store type</td><td><%=tableDesc.getMeta().getStoreType()%></td></tr> + <tr><td>Store type</td><td><%=tableDesc.getMeta().getDataFormat()%></td></tr> <tr><td># rows</td><td><%=(tableDesc.hasStats() ? ("" + tableDesc.getStats().getNumRows()) : "-")%></td></tr> <tr><td>Volume</td><td><%=(tableDesc.hasStats() ? FileUtil.humanReadableByteCount(tableDesc.getStats().getNumBytes(),true) : "-")%></td></tr> <tr><td>Options</td><td><%=optionStr%></td></tr> http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 3531ce7..5599c7f 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -24,9 +24,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.ContentSummary; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.BuiltinStorages; import org.apache.tajo.OverridableConf; @@ -1401,7 +1398,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex private void updatePhysicalInfo(TableDesc desc) { // FAKEFILE is used for test - if (!desc.getMeta().getStoreType().equals("SYSTEM") && !desc.getMeta().getStoreType().equals("FAKEFILE")) { + if (!desc.getMeta().getDataFormat().equals("SYSTEM") && !desc.getMeta().getDataFormat().equals("FAKEFILE")) { try { if (desc.getStats() != null) { desc.getStats().setNumBytes(storage.getTableVolumn(desc.getUri())); @@ -1844,7 +1841,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex insertNode.setUri(targetUri); if (expr.hasStorageType()) { - insertNode.setStorageType(CatalogUtil.getBackwardCompitablityStoreType(expr.getStorageType())); + insertNode.setDataFormat(CatalogUtil.getBackwardCompitableDataFormat(expr.getDataFormat())); } if (expr.hasParams()) { KeyValueSet options = new KeyValueSet(); @@ -1891,7 +1888,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex createTableNode.setTableSchema(baseTable.getSchema()); createTableNode.setPartitionMethod(partitionDesc); - createTableNode.setStorageType(CatalogUtil.getBackwardCompitablityStoreType(baseTable.getMeta().getStoreType())); + createTableNode.setDataFormat(CatalogUtil.getBackwardCompitableDataFormat(baseTable.getMeta().getDataFormat())); createTableNode.setOptions(baseTable.getMeta().getOptions()); createTableNode.setExternal(baseTable.isExternal()); @@ -1927,9 +1924,9 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex createTableNode.setUri(getCreatedTableURI(context, expr)); if (expr.hasStorageType()) { // If storage type (using clause) is specified - createTableNode.setStorageType(CatalogUtil.getBackwardCompitablityStoreType(expr.getStorageType())); + createTableNode.setDataFormat(CatalogUtil.getBackwardCompitableDataFormat(expr.getStorageType())); } else { // otherwise, default type - createTableNode.setStorageType(BuiltinStorages.TEXT); + createTableNode.setDataFormat(BuiltinStorages.TEXT); } // Set default storage properties to table http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-plan/src/main/java/org/apache/tajo/plan/TablePropertyUtil.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/TablePropertyUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/TablePropertyUtil.java index 62fa706..4bf2734 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/TablePropertyUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/TablePropertyUtil.java @@ -38,10 +38,10 @@ public class TablePropertyUtil { * @param node CreateTableNode */ public static void setTableProperty(OverridableConf context, CreateTableNode node) { - String storeType = node.getStorageType(); + String dataFormat = node.getStorageType(); KeyValueSet property = node.getOptions(); - if (storeType.equalsIgnoreCase("TEXT") || storeType.equalsIgnoreCase("TEXT")) { + if (dataFormat.equalsIgnoreCase("TEXT") || dataFormat.equalsIgnoreCase("TEXT")) { setSessionToProperty(context, SessionVars.NULL_CHAR, property, StorageConstants.TEXT_NULL); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java index 9a169dd..bf29a2d 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java @@ -124,7 +124,7 @@ public class CreateTableNode extends StoreTableNode implements Cloneable { } public String toString() { - return "CreateTable (table=" + tableName + ", external=" + external + ", storeType=" + storageType + + return "CreateTable (table=" + tableName + ", external=" + external + ", dataFormat=" + storageType + ", ifNotExists=" + ifNotExists +")"; } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java index cb3dcee..4030b69 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java @@ -48,7 +48,7 @@ public class InsertNode extends StoreTableNode implements Cloneable { setUri(desc.getUri()); } setOptions(desc.getMeta().getOptions()); - setStorageType(desc.getMeta().getStoreType()); + setDataFormat(desc.getMeta().getDataFormat()); if (desc.hasPartition()) { this.setPartitionMethod(desc.getPartitionMethod()); http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java index a28f01f..30e3260 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java @@ -39,7 +39,7 @@ public abstract class PersistentStoreNode extends UnaryNode implements Cloneable super(pid, nodeType); } - public void setStorageType(String storageType) { + public void setDataFormat(String storageType) { this.storageType = storageType; } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java index c75c3fd..2051dfb 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java @@ -502,7 +502,7 @@ public class LogicalNodeDeserializer { createTable.setOutSchema(convertSchema(protoNode.getOutSchema())); } createTable.setChild(nodeMap.get(persistentStoreProto.getChildSeq())); - createTable.setStorageType(persistentStoreProto.getStorageType()); + createTable.setDataFormat(persistentStoreProto.getStorageType()); createTable.setOptions(new KeyValueSet(persistentStoreProto.getTableProperties())); createTable.setTableName(storeTableNodeSpec.getTableName()); @@ -540,7 +540,7 @@ public class LogicalNodeDeserializer { insertNode.setOutSchema(convertSchema(protoNode.getOutSchema())); } insertNode.setChild(nodeMap.get(persistentStoreProto.getChildSeq())); - insertNode.setStorageType(persistentStoreProto.getStorageType()); + insertNode.setDataFormat(persistentStoreProto.getStorageType()); insertNode.setOptions(new KeyValueSet(persistentStoreProto.getTableProperties())); if (storeTableNodeSpec.hasTableName()) { http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index 80e1e18..44dfc9f 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -159,7 +159,7 @@ public class PlannerUtil { for (LogicalNode node: scanNodes) { scanNode = (ScanNode) node; - isVirtualTable &= (scanNode.getTableDesc().getMeta().getStoreType().equalsIgnoreCase("SYSTEM")); + isVirtualTable &= (scanNode.getTableDesc().getMeta().getDataFormat().equalsIgnoreCase("SYSTEM")); } return !checkIfDDLPlan(rootNode) && hasScanNode && isVirtualTable; @@ -818,7 +818,7 @@ public class PlannerUtil { } } - public static String getStoreType(LogicalPlan plan) { + public static String getDataFormat(LogicalPlan plan) { LogicalRootNode rootNode = plan.getRootBlock().getRoot(); NodeType nodeType = rootNode.getChild().getType(); if (nodeType == NodeType.CREATE_TABLE) { http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index 2c7feac..c16736d 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -190,7 +190,7 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor<PreLogicalPlanVer return true; } - private boolean assertSupportedStoreType(VerificationState state, String name) { + private boolean assertSupportedDataFormat(VerificationState state, String name) { Preconditions.checkNotNull(name); if (name.equalsIgnoreCase("RAW")) { @@ -271,7 +271,7 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor<PreLogicalPlanVer } if (expr.hasStorageType()) { - assertSupportedStoreType(context.state, expr.getStorageType()); + assertSupportedDataFormat(context.state, expr.getStorageType()); } return expr; @@ -298,7 +298,7 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor<PreLogicalPlanVer } if (expr.hasStorageType()) { - assertSupportedStoreType(context.state, expr.getStorageType()); + assertSupportedDataFormat(context.state, expr.getDataFormat()); } if (child != null && child.getType() == OpType.Projection) { http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-plan/src/main/proto/Plan.proto ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto index fa1deeb..0cd0c32 100644 --- a/tajo-plan/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@ -566,7 +566,7 @@ message DataChannelProto { repeated ColumnProto shuffleKeys = 7; optional int32 numOutputs = 9 [default = 1]; - optional string storeType = 10; + optional string dataFormat = 10; } message EnforcerProto { http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java b/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java index 92cb399..56edccc 100644 --- a/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java +++ b/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java @@ -22,11 +22,9 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.plan.logical.GroupbyNode; import org.apache.tajo.plan.logical.JoinNode; -import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.logical.ScanNode; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java index 2e00bc9..859b8b8 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java @@ -103,39 +103,39 @@ public class OldStorageManager { } /** - * Returns the proper Tablespace instance according to the storeType. + * Returns the proper Tablespace instance according to the dataFormat. * * @param tajoConf Tajo system property. - * @param storeType Storage type + * @param dataFormat Storage type * @return * @throws IOException */ - public static Tablespace getStorageManager(TajoConf tajoConf, String storeType) throws IOException { + public static Tablespace getStorageManager(TajoConf tajoConf, String dataFormat) throws IOException { FileSystem fileSystem = TajoConf.getWarehouseDir(tajoConf).getFileSystem(tajoConf); if (fileSystem != null) { - return getStorageManager(tajoConf, fileSystem.getUri(), storeType); + return getStorageManager(tajoConf, fileSystem.getUri(), dataFormat); } else { - return getStorageManager(tajoConf, null, storeType); + return getStorageManager(tajoConf, null, dataFormat); } } /** - * Returns the proper Tablespace instance according to the storeType + * Returns the proper Tablespace instance according to the dataFormat * * @param tajoConf Tajo system property. * @param uri Key that can identify each storage manager(may be a path) - * @param storeType Storage type + * @param dataFormat Storage type * @return * @throws IOException */ public static synchronized Tablespace getStorageManager( - TajoConf tajoConf, URI uri, String storeType) throws IOException { + TajoConf tajoConf, URI uri, String dataFormat) throws IOException { Preconditions.checkNotNull(tajoConf); Preconditions.checkNotNull(uri); - Preconditions.checkNotNull(storeType); + Preconditions.checkNotNull(dataFormat); String typeName; - if (storeType.equalsIgnoreCase("HBASE")) { + if (dataFormat.equalsIgnoreCase("HBASE")) { typeName = "hbase"; } else { typeName = "hdfs"; http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java index 1c6f433..81fbbc0 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java @@ -209,7 +209,7 @@ public abstract class Tablespace { Scanner scanner; - Class<? extends Scanner> scannerClass = getScannerClass(meta.getStoreType()); + Class<? extends Scanner> scannerClass = getScannerClass(meta.getDataFormat()); scanner = OldStorageManager.newScannerInstance(scannerClass, conf, schema, meta, fragment); scanner.setTarget(target.toArray()); @@ -256,7 +256,7 @@ public abstract class Tablespace { Class<? extends Appender> appenderClass; - String handlerName = meta.getStoreType().toLowerCase(); + String handlerName = meta.getDataFormat().toLowerCase(); appenderClass = OldStorageManager.APPENDER_HANDLER_CACHE.get(handlerName); if (appenderClass == null) { appenderClass = conf.getClass( @@ -265,7 +265,7 @@ public abstract class Tablespace { } if (appenderClass == null) { - throw new IOException("Unknown Storage Type: " + meta.getStoreType()); + throw new IOException("Unknown Storage Type: " + meta.getDataFormat()); } appender = OldStorageManager.newAppenderInstance(appenderClass, conf, taskAttemptId, meta, schema, workDir); @@ -274,14 +274,14 @@ public abstract class Tablespace { } /** - * Return the Scanner class for the StoreType that is defined in storage-default.xml. + * Return the Scanner class for the DataFormat that is defined in storage-default.xml. * - * @param storeType store type + * @param dataFormat store type * @return The Scanner class * @throws java.io.IOException */ - public Class<? extends Scanner> getScannerClass(String storeType) throws IOException { - String handlerName = storeType.toLowerCase(); + public Class<? extends Scanner> getScannerClass(String dataFormat) throws IOException { + String handlerName = dataFormat.toLowerCase(); Class<? extends Scanner> scannerClass = OldStorageManager.SCANNER_HANDLER_CACHE.get(handlerName); if (scannerClass == null) { scannerClass = conf.getClass( @@ -290,7 +290,7 @@ public abstract class Tablespace { } if (scannerClass == null) { - throw new IOException("Unknown Storage Type: " + storeType); + throw new IOException("Unknown Storage Type: " + dataFormat); } return scannerClass; http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java index bd46551..d9405e2 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java @@ -47,17 +47,17 @@ public class FragmentConvertor { */ private static final Class<?>[] DEFAULT_FRAGMENT_PARAMS = { ByteString.class }; - public static Class<? extends Fragment> getFragmentClass(Configuration conf, String storeType) + public static Class<? extends Fragment> getFragmentClass(Configuration conf, String dataFormat) throws IOException { - Class<? extends Fragment> fragmentClass = CACHED_FRAGMENT_CLASSES.get(storeType.toLowerCase()); + Class<? extends Fragment> fragmentClass = CACHED_FRAGMENT_CLASSES.get(dataFormat.toLowerCase()); if (fragmentClass == null) { fragmentClass = conf.getClass( - String.format("tajo.storage.fragment.%s.class", storeType.toLowerCase()), null, Fragment.class); - CACHED_FRAGMENT_CLASSES.put(storeType.toLowerCase(), fragmentClass); + String.format("tajo.storage.fragment.%s.class", dataFormat.toLowerCase()), null, Fragment.class); + CACHED_FRAGMENT_CLASSES.put(dataFormat.toLowerCase(), fragmentClass); } if (fragmentClass == null) { - throw new IOException("No such a fragment for " + storeType.toLowerCase()); + throw new IOException("No such a fragment for " + dataFormat.toLowerCase()); } return fragmentClass; @@ -82,9 +82,9 @@ public class FragmentConvertor { public static <T extends Fragment> T convert(Configuration conf, FragmentProto fragment) throws IOException { - Class<T> fragmentClass = (Class<T>) getFragmentClass(conf, fragment.getStoreType().toLowerCase()); + Class<T> fragmentClass = (Class<T>) getFragmentClass(conf, fragment.getDataFormat().toLowerCase()); if (fragmentClass == null) { - throw new IOException("No such a fragment class for " + fragment.getStoreType()); + throw new IOException("No such a fragment class for " + fragment.getDataFormat()); } return convert(fragmentClass, fragment); } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java index 2d3f2c3..18aa515 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java @@ -172,7 +172,7 @@ public class HBaseFragment implements Fragment, Comparable<HBaseFragment>, Clone FragmentProto.Builder fragmentBuilder = FragmentProto.newBuilder(); fragmentBuilder.setId(this.tableName); fragmentBuilder.setContents(builder.buildPartial().toByteString()); - fragmentBuilder.setStoreType(BuiltinStorages.HBASE); + fragmentBuilder.setDataFormat(BuiltinStorages.HBASE); return fragmentBuilder.build(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java index 5f94aae..508962e 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java @@ -230,7 +230,7 @@ public class FileFragment implements Fragment, Comparable<FileFragment>, Cloneab FragmentProto.Builder fragmentBuilder = FragmentProto.newBuilder(); fragmentBuilder.setId(this.tableName); - fragmentBuilder.setStoreType(BuiltinStorages.TEXT); + fragmentBuilder.setDataFormat(BuiltinStorages.TEXT); fragmentBuilder.setContents(builder.buildPartial().toByteString()); return fragmentBuilder.build(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java index 4cdf7df..3143ec2 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java @@ -43,7 +43,7 @@ public abstract class TextLineSerDe { public static ByteBuf getNullChars(TableMeta meta) { byte[] nullCharByteArray; - if (meta.getStoreType().equals("SEQUENCEFILE")) { + if (meta.getDataFormat().equals("SEQUENCEFILE")) { nullCharByteArray = getNullCharsAsBytes(meta, StorageConstants.SEQUENCEFILE_NULL, "\\"); } else { nullCharByteArray = getNullCharsAsBytes(meta); http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java index 67d0646..cc69119 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java @@ -53,12 +53,12 @@ public class TestCompressionStorages { private TajoConf conf; private static String TEST_PATH = "target/test-data/TestCompressionStorages"; - private String storeType; + private String dataFormat; private Path testDir; private FileSystem fs; public TestCompressionStorages(String type) throws IOException { - this.storeType = type; + this.dataFormat = type; conf = new TajoConf(); testDir = CommonTestingUtil.getTestDir(TEST_PATH); @@ -76,44 +76,44 @@ public class TestCompressionStorages { @Test public void testDeflateCodecCompressionData() throws IOException { - storageCompressionTest(storeType, DeflateCodec.class); + storageCompressionTest(dataFormat, DeflateCodec.class); } @Test public void testGzipCodecCompressionData() throws IOException { - if (storeType.equalsIgnoreCase("RCFILE")) { + if (dataFormat.equalsIgnoreCase("RCFILE")) { if( ZlibFactory.isNativeZlibLoaded(conf)) { - storageCompressionTest(storeType, GzipCodec.class); + storageCompressionTest(dataFormat, GzipCodec.class); } - } else if (storeType.equalsIgnoreCase("SEQUENCEFILE")) { + } else if (dataFormat.equalsIgnoreCase("SEQUENCEFILE")) { if( ZlibFactory.isNativeZlibLoaded(conf)) { - storageCompressionTest(storeType, GzipCodec.class); + storageCompressionTest(dataFormat, GzipCodec.class); } } else { - storageCompressionTest(storeType, GzipCodec.class); + storageCompressionTest(dataFormat, GzipCodec.class); } } @Test public void testSnappyCodecCompressionData() throws IOException { if (SnappyCodec.isNativeCodeLoaded()) { - storageCompressionTest(storeType, SnappyCodec.class); + storageCompressionTest(dataFormat, SnappyCodec.class); } } @Test public void testLz4CodecCompressionData() throws IOException { if(NativeCodeLoader.isNativeCodeLoaded() && Lz4Codec.isNativeCodeLoaded()) - storageCompressionTest(storeType, Lz4Codec.class); + storageCompressionTest(dataFormat, Lz4Codec.class); } - private void storageCompressionTest(String storeType, Class<? extends CompressionCodec> codec) throws IOException { + private void storageCompressionTest(String dataFormat, Class<? extends CompressionCodec> codec) throws IOException { Schema schema = new Schema(); schema.addColumn("id", Type.INT4); schema.addColumn("age", Type.FLOAT4); schema.addColumn("name", Type.TEXT); - TableMeta meta = CatalogUtil.newTableMeta(storeType); + TableMeta meta = CatalogUtil.newTableMeta(dataFormat); meta.putOption("compression.codec", codec.getCanonicalName()); meta.putOption("compression.type", SequenceFile.CompressionType.BLOCK.name()); meta.putOption("rcfile.serde", TextSerializerDeserializer.class.getName()); @@ -154,7 +154,7 @@ public class TestCompressionStorages { Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, tablets[0], schema); scanner.init(); - if (storeType.equalsIgnoreCase("SEQUENCEFILE")) { + if (dataFormat.equalsIgnoreCase("SEQUENCEFILE")) { assertTrue(scanner instanceof SequenceFileScanner); Writable key = ((SequenceFileScanner) scanner).getKey(); assertEquals(key.getClass().getCanonicalName(), LongWritable.class.getCanonicalName()); http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java index 4faf8b9..78fbd6b 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java @@ -66,11 +66,11 @@ public class TestMergeScanner { "}\n"; private Path testDir; - private String storeType; + private String dataFormat; private FileSystem fs; - public TestMergeScanner(String storeType) { - this.storeType = storeType; + public TestMergeScanner(String dataFormat) { + this.dataFormat = dataFormat; } @Parameters @@ -83,7 +83,7 @@ public class TestMergeScanner { {"SEQUENCEFILE"}, {"AVRO"}, // RowFile requires Byte-buffer read support, so we omitted RowFile. - //{StoreType.ROWFILE}, + //{DataFormat.ROWFILE}, }); } @@ -106,13 +106,13 @@ public class TestMergeScanner { schema.addColumn("age", Type.INT8); KeyValueSet options = new KeyValueSet(); - TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - meta.setOptions(CatalogUtil.newDefaultProperty(storeType)); - if (storeType.equalsIgnoreCase("AVRO")) { + TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); + meta.setOptions(CatalogUtil.newDefaultProperty(dataFormat)); + if (dataFormat.equalsIgnoreCase("AVRO")) { meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_MULTIPLE_FILES_AVRO_SCHEMA); } - Path table1Path = new Path(testDir, storeType + "_1.data"); + Path table1Path = new Path(testDir, dataFormat + "_1.data"); Appender appender1 = TablespaceManager.getLocalFs().getAppender(null, null, meta, schema, table1Path); appender1.enableStats(); appender1.init(); @@ -134,7 +134,7 @@ public class TestMergeScanner { assertEquals(tupleNum, stat1.getNumRows().longValue()); } - Path table2Path = new Path(testDir, storeType + "_2.data"); + Path table2Path = new Path(testDir, dataFormat + "_2.data"); Appender appender2 = TablespaceManager.getLocalFs().getAppender(null, null, meta, schema, table2Path); appender2.enableStats(); appender2.init(); @@ -166,7 +166,7 @@ public class TestMergeScanner { targetSchema.addColumn(schema.getColumn(2)); Scanner scanner = new MergeScanner(conf, schema, meta, TUtil.newList(fragment), targetSchema); - assertEquals(isProjectableStorage(meta.getStoreType()), scanner.isProjectable()); + assertEquals(isProjectableStorage(meta.getDataFormat()), scanner.isProjectable()); scanner.init(); int totalCounts = 0; @@ -174,7 +174,7 @@ public class TestMergeScanner { while ((tuple = scanner.next()) != null) { totalCounts++; - if (storeType.equalsIgnoreCase("RAW")) { + if (dataFormat.equalsIgnoreCase("RAW")) { assertEquals(4, tuple.size()); assertFalse(tuple.isBlankOrNull(0)); assertFalse(tuple.isBlankOrNull(1)); http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java index 26e16ea..278de45 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java @@ -109,7 +109,7 @@ public class TestStorages { " ]\n" + "}\n"; - private String storeType; + private String dataFormat; private boolean splitable; private boolean statsable; private boolean seekable; @@ -121,14 +121,14 @@ public class TestStorages { throws IOException { final String TEST_PATH = "target/test-data/TestStorages"; - this.storeType = type; + this.dataFormat = type; this.splitable = splitable; this.statsable = statsable; this.seekable = seekable; this.internalType = internalType; conf = new TajoConf(); - if (storeType.equalsIgnoreCase(BuiltinStorages.RCFILE)) { + if (dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) { conf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, 100); } @@ -159,7 +159,7 @@ public class TestStorages { schema.addColumn("id", Type.INT4); schema.addColumn("age", Type.INT8); - TableMeta meta = CatalogUtil.newTableMeta(storeType); + TableMeta meta = CatalogUtil.newTableMeta(dataFormat); Path tablePath = new Path(testDir, "Splitable.data"); FileTablespace sm = TablespaceManager.getLocalFs(); Appender appender = sm.getAppender(meta, schema, tablePath); @@ -214,9 +214,9 @@ public class TestStorages { schema.addColumn("age", Type.INT8); schema.addColumn("score", Type.FLOAT4); - TableMeta meta = CatalogUtil.newTableMeta(storeType); - meta.setOptions(CatalogUtil.newDefaultProperty(storeType)); - if (storeType.equalsIgnoreCase(BuiltinStorages.AVRO)) { + TableMeta meta = CatalogUtil.newTableMeta(dataFormat); + meta.setOptions(CatalogUtil.newDefaultProperty(dataFormat)); + if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) { meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_PROJECTION_AVRO_SCHEMA); } @@ -231,7 +231,7 @@ public class TestStorages { TableStats stat = appender.getStats(); assertEquals(0, stat.getNumRows().longValue()); - if(internalType || BuiltinStorages.TEXT.equals(storeType)) { + if(internalType || BuiltinStorages.TEXT.equals(dataFormat)) { FileStatus fileStatus = fs.getFileStatus(tablePath); assertEquals(0, fileStatus.getLen()); } @@ -252,12 +252,12 @@ public class TestStorages { @Test public void testRCFileSplitable() throws IOException { - if (storeType.equalsIgnoreCase(BuiltinStorages.RCFILE)) { + if (dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) { Schema schema = new Schema(); schema.addColumn("id", Type.INT4); schema.addColumn("age", Type.INT8); - TableMeta meta = CatalogUtil.newTableMeta(storeType); + TableMeta meta = CatalogUtil.newTableMeta(dataFormat); Path tablePath = new Path(testDir, "Splitable.data"); FileTablespace sm = TablespaceManager.getLocalFs(); Appender appender = sm.getAppender(meta, schema, tablePath); @@ -312,9 +312,9 @@ public class TestStorages { schema.addColumn("age", Type.INT8); schema.addColumn("score", Type.FLOAT4); - TableMeta meta = CatalogUtil.newTableMeta(storeType); - meta.setOptions(CatalogUtil.newDefaultProperty(storeType)); - if (storeType.equalsIgnoreCase(BuiltinStorages.AVRO)) { + TableMeta meta = CatalogUtil.newTableMeta(dataFormat); + meta.setOptions(CatalogUtil.newDefaultProperty(dataFormat)); + if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) { meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_PROJECTION_AVRO_SCHEMA); } @@ -370,7 +370,7 @@ public class TestStorages { @Test public void testVariousTypes() throws IOException { - boolean handleProtobuf = !storeType.equalsIgnoreCase(BuiltinStorages.JSON); + boolean handleProtobuf = !dataFormat.equalsIgnoreCase(BuiltinStorages.JSON); Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -389,9 +389,9 @@ public class TestStorages { } KeyValueSet options = new KeyValueSet(); - TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - meta.setOptions(CatalogUtil.newDefaultProperty(storeType)); - if (storeType.equalsIgnoreCase(BuiltinStorages.AVRO)) { + TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); + meta.setOptions(CatalogUtil.newDefaultProperty(dataFormat)); + if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) { String path = JavaResourceUtil.getResourceURL("dataset/testVariousTypes.avsc").toString(); meta.putOption(StorageConstants.AVRO_SCHEMA_URL, path); } @@ -443,7 +443,7 @@ public class TestStorages { @Test public void testNullHandlingTypes() throws IOException { - boolean handleProtobuf = !storeType.equalsIgnoreCase(BuiltinStorages.JSON); + boolean handleProtobuf = !dataFormat.equalsIgnoreCase(BuiltinStorages.JSON); Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -463,13 +463,13 @@ public class TestStorages { } KeyValueSet options = new KeyValueSet(); - TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - meta.setOptions(CatalogUtil.newDefaultProperty(storeType)); + TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); + meta.setOptions(CatalogUtil.newDefaultProperty(dataFormat)); meta.putOption(StorageConstants.TEXT_NULL, "\\\\N"); meta.putOption(StorageConstants.RCFILE_NULL, "\\\\N"); meta.putOption(StorageConstants.RCFILE_SERDE, TextSerializerDeserializer.class.getName()); meta.putOption(StorageConstants.SEQUENCEFILE_NULL, "\\"); - if (storeType.equalsIgnoreCase("AVRO")) { + if (dataFormat.equalsIgnoreCase("AVRO")) { meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_NULL_HANDLING_TYPES_AVRO_SCHEMA); } @@ -540,7 +540,7 @@ public class TestStorages { @Test public void testRCFileTextSerializeDeserialize() throws IOException { - if(!storeType.equalsIgnoreCase(BuiltinStorages.RCFILE)) return; + if(!dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) return; Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -558,7 +558,7 @@ public class TestStorages { schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); KeyValueSet options = new KeyValueSet(); - TableMeta meta = CatalogUtil.newTableMeta(storeType, options); + TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); meta.putOption(StorageConstants.CSVFILE_SERDE, TextSerializerDeserializer.class.getName()); Path tablePath = new Path(testDir, "testVariousTypes.data"); @@ -609,7 +609,7 @@ public class TestStorages { @Test public void testRCFileBinarySerializeDeserialize() throws IOException { - if(!storeType.equalsIgnoreCase(BuiltinStorages.RCFILE)) return; + if(!dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) return; Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -627,7 +627,7 @@ public class TestStorages { schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); KeyValueSet options = new KeyValueSet(); - TableMeta meta = CatalogUtil.newTableMeta(storeType, options); + TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); meta.putOption(StorageConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName()); Path tablePath = new Path(testDir, "testVariousTypes.data"); @@ -678,7 +678,7 @@ public class TestStorages { @Test public void testSequenceFileTextSerializeDeserialize() throws IOException { - if(!storeType.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return; + if(!dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return; Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -696,7 +696,7 @@ public class TestStorages { schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); KeyValueSet options = new KeyValueSet(); - TableMeta meta = CatalogUtil.newTableMeta(storeType, options); + TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); meta.putOption(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName()); Path tablePath = new Path(testDir, "testVariousTypes.data"); @@ -751,7 +751,7 @@ public class TestStorages { @Test public void testSequenceFileBinarySerializeDeserialize() throws IOException { - if(!storeType.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return; + if(!dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) return; Schema schema = new Schema(); schema.addColumn("col1", Type.BOOLEAN); @@ -769,7 +769,7 @@ public class TestStorages { schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())); KeyValueSet options = new KeyValueSet(); - TableMeta meta = CatalogUtil.newTableMeta(storeType, options); + TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); meta.putOption(StorageConstants.SEQUENCEFILE_SERDE, BinarySerializerDeserializer.class.getName()); Path tablePath = new Path(testDir, "testVariousTypes.data"); @@ -825,14 +825,14 @@ public class TestStorages { @Test public void testTime() throws IOException { - if (storeType.equalsIgnoreCase(BuiltinStorages.TEXT) || internalType) { + if (dataFormat.equalsIgnoreCase(BuiltinStorages.TEXT) || internalType) { Schema schema = new Schema(); schema.addColumn("col1", Type.DATE); schema.addColumn("col2", Type.TIME); schema.addColumn("col3", Type.TIMESTAMP); KeyValueSet options = new KeyValueSet(); - TableMeta meta = CatalogUtil.newTableMeta(storeType, options); + TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); Path tablePath = new Path(testDir, "testTime.data"); FileTablespace sm = TablespaceManager.getLocalFs(); @@ -874,7 +874,7 @@ public class TestStorages { schema.addColumn("age", Type.INT8); schema.addColumn("comment", Type.TEXT); - TableMeta meta = CatalogUtil.newTableMeta(storeType); + TableMeta meta = CatalogUtil.newTableMeta(dataFormat); Path tablePath = new Path(testDir, "Seekable.data"); FileTablespace sm = TablespaceManager.getLocalFs(); FileAppender appender = (FileAppender) sm.getAppender(meta, schema, tablePath); @@ -952,12 +952,12 @@ public class TestStorages { schema.addColumn("col5", Type.INT8); KeyValueSet options = new KeyValueSet(); - TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - if (storeType.equalsIgnoreCase(BuiltinStorages.AVRO)) { + TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); + if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) { meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_MAX_VALUE_AVRO_SCHEMA); } - if (storeType.equalsIgnoreCase(BuiltinStorages.RAW)) { + if (dataFormat.equalsIgnoreCase(BuiltinStorages.RAW)) { OldStorageManager.clearCache(); /* TAJO-1250 reproduce BufferOverflow of RAWFile */ int headerSize = 4 + 2 + 1; //Integer record length + Short null-flag length + 1 byte null flags @@ -1006,9 +1006,9 @@ public class TestStorages { @Test public void testLessThanSchemaSize() throws IOException { /* Internal storage must be same with schema size */ - if (internalType || storeType.equalsIgnoreCase(BuiltinStorages.AVRO) - || storeType.equalsIgnoreCase(BuiltinStorages.PARQUET) - || storeType.equalsIgnoreCase(BuiltinStorages.ORC)) { + if (internalType || dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO) + || dataFormat.equalsIgnoreCase(BuiltinStorages.PARQUET) + || dataFormat.equalsIgnoreCase(BuiltinStorages.ORC)) { return; } @@ -1018,8 +1018,8 @@ public class TestStorages { dataSchema.addColumn("col3", Type.INT2); KeyValueSet options = new KeyValueSet(); - TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - meta.setOptions(CatalogUtil.newDefaultProperty(storeType)); + TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); + meta.setOptions(CatalogUtil.newDefaultProperty(dataFormat)); Path tablePath = new Path(testDir, "testLessThanSchemaSize.data"); FileTablespace sm = TablespaceManager.getLocalFs(); @@ -1071,10 +1071,10 @@ public class TestStorages { @Test public final void testInsertFixedCharTypeWithOverSize() throws Exception { - if (!storeType.equalsIgnoreCase(BuiltinStorages.TEXT) && - !storeType.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE) && - !storeType.equalsIgnoreCase(BuiltinStorages.RCFILE) && - !storeType.equalsIgnoreCase(BuiltinStorages.PARQUET)) { + if (!dataFormat.equalsIgnoreCase(BuiltinStorages.TEXT) && + !dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE) && + !dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE) && + !dataFormat.equalsIgnoreCase(BuiltinStorages.PARQUET)) { return; } @@ -1082,10 +1082,10 @@ public class TestStorages { dataSchema.addColumn("col1", Type.CHAR); KeyValueSet options = new KeyValueSet(); - TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - meta.setOptions(CatalogUtil.newDefaultProperty(storeType)); + TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options); + meta.setOptions(CatalogUtil.newDefaultProperty(dataFormat)); - Path tablePath = new Path(testDir, "test_storetype_oversize.data"); + Path tablePath = new Path(testDir, "test_dataformat_oversize.data"); FileTablespace sm = TablespaceManager.getLocalFs(); Appender appender = sm.getAppender(meta, dataSchema, tablePath); appender.init(); http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java index 5437b0d..30cea60 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java @@ -54,10 +54,10 @@ public class TestBSTIndex { private static final String TEST_PATH = "target/test-data/TestIndex"; private Path testDir; private FileSystem fs; - private String storeType; + private String dataFormat; public TestBSTIndex(String type) { - this.storeType = type; + this.dataFormat = type; conf = new TajoConf(); conf.setVar(TajoConf.ConfVars.ROOT_DIR, TEST_PATH); schema = new Schema(); @@ -85,9 +85,9 @@ public class TestBSTIndex { @Test public void testFindValue() throws IOException { - meta = CatalogUtil.newTableMeta(storeType); + meta = CatalogUtil.newTableMeta(dataFormat); - Path tablePath = new Path(testDir, "testFindValue_" + storeType); + Path tablePath = new Path(testDir, "testFindValue_" + dataFormat); Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(meta, schema, tablePath); appender.init(); @@ -118,13 +118,13 @@ public class TestBSTIndex { BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); BSTIndex bst = new BSTIndex(conf); - BSTIndexWriter creater = bst.getIndexWriter(new Path(testDir, "testFindValue_" + storeType + ".idx"), + BSTIndexWriter creater = bst.getIndexWriter(new Path(testDir, "testFindValue_" + dataFormat + ".idx"), BSTIndex.TWO_LEVEL_INDEX, keySchema, comp); creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -146,9 +146,9 @@ public class TestBSTIndex { scanner.close(); tuple = new VTuple(keySchema.size()); - BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testFindValue_" + storeType + ".idx"), keySchema, comp); + BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testFindValue_" + dataFormat + ".idx"), keySchema, comp); reader.open(); - scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -176,9 +176,9 @@ public class TestBSTIndex { @Test public void testBuildIndexWithAppender() throws IOException { - meta = CatalogUtil.newTableMeta(storeType); + meta = CatalogUtil.newTableMeta(dataFormat); - Path tablePath = new Path(testDir, "testBuildIndexWithAppender_" + storeType); + Path tablePath = new Path(testDir, "testBuildIndexWithAppender_" + dataFormat); FileAppender appender = (FileAppender) ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(meta, schema, tablePath); appender.init(); @@ -194,7 +194,7 @@ public class TestBSTIndex { BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); BSTIndex bst = new BSTIndex(conf); - BSTIndexWriter creater = bst.getIndexWriter(new Path(testDir, "testBuildIndexWithAppender_" + storeType + ".idx"), + BSTIndexWriter creater = bst.getIndexWriter(new Path(testDir, "testBuildIndexWithAppender_" + dataFormat + ".idx"), BSTIndex.TWO_LEVEL_INDEX, keySchema, comp); creater.setLoadNum(LOAD_NUM); creater.open(); @@ -225,10 +225,10 @@ public class TestBSTIndex { FileFragment tablet = new FileFragment("table1_1", status.getPath(), 0, fileLen); tuple = new VTuple(keySchema.size()); - BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testBuildIndexWithAppender_" + storeType + ".idx"), + BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testBuildIndexWithAppender_" + dataFormat + ".idx"), keySchema, comp); reader.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -256,9 +256,9 @@ public class TestBSTIndex { @Test public void testFindOmittedValue() throws IOException { - meta = CatalogUtil.newTableMeta(storeType); + meta = CatalogUtil.newTableMeta(dataFormat); - Path tablePath = StorageUtil.concatPath(testDir, "testFindOmittedValue_" + storeType); + Path tablePath = StorageUtil.concatPath(testDir, "testFindOmittedValue_" + dataFormat); Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple; @@ -287,12 +287,12 @@ public class TestBSTIndex { BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); BSTIndex bst = new BSTIndex(conf); - BSTIndexWriter creater = bst.getIndexWriter(new Path(testDir, "testFindOmittedValue_" + storeType + ".idx"), + BSTIndexWriter creater = bst.getIndexWriter(new Path(testDir, "testFindOmittedValue_" + dataFormat + ".idx"), BSTIndex.TWO_LEVEL_INDEX, keySchema, comp); creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -313,7 +313,7 @@ public class TestBSTIndex { creater.close(); scanner.close(); - BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testFindOmittedValue_" + storeType + ".idx"), + BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testFindOmittedValue_" + dataFormat + ".idx"), keySchema, comp); reader.open(); for (int i = 1; i < TUPLE_NUM - 1; i += 2) { @@ -327,9 +327,9 @@ public class TestBSTIndex { @Test public void testFindNextKeyValue() throws IOException { - meta = CatalogUtil.newTableMeta(storeType); + meta = CatalogUtil.newTableMeta(dataFormat); - Path tablePath = new Path(testDir, "testFindNextKeyValue_" + storeType); + Path tablePath = new Path(testDir, "testFindNextKeyValue_" + dataFormat); Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(meta, schema, tablePath); appender.init(); @@ -360,12 +360,12 @@ public class TestBSTIndex { BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); BSTIndex bst = new BSTIndex(conf); - BSTIndexWriter creater = bst.getIndexWriter(new Path(testDir, "testFindNextKeyValue_" + storeType + ".idx"), + BSTIndexWriter creater = bst.getIndexWriter(new Path(testDir, "testFindNextKeyValue_" + dataFormat + ".idx"), BSTIndex.TWO_LEVEL_INDEX, keySchema, comp); creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -386,10 +386,10 @@ public class TestBSTIndex { creater.close(); scanner.close(); - BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testFindNextKeyValue_" + storeType + ".idx"), + BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testFindNextKeyValue_" + dataFormat + ".idx"), keySchema, comp); reader.open(); - scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -420,9 +420,9 @@ public class TestBSTIndex { @Test public void testFindNextKeyOmittedValue() throws IOException { - meta = CatalogUtil.newTableMeta(storeType); + meta = CatalogUtil.newTableMeta(dataFormat); - Path tablePath = new Path(testDir, "testFindNextKeyOmittedValue_" + storeType); + Path tablePath = new Path(testDir, "testFindNextKeyOmittedValue_" + dataFormat); Appender appender = (((FileTablespace) TablespaceManager.getLocalFs())) .getAppender(meta, schema, tablePath); appender.init(); @@ -453,12 +453,12 @@ public class TestBSTIndex { BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); BSTIndex bst = new BSTIndex(conf); - BSTIndexWriter creater = bst.getIndexWriter(new Path(testDir, "testFindNextKeyOmittedValue_" + storeType + ".idx"), + BSTIndexWriter creater = bst.getIndexWriter(new Path(testDir, "testFindNextKeyOmittedValue_" + dataFormat + ".idx"), BSTIndex.TWO_LEVEL_INDEX, keySchema, comp); creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -479,10 +479,10 @@ public class TestBSTIndex { creater.close(); scanner.close(); - BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testFindNextKeyOmittedValue_" + storeType + ".idx"), + BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testFindNextKeyOmittedValue_" + dataFormat + ".idx"), keySchema, comp); reader.open(); - scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -502,9 +502,9 @@ public class TestBSTIndex { @Test public void testFindMinValue() throws IOException { - meta = CatalogUtil.newTableMeta(storeType); + meta = CatalogUtil.newTableMeta(dataFormat); - Path tablePath = new Path(testDir, "testFindMinValue" + storeType); + Path tablePath = new Path(testDir, "testFindMinValue" + dataFormat); Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(meta, schema, tablePath); appender.init(); @@ -535,12 +535,12 @@ public class TestBSTIndex { BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); BSTIndex bst = new BSTIndex(conf); - BSTIndexWriter creater = bst.getIndexWriter(new Path(testDir, "testFindMinValue_" + storeType + ".idx"), + BSTIndexWriter creater = bst.getIndexWriter(new Path(testDir, "testFindMinValue_" + dataFormat + ".idx"), BSTIndex.TWO_LEVEL_INDEX, keySchema, comp); creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -563,10 +563,10 @@ public class TestBSTIndex { tuple = new VTuple(keySchema.size()); - BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testFindMinValue_" + storeType + ".idx"), + BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testFindMinValue_" + dataFormat + ".idx"), keySchema, comp); reader.open(); - scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -588,9 +588,9 @@ public class TestBSTIndex { @Test public void testMinMax() throws IOException { - meta = CatalogUtil.newTableMeta(storeType); + meta = CatalogUtil.newTableMeta(dataFormat); - Path tablePath = new Path(testDir, "testMinMax_" + storeType); + Path tablePath = new Path(testDir, "testMinMax_" + dataFormat); Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(meta, schema, tablePath); appender.init(); @@ -621,12 +621,12 @@ public class TestBSTIndex { BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); BSTIndex bst = new BSTIndex(conf); - BSTIndexWriter creater = bst.getIndexWriter(new Path(testDir, "testMinMax_" + storeType + ".idx"), + BSTIndexWriter creater = bst.getIndexWriter(new Path(testDir, "testMinMax_" + dataFormat + ".idx"), BSTIndex.TWO_LEVEL_INDEX, keySchema, comp); creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -647,7 +647,7 @@ public class TestBSTIndex { creater.close(); scanner.close(); - BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testMinMax_" + storeType + ".idx"), + BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testMinMax_" + dataFormat + ".idx"), keySchema, comp); reader.open(); @@ -694,9 +694,9 @@ public class TestBSTIndex { @Test public void testConcurrentAccess() throws IOException, InterruptedException { - meta = CatalogUtil.newTableMeta(storeType); + meta = CatalogUtil.newTableMeta(dataFormat); - Path tablePath = new Path(testDir, "testConcurrentAccess_" + storeType); + Path tablePath = new Path(testDir, "testConcurrentAccess_" + dataFormat); Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(meta, schema, tablePath); appender.init(); @@ -728,12 +728,12 @@ public class TestBSTIndex { BaseTupleComparator comp = new BaseTupleComparator(keySchema, sortKeys); BSTIndex bst = new BSTIndex(conf); - BSTIndexWriter creater = bst.getIndexWriter(new Path(testDir, "testConcurrentAccess_" + storeType + ".idx"), + BSTIndexWriter creater = bst.getIndexWriter(new Path(testDir, "testConcurrentAccess_" + dataFormat + ".idx"), BSTIndex.TWO_LEVEL_INDEX, keySchema, comp); creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -754,7 +754,7 @@ public class TestBSTIndex { creater.close(); scanner.close(); - BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testConcurrentAccess_" + storeType + ".idx"), + BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testConcurrentAccess_" + dataFormat + ".idx"), keySchema, comp); reader.open(); @@ -776,9 +776,9 @@ public class TestBSTIndex { @Test public void testFindValueDescOrder() throws IOException { - meta = CatalogUtil.newTableMeta(storeType); + meta = CatalogUtil.newTableMeta(dataFormat); - Path tablePath = new Path(testDir, "testFindValueDescOrder_" + storeType); + Path tablePath = new Path(testDir, "testFindValueDescOrder_" + dataFormat); Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(meta, schema, tablePath); appender.init(); @@ -811,12 +811,12 @@ public class TestBSTIndex { BSTIndex bst = new BSTIndex(conf); - BSTIndexWriter creater = bst.getIndexWriter(new Path(testDir, "testFindValueDescOrder_" + storeType + ".idx"), + BSTIndexWriter creater = bst.getIndexWriter(new Path(testDir, "testFindValueDescOrder_" + dataFormat + ".idx"), BSTIndex.TWO_LEVEL_INDEX, keySchema, comp); creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -839,10 +839,10 @@ public class TestBSTIndex { tuple = new VTuple(keySchema.size()); - BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testFindValueDescOrder_" + storeType + ".idx"), + BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testFindValueDescOrder_" + dataFormat + ".idx"), keySchema, comp); reader.open(); - scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -870,9 +870,9 @@ public class TestBSTIndex { @Test public void testFindNextKeyValueDescOrder() throws IOException { - meta = CatalogUtil.newTableMeta(storeType); + meta = CatalogUtil.newTableMeta(dataFormat); - Path tablePath = new Path(testDir, "testFindNextKeyValueDescOrder_" + storeType); + Path tablePath = new Path(testDir, "testFindNextKeyValueDescOrder_" + dataFormat); Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(meta, schema, tablePath); appender.init(); @@ -904,11 +904,11 @@ public class TestBSTIndex { BSTIndex bst = new BSTIndex(conf); BSTIndexWriter creater = bst.getIndexWriter(new Path(testDir, - "testFindNextKeyValueDescOrder_" + storeType + ".idx"), BSTIndex.TWO_LEVEL_INDEX, keySchema, comp); + "testFindNextKeyValueDescOrder_" + dataFormat + ".idx"), BSTIndex.TWO_LEVEL_INDEX, keySchema, comp); creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -930,14 +930,14 @@ public class TestBSTIndex { scanner.close(); - BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testFindNextKeyValueDescOrder_" + storeType + ".idx"), + BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testFindNextKeyValueDescOrder_" + dataFormat + ".idx"), keySchema, comp); reader.open(); assertEquals(keySchema, reader.getKeySchema()); assertEquals(comp, reader.getComparator()); - scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java index a001492..8262073 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java @@ -111,7 +111,7 @@ public class TestSingleCSVFileBSTIndex { creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner fileScanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()) + SeekableScanner fileScanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()) .getSeekableScanner(meta, schema, tablet.getProto(), schema); fileScanner.init(); Tuple keyTuple; @@ -136,7 +136,7 @@ public class TestSingleCSVFileBSTIndex { BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "FindValueInCSV.idx"), keySchema, comp); reader.open(); - fileScanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()) + fileScanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()) .getSeekableScanner(meta, schema, tablet.getProto(), schema); fileScanner.init(); for (int i = 0; i < TUPLE_NUM - 1; i++) { @@ -202,7 +202,7 @@ public class TestSingleCSVFileBSTIndex { creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner fileScanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()) + SeekableScanner fileScanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()) .getSeekableScanner(meta, schema, tablet.getProto(), schema); fileScanner.init(); Tuple keyTuple; @@ -224,7 +224,7 @@ public class TestSingleCSVFileBSTIndex { BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "FindNextKeyValueInCSV.idx"), keySchema, comp); reader.open(); - fileScanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()) + fileScanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()) .getSeekableScanner(meta, schema, tablet.getProto(), schema); fileScanner.init(); Tuple result; http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcFragment.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcFragment.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcFragment.java index bf9536e..1c963fd 100644 --- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcFragment.java +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcFragment.java @@ -94,7 +94,7 @@ public class JdbcFragment implements Fragment, Comparable<JdbcFragment>, Cloneab CatalogProtos.FragmentProto.Builder fragmentBuilder = CatalogProtos.FragmentProto.newBuilder(); fragmentBuilder.setId(this.inputSourceId); - fragmentBuilder.setStoreType("JDBC"); + fragmentBuilder.setDataFormat("JDBC"); fragmentBuilder.setContents(builder.buildPartial().toByteString()); return fragmentBuilder.build(); }
