Continue Cleaning Up File References and Splits 1. Make FileSplit an abstract class with two subclasses; Managed and Unmanaged. A Managed FileSplit can be mapped in a new subclass MappedFileSplit that maps a relative path to an IO device. UnmanagedFileSplit is for files outside the io devices. 2. Remove all usages of absolute paths in file split in test cases. The only remaining place is the write statement. 3. Fix some of the hidden issues in the tests that were working because of our use of the absolute paths. 4. Revert the decision of selecting the IO device to the CC.
Change-Id: I166af8f9b3a2257f94d7b05db94888fb7cb4c79e Reviewed-on: https://asterix-gerrit.ics.uci.edu/1359 Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: abdullah alamoudi <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/72aa1b74 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/72aa1b74 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/72aa1b74 Branch: refs/heads/master Commit: 72aa1b7422b60031d0136926661b802d1e6acb14 Parents: 3584bdd Author: Abdullah Alamoudi <[email protected]> Authored: Sun Nov 27 22:57:08 2016 -0800 Committer: abdullah alamoudi <[email protected]> Committed: Mon Nov 28 08:15:19 2016 -0800 ---------------------------------------------------------------------- .../LangExpressionToPlanTranslator.java | 3 +- .../api/http/servlet/ConnectorAPIServlet.java | 12 +- .../asterix/app/external/FeedOperations.java | 6 +- .../app/nc/AsterixNCAppRuntimeContext.java | 5 +- .../asterix/app/translator/QueryTranslator.java | 3 +- .../apache/asterix/file/DatasetOperations.java | 6 +- .../http/servlet/ConnectorAPIServletTest.java | 16 +- .../app/bootstrap/TestNodeController.java | 5 +- .../common/context/AsterixFileMapManager.java | 8 +- .../AsterixVirtualBufferCacheProvider.java | 2 +- .../common/context/DatasetLifecycleManager.java | 46 +- .../asterix/common/transactions/Resource.java | 14 + .../asterix/common/utils/StoragePathUtil.java | 3 +- .../factory/LocalFSInputStreamFactory.java | 22 +- ...xternalDatasetIndexesOperatorDescriptor.java | 2 +- .../apache/asterix/external/util/FeedUtils.java | 2 +- .../metadata/bootstrap/MetadataBootstrap.java | 6 +- .../ExternalBTreeLocalResourceMetadata.java | 2 +- ...rnalBTreeWithBuddyLocalResourceMetadata.java | 2 +- .../ExternalRTreeLocalResourceMetadata.java | 2 +- .../resource/LSMBTreeLocalResourceMetadata.java | 14 +- .../LSMInvertedIndexLocalResourceMetadata.java | 13 +- .../resource/LSMRTreeLocalResourceMetadata.java | 13 +- .../PersistentLocalResourceRepository.java | 16 +- ...ersistentLocalResourceRepositoryFactory.java | 6 +- .../piglet/metadata/FileSplitUtils.java | 3 +- .../piglet/metadata/PigletMetadataProvider.java | 4 +- .../data/device0/data/simple/int-part1.tbl | 7 + .../data/device0/data/simple/int-part2.tbl | 6 + .../data/simple/int-string-part1-split-0.tbl | 4 + .../data/simple/int-string-part1-split-1.tbl | 3 + .../device0/data/simple/int-string-part1.tbl | 7 + .../device0/data/tpch0.001/customer-part1.tbl | 75 + .../data/device0/data/tpch0.001/customer.tbl | 150 + .../data/device0/data/tpch0.001/lineitem.tbl | 6005 +++ .../data/device0/data/tpch0.001/nation.tbl | 25 + .../device0/data/tpch0.001/orders-part1.tbl | 750 + .../data/device0/data/tpch0.001/part.tbl | 200 + .../data/device0/data/tpch0.001/partsupp.tbl | 800 + .../data/device0/data/tpch0.001/region.tbl | 5 + .../data/device0/data/tpch0.001/supplier.tbl | 10 + .../data/device0/data/tpch0.001/tpch.ddl | 70 + .../device1/data/tpch0.001/customer-part2.tbl | 75 + .../device1/data/tpch0.001/orders-part2.tbl | 750 + .../data/device1/data/tpch0.001/orders.tbl | 1500 + .../algebricks-tests/data/simple/int-part1.tbl | 7 - .../algebricks-tests/data/simple/int-part2.tbl | 6 - .../data/simple/int-string-part1-split-0.tbl | 4 - .../data/simple/int-string-part1-split-1.tbl | 3 - .../data/simple/int-string-part1.tbl | 7 - .../data/tpch0.001/customer-part1.tbl | 75 - .../data/tpch0.001/customer-part2.tbl | 75 - .../data/tpch0.001/customer.tbl | 150 - .../data/tpch0.001/lineitem.tbl | 6005 --- .../algebricks-tests/data/tpch0.001/nation.tbl | 25 - .../data/tpch0.001/orders-part1.tbl | 750 - .../data/tpch0.001/orders-part2.tbl | 750 - .../algebricks-tests/data/tpch0.001/orders.tbl | 1500 - .../algebricks-tests/data/tpch0.001/part.tbl | 200 - .../data/tpch0.001/partsupp.tbl | 800 - .../algebricks-tests/data/tpch0.001/region.tbl | 5 - .../data/tpch0.001/supplier.tbl | 10 - .../algebricks-tests/data/tpch0.001/tpch.ddl | 70 - .../algebricks/algebricks-tests/pom.xml | 4 + .../tests/pushruntime/PushRuntimeTest.java | 77 +- .../util/AlgebricksHyracksIntegrationUtil.java | 14 +- .../org/apache/hyracks/api/io/FileSplit.java | 41 +- .../org/apache/hyracks/api/io/IIOManager.java | 30 +- .../apache/hyracks/api/io/ManagedFileSplit.java | 69 + .../apache/hyracks/api/io/MappedFileSplit.java | 66 + .../hyracks/api/io/UnmanagedFileSplit.java | 66 + .../org/apache/hyracks/control/nc/Joblet.java | 3 +- .../control/nc/NodeControllerService.java | 14 +- .../org/apache/hyracks/control/nc/Task.java | 9 +- .../apache/hyracks/control/nc/io/IOManager.java | 22 +- .../control/nc/io/WorkspaceFileFactory.java | 5 +- .../nc/partitions/MaterializedPartition.java | 4 +- .../partitions/MaterializedPartitionWriter.java | 3 +- .../MaterializingPipelinedPartition.java | 5 +- .../hyracks/hyracks-dataflow-std/pom.xml | 1 - .../AbstractFileWriteOperatorDescriptor.java | 25 +- .../std/file/FileRemoveOperatorDescriptor.java | 2 +- .../dataflow/std/file/IFileSplitProvider.java | 3 +- .../dataflow/std/file/IRecordWriter.java | 4 +- .../file/LineFileWriteOperatorDescriptor.java | 23 +- .../file/RecordFileScanOperatorDescriptor.java | 3 +- .../hyracks/dataflow/std/file/RecordWriter.java | 35 +- .../btree/client/InsertPipelineExample.java | 8 +- .../examples/btree/client/JobHelper.java | 6 +- .../client/PrimaryIndexBulkLoadExample.java | 6 +- .../btree/client/PrimaryIndexSearchExample.java | 6 +- .../client/SecondaryIndexBulkLoadExample.java | 9 +- .../client/SecondaryIndexSearchExample.java | 8 +- .../data/cleanednumbereddblptitles.txt | 50 - .../hyracks-integration-tests/data/dblp.txt | 10 - .../data/device0/data/nc1/words.txt | 1000 + .../data/orders-with-locations-part1.txt | 557 + .../data/orders-with-locations-part2.txt | 193 + .../device0/data/tpch0.001/customer-part1.tbl | 75 + .../data/device0/data/tpch0.001/customer.tbl | 150 + .../data/device0/data/tpch0.001/customer3.tbl | 20000 +++++++++ .../data/device0/data/tpch0.001/customer4.tbl | 40000 +++++++++++++++++ .../data/device0/data/tpch0.001/lineitem.tbl | 6005 +++ .../data/device0/data/tpch0.001/nation.tbl | 25 + .../device0/data/tpch0.001/orders-part1.tbl | 750 + .../data/device0/data/tpch0.001/part.tbl | 200 + .../data/device0/data/tpch0.001/partsupp.tbl | 800 + .../data/device0/data/tpch0.001/region.tbl | 5 + .../data/device0/data/tpch0.001/supplier.tbl | 10 + .../data/device0/data/tpch0.001/tpch.ddl | 70 + .../device0/data/tpch0.002/orders-part2.tbl | 750 + .../device1/data/cleanednumbereddblptitles.txt | 50 + .../data/device1/data/dblp.txt | 10 + .../data/device1/data/spatial.txt | 1079 + .../device1/data/tpch0.001/customer-part2.tbl | 75 + .../device1/data/tpch0.001/orders-part2.tbl | 750 + .../data/device1/data/tpch0.001/orders.tbl | 1500 + .../data/device1/data/tpch0.001/orders1.tbl | 100 + .../data/device1/data/tpch0.001/orders4.tbl | 40000 +++++++++++++++++ .../data/device1/data/tpch0.002/lineitem.tbl | 6005 +++ .../data/device1/data/wordcount.tsv | 17217 +++++++ .../data/device1/data/words.txt | 1000 + .../data/orders-with-locations-part1.txt | 557 - .../data/orders-with-locations-part2.txt | 193 - .../hyracks-integration-tests/data/spatial.txt | 1079 - .../data/tpch0.001/customer-part1.tbl | 75 - .../data/tpch0.001/customer-part2.tbl | 75 - .../data/tpch0.001/customer.tbl | 150 - .../data/tpch0.001/customer3.tbl | 20000 --------- .../data/tpch0.001/customer4.tbl | 40000 ----------------- .../data/tpch0.001/lineitem.tbl | 6005 --- .../data/tpch0.001/nation.tbl | 25 - .../data/tpch0.001/orders-part1.tbl | 750 - .../data/tpch0.001/orders-part2.tbl | 750 - .../data/tpch0.001/orders.tbl | 1500 - .../data/tpch0.001/orders1.tbl | 100 - .../data/tpch0.001/orders4.tbl | 40000 ----------------- .../data/tpch0.001/part.tbl | 200 - .../data/tpch0.001/partsupp.tbl | 800 - .../data/tpch0.001/region.tbl | 5 - .../data/tpch0.001/supplier.tbl | 10 - .../data/tpch0.001/tpch.ddl | 70 - .../data/wordcount.tsv | 17217 ------- .../hyracks-integration-tests/data/words.txt | 1000 - .../am/btree/AbstractBTreeOperatorTest.java | 43 +- .../BTreePrimaryIndexScanOperatorTest.java | 3 +- .../BTreePrimaryIndexSearchOperatorTest.java | 3 +- .../BTreePrimaryIndexStatsOperatorTest.java | 3 +- .../BTreeSecondaryIndexInsertOperatorTest.java | 3 +- .../BTreeSecondaryIndexSearchOperatorTest.java | 7 +- .../BTreeSecondaryIndexUpsertOperatorTest.java | 3 +- .../am/common/ITreeIndexOperatorTestHelper.java | 7 - .../am/common/LSMTreeOperatorTestHelper.java | 55 - .../tests/am/common/TreeOperatorTestHelper.java | 27 +- .../LSMBTreePrimaryIndexScanOperatorTest.java | 5 +- .../LSMBTreePrimaryIndexSearchOperatorTest.java | 5 +- ...SMBTreeSecondaryIndexInsertOperatorTest.java | 5 +- ...SMBTreeSecondaryIndexSearchOperatorTest.java | 5 +- ...SMRTreeSecondaryIndexInsertOperatorTest.java | 5 +- ...SMRTreeSecondaryIndexSearchOperatorTest.java | 5 +- ...rTuplesSecondaryIndexInsertOperatorTest.java | 5 +- ...rTuplesSecondaryIndexSearchOperatorTest.java | 5 +- .../am/rtree/AbstractRTreeOperatorTest.java | 25 +- .../RTreeSecondaryIndexInsertOperatorTest.java | 16 +- .../RTreeSecondaryIndexScanOperatorTest.java | 12 +- .../RTreeSecondaryIndexSearchOperatorTest.java | 14 +- .../RTreeSecondaryIndexStatsOperatorTest.java | 5 +- .../comm/SerializationDeserializationTest.java | 4 +- .../integration/AbstractIntegrationTest.java | 46 +- .../AbstractMultiNCIntegrationTest.java | 6 +- .../tests/integration/AggregationTest.java | 5 +- .../tests/integration/CountOfCountsTest.java | 26 +- .../tests/integration/HeapSortMergeTest.java | 19 +- .../tests/integration/JobFailureTest.java | 2 +- .../integration/LocalityAwareConnectorTest.java | 14 +- .../integration/ReplicateOperatorTest.java | 8 +- .../tests/integration/ScanPrintTest.java | 24 +- .../tests/integration/SortMergeTest.java | 13 +- ...TPCHCustomerOptimizedHybridHashJoinTest.java | 60 +- .../TPCHCustomerOrderHashJoinTest.java | 96 +- .../TPCHCustomerOrderNestedLoopJoinTest.java | 51 +- .../hyracks/tests/integration/UnionTest.java | 5 +- .../integration/VSizeFrameSortMergeTest.java | 15 +- .../examples/text/client/WordCountMain.java | 3 +- .../hyracks/examples/tpch/client/Common.java | 18 +- .../hyracks/examples/tpch/client/Groupby.java | 12 +- .../hyracks/examples/tpch/client/Sort.java | 5 +- .../am/common/dataflow/IndexDataflowHelper.java | 2 +- .../TreeIndexStatsOperatorNodePushable.java | 2 +- .../am/common/util/IndexFileNameUtil.java | 2 +- .../storage/am/lsm/btree/impls/LSMBTree.java | 2 +- .../am/lsm/btree/impls/LSMBTreeFileManager.java | 16 +- .../impls/LSMBTreeWithBuddyFileManager.java | 27 +- .../impls/AbstractLSMIndexFileManager.java | 14 +- .../invertedindex/impls/LSMInvertedIndex.java | 6 +- .../impls/LSMInvertedIndexFileManager.java | 12 +- .../impls/PartitionedLSMInvertedIndex.java | 2 +- .../ondisk/OnDiskInvertedIndexFactory.java | 2 +- .../PartitionedOnDiskInvertedIndexFactory.java | 2 +- .../invertedindex/util/InvertedIndexUtils.java | 6 +- .../am/lsm/rtree/impls/AbstractLSMRTree.java | 4 +- .../am/lsm/rtree/impls/LSMRTreeFileManager.java | 24 +- .../am/btree/OrderedIndexMultiThreadTest.java | 9 +- .../am/rtree/AbstractRTreeMultiThreadTest.java | 13 +- .../hyracks/test/support/TestJobletContext.java | 2 +- .../test/support/TestNCApplicationContext.java | 4 +- .../TestStorageManagerComponentHolder.java | 5 +- .../hyracks/test/support/TestTaskContext.java | 2 +- .../util/BloomFilterTestHarness.java | 2 +- .../storage/am/btree/util/BTreeTestHarness.java | 2 +- .../am/lsm/btree/LSMBTreeBulkLoadTest.java | 3 +- .../am/lsm/btree/LSMBTreeDeleteTest.java | 3 +- .../am/lsm/btree/LSMBTreeExamplesTest.java | 3 +- .../am/lsm/btree/LSMBTreeInsertTest.java | 3 +- .../storage/am/lsm/btree/LSMBTreeMergeTest.java | 3 +- .../am/lsm/btree/LSMBTreeMultiBulkLoadTest.java | 3 +- .../am/lsm/btree/LSMBTreeUpdateTest.java | 3 +- .../multithread/LSMBTreeMultiThreadTest.java | 3 +- .../am/lsm/btree/perf/BTreeBulkLoadRunner.java | 4 +- .../storage/am/lsm/btree/perf/BTreeRunner.java | 3 +- .../am/lsm/btree/perf/InMemoryBTreeRunner.java | 5 +- .../am/lsm/btree/perf/LSMTreeRunner.java | 5 +- .../am/lsm/btree/util/LSMBTreeTestHarness.java | 5 +- .../am/lsm/common/DummyLSMIndexFileManager.java | 2 +- .../am/lsm/common/LSMIndexFileManagerTest.java | 9 +- .../am/lsm/common/VirtualBufferCacheTest.java | 2 +- .../common/AbstractInvertedIndexTest.java | 8 +- .../common/LSMInvertedIndexTestHarness.java | 5 +- .../LSMInvertedIndexMultiThreadTest.java | 11 +- .../OnDiskInvertedIndexLifecycleTest.java | 4 +- .../util/LSMInvertedIndexTestContext.java | 4 +- .../am/lsm/rtree/LSMRTreeBulkLoadTest.java | 3 +- .../am/lsm/rtree/LSMRTreeDeleteTest.java | 3 +- .../am/lsm/rtree/LSMRTreeExamplesTest.java | 3 +- .../am/lsm/rtree/LSMRTreeInsertTest.java | 3 +- .../storage/am/lsm/rtree/LSMRTreeMergeTest.java | 3 +- ...SMRTreeWithAntiMatterTuplesBulkLoadTest.java | 3 +- .../LSMRTreeWithAntiMatterTuplesDeleteTest.java | 3 +- ...SMRTreeWithAntiMatterTuplesExamplesTest.java | 3 +- .../LSMRTreeWithAntiMatterTuplesInsertTest.java | 3 +- .../LSMRTreeWithAntiMatterTuplesMergeTest.java | 3 +- .../multithread/LSMRTreeMultiThreadTest.java | 3 +- ...TreeWithAntiMatterTuplesMultiThreadTest.java | 3 +- .../am/lsm/rtree/util/LSMRTreeTestHarness.java | 5 +- .../storage/am/rtree/RTreeBulkLoadTest.java | 3 +- .../storage/am/rtree/RTreeDeleteTest.java | 3 +- .../storage/am/rtree/RTreeExamplesTest.java | 3 +- .../storage/am/rtree/RTreeInsertTest.java | 3 +- .../storage/am/rtree/RTreeSearchCursorTest.java | 4 +- .../rtree/multithread/RTreeMultiThreadTest.java | 3 +- .../am/rtree/utils/AbstractRTreeTest.java | 3 +- .../am/rtree/utils/RTreeTestHarness.java | 5 +- .../common/BufferCacheRegressionTest.java | 6 +- .../hyracks/storage/common/BufferCacheTest.java | 20 +- 254 files changed, 149768 insertions(+), 141965 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java index fc982d8..977a7b0 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java @@ -157,6 +157,7 @@ import org.apache.hyracks.algebricks.core.algebra.properties.LocalOrderProperty; import org.apache.hyracks.algebricks.core.algebra.properties.OrderColumn; import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil; import org.apache.hyracks.api.io.FileSplit; +import org.apache.hyracks.api.io.ManagedFileSplit; /** * Each visit returns a pair of an operator and a variable. The variable @@ -735,7 +736,7 @@ class LangExpressionToPlanTranslator String filePath = outputDir + System.getProperty("file.separator") + OUTPUT_FILE_PREFIX + outputFileID.incrementAndGet(); AsterixMetadataProperties metadataProperties = AsterixAppContextInfo.INSTANCE.getMetadataProperties(); - return new FileSplit(metadataProperties.getMetadataNodeName(), filePath, true); + return new ManagedFileSplit(metadataProperties.getMetadataNodeName(), filePath); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java index 7982f12..ac1a5aa 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java @@ -140,8 +140,7 @@ public class ConnectorAPIServlet extends HttpServlet { for (FileSplit split : fileSplits) { String ipAddress = nodeMap.get(split.getNodeName()).getNetworkAddress().getAddress().toString(); String path = split.getPath(); - boolean relative = split.isManaged(); - FilePartition partition = new FilePartition(ipAddress, path, relative); + FilePartition partition = new FilePartition(ipAddress, path); partititons.put(partition.toJSONObject()); } // Generates the response object which contains the splits. @@ -152,12 +151,10 @@ public class ConnectorAPIServlet extends HttpServlet { class FilePartition { private final String ipAddress; private final String path; - private final boolean relative; - public FilePartition(String ipAddress, String path, boolean relative) { + public FilePartition(String ipAddress, String path) { this.ipAddress = ipAddress; this.path = path; - this.relative = relative; } public String getIPAddress() { @@ -168,10 +165,6 @@ class FilePartition { return path; } - public boolean isRelative() { - return relative; - } - @Override public String toString() { return ipAddress + ":" + path; @@ -181,7 +174,6 @@ class FilePartition { JSONObject partition = new JSONObject(); partition.put("ip", ipAddress); partition.put("path", path); - partition.put("relative", relative); return partition; } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedOperations.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedOperations.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedOperations.java index 66a55c7..34746d3 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedOperations.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedOperations.java @@ -87,7 +87,7 @@ public class FeedOperations { AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, nullSink, ingesterPc); spec.connect(new OneToOneConnectorDescriptor(spec), feedIngestor, 0, nullSink, 0); spec.addRoot(nullSink); - return new Pair<JobSpecification, IAdapterFactory>(spec, adapterFactory); + return new Pair<>(spec, adapterFactory); } /** @@ -131,7 +131,7 @@ public class FeedOperations { AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, nullSink, messengerPc); spec.connect(new OneToOneConnectorDescriptor(spec), feedMessenger, 0, nullSink, 0); spec.addRoot(nullSink); - return new Pair<JobSpecification, Boolean>(spec, terminateIntakeJob); + return new Pair<>(spec, terminateIntakeJob); } catch (AlgebricksException e) { throw new AsterixException(e); @@ -146,7 +146,7 @@ public class FeedOperations { new AlgebricksAbsolutePartitionConstraint(locations.toArray(new String[] {})); FeedMessageOperatorDescriptor feedMessenger = new FeedMessageOperatorDescriptor(jobSpec, feedConenctionId, feedMessage); - return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(feedMessenger, partitionConstraint); + return new Pair<>(feedMessenger, partitionConstraint); } private static Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildDisconnectFeedMessengerRuntime( http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/AsterixNCAppRuntimeContext.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/AsterixNCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/AsterixNCAppRuntimeContext.java index 9818116..af50b71 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/AsterixNCAppRuntimeContext.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/AsterixNCAppRuntimeContext.java @@ -80,7 +80,6 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.api.lifecycle.ILifeCycleComponent; import org.apache.hyracks.api.lifecycle.ILifeCycleComponentManager; -import org.apache.hyracks.control.nc.io.IOManager; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker; @@ -125,7 +124,7 @@ public class AsterixNCAppRuntimeContext implements IAsterixAppRuntimeContext, IA private ILSMIOOperationScheduler lsmIOScheduler; private PersistentLocalResourceRepository localResourceRepository; - private IOManager ioManager; + private IIOManager ioManager; private boolean isShuttingdown; private ActiveManager activeManager; @@ -168,7 +167,7 @@ public class AsterixNCAppRuntimeContext implements IAsterixAppRuntimeContext, IA Logger.getLogger("org.apache.asterix").setLevel(externalProperties.getLogLevel()); Logger.getLogger("org.apache.hyracks").setLevel(externalProperties.getLogLevel()); - ioManager = (IOManager) ncApplicationContext.getIoManager(); + ioManager = ncApplicationContext.getIoManager(); threadExecutor = new AsterixThreadExecutor(ncApplicationContext.getThreadFactory()); fileMapManager = new AsterixFileMapManager(ioManager); ICacheMemoryAllocator allocator = new HeapBufferAllocator(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index 36adb1f..7bf5195 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -198,6 +198,7 @@ import org.apache.hyracks.api.dataset.IHyracksDataset; import org.apache.hyracks.api.dataset.ResultSetId; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileSplit; +import org.apache.hyracks.api.io.UnmanagedFileSplit; import org.apache.hyracks.api.job.JobId; import org.apache.hyracks.api.job.JobSpecification; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory; @@ -419,7 +420,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen throws InstantiationException, IllegalAccessException, ClassNotFoundException { WriteStatement ws = (WriteStatement) stmt; File f = new File(ws.getFileName()); - FileSplit outputFile = new FileSplit(ws.getNcName().getValue(), f.getPath(), false); + FileSplit outputFile = new UnmanagedFileSplit(ws.getNcName().getValue(), f.getPath()); IAWriterFactory writerFactory = null; if (ws.getWriterClassName() != null) { writerFactory = (IAWriterFactory) Class.forName(ws.getWriterClassName()).newInstance(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/DatasetOperations.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/DatasetOperations.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/DatasetOperations.java index 675cb80..f970e50 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/DatasetOperations.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/DatasetOperations.java @@ -174,7 +174,7 @@ public class DatasetOperations { FileSplit[] fs = splitsAndConstraint.first.getFileSplits(); StringBuilder sb = new StringBuilder(); for (int i = 0; i < fs.length; i++) { - sb.append(stringOf(fs[i]) + " "); + sb.append(fs[i] + " "); } LOGGER.info("CREATING File Splits: " + sb.toString()); @@ -204,10 +204,6 @@ public class DatasetOperations { return spec; } - private static String stringOf(FileSplit fs) { - return fs.getNodeName() + ":" + fs.getPath(); - } - public static JobSpecification compactDatasetJobSpec(Dataverse dataverse, String datasetName, MetadataProvider metadata) throws AsterixException, AlgebricksException { String dataverseName = dataverse.getDataverseName(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorAPIServletTest.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorAPIServletTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorAPIServletTest.java index 5d87b00..71b2df5 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorAPIServletTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorAPIServletTest.java @@ -49,6 +49,7 @@ import org.apache.hyracks.api.client.IHyracksClientConnection; import org.apache.hyracks.api.client.NodeControllerInfo; import org.apache.hyracks.api.comm.NetworkAddress; import org.apache.hyracks.api.io.FileSplit; +import org.apache.hyracks.api.io.ManagedFileSplit; import org.json.JSONArray; import org.json.JSONObject; import org.json.JSONTokener; @@ -123,8 +124,8 @@ public class ConnectorAPIServletTest { ConnectorAPIServlet servlet = new ConnectorAPIServlet(); JSONObject actualResponse = new JSONObject(); FileSplit[] splits = new FileSplit[2]; - splits[0] = new FileSplit("asterix_nc1", "foo1", true); - splits[1] = new FileSplit("asterix_nc2", "foo2", true); + splits[0] = new ManagedFileSplit("asterix_nc1", "foo1"); + splits[1] = new ManagedFileSplit("asterix_nc2", "foo2"); Map<String, NodeControllerInfo> nodeMap = new HashMap<>(); NodeControllerInfo mockInfo1 = mock(NodeControllerInfo.class); NodeControllerInfo mockInfo2 = mock(NodeControllerInfo.class); @@ -141,12 +142,9 @@ public class ConnectorAPIServletTest { // Calls ConnectorAPIServlet.formResponseObject. nodeMap.put("asterix_nc1", mockInfo1); nodeMap.put("asterix_nc2", mockInfo2); - PA.invokeMethod(servlet, - "formResponseObject(" + JSONObject.class.getName() + ", " + FileSplit.class.getName() + "[], " - + ARecordType.class.getName() + ", " + String.class.getName() + ", boolean, " + Map.class - .getName() + ")", - actualResponse, splits, recordType, primaryKey, true, nodeMap); - + PA.invokeMethod(servlet, "formResponseObject(" + JSONObject.class.getName() + ", " + FileSplit.class.getName() + + "[], " + ARecordType.class.getName() + ", " + String.class.getName() + ", boolean, " + Map.class + .getName() + ")", actualResponse, splits, recordType, primaryKey, true, nodeMap); // Constructs expected response. JSONObject expectedResponse = new JSONObject(); expectedResponse.put("temp", true); @@ -156,11 +154,9 @@ public class ConnectorAPIServletTest { JSONObject element1 = new JSONObject(); element1.put("ip", "127.0.0.1"); element1.put("path", splits[0].getPath()); - element1.put("relative", true); JSONObject element2 = new JSONObject(); element2.put("ip", "127.0.0.2"); element2.put("path", splits[1].getPath()); - element2.put("relative", true); splitsArray.put(element1); splitsArray.put(element2); expectedResponse.put("splits", splitsArray); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java index 6ab4b80..e2c6a04 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java @@ -66,6 +66,7 @@ import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.dataflow.value.RecordDescriptor; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileSplit; +import org.apache.hyracks.api.io.ManagedFileSplit; import org.apache.hyracks.api.job.JobId; import org.apache.hyracks.api.job.JobSpecification; import org.apache.hyracks.api.util.HyracksConstants; @@ -250,8 +251,8 @@ public class TestNodeController { } public ConstantFileSplitProvider getFileSplitProvider(Dataset dataset) { - FileSplit fileSplit = new FileSplit(ExecutionTestUtil.integrationUtil.ncs[0].getId(), - dataset.getDataverseName() + File.separator + dataset.getDatasetName(), true); + FileSplit fileSplit = new ManagedFileSplit(ExecutionTestUtil.integrationUtil.ncs[0].getId(), + dataset.getDataverseName() + File.separator + dataset.getDatasetName()); return new ConstantFileSplitProvider(new FileSplit[] { fileSplit }); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixFileMapManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixFileMapManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixFileMapManager.java index b330271..e74659d 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixFileMapManager.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixFileMapManager.java @@ -23,18 +23,18 @@ import java.util.Map; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; -import org.apache.hyracks.control.nc.io.IOManager; +import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.common.file.IFileMapManager; public class AsterixFileMapManager implements IFileMapManager { private static final long serialVersionUID = 1L; - private final transient IOManager ioManager; + private final transient IIOManager ioManager; private final Map<Integer, String> id2nameMap = new HashMap<>(); private final Map<String, Integer> name2IdMap = new HashMap<>(); private int idCounter = 0; - public AsterixFileMapManager(IOManager ioManager) { + public AsterixFileMapManager(IIOManager ioManager) { this.ioManager = ioManager; } @@ -44,7 +44,7 @@ public class AsterixFileMapManager implements IFileMapManager { if (fName == null) { throw new HyracksDataException("No mapping found for id: " + fileId); } - return ioManager.getFileRef(fName, false); + return ioManager.resolveAbsolutePath(fName); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java index 685d12d..6fbbfe4 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java @@ -46,7 +46,7 @@ public class AsterixVirtualBufferCacheProvider implements IVirtualBufferCachePro final int partition = ctx.getTaskAttemptId().getTaskId().getPartition(); IIOManager ioManager = ctx.getIOManager(); FileSplit fileSplit = fileSplitProvider.getFileSplits()[partition]; - FileReference fileRef = ioManager.getFileRef(fileSplit.getPath(), fileSplit.isManaged()); + FileReference fileRef = fileSplit.getFileReference(ioManager); IODeviceHandle device = fileRef.getDeviceHandle(); List<IODeviceHandle> devices = ioManager.getIODevices(); int deviceId = 0; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java index a0fe996..fd4a9b5 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java @@ -166,34 +166,34 @@ public class DatasetLifecycleManager implements IDatasetLifecycleManager, ILifeC @Override public synchronized void open(String resourcePath) throws HyracksDataException { - validateDatasetLifecycleManagerState(); - int did = getDIDfromResourcePath(resourcePath); - long resourceID = getResourceIDfromResourcePath(resourcePath); + validateDatasetLifecycleManagerState(); + int did = getDIDfromResourcePath(resourcePath); + long resourceID = getResourceIDfromResourcePath(resourcePath); - DatasetResource dsr = datasets.get(did); - DatasetInfo dsInfo = dsr.getDatasetInfo(); - if (dsInfo == null || !dsInfo.isRegistered()) { - throw new HyracksDataException( - "Failed to open index with resource ID " + resourceID + " since it does not exist."); - } + DatasetResource dsr = datasets.get(did); + DatasetInfo dsInfo = dsr.getDatasetInfo(); + if (dsInfo == null || !dsInfo.isRegistered()) { + throw new HyracksDataException( + "Failed to open index with resource ID " + resourceID + " since it does not exist."); + } - IndexInfo iInfo = dsInfo.getIndexes().get(resourceID); - if (iInfo == null) { - throw new HyracksDataException( - "Failed to open index with resource ID " + resourceID + " since it does not exist."); - } + IndexInfo iInfo = dsInfo.getIndexes().get(resourceID); + if (iInfo == null) { + throw new HyracksDataException( + "Failed to open index with resource ID " + resourceID + " since it does not exist."); + } - dsr.open(true); - dsr.touch(); + dsr.open(true); + dsr.touch(); - if (!iInfo.isOpen()) { - ILSMOperationTracker opTracker = iInfo.getIndex().getOperationTracker(); - synchronized (opTracker) { - iInfo.getIndex().activate(); - } - iInfo.setOpen(true); + if (!iInfo.isOpen()) { + ILSMOperationTracker opTracker = iInfo.getIndex().getOperationTracker(); + synchronized (opTracker) { + iInfo.getIndex().activate(); } - iInfo.touch(); + iInfo.setOpen(true); + } + iInfo.touch(); } private boolean evictCandidateDataset() throws HyracksDataException { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/Resource.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/Resource.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/Resource.java index d649782..e781029 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/Resource.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/Resource.java @@ -19,10 +19,13 @@ package org.apache.asterix.common.transactions; import java.io.Serializable; +import java.util.List; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.io.IIOManager; +import org.apache.hyracks.api.io.IODeviceHandle; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex; import org.apache.hyracks.storage.common.file.LocalResource; @@ -58,4 +61,15 @@ public abstract class Resource implements Serializable { public abstract ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, LocalResource resource) throws HyracksDataException; + + public static int getIoDeviceNum(IIOManager ioManager, IODeviceHandle deviceHandle) { + List<IODeviceHandle> ioDevices = ioManager.getIODevices(); + for (int i = 0; i < ioDevices.size(); i++) { + IODeviceHandle device = ioDevices.get(i); + if (device == deviceHandle) { + return i; + } + } + return -1; + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java index 5816450..34af5c3 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java @@ -26,6 +26,7 @@ import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConst import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileSplit; +import org.apache.hyracks.api.io.MappedFileSplit; import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.log4j.Level; @@ -50,7 +51,7 @@ public class StoragePathUtil { } public static FileSplit getFileSplitForClusterPartition(ClusterPartition partition, String relativePath) { - return new FileSplit(partition.getActiveNodeId(), relativePath, true); + return new MappedFileSplit(partition.getActiveNodeId(), relativePath, partition.getIODeviceNum()); } public static String prepareStoragePartitionPath(String storageDirName, int partitonId) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java index 7aeb0bc..f877796 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/LocalFSInputStreamFactory.java @@ -39,7 +39,7 @@ import org.apache.asterix.external.util.NodeResolverFactory; import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.api.io.FileSplit; +import org.apache.hyracks.api.io.UnmanagedFileSplit; public class LocalFSInputStreamFactory implements IInputStreamFactory { @@ -49,7 +49,7 @@ public class LocalFSInputStreamFactory implements IInputStreamFactory { protected static final Logger LOGGER = Logger.getLogger(LocalFSInputStreamFactory.class.getName()); protected static INodeResolver nodeResolver; protected Map<String, String> configuration; - protected FileSplit[] inputFileSplits; + protected UnmanagedFileSplit[] inputFileSplits; protected boolean isFeed; protected String expression; // transient fields (They don't need to be serialized and transferred) @@ -64,7 +64,7 @@ public class LocalFSInputStreamFactory implements IInputStreamFactory { ArrayList<Path> inputResources = new ArrayList<>(); for (int i = 0; i < inputFileSplits.length; i++) { if (inputFileSplits[i].getNodeName().equals(nodeName)) { - inputResources.add(inputFileSplits[i].getFile(ctx.getIOManager()).toPath()); + inputResources.add(inputFileSplits[i].getFile().toPath()); } } watcher = new FileSystemWatcher(inputResources, expression, isFeed); @@ -100,9 +100,9 @@ public class LocalFSInputStreamFactory implements IInputStreamFactory { private void configureFileSplits(String[] splits) throws AsterixException { INodeResolver resolver = getNodeResolver(); if (inputFileSplits == null) { - inputFileSplits = new FileSplit[splits.length]; - String nodeName; - String nodeLocalPath; + inputFileSplits = new UnmanagedFileSplit[splits.length]; + String node; + String path; int count = 0; String trimmedValue; for (String splitPath : splits) { @@ -111,10 +111,9 @@ public class LocalFSInputStreamFactory implements IInputStreamFactory { throw new AsterixException( "Invalid path: " + splitPath + "\nUsage- path=\"Host://Absolute File Path\""); } - nodeName = resolver.resolveNode(trimmedValue.split(":")[0]); - nodeLocalPath = trimmedValue.split("://")[1]; - FileSplit fileSplit = new FileSplit(nodeName, nodeLocalPath, false); - inputFileSplits[count++] = fileSplit; + node = resolver.resolveNode(trimmedValue.split(":")[0]); + path = trimmedValue.split("://")[1]; + inputFileSplits[count++] = new UnmanagedFileSplit(node, path); } } } @@ -122,8 +121,7 @@ public class LocalFSInputStreamFactory implements IInputStreamFactory { private void configurePartitionConstraint() throws AsterixException { Set<String> locs = new TreeSet<>(); for (int i = 0; i < inputFileSplits.length; i++) { - String location = inputFileSplits[i].getNodeName(); - locs.add(location); + locs.add(inputFileSplits[i].getNodeName()); } constraints = new AlgebricksAbsolutePartitionConstraint(locs.toArray(new String[locs.size()])); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractExternalDatasetIndexesOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractExternalDatasetIndexesOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractExternalDatasetIndexesOperatorDescriptor.java index 078e4f8..8080f81 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractExternalDatasetIndexesOperatorDescriptor.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractExternalDatasetIndexesOperatorDescriptor.java @@ -81,7 +81,7 @@ public abstract class AbstractExternalDatasetIndexesOperatorDescriptor public void initialize() throws HyracksDataException { try { FileSplit fileSplit = fileIndexInfo.getFileSplitProvider().getFileSplits()[partition]; - FileReference fileRef = fileSplit.getFileRef(ctx.getIOManager()); + FileReference fileRef = fileSplit.getFileReference(ctx.getIOManager()); // only in partition of device id = 0, we perform the operation on the files index if (fileRef.getDeviceHandle() == ctx.getIOManager().getIODevices().get(0)) { performOpOnIndex(filesIndexDataflowHelperFactory, ctx, fileIndexInfo, partition); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java index 303f76d..2ca9f82 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java @@ -100,7 +100,7 @@ public class FeedUtils { } public static FileReference getAbsoluteFileRef(String relativePath, int ioDeviceId, IIOManager ioManager) { - return ioManager.getFileRef(ioDeviceId, relativePath); + return ioManager.getFileReference(ioDeviceId, relativePath); } public static FeedLogManager getFeedLogManager(IHyracksTaskContext ctx, int partition, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java index 17ef709..63f53a6 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java @@ -341,7 +341,7 @@ public class MetadataBootstrap { String metadataPartitionPath = StoragePathUtil.prepareStoragePartitionPath( ClusterProperties.INSTANCE.getStorageDirectoryName(), metadataPartition.getPartitionId()); String resourceName = metadataPartitionPath + File.separator + index.getFileNameRelativePath(); - FileReference file = ioManager.getFileRef(metadataDeviceId, resourceName); + FileReference file = ioManager.getFileReference(metadataDeviceId, resourceName); // this should not be done this way. dataset lifecycle manager shouldn't return virtual buffer caches for // a dataset that was not yet created @@ -388,7 +388,9 @@ public class MetadataBootstrap { + " to intialize as a new instance. (WARNING: all data will be lost.)"); } resourceID = resource.getId(); - assert (index.getResourceID() == resource.getId()); + if (index.getResourceID() != resource.getId()) { + throw new HyracksDataException("Resource Id doesn't match expected metadata index resource id"); + } lsmBtree = (LSMBTree) dataLifecycleManager.get(file.getRelativePath()); if (lsmBtree == null) { lsmBtree = LSMBTreeUtils.createLSMTree(ioManager, virtualBufferCaches, file, bufferCache, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java index 3083cfe..79f2780 100644 --- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java +++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java @@ -50,7 +50,7 @@ public class ExternalBTreeLocalResourceMetadata extends LSMBTreeLocalResourceMet public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, LocalResource resource) throws HyracksDataException { IIOManager ioManager = runtimeContextProvider.getIOManager(); - FileReference file = ioManager.getFileRef(resource.getPath(), true); + FileReference file = ioManager.resolve(resource.getPath()); LSMBTree lsmBTree = LSMBTreeUtils.createExternalBTree(ioManager, file, runtimeContextProvider.getBufferCache(), runtimeContextProvider.getFileMapManager(), typeTraits, cmpFactories, bloomFilterKeyFields, runtimeContextProvider.getBloomFilterFalsePositiveRate(), http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java index 3a486a5..32aa764 100644 --- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java +++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java @@ -63,7 +63,7 @@ public class ExternalBTreeWithBuddyLocalResourceMetadata extends Resource { public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, LocalResource resource) throws HyracksDataException { IIOManager ioManager = runtimeContextProvider.getIOManager(); - FileReference file = ioManager.getFileRef(resource.getPath(), true); + FileReference file = ioManager.resolve(resource.getPath()); return LSMBTreeUtils.createExternalBTreeWithBuddy(ioManager, file, runtimeContextProvider.getBufferCache(), runtimeContextProvider.getFileMapManager(), typeTraits, btreeCmpFactories, runtimeContextProvider.getBloomFilterFalsePositiveRate(), http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java index 7f6c4a3..e3663fd 100644 --- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java +++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java @@ -58,7 +58,7 @@ public class ExternalRTreeLocalResourceMetadata extends LSMRTreeLocalResourceMet public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, LocalResource resource) throws HyracksDataException { IIOManager ioManager = runtimeContextProvider.getIOManager(); - FileReference file = ioManager.getFileRef(resource.getPath(), true); + FileReference file = ioManager.resolve(resource.getPath()); try { return LSMRTreeUtils.createExternalRTree(ioManager, file, runtimeContextProvider.getBufferCache(), runtimeContextProvider.getFileMapManager(), typeTraits, rtreeCmpFactories, btreeCmpFactories, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java index 9c52241..8eeffd2 100644 --- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java +++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java @@ -18,7 +18,6 @@ */ package org.apache.asterix.transaction.management.resource; -import java.util.List; import java.util.Map; import org.apache.asterix.common.api.IDatasetLifecycleManager; @@ -31,7 +30,6 @@ import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; -import org.apache.hyracks.api.io.IODeviceHandle; import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree; import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex; @@ -76,16 +74,8 @@ public class LSMBTreeLocalResourceMetadata extends Resource { public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, LocalResource resource) throws HyracksDataException { IIOManager ioManager = runtimeContextProvider.getIOManager(); - FileReference file = ioManager.getFileRef(resource.getPath(), true); - List<IODeviceHandle> ioDevices = ioManager.getIODevices(); - int ioDeviceNum = 0; - for (int i = 0; i < ioDevices.size(); i++) { - IODeviceHandle device = ioDevices.get(i); - if (device == file.getDeviceHandle()) { - ioDeviceNum = i; - break; - } - } + FileReference file = ioManager.resolve(resource.getPath()); + int ioDeviceNum = Resource.getIoDeviceNum(ioManager, file.getDeviceHandle()); final IDatasetLifecycleManager datasetLifecycleManager = runtimeContextProvider.getDatasetLifecycleManager(); LSMBTree lsmBTree = LSMBTreeUtils.createLSMTree(ioManager, datasetLifecycleManager.getVirtualBufferCaches( datasetId(), http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java index 06bf298..fc63bbf 100644 --- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java +++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java @@ -30,7 +30,6 @@ import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; -import org.apache.hyracks.api.io.IODeviceHandle; import org.apache.hyracks.storage.am.common.api.IndexException; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory; @@ -80,16 +79,8 @@ public class LSMInvertedIndexLocalResourceMetadata extends Resource { public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, LocalResource resource) throws HyracksDataException { IIOManager ioManager = runtimeContextProvider.getIOManager(); - FileReference file = ioManager.getFileRef(resource.getPath(), true); - List<IODeviceHandle> ioDevices = ioManager.getIODevices(); - int ioDeviceNum = 0; - for (int i = 0; i < ioDevices.size(); i++) { - IODeviceHandle device = ioDevices.get(i); - if (device == file.getDeviceHandle()) { - ioDeviceNum = i; - break; - } - } + FileReference file = ioManager.resolve(resource.getPath()); + int ioDeviceNum = Resource.getIoDeviceNum(ioManager, file.getDeviceHandle()); List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getDatasetLifecycleManager() .getVirtualBufferCaches(datasetId(), ioDeviceNum); try { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java index 73c2dd9..7177391 100644 --- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java +++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java @@ -31,7 +31,6 @@ import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; -import org.apache.hyracks.api.io.IODeviceHandle; import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory; import org.apache.hyracks.storage.am.common.api.TreeIndexException; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex; @@ -81,16 +80,8 @@ public class LSMRTreeLocalResourceMetadata extends Resource { public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, LocalResource resource) throws HyracksDataException { IIOManager ioManager = runtimeContextProvider.getIOManager(); - FileReference file = ioManager.getFileRef(resource.getPath(), true); - List<IODeviceHandle> ioDevices = ioManager.getIODevices(); - int ioDeviceNum = 0; - for (int i = 0; i < ioDevices.size(); i++) { - IODeviceHandle device = ioDevices.get(i); - if (device == file.getDeviceHandle()) { - ioDeviceNum = i; - break; - } - } + FileReference file = ioManager.resolve(resource.getPath()); + int ioDeviceNum = Resource.getIoDeviceNum(ioManager, file.getDeviceHandle()); List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getDatasetLifecycleManager() .getVirtualBufferCaches(datasetId(), ioDeviceNum); try { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java index 9ebb8e7..8600efb 100644 --- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java +++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java @@ -49,11 +49,11 @@ import org.apache.asterix.common.utils.StoragePathUtil; import org.apache.commons.io.FileUtils; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; +import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.api.io.IODeviceHandle; import org.apache.hyracks.api.replication.IReplicationJob.ReplicationExecutionType; import org.apache.hyracks.api.replication.IReplicationJob.ReplicationJobType; import org.apache.hyracks.api.replication.IReplicationJob.ReplicationOperation; -import org.apache.hyracks.control.nc.io.IOManager; import org.apache.hyracks.storage.am.common.frames.LIFOMetaDataFrame; import org.apache.hyracks.storage.common.file.ILocalResourceRepository; import org.apache.hyracks.storage.common.file.LocalResource; @@ -74,7 +74,7 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito private static final FilenameFilter METADATA_FILES_FILTER = (File dir, String name) -> name.equalsIgnoreCase( METADATA_FILE_NAME); // Finals - private final IOManager ioManager; + private final IIOManager ioManager; private final String[] mountPoints; private final String nodeId; private final Cache<String, LocalResource> resourceCache; @@ -87,7 +87,7 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito private IReplicationManager replicationManager; private Set<Integer> nodeInactivePartitions; - public PersistentLocalResourceRepository(IOManager ioManager, List<IODeviceHandle> devices, String nodeId, + public PersistentLocalResourceRepository(IIOManager ioManager, List<IODeviceHandle> devices, String nodeId, AsterixMetadataProperties metadataProperties) throws HyracksDataException { this.ioManager = ioManager; mountPoints = new String[devices.size()]; @@ -187,7 +187,7 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito @Override public synchronized void insert(LocalResource resource) throws HyracksDataException { String relativePath = getFileName(resource.getPath(), resource.getId()); - FileReference resourceFile = ioManager.getFileRef(relativePath, true); + FileReference resourceFile = ioManager.resolve(relativePath); if (resourceFile.getFile().exists()) { throw new HyracksDataException("Duplicate resource: " + resourceFile.getAbsolutePath()); } else { @@ -230,10 +230,10 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito } } - private static FileReference getLocalResourceFileByName(IOManager ioManager, String resourcePath) + private static FileReference getLocalResourceFileByName(IIOManager ioManager, String resourcePath) throws HyracksDataException { String fileName = resourcePath + File.separator + METADATA_FILE_NAME; - return ioManager.getFileRef(fileName, true); + return ioManager.resolve(fileName); } public Map<Long, LocalResource> loadAndGetAllResources() throws HyracksDataException { @@ -420,7 +420,7 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito * @param ioDeviceId * @return A file reference to the storage metadata file. */ - private static FileReference getStorageMetadataFile(IOManager ioManager, String nodeId, + private static FileReference getStorageMetadataFile(IIOManager ioManager, String nodeId, int ioDeviceId) { String storageMetadataFileName = STORAGE_METADATA_DIRECTORY + File.separator + nodeId + "_" + "iodevice" + ioDeviceId + File.separator @@ -435,7 +435,7 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito * @return A file reference to the storage root directory if exists, otherwise null. * @throws HyracksDataException */ - public static File getStorageRootDirectoryIfExists(IOManager ioManager, String nodeId, + public static File getStorageRootDirectoryIfExists(IIOManager ioManager, String nodeId, int ioDeviceId) throws HyracksDataException { File storageRootDir = null; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java index 1c28e17..b46806a 100644 --- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java +++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java @@ -20,16 +20,16 @@ package org.apache.asterix.transaction.management.resource; import org.apache.asterix.common.config.AsterixMetadataProperties; import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.control.nc.io.IOManager; +import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.common.file.ILocalResourceRepository; import org.apache.hyracks.storage.common.file.ILocalResourceRepositoryFactory; public class PersistentLocalResourceRepositoryFactory implements ILocalResourceRepositoryFactory { - private final IOManager ioManager; + private final IIOManager ioManager; private final String nodeId; private final AsterixMetadataProperties metadataProperties; - public PersistentLocalResourceRepositoryFactory(IOManager ioManager, String nodeId, + public PersistentLocalResourceRepositoryFactory(IIOManager ioManager, String nodeId, AsterixMetadataProperties metadataProperties) { this.ioManager = ioManager; this.nodeId = nodeId; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java b/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java index d01bae5..5d9576f 100644 --- a/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java +++ b/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java @@ -21,6 +21,7 @@ package org.apache.hyracks.algebricks.examples.piglet.metadata; import java.io.File; import org.apache.hyracks.api.io.FileSplit; +import org.apache.hyracks.api.io.UnmanagedFileSplit; public class FileSplitUtils { public static FileSplit[] parseFileSplits(String fileSplits) { @@ -32,7 +33,7 @@ public class FileSplitUtils { if (idx < 0) { throw new IllegalArgumentException("File split " + s + " not well formed"); } - fSplits[i] = new FileSplit(s.substring(0, idx), new File(s.substring(idx + 1)).getAbsolutePath(), false); + fSplits[i] = new UnmanagedFileSplit(s.substring(0, idx), new File(s.substring(idx + 1)).getAbsolutePath()); } return fSplits; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java b/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java index 37906c1..35b433b 100644 --- a/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java +++ b/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java @@ -147,8 +147,8 @@ public class PigletMetadataProvider implements IMetadataProvider<String, String> try { prf = new SinkWriterRuntimeFactory(printColumns, printerFactories, fileSplits[0].getFile(null), PrinterBasedWriterFactory.INSTANCE, inputDesc); - AlgebricksAbsolutePartitionConstraint constraint = new AlgebricksAbsolutePartitionConstraint(locations); - return new Pair<>(prf, constraint); + AlgebricksAbsolutePartitionConstraint constraint = new AlgebricksAbsolutePartitionConstraint(locations); + return new Pair<>(prf, constraint); } catch (HyracksDataException e) { throw new AlgebricksException(e); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-part1.tbl ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-part1.tbl b/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-part1.tbl new file mode 100644 index 0000000..91a5152 --- /dev/null +++ b/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-part1.tbl @@ -0,0 +1,7 @@ +0| +2| +4| +6| +8| +10| +12| http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-part2.tbl ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-part2.tbl b/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-part2.tbl new file mode 100644 index 0000000..a481d5d --- /dev/null +++ b/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-part2.tbl @@ -0,0 +1,6 @@ +1| +3| +5| +7| +9| +11| http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part1-split-0.tbl ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part1-split-0.tbl b/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part1-split-0.tbl new file mode 100644 index 0000000..0ea8a88 --- /dev/null +++ b/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part1-split-0.tbl @@ -0,0 +1,4 @@ +0,first branch1 +0,first branch2 +0,first branch3 +0,first branch4 http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part1-split-1.tbl ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part1-split-1.tbl b/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part1-split-1.tbl new file mode 100644 index 0000000..53588ef --- /dev/null +++ b/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part1-split-1.tbl @@ -0,0 +1,3 @@ +1,second branch1 +1,second branch2 +1,second branch3 http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part1.tbl ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part1.tbl b/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part1.tbl new file mode 100644 index 0000000..ceb859a --- /dev/null +++ b/hyracks-fullstack/algebricks/algebricks-tests/data/device0/data/simple/int-string-part1.tbl @@ -0,0 +1,7 @@ +0|first branch1 +1|second branch1 +0|first branch2 +1|second branch2 +0|first branch3 +1|second branch3 +0|first branch4 \ No newline at end of file
