Yingyi Bu has uploaded a new change for review. https://asterix-gerrit.ics.uci.edu/324
Change subject: allow btree search to read unregistered indexes. ...................................................................... allow btree search to read unregistered indexes. Change-Id: I1c7cc7752dd7b6d5261ff02d591f25d76d442a1c --- M hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileSplit.java M hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/AbstractBTreeOperatorTest.java M hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/AbstractfWordInvertedIndexTest.java M hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java M hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java M hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java M hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java M hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/TransientLocalResourceFactoryProvider.java 8 files changed, 44 insertions(+), 30 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/hyracks refs/changes/24/324/1 diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileSplit.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileSplit.java index 8d1e370..a918869 100644 --- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileSplit.java +++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileSplit.java @@ -3,9 +3,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * you may obtain a copy of the License from - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -44,6 +44,12 @@ this.ioDeviceId = 0; } + public FileSplit(String nodeName, String path, int ioDeviceId) { + this.nodeName = nodeName; + this.file = new FileReference(new File(path)); + this.ioDeviceId = ioDeviceId; + } + public String getNodeName() { return nodeName; } diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/AbstractBTreeOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/AbstractBTreeOperatorTest.java index 1eb0ee9..bb100ae 100644 --- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/AbstractBTreeOperatorTest.java +++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/AbstractBTreeOperatorTest.java @@ -139,7 +139,7 @@ public void createPrimaryIndex() throws Exception { JobSpecification spec = new JobSpecification(); - TransientLocalResourceFactoryProvider localResourceFactoryProvider = new TransientLocalResourceFactoryProvider(); + TransientLocalResourceFactoryProvider localResourceFactoryProvider = TransientLocalResourceFactoryProvider.INSTANCE; TreeIndexCreateOperatorDescriptor primaryCreateOp = new TreeIndexCreateOperatorDescriptor(spec, storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, primaryComparatorFactories, primaryBloomFilterKeyFields, dataflowHelperFactory, localResourceFactoryProvider, @@ -195,7 +195,7 @@ public void createSecondaryIndex() throws Exception { JobSpecification spec = new JobSpecification(); - TransientLocalResourceFactoryProvider localResourceFactoryProvider = new TransientLocalResourceFactoryProvider(); + TransientLocalResourceFactoryProvider localResourceFactoryProvider = TransientLocalResourceFactoryProvider.INSTANCE; TreeIndexCreateOperatorDescriptor secondaryCreateOp = new TreeIndexCreateOperatorDescriptor(spec, storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits, secondaryComparatorFactories, secondaryBloomFilterKeyFields, dataflowHelperFactory, diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/AbstractfWordInvertedIndexTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/AbstractfWordInvertedIndexTest.java index a77a71d..3de87bb 100644 --- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/AbstractfWordInvertedIndexTest.java +++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/AbstractfWordInvertedIndexTest.java @@ -164,7 +164,7 @@ public void createPrimaryIndex() throws Exception { JobSpecification spec = new JobSpecification(); - TransientLocalResourceFactoryProvider localResourceFactoryProvider = new TransientLocalResourceFactoryProvider(); + TransientLocalResourceFactoryProvider localResourceFactoryProvider = TransientLocalResourceFactoryProvider.INSTANCE; TreeIndexCreateOperatorDescriptor primaryCreateOp = new TreeIndexCreateOperatorDescriptor(spec, storageManager, lcManagerProvider, primaryFileSplitProvider, primaryTypeTraits, primaryComparatorFactories, null, btreeDataflowHelperFactory, localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE); @@ -175,7 +175,7 @@ public void createInvertedIndex() throws Exception { JobSpecification spec = new JobSpecification(); - ILocalResourceFactoryProvider localResourceFactoryProvider = new TransientLocalResourceFactoryProvider(); + ILocalResourceFactoryProvider localResourceFactoryProvider = TransientLocalResourceFactoryProvider.INSTANCE; LSMInvertedIndexCreateOperatorDescriptor invIndexCreateOp = new LSMInvertedIndexCreateOperatorDescriptor(spec, storageManager, btreeFileSplitProvider, lcManagerProvider, tokenTypeTraits, tokenComparatorFactories, invListsTypeTraits, invListsComparatorFactories, tokenizerFactory, invertedIndexDataflowHelperFactory, diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java index 76ea772..9bed169 100644 --- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java +++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java @@ -200,7 +200,7 @@ protected void createPrimaryIndex() throws Exception { JobSpecification spec = new JobSpecification(); - TransientLocalResourceFactoryProvider localResourceFactoryProvider = new TransientLocalResourceFactoryProvider(); + TransientLocalResourceFactoryProvider localResourceFactoryProvider = TransientLocalResourceFactoryProvider.INSTANCE; TreeIndexCreateOperatorDescriptor primaryCreateOp = new TreeIndexCreateOperatorDescriptor(spec, storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, primaryComparatorFactories, null, btreeDataflowHelperFactory, localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE); @@ -261,7 +261,7 @@ protected void createSecondaryIndex() throws Exception { JobSpecification spec = new JobSpecification(); - TransientLocalResourceFactoryProvider localResourceFactoryProvider = new TransientLocalResourceFactoryProvider(); + TransientLocalResourceFactoryProvider localResourceFactoryProvider = TransientLocalResourceFactoryProvider.INSTANCE; TreeIndexCreateOperatorDescriptor secondaryCreateOp = new TreeIndexCreateOperatorDescriptor(spec, storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits, secondaryComparatorFactories, null, rtreeDataflowHelperFactory, localResourceFactoryProvider, diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java index 48f253c..edf8b49 100644 --- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java +++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java @@ -3,9 +3,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * you may obtain a copy of the License from - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,7 +30,7 @@ import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory; import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface; -import edu.uci.ics.hyracks.storage.common.file.NoOpLocalResourceFactoryProvider; +import edu.uci.ics.hyracks.storage.common.file.TransientLocalResourceFactoryProvider; public class BTreeSearchOperatorDescriptor extends AbstractTreeIndexOperatorDescriptor { @@ -53,7 +53,7 @@ int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes) { super(spec, 1, 1, recDesc, storageManager, lifecycleManagerProvider, fileSplitProvider, typeTraits, comparatorFactories, bloomFilterKeyFields, dataflowHelperFactory, null, retainInput, retainNull, - nullWriterFactory, NoOpLocalResourceFactoryProvider.INSTANCE, searchOpCallbackProvider, + nullWriterFactory, TransientLocalResourceFactoryProvider.INSTANCE, searchOpCallbackProvider, NoOpOperationCallbackFactory.INSTANCE); this.lowKeyFields = lowKeyFields; this.highKeyFields = highKeyFields; diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java index d67f585..c6fb4dd 100644 --- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java +++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java @@ -3,9 +3,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * you may obtain a copy of the License from - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -76,20 +76,15 @@ index = createIndexInstance(); } - // The previous resource ID needs to be removed since calling IIndex.create() may possibly destroy - // any physical artifact that the LocalResourceRepository is managing (e.g. a file containing the resource ID). + // The previous resource ID needs to be removed since calling IIndex.create() may possibly destroy + // any physical artifact that the LocalResourceRepository is managing (e.g. a file containing the resource ID). // Once the index has been created, a new resource ID can be generated. if (resourceID != -1) { localResourceRepository.deleteResourceByName(file.getFile().getPath()); } index.create(); try { - //TODO Create LocalResource through LocalResourceFactory interface - resourceID = resourceIdFactory.createId(); - ILocalResourceFactory localResourceFactory = opDesc.getLocalResourceFactoryProvider() - .getLocalResourceFactory(); - localResourceRepository.insert(localResourceFactory.createLocalResource(resourceID, file.getFile() - .getPath(), partition)); + resourceID = addLocalResource(); } catch (IOException e) { throw new HyracksDataException(e); } @@ -101,9 +96,8 @@ public void open() throws HyracksDataException { synchronized (lcManager) { long resourceID = getResourceID(); - if (resourceID == -1) { - throw new HyracksDataException("Index does not have a valid resource ID. Has it been created yet?"); + resourceID = addLocalResource(); } index = lcManager.getIndex(resourceID); @@ -159,4 +153,13 @@ public IHyracksTaskContext getTaskContext() { return ctx; } -} \ No newline at end of file + + // Adds the local resource if it is missing. + private long addLocalResource() throws HyracksDataException { + long resourceID = resourceIdFactory.createId(); + ILocalResourceFactory localResourceFactory = opDesc.getLocalResourceFactoryProvider().getLocalResourceFactory(); + localResourceRepository.insert(localResourceFactory.createLocalResource(resourceID, file.getFile().getPath(), + partition)); + return resourceID; + } +} diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java index 92e178a..77dea7f 100644 --- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java +++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java @@ -26,7 +26,7 @@ import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider; import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory; import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface; -import edu.uci.ics.hyracks.storage.common.file.NoOpLocalResourceFactoryProvider; +import edu.uci.ics.hyracks.storage.common.file.TransientLocalResourceFactoryProvider; public class TreeIndexBulkLoadOperatorDescriptor extends AbstractTreeIndexOperatorDescriptor { @@ -46,7 +46,7 @@ IIndexDataflowHelperFactory dataflowHelperFactory) { super(spec, 1, 1, recDesc, storageManager, lifecycleManagerProvider, fileSplitProvider, typeTraits, comparatorFactories, bloomFilterKeyFields, dataflowHelperFactory, null, false, false, null, - NoOpLocalResourceFactoryProvider.INSTANCE, NoOpOperationCallbackFactory.INSTANCE, + TransientLocalResourceFactoryProvider.INSTANCE, NoOpOperationCallbackFactory.INSTANCE, NoOpOperationCallbackFactory.INSTANCE); this.fieldPermutation = fieldPermutation; this.fillFactor = fillFactor; diff --git a/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/TransientLocalResourceFactoryProvider.java b/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/TransientLocalResourceFactoryProvider.java index 7d5eb3a..a8565eb 100644 --- a/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/TransientLocalResourceFactoryProvider.java +++ b/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/TransientLocalResourceFactoryProvider.java @@ -3,9 +3,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * you may obtain a copy of the License from - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,8 +15,13 @@ package edu.uci.ics.hyracks.storage.common.file; public class TransientLocalResourceFactoryProvider implements ILocalResourceFactoryProvider { - + private static final long serialVersionUID = 1L; + public static TransientLocalResourceFactoryProvider INSTANCE = new TransientLocalResourceFactoryProvider(); + + private TransientLocalResourceFactoryProvider() { + + } @Override public ILocalResourceFactory getLocalResourceFactory() { -- To view, visit https://asterix-gerrit.ics.uci.edu/324 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I1c7cc7752dd7b6d5261ff02d591f25d76d442a1c Gerrit-PatchSet: 1 Gerrit-Project: hyracks Gerrit-Branch: master Gerrit-Owner: Yingyi Bu <[email protected]>
