http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/PrefixMergePolicy.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/PrefixMergePolicy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/PrefixMergePolicy.java index 36cb958..5f36339 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/PrefixMergePolicy.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/PrefixMergePolicy.java @@ -27,6 +27,7 @@ import java.util.Map; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.common.api.IndexException; import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.ComponentState; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex; @@ -42,15 +43,15 @@ public class PrefixMergePolicy implements ILSMMergePolicy { public void diskComponentAdded(final ILSMIndex index, boolean fullMergeIsRequested) throws HyracksDataException, IndexException { - List<ILSMComponent> immutableComponents = new ArrayList<ILSMComponent>(index.getImmutableComponents()); + ArrayList<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getImmutableComponents()); if (!areComponentsReadableWritableState(immutableComponents)) { return; } if (fullMergeIsRequested) { - ILSMIndexAccessor accessor = index.createAccessor(NoOpOperationCallback.INSTANCE, - NoOpOperationCallback.INSTANCE); + ILSMIndexAccessor accessor = + index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); accessor.scheduleFullMerge(index.getIOOperationCallback()); return; } @@ -130,7 +131,7 @@ public class PrefixMergePolicy implements ILSMMergePolicy { * there will be no new merge either in this situation. */ - List<ILSMComponent> immutableComponents = index.getImmutableComponents(); + List<ILSMDiskComponent> immutableComponents = index.getImmutableComponents(); int mergableImmutableComponentCount = getMergableImmutableComponentCount(immutableComponents); // [case 1] @@ -166,7 +167,7 @@ public class PrefixMergePolicy implements ILSMMergePolicy { * @param immutableComponents * @return true if there is an ongoing merge operation, false otherwise. */ - private boolean isMergeOngoing(List<ILSMComponent> immutableComponents) { + private boolean isMergeOngoing(List<ILSMDiskComponent> immutableComponents) { int size = immutableComponents.size(); for (int i = 0; i < size; i++) { if (immutableComponents.get(i).getState() == ComponentState.READABLE_MERGING) { @@ -184,10 +185,10 @@ public class PrefixMergePolicy implements ILSMMergePolicy { * @param immutableComponents * @return the number of mergable component */ - private int getMergableImmutableComponentCount(List<ILSMComponent> immutableComponents) { + private int getMergableImmutableComponentCount(List<ILSMDiskComponent> immutableComponents) { int count = 0; for (ILSMComponent c : immutableComponents) { - long componentSize = ((AbstractDiskLSMComponent) c).getComponentSize(); + long componentSize = ((ILSMDiskComponent) c).getComponentSize(); //stop when the first non-mergable component is found. if (c.getState() != ComponentState.READABLE_UNWRITABLE || componentSize > maxMergableComponentSize) { break; @@ -203,7 +204,7 @@ public class PrefixMergePolicy implements ILSMMergePolicy { * @param immutableComponents * @return true if all components are of READABLE_UNWRITABLE state, false otherwise. */ - private boolean areComponentsReadableWritableState(List<ILSMComponent> immutableComponents) { + private boolean areComponentsReadableWritableState(List<ILSMDiskComponent> immutableComponents) { for (ILSMComponent c : immutableComponents) { if (c.getState() != ComponentState.READABLE_UNWRITABLE) { return false; @@ -225,7 +226,7 @@ public class PrefixMergePolicy implements ILSMMergePolicy { // all such components for which the sum of their sizes exceeds MaxMrgCompSz. Schedule a merge of those components into a new component. // 2. If a merge from 1 doesn't happen, see if the set of candidate components for merging exceeds MaxTolCompCnt. If so, schedule // a merge all of the current candidates into a new single component. - List<ILSMComponent> immutableComponents = new ArrayList<ILSMComponent>(index.getImmutableComponents()); + List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getImmutableComponents()); // Reverse the components order so that we look at components from oldest to newest. Collections.reverse(immutableComponents); @@ -233,7 +234,7 @@ public class PrefixMergePolicy implements ILSMMergePolicy { int startIndex = -1; for (int i = 0; i < immutableComponents.size(); i++) { ILSMComponent c = immutableComponents.get(i); - long componentSize = ((AbstractDiskLSMComponent) c).getComponentSize(); + long componentSize = ((ILSMDiskComponent) c).getComponentSize(); if (componentSize > maxMergableComponentSize) { startIndex = i; totalSize = 0; @@ -243,14 +244,14 @@ public class PrefixMergePolicy implements ILSMMergePolicy { boolean isLastComponent = i + 1 == immutableComponents.size() ? true : false; if (totalSize > maxMergableComponentSize || (isLastComponent && i - startIndex >= maxToleranceComponentCount)) { - List<ILSMComponent> mergableComponents = new ArrayList<ILSMComponent>(); + List<ILSMDiskComponent> mergableComponents = new ArrayList<>(); for (int j = startIndex + 1; j <= i; j++) { mergableComponents.add(immutableComponents.get(j)); } // Reverse the components order back to its original order Collections.reverse(mergableComponents); - ILSMIndexAccessor accessor = index.createAccessor(NoOpOperationCallback.INSTANCE, - NoOpOperationCallback.INSTANCE); + ILSMIndexAccessor accessor = + index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); accessor.scheduleMerge(index.getIOOperationCallback(), mergableComponents); return true; }
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java index 9b255d7..d5a8488 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/ThreadCountingTracker.java @@ -26,7 +26,6 @@ import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback; import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexInternal; import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker; import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType; @@ -56,13 +55,11 @@ public class ThreadCountingTracker implements ILSMOperationTracker { public void completeOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback, IModificationOperationCallback modificationCallback) throws HyracksDataException { // Flush will only be handled by last exiting thread. - if (opType == LSMOperationType.MODIFICATION) { - if (threadRefCount.decrementAndGet() == 0 - && ((ILSMIndexInternal) index).hasFlushRequestForCurrentMutableComponent()) { - ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE, - NoOpOperationCallback.INSTANCE); - accessor.scheduleFlush(NoOpIOOperationCallback.INSTANCE); - } + if (opType == LSMOperationType.MODIFICATION && threadRefCount.decrementAndGet() == 0 + && index.hasFlushRequestForCurrentMutableComponent()) { + ILSMIndexAccessor accessor = + index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); + accessor.scheduleFlush(NoOpIOOperationCallback.INSTANCE); } } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/utils/ComponentMetadataUtil.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/utils/ComponentMetadataUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/utils/ComponentMetadataUtil.java new file mode 100644 index 0000000..b55e8ad --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/utils/ComponentMetadataUtil.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.storage.am.lsm.common.utils; + +import java.util.List; + +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.data.std.api.IPointable; +import org.apache.hyracks.data.std.api.IValueReference; +import org.apache.hyracks.data.std.primitive.LongPointable; +import org.apache.hyracks.storage.am.common.freepage.MutableArrayValueReference; +import org.apache.hyracks.storage.am.lsm.common.api.IComponentMetadata; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent; + +public class ComponentMetadataUtil { + + public static final MutableArrayValueReference MARKER_LSN_KEY = + new MutableArrayValueReference("Marker".getBytes()); + public static final long NOT_FOUND = -1L; + + private ComponentMetadataUtil() { + } + + /** + * Get a long value from the metadata of a component or return a default value + * + * @param metadata + * the component's metadata + * @param key + * the key + * @param defaultValue + * the default value + * @return + * the long value if found, the default value otherwise + * @throws HyracksDataException + * If the comopnent was a disk component and an IO error was encountered + */ + public static long getLong(IComponentMetadata metadata, IValueReference key, long defaultValue) + throws HyracksDataException { + IValueReference value = metadata.get(key); + return value == null || value.getLength() == 0 ? defaultValue + : LongPointable.getLong(value.getByteArray(), value.getStartOffset()); + } + + /** + * Get a value from an index's metadata pages. It first, searches the current in memory component + * then searches the other components. in reverse order. + * Note: This method locks on the OpTracker of the index + * + * @param index + * @param key + * @param pointable + * @throws HyracksDataException + */ + public static void get(ILSMIndex index, IValueReference key, IPointable pointable) throws HyracksDataException { + // Lock the opTracker to ensure index components don't change + synchronized (index.getOperationTracker()) { + index.getCurrentMemoryComponent().getMetadata().get(key, pointable); + if (pointable.getLength() == 0) { + // was not found in the in current mutable component, search in the other in memory components + fromImmutableMemoryComponents(index, key, pointable); + if (pointable.getLength() == 0) { + // was not found in the in all in memory components, search in the disk components + fromDiskComponents(index, key, pointable); + } + } + } + } + + private static void fromDiskComponents(ILSMIndex index, IValueReference key, IPointable pointable) + throws HyracksDataException { + for (ILSMDiskComponent c : index.getImmutableComponents()) { + c.getMetadata().get(key, pointable); + if (pointable.getLength() != 0) { + // Found + return; + } + } + } + + private static void fromImmutableMemoryComponents(ILSMIndex index, IValueReference key, IPointable pointable) { + List<ILSMMemoryComponent> memComponents = index.getMemoryComponents(); + int numOtherMemComponents = memComponents.size() - 1; + int next = index.getCurrentMemoryComponentIndex(); + for (int i = 0; i < numOtherMemComponents; i++) { + next = next - 1; + if (next < 0) { + next = memComponents.size() - 1; + } + ILSMMemoryComponent c = index.getMemoryComponents().get(next); + if (c.isReadable()) { + c.getMetadata().get(key, pointable); + if (pointable.getLength() != 0) { + // Found + return; + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml index e03cddc..010a343 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml @@ -16,18 +16,15 @@ ! specific language governing permissions and limitations ! under the License. !--> - <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> <artifactId>hyracks-storage-am-lsm-invertedindex</artifactId> - <parent> <artifactId>hyracks</artifactId> <groupId>org.apache.hyracks</groupId> <version>0.3.1-SNAPSHOT</version> <relativePath>..</relativePath> </parent> - <licenses> <license> <name>Apache License, Version 2.0</name> @@ -36,11 +33,9 @@ <comments>A business-friendly OSS license</comments> </license> </licenses> - <properties> <root.dir>${basedir}/../..</root.dir> </properties> - <dependencies> <dependency> <groupId>org.apache.hyracks</groupId> @@ -93,4 +88,4 @@ <version>${project.version}</version> </dependency> </dependencies> -</project> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java index f39bab6..449c8f9 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/api/IInvertedIndex.java @@ -28,16 +28,16 @@ import org.apache.hyracks.storage.am.common.api.IIndexOperationContext; import org.apache.hyracks.storage.am.common.api.IndexException; public interface IInvertedIndex extends IIndex { - public IInvertedListCursor createInvertedListCursor(); + IInvertedListCursor createInvertedListCursor(); - public void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference searchKey, IIndexOperationContext ictx) + void openInvertedListCursor(IInvertedListCursor listCursor, ITupleReference searchKey, IIndexOperationContext ictx) throws HyracksDataException, IndexException; - public ITypeTraits[] getInvListTypeTraits(); + ITypeTraits[] getInvListTypeTraits(); - public IBinaryComparatorFactory[] getInvListCmpFactories(); + IBinaryComparatorFactory[] getInvListCmpFactories(); - public ITypeTraits[] getTokenTypeTraits(); + ITypeTraits[] getTokenTypeTraits(); - public IBinaryComparatorFactory[] getTokenCmpFactories(); + IBinaryComparatorFactory[] getTokenCmpFactories(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java index 28cfadf..ac06d1c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java @@ -90,7 +90,7 @@ public final class LSMInvertedIndexDataflowHelper extends AbstractLSMIndexDatafl invIndexOpDesc.getTokenComparatorFactories(), invIndexOpDesc.getTokenizerFactory(), diskBufferCache, fileRef.getFile().getAbsolutePath(), bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx.getJobletContext().getApplicationContext()), ioScheduler, - ioOpCallbackFactory.createIOOperationCallback(), invertedIndexFields, filterTypeTraits, + ioOpCallbackFactory.createIoOpCallback(), invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable, (IMetadataPageManagerFactory) opDesc .getPageManagerFactory()); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java index 5138642..4ee3a21 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java @@ -89,7 +89,7 @@ public final class PartitionedLSMInvertedIndexDataflowHelper extends AbstractLSM invIndexOpDesc.getTokenComparatorFactories(), invIndexOpDesc.getTokenizerFactory(), diskBufferCache, fileRef.getFile().getAbsolutePath(), bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx.getJobletContext().getApplicationContext()), ioScheduler, - ioOpCallbackFactory.createIOOperationCallback(), invertedIndexFields, filterTypeTraits, + ioOpCallbackFactory.createIoOpCallback(), invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable, opDesc.getPageManagerFactory()); return invIndex; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java index 403fa09..fc0e4ec 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java @@ -55,14 +55,14 @@ import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy; @@ -94,7 +94,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex // On-disk components. // For creating inverted indexes in flush and merge. - protected final ILSMComponentFactory componentFactory; + protected final ILSMDiskComponentFactory componentFactory; // Type traits and comparators for tokens and inverted-list elements. protected final ITypeTraits[] invListTypeTraits; @@ -109,13 +109,14 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory deletedKeysBTreeFactory, BloomFilterFactory bloomFilterFactory, ILSMComponentFilterFactory filterFactory, ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, - double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, - ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, - ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, - IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, - ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, - int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, - boolean durable) throws IndexException, HyracksDataException { + double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager, + IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits, + IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, + IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, + ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, + ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, int[] filterFields, + int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) + throws IndexException, HyracksDataException { super(ioManager, virtualBufferCaches, diskInvIndexFactory.getBufferCache(), fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory, filterManager, filterFields, durable); @@ -134,17 +135,15 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex int i = 0; for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) { - InMemoryInvertedIndex memInvIndex = createInMemoryInvertedIndex(virtualBufferCache, - new VirtualFreePageManager(virtualBufferCache), i); + InMemoryInvertedIndex memInvIndex = + createInMemoryInvertedIndex(virtualBufferCache, new VirtualFreePageManager(virtualBufferCache), i); BTree deleteKeysBTree = BTreeUtils.createBTree(virtualBufferCache, - new VirtualFreePageManager(virtualBufferCache), - virtualBufferCache.getFileMapProvider(), invListTypeTraits, invListCmpFactories, - BTreeLeafFrameType.REGULAR_NSM, ioManager.resolveAbsolutePath(fileManager.getBaseDir() - + "_virtual_del_" - + i)); - LSMInvertedIndexMemoryComponent mutableComponent = new LSMInvertedIndexMemoryComponent(memInvIndex, - deleteKeysBTree, virtualBufferCache, i == 0 ? true : false, - filterFactory == null ? null : filterFactory.createLSMComponentFilter()); + new VirtualFreePageManager(virtualBufferCache), virtualBufferCache.getFileMapProvider(), + invListTypeTraits, invListCmpFactories, BTreeLeafFrameType.REGULAR_NSM, + ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_del_" + i)); + LSMInvertedIndexMemoryComponent mutableComponent = + new LSMInvertedIndexMemoryComponent(memInvIndex, deleteKeysBTree, virtualBufferCache, + i == 0 ? true : false, filterFactory == null ? null : filterFactory.createFilter()); memoryComponents.add(mutableComponent); ++i; } @@ -168,7 +167,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex } try { - List<ILSMComponent> immutableComponents = diskComponents; + List<ILSMDiskComponent> immutableComponents = diskComponents; immutableComponents.clear(); List<LSMComponentFileReferences> validFileReferences = fileManager.cleanupAndGetValidFiles(); for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) { @@ -196,8 +195,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex } clearMemoryComponents(); - List<ILSMComponent> immutableComponents = diskComponents; - for (ILSMComponent c : immutableComponents) { + List<ILSMDiskComponent> immutableComponents = diskComponents; + for (ILSMDiskComponent c : immutableComponents) { LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c; component.getBloomFilter().deactivate(); component.getInvIndex().deactivate(); @@ -216,7 +215,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex } if (flushOnExit) { BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback); - ILSMIndexAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); + ILSMIndexAccessor accessor = + createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); accessor.scheduleFlush(cb); try { cb.waitForIO(); @@ -225,8 +225,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex } } - List<ILSMComponent> immutableComponents = diskComponents; - for (ILSMComponent c : immutableComponents) { + List<ILSMDiskComponent> immutableComponents = diskComponents; + for (ILSMDiskComponent c : immutableComponents) { LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c; component.getBloomFilter().deactivate(); component.getInvIndex().deactivate(); @@ -252,8 +252,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex mutableComponent.getInvIndex().destroy(); mutableComponent.getDeletedKeysBTree().destroy(); } - List<ILSMComponent> immutableComponents = diskComponents; - for (ILSMComponent c : immutableComponents) { + List<ILSMDiskComponent> immutableComponents = diskComponents; + for (ILSMDiskComponent c : immutableComponents) { LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c; component.getInvIndex().destroy(); component.getDeletedKeysBTree().destroy(); @@ -264,7 +264,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex @Override public void getOperationalComponents(ILSMIndexOperationContext ctx) throws HyracksDataException { - List<ILSMComponent> immutableComponents = diskComponents; + List<ILSMDiskComponent> immutableComponents = diskComponents; List<ILSMComponent> operationalComponents = ctx.getComponentHolder(); int cmc = currentMutableComponentId.get(); ctx.setCurrentMutableComponentId(cmc); @@ -322,7 +322,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex * - Insert key into deleted-keys BTree. */ @Override - public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException { + public void modify(IIndexOperationContext ictx, ITupleReference tuple) + throws HyracksDataException, IndexException { LSMInvertedIndexOpContext ctx = (LSMInvertedIndexOpContext) ictx; // TODO: This is a hack to support logging properly in ASTERIX. // The proper undo operations are only dependent on the after image so @@ -386,8 +387,9 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex IIndexAccessor invIndexAccessor = ((LSMInvertedIndexMemoryComponent) component).getInvIndex() .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); indexAccessors.add(invIndexAccessor); - IIndexAccessor deletedKeysAccessor = ((LSMInvertedIndexMemoryComponent) component).getDeletedKeysBTree() - .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); + IIndexAccessor deletedKeysAccessor = + ((LSMInvertedIndexMemoryComponent) component).getDeletedKeysBTree() + .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); deletedKeysBTreeAccessors.add(deletedKeysAccessor); } else { IIndexAccessor invIndexAccessor = ((LSMInvertedIndexDiskComponent) component).getInvIndex() @@ -420,8 +422,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex .getDeletedKeysBTree().getLeafFrameFactory(), ictx, includeMutableComponent, lsmHarness, operationalComponents); } else { - LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) memoryComponents - .get(currentMutableComponentId.get()); + LSMInvertedIndexMemoryComponent mutableComponent = + (LSMInvertedIndexMemoryComponent) memoryComponents.get(currentMutableComponentId.get()); InMemoryInvertedIndex memInvIndex = (InMemoryInvertedIndex) mutableComponent.getInvIndex(); MultiComparator tokensAndKeysCmp = MultiComparator.create(memInvIndex.getBTree().getComparatorFactories()); initState = new LSMInvertedIndexRangeSearchCursorInitialState(tokensAndKeysCmp, keyCmp, keysOnlyTuple, @@ -452,8 +454,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex throws HyracksDataException { ILSMComponent flushingComponent = ctx.getComponentHolder().get(0); LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference(); - LSMInvertedIndexOpContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE, - NoOpOperationCallback.INSTANCE); + LSMInvertedIndexOpContext opCtx = + createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); opCtx.setOperation(IndexOperation.FLUSH); opCtx.getComponentHolder().add(flushingComponent); ioScheduler.scheduleOperation(new LSMInvertedIndexFlushOperation( @@ -463,18 +465,18 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex } @Override - public ILSMComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException { + public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException, IndexException { LSMInvertedIndexFlushOperation flushOp = (LSMInvertedIndexFlushOperation) operation; // Create an inverted index instance to be bulk loaded. - LSMInvertedIndexDiskComponent component = createDiskInvIndexComponent(componentFactory, - flushOp.getDictBTreeFlushTarget(), flushOp.getDeletedKeysBTreeFlushTarget(), - flushOp.getBloomFilterFlushTarget(), true); + LSMInvertedIndexDiskComponent component = + createDiskInvIndexComponent(componentFactory, flushOp.getDictBTreeFlushTarget(), + flushOp.getDeletedKeysBTreeFlushTarget(), flushOp.getBloomFilterFlushTarget(), true); IInvertedIndex diskInvertedIndex = component.getInvIndex(); // Create a scan cursor on the BTree underlying the in-memory inverted index. - LSMInvertedIndexMemoryComponent flushingComponent = (LSMInvertedIndexMemoryComponent) flushOp - .getFlushingComponent(); + LSMInvertedIndexMemoryComponent flushingComponent = + (LSMInvertedIndexMemoryComponent) flushOp.getFlushingComponent(); InMemoryInvertedIndexAccessor memInvIndexAccessor = (InMemoryInvertedIndexAccessor) flushingComponent .getInvIndex().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); BTreeAccessor memBTreeAccessor = memInvIndexAccessor.getBTreeAccessor(); @@ -496,10 +498,11 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex List<ITupleReference> filterTuples = new ArrayList<>(); filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple()); filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple()); - filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples); - filterManager.writeFilterInfo(component.getLSMComponentFilter(), + filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples); + filterManager.writeFilter(component.getLSMComponentFilter(), ((OnDiskInvertedIndex) component.getInvIndex()).getBTree()); } + flushingComponent.getMetadata().copy(component.getMetadata()); invIndexBulkLoader.end(); IIndexAccessor deletedKeysBTreeAccessor = flushingComponent.getDeletedKeysBTree() @@ -518,8 +521,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex } int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numBTreeTuples); - BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement, - bloomFilterFalsePositiveRate); + BloomFilterSpecification bloomFilterSpec = + BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate); // Create an BTree instance for the deleted keys. BTree diskDeletedKeysBTree = component.getDeletedKeysBTree(); @@ -551,8 +554,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex @Override public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback) throws HyracksDataException, IndexException { - LSMInvertedIndexOpContext ictx = createOpContext(NoOpOperationCallback.INSTANCE, - NoOpOperationCallback.INSTANCE); + LSMInvertedIndexOpContext ictx = + createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); ictx.setOperation(IndexOperation.MERGE); List<ILSMComponent> mergingComponents = ctx.getComponentHolder(); IIndexCursor cursor = new LSMInvertedIndexRangeSearchCursor(ictx); @@ -561,20 +564,21 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex OnDiskInvertedIndex firstInvIndex = (OnDiskInvertedIndex) firstComponent.getInvIndex(); String firstFileName = firstInvIndex.getBTree().getFileReference().getFile().getName(); - LSMInvertedIndexDiskComponent lastComponent = (LSMInvertedIndexDiskComponent) mergingComponents - .get(mergingComponents.size() - 1); + LSMInvertedIndexDiskComponent lastComponent = + (LSMInvertedIndexDiskComponent) mergingComponents.get(mergingComponents.size() - 1); OnDiskInvertedIndex lastInvIndex = (OnDiskInvertedIndex) lastComponent.getInvIndex(); String lastFileName = lastInvIndex.getBTree().getFileReference().getFile().getName(); - LSMComponentFileReferences relMergeFileRefs = fileManager.getRelMergeFileReference(firstFileName, lastFileName); - ILSMIndexAccessorInternal accessor = new LSMInvertedIndexAccessor(lsmHarness, ctx); + LSMComponentFileReferences relMergeFileRefs = + fileManager.getRelMergeFileReference(firstFileName, lastFileName); + ILSMIndexAccessor accessor = new LSMInvertedIndexAccessor(lsmHarness, ctx); ioScheduler.scheduleOperation(new LSMInvertedIndexMergeOperation(accessor, mergingComponents, cursor, relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir())); } @Override - public ILSMComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException { + public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException, IndexException { LSMInvertedIndexMergeOperation mergeOp = (LSMInvertedIndexMergeOperation) operation; IIndexCursor cursor = mergeOp.getCursor(); @@ -585,9 +589,9 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex search(opCtx, cursor, mergePred); // Create an inverted index instance. - LSMInvertedIndexDiskComponent component = createDiskInvIndexComponent(componentFactory, - mergeOp.getDictBTreeMergeTarget(), mergeOp.getDeletedKeysBTreeMergeTarget(), - mergeOp.getBloomFilterMergeTarget(), true); + LSMInvertedIndexDiskComponent component = + createDiskInvIndexComponent(componentFactory, mergeOp.getDictBTreeMergeTarget(), + mergeOp.getDeletedKeysBTreeMergeTarget(), mergeOp.getBloomFilterMergeTarget(), true); IInvertedIndex mergedDiskInvertedIndex = component.getInvIndex(); @@ -597,8 +601,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex .get(diskComponents.size() - 1)) { // Keep the deleted tuples since the oldest disk component is not included in the merge operation - LSMInvertedIndexDeletedKeysBTreeMergeCursor btreeCursor = new LSMInvertedIndexDeletedKeysBTreeMergeCursor( - opCtx); + LSMInvertedIndexDeletedKeysBTreeMergeCursor btreeCursor = + new LSMInvertedIndexDeletedKeysBTreeMergeCursor(opCtx); search(opCtx, btreeCursor, mergePred); BTree btree = component.getDeletedKeysBTree(); @@ -611,8 +615,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex } int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements); - BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement, - bloomFilterFalsePositiveRate); + BloomFilterSpecification bloomFilterSpec = + BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate); IIndexBulkLoader builder = component.getBloomFilter().createBuilder(numElements, bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements()); try { @@ -655,8 +659,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex filterTuples.add(max); } } - filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples); - filterManager.writeFilterInfo(component.getLSMComponentFilter(), + filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples); + filterManager.writeFilter(component.getLSMComponentFilter(), ((OnDiskInvertedIndex) component.getInvIndex()).getBTree()); } invIndexBulkLoader.end(); @@ -664,12 +668,6 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex return component; } - private ILSMComponent createBulkLoadTarget() throws HyracksDataException, IndexException { - LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference(); - return createDiskInvIndexComponent(componentFactory, componentFileRefs.getInsertIndexFileReference(), - componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true); - } - @Override public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex) throws IndexException { @@ -681,7 +679,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex } public class LSMInvertedIndexBulkLoader implements IIndexBulkLoader { - private final ILSMComponent component; + private final ILSMDiskComponent component; private final IIndexBulkLoader invIndexBulkLoader; private final IIndexBulkLoader deletedKeysBTreeBulkLoader; private boolean cleanedUpArtifacts = false; @@ -763,7 +761,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex public void end() throws IndexException, HyracksDataException { if (!cleanedUpArtifacts) { if (component.getLSMComponentFilter() != null) { - filterManager.writeFilterInfo(component.getLSMComponentFilter(), + filterManager.writeFilter(component.getLSMComponentFilter(), (((OnDiskInvertedIndex) ((LSMInvertedIndexDiskComponent) component).getInvIndex()) .getBTree())); } @@ -789,20 +787,27 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex deletedKeysBTreeBulkLoader.abort(); } } + + private ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException, IndexException { + LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference(); + return createDiskInvIndexComponent(componentFactory, componentFileRefs.getInsertIndexFileReference(), + componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), + true); + } } protected InMemoryInvertedIndex createInMemoryInvertedIndex(IVirtualBufferCache virtualBufferCache, VirtualFreePageManager virtualFreePageManager, int id) throws IndexException, HyracksDataException { return InvertedIndexUtils.createInMemoryBTreeInvertedindex(virtualBufferCache, virtualFreePageManager, - invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, ioManager - .resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_vocab_" + id)); + invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, + ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_vocab_" + id)); } - protected LSMInvertedIndexDiskComponent createDiskInvIndexComponent(ILSMComponentFactory factory, + protected LSMInvertedIndexDiskComponent createDiskInvIndexComponent(ILSMDiskComponentFactory factory, FileReference dictBTreeFileRef, FileReference btreeFileRef, FileReference bloomFilterFileRef, boolean create) throws HyracksDataException, IndexException { - LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) factory.createLSMComponentInstance( - new LSMComponentFileReferences(dictBTreeFileRef, btreeFileRef, bloomFilterFileRef)); + LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) factory + .createComponent(new LSMComponentFileReferences(dictBTreeFileRef, btreeFileRef, bloomFilterFileRef)); if (create) { component.getInvIndex().create(); component.getDeletedKeysBTree().create(); @@ -813,14 +818,14 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex component.getBloomFilter().activate(); // Will be closed during cleanup of merge(). if (component.getLSMComponentFilter() != null && !create) { - filterManager.readFilterInfo(component.getLSMComponentFilter(), + filterManager.readFilter(component.getLSMComponentFilter(), ((OnDiskInvertedIndex) component.getInvIndex()).getBTree()); } return component; } @Override - public ILSMIndexAccessorInternal createAccessor(IModificationOperationCallback modificationCallback, + public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) throws HyracksDataException { return new LSMInvertedIndexAccessor(lsmHarness, createOpContext(modificationCallback, searchCallback)); } @@ -878,7 +883,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex } @Override - public void markAsValid(ILSMComponent lsmComponent) throws HyracksDataException { + public void markAsValid(ILSMDiskComponent lsmComponent) throws HyracksDataException { LSMInvertedIndexDiskComponent invIndexComponent = (LSMInvertedIndexDiskComponent) lsmComponent; OnDiskInvertedIndex invIndex = (OnDiskInvertedIndex) invIndexComponent.getInvIndex(); IBufferCache bufferCache = invIndex.getBufferCache(); @@ -895,8 +900,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex @Override public void validate() throws HyracksDataException { validateMemoryComponents(); - List<ILSMComponent> immutableComponents = diskComponents; - for (ILSMComponent c : immutableComponents) { + List<ILSMDiskComponent> immutableComponents = diskComponents; + for (ILSMDiskComponent c : immutableComponents) { LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c; component.getInvIndex().validate(); component.getDeletedKeysBTree().validate(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java index 4c4a552..cef4257 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java @@ -26,17 +26,17 @@ import org.apache.hyracks.storage.am.common.api.IIndexCursor; import org.apache.hyracks.storage.am.common.api.ISearchPredicate; import org.apache.hyracks.storage.am.common.api.IndexException; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext; import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor; -public class LSMInvertedIndexAccessor implements ILSMIndexAccessorInternal, IInvertedIndexAccessor { +public class LSMInvertedIndexAccessor implements ILSMIndexAccessor, IInvertedIndexAccessor { protected final ILSMHarness lsmHarness; protected final ILSMIndexOperationContext ctx; @@ -93,7 +93,7 @@ public class LSMInvertedIndexAccessor implements ILSMIndexAccessorInternal, IInv } @Override - public void scheduleMerge(ILSMIOOperationCallback callback, List<ILSMComponent> components) + public void scheduleMerge(ILSMIOOperationCallback callback, List<ILSMDiskComponent> components) throws HyracksDataException, IndexException { ctx.setOperation(IndexOperation.MERGE); ctx.getComponentsToBeMerged().clear(); @@ -102,7 +102,7 @@ public class LSMInvertedIndexAccessor implements ILSMIndexAccessorInternal, IInv } @Override - public void scheduleReplication(List<ILSMComponent> lsmComponents, boolean bulkload, LSMOperationType opType) + public void scheduleReplication(List<ILSMDiskComponent> lsmComponents, boolean bulkload, LSMOperationType opType) throws HyracksDataException { ctx.setOperation(IndexOperation.REPLICATE); ctx.getComponentsToBeReplicated().clear(); @@ -127,8 +127,8 @@ public class LSMInvertedIndexAccessor implements ILSMIndexAccessorInternal, IInv } @Override - public void rangeSearch(IIndexCursor cursor, ISearchPredicate searchPred) throws IndexException, - HyracksDataException { + public void rangeSearch(IIndexCursor cursor, ISearchPredicate searchPred) + throws IndexException, HyracksDataException { search(cursor, searchPred); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java index 799a3da..8a12dea 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java @@ -21,20 +21,21 @@ package org.apache.hyracks.storage.am.lsm.invertedindex.impls; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter; import org.apache.hyracks.storage.am.btree.impls.BTree; +import org.apache.hyracks.storage.am.common.api.IMetadataPageManager; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter; -import org.apache.hyracks.storage.am.lsm.common.impls.AbstractDiskLSMComponent; +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponent; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex; import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex; -public class LSMInvertedIndexDiskComponent extends AbstractDiskLSMComponent { +public class LSMInvertedIndexDiskComponent extends AbstractLSMDiskComponent { private final IInvertedIndex invIndex; private final BTree deletedKeysBTree; private final BloomFilter bloomFilter; public LSMInvertedIndexDiskComponent(IInvertedIndex invIndex, BTree deletedKeysBTree, BloomFilter bloomFilter, - ILSMComponentFilter filter) { - super(filter); + ILSMComponentFilter filter) throws HyracksDataException { + super((IMetadataPageManager) deletedKeysBTree.getPageManager(), filter); this.invIndex = invIndex; this.deletedKeysBTree = deletedKeysBTree; this.bloomFilter = bloomFilter; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentFactory.java index a6f80ce..270d3a1 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentFactory.java @@ -22,16 +22,13 @@ package org.apache.hyracks.storage.am.lsm.invertedindex.impls; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory; import org.apache.hyracks.storage.am.btree.impls.BTree; -import org.apache.hyracks.storage.am.common.api.IndexException; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexFactory; -import org.apache.hyracks.storage.common.buffercache.IBufferCache; -public class LSMInvertedIndexDiskComponentFactory implements ILSMComponentFactory { +public class LSMInvertedIndexDiskComponentFactory implements ILSMDiskComponentFactory { private final OnDiskInvertedIndexFactory diskInvIndexFactory; private final TreeIndexFactory<BTree> btreeFactory; private final BloomFilterFactory bloomFilterFactory; @@ -47,16 +44,11 @@ public class LSMInvertedIndexDiskComponentFactory implements ILSMComponentFactor } @Override - public ILSMComponent createLSMComponentInstance(LSMComponentFileReferences cfr) throws IndexException, - HyracksDataException { - return new LSMInvertedIndexDiskComponent(diskInvIndexFactory.createIndexInstance(cfr - .getInsertIndexFileReference()), btreeFactory.createIndexInstance(cfr.getDeleteIndexFileReference()), + public LSMInvertedIndexDiskComponent createComponent(LSMComponentFileReferences cfr) throws HyracksDataException { + return new LSMInvertedIndexDiskComponent( + diskInvIndexFactory.createIndexInstance(cfr.getInsertIndexFileReference()), + btreeFactory.createIndexInstance(cfr.getDeleteIndexFileReference()), bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()), - filterFactory == null ? null : filterFactory.createLSMComponentFilter()); - } - - @Override - public IBufferCache getBufferCache() { - return diskInvIndexFactory.getBufferCache(); + filterFactory == null ? null : filterFactory.createFilter()); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java index b38d23a..03ba304 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java @@ -30,10 +30,10 @@ import org.apache.hyracks.storage.am.common.api.IndexException; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; public class LSMInvertedIndexFlushOperation implements ILSMIOOperation, Comparable<LSMInvertedIndexFlushOperation> { - private final ILSMIndexAccessorInternal accessor; + private final ILSMIndexAccessor accessor; private final ILSMComponent flushingComponent; private final FileReference dictBTreeFlushTarget; private final FileReference deletedKeysBTreeFlushTarget; @@ -41,7 +41,7 @@ public class LSMInvertedIndexFlushOperation implements ILSMIOOperation, Comparab private final ILSMIOOperationCallback callback; private final String indexIdentifier; - public LSMInvertedIndexFlushOperation(ILSMIndexAccessorInternal accessor, ILSMComponent flushingComponent, + public LSMInvertedIndexFlushOperation(ILSMIndexAccessor accessor, ILSMComponent flushingComponent, FileReference dictBTreeFlushTarget, FileReference deletedKeysBTreeFlushTarget, FileReference bloomFilterFlushTarget, ILSMIOOperationCallback callback, String indexIdentifier) { this.accessor = accessor; @@ -60,7 +60,7 @@ public class LSMInvertedIndexFlushOperation implements ILSMIOOperation, Comparab @Override public Set<IODeviceHandle> getWriteDevices() { - Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>(); + Set<IODeviceHandle> devs = new HashSet<>(); devs.add(dictBTreeFlushTarget.getDeviceHandle()); devs.add(deletedKeysBTreeFlushTarget.getDeviceHandle()); devs.add(bloomFilterFlushTarget.getDeviceHandle()); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMemoryComponent.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMemoryComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMemoryComponent.java index d5a9237..c164cf9 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMemoryComponent.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMemoryComponent.java @@ -23,10 +23,10 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter; import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; -import org.apache.hyracks.storage.am.lsm.common.impls.AbstractMemoryLSMComponent; +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMMemoryComponent; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex; -public class LSMInvertedIndexMemoryComponent extends AbstractMemoryLSMComponent { +public class LSMInvertedIndexMemoryComponent extends AbstractLSMMemoryComponent { private final IInvertedIndex invIndex; private final BTree deletedKeysBTree; @@ -47,7 +47,7 @@ public class LSMInvertedIndexMemoryComponent extends AbstractMemoryLSMComponent } @Override - protected void reset() throws HyracksDataException { + public void reset() throws HyracksDataException { super.reset(); invIndex.deactivate(); invIndex.destroy(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java index 7cbf544..c22af32 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java @@ -31,11 +31,11 @@ import org.apache.hyracks.storage.am.common.api.IndexException; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex; public class LSMInvertedIndexMergeOperation implements ILSMIOOperation { - private final ILSMIndexAccessorInternal accessor; + private final ILSMIndexAccessor accessor; private final List<ILSMComponent> mergingComponents; private final IIndexCursor cursor; private final FileReference dictBTreeMergeTarget; @@ -44,7 +44,7 @@ public class LSMInvertedIndexMergeOperation implements ILSMIOOperation { private final ILSMIOOperationCallback callback; private final String indexIdentifier; - public LSMInvertedIndexMergeOperation(ILSMIndexAccessorInternal accessor, List<ILSMComponent> mergingComponents, + public LSMInvertedIndexMergeOperation(ILSMIndexAccessor accessor, List<ILSMComponent> mergingComponents, IIndexCursor cursor, FileReference dictBTreeMergeTarget, FileReference deletedKeysBTreeMergeTarget, FileReference bloomFilterMergeTarget, ILSMIOOperationCallback callback, String indexIdentifier) { this.accessor = accessor; @@ -59,7 +59,7 @@ public class LSMInvertedIndexMergeOperation implements ILSMIOOperation { @Override public Set<IODeviceHandle> getReadDevices() { - Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>(); + Set<IODeviceHandle> devs = new HashSet<>(); for (Object o : mergingComponents) { LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) o; OnDiskInvertedIndex invIndex = (OnDiskInvertedIndex) component.getInvIndex(); @@ -72,7 +72,7 @@ public class LSMInvertedIndexMergeOperation implements ILSMIOOperation { @Override public Set<IODeviceHandle> getWriteDevices() { - Set<IODeviceHandle> devs = new HashSet<IODeviceHandle>(); + Set<IODeviceHandle> devs = new HashSet<>(); devs.add(dictBTreeMergeTarget.getDeviceHandle()); devs.add(deletedKeysBTreeMergeTarget.getDeviceHandle()); devs.add(bloomFilterMergeTarget.getDeviceHandle()); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java index 828e296..c8ff02f 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java @@ -31,7 +31,9 @@ import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor; @@ -41,8 +43,8 @@ public class LSMInvertedIndexOpContext extends AbstractLSMIndexOperationContext private IndexOperation op; private final List<ILSMComponent> componentHolder; - private final List<ILSMComponent> componentsToBeMerged; - private final List<ILSMComponent> componentsToBeReplicated; + private final List<ILSMDiskComponent> componentsToBeMerged; + private final List<ILSMDiskComponent> componentsToBeReplicated; private IModificationOperationCallback modificationCallback; private ISearchOperationCallback searchCallback; @@ -64,12 +66,12 @@ public class LSMInvertedIndexOpContext extends AbstractLSMIndexOperationContext public ISearchPredicate searchPredicate; - public LSMInvertedIndexOpContext(List<ILSMComponent> mutableComponents, + public LSMInvertedIndexOpContext(List<ILSMMemoryComponent> mutableComponents, IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback, int[] invertedIndexFields, int[] filterFields) throws HyracksDataException { - this.componentHolder = new LinkedList<ILSMComponent>(); - this.componentsToBeMerged = new LinkedList<ILSMComponent>(); - this.componentsToBeReplicated = new LinkedList<ILSMComponent>(); + this.componentHolder = new LinkedList<>(); + this.componentsToBeMerged = new LinkedList<>(); + this.componentsToBeReplicated = new LinkedList<>(); this.modificationCallback = modificationCallback; this.searchCallback = searchCallback; @@ -77,8 +79,8 @@ public class LSMInvertedIndexOpContext extends AbstractLSMIndexOperationContext deletedKeysBTreeAccessors = new IIndexAccessor[mutableComponents.size()]; for (int i = 0; i < mutableComponents.size(); i++) { - LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) mutableComponents - .get(i); + LSMInvertedIndexMemoryComponent mutableComponent = + (LSMInvertedIndexMemoryComponent) mutableComponents.get(i); mutableInvIndexAccessors[i] = (IInvertedIndexAccessor) mutableComponent.getInvIndex() .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); deletedKeysBTreeAccessors[i] = mutableComponent.getDeletedKeysBTree() @@ -149,7 +151,7 @@ public class LSMInvertedIndexOpContext extends AbstractLSMIndexOperationContext } @Override - public List<ILSMComponent> getComponentsToBeMerged() { + public List<ILSMDiskComponent> getComponentsToBeMerged() { return componentsToBeMerged; } @@ -164,7 +166,7 @@ public class LSMInvertedIndexOpContext extends AbstractLSMIndexOperationContext } @Override - public List<ILSMComponent> getComponentsToBeReplicated() { + public List<ILSMDiskComponent> getComponentsToBeReplicated() { return componentsToBeReplicated; } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java index bef5248..583c5f4 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedIndex.java @@ -23,7 +23,6 @@ import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; -import org.apache.hyracks.storage.am.btree.exceptions.BTreeException; import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType; import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor; @@ -59,7 +58,7 @@ public class InMemoryInvertedIndex implements IInvertedIndex { public InMemoryInvertedIndex(IBufferCache virtualBufferCache, IPageManager virtualFreePageManager, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, - IBinaryTokenizerFactory tokenizerFactory, FileReference btreeFileRef) throws BTreeException { + IBinaryTokenizerFactory tokenizerFactory, FileReference btreeFileRef) throws HyracksDataException { this.tokenTypeTraits = tokenTypeTraits; this.tokenCmpFactories = tokenCmpFactories; this.invListTypeTraits = invListTypeTraits; @@ -168,8 +167,8 @@ public class InMemoryInvertedIndex implements IInvertedIndex { @Override public IIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) throws HyracksDataException { - return new InMemoryInvertedIndexAccessor(this, new InMemoryInvertedIndexOpContext(btree, tokenCmpFactories, - tokenizerFactory)); + return new InMemoryInvertedIndexAccessor(this, + new InMemoryInvertedIndexOpContext(btree, tokenCmpFactories, tokenizerFactory)); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d718dc4a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java index f9473db..0bea41f 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/PartitionedInMemoryInvertedIndex.java @@ -26,12 +26,11 @@ import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; -import org.apache.hyracks.storage.am.btree.exceptions.BTreeException; import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor; -import org.apache.hyracks.storage.am.common.api.IPageManager; import org.apache.hyracks.storage.am.common.api.IIndexAccessor; import org.apache.hyracks.storage.am.common.api.IIndexOperationContext; import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback; +import org.apache.hyracks.storage.am.common.api.IPageManager; import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback; import org.apache.hyracks.storage.am.common.api.IndexException; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; @@ -53,7 +52,7 @@ public class PartitionedInMemoryInvertedIndex extends InMemoryInvertedIndex impl public PartitionedInMemoryInvertedIndex(IBufferCache memBufferCache, IPageManager memFreePageManager, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, - IBinaryTokenizerFactory tokenizerFactory, FileReference btreeFileRef) throws BTreeException { + IBinaryTokenizerFactory tokenizerFactory, FileReference btreeFileRef) throws HyracksDataException { super(memBufferCache, memFreePageManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, btreeFileRef); } @@ -63,7 +62,8 @@ public class PartitionedInMemoryInvertedIndex extends InMemoryInvertedIndex impl throws HyracksDataException, IndexException { super.insert(tuple, btreeAccessor, ictx); PartitionedInMemoryInvertedIndexOpContext ctx = (PartitionedInMemoryInvertedIndexOpContext) ictx; - PartitionedInvertedIndexTokenizingTupleIterator tupleIter = (PartitionedInvertedIndexTokenizingTupleIterator) ctx.tupleIter; + PartitionedInvertedIndexTokenizingTupleIterator tupleIter = + (PartitionedInvertedIndexTokenizingTupleIterator) ctx.tupleIter; updatePartitionIndexes(tupleIter.getNumTokens()); } @@ -91,8 +91,8 @@ public class PartitionedInMemoryInvertedIndex extends InMemoryInvertedIndex impl @Override public IIndexAccessor createAccessor(IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) throws HyracksDataException { - return new PartitionedInMemoryInvertedIndexAccessor(this, new PartitionedInMemoryInvertedIndexOpContext(btree, - tokenCmpFactories, tokenizerFactory)); + return new PartitionedInMemoryInvertedIndexAccessor(this, + new PartitionedInMemoryInvertedIndexOpContext(btree, tokenCmpFactories, tokenizerFactory)); } @Override @@ -131,8 +131,8 @@ public class PartitionedInMemoryInvertedIndex extends InMemoryInvertedIndex impl // using the last existing partition and re-searching the BTree with an open interval as low key. for (short i = partitionStartIndex; i <= partitionEndIndex; i++) { partSearcher.setNumTokensBoundsInSearchKeys(i, i); - InMemoryInvertedListCursor inMemListCursor = (InMemoryInvertedListCursor) partSearcher - .getCachedInvertedListCursor(); + InMemoryInvertedListCursor inMemListCursor = + (InMemoryInvertedListCursor) partSearcher.getCachedInvertedListCursor(); inMemListCursor.prepare(ctx.btreeAccessor, ctx.btreePred, ctx.tokenFieldsCmp, ctx.btreeCmp); inMemListCursor.reset(searchKey); invListPartitions.addInvertedListCursor(inMemListCursor, i);
