http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java index d4e6c77..8391e5c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java @@ -24,10 +24,12 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Queue; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Executor; @@ -45,7 +47,6 @@ import org.apache.hyracks.api.io.IFileHandle; import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.api.lifecycle.ILifeCycleComponent; import org.apache.hyracks.api.replication.IIOReplicationManager; -import org.apache.hyracks.api.util.IoUtil; import org.apache.hyracks.storage.common.file.BufferedFileHandle; import org.apache.hyracks.storage.common.file.IFileMapManager; @@ -107,7 +108,7 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { closed = false; fifoWriter = new AsyncFIFOPageQueueManager(this); - if (DEBUG) { + if ( DEBUG ) { confiscatedPages = new ArrayList<>(); confiscatedPagesOwner = new HashMap<>(); confiscateLock = new ReentrantLock(); @@ -217,7 +218,7 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { cPage.valid = true; } pageReplacementStrategy.notifyCachePageAccess(cPage); - if (DEBUG) { + if (DEBUG){ pinnedPageOwner.put(cPage, Thread.currentThread().getStackTrace()); } return cPage; @@ -334,9 +335,9 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { } int victimHash = hash(victim.dpid); if (victimHash == hash) { - /* - * Case 2a. - */ + /* + * Case 2a. + */ bucket.bucketLock.lock(); try { if (!victim.pinCount.compareAndSet(0, 1)) { @@ -371,9 +372,9 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { } return victim; } else { - /* - * Case 2b. - */ + /* + * Case 2b. + */ CacheBucket victimBucket = pageMap[victimHash]; if (victimHash < hash) { victimBucket.bucketLock.lock(); @@ -474,7 +475,7 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { } } buffer.append("Number of cached pages: ").append(nCachedPages).append('\n'); - if (DEBUG) { + if (DEBUG){ confiscateLock.lock(); try { buffer.append("Number of confiscated pages: ").append(confiscatedPages.size()).append('\n'); @@ -486,28 +487,29 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { } @Override - public boolean isClean() { + public boolean isClean(){ List<Long> reachableDpids = new LinkedList<>(); synchronized (cachedPages) { for (ICachedPageInternal internalPage : cachedPages) { - CachedPage c = (CachedPage) internalPage; - if (c.confiscated() || c.latch.getReadLockCount() != 0 || c.latch.getWriteHoldCount() != 0) { + CachedPage c = (CachedPage) internalPage; + if (c.confiscated() || + c.latch.getReadLockCount() != 0 || c.latch.getWriteHoldCount() != 0) { return false; } - if (c.valid) { + if (c.valid){ reachableDpids.add(c.dpid); } } } - for (Long l : reachableDpids) { - if (!canFindValidCachedPage(l)) { + for(Long l: reachableDpids){ + if (!canFindValidCachedPage(l)){ return false; } } return true; } - private boolean canFindValidCachedPage(long dpid) { + private boolean canFindValidCachedPage(long dpid){ int hash = hash(dpid); CachedPage cPage = null; CacheBucket bucket = pageMap[hash]; @@ -570,16 +572,14 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { } BufferedFileHandle getFileInfo(int fileId) throws HyracksDataException { - BufferedFileHandle fInfo; synchronized (fileInfoMap) { - fInfo = fileInfoMap.get(fileId); - } - if (fInfo == null) { - throw new HyracksDataException("No such file mapped"); + BufferedFileHandle fInfo = fileInfoMap.get(fileId); + if (fInfo == null) { + throw new HyracksDataException("No such file mapped"); + } + return fInfo; } - return fInfo; } - private BufferCacheHeaderHelper checkoutHeaderHelper() { BufferCacheHeaderHelper helper = headerPageCache.poll(); if (helper == null) { @@ -609,8 +609,8 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { getOffsetForPage(BufferedFileHandle.getPageId(cPage.dpid)), header.prepareWrite(cPage, buf)); - if (bytesWritten != (contiguousLargePages ? pageSize * (totalPages - 1) : 0) - + getPageSizeWithHeader()) { + if (bytesWritten != + (contiguousLargePages ? pageSize * (totalPages - 1) : 0) + getPageSizeWithHeader()) { throw new HyracksDataException("Failed to write completely: " + bytesWritten); } } finally { @@ -636,6 +636,7 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { } } + private int hash(long dpid) { int hashValue = (int) dpid ^ (Integer.reverse((int) (dpid >>> 32)) >>> 1); return hashValue % pageMap.length; @@ -789,39 +790,13 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { } @Override - public int createFile(FileReference fileRef) throws HyracksDataException { + public void createFile(FileReference fileRef) throws HyracksDataException { if (LOGGER.isLoggable(fileOpsLevel)) { LOGGER.log(fileOpsLevel, "Creating file: " + fileRef + " in cache: " + this); } - IoUtil.create(fileRef); - try { - return fileMapManager.registerFile(fileRef); - } catch (Exception e) { - // If file registration failed for any reason, we need to undo the file creation - try { - IoUtil.delete(fileRef); - } catch (Exception deleteException) { - e.addSuppressed(deleteException); - } - throw HyracksDataException.create(e); - } - } - - @Override - public int openFile(FileReference fileRef) throws HyracksDataException { - if (LOGGER.isLoggable(fileOpsLevel)) { - LOGGER.log(fileOpsLevel, "Opening file: " + fileRef + " in cache: " + this); - } - int fileId; synchronized (fileInfoMap) { - if (fileMapManager.isMapped(fileRef)) { - fileId = fileMapManager.lookupFileId(fileRef); - } else { - fileId = fileMapManager.registerFile(fileRef); - } - openFile(fileId); + fileMapManager.registerFile(fileRef); } - return fileId; } @Override @@ -833,9 +808,10 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { BufferedFileHandle fInfo; fInfo = fileInfoMap.get(fileId); if (fInfo == null) { + + // map is full, make room by cleaning up unreferenced files boolean unreferencedFileFound = true; while (fileInfoMap.size() >= maxOpenFiles && unreferencedFileFound) { - // map is full, make room by cleaning up unreferenced files unreferencedFileFound = false; for (Map.Entry<Integer, BufferedFileHandle> entry : fileInfoMap.entrySet()) { if (entry.getValue().getReferenceCount() <= 0) { @@ -853,10 +829,12 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { } } } + if (fileInfoMap.size() >= maxOpenFiles) { throw new HyracksDataException("Could not open fileId " + fileId + ". Max number of files " + maxOpenFiles + " already opened and referenced."); } + // create, open, and map new file reference FileReference fileRef = fileMapManager.lookupFileName(fileId); IFileHandle fh = ioManager.open(fileRef, IIOManager.FileReadWriteMode.READ_WRITE, @@ -961,49 +939,30 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { } @Override - public void deleteFile(FileReference fileRef) throws HyracksDataException { - synchronized (fileInfoMap) { - if (fileMapManager.isMapped(fileRef)) { - int fileId = fileMapManager.lookupFileId(fileRef); - deleteFile(fileId); - return; - } else { - IoUtil.delete(fileRef); - } - } - } - - @Override - public void deleteFile(int fileId) throws HyracksDataException { + public synchronized void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException { if (LOGGER.isLoggable(fileOpsLevel)) { LOGGER.log(fileOpsLevel, "Deleting file: " + fileId + " in cache: " + this); } synchronized (fileInfoMap) { - sweepAndFlush(fileId, false); + sweepAndFlush(fileId, flushDirtyPages); BufferedFileHandle fInfo = null; try { fInfo = fileInfoMap.get(fileId); if (fInfo != null && fInfo.getReferenceCount() > 0) { throw new HyracksDataException("Deleting open file"); } - } catch (Exception e) { - throw HyracksDataException.create(e); } finally { - FileReference fileRef = fileMapManager.unregisterFile(fileId); - try { - if (fInfo != null) { - // Mark the fInfo as deleted, - // such that when its pages are reclaimed in openFile(), - // the pages are not flushed to disk but only invalidated. - synchronized (fInfo) { - if (!fInfo.fileHasBeenDeleted()) { - ioManager.close(fInfo.getFileHandle()); - fInfo.markAsDeleted(); - } + fileMapManager.unregisterFile(fileId); + if (fInfo != null) { + // Mark the fInfo as deleted, + // such that when its pages are reclaimed in openFile(), + // the pages are not flushed to disk but only invalidated. + synchronized (fInfo) { + if (!fInfo.fileHasBeenDeleted()) { + ioManager.close(fInfo.getFileHandle()); + fInfo.markAsDeleted(); } } - } finally { - IoUtil.delete(fileRef); } } } @@ -1052,7 +1011,7 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { @Override public boolean removePage(ICachedPageInternal victimPage) { - CachedPage victim = (CachedPage) victimPage; + CachedPage victim = (CachedPage)victimPage; // Case 1 from findPage() if (victim.dpid < 0) { // new page if (!victim.pinCount.compareAndSet(0, 1)) { @@ -1120,7 +1079,7 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { ICachedPageInternal old = cachedPages.set(victim.cpid, null); if (DEBUG) { assert old == victim; - } ; + }; } return true; } @@ -1158,11 +1117,12 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId) throws HyracksDataException { ICachedPage cachedPage = confiscatePage(dpid, multiplier); - ((ICachedPageInternal) cachedPage).setExtraBlockPageId(extraBlockPageId); + ((ICachedPageInternal)cachedPage).setExtraBlockPageId(extraBlockPageId); return cachedPage; } - private ICachedPage confiscatePage(long dpid, int multiplier) throws HyracksDataException { + private ICachedPage confiscatePage(long dpid, int multiplier) + throws HyracksDataException { return getPageLoop(dpid, multiplier, true); } @@ -1254,7 +1214,8 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { return null; } - private ICachedPage getPageLoop(long dpid, int multiplier, boolean confiscate) throws HyracksDataException { + private ICachedPage getPageLoop(long dpid, int multiplier, boolean confiscate) + throws HyracksDataException { final long startingPinCount = DEBUG ? masterPinCount.get() : -1; int cycleCount = 0; try { @@ -1298,16 +1259,16 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { if (cycleCount > MAX_PIN_ATTEMPT_CYCLES) { cycleCount = 0; // suppress warning below throw new HyracksDataException("Unable to find free page in buffer cache after " - + MAX_PIN_ATTEMPT_CYCLES + " cycles (buffer cache undersized?)" - + (DEBUG ? " ; " + (masterPinCount.get() - startingPinCount) - + " successful pins since start of cycle" : "")); + + MAX_PIN_ATTEMPT_CYCLES + " cycles (buffer cache undersized?)" + (DEBUG ? " ; " + + (masterPinCount.get() - startingPinCount) + " successful pins since start of cycle" + : "")); } } } finally { if (cycleCount > PIN_ATTEMPT_CYCLES_WARNING_THRESHOLD && LOGGER.isLoggable(Level.WARNING)) { - LOGGER.warning("Took " + cycleCount + " cycles to find free page in buffer cache. (buffer cache " - + "undersized?)" + (DEBUG ? " ; " + (masterPinCount.get() - startingPinCount) - + " successful pins since start of cycle" : "")); + LOGGER.warning("Took " + cycleCount + " cycles to find free page in buffer cache. (buffer cache " + + "undersized?)" + (DEBUG ? " ; " + (masterPinCount.get() - startingPinCount) + + " successful pins since start of cycle" : "")); } } } @@ -1321,7 +1282,7 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { public void returnPage(ICachedPage page, boolean reinsert) { CachedPage cPage = (CachedPage) page; CacheBucket bucket; - if (!page.confiscated()) { + if (!page.confiscated()){ return; } if (reinsert) { @@ -1337,7 +1298,7 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { cPage.next = bucket.cachedPage; bucket.cachedPage = cPage; cPage.pinCount.decrementAndGet(); - if (DEBUG) { + if (DEBUG){ assert cPage.pinCount.get() == 0; assert cPage.latch.getReadLockCount() == 0; assert cPage.latch.getWriteHoldCount() == 0; @@ -1353,7 +1314,7 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { } else { cPage.invalidate(); cPage.pinCount.decrementAndGet(); - if (DEBUG) { + if (DEBUG){ assert cPage.pinCount.get() == 0; assert cPage.latch.getReadLockCount() == 0; assert cPage.latch.getWriteHoldCount() == 0; @@ -1402,23 +1363,23 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent { * _ONLY_ call this if you absolutely, positively know this file has no dirty pages in the cache! * Bypasses the normal lifecycle of a file handle and evicts all references to it immediately. */ - public void purgeHandle(int fileId) throws HyracksDataException { - synchronized (fileInfoMap) { - BufferedFileHandle fh = fileInfoMap.get(fileId); - if (fh != null) { - ioManager.close(fh.getFileHandle()); - fileInfoMap.remove(fileId); - fileMapManager.unregisterFile(fileId); - } + public void purgeHandle(int fileId) throws HyracksDataException{ + synchronized(fileInfoMap){ + BufferedFileHandle fh = fileInfoMap.get(fileId); + if (fh != null){ + ioManager.close(fh.getFileHandle()); + fileInfoMap.remove(fileId); + fileMapManager.unregisterFile(fileId); + } } } static class BufferCacheHeaderHelper { private static final int FRAME_MULTIPLIER_OFF = 0; - private static final int EXTRA_BLOCK_PAGE_ID_OFF = FRAME_MULTIPLIER_OFF + 4; // 4 + private static final int EXTRA_BLOCK_PAGE_ID_OFF = FRAME_MULTIPLIER_OFF + 4; // 4 private final ByteBuffer buf; - private final ByteBuffer[] array; + private final ByteBuffer [] array; private BufferCacheHeaderHelper(int pageSize) { buf = ByteBuffer.allocate(RESERVED_HEADER_BYTES + pageSize);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java index 8f7a965..4461069 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java @@ -52,10 +52,9 @@ public class DebugBufferCache implements IBufferCache { } @Override - public int createFile(FileReference fileRef) throws HyracksDataException { - int fileId = bufferCache.createFile(fileRef); + public void createFile(FileReference fileRef) throws HyracksDataException { + bufferCache.createFile(fileRef); createFileCount.addAndGet(1); - return fileId; } @Override @@ -71,8 +70,8 @@ public class DebugBufferCache implements IBufferCache { } @Override - public void deleteFile(int fileId) throws HyracksDataException { - bufferCache.deleteFile(fileId); + public void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException { + bufferCache.deleteFile(fileId, flushDirtyPages); deleteFileCount.addAndGet(1); } @@ -250,16 +249,4 @@ public class DebugBufferCache implements IBufferCache { throws HyracksDataException { bufferCache.resizePage(page, multiplier, extraPageBlockHelper); } - - @Override - public int openFile(FileReference fileRef) throws HyracksDataException { - openFileCount.incrementAndGet(); - return bufferCache.openFile(fileRef); - } - - @Override - public void deleteFile(FileReference file) throws HyracksDataException { - deleteFileCount.incrementAndGet(); - bufferCache.deleteFile(file); - } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java index 789f7b7..a3fc0e2 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java @@ -28,65 +28,13 @@ public interface IBufferCache { int INVALID_PAGEID = -1; int RESERVED_HEADER_BYTES = 8; - /** - * Create file on disk - * - * @param fileRef - * the file to create - * @return the file id - * @throws HyracksDataException - * if the file already exists or attempt to create the file failed - */ - int createFile(FileReference fileRef) throws HyracksDataException; - - /** - * Open the file and register it (if not registered) with the file map manager - * - * @param fileRef - * the file to open - * @return the file id - * @throws HyracksDataException - * if the file doesn't exist or buffer cache failed to open the file - */ - int openFile(FileReference fileRef) throws HyracksDataException; - - /** - * Open the mapped file with the passed file id - * - * @param fileId - * the file id - * @throws HyracksDataException - * if the file doesn't exist or buffer cache fails to open the file - */ + void createFile(FileReference fileRef) throws HyracksDataException; + void openFile(int fileId) throws HyracksDataException; - /** - * close the file - * - * @param fileId - * the file id - * @throws HyracksDataException - * if file doesn't exist or is not open - */ void closeFile(int fileId) throws HyracksDataException; - /** - * delete the file from memory and disk - * - * @param fileId - * the file id - * @throws HyracksDataException - * if the file doesn't exist or if a failure to delete takes place - */ - void deleteFile(int fileId) throws HyracksDataException; - - /** - * Delete from memory if registered and from disk - * - * @param file - * @throws HyracksDataException - */ - void deleteFile(FileReference file) throws HyracksDataException; + void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException; ICachedPage tryPin(long dpid) throws HyracksDataException; @@ -126,9 +74,9 @@ public interface IBufferCache { void setPageDiskId(ICachedPage page, long dpid); - boolean isReplicationEnabled(); + public boolean isReplicationEnabled(); - IIOReplicationManager getIOReplicationManager(); + public IIOReplicationManager getIOReplicationManager(); void purgeHandle(int fileId) throws HyracksDataException; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/FileMapManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/FileMapManager.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/FileMapManager.java deleted file mode 100644 index 209974e..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/FileMapManager.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hyracks.storage.common.file; - -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -import org.apache.hyracks.api.exceptions.ErrorCode; -import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.api.io.FileReference; - -public class FileMapManager implements IFileMapManager { - private static final long serialVersionUID = 1L; - - private ConcurrentMap<Integer, FileReference> id2nameMap = new ConcurrentHashMap<>(); - private ConcurrentMap<FileReference, Integer> name2IdMap = new ConcurrentHashMap<>(); - private int idCounter = 0; - - @Override - public FileReference lookupFileName(int fileId) throws HyracksDataException { - FileReference fRef = id2nameMap.get(fileId); - if (fRef == null) { - throw HyracksDataException.create(ErrorCode.NO_MAPPING_FOR_FILE_ID, fileId); - } - return fRef; - } - - @Override - public int lookupFileId(FileReference fileRef) throws HyracksDataException { - Integer fileId = name2IdMap.get(fileRef); - if (fileId == null) { - throw HyracksDataException.create(ErrorCode.NO_MAPPING_FOR_FILENAME, fileRef); - } - return fileId; - } - - @Override - public boolean isMapped(FileReference fileRef) { - return name2IdMap.containsKey(fileRef); - } - - @Override - public boolean isMapped(int fileId) { - return id2nameMap.containsKey(fileId); - } - - @Override - public synchronized FileReference unregisterFile(int fileId) throws HyracksDataException { - FileReference fileRef = id2nameMap.remove(fileId); - if (fileRef == null) { - throw HyracksDataException.create(ErrorCode.NO_MAPPING_FOR_FILE_ID, fileId); - } - name2IdMap.remove(fileRef); - return fileRef; - } - - @Override - public synchronized int registerFile(FileReference fileRef) throws HyracksDataException { - if (isMapped(fileRef)) { - throw HyracksDataException.create(ErrorCode.FILE_ALREADY_MAPPED, fileRef); - } - int fileId = idCounter++; - id2nameMap.put(fileId, fileRef); - name2IdMap.put(fileRef, fileId); - return fileId; - } - -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/IFileMapManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/IFileMapManager.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/IFileMapManager.java index 9a633b3..0d81776 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/IFileMapManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/IFileMapManager.java @@ -34,9 +34,8 @@ public interface IFileMapManager extends IFileMapProvider { * - file reference to register * @throws HyracksDataException * - if a mapping for the file already exists. - * @return the file id */ - int registerFile(FileReference fileRef) throws HyracksDataException; + public void registerFile(FileReference fileRef) throws HyracksDataException; /** * Unregister a file mapping @@ -45,8 +44,7 @@ public interface IFileMapManager extends IFileMapProvider { * - The file id whose mapping is to be unregistered. * @throws HyracksDataException * - If the fileid is not mapped currently in this manager. - * @return the file reference */ - FileReference unregisterFile(int fileId) throws HyracksDataException; + public void unregisterFile(int fileId) throws HyracksDataException; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/TransientFileMapManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/TransientFileMapManager.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/TransientFileMapManager.java new file mode 100644 index 0000000..66d54b1 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/TransientFileMapManager.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.storage.common.file; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.io.FileReference; + +public class TransientFileMapManager implements IFileMapManager { + private static final long serialVersionUID = 1L; + + private Map<Integer, FileReference> id2nameMap = new HashMap<Integer, FileReference>(); + private Map<FileReference, Integer> name2IdMap = new HashMap<FileReference, Integer>(); + private int idCounter = 0; + + @Override + public FileReference lookupFileName(int fileId) throws HyracksDataException { + FileReference fRef = id2nameMap.get(fileId); + if (fRef == null) { + throw new HyracksDataException("No mapping found for id: " + fileId); + } + return fRef; + } + + @Override + public int lookupFileId(FileReference fileRef) throws HyracksDataException { + Integer fileId = name2IdMap.get(fileRef); + if (fileId == null) { + throw new HyracksDataException("No mapping found for name: " + fileRef); + } + return fileId; + } + + @Override + public boolean isMapped(FileReference fileRef) { + return name2IdMap.containsKey(fileRef); + } + + @Override + public boolean isMapped(int fileId) { + return id2nameMap.containsKey(fileId); + } + + @Override + public void unregisterFile(int fileId) throws HyracksDataException { + FileReference fileRef = id2nameMap.remove(fileId); + name2IdMap.remove(fileRef); + } + + @Override + public void registerFile(FileReference fileRef) throws HyracksDataException { + Integer fileId = idCounter++; + id2nameMap.put(fileId, fileRef); + name2IdMap.put(fileRef, fileId); + } + +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/AbstractIndexLifecycleTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/AbstractIndexLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/AbstractIndexLifecycleTest.java index aac4df5..8211685 100644 --- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/AbstractIndexLifecycleTest.java +++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/AbstractIndexLifecycleTest.java @@ -18,13 +18,14 @@ */ package org.apache.hyracks.storage.am.common; -import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.storage.common.IIndex; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.storage.common.IIndex; + public abstract class AbstractIndexLifecycleTest { protected IIndex index; @@ -47,16 +48,10 @@ public abstract class AbstractIndexLifecycleTest { @Test public void validSequenceTest() throws Exception { - // Double create is invalid + // Double create is valid index.create(); Assert.assertTrue(persistentStateExists()); - boolean exceptionCaught = false; - try { - index.create(); - } catch (Exception e) { - exceptionCaught = true; - } - Assert.assertTrue(exceptionCaught); + index.create(); Assert.assertTrue(persistentStateExists()); // Double open is valid @@ -74,6 +69,8 @@ public abstract class AbstractIndexLifecycleTest { // Insert more stuff performInsertions(); + + // Double close is valid index.deactivate(); // Check that the inserted stuff is still there @@ -81,16 +78,10 @@ public abstract class AbstractIndexLifecycleTest { checkInsertions(); index.deactivate(); - // Double destroy is invalid + // Double destroy is valid index.destroy(); Assert.assertFalse(persistentStateExists()); - exceptionCaught = false; - try { - index.destroy(); - } catch (Exception e) { - exceptionCaught = true; - } - Assert.assertTrue(exceptionCaught); + index.destroy(); Assert.assertFalse(persistentStateExists()); } @@ -98,34 +89,20 @@ public abstract class AbstractIndexLifecycleTest { public void invalidSequenceTest1() throws Exception { index.create(); index.activate(); - try { - index.create(); - } finally { - index.deactivate(); - index.destroy(); - } + index.create(); } @Test(expected = HyracksDataException.class) public void invalidSequenceTest2() throws Exception { index.create(); index.activate(); - try { - index.destroy(); - } finally { - index.deactivate(); - index.destroy(); - } + index.destroy(); } @Test(expected = HyracksDataException.class) public void invalidSequenceTest3() throws Exception { index.create(); - try { - index.clear(); - } finally { - index.destroy(); - } + index.clear(); } @Test(expected = HyracksDataException.class) @@ -137,12 +114,7 @@ public abstract class AbstractIndexLifecycleTest { public void invalidSequenceTest5() throws Exception { index.create(); index.activate(); - try { - index.activate(); - } finally { - index.deactivate(); - index.destroy(); - } + index.activate(); } @Test(expected = HyracksDataException.class) @@ -150,10 +122,6 @@ public abstract class AbstractIndexLifecycleTest { index.create(); index.activate(); index.deactivate(); - try { - index.deactivate(); - } finally { - index.destroy(); - } + index.deactivate(); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManager.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManager.java index 350de74..3caeedd 100644 --- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManager.java +++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManager.java @@ -24,6 +24,7 @@ import org.apache.hyracks.storage.common.ILocalResourceRepository; import org.apache.hyracks.storage.common.IResourceLifecycleManager; import org.apache.hyracks.storage.common.IStorageManager; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.file.IFileMapProvider; import org.apache.hyracks.storage.common.file.ResourceIdFactory; public class TestStorageManager implements IStorageManager { @@ -35,6 +36,11 @@ public class TestStorageManager implements IStorageManager { } @Override + public IFileMapProvider getFileMapProvider(INCServiceContext ctx) { + return TestStorageManagerComponentHolder.getFileMapProvider(); + } + + @Override public ILocalResourceRepository getLocalResourceRepository(INCServiceContext ctx) { return TestStorageManagerComponentHolder.getLocalResourceRepository(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManagerComponentHolder.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManagerComponentHolder.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManagerComponentHolder.java index 3060b25..d9c4b85 100644 --- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManagerComponentHolder.java +++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestStorageManagerComponentHolder.java @@ -43,12 +43,12 @@ import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICacheMemoryAllocator; import org.apache.hyracks.storage.common.buffercache.IPageReplacementStrategy; -import org.apache.hyracks.storage.common.file.FileMapManager; import org.apache.hyracks.storage.common.file.IFileMapManager; import org.apache.hyracks.storage.common.file.IFileMapProvider; import org.apache.hyracks.storage.common.file.ILocalResourceRepositoryFactory; import org.apache.hyracks.storage.common.file.ResourceIdFactory; import org.apache.hyracks.storage.common.file.ResourceIdFactoryProvider; +import org.apache.hyracks.storage.common.file.TransientFileMapManager; import org.apache.hyracks.storage.common.file.TransientLocalResourceRepositoryFactory; public class TestStorageManagerComponentHolder { @@ -98,9 +98,9 @@ public class TestStorageManagerComponentHolder { return bufferCache; } - private synchronized static IFileMapProvider getFileMapProvider() { + public synchronized static IFileMapProvider getFileMapProvider() { if (fileMapProvider == null) { - fileMapProvider = new FileMapManager(); + fileMapProvider = new TransientFileMapManager(); } return fileMapProvider; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/BloomFilterTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/BloomFilterTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/BloomFilterTest.java index 35779ac..26d6d85 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/BloomFilterTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/BloomFilterTest.java @@ -62,12 +62,13 @@ public class BloomFilterTest extends AbstractBloomFilterTest { int numElements = 100; int[] keyFields = { 0 }; - BloomFilter bf = new BloomFilter(bufferCache, harness.getFileReference(), keyFields); + BloomFilter bf = new BloomFilter(bufferCache, harness.getFileMapProvider(), harness.getFileReference(), + keyFields); double acceptanleFalsePositiveRate = 0.1; int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements); - BloomFilterSpecification bloomFilterSpec = - BloomCalculations.computeBloomSpec(maxBucketsPerElement, acceptanleFalsePositiveRate); + BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement, + acceptanleFalsePositiveRate); bf.create(); bf.activate(); @@ -120,12 +121,13 @@ public class BloomFilterTest extends AbstractBloomFilterTest { int numElements = 10000; int[] keyFields = { 2, 4, 1 }; - BloomFilter bf = new BloomFilter(bufferCache, harness.getFileReference(), keyFields); + BloomFilter bf = new BloomFilter(bufferCache, harness.getFileMapProvider(), harness.getFileReference(), + keyFields); double acceptanleFalsePositiveRate = 0.1; int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements); - BloomFilterSpecification bloomFilterSpec = - BloomCalculations.computeBloomSpec(maxBucketsPerElement, acceptanleFalsePositiveRate); + BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement, + acceptanleFalsePositiveRate); bf.create(); bf.activate(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/util/BloomFilterTestHarness.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/util/BloomFilterTestHarness.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/util/BloomFilterTestHarness.java index 6ccb2be..04519e5 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/util/BloomFilterTestHarness.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-bloomfilter-test/src/test/java/org/apache/hyracks/storage/am/bloomfilter/util/BloomFilterTestHarness.java @@ -29,6 +29,7 @@ import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.file.IFileMapProvider; import org.apache.hyracks.test.support.TestStorageManagerComponentHolder; import org.apache.hyracks.test.support.TestUtils; @@ -44,6 +45,7 @@ public class BloomFilterTestHarness { protected IIOManager ioManager; protected IHyracksTaskContext ctx; protected IBufferCache bufferCache; + protected IFileMapProvider fileMapProvider; protected FileReference file; protected final Random rnd = new Random(); @@ -68,6 +70,7 @@ public class BloomFilterTestHarness { TestStorageManagerComponentHolder.init(pageSize, numPages, maxOpenFiles); ioManager = ctx.getIoManager(); bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext()); + fileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider(); file = ioManager.getFileReference(0, simpleDateFormat.format(new Date())); rnd.setSeed(RANDOM_SEED); } @@ -85,6 +88,10 @@ public class BloomFilterTestHarness { return bufferCache; } + public IFileMapProvider getFileMapProvider() { + return fileMapProvider; + } + public Random getRandom() { return rnd; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeBulkLoadTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeBulkLoadTest.java index 394f989..76a9078 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeBulkLoadTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeBulkLoadTest.java @@ -50,9 +50,10 @@ public class BTreeBulkLoadTest extends OrderedIndexBulkLoadTest { @SuppressWarnings("rawtypes") @Override protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys, - BTreeLeafFrameType leafType, boolean filtered) throws Exception { - return BTreeTestContext.create(harness.getBufferCache(), harness.getFileReference(), fieldSerdes, numKeys, - leafType, harness.getPageManagerFactory().createPageManager(harness.getBufferCache())); + BTreeLeafFrameType leafType, boolean filtered) throws Exception { + return BTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(), + harness.getFileReference(), fieldSerdes, numKeys, leafType, harness.getPageManagerFactory() + .createPageManager(harness.getBufferCache())); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeDeleteTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeDeleteTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeDeleteTest.java index edf02fc..905fd22 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeDeleteTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeDeleteTest.java @@ -50,9 +50,10 @@ public class BTreeDeleteTest extends OrderedIndexDeleteTest { @SuppressWarnings("rawtypes") @Override protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys, - BTreeLeafFrameType leafType, boolean filtered) throws Exception { - return BTreeTestContext.create(harness.getBufferCache(), harness.getFileReference(), fieldSerdes, numKeys, - leafType, harness.getPageManagerFactory().createPageManager(harness.getBufferCache())); + BTreeLeafFrameType leafType, boolean filtered) throws Exception { + return BTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(), + harness.getFileReference(), fieldSerdes, numKeys, leafType, harness.getPageManagerFactory() + .createPageManager(harness.getBufferCache())); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeExamplesTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeExamplesTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeExamplesTest.java index 18580b0..6c6161f 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeExamplesTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeExamplesTest.java @@ -52,7 +52,7 @@ public class BTreeExamplesTest extends OrderedIndexExamplesTest { ITreeIndexMetadataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory(); LinkedMetaDataPageManager freePageManager = new LinkedMetaDataPageManager(harness.getBufferCache(), metaFrameFactory); - return BTreeUtils.createBTree(harness.getBufferCache(), typeTraits, cmpFactories, + return BTreeUtils.createBTree(harness.getBufferCache(), harness.getFileMapProvider(), typeTraits, cmpFactories, BTreeLeafFrameType.REGULAR_NSM, harness.getFileReference(), freePageManager); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeInsertTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeInsertTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeInsertTest.java index 66c80fa..14810a7 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeInsertTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeInsertTest.java @@ -33,10 +33,10 @@ import org.junit.Before; * Tests the BTree insert operation with strings and integer fields using * various numbers of key and payload fields. Each tests first fills a BTree with * randomly generated tuples. We compare the following operations against expected results: - * 1) Point searches for all tuples - * 2) Ordered scan - * 3) Disk-order scan - * 4) Range search (and prefix search for composite keys) + * 1) Point searches for all tuples + * 2) Ordered scan + * 3) Disk-order scan + * 4) Range search (and prefix search for composite keys) */ public class BTreeInsertTest extends OrderedIndexInsertTest { @@ -59,9 +59,10 @@ public class BTreeInsertTest extends OrderedIndexInsertTest { @SuppressWarnings("rawtypes") @Override protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys, - BTreeLeafFrameType leafType, boolean filtered) throws Exception { - return BTreeTestContext.create(harness.getBufferCache(), harness.getFileReference(), fieldSerdes, numKeys, - leafType, harness.getPageManagerFactory().createPageManager(harness.getBufferCache())); + BTreeLeafFrameType leafType, boolean filtered) throws Exception { + return BTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(), + harness.getFileReference(), fieldSerdes, numKeys, leafType, harness.getPageManagerFactory() + .createPageManager(harness.getBufferCache())); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeLifecycleTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeLifecycleTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeLifecycleTest.java index 7caa40c..a096d59 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeLifecycleTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeLifecycleTest.java @@ -36,8 +36,7 @@ public class BTreeLifecycleTest extends AbstractIndexLifecycleTest { private final TreeIndexTestUtils titu = new OrderedIndexTestUtils(); @SuppressWarnings("rawtypes") - private final ISerializerDeserializer[] fieldSerdes = - new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE }; + private final ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE }; private ITreeIndexFrame frame = null; @@ -47,14 +46,20 @@ public class BTreeLifecycleTest extends AbstractIndexLifecycleTest { @Override public void setup() throws Exception { harness.setUp(); - testCtx = BTreeTestContext.create(harness.getBufferCache(), harness.getFileReference(), fieldSerdes, - fieldSerdes.length, BTreeLeafFrameType.REGULAR_NSM, - harness.getPageManagerFactory().createPageManager(harness.getBufferCache())); + testCtx = BTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(), + harness.getFileReference(), fieldSerdes, fieldSerdes.length, BTreeLeafFrameType.REGULAR_NSM, harness + .getPageManagerFactory().createPageManager(harness.getBufferCache())); index = testCtx.getIndex(); } @Override public void tearDown() throws HyracksDataException { + try { + testCtx.getIndex().deactivate(); + } catch (Exception e) { + } finally { + testCtx.getIndex().destroy(); + } harness.tearDown(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeModificationOperationCallbackTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeModificationOperationCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeModificationOperationCallbackTest.java index 304bf14..defb0bf 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeModificationOperationCallbackTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeModificationOperationCallbackTest.java @@ -37,9 +37,9 @@ public class BTreeModificationOperationCallbackTest extends AbstractModification protected void createIndexInstance() throws Exception { IMetadataPageManagerFactory freePageManagerFactory = new LinkedMetadataPageManagerFactory(); IMetadataPageManager freePageManager = freePageManagerFactory.createPageManager(harness.getBufferCache()); - index = BTreeUtils.createBTree(harness.getBufferCache(), SerdeUtils.serdesToTypeTraits(keySerdes), - SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), BTreeLeafFrameType.REGULAR_NSM, - harness.getFileReference(), freePageManager); + index = BTreeUtils.createBTree(harness.getBufferCache(), harness.getFileMapProvider(), SerdeUtils + .serdesToTypeTraits(keySerdes), SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), + BTreeLeafFrameType.REGULAR_NSM, harness.getFileReference(), freePageManager); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java index cc4f444..1b6f42b 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java @@ -98,16 +98,16 @@ public class BTreeSearchCursorTest extends AbstractBTreeTest { IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory); - BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, - fieldCount, harness.getFileReference()); + BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory, + leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference()); btree.create(); btree.activate(); ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount); ArrayTupleReference tuple = new ArrayTupleReference(); - ITreeIndexAccessor indexAccessor = - btree.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE); + ITreeIndexAccessor indexAccessor = btree.createAccessor(TestOperationCallback.INSTANCE, + TestOperationCallback.INSTANCE); // generate keys int numKeys = 50; @@ -174,16 +174,16 @@ public class BTreeSearchCursorTest extends AbstractBTreeTest { IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory); - BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, - fieldCount, harness.getFileReference()); + BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory, + leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference()); btree.create(); btree.activate(); ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount); ArrayTupleReference tuple = new ArrayTupleReference(); - ITreeIndexAccessor indexAccessor = - btree.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE); + ITreeIndexAccessor indexAccessor = btree.createAccessor(TestOperationCallback.INSTANCE, + TestOperationCallback.INSTANCE); // generate keys int numKeys = 50; @@ -247,16 +247,16 @@ public class BTreeSearchCursorTest extends AbstractBTreeTest { IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory); - BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, - fieldCount, harness.getFileReference()); + BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory, + leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference()); btree.create(); btree.activate(); ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount); ArrayTupleReference tuple = new ArrayTupleReference(); - ITreeIndexAccessor indexAccessor = - btree.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE); + ITreeIndexAccessor indexAccessor = btree.createAccessor(TestOperationCallback.INSTANCE, + TestOperationCallback.INSTANCE); // generate keys int numKeys = 50; @@ -309,8 +309,8 @@ public class BTreeSearchCursorTest extends AbstractBTreeTest { searchCmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator(); MultiComparator searchCmp = new MultiComparator(searchCmps); - RangePredicate rangePred = - new RangePredicate(lowKey, highKey, lowKeyInclusive, highKeyInclusive, searchCmp, searchCmp); + RangePredicate rangePred = new RangePredicate(lowKey, highKey, lowKeyInclusive, highKeyInclusive, searchCmp, + searchCmp); return rangePred; } @@ -356,8 +356,8 @@ public class BTreeSearchCursorTest extends AbstractBTreeTest { ITreeIndexCursor rangeCursor = new BTreeRangeSearchCursor(leafFrame, false); RangePredicate rangePred = createRangePredicate(lowKey, highKey, lowKeyInclusive, highKeyInclusive); - ITreeIndexAccessor indexAccessor = - btree.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE); + ITreeIndexAccessor indexAccessor = btree.createAccessor(TestOperationCallback.INSTANCE, + TestOperationCallback.INSTANCE); indexAccessor.search(rangeCursor, rangePred); try { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchOperationCallbackTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchOperationCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchOperationCallbackTest.java index 003dd39..c9248a6 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchOperationCallbackTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchOperationCallbackTest.java @@ -36,9 +36,10 @@ public class BTreeSearchOperationCallbackTest extends AbstractSearchOperationCal @Override protected void createIndexInstance() throws Exception { ITreeIndexMetadataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory(); - LinkedMetaDataPageManager freePageManager = - new LinkedMetaDataPageManager(harness.getBufferCache(), metaFrameFactory); - index = BTreeUtils.createBTree(harness.getBufferCache(), SerdeUtils.serdesToTypeTraits(keySerdes), + LinkedMetaDataPageManager freePageManager = new LinkedMetaDataPageManager(harness.getBufferCache(), + metaFrameFactory); + index = BTreeUtils.createBTree(harness.getBufferCache(), harness.getFileMapProvider(), + SerdeUtils.serdesToTypeTraits(keySerdes), SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), BTreeLeafFrameType.REGULAR_NSM, harness.getFileReference(), freePageManager); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSortedInsertTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSortedInsertTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSortedInsertTest.java index 08f3cf6..164f699 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSortedInsertTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSortedInsertTest.java @@ -59,9 +59,10 @@ public class BTreeSortedInsertTest extends OrderedIndexSortedInsertTest { @SuppressWarnings("rawtypes") @Override protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys, - BTreeLeafFrameType leafType, boolean filtered) throws Exception { - return BTreeTestContext.create(harness.getBufferCache(), harness.getFileReference(), fieldSerdes, numKeys, - leafType, harness.getPageManagerFactory().createPageManager(harness.getBufferCache())); + BTreeLeafFrameType leafType, boolean filtered) throws Exception { + return BTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(), + harness.getFileReference(), fieldSerdes, numKeys, leafType, harness.getPageManagerFactory() + .createPageManager(harness.getBufferCache())); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeStatsTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeStatsTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeStatsTest.java index 7281da6..ee7e968 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeStatsTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeStatsTest.java @@ -58,6 +58,7 @@ import org.apache.hyracks.storage.am.common.util.TreeIndexBufferCacheWarmup; import org.apache.hyracks.storage.am.common.util.TreeIndexStats; import org.apache.hyracks.storage.am.common.util.TreeIndexStatsGatherer; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.file.IFileMapProvider; import org.apache.hyracks.test.support.TestStorageManagerComponentHolder; import org.apache.hyracks.test.support.TestUtils; import org.junit.Test; @@ -76,6 +77,7 @@ public class BTreeStatsTest extends AbstractBTreeTest { TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES); IBufferCache bufferCache = harness.getBufferCache(); + IFileMapProvider fmp = harness.getFileMapProvider(); // declare fields int fieldCount = 2; @@ -99,7 +101,7 @@ public class BTreeStatsTest extends AbstractBTreeTest { IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory); - BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, + BTree btree = new BTree(bufferCache, fmp, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference()); btree.create(); btree.activate(); @@ -161,15 +163,16 @@ public class BTreeStatsTest extends AbstractBTreeTest { } } - TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, freePageManager, - harness.getFileReference(), btree.getRootPageId()); + int fileId = fmp.lookupFileId(harness.getFileReference()); + TreeIndexStatsGatherer statsGatherer = + new TreeIndexStatsGatherer(bufferCache, freePageManager, fileId, btree.getRootPageId()); TreeIndexStats stats = statsGatherer.gatherStats(leafFrame, interiorFrame, metaFrame); if (LOGGER.isLoggable(Level.INFO)) { LOGGER.info("\n" + stats.toString()); } TreeIndexBufferCacheWarmup bufferCacheWarmup = - new TreeIndexBufferCacheWarmup(bufferCache, freePageManager, harness.getFileReference()); + new TreeIndexBufferCacheWarmup(bufferCache, freePageManager, fileId); bufferCacheWarmup.warmup(leafFrame, metaFrame, new int[] { 1, 2 }, new int[] { 2, 5 }); btree.deactivate(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateSearchTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateSearchTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateSearchTest.java index 78c92eb..78023a2 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateSearchTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateSearchTest.java @@ -82,8 +82,8 @@ public class BTreeUpdateSearchTest extends AbstractBTreeTest { IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame(); IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory); - BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, - fieldCount, harness.getFileReference()); + BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory, + leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference()); btree.create(); btree.activate(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateTest.java index 3374fec..a8567e4 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateTest.java @@ -50,9 +50,10 @@ public class BTreeUpdateTest extends OrderedIndexUpdateTest { @SuppressWarnings("rawtypes") @Override protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys, - BTreeLeafFrameType leafType, boolean filtered) throws Exception { - return BTreeTestContext.create(harness.getBufferCache(), harness.getFileReference(), fieldSerdes, numKeys, - leafType, harness.getPageManagerFactory().createPageManager(harness.getBufferCache())); + BTreeLeafFrameType leafType, boolean filtered) throws Exception { + return BTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(), + harness.getFileReference(), fieldSerdes, numKeys, leafType, harness.getPageManagerFactory() + .createPageManager(harness.getBufferCache())); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpsertTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpsertTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpsertTest.java index bc6a94f..2d10a25 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpsertTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpsertTest.java @@ -33,10 +33,10 @@ import org.junit.Before; * Tests the BTree insert operation with strings and integer fields using * various numbers of key and payload fields. Each tests first fills a BTree with * randomly generated tuples. We compare the following operations against expected results: - * 1) Point searches for all tuples - * 2) Ordered scan - * 3) Disk-order scan - * 4) Range search (and prefix search for composite keys) + * 1) Point searches for all tuples + * 2) Ordered scan + * 3) Disk-order scan + * 4) Range search (and prefix search for composite keys) */ public class BTreeUpsertTest extends OrderedIndexUpsertTest { @@ -59,9 +59,10 @@ public class BTreeUpsertTest extends OrderedIndexUpsertTest { @SuppressWarnings("rawtypes") @Override protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys, - BTreeLeafFrameType leafType, boolean filtered) throws Exception { - return BTreeTestContext.create(harness.getBufferCache(), harness.getFileReference(), fieldSerdes, numKeys, - leafType, harness.getPageManagerFactory().createPageManager(harness.getBufferCache())); + BTreeLeafFrameType leafType, boolean filtered) throws Exception { + return BTreeTestContext.create(harness.getBufferCache(), harness.getFileMapProvider(), + harness.getFileReference(), fieldSerdes, numKeys, leafType, harness.getPageManagerFactory() + .createPageManager(harness.getBufferCache())); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java index 33177b0..2f1b89f 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java @@ -49,6 +49,7 @@ import org.apache.hyracks.storage.common.MultiComparator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.file.BufferedFileHandle; +import org.apache.hyracks.storage.common.file.IFileMapProvider; import org.junit.Assert; import org.junit.Test; @@ -127,8 +128,10 @@ public class FieldPrefixNSMTest extends AbstractBTreeTest { rnd.setSeed(50); IBufferCache bufferCache = harness.getBufferCache(); + IFileMapProvider fileMapProvider = harness.getFileMapProvider(); bufferCache.createFile(harness.getFileReference()); - int btreeFileId = bufferCache.openFile(harness.getFileReference()); + int btreeFileId = fileMapProvider.lookupFileId(harness.getFileReference()); + bufferCache.openFile(btreeFileId); IHyracksTaskContext ctx = harness.getHyracksTaskContext(); ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(btreeFileId, 0), true); try { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageFileAccessTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageFileAccessTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageFileAccessTest.java index 6dcb3a4..d62d0ad 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageFileAccessTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageFileAccessTest.java @@ -24,13 +24,14 @@ import java.util.List; import java.util.Random; import java.util.logging.Level; +import org.junit.Test; + import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.btree.util.AbstractBTreeTest; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.file.BufferedFileHandle; import org.apache.hyracks.storage.common.sync.LatchType; -import org.junit.Test; public class StorageFileAccessTest extends AbstractBTreeTest { public class PinnedLatchedPage { @@ -65,7 +66,7 @@ public class StorageFileAccessTest extends AbstractBTreeTest { private int loopCount = 0; private boolean fileIsOpen = false; private Random rnd = new Random(50); - private List<PinnedLatchedPage> pinnedPages = new LinkedList<>(); + private List<PinnedLatchedPage> pinnedPages = new LinkedList<PinnedLatchedPage>(); public FileAccessWorker(int workerId, IBufferCache bufferCache, FileAccessType fta, int fileId, int maxPages, int maxPinnedPages, int maxLoopCount, int closeFileChance, long thinkTime) { @@ -258,7 +259,8 @@ public class StorageFileAccessTest extends AbstractBTreeTest { public void oneThreadOneFileTest() throws Exception { IBufferCache bufferCache = harness.getBufferCache(); bufferCache.createFile(harness.getFileReference()); - int btreeFileId = bufferCache.openFile(harness.getFileReference()); + int btreeFileId = harness.getFileMapProvider().lookupFileId(harness.getFileReference()); + bufferCache.openFile(btreeFileId); Thread worker = new Thread(new FileAccessWorker(0, harness.getBufferCache(), FileAccessType.FTA_UNLATCHED, btreeFileId, 10, 10, 100, 10, 0)); worker.start(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java index fc58452..fdc5e95 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java @@ -52,7 +52,7 @@ public class BTreeMultiThreadTest extends OrderedIndexMultiThreadTest { @Override protected ITreeIndex createIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields) throws HyracksDataException { - return BTreeUtils.createBTree(harness.getBufferCache(), typeTraits, cmpFactories, + return BTreeUtils.createBTree(harness.getBufferCache(), harness.getFileMapProvider(), typeTraits, cmpFactories, BTreeLeafFrameType.REGULAR_NSM, harness.getFileReference(), harness.getPageManagerFactory().createPageManager(harness.getBufferCache())); } @@ -84,8 +84,9 @@ public class BTreeMultiThreadTest extends OrderedIndexMultiThreadTest { ProbabilityHelper.getUniformProbDist(insertDeleteUpdateUpsertOps.length))); // All operations mixed. - TestOperation[] allOps = new TestOperation[] { TestOperation.INSERT, TestOperation.DELETE, TestOperation.UPDATE, - TestOperation.UPSERT, TestOperation.POINT_SEARCH, TestOperation.SCAN, TestOperation.DISKORDER_SCAN }; + TestOperation[] allOps = new TestOperation[] { TestOperation.INSERT, TestOperation.DELETE, + TestOperation.UPDATE, TestOperation.UPSERT, TestOperation.POINT_SEARCH, TestOperation.SCAN, + TestOperation.DISKORDER_SCAN }; workloadConfs.add(new TestWorkloadConf(allOps, ProbabilityHelper.getUniformProbDist(allOps.length))); return workloadConfs; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/util/BTreeTestContext.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/util/BTreeTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/util/BTreeTestContext.java index c6c43c8..2abd547 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/util/BTreeTestContext.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/util/BTreeTestContext.java @@ -31,6 +31,7 @@ import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.common.api.IPageManager; import org.apache.hyracks.storage.am.common.api.ITreeIndex; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.file.IFileMapProvider; @SuppressWarnings("rawtypes") public class BTreeTestContext extends OrderedIndexTestContext { @@ -51,12 +52,14 @@ public class BTreeTestContext extends OrderedIndexTestContext { return btree.getComparatorFactories(); } - public static BTreeTestContext create(IBufferCache bufferCache, FileReference file, - ISerializerDeserializer[] fieldSerdes, int numKeyFields, BTreeLeafFrameType leafType, - IPageManager pageManager) throws Exception { + public static BTreeTestContext create(IBufferCache bufferCache, IFileMapProvider fileMapProvider, + FileReference file, ISerializerDeserializer[] fieldSerdes, int numKeyFields, BTreeLeafFrameType leafType, + IPageManager pageManager) + throws Exception { ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes); IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields); - BTree btree = BTreeUtils.createBTree(bufferCache, typeTraits, cmpFactories, leafType, file, pageManager); + BTree btree = BTreeUtils.createBTree(bufferCache, fileMapProvider, typeTraits, cmpFactories, leafType, file, + pageManager); BTreeTestContext testCtx = new BTreeTestContext(fieldSerdes, btree); return testCtx; }
