Author: elecharny
Date: Thu Jun 6 06:04:58 2013
New Revision: 1490166
URL: http://svn.apache.org/r1490166
Log:
Updated the way the free pages are managed. Some tests are failing.
Added:
labs/mavibot/trunk/mavibot/src/test/java/org/apache/mavibot/btree/InMemoryBTreeTestOps.java
labs/mavibot/trunk/mavibot/src/test/resources/
labs/mavibot/trunk/mavibot/src/test/resources/log4j.properties
Modified:
labs/mavibot/trunk/mavibot/pom.xml
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/AbstractResult.java
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/ModifyResult.java
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/RecordManager.java
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/SplitResult.java
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/serializer/StringSerializer.java
labs/mavibot/trunk/mavibot/src/test/java/org/apache/mavibot/btree/RecordManagerTest.java
Modified: labs/mavibot/trunk/mavibot/pom.xml
URL:
http://svn.apache.org/viewvc/labs/mavibot/trunk/mavibot/pom.xml?rev=1490166&r1=1490165&r2=1490166&view=diff
==============================================================================
--- labs/mavibot/trunk/mavibot/pom.xml (original)
+++ labs/mavibot/trunk/mavibot/pom.xml Thu Jun 6 06:04:58 2013
@@ -36,6 +36,10 @@
<properties>
<junit.version>4.10</junit.version>
<commons.collections.version>3.2.1</commons.collections.version>
+ <commons.io.version>2.4</commons.io.version>
+ <log4j.version>1.2.17</log4j.version>
+ <slf4j.api.version>1.7.5</slf4j.api.version>
+ <slf4j.log4j12.version>1.7.5</slf4j.log4j12.version>
</properties>
<dependencies>
@@ -55,10 +59,28 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>2.4</version>
- <scope>test</scope>
+ <version>${commons.io.version}</version>
+ </dependency>
+
+ <!-- Logging dependencies -->
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.api.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>${slf4j.log4j12.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <version>${log4j.version}</version>
</dependency>
-
</dependencies>
<build>
Modified:
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/AbstractResult.java
URL:
http://svn.apache.org/viewvc/labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/AbstractResult.java?rev=1490166&r1=1490165&r2=1490166&view=diff
==============================================================================
---
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/AbstractResult.java
(original)
+++
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/AbstractResult.java
Thu Jun 6 06:04:58 2013
@@ -75,4 +75,32 @@ import java.util.List;
{
copiedPage.add( page );
}
+
+
+ public String toString()
+ {
+ StringBuilder sb = new StringBuilder();
+
+ sb.append( "\n copiedPage = <" );
+
+ boolean isFirst = true;
+
+ for ( Page<K, V> copiedPage : getCopiedPages() )
+ {
+ if ( isFirst )
+ {
+ isFirst = false;
+ }
+ else
+ {
+ sb.append( ", " );
+ }
+
+ sb.append( copiedPage.getOffset() );
+ }
+
+ sb.append( ">" );
+
+ return sb.toString();
+ }
}
Modified:
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/ModifyResult.java
URL:
http://svn.apache.org/viewvc/labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/ModifyResult.java?rev=1490166&r1=1490165&r2=1490166&view=diff
==============================================================================
---
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/ModifyResult.java
(original)
+++
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/ModifyResult.java
Thu Jun 6 06:04:58 2013
@@ -97,6 +97,7 @@ import java.util.List;
sb.append( "ModifyResult, old value = " ).append( modifiedValue );
sb.append( ", modifiedPage = " ).append( modifiedPage );
+ sb.append( super.toString() );
return sb.toString();
}
Modified:
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/RecordManager.java
URL:
http://svn.apache.org/viewvc/labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/RecordManager.java?rev=1490166&r1=1490165&r2=1490166&view=diff
==============================================================================
---
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/RecordManager.java
(original)
+++
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/RecordManager.java
Thu Jun 6 06:04:58 2013
@@ -31,6 +31,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.mavibot.btree.exception.BTreeAlreadyManagedException;
import org.apache.mavibot.btree.exception.EndOfFileExceededException;
@@ -72,6 +73,14 @@ public class RecordManager
private long firstFreePage;
private long lastFreePage;
+ /** The list of available free pages */
+ List<PageIO> freePages = new ArrayList<PageIO>();
+
+ /** A counter to track the number of free pages */
+ public AtomicLong nbFreedPages = new AtomicLong( 0 );
+ public AtomicLong nbCreatedPages = new AtomicLong( 0 );
+ public AtomicLong nbReusedPages = new AtomicLong( 0 );
+
/** The offset of the end of the file */
private long endOfFileOffset;
@@ -130,18 +139,21 @@ public class RecordManager
/** A deserializer for Offsets */
private static final LongSerializer OFFSET_SERIALIZER = new
LongSerializer();
-
private static final String REVISION_BTREE_NAME = "_revisionBTree_";
private static final String COPIED_PAGE_BTREE_NAME = "_copiedPageBTree_";
-
+
private static final String OFFSET_BTREE_NAME = "_offsetBTree_";
-
+
private BTree<String, Long> offsetBTree;
-
+
/** A flag set to true if we want to keep old revisions */
private boolean keepRevisions;
-
+
+ /** A global buffer used for free pages */
+ private static ByteBuffer FREE_PAGE_BUFFER;
+
+
/**
* Create a Record manager which will either create the underlying file
* or load an existing one. If a folder is provided, then we will create
@@ -154,6 +166,7 @@ public class RecordManager
this( fileName, DEFAULT_PAGE_SIZE );
}
+
/**
* Create a Record manager which will either create the underlying file
* or load an existing one. If a folder is provider, then we will create
@@ -165,7 +178,9 @@ public class RecordManager
public RecordManager( String fileName, int pageSize )
{
managedBTrees = new LinkedHashMap<String, BTree<?, ?>>();
-
+
+ FREE_PAGE_BUFFER = ByteBuffer.allocateDirect( pageSize );
+
// Open the file or create it
File tmpFile = new File( fileName );
boolean isNewFile = false;
@@ -275,7 +290,8 @@ public class RecordManager
endOfFileOffset = fileChannel.size();
// Now, initialize the Copied Page BTree
- copiedPageBTree = new BTree<RevisionName, long[]>(
COPIED_PAGE_BTREE_NAME, new RevisionNameSerializer(), new LongArraySerializer()
);
+ copiedPageBTree = new BTree<RevisionName, long[]>(
COPIED_PAGE_BTREE_NAME, new RevisionNameSerializer(),
+ new LongArraySerializer() );
// and initialize the Revision BTree
revisionBTree = new BTree<RevisionName, Long>( REVISION_BTREE_NAME,
new RevisionNameSerializer(),
@@ -283,7 +299,7 @@ public class RecordManager
offsetBTree = new BTree<String, Long>( OFFSET_BTREE_NAME, new
StringSerializer(),
new LongSerializer() );
-
+
// Inject these BTrees into the RecordManager
try
{
@@ -355,29 +371,29 @@ public class RecordManager
nextBtreeOffset = revisionBTree.getNextBTreeOffset();
pageIos = readPages( nextBtreeOffset, Long.MAX_VALUE );
-
+
offsetBTree = BTreeFactory.createBTree();
offsetBTree.setBtreeOffset( nextBtreeOffset );
loadBTree( pageIos, offsetBTree );
-
+
// Then process the next ones
-// for ( int i = 2; i < nbBtree; i++ )
-// {
-// // Create the BTree
-// BTree<?, ?> btree = BTreeFactory.createBTree();
-// btree.setBtreeOffset( nextBtreeOffset );
-//
-// // Read the associated pages
-// pageIos = readPages( nextBtreeOffset, Long.MAX_VALUE );
-//
-// // Load the BTree
-// loadBTree( pageIos, btree );
-// nextBtreeOffset = btree.getNextBTreeOffset();
-//
-// // Store it into the managedBtrees map
-// managedBTrees.put( btree.getName(), btree );
-// }
+ // for ( int i = 2; i < nbBtree; i++ )
+ // {
+ // // Create the BTree
+ // BTree<?, ?> btree = BTreeFactory.createBTree();
+ // btree.setBtreeOffset( nextBtreeOffset );
+ //
+ // // Read the associated pages
+ // pageIos = readPages( nextBtreeOffset,
Long.MAX_VALUE );
+ //
+ // // Load the BTree
+ // loadBTree( pageIos, btree );
+ // nextBtreeOffset = btree.getNextBTreeOffset();
+ //
+ // // Store it into the managedBtrees map
+ // managedBTrees.put( btree.getName(), btree );
+ // }
// We are done ! Let's finish with the last initilization parts
endOfFileOffset = fileChannel.size();
@@ -587,22 +603,22 @@ public class RecordManager
for ( int i = 0; i < nbElems; i++ )
{
ElementHolder valueHolder;
-
- if( btree.isAllowDuplicates() )
+
+ if ( btree.isAllowDuplicates() )
{
String value = StringSerializer.INSTANCE.deserialize(
byteBuffer );
BTree dupValueContainer = getManagedTree( value );
- valueHolder = new DuplicateKeyMemoryHolder( btree,
dupValueContainer );
+ valueHolder = new DuplicateKeyMemoryHolder( btree,
dupValueContainer );
}
else
{
Object value = btree.getValueSerializer().deserialize(
byteBuffer );
-
+
valueHolder = new MemoryHolder( btree, value );
}
BTreeFactory.setValue( ( ( Leaf ) page ), i, valueHolder );
-
+
Object key = btree.getKeySerializer().deserialize( byteBuffer
);
BTreeFactory.setKey( page, i, key );
@@ -956,7 +972,7 @@ public class RecordManager
// The allowDuplicates flag
position = store( position, ( btree.isAllowDuplicates() ? 1 : 0 ),
pageIos );
-
+
// And flush the pages to disk now
flushPages( pageIos );
flushPages( rootPageIos );
@@ -978,7 +994,7 @@ public class RecordManager
lastAddedBTreeOffset = btreeOffset;
offsetBTree.insert( name, btreeOffset, 0 );
-
+
// Last, not least, update the number of managed BTrees in the header
updateRecordManagerHeader();
}
@@ -1077,7 +1093,7 @@ public class RecordManager
}
else
{
- if( btree.isAllowDuplicates() )
+ if ( btree.isAllowDuplicates() )
{
DuplicateKeyMemoryHolder value = (
DuplicateKeyMemoryHolder ) ( ( Leaf ) page ).getValue( pos );
String duplicateContainerName = ( ( BTree )
value.getValue( btree ) ).getName();
@@ -1088,7 +1104,7 @@ public class RecordManager
ElementHolder value = ( ( Leaf ) page ).getValue( pos
);
buffer = btree.getValueSerializer().serialize(
value.getValue( btree ) );
}
-
+
serializedData.add( buffer );
dataSize += buffer.length;
}
@@ -1681,6 +1697,8 @@ public class RecordManager
{
if ( firstFreePage == NO_PAGE )
{
+ nbCreatedPages.incrementAndGet();
+
// We don't have any free page. Reclaim some new page at the end
// of the file
PageIO newPage = new PageIO( endOfFileOffset );
@@ -1693,16 +1711,34 @@ public class RecordManager
newPage.setNextPage( NO_PAGE );
newPage.setSize( 0 );
+ LOG.debug( "Created a new page at offset {}", newPage.getOffset()
);
+
return newPage;
}
else
{
- // We have some existing free page. Fetch one from there.
+ nbReusedPages.incrementAndGet();
+
+ // We have some existing free page. Fetch it from disk
PageIO pageIo = fetchPage( firstFreePage );
- // Point to the next free page
+ // Update the firstFreePage pointer
firstFreePage = pageIo.getNextPage();
+ pageIo.setNextPage( NO_PAGE );
+ pageIo.setSize( 0 );
+
+ LOG.debug( "Reused page at offset {}", pageIo.getOffset() );
+
+ // If we don't have any more free page, update the last free page
pointer too
+ if ( firstFreePage == NO_PAGE )
+ {
+ lastFreePage = NO_PAGE;
+ }
+
+ // Update the header
+ updateRecordManagerHeader();
+
return pageIo;
}
}
@@ -1834,13 +1870,13 @@ public class RecordManager
{
// TODO : we must wait for the last write to finish
- for( BTree tree : managedBTrees.values() )
+ for ( BTree tree : managedBTrees.values() )
{
tree.close();
}
-
+
managedBTrees.clear();
-
+
// Write the data
fileChannel.force( true );
@@ -1875,26 +1911,29 @@ public class RecordManager
long firstFreePage = header.getLong();
long lastFreePage = header.getLong();
- System.out.println( "RecordManager" );
- System.out.println( "-------------" );
- System.out.println( " Header " );
- System.out.println( " '" + Strings.dumpBytes( header.array() ) +
"'" );
- System.out.println( " page size : " + pageSize );
- System.out.println( " nbTree : " + nbBTree );
- System.out.println( " firstFreePage : " + firstFreePage );
- System.out.println( " lastFreePage : " + lastFreePage );
+ if ( LOG.isDebugEnabled() )
+ {
+ LOG.debug( "RecordManager" );
+ LOG.debug( "-------------" );
+ LOG.debug( " Header " );
+ LOG.debug( " '{}'", Strings.dumpBytes( header.array() ) );
+ LOG.debug( " page size : {}", pageSize );
+ LOG.debug( " nbTree : {}", nbBTree );
+ LOG.debug( " firstFreePage : {}", firstFreePage );
+ LOG.debug( " lastFreePage : {}", lastFreePage );
+ }
long position = HEADER_SIZE;
// Dump the BTrees
for ( int i = 0; i < nbBTree; i++ )
{
- System.out.println( " Btree[" + i + "]" );
+ LOG.debug( " Btree[{}]", i );
PageIO[] pageIos = readPages( position, Long.MAX_VALUE );
for ( PageIO pageIo : pageIos )
{
- System.out.println( " " + pageIo );
+ LOG.debug( " {}", pageIo );
}
}
}
@@ -1919,11 +1958,11 @@ public class RecordManager
public Set<String> getManagedTrees()
{
Set<String> btrees = new HashSet<String>( managedBTrees.keySet() );
-
+
btrees.remove( COPIED_PAGE_BTREE_NAME );
btrees.remove( REVISION_BTREE_NAME );
btrees.remove( OFFSET_BTREE_NAME );
-
+
return btrees;
}
@@ -1937,12 +1976,12 @@ public class RecordManager
*/
/* No qualifier */void storeRootPage( BTree btree, Page rootPage ) throws
IOException
{
- if( !isKeepRevisions() )
+ if ( !isKeepRevisions() )
{
return;
}
-
- if( ( btree == copiedPageBTree ) || ( btree == revisionBTree ) || (
btree == offsetBTree ) )
+
+ if ( ( btree == copiedPageBTree ) || ( btree == revisionBTree ) || (
btree == offsetBTree ) )
{
return;
}
@@ -1989,131 +2028,136 @@ public class RecordManager
*/
public BTree getManagedTree( String name )
{
- if( name.startsWith( "1.3.6.1.4.1.18060.0.4.1.2.50" ) )
+ if ( name.startsWith( "1.3.6.1.4.1.18060.0.4.1.2.50" ) )
{
- System.out.println("");
+ LOG.debug( "" );
}
-
+
BTree tree = managedBTrees.get( name );
-
- if( tree == null )
+
+ if ( tree == null )
{
try
{
- if( !offsetBTree.hasKey( name ) )
+ if ( !offsetBTree.hasKey( name ) )
{
return null;
}
-
+
long treeOffset = offsetBTree.get( name );
-
+
// Create the BTree
BTree<?, ?> btree = BTreeFactory.createBTree();
btree.setBtreeOffset( treeOffset );
-
+
// Read the associated pages
PageIO[] pageIos = readPages( treeOffset, Long.MAX_VALUE );
// Load the BTree
loadBTree( pageIos, btree );
-
+
// Store it into the managedBtrees map
managedBTrees.put( btree.getName(), btree );
-
+
return btree;
}
- catch( Exception e )
+ catch ( Exception e )
{
throw new RuntimeException( e );
}
}
-
+
return tree;
}
-
-
+
+
/**
- *
- * TODO addFreePages.
- *
- * @param btree
- * @param freePages
+ * Move a list of pages to the free page list. A logical page is
associated with on
+ * or physical PageIO, which are on the disk. We have to move all those
PagIO instance
+ * to the free list, and do the same in memory (we try to keep a reference
to a set of
+ * free pages.
+ *
+ * @param btree The BTree which were owning the pages
+ * @param pages The pages to free
+ * @throws IOException
+ * @throws EndOfFileExceededException
*/
- public void addFreePages( BTree btree, List<Page> freePages )
+ /* Package protected */void addFreePages( BTree btree, List<Page> pages )
throws EndOfFileExceededException,
+ IOException
{
- if( ( btree == copiedPageBTree ) || ( btree == revisionBTree ) || (
btree == offsetBTree ) )
+ if ( ( btree == copiedPageBTree ) || ( btree == revisionBTree ) || (
btree == offsetBTree ) )
{
return;
}
-
- if( ( freePages == null ) || freePages.isEmpty() )
+
+ if ( ( pages == null ) || pages.isEmpty() )
{
return;
}
- // if the btree doesn't keep revisions just add them to the free page
list
- if( !keepRevisions )
+ if ( !keepRevisions )
{
- int start = 0;
- PageIO tailPage = null;
+ // if the btree doesn't keep revisions, we can safely move
+ // the pages to the free page list.
+ // NOTE : potential improvement : we can update the header only
when
+ // we have processed all the logical pages.
+ for ( Page page : pages )
+ {
+ // Retrieve all the PageIO associated with this logical page
+ long pageIoOffset = page.getOffset();
+ long firstPageIoOffset = page.getOffset();
+ long lastPageIoOffset = NO_PAGE;
- ByteBuffer data = ByteBuffer.allocateDirect( pageSize );
+ // Iterate on the pageIOs
+ while ( pageIoOffset != NO_PAGE )
+ {
+ // Keep a track of the current page
+ lastPageIoOffset = pageIoOffset;
- if( firstFreePage == NO_PAGE )
- {
- tailPage = new PageIO( freePages.get( start ).getOffset() );
+ // Retrieve the first physical page from disk
+ PageIO pageIo = fetchPage( pageIoOffset );
- firstFreePage = tailPage.getOffset();
- start = 1;
- }
- else if( lastFreePage != NO_PAGE )
- {
- tailPage = new PageIO( lastFreePage );
- }
-
- tailPage.setData( data );
- tailPage.setNextPage( NO_PAGE );
- tailPage.setSize( 0 );
-
- try
- {
- flushPages( tailPage );
-
- if( freePages.size() > start )
+ // Get the next offset
+ pageIoOffset = pageIo.getNextPage();
+ }
+
+ // Update the pointers
+ if ( firstFreePage == NO_PAGE )
{
- for( ; start < freePages.size(); start++ )
- {
- long nextOffset = freePages.get( start ).getOffset();
- tailPage.setNextPage( nextOffset );
-
- PageIO next = new PageIO( nextOffset );
- data = ByteBuffer.allocateDirect( pageSize );
- next.setData( data );
- next.setNextPage( NO_PAGE );
- next.setSize( 0 );
-
- flushPages( tailPage, next );
- tailPage = next;
- }
+ firstFreePage = firstPageIoOffset;
+ lastFreePage = lastPageIoOffset;
}
-
- lastFreePage = tailPage.getOffset();
- }
- catch( IOException e )
- {
- throw new RuntimeException( e );
+ else
+ {
+ // Fetch the last free pageIO
+ PageIO previousLastFreePage = fetchPage( lastFreePage );
+
+ // update its pointer to the next free pageIo
+ previousLastFreePage.setNextPage( firstPageIoOffset );
+
+ // And flush it to disk
+ flushPages( previousLastFreePage );
+
+ // We can update the lastFreePage offset
+ lastFreePage = lastPageIoOffset;
+ }
+
+ // Last, not least, flush the header
+ updateRecordManagerHeader();
}
}
else
{
- for( Page p : freePages )
+ LOG.debug( "We should not get there" );
+
+ for ( Page p : pages )
{
addFreePage( btree, p );
}
}
}
-
+
/**
*
* TODO addFreePage.
@@ -2127,8 +2171,8 @@ public class RecordManager
{
RevisionName revision = new RevisionName( freePage.getRevision(),
btree.getName() );
long[] offsetArray = null;
-
- if( copiedPageBTree.hasKey( revision ) )
+
+ if ( copiedPageBTree.hasKey( revision ) )
{
offsetArray = copiedPageBTree.get( revision );
long[] tmp = new long[offsetArray.length + 1];
@@ -2139,18 +2183,18 @@ public class RecordManager
{
offsetArray = new long[1];
}
-
+
offsetArray[offsetArray.length - 1] = freePage.getOffset();
-
+
copiedPageBTree.insert( revision, offsetArray, 0 );
}
- catch( Exception e )
+ catch ( Exception e )
{
throw new RuntimeException( e );
}
}
-
-
+
+
/**
* @return the keepRevisions
*/
@@ -2168,7 +2212,7 @@ public class RecordManager
this.keepRevisions = keepRevisions;
}
-
+
/**
* Creates a BTree and automatically adds it to the list of managed btrees
*
@@ -2181,20 +2225,68 @@ public class RecordManager
* @throws BTreeAlreadyManagedException
*/
@SuppressWarnings("all")
- public BTree addBTree( String name, ElementSerializer<?> keySerializer,
ElementSerializer<?> valueSerializer, boolean allowDuplicates ) throws
IOException, BTreeAlreadyManagedException
+ public BTree addBTree( String name, ElementSerializer<?> keySerializer,
ElementSerializer<?> valueSerializer,
+ boolean allowDuplicates ) throws IOException,
BTreeAlreadyManagedException
{
BTreeConfiguration config = new BTreeConfiguration();
-
+
config.setName( name );
config.setKeySerializer( keySerializer );
config.setValueSerializer( valueSerializer );
config.setAllowDuplicates( allowDuplicates );
config.setType( BTreeTypeEnum.MANAGED );
-
+
BTree btree = new BTree( config );
manage( btree );
-
+
return btree;
}
+
+ public String toString()
+ {
+ StringBuilder sb = new StringBuilder();
+
+ sb.append( "RM free pages : [" );
+
+ if ( firstFreePage != NO_PAGE )
+ {
+ long current = firstFreePage;
+ boolean isFirst = true;
+
+ while ( current != NO_PAGE )
+ {
+ if ( isFirst )
+ {
+ isFirst = false;
+ }
+ else
+ {
+ sb.append( ", " );
+ }
+
+ PageIO pageIo;
+
+ try
+ {
+ pageIo = fetchPage( current );
+ sb.append( pageIo.getOffset() );
+ current = pageIo.getNextPage();
+ }
+ catch ( EndOfFileExceededException e )
+ {
+ e.printStackTrace();
+ }
+ catch ( IOException e )
+ {
+ e.printStackTrace();
+ }
+
+ }
+ }
+
+ sb.append( "]" );
+
+ return sb.toString();
+ }
}
Modified:
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/SplitResult.java
URL:
http://svn.apache.org/viewvc/labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/SplitResult.java?rev=1490166&r1=1490165&r2=1490166&view=diff
==============================================================================
---
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/SplitResult.java
(original)
+++
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/SplitResult.java
Thu Jun 6 06:04:58 2013
@@ -112,6 +112,7 @@ import java.util.List;
sb.append( "SplitResult, new pivot = " ).append( pivot );
sb.append( "\n leftPage = " ).append( leftPage );
sb.append( "\n rightPage = " ).append( rightPage );
+ sb.append( super.toString() );
return sb.toString();
}
Modified:
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/serializer/StringSerializer.java
URL:
http://svn.apache.org/viewvc/labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/serializer/StringSerializer.java?rev=1490166&r1=1490165&r2=1490166&view=diff
==============================================================================
---
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/serializer/StringSerializer.java
(original)
+++
labs/mavibot/trunk/mavibot/src/main/java/org/apache/mavibot/btree/serializer/StringSerializer.java
Thu Jun 6 06:04:58 2013
@@ -177,24 +177,39 @@ public class StringSerializer extends Ab
break;
default:
- try
- {
- byte[] strBytes = element.getBytes( "UTF-8" );
-
- bytes = new byte[strBytes.length + 4];
+ char[] chars = element.toCharArray();
+ byte[] tmpBytes = new byte[chars.length * 2];
- System.arraycopy( strBytes, 0, bytes, 4, strBytes.length );
+ int pos = 0;
+ len = 0;
- bytes[0] = ( byte ) ( strBytes.length >>> 24 );
- bytes[1] = ( byte ) ( strBytes.length >>> 16 );
- bytes[2] = ( byte ) ( strBytes.length >>> 8 );
- bytes[3] = ( byte ) ( strBytes.length );
- }
- catch ( UnsupportedEncodingException uee )
+ for ( char c : chars )
{
- // if this happens something is really strange
- throw new RuntimeException( uee );
+ if ( ( c & 0xFF80 ) == 0 )
+ {
+ tmpBytes[pos++] = ( byte ) c;
+ }
+ else if ( ( c & 0xF800 ) == 0 )
+ {
+ tmpBytes[pos++] = ( byte ) ( ( byte ) 0x00C0 | ( byte
) ( ( c & 0x07C0 ) >> 6 ) );
+ tmpBytes[pos++] = ( byte ) ( ( byte ) 0x80 | ( byte )
( c & 0x003F ) );
+ }
+ else
+ {
+ tmpBytes[pos++] = ( byte ) ( ( byte ) 0x80 | ( byte )
( c & 0x001F ) );
+ tmpBytes[pos++] = ( byte ) ( ( byte ) 0x80 | ( byte )
( c & 0x07C0 ) );
+ tmpBytes[pos++] = ( byte ) ( ( byte ) 0xE0 | ( byte )
( c & 0x7800 ) );
+ }
}
+
+ bytes = new byte[pos + 4];
+
+ bytes[0] = ( byte ) ( pos >>> 24 );
+ bytes[1] = ( byte ) ( pos >>> 16 );
+ bytes[2] = ( byte ) ( pos >>> 8 );
+ bytes[3] = ( byte ) ( pos );
+
+ System.arraycopy( tmpBytes, 0, bytes, 4, pos );
}
return bytes;
@@ -246,8 +261,39 @@ public class StringSerializer extends Ab
byte[] bytes = new byte[len];
buffer.get( bytes );
+ char[] chars = new char[len];
+ int clen = 0;
+
+ for ( int i = 0; i < len; i++ )
+ {
+ byte b = bytes[i];
+
+ if ( b >= 0 )
+ {
+ chars[clen++] = ( char ) b;
+ }
+ else
+ {
+ if ( ( b & 0xE0 ) == 0 )
+ {
+ // 3 bytes long char
+ i++;
+ byte b2 = bytes[i];
+ i++;
+ byte b3 = bytes[i];
+ chars[clen++] = ( char ) ( ( ( b & 0x000F ) << 12
) | ( ( b2 & 0x003F ) << 6 ) | ( ( b3 & 0x003F ) ) );
+ }
+ else
+ {
+ // 2 bytes long char
+ i++;
+ byte b2 = bytes[i];
+ chars[clen++] = ( char ) ( ( ( b & 0x001F ) << 6 )
| ( b2 & 0x003F ) );
+ }
+ }
+ }
- return Strings.utf8ToString( bytes );
+ return new String( chars, 0, clen );
}
}
Added:
labs/mavibot/trunk/mavibot/src/test/java/org/apache/mavibot/btree/InMemoryBTreeTestOps.java
URL:
http://svn.apache.org/viewvc/labs/mavibot/trunk/mavibot/src/test/java/org/apache/mavibot/btree/InMemoryBTreeTestOps.java?rev=1490166&view=auto
==============================================================================
---
labs/mavibot/trunk/mavibot/src/test/java/org/apache/mavibot/btree/InMemoryBTreeTestOps.java
(added)
+++
labs/mavibot/trunk/mavibot/src/test/java/org/apache/mavibot/btree/InMemoryBTreeTestOps.java
Thu Jun 6 06:04:58 2013
@@ -0,0 +1,114 @@
+package org.apache.mavibot.btree;
+
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.mavibot.btree.serializer.LongSerializer;
+import org.apache.mavibot.btree.serializer.StringSerializer;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+
+/**
+ * A class to test multi-threaded operations on the btree
+ *
+ * @author <a href="mailto:[email protected]";>Mavibot labs Project</a>
+ */
+public class InMemoryBTreeTestOps
+{
+ /** The btree we use */
+ private static BTree<Long, String> btree;
+
+
+ /**
+ * Create the btree once
+ * @throws IOException If the creation failed
+ */
+ @BeforeClass
+ public static void setup() throws IOException
+ {
+ btree = new BTree<Long, String>( "test", new LongSerializer(), new
StringSerializer() );
+ }
+
+
+ /**
+ * Close the btree
+ */
+ @AfterClass
+ public static void shutdown() throws IOException
+ {
+ btree.close();
+ }
+
+
+ /**
+ * Create a btree with 500 000 elements in it
+ * @throws IOException If the creation failed
+ */
+ private void createTree() throws IOException
+ {
+ Random random = new Random( System.nanoTime() );
+
+ int nbElems = 500000;
+
+ // Create a BTree with 500 000 entries
+ btree.setPageSize( 32 );
+ for ( int i = 0; i < nbElems; i++ )
+ {
+ Long key = ( long ) random.nextLong();
+ String value = Long.toString( key );
+
+ try
+ {
+ btree.insert( key, value );
+
+ if ( i % 100000 == 0 )
+ {
+ System.out.println( "Written " + i + " elements" );
+ }
+ }
+ catch ( Exception e )
+ {
+ e.printStackTrace();
+ System.out.println( btree );
+ System.out.println( "Error while adding " + value );
+ return;
+ }
+ }
+
+ }
+
+
+ @Test
+ public void testCreateTree() throws InterruptedException, IOException
+ {
+
+ long t0 = System.currentTimeMillis();
+
+ // Start the writer
+ createTree();
+ long t1 = System.currentTimeMillis();
+ System.out.println( "Time to create a tree with 500 in memory:" + ( (
t1 - t0 ) ) + " Mseconds" );
+ }
+}
Modified:
labs/mavibot/trunk/mavibot/src/test/java/org/apache/mavibot/btree/RecordManagerTest.java
URL:
http://svn.apache.org/viewvc/labs/mavibot/trunk/mavibot/src/test/java/org/apache/mavibot/btree/RecordManagerTest.java?rev=1490166&r1=1490165&r2=1490166&view=diff
==============================================================================
---
labs/mavibot/trunk/mavibot/src/test/java/org/apache/mavibot/btree/RecordManagerTest.java
(original)
+++
labs/mavibot/trunk/mavibot/src/test/java/org/apache/mavibot/btree/RecordManagerTest.java
Thu Jun 6 06:04:58 2013
@@ -39,7 +39,6 @@ import org.apache.mavibot.btree.exceptio
import org.apache.mavibot.btree.serializer.LongSerializer;
import org.apache.mavibot.btree.serializer.StringSerializer;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -49,7 +48,7 @@ import org.junit.rules.TemporaryFolder;
* test the RecordManager
* @author <a href="mailto:[email protected]";>Mavibot labs Project</a>
*/
-@Ignore("ignoring till RM functionality is standardized")
+//@Ignore("ignoring till RM functionality is standardized")
public class RecordManagerTest
{
private BTree<Long, String> btree = null;
@@ -66,15 +65,15 @@ public class RecordManagerTest
public void createBTree()
{
dataDir = tempFolder.newFolder( UUID.randomUUID().toString() );
-
+
openRecordManagerAndBtree();
try
{
// Create a new BTree
- btree = ( BTree<Long, String> ) recordManager1.addBTree( "test",
new LongSerializer(), new StringSerializer(), false );
+ btree = recordManager1.addBTree( "test", new LongSerializer(), new
StringSerializer(), false );
}
- catch( Exception e )
+ catch ( Exception e )
{
throw new RuntimeException( e );
}
@@ -85,21 +84,21 @@ public class RecordManagerTest
{
try
{
- if( recordManager1 != null )
+ if ( recordManager1 != null )
{
recordManager1.close();
}
-
+
// Now, try to reload the file back
recordManager1 = new RecordManager( dataDir.getAbsolutePath() );
-
+
// load the last created btree
- if( btree != null )
+ if ( btree != null )
{
btree = recordManager1.getManagedTree( btree.getName() );
}
}
- catch( Exception e )
+ catch ( Exception e )
{
throw new RuntimeException( e );
}
@@ -224,19 +223,51 @@ public class RecordManagerTest
* Test the creation of a RecordManager with a BTree containing 100 000
elements
*/
@Test
- @Ignore("This is a performance test")
+ //@Ignore("This is a performance test")
public void testRecordManagerWithBTreeLeafNode100K() throws IOException,
BTreeAlreadyManagedException,
KeyNotFoundException
{
+ // Don't keep any revision
+ recordManager1.setKeepRevisions( false );
+
+ String fileName = dataDir.getAbsolutePath() + "/mavibot.db";
+ File file = new File( fileName );
+ long fileSize = file.length();
+ System.out.println( "----- Size before = " + fileSize );
+
// Now, add some elements in the BTree
long t0 = System.currentTimeMillis();
- for ( long i = 0L; i < 100000L; i++ )
+ for ( Long i = 0L; i < 100000L; i++ )
{
- btree.insert( i, "V" + i );
+ String value = "V" + i;
+ btree.insert( i, value );
+
+ /*
+ if ( !recordManager1.check() )
+ {
+ System.out.println( "Failure while adding element " + i );
+ fail();
+ }
+ */
+
+ if ( i % 10000 == 0 )
+ {
+ fileSize = file.length();
+ System.out.println( "----- Size after insertion of " + i + " =
" + fileSize );
+ System.out.println( recordManager1 );
+ //System.out.println( btree );
+ }
}
long t1 = System.currentTimeMillis();
+ fileSize = file.length();
+ System.out.println( "Size after insertion of 100 000 elements : " +
fileSize );
System.out.println( "Time taken to write 100 000 elements : " + ( t1 -
t0 ) );
+ System.out.println( " Nb elem/s : " + ( ( 100000 * 1000 ) / ( t1 - t0
) ) );
+ System.out.println( "Nb created page " +
recordManager1.nbCreatedPages.get() );
+ System.out.println( "Nb allocated page " +
recordManager1.nbReusedPages.get() );
+ System.out.println( "Nb page we have freed " +
recordManager1.nbFreedPages.get() );
+ System.out.println( recordManager1 );
// Now, try to reload the file back
openRecordManagerAndBtree();
@@ -800,42 +831,43 @@ public class RecordManagerTest
int pageSize = 8;
int numKeys = 2;
String name = "duplicateTree";
-
- BTree<Long,String> dupsTree = new BTree( name, null, new
LongSerializer(), new StringSerializer(), pageSize, true );
-
+
+ BTree<Long, String> dupsTree = new BTree( name, null, new
LongSerializer(), new StringSerializer(), pageSize,
+ true );
+
recordManager1.manage( dupsTree );
-
- for( long i=0; i < numKeys; i++ )
+
+ for ( long i = 0; i < numKeys; i++ )
{
- for( int k=0; k < pageSize + 1; k++ )
+ for ( int k = 0; k < pageSize + 1; k++ )
{
dupsTree.insert( i, String.valueOf( k ) );
}
}
-
+
// Now, try to reload the file back
openRecordManagerAndBtree();
-
+
dupsTree = recordManager1.getManagedTree( name );
-
-// Cursor<Long, String> cursor1 = dupsTree.browse();
-// while( cursor1.hasNext() )
-// {
-// System.out.println( cursor1.next() );
-// }
-// cursor1.close();
-
- for( long i=0; i < numKeys; i++ )
+
+ // Cursor<Long, String> cursor1 = dupsTree.browse();
+ // while( cursor1.hasNext() )
+ // {
+ // System.out.println( cursor1.next() );
+ // }
+ // cursor1.close();
+
+ for ( long i = 0; i < numKeys; i++ )
{
BTree<String, String> values = dupsTree.getValues( i );
-// Cursor<String, String> cursor = values.browse();
-// while( cursor.hasNext() )
-// {
-// System.out.println( cursor.next() );
-// }
-// cursor.close();
-
- for( int k=0; k < pageSize + 1; k++ )
+ // Cursor<String, String> cursor = values.browse();
+ // while( cursor.hasNext() )
+ // {
+ // System.out.println( cursor.next() );
+ // }
+ // cursor.close();
+
+ for ( int k = 0; k < pageSize + 1; k++ )
{
assertTrue( values.hasKey( String.valueOf( k ) ) );
}
Added: labs/mavibot/trunk/mavibot/src/test/resources/log4j.properties
URL:
http://svn.apache.org/viewvc/labs/mavibot/trunk/mavibot/src/test/resources/log4j.properties?rev=1490166&view=auto
==============================================================================
--- labs/mavibot/trunk/mavibot/src/test/resources/log4j.properties (added)
+++ labs/mavibot/trunk/mavibot/src/test/resources/log4j.properties Thu Jun 6
06:04:58 2013
@@ -0,0 +1,32 @@
+#############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#############################################################################
+log4j.rootCategory=OFF, stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%d{HH:mm:ss}] %p
[%c-%X{Replica}] - %m%n
+#log4j.appender.stdout.layout.ConversionPattern=[%d{HH:mm:ss}] %p
[%c-%X{Replica}] %C{1}.%M@%L - %m%n
+
+log4j.appender.file=org.apache.log4j.RollingFileAppender
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=[%d{HH:mm:ss}] %p [%c] - %m%n
+log4j.appender.file.File=/tmp/server-integ.log
+log4j.appender.file.MaxFileSize=7168KB
+log4j.appender.file.MaxBackupIndex=100
+
+#log4j.logger.org=FATAL
+log4j.logger.org.apache.mavibot.btree=ERROR
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
