Author: nextgens Date: 2007-08-17 14:14:29 +0000 (Fri, 17 Aug 2007) New Revision: 14749
Modified: trunk/contrib/bdb/README trunk/contrib/bdb/build.properties trunk/contrib/bdb/build.xml trunk/contrib/bdb/dist/build.properties trunk/contrib/bdb/example.properties trunk/contrib/bdb/src/com/sleepycat/bind/serial/SerialBinding.java trunk/contrib/bdb/src/com/sleepycat/bind/tuple/TupleTupleMarshalledKeyCreator.java trunk/contrib/bdb/src/com/sleepycat/collections/CurrentTransaction.java trunk/contrib/bdb/src/com/sleepycat/collections/TransactionRunner.java trunk/contrib/bdb/src/com/sleepycat/je/Cursor.java trunk/contrib/bdb/src/com/sleepycat/je/Database.java trunk/contrib/bdb/src/com/sleepycat/je/Environment.java trunk/contrib/bdb/src/com/sleepycat/je/EnvironmentStats.java trunk/contrib/bdb/src/com/sleepycat/je/JEVersion.java trunk/contrib/bdb/src/com/sleepycat/je/SecondaryCursor.java trunk/contrib/bdb/src/com/sleepycat/je/cleaner/Cleaner.java trunk/contrib/bdb/src/com/sleepycat/je/cleaner/FileProcessor.java trunk/contrib/bdb/src/com/sleepycat/je/cleaner/FileSelector.java trunk/contrib/bdb/src/com/sleepycat/je/cleaner/TrackedFileSummary.java trunk/contrib/bdb/src/com/sleepycat/je/cleaner/UtilizationProfile.java trunk/contrib/bdb/src/com/sleepycat/je/config/BooleanConfigParam.java trunk/contrib/bdb/src/com/sleepycat/je/config/EnvironmentParams.java trunk/contrib/bdb/src/com/sleepycat/je/dbi/CursorImpl.java trunk/contrib/bdb/src/com/sleepycat/je/dbi/DatabaseImpl.java trunk/contrib/bdb/src/com/sleepycat/je/dbi/DbTree.java trunk/contrib/bdb/src/com/sleepycat/je/dbi/EnvironmentImpl.java trunk/contrib/bdb/src/com/sleepycat/je/dbi/GetMode.java trunk/contrib/bdb/src/com/sleepycat/je/dbi/MemoryBudget.java trunk/contrib/bdb/src/com/sleepycat/je/dbi/SortedLSNTreeWalker.java trunk/contrib/bdb/src/com/sleepycat/je/evictor/Evictor.java trunk/contrib/bdb/src/com/sleepycat/je/incomp/INCompressor.java trunk/contrib/bdb/src/com/sleepycat/je/jca/ra/JEConnection.java trunk/contrib/bdb/src/com/sleepycat/je/jca/ra/JEManagedConnection.java trunk/contrib/bdb/src/com/sleepycat/je/jca/ra/JEManagedConnectionFactory.java trunk/contrib/bdb/src/com/sleepycat/je/jca/ra/ra.xml trunk/contrib/bdb/src/com/sleepycat/je/log/FileReader.java trunk/contrib/bdb/src/com/sleepycat/je/log/INFileReader.java trunk/contrib/bdb/src/com/sleepycat/je/log/LatchedLogManager.java trunk/contrib/bdb/src/com/sleepycat/je/log/LogManager.java trunk/contrib/bdb/src/com/sleepycat/je/log/StatsFileReader.java trunk/contrib/bdb/src/com/sleepycat/je/log/SyncedLogManager.java trunk/contrib/bdb/src/com/sleepycat/je/recovery/Checkpointer.java trunk/contrib/bdb/src/com/sleepycat/je/recovery/RecoveryManager.java trunk/contrib/bdb/src/com/sleepycat/je/tree/BIN.java trunk/contrib/bdb/src/com/sleepycat/je/tree/BINDelta.java trunk/contrib/bdb/src/com/sleepycat/je/tree/DBIN.java trunk/contrib/bdb/src/com/sleepycat/je/tree/IN.java trunk/contrib/bdb/src/com/sleepycat/je/tree/LN.java trunk/contrib/bdb/src/com/sleepycat/je/tree/MapLN.java trunk/contrib/bdb/src/com/sleepycat/je/tree/Tree.java trunk/contrib/bdb/src/com/sleepycat/je/txn/BasicLocker.java trunk/contrib/bdb/src/com/sleepycat/je/txn/DummyLockManager.java trunk/contrib/bdb/src/com/sleepycat/je/txn/LatchedLockManager.java trunk/contrib/bdb/src/com/sleepycat/je/txn/Lock.java trunk/contrib/bdb/src/com/sleepycat/je/txn/LockInfo.java trunk/contrib/bdb/src/com/sleepycat/je/txn/LockManager.java trunk/contrib/bdb/src/com/sleepycat/je/txn/Locker.java trunk/contrib/bdb/src/com/sleepycat/je/txn/ReadCommittedLocker.java trunk/contrib/bdb/src/com/sleepycat/je/txn/SyncedLockManager.java trunk/contrib/bdb/src/com/sleepycat/je/txn/ThreadLocker.java trunk/contrib/bdb/src/com/sleepycat/je/txn/Txn.java trunk/contrib/bdb/src/com/sleepycat/je/txn/WriteLockInfo.java trunk/contrib/bdb/src/com/sleepycat/je/util/DbVerify.java trunk/contrib/bdb/src/com/sleepycat/je/utilint/VLSN.java trunk/contrib/bdb/src/com/sleepycat/persist/StoreConfig.java trunk/contrib/bdb/src/com/sleepycat/persist/impl/PersistCatalog.java trunk/contrib/bdb/src/com/sleepycat/persist/impl/PersistEntityBinding.java trunk/contrib/bdb/src/com/sleepycat/persist/impl/SimpleCatalog.java trunk/contrib/bdb/src/com/sleepycat/persist/impl/Store.java trunk/contrib/bdb/src/com/sleepycat/persist/model/AnnotationModel.java trunk/contrib/bdb/src/com/sleepycat/persist/model/EntityModel.java trunk/contrib/bdb/test/com/sleepycat/collections/test/TransactionTest.java trunk/contrib/bdb/test/com/sleepycat/je/DatabaseTest.java trunk/contrib/bdb/test/com/sleepycat/je/EnvironmentTest.java trunk/contrib/bdb/test/com/sleepycat/je/RunRecoveryFailureTest.java trunk/contrib/bdb/test/com/sleepycat/je/cleaner/CleanerTest.java trunk/contrib/bdb/test/com/sleepycat/je/cleaner/TruncateAndRemoveTest.java trunk/contrib/bdb/test/com/sleepycat/je/dbi/DbCursorDuplicateTest.java trunk/contrib/bdb/test/com/sleepycat/je/dbi/DbCursorTestBase.java trunk/contrib/bdb/test/com/sleepycat/je/evictor/EvictActionTest.java trunk/contrib/bdb/test/com/sleepycat/je/evictor/EvictSelectionTest.java trunk/contrib/bdb/test/com/sleepycat/je/test/DeferredWriteTest.java trunk/contrib/bdb/test/com/sleepycat/je/test/SecondaryTest.java trunk/contrib/bdb/test/com/sleepycat/je/tree/MemorySizeTest.java trunk/contrib/bdb/test/com/sleepycat/je/txn/LockManagerTest.java trunk/contrib/bdb/test/com/sleepycat/je/txn/LockTest.java trunk/contrib/bdb/test/com/sleepycat/persist/test/EvolveClasses.java trunk/contrib/bdb/test/com/sleepycat/persist/test/OperationTest.java trunk/contrib/bdb/test/je.properties Log: freenet-ext: update bdb : je-3.2.23 -> je-3.2.42 (http://download.oracle.com/berkeley-db/je-3.2.42.tar.gz) Modified: trunk/contrib/bdb/README =================================================================== --- trunk/contrib/bdb/README 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/README 2007-08-17 14:14:29 UTC (rev 14749) @@ -1,5 +1,5 @@ -Oracle: Berkeley DB, Java Edition 3.2.23: April 06, 2007 +Oracle: Berkeley DB, Java Edition 3.2.42: August 08, 2007 -This is Berkeley DB, Java Edition, version 3.2.23 from +This is Berkeley DB, Java Edition, version 3.2.42 from Oracle. To view the release and installation documentation, load the distribution file docs/index.html into your web browser. \ No newline at end of file Modified: trunk/contrib/bdb/build.properties =================================================================== --- trunk/contrib/bdb/build.properties 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/build.properties 2007-08-17 14:14:29 UTC (rev 14749) @@ -8,6 +8,9 @@ # Typical j2ee jar for Sun Java Application Server #j2ee.jarfile = c:/j2ee1.4AppServer/lib/j2ee.jar +# Typical j2ee jar for OC4J +#j2ee.jarfile = <OC4J-HOME>/oc4j/j2ee/home/lib/ejb.jar:<OC4J-HOME>/oc4j/j2ee/home/lib/connector.jar:<OC4J-HOME>/oc4j/j2ee/home/lib/oc4j-internal.jar + ########################################################################## # Set example.resources to run the JCA examples. ########################################################################## @@ -15,6 +18,8 @@ #example.resources = c:/j2ee1.4AppServer/lib/appserv-rt.jar # JBOSS #example.resources = <jehome>/examples/resources/jboss +# OC4J +#example.resources = <OC4J-HOME>/je/examples/resources/oc4j/oc4j.jar ########################################################################## # Set example.jca.srcdir to run the JCA examples. @@ -23,6 +28,8 @@ #example.jca.srcdir = <jehome>/examples/jca/jboss # SJSAS #example.jca.srcdir = <jehome>/examples/jca/sjsas8_1 +# OC4J +#example.jca.srcdir = <OC4J-HOME>/je/examples/jca/oc4j ########################################################################## # Set example.jca.descriptorname to run the JCA examples. @@ -31,3 +38,5 @@ #example.jca.descriptorname = jboss.xml # SJSAS #example.jca.descriptorname = sun-ejb-jar.xml +# OC4J +#example.jca.descriptorname = orion-ejb-jar.xml Modified: trunk/contrib/bdb/build.xml =================================================================== --- trunk/contrib/bdb/build.xml 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/build.xml 2007-08-17 14:14:29 UTC (rev 14749) @@ -391,7 +391,7 @@ includes="bin/**,lib/**,dist/build.properties"/> <zip basedir="${basedir}" destfile="src.zip" - excludes="test/rpcserver/**,test/experiments/**,test/regress/*/**,test/**/MiniStress.java,test/**/AbortStress.java,dist/,**/jca/README.txt,examples/com/**,src/com/sleepycat/je/rep/**,examples/je/rep/**,test/com/sleepycat/je/rep/**" + excludes="test/rpcserver/**,test/experiments/**,test/standalone/**,test/regress/*/**,test/**/MiniStress.java,test/**/AbortStress.java,dist/,**/jca/README.txt,examples/com/**,src/com/sleepycat/je/rep/**,examples/je/rep/**,test/com/sleepycat/je/rep/**" includes="src/**,examples/**,test/**,docs/**,ant/**,regress/,build.xml,build.properties,example.properties,README,LICENSE,FindBugsExclude.xml"/> <zip basedir="${basedir}" destfile="${zipfile}" Modified: trunk/contrib/bdb/dist/build.properties =================================================================== --- trunk/contrib/bdb/dist/build.properties 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/dist/build.properties 2007-08-17 14:14:29 UTC (rev 14749) @@ -1,4 +1,4 @@ -release.version=3.2.23 -release.numeric.version=3.2.23 +release.version=3.2.42 +release.numeric.version=3.2.42 release.major=3 release.minor=2 Modified: trunk/contrib/bdb/example.properties =================================================================== --- trunk/contrib/bdb/example.properties 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/example.properties 2007-08-17 14:14:29 UTC (rev 14749) @@ -297,6 +297,13 @@ # je.env.checkLeaks=true # (mutable at run time: false) +# *** Experimental and not fully tested in 3.2.x. *** +# If true, enable eviction of metadata for closed databases. +# The default for JE 3.2.x is false but will be changed to true +# in JE 3.3 and above. +# je.env.dbEviction=false +# (mutable at run time: false) + # If true, use latches instead of synchronized blocks to # implement the lock table and log write mutexes. Latches require # that threads queue to obtain the mutex in question and Modified: trunk/contrib/bdb/src/com/sleepycat/bind/serial/SerialBinding.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/bind/serial/SerialBinding.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/bind/serial/SerialBinding.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2000,2007 Oracle. All rights reserved. * - * $Id: SerialBinding.java,v 1.28.2.1 2007/02/01 14:49:38 cwl Exp $ + * $Id: SerialBinding.java,v 1.28.2.3 2007/06/13 23:01:40 mark Exp $ */ package com.sleepycat.bind.serial; @@ -76,15 +76,14 @@ /** * Returns the class loader to be used during deserialization, or null if * a default class loader should be used. The default implementation of - * this method returns null. + * this method returns + * <code>Thread.currentThread().getContextClassLoader()</code> to use the + * context class loader for the current thread. * * <p>This method may be overriden to return a dynamically determined class - * loader. For example, - * <code>Thread.currentThread().getContextClassLoader()</code> could be - * called to use the context class loader for the curren thread. Or - * <code>getBaseClass().getClassLoader()</code> could be called to use the - * class loader for the base class, assuming that a base class has been - * specified.</p> + * loader. For example, <code>getBaseClass().getClassLoader()</code> could + * be called to use the class loader for the base class, assuming that a + * base class has been specified.</p> * * <p>If this method returns null, a default class loader will be used as * determined by the <code>java.io.ObjectInputStream.resolveClass</code> @@ -92,7 +91,7 @@ */ public ClassLoader getClassLoader() { - return null; + return Thread.currentThread().getContextClassLoader(); } /** Modified: trunk/contrib/bdb/src/com/sleepycat/bind/tuple/TupleTupleMarshalledKeyCreator.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/bind/tuple/TupleTupleMarshalledKeyCreator.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/bind/tuple/TupleTupleMarshalledKeyCreator.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2000,2007 Oracle. All rights reserved. * - * $Id: TupleTupleMarshalledKeyCreator.java,v 1.27.2.1 2007/02/01 14:49:39 cwl Exp $ + * $Id: TupleTupleMarshalledKeyCreator.java,v 1.27.2.2 2007/06/01 21:32:54 mark Exp $ */ package com.sleepycat.bind.tuple; @@ -61,7 +61,6 @@ public boolean nullifyForeignKey(TupleInput dataInput, TupleOutput dataOutput) { - // XXX null primary key input below may be unexpected by the binding MarshalledTupleKeyEntity entity = (MarshalledTupleKeyEntity) binding.entryToObject(null, dataInput); if (entity.nullifyForeignKey(keyName)) { Modified: trunk/contrib/bdb/src/com/sleepycat/collections/CurrentTransaction.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/collections/CurrentTransaction.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/collections/CurrentTransaction.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,11 +3,12 @@ * * Copyright (c) 2000,2007 Oracle. All rights reserved. * - * $Id: CurrentTransaction.java,v 1.46.2.1 2007/02/01 14:49:39 cwl Exp $ + * $Id: CurrentTransaction.java,v 1.46.2.2 2007/04/12 16:13:16 mark Exp $ */ package com.sleepycat.collections; +import java.lang.ref.WeakReference; import java.util.ArrayList; import java.util.List; import java.util.WeakHashMap; @@ -71,10 +72,14 @@ */ static CurrentTransaction getInstanceInternal(Environment env) { synchronized (envMap) { - CurrentTransaction myEnv = (CurrentTransaction) envMap.get(env); + CurrentTransaction myEnv = null; + WeakReference myEnvRef = (WeakReference) envMap.get(env); + if (myEnvRef != null) { + myEnv = (CurrentTransaction) myEnvRef.get(); + } if (myEnv == null) { myEnv = new CurrentTransaction(env); - envMap.put(env, myEnv); + envMap.put(env, new WeakReference(myEnv)); } return myEnv; } Modified: trunk/contrib/bdb/src/com/sleepycat/collections/TransactionRunner.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/collections/TransactionRunner.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/collections/TransactionRunner.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2000,2007 Oracle. All rights reserved. * - * $Id: TransactionRunner.java,v 1.44.2.1 2007/02/01 14:49:40 cwl Exp $ + * $Id: TransactionRunner.java,v 1.44.2.3 2007/06/01 22:26:53 mark Exp $ */ package com.sleepycat.collections; @@ -239,9 +239,9 @@ } catch (Throwable e2) { /* - * XXX We should really throw a 3rd exception that - * wraps both e and e2, to give the user a complete - * set of error information. + * We print this stack trace so that the + * information is not lost when we throw the + * original exception. */ if (DbCompat.TRANSACTION_RUNNER_PRINT_STACK_TRACES) { e2.printStackTrace(); Modified: trunk/contrib/bdb/src/com/sleepycat/je/Cursor.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/Cursor.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/Cursor.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: Cursor.java,v 1.202.2.1 2007/02/01 14:49:41 cwl Exp $ + * $Id: Cursor.java,v 1.202.2.2 2007/06/13 21:22:17 mark Exp $ */ package com.sleepycat.je; @@ -244,12 +244,22 @@ * Javadoc for this public method is generated via * the doc templates in the doc_src directory. */ - public synchronized void close() + public void close() throws DatabaseException { + close(true /*releaseNonTxnLocks*/); + } + + /** + * @param releaseNonTxnLocks should normally be true. See + * CursorImpl.close(boolean) [#15573] + */ + synchronized void close(boolean releaseNonTxnLocks) + throws DatabaseException { + try { checkState(false); - cursorImpl.close(); + cursorImpl.close(releaseNonTxnLocks); if (dbHandle != null) { dbHandle.removeCursor(this); } Modified: trunk/contrib/bdb/src/com/sleepycat/je/Database.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/Database.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/Database.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: Database.java,v 1.216.2.1 2007/02/01 14:49:41 cwl Exp $ + * $Id: Database.java,v 1.216.2.2 2007/07/02 19:54:48 mark Exp $ */ package com.sleepycat.je; @@ -280,6 +280,7 @@ if (databaseImpl != null) { databaseImpl.removeReferringHandle(this); + envHandle.getEnvironmentImpl().releaseDb(databaseImpl); databaseImpl = null; /* @@ -1105,6 +1106,14 @@ envHandle.removeReferringHandle(this); if (databaseImpl != null) { databaseImpl.removeReferringHandle(this); + envHandle.getEnvironmentImpl().releaseDb(databaseImpl); + + /* + * Database.close may be called after an abort. By setting the + * databaseImpl field to null we ensure that close won't call + * releaseDb or endOperation. [#13415] + */ + databaseImpl = null; } } Modified: trunk/contrib/bdb/src/com/sleepycat/je/Environment.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/Environment.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/Environment.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: Environment.java,v 1.179.2.1 2007/02/01 14:49:41 cwl Exp $ + * $Id: Environment.java,v 1.179.2.2 2007/07/02 19:54:48 mark Exp $ */ package com.sleepycat.je; @@ -402,6 +402,7 @@ validateDbConfigAgainstEnv(dbConfig, databaseName); Locker locker = null; + DatabaseImpl database = null; boolean operationOk = false; boolean dbIsClosing = false; try { @@ -434,12 +435,9 @@ locker.isTransactional(); } - DatabaseImpl database = environmentImpl.getDb(locker, - databaseName, - newDb); + database = environmentImpl.getDb(locker, databaseName, newDb); boolean databaseExists = - (database == null) ? false : - ((database.isDeleted()) ? false : true); + (database != null) && !database.isDeleted(); if (databaseExists) { if (dbConfig.getAllowCreate() && @@ -452,6 +450,10 @@ newDb.initExisting(this, locker, database, dbConfig); } else { + /* Release deleted DB. [#13415] */ + environmentImpl.releaseDb(database); + database = null; + /* No database. Create if we're allowed to. */ if (dbConfig.getAllowCreate()) { @@ -493,6 +495,18 @@ locker.setHandleLockOwner(operationOk, newDb, dbIsClosing); locker.operationEnd(operationOk); } + + /* + * Normally releaseDb will be called when the DB is closed, or by + * abort if a transaction is used, or by setHandleLockOwner if a + * non-transactional locker is used. But when the open operation + * fails and the Database.databaseImpl field was not initialized, + * we must call releaseDb here. [#13415] + */ + if ((!operationOk || dbIsClosing) && + newDb.getDatabaseImpl() == null) { + environmentImpl.releaseDb(database); + } } } Modified: trunk/contrib/bdb/src/com/sleepycat/je/EnvironmentStats.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/EnvironmentStats.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/EnvironmentStats.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,12 +3,13 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: EnvironmentStats.java,v 1.43.2.1 2007/02/01 14:49:41 cwl Exp $ + * $Id: EnvironmentStats.java,v 1.43.2.3 2007/05/23 20:31:05 mark Exp $ */ package com.sleepycat.je; import java.io.Serializable; +import java.text.DecimalFormat; import com.sleepycat.je.utilint.DbLsn; @@ -241,6 +242,10 @@ private int nLogBuffers; // number of existing log buffers private long bufferBytes; // cache consumed by the log buffers, // in bytes + private long adminBytes; // part of cache used by transactions, + // log cleaning metadata, and other + // administrative structures + private long lockBytes; // part of cache used by locks /* * Log activity @@ -271,6 +276,11 @@ * chunk size controlled by je.log.iteratorReadSize is too small. */ private long nRepeatIteratorReads; + + /* + * Approximation of the total log size in bytes. + */ + private long totalLogSize; /** * Internal use only. @@ -344,6 +354,7 @@ nRepeatFaultReads = 0; nTempBufferWrites = 0; nRepeatIteratorReads = 0; + totalLogSize = 0; } /** @@ -699,6 +710,23 @@ } /** + * The number of bytes of JE cache used for holding transaction objects, + * log cleaning metadata, and other administrative structures. This is a + * subset of cacheDataBytes. + */ + public long getAdminBytes() { + return adminBytes; + } + + /** + * The number of bytes of JE cache used for holding lock objects. + * This is a subset of cacheDataBytes. + */ + public long getLockBytes() { + return lockBytes; + } + + /** * Javadoc for this public method is generated via * the doc templates in the doc_src directory. */ @@ -750,6 +778,14 @@ * Javadoc for this public method is generated via * the doc templates in the doc_src directory. */ + public long getTotalLogSize() { + return totalLogSize; + } + + /** + * Javadoc for this public method is generated via + * the doc templates in the doc_src directory. + */ public int getSplitBins() { return splitBins; } @@ -764,6 +800,20 @@ /** * Internal use only. */ + public void setAdminBytes(long adminBytes) { + this.adminBytes = adminBytes; + } + + /** + * Internal use only. + */ + public void setLockBytes(long lockBytes) { + this.lockBytes = lockBytes; + } + + /** + * Internal use only. + */ public void setNNotResident(long nNotResident) { this.nNotResident = nNotResident; } @@ -1100,6 +1150,13 @@ /** * Internal use only. */ + public void setTotalLogSize(long val) { + totalLogSize = val; + } + + /** + * Internal use only. + */ public void setSplitBins(int val) { splitBins = val; } @@ -1109,79 +1166,109 @@ * the doc templates in the doc_src directory. */ public String toString() { + DecimalFormat f = new DecimalFormat("###,###,###,###,###,###,###"); + StringBuffer sb = new StringBuffer(); - sb.append("splitBins=").append(splitBins).append('\n'); - sb.append("dbClosedBins=").append(dbClosedBins).append('\n'); - sb.append("cursorsBins=").append(cursorsBins).append('\n'); - sb.append("nonEmptyBins=").append(nonEmptyBins).append('\n'); - sb.append("processedBins=").append(processedBins).append('\n'); - sb.append("inCompQueueSize=").append(inCompQueueSize).append('\n'); + sb.append("\nCompression stats\n"); + sb.append("splitBins=").append(f.format(splitBins)).append('\n'); + sb.append("dbClosedBins=").append(f.format(dbClosedBins)).append('\n'); + sb.append("cursorsBins=").append(f.format(cursorsBins)).append('\n'); + sb.append("nonEmptyBins=").append(f.format(nonEmptyBins)).append('\n'); + sb.append("processedBins="). + append(f.format(processedBins)).append('\n'); + sb.append("inCompQueueSize="). + append(f.format(inCompQueueSize)).append('\n'); // Evictor - sb.append("nEvictPasses=").append(nEvictPasses).append('\n'); - sb.append("nNodesSelected=").append(nNodesSelected).append('\n'); - sb.append("nNodesScanned=").append(nNodesScanned).append('\n'); + sb.append("\nEviction stats\n"); + sb.append("nEvictPasses=").append(f.format(nEvictPasses)).append('\n'); + sb.append("nNodesSelected="). + append(f.format(nNodesSelected)).append('\n'); + sb.append("nNodesScanned="). + append(f.format(nNodesScanned)).append('\n'); sb.append("nNodesExplicitlyEvicted="). - append(nNodesExplicitlyEvicted).append('\n'); - sb.append("nBINsStripped=").append(nBINsStripped).append('\n'); - sb.append("requiredEvictBytes=").append(requiredEvictBytes). - append('\n'); + append(f.format(nNodesExplicitlyEvicted)).append('\n'); + sb.append("nBINsStripped="). + append(f.format(nBINsStripped)).append('\n'); + sb.append("requiredEvictBytes="). + append(f.format(requiredEvictBytes)).append('\n'); // Checkpointer - sb.append("nCheckpoints=").append(nCheckpoints).append('\n'); - sb.append("lastCheckpointId=").append(lastCheckpointId).append('\n'); - sb.append("nFullINFlush=").append(nFullINFlush).append('\n'); - sb.append("nFullBINFlush=").append(nFullBINFlush).append('\n'); - sb.append("nDeltaINFlush=").append(nDeltaINFlush).append('\n'); + sb.append("\nCheckpoint stats\n"); + sb.append("nCheckpoints=").append(f.format(nCheckpoints)).append('\n'); + sb.append("lastCheckpointId="). + append(f.format(lastCheckpointId)).append('\n'); + sb.append("nFullINFlush=").append(f.format(nFullINFlush)).append('\n'); + sb.append("nFullBINFlush="). + append(f.format(nFullBINFlush)).append('\n'); + sb.append("nDeltaINFlush="). + append(f.format(nDeltaINFlush)).append('\n'); sb.append("lastCheckpointStart="). append(DbLsn.getNoFormatString(lastCheckpointStart)).append('\n'); sb.append("lastCheckpointEnd="). append(DbLsn.getNoFormatString(lastCheckpointEnd)).append('\n'); // Cleaner - sb.append("cleanerBacklog=").append(cleanerBacklog).append('\n'); - sb.append("nCleanerRuns=").append(nCleanerRuns).append('\n'); - sb.append("nCleanerDeletions=").append(nCleanerDeletions).append('\n'); - sb.append("nINsObsolete=").append(nINsObsolete).append('\n'); - sb.append("nINsCleaned=").append(nINsCleaned).append('\n'); - sb.append("nINsDead=").append(nINsDead).append('\n'); - sb.append("nINsMigrated=").append(nINsMigrated).append('\n'); - sb.append("nLNsObsolete=").append(nLNsObsolete).append('\n'); - sb.append("nLNsCleaned=").append(nLNsCleaned).append('\n'); - sb.append("nLNsDead=").append(nLNsDead).append('\n'); - sb.append("nLNsLocked=").append(nLNsLocked).append('\n'); - sb.append("nLNsMigrated=").append(nLNsMigrated).append('\n'); - sb.append("nLNsMarked=").append(nLNsMarked).append('\n'); + sb.append("\nCleaner stats\n"); + sb.append("cleanerBacklog="). + append(f.format(cleanerBacklog)).append('\n'); + sb.append("nCleanerRuns="). + append(f.format(nCleanerRuns)).append('\n'); + sb.append("nCleanerDeletions="). + append(f.format(nCleanerDeletions)).append('\n'); + sb.append("nINsObsolete=").append(f.format(nINsObsolete)).append('\n'); + sb.append("nINsCleaned=").append(f.format(nINsCleaned)).append('\n'); + sb.append("nINsDead=").append(f.format(nINsDead)).append('\n'); + sb.append("nINsMigrated=").append(f.format(nINsMigrated)).append('\n'); + sb.append("nLNsObsolete=").append(f.format(nLNsObsolete)).append('\n'); + sb.append("nLNsCleaned=").append(f.format(nLNsCleaned)).append('\n'); + sb.append("nLNsDead=").append(f.format(nLNsDead)).append('\n'); + sb.append("nLNsLocked=").append(f.format(nLNsLocked)).append('\n'); + sb.append("nLNsMigrated=").append(f.format(nLNsMigrated)).append('\n'); + sb.append("nLNsMarked=").append(f.format(nLNsMarked)).append('\n'); sb.append("nLNQueueHits="). - append(nLNQueueHits).append('\n'); + append(f.format(nLNQueueHits)).append('\n'); sb.append("nPendingLNsProcessed="). - append(nPendingLNsProcessed).append('\n'); + append(f.format(nPendingLNsProcessed)).append('\n'); sb.append("nMarkedLNsProcessed="). - append(nMarkedLNsProcessed).append('\n'); + append(f.format(nMarkedLNsProcessed)).append('\n'); sb.append("nToBeCleanedLNsProcessed="). - append(nToBeCleanedLNsProcessed).append('\n'); + append(f.format(nToBeCleanedLNsProcessed)).append('\n'); sb.append("nClusterLNsProcessed="). - append(nClusterLNsProcessed).append('\n'); + append(f.format(nClusterLNsProcessed)).append('\n'); sb.append("nPendingLNsLocked="). - append(nPendingLNsLocked).append('\n'); + append(f.format(nPendingLNsLocked)).append('\n'); sb.append("nCleanerEntriesRead="). - append(nCleanerEntriesRead).append('\n'); + append(f.format(nCleanerEntriesRead)).append('\n'); // Cache - sb.append("nNotResident=").append(nNotResident).append('\n'); - sb.append("nCacheMiss=").append(nCacheMiss).append('\n'); - sb.append("nLogBuffers=").append(nLogBuffers).append('\n'); - sb.append("bufferBytes=").append(bufferBytes).append('\n'); - sb.append("cacheDataBytes=").append(cacheDataBytes).append('\n'); - sb.append("cacheTotalBytes=").append(getCacheTotalBytes()). - append('\n'); - sb.append("nFSyncs=").append(nFSyncs).append('\n'); - sb.append("nFSyncRequests=").append(nFSyncRequests).append('\n'); - sb.append("nFSyncTimeouts=").append(nFSyncTimeouts).append('\n'); - sb.append("nRepeatFaultReads=").append(nRepeatFaultReads).append('\n'); - sb.append("nTempBufferWrite=").append(nTempBufferWrites).append('\n'); + sb.append("\nCache stats\n"); + sb.append("nNotResident=").append(f.format(nNotResident)).append('\n'); + sb.append("nCacheMiss=").append(f.format(nCacheMiss)).append('\n'); + sb.append("nLogBuffers=").append(f.format(nLogBuffers)).append('\n'); + sb.append("bufferBytes=").append(f.format(bufferBytes)).append('\n'); + sb.append("cacheDataBytes="). + append(f.format(cacheDataBytes)).append('\n'); + sb.append("adminBytes=").append(f.format(adminBytes)).append('\n'); + sb.append("lockBytes=").append(f.format(lockBytes)).append('\n'); + sb.append("cacheTotalBytes="). + append(f.format(getCacheTotalBytes())).append('\n'); + + // Logging + sb.append("\nLogging stats\n"); + sb.append("nFSyncs=").append(f.format(nFSyncs)).append('\n'); + sb.append("nFSyncRequests="). + append(f.format(nFSyncRequests)).append('\n'); + sb.append("nFSyncTimeouts="). + append(f.format(nFSyncTimeouts)).append('\n'); + sb.append("nRepeatFaultReads="). + append(f.format(nRepeatFaultReads)).append('\n'); + sb.append("nTempBufferWrite="). + append(f.format(nTempBufferWrites)).append('\n'); sb.append("nRepeatIteratorReads="). - append(nRepeatIteratorReads).append('\n'); + append(f.format(nRepeatIteratorReads)).append('\n'); + sb.append("totalLogSize="). + append(f.format(totalLogSize)).append('\n'); return sb.toString(); } Modified: trunk/contrib/bdb/src/com/sleepycat/je/JEVersion.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/JEVersion.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/JEVersion.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: JEVersion.java,v 1.93.2.9 2007/04/04 14:27:33 cwl Exp $ + * $Id: JEVersion.java,v 1.93.2.20 2007/08/06 16:43:22 cwl Exp $ */ package com.sleepycat.je; @@ -19,7 +19,7 @@ * the doc templates in the doc_src directory. */ public static final JEVersion CURRENT_VERSION = - new JEVersion(3, 2, 23, null); + new JEVersion(3, 2, 42, null); private int majorNum; private int minorNum; Modified: trunk/contrib/bdb/src/com/sleepycat/je/SecondaryCursor.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/SecondaryCursor.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/SecondaryCursor.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: SecondaryCursor.java,v 1.35.2.1 2007/02/01 14:49:41 cwl Exp $ + * $Id: SecondaryCursor.java,v 1.35.2.2 2007/06/13 21:22:17 mark Exp $ */ package com.sleepycat.je; @@ -704,7 +704,12 @@ Locker locker = cursorImpl.getLocker(); Cursor cursor = null; try { - cursor = new Cursor(primaryDb, locker, null); + + /* + * Use Cursor constructor with DatabaseImpl parameter so that the + * non-transactional locker is used in the primary cursor. [#15573] + */ + cursor = new Cursor(primaryDb.getDatabaseImpl(), locker, null); OperationStatus status = cursor.search(pKey, data, lockMode, SearchMode.SET); if (status != OperationStatus.SUCCESS) { @@ -779,8 +784,14 @@ } return OperationStatus.SUCCESS; } finally { + + /* + * Do not release non-transactional locks when closing the primary + * cursor. They are held until all locks for this operation are + * released by the secondary cursor. [#15573] + */ if (cursor != null) { - cursor.close(); + cursor.close(false /*releaseNonTxnLocks*/); } } } Modified: trunk/contrib/bdb/src/com/sleepycat/je/cleaner/Cleaner.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/cleaner/Cleaner.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/cleaner/Cleaner.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: Cleaner.java,v 1.183.2.2 2007/03/08 22:32:53 mark Exp $ + * $Id: Cleaner.java,v 1.183.2.5 2007/07/02 19:54:48 mark Exp $ */ package com.sleepycat.je.cleaner; @@ -405,6 +405,7 @@ stat.setNPendingLNsLocked(nPendingLNsLocked); stat.setNCleanerEntriesRead(nEntriesRead); stat.setNRepeatIteratorReads(nRepeatIteratorReads); + stat.setTotalLogSize(profile.getTotalLogSize()); if (config.getClear()) { nCleanerRuns = 0; @@ -430,6 +431,13 @@ } } + /** + * For unit testing. + */ + void injectFileForCleaning(Long fileNum) { + fileSelector.putBackFileForCleaning(fileNum); + } + /** * Deletes all files that are safe-to-delete, if there are no read/only * processes and concurrent backups. @@ -632,19 +640,23 @@ DatabaseId dbId = info.getDbId(); DatabaseImpl db = dbMapTree.getDb(dbId, lockTimeout); + try { + byte[] key = info.getKey(); + byte[] dupKey = info.getDupKey(); + LN ln = info.getLN(); - byte[] key = info.getKey(); - byte[] dupKey = info.getDupKey(); - LN ln = info.getLN(); + /* Evict before processing each entry. */ + if (DO_CRITICAL_EVICTION) { + env.getEvictor(). + doCriticalEviction(true); // backgroundIO + } - /* Evict before processing each entry. */ - if (DO_CRITICAL_EVICTION) { - env.getEvictor().doCriticalEviction(true); // backgroundIO + processPendingLN + (ln, db, key, dupKey, location); + } finally { + dbMapTree.releaseDb(db); } - processPendingLN - (ln, db, key, dupKey, location); - /* Sleep if background read/write limit was exceeded. */ env.sleepAfterBackgroundIO(); } @@ -655,8 +667,12 @@ for (int i = 0; i < pendingDBs.length; i += 1) { DatabaseId dbId = pendingDBs[i]; DatabaseImpl db = dbMapTree.getDb(dbId, lockTimeout); - if (db == null || db.isDeleteFinished()) { - fileSelector.removePendingDB(dbId); + try { + if (db == null || db.isDeleteFinished()) { + fileSelector.removePendingDB(dbId); + } + } finally { + dbMapTree.releaseDb(db); } } } Modified: trunk/contrib/bdb/src/com/sleepycat/je/cleaner/FileProcessor.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/cleaner/FileProcessor.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/cleaner/FileProcessor.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: FileProcessor.java,v 1.17.2.1 2007/02/01 14:49:42 cwl Exp $ + * $Id: FileProcessor.java,v 1.17.2.6 2007/07/02 19:54:48 mark Exp $ */ package com.sleepycat.je.cleaner; @@ -25,6 +25,7 @@ import com.sleepycat.je.dbi.EnvironmentImpl; import com.sleepycat.je.dbi.MemoryBudget; import com.sleepycat.je.log.CleanerFileReader; +import com.sleepycat.je.log.LogFileNotFoundException; import com.sleepycat.je.tree.BIN; import com.sleepycat.je.tree.ChildReference; import com.sleepycat.je.tree.DIN; @@ -239,6 +240,7 @@ */ resetPerRunCounters(); boolean finished = false; + boolean fileDeleted = false; long fileNumValue = fileNum.longValue(); int runId = ++cleaner.nCleanerRuns; try { @@ -261,11 +263,28 @@ accumulatePerRunCounters(); finished = true; } - } catch (IOException IOE) { - Tracer.trace(env, "Cleaner", "doClean", "", IOE); - throw new DatabaseException(IOE); + } catch (LogFileNotFoundException e) { + + /* + * File was deleted. Although it is possible that the file was + * deleted externally it is much more likely that the file was + * deleted normally after being cleaned earlier (this was + * observed prior to JE 3.2.29), and that we are mistakedly + * processing the file repeatedly. Since the file does not + * exist, ignore the error so that the cleaner will continue. + * The tracing below will indicate that the file was deleted. + * Remove the file completely from the FileSelector and + * UtilizationProfile so that we don't repeatedly attempt to + * process it. [#15528] + */ + fileDeleted = true; + profile.removeFile(fileNum); + fileSelector.removeAllFileReferences(fileNum); + } catch (IOException e) { + Tracer.trace(env, "Cleaner", "doClean", "", e); + throw new DatabaseException(e); } finally { - if (!finished) { + if (!finished && !fileDeleted) { fileSelector.putBackFileForCleaning(fileNum); } String traceMsg = @@ -273,6 +292,7 @@ " on file 0x" + Long.toHexString(fileNumValue) + " invokedFromDaemon=" + invokedFromDaemon + " finished=" + finished + + " fileDeleted=" + fileDeleted + " nEntriesRead=" + nEntriesReadThisRun + " nINsObsolete=" + nINsObsoleteThisRun + " nINsCleaned=" + nINsCleanedThisRun + @@ -370,8 +390,13 @@ */ Set checkPendingDbSet = new HashSet(); - /* Use local caching to reduce DbTree.getDb overhead. */ + /* + * Use local caching to reduce DbTree.getDb overhead. Do not call + * releaseDb after getDb with the dbCache, since the entire dbCache + * will be released at the end of thie method. + */ Map dbCache = new HashMap(); + DbTree dbMapTree = env.getDbMapTree(); try { /* Create the file reader. */ @@ -380,7 +405,6 @@ /* Validate all entries before ever deleting a file. */ reader.setAlwaysValidateChecksum(true); - DbTree dbMapTree = env.getDbMapTree(); TreeLocation location = new TreeLocation(); int nProcessedLNs = 0; @@ -507,7 +531,6 @@ DatabaseImpl db = dbMapTree.getDb (dbId, cleaner.lockTimeout, dbCache); targetIN.setDatabase(db); - processIN(targetIN, db, logLsn, deferredWriteDbs); } else if (isRoot) { @@ -545,6 +568,9 @@ /* Subtract the overhead of this method from the budget. */ budget.updateMiscMemoryUsage(0 - adjustMem); + /* Release all cached DBs. */ + dbMapTree.releaseDbs(dbCache); + /* Allow flushing of TFS when cleaning is complete. */ if (tfs != null) { tfs.setAllowFlush(true); @@ -579,6 +605,10 @@ long logLsn = DbLsn.makeLsn (fileNum.longValue(), offset.longValue()); + /* + * Do not call releaseDb after this getDb, since the entire dbCache + * will be released later. + */ DatabaseImpl db = env.getDbMapTree().getDb (info.getDbId(), cleaner.lockTimeout, dbCache); @@ -1010,15 +1040,11 @@ long treeLsn = result.parent.getLsn(result.index); /* - * Any entry that is in the log should not have a NULL_LSN in the - * in-memory tree, even for a deferred write database. The - * in-memory tree should always have a lsn that shows the last - * on-disk version. + * The IN in the tree is a never-written IN for a DW db so the IN + * in the file is obsolete. [#15588] */ if (treeLsn == DbLsn.NULL_LSN) { - throw new DatabaseException - ("Deferred Write IN should not have a NULL_LSN " + - " logLsn=" + DbLsn.getNoFormatString(logLsn)); + return null; } int compareVal = DbLsn.compareTo(treeLsn, logLsn); @@ -1106,10 +1132,12 @@ return null; } - /* bozo, how can the root's lsn be less that the logLsn? - This may have been an artifact of when we didn't properly - propagate the logging of the rootIN up to the root - ChildReference. Remove? */ + /* + * A root LSN less than the log LSN must be an artifact of when we + * didn't properly propagate the logging of the rootIN up to the + * root ChildReference. We still do this for compatibility with + * old log versions but may be able to remove it in the future. + */ if (DbLsn.compareTo(root.getLsn(), logLsn) <= 0) { IN rootIN = (IN) root.fetchTarget(db, null); rootIN.latch(Cleaner.UPDATE_GENERATION); @@ -1171,16 +1199,6 @@ } /** - * XXX: Was this intended to override Thread.toString()? If so it no - * longer does, because we separated Thread from DaemonThread. - */ - public String toString() { - StringBuffer sb = new StringBuffer(); - sb.append("<Cleaner name=\"").append(name).append("\"/>"); - return sb.toString(); - } - - /** * A cache of LNInfo by LSN offset. Used to hold a set of LNs that are * to be processed. Keeps track of memory used, and when full (over * budget) the next offset should be queried and removed. Modified: trunk/contrib/bdb/src/com/sleepycat/je/cleaner/FileSelector.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/cleaner/FileSelector.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/cleaner/FileSelector.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: FileSelector.java,v 1.15.2.1 2007/02/01 14:49:42 cwl Exp $ + * $Id: FileSelector.java,v 1.15.2.2 2007/05/31 21:55:32 mark Exp $ */ package com.sleepycat.je.cleaner; @@ -253,6 +253,18 @@ } /** + * Removes all references to a file. + */ + synchronized void removeAllFileReferences(Long file) { + toBeCleanedFiles.remove(file); + beingCleanedFiles.remove(file); + cleanedFiles.remove(file); + checkpointedFiles.remove(file); + fullyProcessedFiles.remove(file); + safeToDeleteFiles.remove(file); + } + + /** * When file cleaning is aborted, move the file back from the being-cleaned * set to the to-be-cleaned set. */ Modified: trunk/contrib/bdb/src/com/sleepycat/je/cleaner/TrackedFileSummary.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/cleaner/TrackedFileSummary.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/cleaner/TrackedFileSummary.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: TrackedFileSummary.java,v 1.9.2.1 2007/02/01 14:49:42 cwl Exp $ + * $Id: TrackedFileSummary.java,v 1.9.2.2 2007/05/15 14:48:39 mark Exp $ */ package com.sleepycat.je.cleaner; @@ -124,10 +124,12 @@ add(other); /* - * Add the offsets. The memory budget has already been updated for the - * offsets to be added, so we only need to account for a difference - * when we merge them. + * Add the offsets and the memory used [#15505] by the other tracker. + * The memory budget has already been updated for the offsets to be + * added, so we only need to account for a possible difference of one + * segment when we merge them. */ + memSize += other.memSize; if (other.obsoleteOffsets != null) { if (obsoleteOffsets != null) { /* Merge the other offsets into our list. */ Modified: trunk/contrib/bdb/src/com/sleepycat/je/cleaner/UtilizationProfile.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/cleaner/UtilizationProfile.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/cleaner/UtilizationProfile.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: UtilizationProfile.java,v 1.52.2.2 2007/03/07 00:40:24 mark Exp $ + * $Id: UtilizationProfile.java,v 1.52.2.7 2007/07/02 19:54:48 mark Exp $ */ package com.sleepycat.je.cleaner; @@ -95,6 +95,7 @@ private SortedMap fileSummaryMap; // cached fileNum -> FileSummary private boolean cachePopulated; private boolean rmwFixEnabled; + private long totalLogSize; /** * Minimum overall utilization threshold that triggers cleaning. Is @@ -176,6 +177,29 @@ } /** + * Returns an approximation of the total log size. Used for stats. + */ + long getTotalLogSize() { + + /* Start with the size known to the profile. */ + long size = totalLogSize; + + /* + * Add sizes that are known to the tracker but are not yet in the + * profile. The FileSummary.totalSize field is the delta for new + * log entries added. Typically the last log file is only one that + * will have a delta, but previous files may also not have been added + * to the profile yet. + */ + TrackedFileSummary[] trackedFiles = tracker.getTrackedFiles(); + for (int i = 0; i < trackedFiles.length; i += 1) { + size += trackedFiles[i].totalSize; + } + + return size; + } + + /** * Returns the cheapest file to clean from the given list of files. This * method is used to select the first file to be cleaned in the batch of * to-be-cleaned files. @@ -582,10 +606,13 @@ assert cachePopulated; /* Remove from the cache. */ - if (fileSummaryMap.remove(fileNum) != null) { + FileSummary oldSummary = + (FileSummary) fileSummaryMap.remove(fileNum); + if (oldSummary != null) { MemoryBudget mb = env.getMemoryBudget(); mb.updateMiscMemoryUsage (0 - MemoryBudget.UTILIZATION_PROFILE_ENTRY); + totalLogSize -= oldSummary.totalSize; } } @@ -711,14 +738,29 @@ /* * An obsolete node may have been counted after its file was - * deleted, for example, when compressing a BIN. Do not insert - * a new profile record if no corresponding log file exists. + * deleted, for example, when compressing a BIN. Do not insert a + * new profile record if no corresponding log file exists. But if + * the file number is greater than the last known file, this is a + * new file that has been buffered but not yet flushed to disk; in + * that case we should insert a new profile record. */ - File file = new File - (env.getFileManager().getFullFileName - (fileNum, FileManager.JE_SUFFIX)); - if (!file.exists()) { - return null; + if (!fileSummaryMap.isEmpty() && + fileNum < ((Long) fileSummaryMap.lastKey()).longValue()) { + File file = new File + (env.getFileManager().getFullFileName + (fileNum, FileManager.JE_SUFFIX)); + if (!file.exists()) { + + /* + * File was deleted by the cleaner. Remove it from the + * UtilizationTracker and return. Note that a file is + * normally removed from the tracker by + * FileSummaryLN.writeToLog method when it is called via + * insertFileSummary below. [#15512] + */ + env.getLogManager().removeTrackedFile(tfs); + return null; + } } summary = new FileSummary(); @@ -732,6 +774,7 @@ FileSummary tmp = new FileSummary(); tmp.add(summary); tmp.add(tfs); + totalLogSize += tfs.totalSize; int sequence = tmp.getEntriesCounted(); /* Insert an LN with the existing and tracked summary info. */ @@ -898,6 +941,8 @@ int oldMemorySize = fileSummaryMap.size() * MemoryBudget.UTILIZATION_PROFILE_ENTRY; + totalLogSize = 0; + /* * It is possible to have an undeleted FileSummaryLN in the database * for a deleted log file if we crash after deleting a file but before @@ -959,7 +1004,9 @@ if (Arrays.binarySearch(existingFiles, fileNumLong) >= 0) { /* File exists, cache the FileSummaryLN. */ - fileSummaryMap.put(fileNumLong, ln.getBaseSummary()); + FileSummary summary = ln.getBaseSummary(); + fileSummaryMap.put(fileNumLong, summary); + totalLogSize += summary.totalSize; /* * Update old version records to the new version. A @@ -1101,6 +1148,12 @@ boolean operationOk = false; try { autoTxn = new AutoTxn(env, new TransactionConfig()); + + /* + * releaseDb is not called after this getDb or createDb because we + * want to prohibit eviction of this database until the environment + * is closed. + */ DatabaseImpl db = dbTree.getDb (autoTxn, DbTree.UTILIZATION_DB_NAME, null); if (db == null) { @@ -1256,20 +1309,21 @@ DatabaseId dbId = entry.getDbId(); DatabaseImpl db = env.getDbMapTree().getDb(dbId); - /* - * The whole database is gone, so this LN is obsolete. No need - * to worry about delete cleanup; this is just verification and - * no cleaning is done. - */ - if (db == null || db.isDeleted()) { - return true; - } - /* * Search down to the bottom most level for the parent of this LN. */ BIN bin = null; try { + + /* + * The whole database is gone, so this LN is obsolete. No need + * to worry about delete cleanup; this is just verification and + * no cleaning is done. + */ + if (db == null || db.isDeleted()) { + return true; + } + Tree tree = db.getTree(); TreeLocation location = new TreeLocation(); boolean parentFound = tree.getParentBINForChildLN @@ -1307,6 +1361,7 @@ " was found in tree."); return false; } finally { + env.getDbMapTree().releaseDb(db); if (bin != null) { bin.releaseLatch(); } Modified: trunk/contrib/bdb/src/com/sleepycat/je/config/BooleanConfigParam.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/config/BooleanConfigParam.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/config/BooleanConfigParam.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: BooleanConfigParam.java,v 1.25.2.1 2007/02/01 14:49:43 cwl Exp $ + * $Id: BooleanConfigParam.java,v 1.25.2.2 2007/06/04 17:03:30 linda Exp $ */ package com.sleepycat.je.config; @@ -20,7 +20,7 @@ * Set a boolean parameter w/default. * @param configName * @param defaultValue - * @param forReplication TODO + * @param forReplication true if param is only used for replication */ BooleanConfigParam(String configName, boolean defaultValue, Modified: trunk/contrib/bdb/src/com/sleepycat/je/config/EnvironmentParams.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/config/EnvironmentParams.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/config/EnvironmentParams.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: EnvironmentParams.java,v 1.84.2.3 2007/03/07 01:24:35 mark Exp $ + * $Id: EnvironmentParams.java,v 1.84.2.4 2007/07/02 19:54:49 mark Exp $ */ package com.sleepycat.je.config; @@ -207,6 +207,16 @@ false, // mutable false, // forReplication "# If true, use shared latches for Internal Nodes (INs).\n"); + + public static final BooleanConfigParam ENV_DB_EVICTION = + new BooleanConfigParam("je.env.dbEviction", + false, // default + false, // mutable + false, // forReplication + "# *** Experimental and not fully tested in 3.2.x. ***\n" + + "# If true, enable eviction of metadata for closed databases.\n" + + "# The default for JE 3.2.x is false but will be changed to true\n" + + "# in JE 3.3 and above."); public static final IntConfigParam ADLER32_CHUNK_SIZE = new IntConfigParam("je.adler32.chunkSize", Modified: trunk/contrib/bdb/src/com/sleepycat/je/dbi/CursorImpl.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/dbi/CursorImpl.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/dbi/CursorImpl.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: CursorImpl.java,v 1.320.2.1 2007/02/01 14:49:44 cwl Exp $ + * $Id: CursorImpl.java,v 1.320.2.3 2007/06/13 21:22:17 mark Exp $ */ package com.sleepycat.je.dbi; @@ -646,10 +646,26 @@ } /** + * Close a cursor with releaseNonTxnLocks=true. + */ + public void close() + throws DatabaseException { + + close(true /*releaseNonTxnLocks*/); + } + + /** * Close a cursor. + * + * @param releaseNonTxnLocks should normally be true to release + * non-transactional locks when a cursor is closed. However, some + * operations may wish to hold non-transactional locks if the locker is + * re-used in another cursor. For example, see + * SecondaryCursor.readPrimaryAfterGet. [#15573] + * * @throws DatabaseException if the cursor was previously closed. */ - public void close() + public void close(boolean releaseNonTxnLocks) throws DatabaseException { assert assertCursorState(false) : dumpToString(true); @@ -657,7 +673,7 @@ removeCursor(); locker.unRegisterCursor(this); - if (!retainNonTxnLocks) { + if (releaseNonTxnLocks && !retainNonTxnLocks) { locker.releaseNonTxnLocks(); } @@ -1178,10 +1194,16 @@ /* Check that data compares equal before replacing it. */ boolean keysEqual = false; + /* + * Do not use a custom duplicate comaprator here because we do + * not support replacing the data if it is different, even if + * the custom comparator considers it equal. If we were to + * support this we would have to update the key in the BIN + * slot. [#15527] + */ if (foundDataBytes != null) { - keysEqual = Key.compareKeys - (foundDataBytes, newData, - database.getDuplicateComparator()) == 0; + keysEqual = + Key.compareKeys(foundDataBytes, newData, null) == 0; } if (!keysEqual) { Modified: trunk/contrib/bdb/src/com/sleepycat/je/dbi/DatabaseImpl.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/dbi/DatabaseImpl.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/dbi/DatabaseImpl.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: DatabaseImpl.java,v 1.157.2.5 2007/03/09 17:37:09 linda Exp $ + * $Id: DatabaseImpl.java,v 1.157.2.7 2007/07/02 19:54:49 mark Exp $ */ package com.sleepycat.je.dbi; @@ -92,6 +92,7 @@ private BtreeStats stats; // most recent btree stats w/ !DB_FAST_STAT private long eofNodeId; // Logical EOF node for range locking private short deleteState; // one of four delete states. + private int useCount = 0; // If non-zero, eviction is prohibited /* * The user defined Btree and duplicate comparison functions, if specified. @@ -220,13 +221,20 @@ } /** - * Clone. For now just pass off to the super class for a field-by-field - * copy. + * Clone. For the most part, just pass off to the super class for a + * field-by-field copy. */ - public Object clone() - throws CloneNotSupportedException { + public DatabaseImpl cloneDb() + throws DatabaseException { - return super.clone(); + try { + DatabaseImpl newDb = (DatabaseImpl) clone(); + /* The cloned DB could have a non-zero use count. [#13415] */ + newDb.useCount = 0; + return newDb; + } catch (CloneNotSupportedException e) { + throw new DatabaseException(e); + } } /** @@ -436,6 +444,69 @@ } /** + * Increments the use count of this DB to prevent it from being + * evicted. Called by the DbTree.createDb/getDb methods that return a + * DatabaseImpl. Must be called while holding a lock on the MapLN. See + * isInUse. [#13415] + */ + void incrementUseCount() { + if (envImpl.getDbEviction()) { + /* Synchronize to update useCount atomically. */ + synchronized (this) { + useCount += 1; + } + } + } + + /** + * Decrements the use count of this DB, allowing it to be evicted if the + * use count reaches zero. Called via DbTree.releaseDb to release a + * DatabaseImpl that was returned by a DbTree.createDb/getDb method. See + * isInUse. [#13415] + */ + void decrementUseCount() { + if (envImpl.getDbEviction()) { + /* Synchronize to update useCount atomically. */ + synchronized (this) { + assert useCount > 0; + useCount -= 1; + } + } + } + + /** + * Returns whether this DB is in use and cannot be evicted. Called by + * MapLN.isEvictable while holding a write-lock on the MapLN and a latch on + * its parent BIN. [#13415] + * + * When isInUse returns false (while holding a write-lock on the MapLN and + * a latch on the parent BIN), it guarantees that the database object + * is not in use and cannot be acquired by another thread (via + * DbTree.createDb/getDb) until both the MapLN lock and BIN latch are + * released. This guarantee is due to the fact that DbTree.createDb/getDb + * only increment the use count while holding a read-lock on the MapLN. + * Therefore, it is safe to evict the MapLN when isInUse returns false. + * + * When isInUse returns true, it is possible that another thread may + * decrement the use count at any time, since no locking or latching is + * performed when calling DbTree.releaseDb (which calls decrementUseCount). + * Therefore, it is not guaranteed that the MapLN is in use when isInUse + * returns true. A true result means: the DB may be in use, so it is not + * safe to evict it. + */ + public boolean isInUse() { + if (envImpl.getDbEviction()) { + /* Synchronize to read the up-to-date value of useCount. */ + synchronized (this) { + return (useCount > 0); + } + } else { + /* Always prohibit eviction when je.env.dbEviction=false. */ + return true; + } + } + + /** * Flush all dirty nodes for this database to disk. */ public synchronized void sync(boolean flushLog) @@ -592,6 +663,8 @@ (snapshot.getTrackedFiles()); } finally { deleteState = DELETED; + /* releaseDb to balance getDb called by truncate/remove. */ + envImpl.releaseDb(this); } } @@ -1209,10 +1282,16 @@ in.latch(); try { int index = inEntry.index; - if (in.isEntryKnownDeleted(index) || - in.getLsn(index) != lsn) { - return null; - } + if (index < 0) { + /* Negative index signifies a DupCountLN. */ + DIN din = (DIN) in; + return din.getDupCountLN(); + } else { + if (in.isEntryKnownDeleted(index) || + in.getLsn(index) != lsn) { + return null; + } + } return in.fetchTarget(index); } finally { in.releaseLatch(); Modified: trunk/contrib/bdb/src/com/sleepycat/je/dbi/DbTree.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/dbi/DbTree.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/dbi/DbTree.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: DbTree.java,v 1.170.2.1 2007/02/01 14:49:44 cwl Exp $ + * $Id: DbTree.java,v 1.170.2.2 2007/07/02 19:54:49 mark Exp $ */ package com.sleepycat.je.dbi; @@ -12,6 +12,7 @@ import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.Map; @@ -42,7 +43,69 @@ import com.sleepycat.je.txn.Locker; /** - * Represents the DatabaseImpl Naming Tree. + * DbTree represents the database directory for this environment. DbTree is + * itself implemented through two databases. The nameDatabase maps + * databaseName-> an internal databaseId. The idDatabase maps + * databaseId->DatabaseImpl. + * + * For example, suppose we have two databases, foo and bar. We have the + * following structure: + * + * nameDatabase idDatabase + * IN IN + * | | + * BIN BIN + * +-------------+--------+ +---------------+--------+ + * . | | . | | + * NameLNs NameLN NameLN MapLNs for MapLN MapLN + * for internal key=bar key=foo internal dbs key=53 key=79 + * dbs data= data= data= data= + * dbId79 dbId53 DatabaseImpl DatabaseImpl + * | | + * Tree for foo Tree for bar + * | | + * root IN root IN + * + * Databases, Cursors, the cleaner, compressor, and other entities have + * references to DatabaseImpls. It's important that object identity is properly + * maintained, and that all constituents reference the same DatabaseImpl for + * the same db, lest they develop disparate views of the in-memory database; + * corruption would ensue. To ensure that, all entities must obtain their + * DatabaseImpl by going through the idDatabase. + * + * DDL type operations such as create, rename, remove and truncate get their + * transactional semantics by transactionally locking the NameLN appropriately. + * A read-lock on the NameLN, called a handle lock, is maintained for all DBs + * opened via the public API (openDatabase). This prevents them from being + * renamed or removed while open. + * + * However, for internal database operations, no handle lock on the NameLN is + * acacuiqred and MapLNs are locked with short-lived non-transactional Lockers. + * An entity that is trying to get a reference to the DatabaseImpl gets a short + * lived read lock just for the fetch of the MapLN. A write lock on the MapLN + * is taken when the database is created, deleted, or when the MapLN is + * evicted. (see DatabaseImpl.isInUse()) + * + * The nameDatabase operates pretty much as a regular application database in + * terms of eviction and recovery. The idDatabase requires special treatment + * for both eviction and recovery. + * + * The issues around eviction of the idDatabase center on the need to ensure + * that there are no other current references to the DatabaseImpl other than + * that held by the mapLN. The presence of a current reference would both make + * the DatabaseImpl not GC'able, and more importantly, would lead to object + * identify confusion later on. For example, if the MapLN is evicted while + * there is a current reference to its DatabaseImpl, and then refetched, there + * will be two in-memory versions of the DatabaseImpl. Since locks on the + * idDatabase are short lived, DatabaseImpl.useCount acts as a reference count + * of active current references. DatabaseImpl.useCount must be modified and + * read in conjunction with appropropriate locking on the MapLN. See + * DatabaseImpl.isInUse() for details. + * + * This reference count checking is only needed when the entire MapLN is + * evicted. It's possible to evict only the root IN of the database in + * question, since that doesn't interfere with the DatabaseImpl object + * identity. */ public class DbTree implements Loggable { @@ -151,6 +214,10 @@ /** * Create a database. * + * Increments the use count of the new DB to prevent it from being evicted. + * releaseDb should be called when the returned object is no longer used, + * to allow it to be evicted. See DatabaseImpl.isInUse. [#13415] + * * Do not evict (do not call CursorImpl.setAllowEviction(true)) during low * level DbTree operation. [#15176] * @@ -193,6 +260,8 @@ idDbLocker = new BasicLocker(envImpl); idCursor = new CursorImpl(idDatabase, idDbLocker); idCursor.putLN(newId.getBytes(), new MapLN(newDb), false); + /* Increment DB use count with lock held. */ + newDb.incrementUseCount(); operationOk = true; } catch (UnsupportedEncodingException UEE) { throw new DatabaseException(UEE); @@ -212,6 +281,7 @@ return newDb; } + /** * Called by the Tree to propagate a root change. If the tree is a data * database, we will write the MapLn that represents this db to the log. If @@ -341,9 +411,10 @@ ("Attempted to " + action + " non-existent database " + databaseName); } - result.nameCursor = new CursorImpl(nameDatabase, locker); + boolean success = false; + try { + result.nameCursor = new CursorImpl(nameDatabase, locker); - try { /* Position the cursor at the specified NameLN. */ DatabaseEntry key = new DatabaseEntry(databaseName.getBytes("UTF-8")); @@ -374,14 +445,17 @@ databaseName + "," + handleCount + " open Dbs exist"); } + success = true; } catch (UnsupportedEncodingException UEE) { - result.nameCursor.releaseBIN(); - result.nameCursor.close(); throw new DatabaseException(UEE); - } catch (DatabaseException e) { - result.nameCursor.releaseBIN(); - result.nameCursor.close(); - throw e; + } finally { + if (!success) { + releaseDb(result.dbImpl); + if (result.nameCursor != null) { + result.nameCursor.releaseBIN(); + result.nameCursor.close(); + } + } } return result; @@ -400,8 +474,8 @@ throws DatabaseException { CursorImpl nameCursor = null; + NameLockResult result = lockNameLN(locker, databaseName, "rename"); try { - NameLockResult result = lockNameLN(locker, databaseName, "rename"); nameCursor = result.nameCursor; if (nameCursor == null) { return false; @@ -422,6 +496,7 @@ } catch (UnsupportedEncodingException UEE) { throw new DatabaseException(UEE); } finally { + releaseDb(result.dbImpl); if (nameCursor != null) { nameCursor.releaseBIN(); nameCursor.close(); @@ -436,8 +511,8 @@ throws DatabaseException { CursorImpl nameCursor = null; + NameLockResult result = lockNameLN(locker, databaseName, "remove"); try { - NameLockResult result = lockNameLN(locker, databaseName, "remove"); nameCursor = result.nameCursor; if (nameCursor == null) { return; @@ -456,6 +531,9 @@ * Schedule database for final deletion during commit. This * should be the last action taken, since this will take * effect immediately for non-txnal lockers. + * + * Do not call releaseDb here on result.dbImpl, since that is + * taken care of by markDeleteAtTxnEnd. */ locker.markDeleteAtTxnEnd(result.dbImpl, true); } @@ -484,9 +562,8 @@ CursorImpl nameCursor = null; Locker idDbLocker = null; + NameLockResult result = lockNameLN(locker, databaseName, "truncate"); try { - NameLockResult result = lockNameLN(locker, databaseName, - "truncate"); nameCursor = result.nameCursor; if (nameCursor == null) { return 0; @@ -497,7 +574,8 @@ * nameLN refer to the id of the new database. */ DatabaseId newId = new DatabaseId(getNextDbId()); - DatabaseImpl newDb = (DatabaseImpl) result.dbImpl.clone(); + DatabaseImpl newDb = result.dbImpl.cloneDb(); + newDb.incrementUseCount(); newDb.setId(newId); newDb.setTree(new Tree(newDb)); @@ -538,6 +616,9 @@ /* * Marking the lockers should be the last action, since * it takes effect immediately for non-txnal lockers. + * + * Do not call releaseDb here on result.dbImpl or newDb, since + * that is taken care of by markDeleteAtTxnEnd. */ /* Schedule old database for deletion if txn commits. */ @@ -548,9 +629,6 @@ return recordCount; } - - } catch (CloneNotSupportedException CNSE) { - throw new DatabaseException(CNSE); } finally { if (nameCursor != null) { nameCursor.releaseBIN(); @@ -655,9 +733,18 @@ */ DatabaseImpl newDb; DatabaseId newId = new DatabaseId(getNextDbId()); - newDb = (DatabaseImpl) oldDatabase.clone(); + newDb = oldDatabase.cloneDb(); + newDb.incrementUseCount(); newDb.setId(newId); newDb.setTree(new Tree(newDb)); + + /* + * The non-deprecated truncate starts with an old database with a + * incremented use count because lockNameLN is called, which calls + * getDb. To normalize the situation here we must increment it, + * since we don't call lockNameLN/getDb. + */ + oldDatabase.incrementUseCount(); /* Insert the new db into id -> name map */ CursorImpl idCursor = null; @@ -694,8 +781,6 @@ nameCursor.putCurrent(dataDbt, null, null); return new TruncateResult(newDb, (int) count); - } catch (CloneNotSupportedException CNSE) { - throw new DatabaseException(CNSE); } catch (UnsupportedEncodingException UEE) { throw new DatabaseException(UEE); } finally { @@ -721,6 +806,7 @@ throws DatabaseException { try { + /* Use count is not incremented for idDatabase and nameDatabase. */ if (databaseName.equals(ID_DB_NAME)) { return idDatabase; } else if (databaseName.equals(NAME_DB_NAME)) { @@ -734,7 +820,6 @@ CursorImpl nameCursor = null; DatabaseId id = null; try { - nameCursor = new CursorImpl(nameDatabase, nameLocker); DatabaseEntry keyDbt = new DatabaseEntry(databaseName.getBytes("UTF-8")); @@ -824,6 +909,11 @@ * be specified by daemons with their own timeout configuration. public * for unit tests. * + * Increments the use count of the given DB to prevent it from being + * evicted. releaseDb should be called when the returned object is no + * longer used, to allow it to be evicted. See DatabaseImpl.isInUse. + * [#13415] + * * Do not evict (do not call CursorImpl.setAllowEviction(true)) during low * level DbTree operation. [#15176] */ @@ -866,6 +956,8 @@ idCursor.getCurrentLNAlreadyLatched(LockType.READ); assert mapLN != null; /* Should be locked. */ foundDbImpl = mapLN.getDatabase(); + /* Increment DB use count with lock held. */ + foundDbImpl.incrementUseCount(); } break; } catch (DeadlockException DE) { @@ -896,6 +988,33 @@ } } + /** + * Decrements the use count of the given DB, allowing it to be evicted if + * the use count reaches zero. Must be called to release a DatabaseImpl + * that was returned by a method in this class. See DatabaseImpl.isInUse. + * [#13415] + */ + public void releaseDb(DatabaseImpl db) { + /* Use count is not incremented for idDatabase and nameDatabase. */ + if (db != null && + db != idDatabase && + db != nameDatabase) { + db.decrementUseCount(); + } + } + + /** + * Calls releaseDb for all DBs in the given map of DatabaseId to + * DatabaseImpl. See getDb(DatabaseId, long, Map). [#13415] + */ + public void releaseDbs(Map dbCache) { + if (dbCache != null) { + for (Iterator i = dbCache.values().iterator(); i.hasNext();) { + releaseDb((DatabaseImpl) i.next()); + } + } + } + /* * We need to cache a database name in the dbImpl for later use in error * messages, when it may be unsafe to walk the mapping tree. Finding a Modified: trunk/contrib/bdb/src/com/sleepycat/je/dbi/EnvironmentImpl.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/dbi/EnvironmentImpl.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/dbi/EnvironmentImpl.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: EnvironmentImpl.java,v 1.256.2.6 2007/04/04 18:36:07 cwl Exp $ + * $Id: EnvironmentImpl.java,v 1.256.2.7 2007/07/02 19:54:49 mark Exp $ */ package com.sleepycat.je.dbi; @@ -97,6 +97,7 @@ private static boolean useSharedLatchesForINs; /* true if offset tracking should be used for deferred write dbs. */ private boolean deferredWriteTemp; + private boolean dbEviction; private MemoryBudget memoryBudget; private static int adler32ChunkSize; @@ -277,9 +278,10 @@ configManager.getBoolean(EnvironmentParams.LOG_MEMORY_ONLY); useSharedLatchesForINs = configManager.getBoolean(EnvironmentParams.ENV_SHARED_LATCHES); + dbEviction = + configManager.getBoolean(EnvironmentParams.ENV_DB_EVICTION); adler32ChunkSize = configManager.getInt(EnvironmentParams.ADLER32_CHUNK_SIZE); - exceptionListener = envConfig.getExceptionListener(); /* @@ -1343,6 +1345,13 @@ return useSharedLatchesForINs; } + /** + * Returns whether DB/MapLN eviction is enabled. + */ + boolean getDbEviction() { + return dbEviction; + } + public boolean getDeferredWriteTemp() { return deferredWriteTemp; } @@ -1355,7 +1364,13 @@ return adler32ChunkSize; } - /* DatabaseImpl access. */ + /** + * Creates a new database object given a database name. + * + * Increments the use count of the new DB to prevent it from being evicted. + * releaseDb should be called when the returned object is no longer used, + * to allow it to be evicted. See DatabaseImpl.isInUse. [#13415] + */ public DatabaseImpl createDb(Locker locker, String databaseName, DatabaseConfig dbConfig, @@ -1371,6 +1386,11 @@ /** * Get a database object given a database name. * + * Increments the use count of the given DB to prevent it from being + * evicted. releaseDb should be called when the returned object is no + * longer used, to allow it to be evicted. See DatabaseImpl.isInUse. + * [#13415] + * * @param databaseName target database. * * @return null if database doesn't exist. @@ -1383,6 +1403,16 @@ return dbMapTree.getDb(locker, databaseName, databaseHandle); } + /** + * Decrements the use count of the given DB, allowing it to be evicted if + * the use count reaches zero. Must be called to release a DatabaseImpl + * that was returned by createDb or getDb. See DatabaseImpl.isInUse. + * [#13415] + */ + public void releaseDb(DatabaseImpl db) { + dbMapTree.releaseDb(db); + } + public List getDbNames() throws DatabaseException { Modified: trunk/contrib/bdb/src/com/sleepycat/je/dbi/GetMode.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/dbi/GetMode.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/dbi/GetMode.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: GetMode.java,v 1.7.2.1 2007/02/01 14:49:44 cwl Exp $ + * $Id: GetMode.java,v 1.7.2.2 2007/08/06 16:00:06 cwl Exp $ */ package com.sleepycat.je.dbi; @@ -21,12 +21,12 @@ this.forward = forward; } - public static final GetMode NEXT = new GetMode("NEXT", true); - public static final GetMode PREV = new GetMode("PREV", false); - public static final GetMode NEXT_DUP = new GetMode("NEXT_DUP", true); - public static final GetMode PREV_DUP = new GetMode("PREV_DUP", false); - public static final GetMode NEXT_NODUP = new GetMode("NEXT_NODUP", true); - public static final GetMode PREV_NODUP = new GetMode("PREV_NODUP", false); + public static final GetMode NEXT = new GetMode("NEXT", true); + public static final GetMode PREV = new GetMode("PREV", false); + public static final GetMode NEXT_DUP = new GetMode("NEXT_DUP", true); + public static final GetMode PREV_DUP = new GetMode("PREV_DUP", false); + public static final GetMode NEXT_NODUP = new GetMode("NEXT_NODUP", true); + public static final GetMode PREV_NODUP = new GetMode("PREV_NODUP", false); public final boolean isForward() { return forward; Modified: trunk/contrib/bdb/src/com/sleepycat/je/dbi/MemoryBudget.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/dbi/MemoryBudget.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/dbi/MemoryBudget.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: MemoryBudget.java,v 1.54.2.2 2007/02/14 19:46:42 linda Exp $ + * $Id: MemoryBudget.java,v 1.54.2.6 2007/07/13 02:32:05 cwl Exp $ */ package com.sleepycat.je.dbi; @@ -131,16 +131,16 @@ private final static int KEY_OVERHEAD_64 = 24; // 24 - private final static int LOCK_OVERHEAD_32 = 32; - private final static int LOCK_OVERHEAD_64 = 56; + private final static int LOCK_OVERHEAD_32 = 24; + private final static int LOCK_OVERHEAD_64 = 48; // 25 private final static int LOCKINFO_OVERHEAD_32 = 16; private final static int LOCKINFO_OVERHEAD_64 = 32; // 37 - private final static int WRITE_LOCKINFO_OVERHEAD_32 = 32; - private final static int WRITE_LOCKINFO_OVERHEAD_64 = 40; + private final static int WRITE_LOCKINFO_OVERHEAD_32 = 24; + private final static int WRITE_LOCKINFO_OVERHEAD_64 = 32; /* * Txn memory is the size for the Txn + a hashmap entry @@ -347,8 +347,7 @@ /* * Amount of memory cached for locks. Protected by the - * LockManager.lockTableLatches[lockTableIndex]. Individual elements of - * array may be negative, but the sum will be >= 0. + * LockManager.lockTableLatches[lockTableIndex]. */ private long[] lockMemoryUsage; @@ -391,7 +390,7 @@ envImpl.addConfigObserver(this); /* Peform first time budget initialization. */ - reset(configManager); + reset(configManager, true); /* * Calculate IN and BIN overheads, which are a function of @@ -418,7 +417,7 @@ * hasn't changed, since that is expensive and may cause I/O. */ long oldLogBufferBudget = logBufferBudget; - reset(configManager); + reset(configManager, false); if (oldLogBufferBudget != logBufferBudget) { envImpl.getLogManager().resetPool(configManager); } @@ -427,7 +426,8 @@ /** * Initialize at construction time and when the cache is resized. */ - private void reset(DbConfigManager configManager) + private void reset(DbConfigManager configManager, + boolean resetLockMemoryUsage) throws DatabaseException { /* @@ -530,9 +530,11 @@ logBufferBudget = newLogBufferBudget; trackerBudget = newTrackerBudget; cacheBudget = newMaxMemory - newLogBufferBudget; - nLockTables = - configManager.getInt(EnvironmentParams.N_LOCK_TABLES); - lockMemoryUsage = new long[nLockTables]; + if (resetLockMemoryUsage) { + nLockTables = + configManager.getInt(EnvironmentParams.N_LOCK_TABLES); + lockMemoryUsage = new long[nLockTables]; + } } /** @@ -644,6 +646,12 @@ } public long getCacheMemoryUsage() { + long accLockMemoryUsage = accumulateLockUsage(); + + return treeMemoryUsage + miscMemoryUsage + accLockMemoryUsage; + } + + private long accumulateLockUsage() { long accLockMemoryUsage = 0; if (nLockTables == 1) { accLockMemoryUsage = lockMemoryUsage[0]; @@ -652,7 +660,7 @@ accLockMemoryUsage += lockMemoryUsage[i]; } } - return treeMemoryUsage + miscMemoryUsage + accLockMemoryUsage; + return accLockMemoryUsage; } /** @@ -662,6 +670,13 @@ return treeMemoryUsage; } + /** + * Used for unit testing. + */ + public long getMiscMemoryUsage() { + return miscMemoryUsage; + } + public long getLogBufferBudget() { return logBufferBudget; } @@ -717,5 +732,7 @@ void loadStats(StatsConfig config, EnvironmentStats stats) { stats.setCacheDataBytes(getCacheMemoryUsage()); + stats.setAdminBytes(miscMemoryUsage); + stats.setLockBytes(accumulateLockUsage()); } } Modified: trunk/contrib/bdb/src/com/sleepycat/je/dbi/SortedLSNTreeWalker.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/dbi/SortedLSNTreeWalker.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/dbi/SortedLSNTreeWalker.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: SortedLSNTreeWalker.java,v 1.17.2.2 2007/03/07 01:24:36 mark Exp $ + * $Id: SortedLSNTreeWalker.java,v 1.17.2.3 2007/05/01 19:27:23 mark Exp $ */ package com.sleepycat.je.dbi; @@ -209,7 +209,7 @@ MemoryBudget mb = envImpl.getMemoryBudget(); inList.latchMajor(); try { - /* consolidate the INList first. */ + /* Consolidate the INList first. */ inList.latchMinorAndDumpAddedINs(); Iterator iter = inList.iterator(); @@ -419,6 +419,8 @@ DupCountLN dcl = (DupCountLN) din.getDupCountLN(); callback.processDupCount(dcl.getDupCount()); } else { + /* Negative index signifies a DupCountLN. */ + addToLsnINMap(new Long(lsn), in, -1); Node node = fetchLSN(lsn, lnKeyEntry); callback.processLSN (lsn, LogEntryType.LOG_DUPCOUNTLN, node, @@ -481,6 +483,9 @@ */ } + /** + * @param index a negative index signifies a DupCountLN. + */ protected void addToLsnINMap(Long lsn, IN in, int index) { } Modified: trunk/contrib/bdb/src/com/sleepycat/je/evictor/Evictor.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/evictor/Evictor.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/evictor/Evictor.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: Evictor.java,v 1.86.2.4 2007/03/07 01:24:37 mark Exp $ + * $Id: Evictor.java,v 1.86.2.6 2007/07/02 19:54:50 mark Exp $ */ package com.sleepycat.je.evictor; @@ -31,10 +31,12 @@ import com.sleepycat.je.log.LogManager; import com.sleepycat.je.recovery.Checkpointer; import com.sleepycat.je.tree.BIN; +import com.sleepycat.je.tree.ChildReference; import com.sleepycat.je.tree.IN; import com.sleepycat.je.tree.Node; import com.sleepycat.je.tree.SearchResult; import com.sleepycat.je.tree.Tree; +import com.sleepycat.je.tree.WithRootLatched; import com.sleepycat.je.utilint.DaemonThread; import com.sleepycat.je.utilint.DbLsn; import com.sleepycat.je.utilint.TestHook; @@ -133,12 +135,6 @@ active = false; } - public String toString() { - StringBuffer sb = new StringBuffer(); - sb.append("<Evictor name=\"").append(name).append("\"/>"); - return sb.toString(); - } - /** * Evictor doesn't have a work queue so just throw an exception if it's * ever called. @@ -361,8 +357,13 @@ break; } else { assert evictProfile.count(target);//intentional side effect - evictBytes += evict - (inList, target, scanIter, backgroundIO, tracker); + if (target.isDbRoot()) { + evictBytes += evictRoot + (inList, target, scanIter, backgroundIO); + } else { + evictBytes += evict + (inList, target, scanIter, backgroundIO, tracker); + } } nBatchSets++; } @@ -516,15 +517,6 @@ } /* - * Don't evict the DatabaseImpl Id Mapping Tree (db 0), both - * for object identity reasons and because the id mapping tree - * should stay cached. - */ - if (db.getId().equals(DbTree.ID_DB_ID)) { - continue; - } - - /* * If this is a read only database and we have at least one * target, skip any dirty INs (recovery dirties INs even in a * read-only environment). We take at least one target so we @@ -619,6 +611,18 @@ * in a non-duplicate tree. This isn't always optimimal, but is the best * we can do considering that BINs in duplicate trees may contain a mix of * LNs and DINs. + * + * BINs in the mapping tree are also assigned the same level as user DB + * BINs. When doing by-level eviction (lruOnly=false), this seems + * counter-intuitive since we should evict user DB nodes before mapping DB + * nodes. But that does occur because mapping DB INs referencing an open + * DB are unevictable. The level is only used for selecting among + * evictable nodes. + * + * If we did NOT normalize the level for the mapping DB, then INs for + * closed evictable DBs would not be evicted until after all nodes in all + * user DBs were evicted. If there were large numbers of closed DBs, this + * would have a negative performance impact. */ public int normalizeLevel(IN in, int evictType) { @@ -632,6 +636,75 @@ } /** + * Evict this DB root node. [#13415] + * @return number of bytes evicted. + */ + private long evictRoot(final INList inList, + final IN target, + final ScanIterator scanIter, + final boolean backgroundIO) + throws DatabaseException { + + final DatabaseImpl db = target.getDatabase(); + + class RootEvictor implements WithRootLatched { + + boolean flushed = false; + long evictBytes = 0; + + public IN doWork(ChildReference root) + throws DatabaseException { + + IN rootIN = (IN) root.fetchTarget(db, null); + rootIN.latch(false); + try { + /* Re-check that all conditions still hold. */ + if (rootIN == target && + rootIN.isDbRoot() && + rootIN.isEvictable()) { + + /* Flush if dirty. */ + if (!envImpl.isReadOnly() && rootIN.getDirty()) { + long newLsn = rootIN.log + (envImpl.getLogManager(), + false, // allowDeltas + isProvisionalRequired(rootIN), + true, // proactiveMigration + backgroundIO, + null); // parent + root.setLsn(newLsn); + flushed = true; + } + + /* Take off the INList and adjust memory budget. */ + scanIter.mark(); + inList.removeLatchAlreadyHeld(rootIN); + scanIter.resetToMark(); + evictBytes = rootIN.getInMemorySize(); + + /* Evict IN. */ + root.clearTarget(); + } + } finally { + rootIN.releaseLatch(); + } + return null; + } + } + + /* Attempt to evict the DB root IN. */ + RootEvictor evictor = new RootEvictor(); + db.getTree().withRootLatchedExclusive(evictor); + + /* If the root IN was flushed, write the dirtied MapLN. */ + if (evictor.flushed) { + envImpl.getDbMapTree().modifyDbRoot(db); + } + + return evictor.evictBytes; + } + + /** * Strip or evict this node. * @return number of bytes evicted. */ Modified: trunk/contrib/bdb/src/com/sleepycat/je/incomp/INCompressor.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/incomp/INCompressor.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/incomp/INCompressor.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: INCompressor.java,v 1.125.2.2 2007/03/07 01:24:37 mark Exp $ + * $Id: INCompressor.java,v 1.125.2.5 2007/07/02 19:54:50 mark Exp $ */ package com.sleepycat.je.incomp; @@ -97,12 +97,6 @@ binRefQueueSync = new Object(); } - public String toString() { - StringBuffer sb = new StringBuffer(); - sb.append("<INCompressor name=\"").append(name).append("\"/>"); - return sb.toString(); - } - synchronized public void clearEnv() { env = null; } @@ -126,20 +120,28 @@ queueSnapshot = new ArrayList(binRefQueue.values()); } - /* Use local caching to reduce DbTree.getDb overhead. */ + /* + * Use local caching to reduce DbTree.getDb overhead. Do not call + * releaseDb after each getDb, since the entire dbCache will be + * released at the end. + */ + DbTree dbTree = env.getDbMapTree(); Map dbCache = new HashMap(); - - Iterator it = queueSnapshot.iterator(); - while (it.hasNext()) { - BINReference binRef = (BINReference) it.next(); - DatabaseImpl db = env.getDbMapTree().getDb - (binRef.getDatabaseId(), lockTimeout, dbCache); - BIN bin = searchForBIN(db, binRef); - if (bin != null) { - bin.verifyCursors(); - bin.releaseLatch(); - } - } + try { + Iterator it = queueSnapshot.iterator(); + while (it.hasNext()) { + BINReference binRef = (BINReference) it.next(); + DatabaseImpl db = dbTree.getDb + (binRef.getDatabaseId(), lockTimeout, dbCache); + BIN bin = searchForBIN(db, binRef); + if (bin != null) { + bin.verifyCursors(); + bin.releaseLatch(); + } + } + } finally { + dbTree.releaseDbs(dbCache); + } } /** @@ -462,8 +464,9 @@ env.getUtilizationProfile().countAndLogSummaries (summaries); } - + } finally { + dbTree.releaseDbs(dbCache); assert LatchSupport.countLatchesHeld() == 0; accumulatePerRunCounters(); } @@ -790,7 +793,10 @@ Map dbCache) throws DatabaseException { - /* Find the database. */ + /* + * Find the database. Do not call releaseDb after this getDb, since + * the entire dbCache will be released later. + */ binSearch.db = dbTree.getDb (binRef.getDatabaseId(), lockTimeout, dbCache); if ((binSearch.db == null) ||(binSearch.db.isDeleted())) { Modified: trunk/contrib/bdb/src/com/sleepycat/je/jca/ra/JEConnection.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/jca/ra/JEConnection.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/jca/ra/JEConnection.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: JEConnection.java,v 1.13.2.1 2007/02/01 14:49:45 cwl Exp $ + * $Id: JEConnection.java,v 1.13.2.2 2007/05/22 20:36:39 cwl Exp $ */ package com.sleepycat.je.jca.ra; @@ -24,82 +24,35 @@ * <JEHOME>/examples/jca/simple/SimpleBean.java for more information on * how to build the resource adaptor and use a JEConnection. */ -public class JEConnection { +public interface JEConnection { - private JEManagedConnection mc; - private JELocalTransaction txn; + public void setManagedConnection(JEManagedConnection mc, + JELocalTransaction lt); - public JEConnection(JEManagedConnection mc) { - this.mc = mc; - } + public JELocalTransaction getLocalTransaction(); - protected void setManagedConnection(JEManagedConnection mc, - JELocalTransaction lt) { - this.mc = mc; - if (txn == null) { - txn = lt; - } - } + public void setLocalTransaction(JELocalTransaction txn); - JELocalTransaction getLocalTransaction() { - return txn; - } - - void setLocalTransaction(JELocalTransaction txn) { - this.txn = txn; - } - public Environment getEnvironment() - throws ResourceException { + throws ResourceException; - return mc.getEnvironment(); - } - public Database openDatabase(String name, DatabaseConfig config) - throws DatabaseException { + throws DatabaseException; - return mc.openDatabase(name, config); - } - public SecondaryDatabase openSecondaryDatabase(String name, Database primaryDatabase, SecondaryConfig config) - throws DatabaseException { + throws DatabaseException; - return mc.openSecondaryDatabase(name, primaryDatabase, config); - } - public void removeDatabase(String databaseName) - throws DatabaseException { + throws DatabaseException; - mc.removeDatabase(databaseName); - } - public long truncateDatabase(String databaseName, boolean returnCount) - throws DatabaseException { + throws DatabaseException; - return mc.truncateDatabase(databaseName, returnCount); - } - public Transaction getTransaction() - throws ResourceException { + throws ResourceException; - if (txn == null) { - return null; - } - - try { - return txn.getTransaction(); - } catch (DatabaseException DE) { - ResourceException ret = new ResourceException(DE.toString()); - ret.initCause(DE); - throw ret; - } - } - public void close() - throws JEException { - - mc.close(); - } + throws JEException; } Modified: trunk/contrib/bdb/src/com/sleepycat/je/jca/ra/JEManagedConnection.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/jca/ra/JEManagedConnection.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/jca/ra/JEManagedConnection.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: JEManagedConnection.java,v 1.13.2.1 2007/02/01 14:49:45 cwl Exp $ + * $Id: JEManagedConnection.java,v 1.13.2.2 2007/05/22 20:36:39 cwl Exp $ */ package com.sleepycat.je.jca.ra; @@ -67,7 +67,7 @@ throws ResourceException { if (conn == null) { - conn = new JEConnection(this); + conn = new JEConnectionImpl(this); } return conn; } @@ -115,7 +115,7 @@ public void associateConnection(Object connection) throws ResourceException { - conn = (JEConnection) connection; + conn = (JEConnectionImpl) connection; conn.setManagedConnection(this, savedLT); savedLT = null; } Modified: trunk/contrib/bdb/src/com/sleepycat/je/jca/ra/JEManagedConnectionFactory.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/jca/ra/JEManagedConnectionFactory.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/jca/ra/JEManagedConnectionFactory.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: JEManagedConnectionFactory.java,v 1.9.2.1 2007/02/01 14:49:45 cwl Exp $ + * $Id: JEManagedConnectionFactory.java,v 1.9.2.2 2007/05/22 20:36:39 cwl Exp $ */ package com.sleepycat.je.jca.ra; @@ -26,6 +26,9 @@ public class JEManagedConnectionFactory implements ManagedConnectionFactory, Serializable { + private String userName; + private String password; + public JEManagedConnectionFactory() { } @@ -76,6 +79,22 @@ return null; } + public void setUserName(String userName) { + this.userName = userName; + } + + public String getUserName() { + return userName; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getPassword() { + return password; + } + public void setLogWriter(PrintWriter out) throws ResourceException { Modified: trunk/contrib/bdb/src/com/sleepycat/je/jca/ra/ra.xml =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/jca/ra/ra.xml 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/jca/ra/ra.xml 2007-08-17 14:14:29 UTC (rev 14749) @@ -24,7 +24,7 @@ </connectionfactory-impl-class> <connection-interface>com.sleepycat.je.jca.ra.JEConnection </connection-interface> - <connection-impl-class>com.sleepycat.je.jca.ra.JEConnection + <connection-impl-class>com.sleepycat.je.jca.ra.JEConnectionImpl </connection-impl-class> <transaction-support>LocalTransaction</transaction-support> <!-- Modified: trunk/contrib/bdb/src/com/sleepycat/je/log/FileReader.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/log/FileReader.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/log/FileReader.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: FileReader.java,v 1.99.2.3 2007/04/04 14:28:22 cwl Exp $ + * $Id: FileReader.java,v 1.99.2.4 2007/05/31 21:55:32 mark Exp $ */ package com.sleepycat.je.log; @@ -267,7 +267,8 @@ readBasicHeader(dataBuffer); if (currentEntryHeader.getReplicate()) { - dataBuffer = readData(currentEntryHeader.getVariablePortionSize(), true); + dataBuffer = readData + (currentEntryHeader.getVariablePortionSize(), true); currentEntryHeader.readVariablePortion(dataBuffer); } @@ -338,26 +339,38 @@ } catch (DatabaseException e) { eof = true; /* Report on error. */ - LogEntryType problemType = - LogEntryType.findType(currentEntryHeader.getType(), - currentEntryHeader.getVersion()); - Tracer.trace(envImpl, "FileReader", "readNextEntry", - "Halted log file reading at file 0x" + - Long.toHexString(readBufferFileNum) + - " offset 0x" + - Long.toHexString(nextEntryOffset) + - " offset(decimal)=" + nextEntryOffset + - ":\nentry="+ problemType + - "(typeNum=" + currentEntryHeader.getType() + - ",version=" + currentEntryHeader.getVersion() + - ")\nprev=0x" + - Long.toHexString(currentEntryPrevOffset) + - "\nsize=" + currentEntryHeader.getItemSize() + - "\nNext entry should be at 0x" + - Long.toHexString((nextEntryOffset + + if (currentEntryHeader != null) { + LogEntryType problemType = + LogEntryType.findType(currentEntryHeader.getType(), + currentEntryHeader.getVersion()); + Tracer.trace(envImpl, "FileReader", "readNextEntry", + "Halted log file reading at file 0x" + + Long.toHexString(readBufferFileNum) + + " offset 0x" + + Long.toHexString(nextEntryOffset) + + " offset(decimal)=" + nextEntryOffset + + ":\nentry="+ problemType + + "(typeNum=" + currentEntryHeader.getType() + + ",version=" + currentEntryHeader.getVersion() + + ")\nprev=0x" + + Long.toHexString(currentEntryPrevOffset) + + "\nsize=" + currentEntryHeader.getItemSize() + + "\nNext entry should be at 0x" + + Long.toHexString((nextEntryOffset + currentEntryHeader.getSize() + currentEntryHeader.getItemSize())) + - "\n:", e); + "\n:", e); + } else { + Tracer.trace(envImpl, "FileReader", "readNextEntry", + "Halted log file reading at file 0x" + + Long.toHexString(readBufferFileNum) + + " offset 0x" + + Long.toHexString(nextEntryOffset) + + " offset(decimal)=" + nextEntryOffset + + " prev=0x" + + Long.toHexString(currentEntryPrevOffset), + e); + } throw e; } return foundEntry; Modified: trunk/contrib/bdb/src/com/sleepycat/je/log/INFileReader.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/log/INFileReader.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/log/INFileReader.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: INFileReader.java,v 1.52.2.2 2007/03/08 22:32:54 mark Exp $ + * $Id: INFileReader.java,v 1.52.2.3 2007/08/06 16:00:20 cwl Exp $ */ package com.sleepycat.je.log; @@ -293,8 +293,8 @@ * Do partial load of db and txn id tracking entries if necessary. * Note that these entries do not overlap with targetLogEntry. * - * XXX We're doing a full load for now, since LNLogEntry does not - * read the db and txn id in a partial load, only the node id. + * We're doing a full load for now, since LNLogEntry does not read + * the db and txn id in a partial load, only the node id. */ LNLogEntry lnEntry = null; if (dbIdTrackingEntry != null) { Modified: trunk/contrib/bdb/src/com/sleepycat/je/log/LatchedLogManager.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/log/LatchedLogManager.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/log/LatchedLogManager.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: LatchedLogManager.java,v 1.17.2.1 2007/02/01 14:49:47 cwl Exp $ + * $Id: LatchedLogManager.java,v 1.17.2.2 2007/06/13 03:55:37 mark Exp $ */ package com.sleepycat.je.log; @@ -83,7 +83,21 @@ logWriteLatch.release(); } } + + /** + * @see LogManager#removeTrackedFile + */ + public void removeTrackedFile(TrackedFileSummary tfs) + throws DatabaseException { + logWriteLatch.acquire(); + try { + removeTrackedFileInternal(tfs); + } finally { + logWriteLatch.release(); + } + } + /** * @see LogManager#countObsoleteLNs */ Modified: trunk/contrib/bdb/src/com/sleepycat/je/log/LogManager.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/log/LogManager.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/log/LogManager.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: LogManager.java,v 1.163.2.3 2007/03/09 21:04:12 mark Exp $ + * $Id: LogManager.java,v 1.163.2.4 2007/06/13 03:55:37 mark Exp $ */ package com.sleepycat.je.log; @@ -821,6 +821,16 @@ } /** + * Removes the tracked summary for the given file. + */ + abstract public void removeTrackedFile(TrackedFileSummary tfs) + throws DatabaseException; + + protected void removeTrackedFileInternal(TrackedFileSummary tfs) { + tfs.reset(); + } + + /** * Count node as obsolete under the log write latch. This is done here * because the log write latch is managed here, and all utilization * counting must be performed under the log write latch. Modified: trunk/contrib/bdb/src/com/sleepycat/je/log/StatsFileReader.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/log/StatsFileReader.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/log/StatsFileReader.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: StatsFileReader.java,v 1.15.2.1 2007/02/01 14:49:47 cwl Exp $ + * $Id: StatsFileReader.java,v 1.15.2.2 2007/06/04 17:03:30 linda Exp $ */ package com.sleepycat.je.log; @@ -187,7 +187,7 @@ 8 bytes txn id 8 bytes lastlogged LSN (backpointer for txn) */ - /** BOZO -- the header size is undercounted for replication */ + int overhead = (info.count*46) + info.headerBytes; realTotalBytes += (info.totalBytes-overhead); } Modified: trunk/contrib/bdb/src/com/sleepycat/je/log/SyncedLogManager.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/log/SyncedLogManager.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/log/SyncedLogManager.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: SyncedLogManager.java,v 1.18.2.1 2007/02/01 14:49:47 cwl Exp $ + * $Id: SyncedLogManager.java,v 1.18.2.2 2007/06/13 03:55:37 mark Exp $ */ package com.sleepycat.je.log; @@ -77,7 +77,18 @@ return getUnflushableTrackedSummaryInternal(file); } } + + /** + * @see LogManager#removeTrackedFile + */ + public void removeTrackedFile(TrackedFileSummary tfs) + throws DatabaseException { + synchronized (logWriteLatch) { + removeTrackedFileInternal(tfs); + } + } + /** * @see LogManager#countObsoleteLNs */ Modified: trunk/contrib/bdb/src/com/sleepycat/je/recovery/Checkpointer.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/recovery/Checkpointer.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/recovery/Checkpointer.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: Checkpointer.java,v 1.140.2.2 2007/03/07 01:24:39 mark Exp $ + * $Id: Checkpointer.java,v 1.140.2.3 2007/06/01 21:32:56 mark Exp $ */ package com.sleepycat.je.recovery; @@ -146,12 +146,6 @@ checkpointId = lastCheckpointId; } - public String toString() { - StringBuffer sb = new StringBuffer(); - sb.append("<Checkpointer name=\"").append(name).append("\"/>"); - return sb.toString(); - } - /** * Load stats. */ Modified: trunk/contrib/bdb/src/com/sleepycat/je/recovery/RecoveryManager.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/recovery/RecoveryManager.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/recovery/RecoveryManager.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: RecoveryManager.java,v 1.211.2.3 2007/03/28 15:53:44 cwl Exp $ + * $Id: RecoveryManager.java,v 1.211.2.4 2007/07/02 19:54:51 mark Exp $ */ package com.sleepycat.je.recovery; @@ -651,8 +651,12 @@ DatabaseId dbId = reader.getDatabaseId(); if (dbId.equals(DbTree.ID_DB_ID)) { DatabaseImpl db = dbMapTree.getDb(dbId); - replayOneIN(reader, db, false, recorder); - info.numMapINs++; + try { + replayOneIN(reader, db, false, recorder); + info.numMapINs++; + } finally { + dbMapTree.releaseDb(db); + } } } @@ -731,18 +735,24 @@ } if (isTarget) { DatabaseImpl db = dbMapTree.getDb(dbId); - if (db == null) { - // This db has been deleted, ignore the entry. - } else { - replayOneIN(reader, db, requireExactMatch, recorder); - numINsSeen++; + try { + if (db == null) { + // This db has been deleted, ignore the entry. + } else { + replayOneIN(reader, db, requireExactMatch, + recorder); + numINsSeen++; - /* - * Add any db that we encounter IN's for because - * they'll be part of the in-memory tree and therefore - * should be included in the INList rebuild. - */ - inListRebuildDbIds.add(dbId); + /* + * Add any db that we encounter IN's for because + * they'll be part of the in-memory tree and + * therefore should be included in the INList + * rebuild. + */ + inListRebuildDbIds.add(dbId); + } + } finally { + dbMapTree.releaseDb(db); } } } @@ -787,13 +797,17 @@ DatabaseId dbId = reader.getDatabaseId(); if (targetDbs.contains(dbId)) { DatabaseImpl db = dbMapTree.getDb(dbId); - if (db == null) { - // This db has been deleted, ignore the entry. - } else { - replayOneIN(reader, - db, - true, // requireExactMatch, - null); // level recorder + try { + if (db == null) { + // This db has been deleted, ignore the entry. + } else { + replayOneIN(reader, + db, + true, // requireExactMatch, + null); // level recorder + } + } finally { + dbMapTree.releaseDb(db); } } } @@ -934,46 +948,49 @@ reader.getAbortKnownDeleted(); DatabaseId dbId = reader.getDatabaseId(); DatabaseImpl db = dbMapTree.getDb(dbId); - - /* Database may be null if it's been deleted. */ - if (db != null) { - ln.postFetchInit(db, logLsn); - try { - undo(detailedTraceLevel, - db, - location, - ln, - reader.getKey(), - reader.getDupTreeKey(), - logLsn, - abortLsn, - abortKnownDeleted, - info, - true); - } finally { - if (location.bin != null) { - location.bin.releaseLatchIfOwner(); - } - } - /* Undo utilization info. */ - TxnNodeId txnNodeId = - new TxnNodeId(reader.getNodeId(), - txnId.longValue()); - undoUtilizationInfo(ln, logLsn, abortLsn, - abortKnownDeleted, - reader.getLastEntrySize(), - txnNodeId, - countedFileSummaries, - countedAbortLsnNodes); + try { + /* Database may be null if it's been deleted. */ + if (db != null) { + ln.postFetchInit(db, logLsn); + try { + undo(detailedTraceLevel, + db, + location, + ln, + reader.getKey(), + reader.getDupTreeKey(), + logLsn, + abortLsn, + abortKnownDeleted, + info, + true); + } finally { + if (location.bin != null) { + location.bin.releaseLatchIfOwner(); + } + } + /* Undo utilization info. */ + TxnNodeId txnNodeId = + new TxnNodeId(reader.getNodeId(), + txnId.longValue()); + undoUtilizationInfo(ln, logLsn, abortLsn, + abortKnownDeleted, + reader.getLastEntrySize(), + txnNodeId, + countedFileSummaries, + countedAbortLsnNodes); - /* - * Add any db that we encounter LN's for because - * they'll be part of the in-memory tree and - * therefore should be included in the INList - * rebuild. - */ - inListRebuildDbIds.add(dbId); - } + /* + * Add any db that we encounter LN's for + * because they'll be part of the in-memory + * tree and therefore should be included in the + * INList rebuild. + */ + inListRebuildDbIds.add(dbId); + } + } finally { + dbMapTree.releaseDb(db); + } } } else if (reader.isPrepare()) { @@ -1075,57 +1092,61 @@ LN ln = reader.getLN(); DatabaseId dbId = reader.getDatabaseId(); DatabaseImpl db = dbMapTree.getDb(dbId); - long logLsn = reader.getLastLsn(); - long treeLsn = DbLsn.NULL_LSN; + try { + long logLsn = reader.getLastLsn(); + long treeLsn = DbLsn.NULL_LSN; - /* Database may be null if it's been deleted. */ - if (db != null) { - ln.postFetchInit(db, logLsn); + /* Database may be null if it's been deleted. */ + if (db != null) { + ln.postFetchInit(db, logLsn); - if (preparedTxn != null) { - preparedTxn.addLogInfo(logLsn); + if (preparedTxn != null) { + preparedTxn.addLogInfo(logLsn); - /* - * We're reconstructing a prepared, but not - * finished, transaction. We know that there - * was a write lock on this LN since it exists - * in the log under this txnId. - */ - preparedTxn.lock - (ln.getNodeId(), LockType.WRITE, - false /*noWait*/, db); - } + /* + * We're reconstructing a prepared, but not + * finished, transaction. We know that + * there was a write lock on this LN since + * it exists in the log under this txnId. + */ + preparedTxn.lock + (ln.getNodeId(), LockType.WRITE, + false /*noWait*/, db); + } - treeLsn = redo(db, - location, - ln, - reader.getKey(), - reader.getDupTreeKey(), - logLsn, - info); + treeLsn = redo(db, + location, + ln, + reader.getKey(), + reader.getDupTreeKey(), + logLsn, + info); - /* - * Add any db that we encounter LN's for because - * they'll be part of the in-memory tree and - * therefore should be included in the INList - * rebuild. - */ - inListRebuildDbIds.add(dbId); - } + /* + * Add any db that we encounter LN's for + * because they'll be part of the in-memory + * tree and therefore should be included in the + * INList rebuild. + */ + inListRebuildDbIds.add(dbId); + } - /* Redo utilization info. */ - TxnNodeId txnNodeId = null; - if (txnId != null) { - txnNodeId = new TxnNodeId(reader.getNodeId(), - txnId.longValue()); + /* Redo utilization info. */ + TxnNodeId txnNodeId = null; + if (txnId != null) { + txnNodeId = new TxnNodeId(reader.getNodeId(), + txnId.longValue()); + } + redoUtilizationInfo(logLsn, treeLsn, + reader.getAbortLsn(), + reader.getAbortKnownDeleted(), + reader.getLastEntrySize(), + reader.getKey(), + ln, txnNodeId, + countedAbortLsnNodes); + } finally { + dbMapTree.releaseDb(db); } - redoUtilizationInfo(logLsn, treeLsn, - reader.getAbortLsn(), - reader.getAbortKnownDeleted(), - reader.getLastEntrySize(), - reader.getKey(), - ln, txnNodeId, - countedAbortLsnNodes); } } } @@ -1152,9 +1173,13 @@ /* We already did the map tree, don't do it again. */ if (!dbId.equals(DbTree.ID_DB_ID)) { DatabaseImpl db = env.getDbMapTree().getDb(dbId); - if (db != null) { - db.getTree().rebuildINList(); - } + try { + if (db != null) { + db.getTree().rebuildINList(); + } + } finally { + env.releaseDb(db); + } } } } Modified: trunk/contrib/bdb/src/com/sleepycat/je/tree/BIN.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/tree/BIN.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/tree/BIN.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: BIN.java,v 1.188.2.3 2007/03/08 22:32:58 mark Exp $ + * $Id: BIN.java,v 1.188.2.6 2007/07/02 19:54:52 mark Exp $ */ package com.sleepycat.je.tree; @@ -19,6 +19,7 @@ import com.sleepycat.je.dbi.DatabaseId; import com.sleepycat.je.dbi.DatabaseImpl; import com.sleepycat.je.dbi.DbConfigManager; +import com.sleepycat.je.dbi.DbTree; import com.sleepycat.je.dbi.EnvironmentImpl; import com.sleepycat.je.dbi.MemoryBudget; import com.sleepycat.je.log.LogEntryType; @@ -219,18 +220,42 @@ /** * @Override */ - boolean hasNonLNChildren() { + boolean hasPinnedChildren() { - for (int i = 0; i < getNEntries(); i++) { - Node node = getTarget(i); - if (node != null) { - if (!(node instanceof LN)) { - return true; + DatabaseImpl db = getDatabase(); + + /* + * For the mapping DB, if any MapLN is resident we cannot evict this + * BIN. If a MapLN was not previously stripped, then the DB may be + * open. [#13415] + */ + if (db.getId().equals(DbTree.ID_DB_ID)) { + return hasResidentChildren(); + + /* + * For other DBs, if there are no duplicates allowed we can always + * evict this BIN because its children are limited to LNs. When + * logging the BIN, any dirty LNs will be logged and non-dirty LNs can + * be discarded. + */ + } else if (!db.getSortedDuplicates()) { + return false; + + /* + * If duplicates are allowed, we disallow eviction of this BIN if it + * has any non-LN (DIN) children. + */ + } else { + for (int i = 0; i < getNEntries(); i++) { + Node node = getTarget(i); + if (node != null) { + if (!(node instanceof LN)) { + return true; + } } } + return false; } - - return false; } /** @@ -244,6 +269,25 @@ Node node = getTarget(i); if (node != null) { if (node instanceof LN) { + LN ln = (LN) node; + + /* + * If the LN is not evictable, we may neither strip the LN + * nor evict the node. isEvictableInexact is used here as + * a fast check, to avoid the overhead of acquiring a + * handle lock while selecting an IN for eviction. See + * evictInternal which will call LN.isEvictable to acquire + * an handle lock and guarantee that another thread cannot + * open the MapLN. [#13415] + */ + if (!ln.isEvictableInexact()) { + return MAY_NOT_EVICT; + } + + /* + * If the cleaner allows eviction, then this LN may be + * stripped. + */ if (cleaner.isEvictable(this, i)) { return MAY_EVICT_LNS; } @@ -667,7 +711,13 @@ binRef.hasDeletedKey(new Key(getKey(i)))) { if (canFetch) { - n = fetchTarget(i); + if (db.isDeferredWrite() && + getLsn(i) == DbLsn.NULL_LSN) { + /* Null LSNs are ok in DW. [#15588] */ + n = null; + } else { + n = fetchTarget(i); + } } else { n = getTarget(i); if (n == null) { @@ -846,65 +896,74 @@ Node n = getTarget(index); - /* Don't strip LNs that the cleaner will be migrating. */ - if (n instanceof LN && - cleaner.isEvictable(this, index)) { - - /* Log target if necessary. */ + if (n instanceof LN) { LN ln = (LN) n; - if (ln.isDirty()) { - DatabaseImpl dbImpl = getDatabase(); - /* - * Only deferred write databases should have dirty LNs. This - * is an overly stringent assertion, because a db can flop - * between dw and durable mode, but is used currently for our - * regression testing. - */ - assert dbImpl.isDeferredWrite(); + /* + * Don't evict MapLNs for open databases (LN.isEvictable) [#13415]. + * And don't strip LNs that the cleaner will be migrating + * (Cleaner.isEvictable). + */ + if (ln.isEvictable() && + cleaner.isEvictable(this, index)) { - /* - * For DW we don't know the size of the last logged LN, so we - * pass zero for the obsolete size. - */ - EnvironmentImpl envImpl = dbImpl.getDbEnvironment(); - long obsoleteLsn = getObsoleteLsnForDWLogging(envImpl, index); - int obsoleteSize = ln.getLastLoggedSize(); - long lsn = ln.log(envImpl, - dbImpl.getId(), - getKey(index), - obsoleteLsn, - obsoleteSize, - null, // locker - true); // backgroundIO - updateEntry(index, lsn); - } + boolean force = getDatabase().isDeferredWrite() && + getLsn(index) == DbLsn.NULL_LSN; + /* Log target if necessary. */ + logDirtyLN(index, (LN) n, force); - /* Clear target. */ - setTarget(index, null); + /* Clear target. */ + setTarget(index, null); - return n.getMemorySizeIncludedByParent(); - } else { - return 0; + return n.getMemorySizeIncludedByParent(); + } } + return 0; } /** - * Returns the old LSN to be counted as obsolete during logging of a dirty - * deferred-write LN. If there is no old LSN, NULL_LSN is returned. If - * je.deferredWrite.temp=false, return the file number with a zero offset - * so that inexact counting is used; we cannot guarantee obsoleteness until - * the parent tree is flushed. [#15365] + * Logs the LN at the given index if it is dirty. */ - private long getObsoleteLsnForDWLogging(EnvironmentImpl envImpl, - int index) { - long lsn = getLsn(index); - if (lsn != DbLsn.NULL_LSN) { - if (!envImpl.getDeferredWriteTemp()) { - lsn = DbLsn.makeLsn(DbLsn.getFileNumber(lsn), 0); + private void logDirtyLN(int index, LN ln, boolean force) + throws DatabaseException { + + if (ln.isDirty() || force) { + DatabaseImpl dbImpl = getDatabase(); + EnvironmentImpl envImpl = dbImpl.getDbEnvironment(); + + /* Only deferred write databases should have dirty LNs. */ + assert dbImpl.isDeferredWrite(); + + /* Log the LN with the main tree key. */ + byte[] key = containsDuplicates() ? getDupKey() : getKey(index); + + /* + * Get the old LSN to be counted as obsolete during logging of a + * dirty deferred-write LN. If there is no old LSN, NULL_LSN is + * returned. If je.deferredWrite.temp=false, return the file + * number with a zero offset so that inexact counting is used; we + * cannot guarantee obsoleteness until the parent tree is flushed. + * [#15365] + */ + long obsoleteLsn = getLsn(index); + if (obsoleteLsn != DbLsn.NULL_LSN) { + if (!envImpl.getDeferredWriteTemp()) { + obsoleteLsn = DbLsn.makeLsn + (DbLsn.getFileNumber(obsoleteLsn), 0); + } } + int obsoleteSize = ln.getLastLoggedSize(); + + /* No need to lock, this is non-txnal. */ + long lsn = ln.log(dbImpl.getDbEnvironment(), + dbImpl.getId(), + key, + obsoleteLsn, + obsoleteSize, + null, // locker + true); // backgroundIO + updateEntry(index, lsn); } - return lsn; } /* For debugging. Overrides method in IN. */ @@ -1008,28 +1067,7 @@ if (node != null) { if (node instanceof LN) { - LN ln = (LN) node; - - /* No need to lock, this is non-txnal. */ - if (ln.isDirty()) { - - /* - * For DW we don't know the size of the last logged LN, - * so we pass zero for the obsolete size. - */ - long obsoleteLsn = - getObsoleteLsnForDWLogging(envImpl, i); - int obsoleteSize = ln.getLastLoggedSize(); - long childLsn = ln.log(envImpl, - getDatabase().getId(), - getKey(i), - obsoleteLsn, - obsoleteSize, - null, // locker - true); // backgroundIO - updateEntry(i, childLsn); - } - + logDirtyLN(i, (LN) node, false); } else { DIN din = (DIN) node; din.latch(false); @@ -1139,39 +1177,13 @@ DatabaseId dbId = getDatabase().getId(); EnvironmentImpl envImpl = getDatabase().getDbEnvironment(); + boolean isDeferredWrite = getDatabase().isDeferredWrite(); for (int i = 0; i < getNEntries(); i++) { Node node = getTarget(i); if ((node != null) && (node instanceof LN)) { - - LN ln = (LN) node; - if (ln.isDirty()) { - - /* - * Only deferred write databases should have dirty - * LNs. This is an overly stringent assertion, because a - * db can flop between dw and durable mode, but is used - * currently for our regression testing. - */ - assert getDatabase().isDeferredWrite(); - - /* - * For DW we don't know the size of the last logged LN, so - * we pass zero for the obsolete size. - */ - long obsoleteLsn = getObsoleteLsnForDWLogging(envImpl, i); - int obsoleteSize = ln.getLastLoggedSize(); - long lsn = ln.log(envImpl, - dbId, - getKey(i), - null, // delDupKey - obsoleteLsn, - obsoleteSize, - null, // locker - true, // backgroundIO - false); // Is provisional. - updateEntry(i, lsn); - } + logDirtyLN(i, (LN) node, + (getLsn(i) == DbLsn.NULL_LSN && isDeferredWrite)); } } } Modified: trunk/contrib/bdb/src/com/sleepycat/je/tree/BINDelta.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/tree/BINDelta.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/tree/BINDelta.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: BINDelta.java,v 1.44.2.1 2007/02/01 14:49:51 cwl Exp $ + * $Id: BINDelta.java,v 1.44.2.2 2007/07/02 19:54:52 mark Exp $ */ package com.sleepycat.je.tree; @@ -99,59 +99,67 @@ /* Get the last full version of this BIN. */ BIN fullBIN = (BIN) env.getLogManager().get(lastFullLsn); DatabaseImpl db = env.getDbMapTree().getDb(dbId); + try { - /* - * In effect, call fullBIN.postFetchInit(db) here. But we don't want - * to do that since it will put fullBIN on the INList. Since this is - * either recovery or during the Cleaner run, we don't want it on the - * INList. - */ - fullBIN.setDatabase(db); - fullBIN.setLastFullLsn(lastFullLsn); - - /* Process each delta. */ - fullBIN.latch(); - for (int i = 0; i < deltas.size(); i++) { - DeltaInfo info = (DeltaInfo) deltas.get(i); + /* + * In effect, call fullBIN.postFetchInit(db) here. But we don't + * want to do that since it will put fullBIN on the INList. Since + * this is either recovery or during the Cleaner run, we don't want + * it on the INList. + */ + fullBIN.setDatabase(db); + fullBIN.setLastFullLsn(lastFullLsn); + + /* Process each delta. */ + fullBIN.latch(); + for (int i = 0; i < deltas.size(); i++) { + DeltaInfo info = (DeltaInfo) deltas.get(i); - /* - * The BINDelta holds the authoritative version of each entry. In - * all cases, its entry should supercede the entry in the full - * BIN. This is true even if the BIN Delta's entry is knownDeleted - * or if the full BIN's version is knownDeleted. Therefore we use - * the flavor of findEntry that will return a knownDeleted entry if - * the entry key matches (i.e. true, false) but still indicates - * exact matches with the return index. findEntry only returns - * deleted entries if third arg is false, but we still need to know - * if it's an exact match or not so indicateExact is true. - */ - int foundIndex = fullBIN.findEntry(info.getKey(), true, false); - if (foundIndex >= 0 && - (foundIndex & IN.EXACT_MATCH) != 0) { - foundIndex &= ~IN.EXACT_MATCH; + /* + * The BINDelta holds the authoritative version of each entry. + * In all cases, its entry should supercede the entry in the + * full BIN. This is true even if the BIN Delta's entry is + * knownDeleted or if the full BIN's version is knownDeleted. + * Therefore we use the flavor of findEntry that will return a + * knownDeleted entry if the entry key matches (i.e. true, + * false) but still indicates exact matches with the return + * index. findEntry only returns deleted entries if third arg + * is false, but we still need to know if it's an exact match + * or not so indicateExact is true. + */ + int foundIndex = fullBIN.findEntry(info.getKey(), true, false); + if (foundIndex >= 0 && + (foundIndex & IN.EXACT_MATCH) != 0) { + foundIndex &= ~IN.EXACT_MATCH; - /* - * The entry exists in the full version, update it with the - * delta info. - */ - if (info.isKnownDeleted()) { - fullBIN.setKnownDeleted(foundIndex); + /* + * The entry exists in the full version, update it with the + * delta info. + */ + if (info.isKnownDeleted()) { + fullBIN.setKnownDeleted(foundIndex); + } else { + fullBIN.updateEntry + (foundIndex, info.getLsn(), info.getState()); + } } else { - fullBIN.updateEntry - (foundIndex, info.getLsn(), info.getState()); + + /* + * The entry doesn't exist, add a new entry from the delta. + */ + if (!info.isKnownDeleted()) { + ChildReference entry = + new ChildReference(null, + info.getKey(), + info.getLsn(), + info.getState()); + boolean insertOk = fullBIN.insertEntry(entry); + assert insertOk; + } } - } else { - /* The entry doesn't exist, add a new entry from the delta. */ - if (!info.isKnownDeleted()) { - ChildReference entry = - new ChildReference(null, - info.getKey(), - info.getLsn(), - info.getState()); - boolean insertOk = fullBIN.insertEntry(entry); - assert insertOk; - } } + } finally { + env.releaseDb(db); } /* Modified: trunk/contrib/bdb/src/com/sleepycat/je/tree/DBIN.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/tree/DBIN.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/tree/DBIN.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: DBIN.java,v 1.70.2.1 2007/02/01 14:49:51 cwl Exp $ + * $Id: DBIN.java,v 1.70.2.2 2007/07/02 19:54:52 mark Exp $ */ package com.sleepycat.je.tree; @@ -180,7 +180,7 @@ /** * @Override */ - boolean hasNonLNChildren() { + boolean hasPinnedChildren() { return false; } Modified: trunk/contrib/bdb/src/com/sleepycat/je/tree/IN.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/tree/IN.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/tree/IN.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: IN.java,v 1.295.2.3 2007/03/14 01:49:45 cwl Exp $ + * $Id: IN.java,v 1.295.2.5 2007/07/02 19:54:52 mark Exp $ */ package com.sleepycat.je.tree; @@ -912,7 +912,7 @@ * Returns true if the given state is known deleted. */ static boolean isStatePendingDeleted(byte state) { - return ((state & KNOWN_DELETED_BIT) != 0); + return ((state & PENDING_DELETED_BIT) != 0); } /** @@ -2368,14 +2368,23 @@ } /* - * An IN can be evicted if its resident children are all LNs, because - * those children can be logged and stripped before this node is - * evicted. + * An IN can be evicted only if its resident children are all evictable + * LNs, because those children can be logged (if dirty) and stripped + * before this node is evicted. Non-LN children or pinned LNs (MapLNs + * for open DBs) will prevent eviction. */ - if (hasNonLNChildren()) { + if (hasPinnedChildren()) { return false; } + for (int i = 0; i < getNEntries(); i++) { + /* Target and LSN can be null in DW. Not evictable in that case. */ + if (getLsn(i) == DbLsn.NULL_LSN && + getTarget(i) == null) { + return false; + } + } + return true; } @@ -2407,7 +2416,32 @@ */ boolean isEvictionProhibited() { - return isDbRoot(); + if (isDbRoot()) { + + /* + * Disallow eviction of a dirty DW DB root, since logging the MapLN + * (via DbTree.modifyDbRoot) will make the all other changes to the + * DW DB effectively non-provisional (durable). This implies that + * a DW DB root cannot be evicted until it is synced (or removed). + * [#13415] + */ + if (databaseImpl.isDeferredWrite() && getDirty()) { + return true; + } + + /* + * Disallow eviction of the mapping and naming DB roots, because + * the use count is not incremented for these DBs. In addition, + * their eviction and re-fetching is a special case that is not + * worth supporting. [#13415] + */ + DatabaseId dbId = databaseImpl.getId(); + if (dbId.equals(DbTree.ID_DB_ID) || + dbId.equals(DbTree.NAME_DB_ID)) { + return true; + } + } + return false; } /** @@ -2415,7 +2449,7 @@ * For an IN, that equates to whether there are any resident children * at all. */ - boolean hasNonLNChildren() { + boolean hasPinnedChildren() { return hasResidentChildren(); } @@ -2430,9 +2464,10 @@ } /** - * Returns whether any child is non-null. + * Returns whether any child is non-null. Is final to indicate it is not + * overridden (unlike hasPinnedChildren, isEvictionProhibited, etc). */ - private boolean hasResidentChildren() { + final boolean hasResidentChildren() { for (int i = 0; i < getNEntries(); i++) { if (getTarget(i) != null) { Modified: trunk/contrib/bdb/src/com/sleepycat/je/tree/LN.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/tree/LN.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/tree/LN.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: LN.java,v 1.125.2.4 2007/03/09 17:37:09 linda Exp $ + * $Id: LN.java,v 1.125.2.5 2007/07/02 19:54:52 mark Exp $ */ package com.sleepycat.je.tree; @@ -148,6 +148,27 @@ } /** + * Returns true by default, but is overridden by MapLN to prevent eviction + * of open databases. This method is meant to be a fast but not guaranteed + * check and is used during selection of BINs for LN stripping. [#13415] + */ + boolean isEvictableInexact() { + return true; + } + + /** + * Returns true by default, but is overridden by MapLN to prevent eviction + * of open databases. This method is meant to be a guaranteed check and is + * used after a BIN has been selected for LN stripping but before actually + * stripping an LN. [#13415] + */ + boolean isEvictable() + throws DatabaseException { + + return true; + } + + /** * A LN can never be a child in the search chain. */ protected boolean isSoughtNode(long nid, boolean updateGeneration) { Modified: trunk/contrib/bdb/src/com/sleepycat/je/tree/MapLN.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/tree/MapLN.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/tree/MapLN.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: MapLN.java,v 1.69.2.2 2007/03/08 22:33:00 mark Exp $ + * $Id: MapLN.java,v 1.69.2.3 2007/07/02 19:54:52 mark Exp $ */ package com.sleepycat.je.tree; @@ -16,6 +16,10 @@ import com.sleepycat.je.log.LogEntryType; import com.sleepycat.je.log.LogException; import com.sleepycat.je.log.LogUtils; +import com.sleepycat.je.txn.BasicLocker; +import com.sleepycat.je.txn.LockGrantType; +import com.sleepycat.je.txn.LockResult; +import com.sleepycat.je.txn.LockType; /** * A MapLN represents a Leaf Node in the JE DatabaseImpl Naming Tree. @@ -66,6 +70,62 @@ } /** + * Does a fast check without acquiring the MapLN write-lock. This is + * important because the overhead of requesting the lock is significant and + * unnecessary if this DB is open or the root IN is resident. When there + * are lots of databases open, this method will be called often during + * selection of BINs for eviction. [#13415] + * @Override + */ + boolean isEvictableInexact() { + return !databaseImpl.isInUse() && + !databaseImpl.getTree().isRootResident(); + } + + /** + * Does a guaranteed check by acquiring the write-lock and then calling + * isEvictableInexact. [#13415] + * @Override + */ + boolean isEvictable() + throws DatabaseException { + + boolean evictable = false; + + /* To prevent DB open, get a write-lock on the MapLN. */ + BasicLocker locker = new BasicLocker(databaseImpl.getDbEnvironment()); + try { + LockResult lockResult = locker.nonBlockingLock + (getNodeId(), LockType.WRITE, databaseImpl); + + /* + * The isEvictableInexact result is guaranteed to hold true during + * LN stripping if it is still true after acquiring the write-lock. + */ + if (lockResult.getLockGrant() != LockGrantType.DENIED && + isEvictableInexact()) { + + /* + * While holding both a write-lock on the MapLN, we are + * guaranteed that the DB is not currently open. It cannot be + * subsequently opened until the BIN latch is released, since + * the BIN latch will block DbTree.getDb (called during DB + * open). We will evict the LN before releasing the BIN latch. + * After releasing the BIN latch, if a DB open is waiting on + * getDb, then it will proceed, fetch the evicted LN and open + * the DB normally. + */ + evictable = true; + } + } finally { + /* Release the write-lock. The BIN latch is still held. */ + locker.operationEnd(); + } + + return evictable; + } + + /** * Initialize a node that has been faulted in from the log. */ public void postFetchInit(DatabaseImpl db, long sourceLsn) Modified: trunk/contrib/bdb/src/com/sleepycat/je/tree/Tree.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/tree/Tree.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/tree/Tree.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: Tree.java,v 1.418.2.3 2007/04/04 14:28:59 cwl Exp $ + * $Id: Tree.java,v 1.418.2.5 2007/07/02 19:54:52 mark Exp $ */ package com.sleepycat.je.tree; @@ -219,6 +219,17 @@ return true; } + /** + * Perform a fast check to see if the root IN is resident. No latching is + * performed. To ensure that the root IN is not loaded by another thread, + * this method should be called while holding a write lock on the MapLN. + * That will prevent opening the DB in another thread, and potentially + * loading the root IN. [#13415] + */ + public boolean isRootResident() { + return root != null && root.getTarget() != null; + } + /* * Class that overrides fetchTarget() so that if the rootLatch is not * held exclusively when the root is fetched, we upgrade it to exclusive. @@ -2817,6 +2828,15 @@ currentLock = cursor.lockLNDeletedAllowed (currentLN, LockType.WRITE); currentLN = currentLock.getLN(); + + /* + * The BIN may have been latched while locking above. + * Release the latch here because we released it above + * to improve concurrency, and we will latch it again + * below to increment the duplicate count. [#15574] + */ + cursor.releaseBIN(); + /* The DBIN/index may have changed while locking. */ dupBin = cursor.getDupBIN(); dupIndex = cursor.getDupIndex(); Modified: trunk/contrib/bdb/src/com/sleepycat/je/txn/BasicLocker.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/txn/BasicLocker.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/txn/BasicLocker.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: BasicLocker.java,v 1.84.2.1 2007/02/01 14:49:52 cwl Exp $ + * $Id: BasicLocker.java,v 1.84.2.4 2007/07/13 02:32:05 cwl Exp $ */ package com.sleepycat.je.txn; @@ -14,6 +14,7 @@ import com.sleepycat.je.Database; import com.sleepycat.je.DatabaseException; +import com.sleepycat.je.DbInternal; import com.sleepycat.je.LockStats; import com.sleepycat.je.dbi.CursorImpl; import com.sleepycat.je.dbi.DatabaseImpl; @@ -37,8 +38,11 @@ * * There's no need to track memory utilization for these non-txnal lockers, * because the lockers are short lived. + * + * ownedLock and ownedLockSet contains nids, not locks. We only need locks + * for the stats, so we can look them up on demand. */ - private Lock ownedLock; + private Long ownedLock; private Set ownedLockSet; /** @@ -186,14 +190,14 @@ * graph "display". [#9544] */ if (ownedLock != null) { - lockManager.release(ownedLock, this); + lockManager.release(ownedLock.longValue(), this); ownedLock = null; } if (ownedLockSet != null) { Iterator iter = ownedLockSet.iterator(); while (iter.hasNext()) { - Lock l = (Lock) iter.next(); - lockManager.release(l, this); + Long nid = (Long) iter.next(); + lockManager.release(nid.longValue(), this); } /* Now clear lock collection. */ @@ -214,14 +218,22 @@ /** * Transfer any MapLN locks to the db handle. */ - public void setHandleLockOwner(boolean ignore /*operationOK*/, + public void setHandleLockOwner(boolean operationOK, Database dbHandle, boolean dbIsClosing) throws DatabaseException { if (dbHandle != null) { - if (!dbIsClosing) { + if (operationOK && !dbIsClosing) { transferHandleLockToHandle(dbHandle); + } else { + + /* + * Release DB if there is a failure. This is done by Txn abort + * by calling Database.invalidate, but for a non-transactional + * locker must be done here. [#13415] + */ + envImpl.releaseDb(DbInternal.dbGetDatabaseImpl(dbHandle)); } unregisterHandle(dbHandle); } @@ -267,7 +279,10 @@ throws DatabaseException { if (deleteAtCommit) { + /* releaseDb will be called by releaseDeletedINs. */ db.deleteAndReleaseINs(); + } else { + envImpl.releaseDb(db); } } @@ -275,36 +290,37 @@ * Add a lock to set owned by this transaction. */ void addLock(Long nodeId, - Lock lock, LockType type, LockGrantType grantStatus) throws DatabaseException { - if (ownedLock == lock || + if ((ownedLock != null && + ownedLock.equals(nodeId)) || (ownedLockSet != null && - ownedLockSet.contains(lock))) { + ownedLockSet.contains(nodeId))) { return; // Already owned } if (ownedLock == null) { - ownedLock = lock; + ownedLock = nodeId; } else { if (ownedLockSet == null) { ownedLockSet = new HashSet(); } - ownedLockSet.add(lock); + ownedLockSet.add(nodeId); } } /** * Remove a lock from the set owned by this txn. */ - void removeLock(long nodeId, Lock lock) + void removeLock(long nodeId) throws DatabaseException { - if (lock == ownedLock) { + if (ownedLock != null && + ownedLock.longValue() == nodeId) { ownedLock = null; } else if (ownedLockSet != null) { - ownedLockSet.remove(lock); + ownedLockSet.remove(new Long(nodeId)); } } @@ -331,22 +347,28 @@ throws DatabaseException { if (ownedLock != null) { - if (ownedLock.isOwnedWriteLock(this)) { - stats.setNWriteLocks(stats.getNWriteLocks() + 1); - } else { - stats.setNReadLocks(stats.getNReadLocks() + 1); - } + Lock l = lockManager.lookupLock(ownedLock); + if (l != null) { + if (l.isOwnedWriteLock(this)) { + stats.setNWriteLocks(stats.getNWriteLocks() + 1); + } else { + stats.setNReadLocks(stats.getNReadLocks() + 1); + } + } } if (ownedLockSet != null) { Iterator iter = ownedLockSet.iterator(); while (iter.hasNext()) { - Lock l = (Lock) iter.next(); - if (l.isOwnedWriteLock(this)) { - stats.setNWriteLocks(stats.getNWriteLocks() + 1); - } else { - stats.setNReadLocks(stats.getNReadLocks() + 1); - } + Long nid = (Long) iter.next(); + Lock l = lockManager.lookupLock(nid); + if (l != null) { + if (l.isOwnedWriteLock(this)) { + stats.setNWriteLocks(stats.getNWriteLocks() + 1); + } else { + stats.setNReadLocks(stats.getNReadLocks() + 1); + } + } } } return stats; Modified: trunk/contrib/bdb/src/com/sleepycat/je/txn/DummyLockManager.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/txn/DummyLockManager.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/txn/DummyLockManager.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: DummyLockManager.java,v 1.4.2.1 2007/02/01 14:49:52 cwl Exp $ + * $Id: DummyLockManager.java,v 1.4.2.3 2007/07/13 13:13:23 cwl Exp $ */ package com.sleepycat.je.txn; @@ -28,6 +28,15 @@ } /** + * @see LockManager#lookupLock + */ + protected Lock lookupLock(Long nodeId) + throws DatabaseException { + + return null; + } + + /** * @see LockManager#attemptLock */ protected LockAttemptResult attemptLock(Long nodeId, @@ -60,9 +69,7 @@ * @see LockManager#releaseAndNotifyTargets */ protected Set releaseAndFindNotifyTargets(long nodeId, - Lock lock, - Locker locker, - boolean removeFromLocker) + Locker locker) throws DatabaseException { return null; Modified: trunk/contrib/bdb/src/com/sleepycat/je/txn/LatchedLockManager.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/txn/LatchedLockManager.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/txn/LatchedLockManager.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: LatchedLockManager.java,v 1.11.2.1 2007/02/01 14:49:52 cwl Exp $ + * $Id: LatchedLockManager.java,v 1.11.2.2 2007/07/13 02:32:05 cwl Exp $ */ package com.sleepycat.je.txn; @@ -28,6 +28,22 @@ } /** + * @see LockManager#lookupLock + */ + protected Lock lookupLock(Long nodeId) + throws DatabaseException { + + int lockTableIndex = getLockTableIndex(nodeId); + Latch latch = lockTableLatches[lockTableIndex]; + latch.acquire(); + try { + return lookupLockInternal(nodeId, lockTableIndex); + } finally { + latch.release(); + } + } + + /** * @see LockManager#attemptLock */ protected LockAttemptResult attemptLock(Long nodeId, @@ -79,21 +95,16 @@ * @see LockManager#releaseAndNotifyTargets */ protected Set releaseAndFindNotifyTargets(long nodeId, - Lock lock, - Locker locker, - boolean removeFromLocker) + Locker locker) throws DatabaseException { long nid = nodeId; - if (nid == -1) { - nid = lock.getNodeId().longValue(); - } int lockTableIndex = getLockTableIndex(nid); Latch latch = lockTableLatches[lockTableIndex]; latch.acquire(); try { return releaseAndFindNotifyTargetsInternal - (nodeId, lock, locker, removeFromLocker, lockTableIndex); + (nodeId, locker, lockTableIndex); } finally { latch.release(); } Modified: trunk/contrib/bdb/src/com/sleepycat/je/txn/Lock.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/txn/Lock.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/txn/Lock.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: Lock.java,v 1.60.2.1 2007/02/01 14:49:52 cwl Exp $ + * $Id: Lock.java,v 1.60.2.2 2007/07/13 02:32:05 cwl Exp $ */ package com.sleepycat.je.txn; @@ -22,7 +22,7 @@ * A Lock embodies the lock state of a NodeId. It includes a set of owners and * a list of waiters. */ -public class Lock { +class Lock { private static final int REMOVE_LOCKINFO_OVERHEAD = 0 - MemoryBudget.LOCKINFO_OVERHEAD; @@ -52,23 +52,13 @@ private Set ownerSet; private LockInfo firstWaiter; private List waiterList; - private Long nodeId; /** * Create a Lock. */ - Lock(Long nodeId) { - this.nodeId = nodeId; + Lock() { } - /* For the Sizeof program. */ - public Lock() { - } - - Long getNodeId() { - return nodeId; - } - /** * The first waiter goes into the firstWaiter member variable. Once the * waiterList is made, all appended waiters go into waiterList, even after @@ -709,7 +699,8 @@ * Transfer a lock from one transaction to another. Make sure that this * destination locker is only present as a single reader or writer. */ - LockType transfer(Locker currentLocker, + LockType transfer(Long nodeId, + Locker currentLocker, Locker destLocker, MemoryBudget mb, int lockTableIndex) @@ -727,7 +718,7 @@ firstOwner = null; numRemovedLockInfos++; } else if (firstOwner.getLocker() == currentLocker) { - lockType = setNewLocker(firstOwner, destLocker); + lockType = setNewLocker(nodeId, firstOwner, destLocker); } } @@ -739,7 +730,7 @@ iter.remove(); numRemovedLockInfos++; } else if (owner.getLocker() == currentLocker) { - lockType = setNewLocker(owner, destLocker); + lockType = setNewLocker(nodeId, owner, destLocker); } } } @@ -766,12 +757,13 @@ return lockType; } - private LockType setNewLocker(LockInfo owner, Locker destLocker) + private LockType setNewLocker(Long nodeId, + LockInfo owner, + Locker destLocker) throws DatabaseException { owner.setLocker(destLocker); - destLocker.addLock(nodeId, this, owner.getLockType(), - LockGrantType.NEW); + destLocker.addLock(nodeId, owner.getLockType(), LockGrantType.NEW); return owner.getLockType(); } @@ -780,7 +772,8 @@ * for case where a write handle lock is being transferred to multiple read * handles. */ - LockType transferMultiple(Locker currentLocker, + LockType transferMultiple(Long nodeId, + Locker currentLocker, Locker[] destLockers, MemoryBudget mb, int lockTableIndex) @@ -790,7 +783,8 @@ LockInfo oldOwner = null; if (destLockers.length == 1) { - return transfer(currentLocker, destLockers[0], mb, lockTableIndex); + return transfer(nodeId, currentLocker, destLockers[0], + mb, lockTableIndex); } else { /* @@ -822,7 +816,8 @@ * Create the clones */ if (firstOwner != null) { - oldOwner = cloneLockInfo(firstOwner, + oldOwner = cloneLockInfo(nodeId, + firstOwner, currentLocker, destLockers, mb, @@ -833,7 +828,8 @@ Iterator ownersIter = ownerSet.iterator(); while (ownersIter.hasNext()) { LockInfo o = (LockInfo) ownersIter.next(); - oldOwner = cloneLockInfo(o, + oldOwner = cloneLockInfo(nodeId, + o, currentLocker, destLockers, mb, @@ -879,7 +875,8 @@ * If oldOwner is the current owner, clone it and transform it into a dest * locker. */ - private LockInfo cloneLockInfo(LockInfo oldOwner, + private LockInfo cloneLockInfo(Long nodeId, + LockInfo oldOwner, Locker currentLocker, Locker[] destLockers, MemoryBudget mb, @@ -892,8 +889,8 @@ for (int i = 0; i < destLockers.length; i++) { LockInfo clonedLockInfo = (LockInfo) oldOwner.clone(); clonedLockInfo.setLocker(destLockers[i]); - destLockers[i].addLock(nodeId, this, lockType, - LockGrantType.NEW); + destLockers[i].addLock(nodeId, lockType, + LockGrantType.NEW); addOwner(clonedLockInfo, mb, lockTableIndex); } return oldOwner; @@ -970,7 +967,7 @@ */ public String toString() { StringBuffer sb = new StringBuffer(); - sb.append(" NodeId:").append(nodeId); + sb.append(" LockAddr:").append(System.identityHashCode(this)); sb.append(" Owners:"); if (nOwners() == 0) { sb.append(" (none)"); Modified: trunk/contrib/bdb/src/com/sleepycat/je/txn/LockInfo.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/txn/LockInfo.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/txn/LockInfo.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: LockInfo.java,v 1.28.2.1 2007/02/01 14:49:53 cwl Exp $ + * $Id: LockInfo.java,v 1.28.2.2 2007/07/13 02:32:05 cwl Exp $ */ package com.sleepycat.je.txn; @@ -17,6 +17,8 @@ /** * LockInfo is a class that embodies information about a lock instance. The * holding thread and the locktype are all contained in the object. + * + * This class is public for unit tests. */ public class LockInfo implements Cloneable { private Locker locker; Modified: trunk/contrib/bdb/src/com/sleepycat/je/txn/LockManager.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/txn/LockManager.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/txn/LockManager.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: LockManager.java,v 1.118.2.1 2007/02/01 14:49:53 cwl Exp $ + * $Id: LockManager.java,v 1.118.2.3 2007/07/13 02:32:05 cwl Exp $ */ package com.sleepycat.je.txn; @@ -285,12 +285,24 @@ assert EnvironmentImpl.maybeForceYield(); } - locker.addLock(nid, result.useLock, type, result.lockGrant); + locker.addLock(nid, type, result.lockGrant); return result.lockGrant; } } + abstract protected Lock lookupLock(Long nodeId) + throws DatabaseException; + + protected Lock lookupLockInternal(Long nodeId, int lockTableIndex) + throws DatabaseException { + + /* Get the target lock. */ + Map lockTable = lockTables[lockTableIndex]; + Lock useLock = (Lock) lockTable.get(nodeId); + return useLock; + } + abstract protected LockAttemptResult attemptLock(Long nodeId, Locker locker, @@ -312,7 +324,7 @@ Map lockTable = lockTables[lockTableIndex]; Lock useLock = (Lock) lockTable.get(nodeId); if (useLock == null) { - useLock = new Lock(nodeId); + useLock = new Lock(); lockTable.put(nodeId, useLock); memoryBudget.updateLockMemoryUsage(TOTAL_LOCK_OVERHEAD, lockTableIndex); @@ -330,7 +342,7 @@ /* Was the attempt successful? */ if ((lockGrant == LockGrantType.NEW) || (lockGrant == LockGrantType.PROMOTION)) { - locker.addLock(nodeId, useLock, type, lockGrant); + locker.addLock(nodeId, type, lockGrant); success = true; } else if (lockGrant == LockGrantType.EXISTING) { success = true; @@ -424,51 +436,12 @@ * @return true if the lock is released successfully, false if * the lock is not currently being held. */ - boolean release(long nodeId, Locker locker) + boolean release(long nodeId, Locker locker) throws DatabaseException { - return release(nodeId, null, locker, true); - } - - /** - * Release a lock and possibly notify any waiters that they have been - * granted the lock. - * - * @param lock The lock to release - * - * @return true if the lock is released successfully, false if the lock is - * not currently being held. - */ - boolean release(Lock lock, Locker locker) - throws DatabaseException { - - return release(-1, lock, locker, false); - } - - /** - * Do the work of releasing a lock and notifying any waiters that they have - * been granted the lock. - * - * @param lock The lock to release. If null, use nodeId to find lock - * @param nodeId The node ID of the lock to release, if lock is null. May - * not be valid if lock is not null. MUST be valid if removeFromLocker is - * true - * @param locker - * @param removeFromLocker true if we're responsible for - * - * @return true if the lock is released successfully, false if the lock is - * not currently being held. - */ - private boolean release(long nodeId, - Lock lock, - Locker locker, - boolean removeFromLocker) - throws DatabaseException { - synchronized (locker) { Set newOwners = - releaseAndFindNotifyTargets(nodeId, lock, locker, - removeFromLocker); + releaseAndFindNotifyTargets(nodeId, locker); if (newOwners == null) { return false; @@ -508,9 +481,7 @@ */ protected abstract Set releaseAndFindNotifyTargets(long nodeId, - Lock lock, - Locker locker, - boolean removeFromLocker) + Locker locker) throws DatabaseException; /** @@ -518,20 +489,14 @@ */ protected Set releaseAndFindNotifyTargetsInternal(long nodeId, - Lock lock, Locker locker, - boolean removeFromLocker, int lockTableIndex) throws DatabaseException { - Lock useLock = lock; - Map lockTable = lockTables[lockTableIndex]; + Lock useLock = (Lock) lockTable.get(new Long(nodeId)); + if (useLock == null) { - useLock = (Lock) lockTable.get(new Long(nodeId)); - } - - if (useLock == null) { /* Lock doesn't exist. */ return null; } @@ -543,20 +508,10 @@ return null; } - /* - * If desired, remove it from the locker's bag. Used when we don't need - * to hang onto the lock after release -- like null txns or locks on - * deleted LNs. - */ - if (removeFromLocker) { - assert nodeId != -1; - locker.removeLock(nodeId, useLock); - } - /* If it's not in use at all, remove it from the lock table. */ if ((useLock.nWaiters() == 0) && (useLock.nOwners() == 0)) { - lockTables[lockTableIndex].remove(useLock.getNodeId()); + lockTables[lockTableIndex].remove(new Long(nodeId)); memoryBudget.updateLockMemoryUsage(REMOVE_TOTAL_LOCK_OVERHEAD, lockTableIndex); } @@ -592,9 +547,9 @@ if (demoteToRead) { useLock.demote(owningLocker); } - useLock.transfer(owningLocker, destLocker, + useLock.transfer(new Long(nodeId), owningLocker, destLocker, memoryBudget, lockTableIndex); - owningLocker.removeLock(nodeId, useLock); + owningLocker.removeLock(nodeId); } /** @@ -623,9 +578,9 @@ assert useLock != null : "Transfer, lock " + nodeId + " was null"; useLock.demote(owningLocker); - useLock.transferMultiple(owningLocker, destLockers, + useLock.transferMultiple(new Long(nodeId), owningLocker, destLockers, memoryBudget, lockTableIndex); - owningLocker.removeLock(nodeId, useLock); + owningLocker.removeLock(nodeId); } /** @@ -964,7 +919,7 @@ /* Found a cycle. */ StringBuffer ret = new StringBuffer(); ret.append("Transaction ").append(locker.toString()); - ret.append(" owns ").append(lock.getNodeId()); + ret.append(" owns ").append(System.identityHashCode(lock)); ret.append(" ").append(info).append("\n"); ret.append("Transaction ").append(locker.toString()); ret.append(" waits for "); @@ -972,7 +927,7 @@ ret.append(" nothing"); } else { ret.append(" node "); - ret.append(waitsFor.getNodeId()); + ret.append(System.identityHashCode(waitsFor)); } ret.append("\n"); return ret; @@ -983,8 +938,8 @@ rootLocker); if (sb != null) { String waitInfo = - "Transaction " + locker + " waits for node " + - waitsFor.getNodeId() + "\n"; + "Transaction " + locker + " waits for " + + waitsFor + "\n"; sb.insert(0, waitInfo); return sb; } Modified: trunk/contrib/bdb/src/com/sleepycat/je/txn/Locker.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/txn/Locker.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/txn/Locker.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: Locker.java,v 1.101.2.1 2007/02/01 14:49:53 cwl Exp $ + * $Id: Locker.java,v 1.101.2.3 2007/07/13 02:32:05 cwl Exp $ */ package com.sleepycat.je.txn; @@ -295,13 +295,8 @@ public void releaseLock(long nodeId) throws DatabaseException { - /* - * If successful, the lock manager will call back to the transaction - * and remove the lock from the lock collection. Done this way because - * we can't get a handle on the lock without creating another object - * XXX: bite the bullet, new a holder object, pass it back? - */ lockManager.release(nodeId, this); + removeLock(nodeId); } /** @@ -482,7 +477,6 @@ * Add a lock to set owned by this transaction. */ abstract void addLock(Long nodeId, - Lock lock, LockType type, LockGrantType grantStatus) throws DatabaseException; @@ -499,7 +493,7 @@ * LockManager.release, the lock manager will call this when its releasing * a lock. */ - abstract void removeLock(long nodeId, Lock lock) + abstract void removeLock(long nodeId) throws DatabaseException; /** @@ -560,7 +554,6 @@ */ public void addToHandleMaps(Long handleLockId, Database databaseHandle) { - // TODO: mutex after measurement Set dbHandleSet = null; if (handleLockToHandleMap == null) { Modified: trunk/contrib/bdb/src/com/sleepycat/je/txn/ReadCommittedLocker.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/txn/ReadCommittedLocker.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/txn/ReadCommittedLocker.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: ReadCommittedLocker.java,v 1.6.2.1 2007/02/01 14:49:53 cwl Exp $ + * $Id: ReadCommittedLocker.java,v 1.6.2.2 2007/07/13 02:32:05 cwl Exp $ */ package com.sleepycat.je.txn; @@ -80,6 +80,7 @@ if (!lockManager.release(nodeId, this)) { lockManager.release(nodeId, getBuddy()); } + removeLock(nodeId); } /** Modified: trunk/contrib/bdb/src/com/sleepycat/je/txn/SyncedLockManager.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/txn/SyncedLockManager.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/txn/SyncedLockManager.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: SyncedLockManager.java,v 1.11.2.1 2007/02/01 14:49:53 cwl Exp $ + * $Id: SyncedLockManager.java,v 1.11.2.2 2007/07/13 02:32:05 cwl Exp $ */ package com.sleepycat.je.txn; @@ -45,6 +45,18 @@ } /** + * @see LockManager#lookupLock + */ + protected Lock lookupLock(Long nodeId) + throws DatabaseException { + + int lockTableIndex = getLockTableIndex(nodeId); + synchronized(lockTableLatches[lockTableIndex]) { + return lookupLockInternal(nodeId, lockTableIndex); + } + } + + /** * @see LockManager#makeTimeoutMsg */ protected String makeTimeoutMsg(String lockOrTxn, @@ -70,19 +82,14 @@ * @see LockManager#releaseAndNotifyTargets */ protected Set releaseAndFindNotifyTargets(long nodeId, - Lock lock, - Locker locker, - boolean removeFromLocker) + Locker locker) throws DatabaseException { long nid = nodeId; - if (nid == -1) { - nid = lock.getNodeId().longValue(); - } int lockTableIndex = getLockTableIndex(nid); synchronized(lockTableLatches[lockTableIndex]) { return releaseAndFindNotifyTargetsInternal - (nodeId, lock, locker, removeFromLocker, lockTableIndex); + (nodeId, locker, lockTableIndex); } } Modified: trunk/contrib/bdb/src/com/sleepycat/je/txn/ThreadLocker.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/txn/ThreadLocker.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/txn/ThreadLocker.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: ThreadLocker.java,v 1.14.2.2 2007/03/28 15:53:44 cwl Exp $ + * $Id: ThreadLocker.java,v 1.14.2.3 2007/05/23 13:44:52 mark Exp $ */ package com.sleepycat.je.txn; @@ -35,10 +35,9 @@ if (thread != Thread.currentThread()) { throw new DatabaseException - ("A per-thread transaction was created in " + thread + - " but used in " + Thread.currentThread() + - "\nPerhaps you are using a non-Transactional Cursor " + - "in multiple threads."); + ("Non-transactional Cursors may not be used in multiple " + + "threads; Cursor was created in " + thread + + " but used in " + Thread.currentThread()); } } Modified: trunk/contrib/bdb/src/com/sleepycat/je/txn/Txn.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/txn/Txn.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/txn/Txn.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: Txn.java,v 1.148.2.5 2007/04/04 14:29:22 cwl Exp $ + * $Id: Txn.java,v 1.148.2.7 2007/07/13 02:32:05 cwl Exp $ */ package com.sleepycat.je.txn; @@ -79,7 +79,7 @@ * A Txn can be used by multiple threads. Modification to the read and * write lock collections is done by synchronizing on the txn. */ - private Set readLocks; + private Set readLocks; // Set<Long> (nodeIds) private Map writeInfo; // key=nodeid, data = WriteLockInfo private final int READ_LOCK_OVERHEAD = MemoryBudget.HASHSET_ENTRY_OVERHEAD; @@ -399,7 +399,7 @@ if (handleLockToHandleMap != null) { Iterator handleLockIter = handleLockToHandleMap.entrySet().iterator(); - while (handleLockIter.hasNext()){ + while (handleLockIter.hasNext()) { Map.Entry entry = (Map.Entry) handleLockIter.next(); Long nodeId = (Long) entry.getKey(); if (writeInfo != null) { @@ -464,13 +464,14 @@ Set alreadyCountedLsnSet = new HashSet(); /* Release all write locks, clear lock collection. */ - Iterator iter = writeInfo.values().iterator(); + Iterator iter = writeInfo.entrySet().iterator(); while (iter.hasNext()) { - WriteLockInfo info = (WriteLockInfo) iter.next(); - lockManager.release(info.lock, this); - /* Count obsolete LSNs for released write locks. */ - countWriteAbortLSN(info, alreadyCountedLsnSet); - } + Map.Entry entry = (Map.Entry) iter.next(); + Long nodeId = (Long) entry.getKey(); + lockManager.release(nodeId.longValue(), this); + countWriteAbortLSN((WriteLockInfo) entry.getValue(), + alreadyCountedLsnSet); + } writeInfo = null; /* Count obsolete LSNs for transferred write locks. */ @@ -676,7 +677,7 @@ if (handleToHandleLockMap != null) { Iterator handleIter = handleToHandleLockMap.keySet().iterator(); - while (handleIter.hasNext()){ + while (handleIter.hasNext()) { Database handle = (Database) handleIter.next(); DbInternal.dbInvalidate(handle); } @@ -776,11 +777,13 @@ throws DatabaseException { int numWriteLocks = writeInfo.size(); + /* Release all write locks, clear lock collection. */ - Iterator iter = writeInfo.values().iterator(); + Iterator iter = writeInfo.entrySet().iterator(); while (iter.hasNext()) { - WriteLockInfo info = (WriteLockInfo) iter.next(); - lockManager.release(info.lock, this); + Map.Entry entry = (Map.Entry) iter.next(); + Long nodeId = (Long) entry.getKey(); + lockManager.release(nodeId.longValue(), this); } writeInfo = null; return numWriteLocks; @@ -794,8 +797,8 @@ numReadLocks = readLocks.size(); Iterator iter = readLocks.iterator(); while (iter.hasNext()) { - Lock rLock = (Lock) iter.next(); - lockManager.release(rLock, this); + Long rLock = (Long) iter.next(); + lockManager.release(rLock.longValue(), this); } readLocks = null; } @@ -860,6 +863,8 @@ delta += MemoryBudget.HASHSET_ENTRY_OVERHEAD + MemoryBudget.OBJECT_OVERHEAD; updateMemoryUsage(delta); + + /* releaseDb will be called by cleanupDatabaseImpls. */ } } @@ -888,7 +893,7 @@ * operations like removeDatabase(), truncateDatabase(). * * This method must be called outside the synchronization on this txn, - * because it calls deleteAndReleaseINs, which gets the TxnManager's + * because it calls releaseDeletedINs, which gets the TxnManager's * allTxns latch. The checkpointer also gets the allTxns latch, and within * that latch, needs to synchronize on individual txns, so we must avoid a * latching hiearchy conflict. @@ -906,7 +911,10 @@ for (int i = 0; i < infoArray.length; i += 1) { DatabaseCleanupInfo info = infoArray[i]; if (info.deleteAtCommit == isCommit) { + /* releaseDb will be called by releaseDeletedINs. */ info.dbImpl.releaseDeletedINs(); + } else { + envImpl.releaseDb(info.dbImpl); } } deletedDatabases = null; @@ -917,7 +925,6 @@ * Add lock to the appropriate queue. */ void addLock(Long nodeId, - Lock lock, LockType type, LockGrantType grantStatus) throws DatabaseException { @@ -932,29 +939,29 @@ } writeInfo.put(nodeId, - new WriteLockInfo(lock)); + new WriteLockInfo()); delta += WRITE_LOCK_OVERHEAD; if ((grantStatus == LockGrantType.PROMOTION) || (grantStatus == LockGrantType.WAIT_PROMOTION)) { - readLocks.remove(lock); + readLocks.remove(nodeId); delta -= READ_LOCK_OVERHEAD; } updateMemoryUsage(delta); } else { - addReadLock(lock); + addReadLock(nodeId); } } } - private void addReadLock(Lock lock) { + private void addReadLock(Long nodeId) { int delta = 0; if (readLocks == null) { readLocks = new HashSet(); delta = MemoryBudget.HASHSET_OVERHEAD; } - readLocks.add(lock); + readLocks.add(nodeId); delta += READ_LOCK_OVERHEAD; updateMemoryUsage(delta); } @@ -965,7 +972,7 @@ * a lock. Usually done because the transaction doesn't need to really keep * the lock, i.e for a deleted record. */ - void removeLock(long nodeId, Lock lock) + void removeLock(long nodeId) throws DatabaseException { /* @@ -978,7 +985,7 @@ */ synchronized (this) { if ((readLocks != null) && - readLocks.remove(lock)) { + readLocks.remove(new Long(nodeId))) { updateMemoryUsage(0 - READ_LOCK_OVERHEAD); } else if ((writeInfo != null) && (writeInfo.remove(new Long(nodeId)) != null)) { @@ -1003,7 +1010,7 @@ assert found : "Couldn't find lock for Node " + nodeId + " in writeInfo Map."; - addReadLock(lock); + addReadLock(new Long(nodeId)); } } Modified: trunk/contrib/bdb/src/com/sleepycat/je/txn/WriteLockInfo.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/txn/WriteLockInfo.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/txn/WriteLockInfo.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: WriteLockInfo.java,v 1.14.2.1 2007/02/01 14:49:53 cwl Exp $ + * $Id: WriteLockInfo.java,v 1.14.2.2 2007/07/13 02:32:06 cwl Exp $ */ package com.sleepycat.je.txn; @@ -15,8 +15,6 @@ * the correct abort LSN. */ public class WriteLockInfo { - /* Write lock for node. */ - Lock lock; /* * The original LSN. This is stored in the LN log entry. May be null if @@ -49,23 +47,13 @@ static final WriteLockInfo basicWriteLockInfo = new WriteLockInfo(); - WriteLockInfo(Lock lock) { - this.lock = lock; + public WriteLockInfo() { abortLsn = DbLsn.NULL_LSN; abortKnownDeleted = false; neverLocked = true; createdThisTxn = false; } - /* public for Sizeof program. */ - public WriteLockInfo() { - this.lock = null; - abortLsn = DbLsn.NULL_LSN; - abortKnownDeleted = true; - neverLocked = true; - createdThisTxn = false; - } - public boolean getAbortKnownDeleted() { return abortKnownDeleted; } Modified: trunk/contrib/bdb/src/com/sleepycat/je/util/DbVerify.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/util/DbVerify.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/util/DbVerify.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: DbVerify.java,v 1.42.2.1 2007/02/01 14:49:54 cwl Exp $ + * $Id: DbVerify.java,v 1.42.2.2 2007/07/02 19:54:53 mark Exp $ */ package com.sleepycat.je.util; @@ -251,9 +251,13 @@ try { DatabaseImpl dbImpl = dbMapTree.getDb(null, targetDb, null); - if (!verifyOneDbImpl(dbImpl, targetDb, - verifyConfig, out)) { - ret = false; + try { + if (!verifyOneDbImpl(dbImpl, targetDb, + verifyConfig, out)) { + ret = false; + } + } finally { + dbMapTree.releaseDb(dbImpl); } } finally { Tracer.trace(Level.INFO, envImpl, Modified: trunk/contrib/bdb/src/com/sleepycat/je/utilint/VLSN.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/je/utilint/VLSN.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/je/utilint/VLSN.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -92,13 +92,11 @@ } - /* BOZO, remove this. */ public int getContentSize() { return getLogSize(); } /** - * BOZO, remove this * @param buffer is the destination buffer */ public void writeToBuffer(ByteBuffer buffer) { Modified: trunk/contrib/bdb/src/com/sleepycat/persist/StoreConfig.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/persist/StoreConfig.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/persist/StoreConfig.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: StoreConfig.java,v 1.13.2.1 2007/02/01 14:49:55 cwl Exp $ + * $Id: StoreConfig.java,v 1.13.2.2 2007/06/14 13:06:04 mark Exp $ */ package com.sleepycat.persist; @@ -42,6 +42,7 @@ private boolean transactional; private boolean readOnly; private boolean deferredWrite; + private boolean secondaryBulkLoad; private EntityModel model; private Mutations mutations; @@ -166,6 +167,44 @@ } /** + * Sets the bulk-load-secondaries configuration property. By default this + * property is false. + * + * <p>This property is true to cause the initial creation of secondary + * indices to be performed as a bulk load. If this property is true and + * {@link EntityStore#getSecondaryIndex EntityStore.getSecondaryIndex} has + * never been called for a secondary index, that secondary index will not + * be created or written as records are written to the primary index. In + * addition, if that secondary index defines a foreign key constraint, the + * constraint will not be enforced.</p> + * + * <p>The secondary index will be populated later when the {code + * getSecondaryIndex} method is called for the first time for that index, + * or when the store is closed and re-opened with this property set to + * false and the primary index is obtained. In either case, the secondary + * index is populated by reading through the entire primary index and + * adding records to the secondary index as needed. While populating the + * secondary, foreign key constraints will be enforced and an exception is + * thrown if a constraint is violated.</p> + * + * <p>When loading a primary index along with secondary indexes from a + * large input data set, configuring a bulk load of the secondary indexes + * is sometimes more performant than updating the secondary indexes each + * time the primary index is updated. The absence of foreign key + * constraints during the load also provides more flexibility.</p> + */ + public void setSecondaryBulkLoad(boolean secondaryBulkLoad) { + this.secondaryBulkLoad = secondaryBulkLoad; + } + + /** + * Returns the bulk-load-secondaries configuration property. + */ + public boolean getSecondaryBulkLoad() { + return secondaryBulkLoad; + } + + /** * Sets the entity model that defines entity classes and index keys. * * <p>If null is specified or this method is not called, an {@link Modified: trunk/contrib/bdb/src/com/sleepycat/persist/impl/PersistCatalog.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/persist/impl/PersistCatalog.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/persist/impl/PersistCatalog.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: PersistCatalog.java,v 1.33.2.1 2007/02/01 14:49:56 cwl Exp $ + * $Id: PersistCatalog.java,v 1.33.2.3 2007/06/14 13:06:05 mark Exp $ */ package com.sleepycat.persist.impl; @@ -552,7 +552,8 @@ ClassMetadata metadata = model.getClassMetadata(className); if (metadata == null) { throw new IllegalArgumentException - ("Class is not persistent: " + className); + ("Class could not be loaded or is not persistent: " + + className); } if (metadata.getCompositeKeyFields() != null && (metadata.getPrimaryKey() != null || @@ -660,7 +661,7 @@ if (entityFormat != null && entityFormat != format) { try { store.openSecondaryIndexes - (entityFormat.getEntityMetadata()); + (null, entityFormat.getEntityMetadata(), null); } catch (DatabaseException e) { throw new RuntimeExceptionWrapper(e); } Modified: trunk/contrib/bdb/src/com/sleepycat/persist/impl/PersistEntityBinding.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/persist/impl/PersistEntityBinding.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/persist/impl/PersistEntityBinding.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: PersistEntityBinding.java,v 1.16.2.1 2007/02/01 14:49:56 cwl Exp $ + * $Id: PersistEntityBinding.java,v 1.16.2.2 2007/05/01 04:08:35 mark Exp $ */ package com.sleepycat.persist.impl; @@ -11,6 +11,7 @@ import com.sleepycat.bind.EntityBinding; import com.sleepycat.bind.tuple.TupleBase; import com.sleepycat.je.DatabaseEntry; +import com.sleepycat.persist.model.EntityModel; import com.sleepycat.persist.raw.RawObject; /** @@ -20,7 +21,7 @@ */ public class PersistEntityBinding implements EntityBinding { - Catalog catalog; + PersistCatalog catalog; Format entityFormat; boolean rawAccess; PersistKeyAssigner keyAssigner; @@ -28,7 +29,7 @@ /** * Creates a key binding for a given entity class. */ - public PersistEntityBinding(Catalog catalog, + public PersistEntityBinding(PersistCatalog catalog, String entityClassName, boolean rawAccess) { this.catalog = catalog; @@ -41,7 +42,7 @@ } else { Class entityCls; try { - entityCls = Class.forName(entityClassName); + entityCls = EntityModel.classForName(entityClassName); } catch (ClassNotFoundException e) { throw new IllegalArgumentException(e); } Modified: trunk/contrib/bdb/src/com/sleepycat/persist/impl/SimpleCatalog.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/persist/impl/SimpleCatalog.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/persist/impl/SimpleCatalog.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: SimpleCatalog.java,v 1.19.2.1 2007/02/01 14:49:56 cwl Exp $ + * $Id: SimpleCatalog.java,v 1.19.2.2 2007/05/01 04:08:35 mark Exp $ */ package com.sleepycat.persist.impl; @@ -15,6 +15,7 @@ import java.util.Map; import java.util.NoSuchElementException; +import com.sleepycat.persist.model.EntityModel; import com.sleepycat.persist.raw.RawObject; /** @@ -82,7 +83,7 @@ cls = primitiveTypeToWrapper.get(cls); } else { try { - cls = Class.forName(className); + cls = EntityModel.classForName(className); } catch (ClassNotFoundException e) { throw new IllegalArgumentException ("Key class not found: " + className); @@ -106,7 +107,7 @@ Class cls = keywordToPrimitive.get(className); if (cls == null) { - cls = Class.forName(className); + cls = EntityModel.classForName(className); } return cls; } Modified: trunk/contrib/bdb/src/com/sleepycat/persist/impl/Store.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/persist/impl/Store.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/persist/impl/Store.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: Store.java,v 1.20.2.2 2007/02/10 02:58:19 mark Exp $ + * $Id: Store.java,v 1.20.2.5 2007/06/14 13:06:05 mark Exp $ */ package com.sleepycat.persist.impl; @@ -330,18 +330,27 @@ (keyBinding, entityBinding, getSequence(seqName)); } - /* Use a single transaction for all opens. */ + /* + * Use a single transaction for opening the primary DB and its + * secondaries. If opening any secondary fails, abort the + * transaction and undo the changes to the state of the store. + * Also support undo if the store is non-transactional. + */ Transaction txn = null; DatabaseConfig dbConfig = getPrimaryConfig(entityMeta); if (dbConfig.getTransactional() && env.getThreadTransaction() == null) { txn = env.beginTransaction(null, null); } + PrimaryOpenState priOpenState = + new PrimaryOpenState(entityClassName); boolean success = false; try { + /* Open the primary database. */ String dbName = storePrefix + entityClassName; Database db = env.openDatabase(txn, dbName, dbConfig); + priOpenState.addDatabase(db); /* Create index object. */ priIndex = new PrimaryIndex @@ -356,7 +365,7 @@ /* If not read-only, open all associated secondaries. */ if (!dbConfig.getReadOnly()) { - openSecondaryIndexes(entityMeta); + openSecondaryIndexes(txn, entityMeta, priOpenState); /* * To enable foreign key contratints, also open all primary @@ -373,12 +382,17 @@ } success = true; } finally { - if (txn != null) { - if (success) { + if (success) { + if (txn != null) { txn.commit(); + } + } else { + if (txn != null) { + txn.abort(); } else { - txn.abort(); + priOpenState.closeDatabases(); } + priOpenState.undoState(); } } } @@ -386,7 +400,75 @@ } /** + * Holds state information about opening a primary index and its secondary + * indexes. Used to undo the state of this object if the transaction + * opening the primary and secondaries aborts. Also used to close all + * databases opened during this process for a non-transactional store. + */ + private class PrimaryOpenState { + + private String entityClassName; + private IdentityHashMap<Database,Object> databases; + private Set<String> secNames; + + PrimaryOpenState(String entityClassName) { + this.entityClassName = entityClassName; + databases = new IdentityHashMap<Database,Object>(); + secNames = new HashSet<String>(); + } + + /** + * Save a database that was opening during this operation. + */ + void addDatabase(Database db) { + databases.put(db, null); + } + + /** + * Save the name of a secondary index that was opening during this + * operation. + */ + void addSecondaryName(String secName) { + secNames.add(secName); + } + + /** + * Close any databases opened during this operation when it fails. + * This method should be called if a non-transactional operation fails, + * since we cannot rely on the transaction abort to cleanup any + * databases that were opened. + */ + void closeDatabases() { + for (Database db : databases.keySet()) { + try { + db.close(); + } catch (Exception ignored) { + } + } + } + + /** + * Reset all state information when this operation fails. This method + * should be called for both transactional and non-transsactional + * operation. + */ + void undoState() { + priIndexMap.remove(entityClassName); + for (String secName : secNames) { + secIndexMap.remove(secName); + } + for (Database db : databases.keySet()) { + deferredWriteDatabases.remove(db); + } + } + } + + /** * Opens a primary index related via a foreign key (relatedEntity). + * Related indexes are not opened in the same transaction used by the + * caller to open a primary or secondary. It is OK to leave the related + * index open when the caller's transaction aborts. It is only important + * to open a primary and its secondaries atomically. */ private PrimaryIndex getRelatedIndex(String relatedClsName) throws DatabaseException { @@ -404,7 +486,7 @@ relatedKeyClsName = null; } else { try { - relatedCls = Class.forName(relatedClsName); + relatedCls = EntityModel.classForName(relatedClsName); } catch (ClassNotFoundException e) { throw new IllegalArgumentException ("Related entity class not found: " + @@ -415,7 +497,12 @@ relatedKeyCls = SimpleCatalog.keyClassForName(relatedKeyClsName); } - /* XXX Check to make sure cycles are not possible here. */ + + /* + * Cycles are prevented here by adding primary indexes to the + * priIndexMap as soon as they are created, before opening related + * indexes. + */ relatedIndex = getPrimaryIndex (relatedKeyCls, relatedKeyClsName, relatedCls, relatedClsName); @@ -491,7 +578,8 @@ secIndex = openSecondaryIndex (null, primaryIndex, entityClass, entityMeta, - keyClass, keyClassName, secKeyMeta, secName); + keyClass, keyClassName, secKeyMeta, secName, + false /*doNotCreate*/, null /*priOpenState*/); } return secIndex; } @@ -503,7 +591,9 @@ * EntityStore.getSubclassIndex has not been previously called for that * class. [#15247] */ - synchronized void openSecondaryIndexes(EntityMetadata entityMeta) + synchronized void openSecondaryIndexes(Transaction txn, + EntityMetadata entityMeta, + PrimaryOpenState priOpenState) throws DatabaseException { String entityClassName = entityMeta.getClassName(); @@ -520,14 +610,16 @@ secIndexMap.get(secName); if (secIndex == null) { String keyClassName = getSecKeyClass(secKeyMeta); - /* XXX: should not require class in raw mode. */ + /* RawMode: should not require class. */ Class keyClass = SimpleCatalog.keyClassForName(keyClassName); openSecondaryIndex - (null, priIndex, entityClass, entityMeta, + (txn, priIndex, entityClass, entityMeta, keyClass, keyClassName, secKeyMeta, makeSecName - (entityClassName, secKeyMeta.getKeyName())); + (entityClassName, secKeyMeta.getKeyName()), + storeConfig.getSecondaryBulkLoad() /*doNotCreate*/, + priOpenState); } } } @@ -544,7 +636,9 @@ Class<SK> keyClass, String keyClassName, SecondaryKeyMetadata secKeyMeta, - String secName) + String secName, + boolean doNotCreate, + PrimaryOpenState priOpenState) throws DatabaseException { assert !secIndexMap.containsKey(secName); @@ -572,8 +666,33 @@ PersistKeyBinding keyBinding = getKeyBinding(keyClassName); - SecondaryDatabase db = - env.openSecondaryDatabase(txn, dbName, priDb, config); + /* + * doNotCreate is true when StoreConfig.getSecondaryBulkLoad is true + * and we are opening a secondary as a side effect of opening a + * primary, i.e., getSecondaryIndex is not being called. If + * doNotCreate is true and the database does not exist, we silently + * ignore the DatabaseNotFoundException and return null. When + * getSecondaryIndex is subsequently called, the secondary database + * will be created and populated from the primary -- a bulk load. + */ + SecondaryDatabase db; + boolean saveAllowCreate = config.getAllowCreate(); + try { + if (doNotCreate) { + config.setAllowCreate(false); + } + db = env.openSecondaryDatabase(txn, dbName, priDb, config); + } catch (DatabaseNotFoundException e) { + if (doNotCreate) { + return null; + } else { + throw e; + } + } finally { + if (doNotCreate) { + config.setAllowCreate(saveAllowCreate); + } + } SecondaryIndex<SK,PK,E2> secIndex = new SecondaryIndex (db, null, primaryIndex, keyClass, keyBinding); @@ -582,6 +701,10 @@ if (DbCompat.getDeferredWrite(config)) { deferredWriteDatabases.put(db, null); } + if (priOpenState != null) { + priOpenState.addDatabase(db); + priOpenState.addSecondaryName(secName); + } return secIndex; } @@ -953,7 +1076,8 @@ EntityMetadata meta = model.getEntityMetadata(clsName); if (meta == null) { throw new IllegalArgumentException - ("Not an entity class: " + clsName); + ("Class could not be loaded or is not an entity class: " + + clsName); } return meta; } Modified: trunk/contrib/bdb/src/com/sleepycat/persist/model/AnnotationModel.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/persist/model/AnnotationModel.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/persist/model/AnnotationModel.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: AnnotationModel.java,v 1.16.2.1 2007/02/01 14:49:57 cwl Exp $ + * $Id: AnnotationModel.java,v 1.16.2.2 2007/05/01 04:08:36 mark Exp $ */ package com.sleepycat.persist.model; @@ -86,7 +86,7 @@ if (metadata == null) { Class<?> type; try { - type = Class.forName(className); + type = EntityModel.classForName(className); } catch (ClassNotFoundException e) { return null; } @@ -320,7 +320,7 @@ /* Load superclass metadata. */ Class cls; try { - cls = Class.forName(data.getClassName()); + cls = EntityModel.classForName(data.getClassName()); } catch (ClassNotFoundException e) { throw new IllegalStateException(e); } Modified: trunk/contrib/bdb/src/com/sleepycat/persist/model/EntityModel.java =================================================================== --- trunk/contrib/bdb/src/com/sleepycat/persist/model/EntityModel.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/src/com/sleepycat/persist/model/EntityModel.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: EntityModel.java,v 1.14.2.1 2007/02/01 14:49:57 cwl Exp $ + * $Id: EntityModel.java,v 1.14.2.3 2007/06/13 23:18:02 mark Exp $ */ package com.sleepycat.persist.model; @@ -88,7 +88,8 @@ * @throws IllegalStateException if this method is called for a model that * is associated with an open store. * - * @throws IllegalArgumentException if the given class is not persistent. + * @throws IllegalArgumentException if the given class is not persistent + * or has a different class loader than previously registered classes. */ public final void registerClass(Class persistentClass) { if (catalog != null) { @@ -228,4 +229,20 @@ public final Object convertRawObject(RawObject raw) { return catalog.convertRawObject(raw, null); } + + /** + * Calls Class.forName with the current thread context class loader. This + * method should be called by entity model implementations instead of + * calling Class.forName whenever loading an application class. + */ + public static Class classForName(String className) + throws ClassNotFoundException { + + try { + return Class.forName(className, true /*initialize*/, + Thread.currentThread().getContextClassLoader()); + } catch (ClassNotFoundException e) { + return Class.forName(className); + } + } } Modified: trunk/contrib/bdb/test/com/sleepycat/collections/test/TransactionTest.java =================================================================== --- trunk/contrib/bdb/test/com/sleepycat/collections/test/TransactionTest.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/test/com/sleepycat/collections/test/TransactionTest.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,11 +3,12 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: TransactionTest.java,v 1.46.2.1 2007/02/01 14:50:03 cwl Exp $ + * $Id: TransactionTest.java,v 1.46.2.3 2007/05/23 20:27:53 linda Exp $ */ package com.sleepycat.collections.test; +import java.io.File; import java.util.Iterator; import java.util.List; import java.util.SortedSet; @@ -29,9 +30,12 @@ import com.sleepycat.je.Database; import com.sleepycat.je.DatabaseConfig; import com.sleepycat.je.DatabaseEntry; +import com.sleepycat.je.DatabaseException; import com.sleepycat.je.Environment; +import com.sleepycat.je.EnvironmentConfig; import com.sleepycat.je.Transaction; import com.sleepycat.je.TransactionConfig; +import com.sleepycat.je.util.TestUtils; import com.sleepycat.util.RuntimeExceptionWrapper; /** @@ -580,6 +584,73 @@ assertNull(currentTxn.getTransaction()); } + // <!-- begin JE only --> + + /** + * Tests that the CurrentTransaction static WeakHashMap does indeed allow + * GC to reclaim tine environment when it is closed. At one point this was + * not working because the value object in the map has a reference to the + * environment. This was fixed by wrapping the value in a WeakReference. + * [#15444] + */ + public void testCurrentTransactionGC() + throws Exception { + + /* + * This test can have indeterminate results because it depends on + * a finalize count, so it's not part of the default run. + */ + if (!TestUtils.runLongTests()) { + return; + } + + final StringBuffer finalizedFlag = new StringBuffer(); + + class MyEnv extends Environment { + + MyEnv(File home, EnvironmentConfig config) + throws DatabaseException { + + super(home, config); + } + + protected void finalize() { + finalizedFlag.append('.'); + } + } + + MyEnv myEnv = new MyEnv(env.getHome(), env.getConfig()); + CurrentTransaction myCurrTxn = CurrentTransaction.getInstance(myEnv); + + store.close(); + store = null; + map = null; + + env.close(); + env = null; + + myEnv.close(); + myEnv = null; + + myCurrTxn = null; + currentTxn = null; + + byte[] x = null; + try { + x = new byte[Integer.MAX_VALUE - 1]; + } catch (OutOfMemoryError expected) { + } + assertNull(x); + + for (int i = 0; i < 10; i += 1) { + System.gc(); + } + + assertTrue(finalizedFlag.length() > 0); + } + + // <!-- end JE only --> + private synchronized void doReadUncommitted(StoredSortedMap dirtyMap) throws Exception { Modified: trunk/contrib/bdb/test/com/sleepycat/je/DatabaseTest.java =================================================================== --- trunk/contrib/bdb/test/com/sleepycat/je/DatabaseTest.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/test/com/sleepycat/je/DatabaseTest.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: DatabaseTest.java,v 1.103.2.1 2007/02/01 14:50:04 cwl Exp $ + * $Id: DatabaseTest.java,v 1.103.2.3 2007/06/13 13:58:40 mark Exp $ */ package com.sleepycat.je; @@ -1079,6 +1079,44 @@ } /** + * Preloads a dup database to verify the bug fix for preloading dups on + * 3.2.x. [#15365] + */ + public void testPreloadDups() + throws Throwable { + + Database myDb = initEnvAndDb + (false, true /*allowDuplicates*/, false, false, null); + DatabaseEntry key = new DatabaseEntry(); + DatabaseEntry data = new DatabaseEntry(); + key.setData(TestUtils.getTestArray(0)); + data.setData(TestUtils.getTestArray(0)); + assertSame(OperationStatus.SUCCESS, + myDb.putNoDupData(null, key, data)); + data.setData(TestUtils.getTestArray(1)); + assertSame(OperationStatus.SUCCESS, + myDb.putNoDupData(null, key, data)); + + /* Close and reopen. */ + myDb.close(); + env.close(); + myDb = initEnvAndDb + (false, true /*allowDuplicates*/, false, false, null); + + /* + * Preload the entire database. Before the bug fix, an assertion would + * fire in DatabaseImpl.PreloadLSNTreeWalker.fetchLSN when processing + * the DupCountLN. + */ + PreloadConfig conf = new PreloadConfig(); + conf.setMaxBytes(100000); + myDb.preload(conf); + + myDb.close(); + env.close(); + } + + /** * Test preload(N, 0) where N > cache size (throws IllArgException). */ public void testPreloadBytesExceedsCache() @@ -1206,6 +1244,54 @@ } /** + * Check that the handle lock is not left behind when a non-transactional + * open of a primary DB fails while populating the secondary. [#15558] + */ + public void testFailedNonTxnDbOpen() + throws DatabaseException { + + EnvironmentConfig envConfig = TestUtils.initEnvConfig(); + envConfig.setAllowCreate(true); + env = new Environment(envHome, envConfig); + + DatabaseConfig priConfig = new DatabaseConfig(); + priConfig.setAllowCreate(true); + Database priDb = env.openDatabase(null, "testDB", priConfig); + + priDb.put(null, new DatabaseEntry(new byte[1]), + new DatabaseEntry(new byte[2])); + + SecondaryConfig secConfig = new SecondaryConfig(); + secConfig.setAllowCreate(true); + secConfig.setAllowPopulate(true); + /* Use priDb as foreign key DB for ease of testing. */ + secConfig.setForeignKeyDatabase(priDb); + secConfig.setKeyCreator(new SecondaryKeyCreator() { + public boolean createSecondaryKey(SecondaryDatabase secondary, + DatabaseEntry key, + DatabaseEntry data, + DatabaseEntry result) + throws DatabaseException { + result.setData + (data.getData(), data.getOffset(), data.getSize()); + return true; + } + }); + try { + env.openSecondaryDatabase(null, "testDB2", priDb, secConfig); + fail(); + } catch (DatabaseException e) { + /* Fails because [0,0] does not exist as a key in priDb. */ + assertTrue(e.toString(), + e.toString().indexOf("foreign key not allowed") > 0); + } + + priDb.close(); + env.close(); + env = null; + } + + /** * Set up the environment and db. */ private Database initEnvAndDb(boolean dontRunEvictor, Modified: trunk/contrib/bdb/test/com/sleepycat/je/EnvironmentTest.java =================================================================== --- trunk/contrib/bdb/test/com/sleepycat/je/EnvironmentTest.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/test/com/sleepycat/je/EnvironmentTest.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: EnvironmentTest.java,v 1.187.2.9 2007/04/04 14:27:33 cwl Exp $ + * $Id: EnvironmentTest.java,v 1.187.2.20 2007/08/06 16:43:22 cwl Exp $ */ package com.sleepycat.je; @@ -88,7 +88,7 @@ throws Throwable { try { - assertEquals("Checking version", "3.2.23", + assertEquals("Checking version", "3.2.42", JEVersion.CURRENT_VERSION.getVersionString()); EnvironmentConfig envConfig = TestUtils.initEnvConfig(); Modified: trunk/contrib/bdb/test/com/sleepycat/je/RunRecoveryFailureTest.java =================================================================== --- trunk/contrib/bdb/test/com/sleepycat/je/RunRecoveryFailureTest.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/test/com/sleepycat/je/RunRecoveryFailureTest.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: RunRecoveryFailureTest.java,v 1.34.2.1 2007/02/01 14:50:05 cwl Exp $ + * $Id: RunRecoveryFailureTest.java,v 1.34.2.2 2007/06/13 03:55:37 mark Exp $ */ package com.sleepycat.je; @@ -98,7 +98,12 @@ DatabaseConfig dbConfig = new DatabaseConfig(); dbConfig.setTransactional(true); dbConfig.setAllowCreate(true); - env.openDatabase(txn, "foo", dbConfig); + Database db = env.openDatabase(txn, "foo", dbConfig); + DatabaseEntry key = new DatabaseEntry(new byte[1000]); + DatabaseEntry data = new DatabaseEntry(new byte[1000]); + for (int i = 0; i < 100; i += 1) { + db.put(txn, key, data); + } env.getEnvironmentImpl().getLogManager().flush(); env.getEnvironmentImpl().getFileManager().clear(); @@ -108,16 +113,25 @@ * re-read. Should get a checksum error, which should invalidate * the environment. */ - File file = new File(envHome, "00000001" + FileManager.JE_SUFFIX); - RandomAccessFile starterFile = new RandomAccessFile(file, "rw"); - FileChannel channel = starterFile.getChannel(); - long fileSize = channel.size(); - channel.position(FileManager.firstLogEntryOffset()); - ByteBuffer junkBuffer = ByteBuffer.allocate((int) fileSize); - int written = channel.write(junkBuffer, - FileManager.firstLogEntryOffset()); - assertTrue(written > 0); - starterFile.close(); + long currentFile = DbInternal.envGetEnvironmentImpl(env) + .getFileManager() + .getCurrentFileNum(); + for (int fileNum = 0; fileNum <= currentFile; fileNum += 1) { + File file = new File + (envHome, "0000000" + fileNum + FileManager.JE_SUFFIX); + RandomAccessFile starterFile = + new RandomAccessFile(file, "rw"); + FileChannel channel = starterFile.getChannel(); + long fileSize = channel.size(); + if (fileSize > FileManager.firstLogEntryOffset()) { + ByteBuffer junkBuffer = ByteBuffer.allocate + ((int) fileSize - FileManager.firstLogEntryOffset()); + int written = channel.write + (junkBuffer, FileManager.firstLogEntryOffset()); + assertTrue(written > 0); + starterFile.close(); + } + } try { txn.abort(); Modified: trunk/contrib/bdb/test/com/sleepycat/je/cleaner/CleanerTest.java =================================================================== --- trunk/contrib/bdb/test/com/sleepycat/je/cleaner/CleanerTest.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/test/com/sleepycat/je/cleaner/CleanerTest.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: CleanerTest.java,v 1.87.2.1 2007/02/01 14:50:06 cwl Exp $ + * $Id: CleanerTest.java,v 1.87.2.3 2007/05/31 21:55:33 mark Exp $ */ package com.sleepycat.je.cleaner; @@ -477,6 +477,45 @@ } /** + * Tests that when a file being cleaned is deleted, we ignore the error and + * don't repeatedly try to clean it. This is happening when we mistakedly + * clean a file after it has been queued for deletion. The workaround is + * to catch LogFileNotFoundException in the cleaner and ignore the error. + * We're testing the workaround here by forcing cleaning of deleted files. + * [#15528] + */ + public void testUnexpectedFileDeletion() + throws DatabaseException, IOException { + + initEnv(true, false); + EnvironmentMutableConfig config = exampleEnv.getMutableConfig(); + config.setConfigParam + (EnvironmentParams.ENV_RUN_CLEANER.getName(), "true"); + config.setConfigParam + (EnvironmentParams.CLEANER_MIN_UTILIZATION.getName(), "80"); + exampleEnv.setMutableConfig(config); + + final EnvironmentImpl envImpl = + DbInternal.envGetEnvironmentImpl(exampleEnv); + final Cleaner cleaner = envImpl.getCleaner(); + + Map expectedMap = new HashMap(); + doLargePut(expectedMap, 1000, 1, true); + checkData(expectedMap); + + for (int i = 0; i < 100; i += 1) { + modifyData(expectedMap, 1, true); + checkData(expectedMap); + cleaner.injectFileForCleaning(new Long(0)); + exampleEnv.cleanLog(); + exampleEnv.checkpoint(forceConfig); + } + checkData(expectedMap); + + closeEnv(); + } + + /** * Helper routine. Generates keys with random alpha values while data * is numbered numerically. */ @@ -577,6 +616,7 @@ txn.abort(); } } + /** * Delete data. */ @@ -854,4 +894,108 @@ return count; } + + /** + * Checks that the memory budget is updated properly by the + * UtilizationTracker. Prior to a bug fix [#15505] amounts were added to + * the budget but not subtraced when two TrackedFileSummary objects were + * merged. Merging occurs when a local tracker is added to the global + * tracker. Local trackers are used during recovery, checkpoints, lazy + * compression, and reverse splits. + */ + public void testTrackerMemoryBudget() + throws DatabaseException { + + /* Open environmnet. */ + EnvironmentConfig envConfig = TestUtils.initEnvConfig(); + envConfig.setAllowCreate(true); + envConfig.setTransactional(true); + envConfig.setConfigParam + (EnvironmentParams.ENV_CHECK_LEAKS.getName(), "false"); + envConfig.setConfigParam + (EnvironmentParams.ENV_RUN_CLEANER.getName(), "false"); + envConfig.setConfigParam + (EnvironmentParams.ENV_RUN_INCOMPRESSOR.getName(), "false"); + exampleEnv = new Environment(envHome, envConfig); + EnvironmentImpl envImpl = DbInternal.envGetEnvironmentImpl(exampleEnv); + MemoryBudget budget = envImpl.getMemoryBudget(); + + /* Open database. */ + DatabaseConfig dbConfig = new DatabaseConfig(); + dbConfig.setTransactional(true); + dbConfig.setAllowCreate(true); + exampleDb = exampleEnv.openDatabase(null, "foo", dbConfig); + + /* Insert data. */ + DatabaseEntry key = new DatabaseEntry(); + DatabaseEntry data = new DatabaseEntry(); + for (int i = 1; i <= 200; i += 1) { + IntegerBinding.intToEntry(i, key); + IntegerBinding.intToEntry(i, data); + exampleDb.put(null, key, data); + } + + /* Sav the misc budget baseline. */ + flushTrackedFiles(); + long misc = budget.getMiscMemoryUsage(); + + /* + * Nothing becomes obsolete when inserting and no INs are logged, so + * the budget does not increase. + */ + IntegerBinding.intToEntry(201, key); + exampleDb.put(null, key, data); + assertEquals(misc, budget.getMiscMemoryUsage()); + flushTrackedFiles(); + assertEquals(misc, budget.getMiscMemoryUsage()); + + /* + * Update a record and expect the budget to increase because the old + * LN becomes obsolete. + */ + exampleDb.put(null, key, data); + assertTrue(misc < budget.getMiscMemoryUsage()); + flushTrackedFiles(); + assertEquals(misc, budget.getMiscMemoryUsage()); + + /* + * Delete all records and expect the budget to increase because LNs + * become obsolete. + */ + for (int i = 1; i <= 201; i += 1) { + IntegerBinding.intToEntry(i, key); + exampleDb.delete(null, key); + } + assertTrue(misc < budget.getMiscMemoryUsage()); + flushTrackedFiles(); + assertEquals(misc, budget.getMiscMemoryUsage()); + + /* + * Compress and expect no change to the budget. Prior to the fix for + * [#15505] the assertion below failed because the baseline misc budget + * was not restored. + */ + exampleEnv.compress(); + flushTrackedFiles(); + assertEquals(misc, budget.getMiscMemoryUsage()); + + closeEnv(); + } + + /** + * Flushes all tracked files to subtract tracked info from the misc memory + * budget. + */ + private void flushTrackedFiles() + throws DatabaseException { + + EnvironmentImpl envImpl = DbInternal.envGetEnvironmentImpl(exampleEnv); + UtilizationTracker tracker = envImpl.getUtilizationTracker(); + UtilizationProfile profile = envImpl.getUtilizationProfile(); + + TrackedFileSummary[] files = tracker.getTrackedFiles(); + for (int i = 0; i < files.length; i += 1) { + profile.flushFileSummary(files[i]); + } + } } Modified: trunk/contrib/bdb/test/com/sleepycat/je/cleaner/TruncateAndRemoveTest.java =================================================================== --- trunk/contrib/bdb/test/com/sleepycat/je/cleaner/TruncateAndRemoveTest.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/test/com/sleepycat/je/cleaner/TruncateAndRemoveTest.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: TruncateAndRemoveTest.java,v 1.18.2.1 2007/02/01 14:50:06 cwl Exp $ + * $Id: TruncateAndRemoveTest.java,v 1.18.2.2 2007/07/02 19:54:54 mark Exp $ */ package com.sleepycat.je.cleaner; @@ -60,8 +60,10 @@ private File envHome; private Environment env; private Database db; + private DatabaseImpl dbImpl; private JUnitThread junitThread; private boolean fetchObsoleteSize; + private boolean dbEviction; public TruncateAndRemoveTest() { envHome = new File(System.getProperty(TestUtils.DEST_DIR)); @@ -103,6 +105,7 @@ } db = null; + dbImpl = null; env = null; envHome = null; } @@ -145,6 +148,10 @@ } env = new Environment(envHome, config); + + config = env.getConfig(); + dbEviction = config.getConfigParam + (EnvironmentParams.ENV_DB_EVICTION.getName()).equals("true"); } /** @@ -158,18 +165,30 @@ dbConfig.setTransactional(envConfig.getTransactional()); dbConfig.setAllowCreate(true); db = env.openDatabase(useTxn, dbName, dbConfig); + dbImpl = DbInternal.dbGetDatabaseImpl(db); } /** - * Closes the environment and database. + * Closes the database. */ - private void closeEnv() + private void closeDb() throws DatabaseException { if (db != null) { db.close(); db = null; + dbImpl = null; } + } + + /** + * Closes the environment and database. + */ + private void closeEnv() + throws DatabaseException { + + closeDb(); + if (env != null) { env.close(); env = null; @@ -185,15 +204,22 @@ openEnv(true); openDb(null, DB_NAME1); writeAndCountRecords(null, RECORD_COUNT); - DatabaseId saveId = DbInternal.dbGetDatabaseImpl(db).getId(); - db.close(); - db = null; + DatabaseImpl saveDb = dbImpl; + DatabaseId saveId = dbImpl.getId(); + closeDb(); Transaction txn = env.beginTransaction(null, null); truncate(txn, true); ObsoleteCounts beforeCommit = getObsoleteCounts(); txn.commit(); + /* Make sure use count is decremented when we commit. */ + assertDbInUse(saveDb, false); + openDb(null, DB_NAME1); + saveDb = dbImpl; + closeDb(); + assertDbInUse(saveDb, false); + verifyUtilization(beforeCommit, RECORD_COUNT + // LNs 3, // prev MapLN + deleted MapLN + prev NameLN @@ -212,14 +238,21 @@ openEnv(true); openDb(null, DB_NAME1); writeAndCountRecords(null, RECORD_COUNT); - db.close(); - db = null; + DatabaseImpl saveDb = dbImpl; + closeDb(); Transaction txn = env.beginTransaction(null, null); truncate(txn, true); ObsoleteCounts beforeAbort = getObsoleteCounts(); txn.abort(); + /* Make sure use count is decremented when we abort. */ + assertDbInUse(saveDb, false); + openDb(null, DB_NAME1); + saveDb = dbImpl; + closeDb(); + assertDbInUse(saveDb, false); + /* * The obsolete count should include the records inserted after * the truncate. @@ -244,8 +277,7 @@ openEnv(true); openDb(null, DB_NAME1); writeAndCountRecords(null, RECORD_COUNT); - db.close(); - db = null; + closeDb(); Transaction txn = env.beginTransaction(null, null); truncate(txn, true); @@ -253,12 +285,19 @@ /* populate the database with some more records. */ openDb(txn, DB_NAME1); writeAndCountRecords(txn, RECORD_COUNT/4); - DatabaseId saveId = DbInternal.dbGetDatabaseImpl(db).getId(); - db.close(); - db = null; + DatabaseImpl saveDb = dbImpl; + DatabaseId saveId = dbImpl.getId(); + closeDb(); ObsoleteCounts beforeAbort = getObsoleteCounts(); txn.abort(); + /* Make sure use count is decremented when we abort. */ + assertDbInUse(saveDb, false); + openDb(null, DB_NAME1); + saveDb = dbImpl; + closeDb(); + assertDbInUse(saveDb, false); + /* * The obsolete count should include the records inserted after * the truncate. @@ -286,15 +325,18 @@ openEnv(true); openDb(null, DB_NAME1); writeAndCountRecords(null, RECORD_COUNT); - DatabaseId saveId = DbInternal.dbGetDatabaseImpl(db).getId(); - db.close(); - db = null; + DatabaseImpl saveDb = dbImpl; + DatabaseId saveId = dbImpl.getId(); + closeDb(); Transaction txn = env.beginTransaction(null, null); env.removeDatabase(txn, DB_NAME1); ObsoleteCounts beforeCommit = getObsoleteCounts(); txn.commit(); + /* Make sure use count is decremented when we commit. */ + assertDbInUse(saveDb, false); + verifyUtilization(beforeCommit, /* LNs + old NameLN, old MapLN, delete MapLN */ RECORD_COUNT + 3, @@ -316,12 +358,15 @@ openEnv(false); openDb(null, DB_NAME1); writeAndCountRecords(null, RECORD_COUNT); - DatabaseId saveId = DbInternal.dbGetDatabaseImpl(db).getId(); - db.close(); - db = null; + DatabaseImpl saveDb = dbImpl; + DatabaseId saveId = dbImpl.getId(); + closeDb(); ObsoleteCounts beforeOperation = getObsoleteCounts(); env.removeDatabase(null, DB_NAME1); + /* Make sure use count is decremented. */ + assertDbInUse(saveDb, false); + verifyUtilization(beforeOperation, /* LNs + new NameLN, old NameLN, old MapLN, delete MapLN */ @@ -345,13 +390,16 @@ openEnv(true); openDb(null, DB_NAME1); writeAndCountRecords(null, RECORD_COUNT); - db.close(); - db = null; + DatabaseImpl saveDb = dbImpl; + closeDb(); Transaction txn = env.beginTransaction(null, null); env.removeDatabase(txn, DB_NAME1); ObsoleteCounts beforeAbort = getObsoleteCounts(); txn.abort(); + /* Make sure use count is decremented when we abort. */ + assertDbInUse(saveDb, false); + verifyUtilization(beforeAbort, 0, 0); /* All records should be there. */ @@ -400,10 +448,7 @@ openDb(null, DB_NAME1); writeAndCountRecords(null, RECORD_COUNT); DatabaseId saveId = DbInternal.dbGetDatabaseImpl(db).getId(); - db.close(); - db = null; - env.close(); - env = null; + closeEnv(); /* * Open the environment and remove the database. The @@ -453,10 +498,7 @@ openDb(null, DB_NAME1); writeAndCountRecords(null, RECORD_COUNT); DatabaseId saveId = DbInternal.dbGetDatabaseImpl(db).getId(); - db.close(); - db = null; - env.close(); - env = null; + closeEnv(); /* * Open the environment and remove the database. Pull 1 BIN in. @@ -468,14 +510,17 @@ c.getFirst(new DatabaseEntry(), new DatabaseEntry(), LockMode.DEFAULT)); c.close(); - db.close(); - db = null; + DatabaseImpl saveDb = dbImpl; + closeDb(); Transaction txn = env.beginTransaction(null, null); env.removeDatabase(txn, DB_NAME1); ObsoleteCounts beforeCommit = getObsoleteCounts(); txn.commit(); + /* Make sure use count is decremented when we commit. */ + assertDbInUse(saveDb, false); + verifyUtilization(beforeCommit, /* LNs + old NameLN, old MapLN, delete MapLN */ RECORD_COUNT + 3, @@ -527,11 +572,11 @@ long remainingRecordCount = deleteAll ? 0 : recordCount; env.checkpoint(FORCE_CHECKPOINT); ObsoleteCounts obsoleteCounts = getObsoleteCounts(); - DatabaseImpl dbImpl = DbInternal.dbGetDatabaseImpl(db); - db.close(); - db = null; - assertTrue(!dbImpl.isDeleteFinished()); - assertTrue(!dbImpl.isDeleted()); + DatabaseImpl saveDb = dbImpl; + closeDb(); + assertTrue(!saveDb.isDeleteFinished()); + assertTrue(!saveDb.isDeleted()); + assertDbInUse(saveDb, false); /* Make sure that we wrote a full file's worth of LNs. */ assertTrue(logFiles.size() >= 3); @@ -562,7 +607,7 @@ */ final Object lock = new Object(); - dbImpl.setPendingDeletedHook(new TestHook() { + saveDb.setPendingDeletedHook(new TestHook() { public void doIOHook() throws IOException { throw new UnsupportedOperationException(); @@ -588,8 +633,9 @@ junitThread.start(); lock.wait(); } - assertTrue(!dbImpl.isDeleteFinished()); - assertTrue(dbImpl.isDeleted()); + assertTrue(!saveDb.isDeleteFinished()); + assertTrue(saveDb.isDeleted()); + assertDbInUse(saveDb, true); /* Expect obsolete LNs: NameLN */ obsoleteCounts = verifyUtilization(obsoleteCounts, 1, 0); @@ -616,8 +662,9 @@ e.printStackTrace(); fail(e.toString()); } - assertTrue(dbImpl.isDeleteFinished()); - assertTrue(dbImpl.isDeleted()); + assertTrue(saveDb.isDeleteFinished()); + assertTrue(saveDb.isDeleted()); + assertDbInUse(saveDb, false); /* Expect obsolete LNs: recordCount + MapLN + FSLNs (apprx). */ verifyUtilization(obsoleteCounts, remainingRecordCount + 6, 0); @@ -724,21 +771,26 @@ DatabaseEntry key = new DatabaseEntry(); DatabaseEntry data = new DatabaseEntry(); + boolean opened = false; if (db == null) { openDb(useTxn, DB_NAME1); + opened = true; } Cursor cursor = db.openCursor(useTxn, null); + int count = 0; try { - int count = 0; OperationStatus status = cursor.getFirst(key, data, null); while (status == OperationStatus.SUCCESS) { count += 1; status = cursor.getNext(key, data, null); } - return count; } finally { cursor.close(); } + if (opened) { + closeDb(); + } + return count; } /** @@ -850,6 +902,17 @@ } /** + * Checks whether a given DB has a non-zero use count. Does nothing if + * je.dbEviction is not enabled, since reference counts are only maintained + * if that config parameter is enabled. + */ + private void assertDbInUse(DatabaseImpl db, boolean inUse) { + if (dbEviction) { + assertEquals(inUse, db.isInUse()); + } + } + + /** * Returns true if all files exist, or false if any file is deleted. */ private boolean logFilesExist(Set fileNumbers) { @@ -889,8 +952,7 @@ writeAndCountRecords(null, RECORD_COUNT * 3); env.checkpoint(force); - db.close(); - db = null; + closeDb(); /* Check log files, there should be entries with this database. */ CheckReader checker = new CheckReader(env, dbId, true); Modified: trunk/contrib/bdb/test/com/sleepycat/je/dbi/DbCursorDuplicateTest.java =================================================================== --- trunk/contrib/bdb/test/com/sleepycat/je/dbi/DbCursorDuplicateTest.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/test/com/sleepycat/je/dbi/DbCursorDuplicateTest.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: DbCursorDuplicateTest.java,v 1.53.2.1 2007/02/01 14:50:09 cwl Exp $ + * $Id: DbCursorDuplicateTest.java,v 1.53.2.2 2007/05/23 14:07:30 mark Exp $ */ package com.sleepycat.je.dbi; @@ -306,8 +306,8 @@ (InvocationCountingBtreeComparator) (exampleDb.getConfig().getDuplicateComparator()); - assertTrue(bTreeICC.getInvocationCount() == 1); - assertTrue(dupICC.getInvocationCount() == 2); + assertEquals(1, bTreeICC.getInvocationCount()); + assertEquals(1, dupICC.getInvocationCount()); } catch (Throwable t) { t.printStackTrace(); throw t; @@ -434,7 +434,12 @@ } } - public void testDuplicateReplacementWithComparisonFunction() + /** + * When using a duplicate comparator that does not compare all bytes, + * attempting to change the data for a duplicate data item should cause an + * error even if a byte not compared is changed. [#15527] + */ + public void testDuplicateReplacementFailureWithComparisonFunction1() throws Throwable { try { @@ -464,23 +469,29 @@ StringDbt dataDbt = new StringDbt(); StringBuffer sb = new StringBuffer(foundData); - sb.replace(3, 3, "3"); - sb.setLength(4); + sb.replace(3, 4, "3"); dataDbt.setString(sb.toString()); - assertEquals(OperationStatus.SUCCESS, - cursor.putCurrent(dataDbt)); + try { + cursor.putCurrent(dataDbt); + fail("didn't catch DatabaseException"); + } catch (DatabaseException DBE) { + } } }; dw.setIgnoreDataMap(true); dw.walkData(); - assertTrue(dw.nEntries == 2); } catch (Throwable t) { t.printStackTrace(); throw t; } } - public void testDuplicateReplacementFailureWithComparisonFunction() + /** + * When using a duplicate comparator that compares all bytes, attempting to + * change the data for a duplicate data item should cause an error. + * [#15527] + */ + public void testDuplicateReplacementFailureWithComparisonFunction2() throws Throwable { try { @@ -934,7 +945,7 @@ */ private static Comparator truncatedComparator = new TruncatedComparator(); - protected static class TruncatedComparator implements Comparator { + private static class TruncatedComparator implements Comparator { protected TruncatedComparator() { } Modified: trunk/contrib/bdb/test/com/sleepycat/je/dbi/DbCursorTestBase.java =================================================================== --- trunk/contrib/bdb/test/com/sleepycat/je/dbi/DbCursorTestBase.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/test/com/sleepycat/je/dbi/DbCursorTestBase.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: DbCursorTestBase.java,v 1.94.2.1 2007/02/01 14:50:10 cwl Exp $ + * $Id: DbCursorTestBase.java,v 1.94.2.2 2007/05/23 14:07:30 mark Exp $ */ package com.sleepycat.je.dbi; @@ -287,12 +287,12 @@ if (!ignoreDataMap) { if (dataMap.get(foundKeyString) != null) { - assertTrue(((String) dataMap.get(foundKeyString)). - equals(foundDataString)); + assertEquals(dataMap.get(foundKeyString), + foundDataString); } else if (addedDataMap != null && addedDataMap.get(foundKeyString) != null) { - assertTrue(((String) addedDataMap.get(foundKeyString)). - equals(foundDataString)); + assertEquals(addedDataMap.get(foundKeyString), + foundDataString); } else { fail("didn't find key in either map (" + foundKeyString + Modified: trunk/contrib/bdb/test/com/sleepycat/je/evictor/EvictActionTest.java =================================================================== --- trunk/contrib/bdb/test/com/sleepycat/je/evictor/EvictActionTest.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/test/com/sleepycat/je/evictor/EvictActionTest.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: EvictActionTest.java,v 1.24.2.1 2007/02/01 14:50:11 cwl Exp $ + * $Id: EvictActionTest.java,v 1.24.2.2 2007/07/02 19:54:55 mark Exp $ */ package com.sleepycat.je.evictor; @@ -25,7 +25,10 @@ import com.sleepycat.je.EnvironmentMutableConfig; import com.sleepycat.je.LockMode; import com.sleepycat.je.OperationStatus; +import com.sleepycat.je.Transaction; import com.sleepycat.je.config.EnvironmentParams; +import com.sleepycat.je.dbi.DatabaseImpl; +import com.sleepycat.je.dbi.DbTree; import com.sleepycat.je.dbi.EnvironmentImpl; import com.sleepycat.je.dbi.MemoryBudget; import com.sleepycat.je.junit.JUnitThread; @@ -49,6 +52,8 @@ private File envHome = null; private Environment env = null; private Database db = null; + private int actualLNs = 0; + private int actualINs = 0; public EvictActionTest() { envHome = new File(System.getProperty(TestUtils.DEST_DIR)); @@ -66,8 +71,6 @@ public void tearDown() throws Exception { - TestUtils.removeLogFiles("TearDown", envHome, false); - if (env != null) { try { env.close(); @@ -76,6 +79,12 @@ } } + try { + TestUtils.removeLogFiles("TearDown", envHome, false); + } catch (Throwable e) { + System.out.println("tearDown: " + e); + } + envHome = null; env = null; db = null; @@ -258,6 +267,271 @@ } /** + * We now allow eviction of the root IN of a DB, whether the DB is closed + * or not. Check that basic root eviction works. [#13415] + */ + public void testRootINEviction() + throws DatabaseException { + + DatabaseEntry entry = new DatabaseEntry(new byte[1]); + OperationStatus status; + + openEnv(80, SMALL_CACHE_SIZE); + + DatabaseConfig dbConfig = new DatabaseConfig(); + dbConfig.setAllowCreate(true); + Database db1 = env.openDatabase(null, "db1", dbConfig); + + /* Root starts out null. */ + assertTrue(!isRootResident(db1)); + /* It is created when we insert the first record. */ + status = db1.put(null, entry, entry); + assertSame(OperationStatus.SUCCESS, status); + assertTrue(isRootResident(db1)); + /* It is evicted when necessary. */ + forceEviction(); + assertTrue(!isRootResident(db1)); + /* And fetched again when needed. */ + status = db1.get(null, entry, entry, null); + assertSame(OperationStatus.SUCCESS, status); + assertTrue(isRootResident(db1)); + + /* Deferred write DBs have special rules. */ + dbConfig.setDeferredWrite(true); + Database db2 = env.openDatabase(null, "db2", dbConfig); + status = db2.put(null, entry, entry); + assertSame(OperationStatus.SUCCESS, status); + assertTrue(isRootResident(db2)); + /* Root eviction is disallowed if the root is dirty. */ + forceEviction(); + assertTrue(isRootResident(db2)); + db2.sync(); + forceEviction(); + assertTrue(!isRootResident(db2)); + + db2.close(); + db1.close(); + closeEnv(); + } + + /** + * We now allow eviction of the MapLN and higher level INs in the DB mappng + * tree when DBs are closed. Check that basic mapping tree IN eviction + * works. [#13415] + */ + public void testMappingTreeEviction() + throws DatabaseException { + + DatabaseConfig dbConfig = new DatabaseConfig(); + dbConfig.setAllowCreate(true); + + DatabaseEntry entry = new DatabaseEntry(new byte[1]); + OperationStatus status; + + openEnv(80, SMALL_CACHE_SIZE); + + /* Baseline mappng tree LNs and INs. */ + final int baseLNs = 2; // Utilization DB and test DB + final int baseINs = 2; // Root IN and BIN + checkMappingTree(baseLNs, baseINs); + forceEviction(); + checkMappingTree(baseLNs, baseINs); + + /* + * Create enough DBs to fill up a BIN in the mapping DB. NODE_MAX is + * configured to be 4 in this test. There are already 2 DBs open. + */ + final int nDbs = 4; + Database[] dbs = new Database[nDbs]; + for (int i = 0; i < nDbs; i += 1) { + dbs[i] = env.openDatabase(null, "db" + i, dbConfig); + status = dbs[i].put(null, entry, entry); + assertSame(OperationStatus.SUCCESS, status); + assertTrue(isRootResident(dbs[i])); + } + final int openLNs = baseLNs + nDbs; // Add 1 MapLN per open DB + final int openINs = baseINs + 1; // Add 1 BIN in the mapping tree + checkMappingTree(openLNs, openINs); + forceEviction(); + checkMappingTree(openLNs, openINs); + + /* Close DBs and force eviction. */ + for (int i = 0; i < nDbs; i += 1) { + dbs[i].close(); + } + forceEviction(); + checkMappingTree(baseLNs, baseINs); + + /* Re-open the DBs, opening each DB twice. */ + Database[] dbs2 = new Database[nDbs]; + for (int i = 0; i < nDbs; i += 1) { + dbs[i] = env.openDatabase(null, "db" + i, dbConfig); + dbs2[i] = env.openDatabase(null, "db" + i, dbConfig); + } + checkMappingTree(openLNs, openINs); + forceEviction(); + checkMappingTree(openLNs, openINs); + + /* Close one handle only, MapLN eviction should not occur. */ + for (int i = 0; i < nDbs; i += 1) { + dbs[i].close(); + } + forceEviction(); + checkMappingTree(openLNs, openINs); + + /* Close the other handles, eviction should occur. */ + for (int i = 0; i < nDbs; i += 1) { + dbs2[i].close(); + } + forceEviction(); + checkMappingTree(baseLNs, baseINs); + + closeEnv(); + } + + /** + * Check that opening a database in a transaction and then aborting the + * transaction will decrement the database use count. [#13415] + */ + public void testAbortOpen() + throws DatabaseException { + + EnvironmentConfig envConfig = TestUtils.initEnvConfig(); + envConfig.setAllowCreate(true); + envConfig.setTransactional(true); + envConfig.setConfigParam(EnvironmentParams. + ENV_DB_EVICTION.getName(), "true"); + env = new Environment(envHome, envConfig); + + /* Abort the txn used to open a database. */ + Transaction txn = env.beginTransaction(null, null); + DatabaseConfig dbConfig = new DatabaseConfig(); + dbConfig.setAllowCreate(true); + dbConfig.setTransactional(true); + Database db1 = env.openDatabase(txn, "db1", dbConfig); + DatabaseImpl saveDb = DbInternal.dbGetDatabaseImpl(db1); + txn.abort(); + + /* DB should not be in use and does not have to be closed. */ + assertEquals(false, saveDb.isInUse()); + + /* + * Environment.close will not throw an exception, even though the DB + * has not been closed. The abort took care of cleaning up the handle. + */ + closeEnv(); + + /* + * Try a non-transactional DB open that throws an exception because we + * create it exclusively and it already exists. The use count should + * be decremented. + */ + env = new Environment(envHome, envConfig); + dbConfig.setAllowCreate(true); + dbConfig.setExclusiveCreate(true); + dbConfig.setTransactional(false); + db1 = env.openDatabase(null, "db1", dbConfig); + saveDb = DbInternal.dbGetDatabaseImpl(db1); + try { + env.openDatabase(null, "db1", dbConfig); + fail(); + } catch (DatabaseException e) { + assertTrue(e.getMessage().indexOf("already exists") >= 0); + } + db1.close(); + assertEquals(false, saveDb.isInUse()); + + /* + * Try a non-transactional DB open that throws an exception because we + * change the duplicatesAllowed setting. The use count should be + * decremented. + */ + dbConfig.setSortedDuplicates(true); + dbConfig.setExclusiveCreate(false); + try { + env.openDatabase(null, "db1", dbConfig); + fail(); + } catch (DatabaseException e) { + assertTrue(e.getMessage().indexOf("duplicatesAllowed") >= 0); + } + assertEquals(false, saveDb.isInUse()); + + closeEnv(); + } + + /** + * Check for the expected number of nodes in the mapping DB. + */ + private void checkMappingTree(int expectLNs, int expectINs) + throws DatabaseException { + + IN root = DbInternal.envGetEnvironmentImpl(env) + .getDbMapTree() + .getDb(DbTree.ID_DB_ID) + .getTree() + .getRootIN(false); + actualLNs = 0; + actualINs = 0; + countMappingTree(root); + root.releaseLatch(); + assertEquals("LNs", expectLNs, actualLNs); + assertEquals("INs", expectINs, actualINs); + } + + private void countMappingTree(IN parent) { + actualINs += 1; + for (int i = 0; i < parent.getNEntries(); i += 1) { + if (parent.getTarget(i) != null) { + if (parent.getTarget(i) instanceof IN) { + countMappingTree((IN) parent.getTarget(i)); + } else { + actualLNs += 1; + } + } + } + } + + /** + * Returns whether the root IN is currently resident for the given DB. + */ + private boolean isRootResident(Database dbParam) { + return DbInternal.dbGetDatabaseImpl(dbParam) + .getTree() + .isRootResident(); + } + + /** + * Force eviction by inserting a large record in the pre-opened DB. + */ + private void forceEviction() + throws DatabaseException { + + EnvironmentImpl envImpl = DbInternal.envGetEnvironmentImpl(env); + MemoryBudget mb = envImpl.getMemoryBudget(); + OperationStatus status; + + /* + * Repeat twice to cause a 2nd pass over the INList. The second pass + * evicts BINs that were only stripped of LNs in the first pass. + */ + for (int i = 0; i < 2; i += 1) { + status = db.put(null, new DatabaseEntry(new byte[1]), + new DatabaseEntry(new byte[BIG_CACHE_SIZE])); + assertSame(OperationStatus.SUCCESS, status); + + long preEvictMem = mb.getCacheMemoryUsage(); + env.evictMemory(); + long postEvictMem = mb.getCacheMemoryUsage(); + assertTrue(preEvictMem > postEvictMem); + + status = db.delete(null, new DatabaseEntry(new byte[1])); + assertSame(OperationStatus.SUCCESS, status); + } + + TestUtils.validateNodeMemUsage(envImpl, true); + } + + /** * Open an environment and database. */ private void openEnv(int floor, @@ -292,6 +566,9 @@ EnvironmentParams.LOG_MEM_SIZE_MIN_STRING); envConfig.setConfigParam(EnvironmentParams.NUM_LOG_BUFFERS.getName(), "2"); + /* Enable DB (MapLN) eviction for eviction tests. */ + envConfig.setConfigParam(EnvironmentParams. + ENV_DB_EVICTION.getName(), "true"); /* * Disable critical eviction, we want to test under controlled @@ -318,10 +595,14 @@ private void closeEnv() throws DatabaseException { - db.close(); - db = null; - env.close(); - env = null; + if (db != null) { + db.close(); + db = null; + } + if (env != null) { + env.close(); + env = null; + } } private void insertData(int nKeys) Modified: trunk/contrib/bdb/test/com/sleepycat/je/evictor/EvictSelectionTest.java =================================================================== --- trunk/contrib/bdb/test/com/sleepycat/je/evictor/EvictSelectionTest.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/test/com/sleepycat/je/evictor/EvictSelectionTest.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: EvictSelectionTest.java,v 1.14.2.1 2007/02/01 14:50:11 cwl Exp $ + * $Id: EvictSelectionTest.java,v 1.14.2.2 2007/07/02 19:54:55 mark Exp $ */ package com.sleepycat.je.evictor; @@ -239,10 +239,6 @@ continue; } - if (in.getDatabase().getId().equals(DbTree.ID_DB_ID)) { - continue; - } - int evictType = in.getEvictionType(); if (evictType == IN.MAY_NOT_EVICT) { continue; Modified: trunk/contrib/bdb/test/com/sleepycat/je/test/DeferredWriteTest.java =================================================================== --- trunk/contrib/bdb/test/com/sleepycat/je/test/DeferredWriteTest.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/test/com/sleepycat/je/test/DeferredWriteTest.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: DeferredWriteTest.java,v 1.5.2.2 2007/02/01 14:50:19 cwl Exp $ + * $Id: DeferredWriteTest.java,v 1.5.2.3 2007/05/01 19:32:05 mark Exp $ */ package com.sleepycat.je.test; @@ -379,6 +379,90 @@ } } + /** + * Performs a basic check of deferred-write w/duplicates for verifying the + * fix to duplicate logging on 3.2.x. [#15365] + */ + public void testDups() + throws DatabaseException { + + EnvironmentConfig envConfig = getEnvConfig(false); + env = new Environment(envHome, envConfig); + + DatabaseConfig dbConfig = new DatabaseConfig(); + dbConfig.setAllowCreate(true); + dbConfig.setDeferredWrite(true); + dbConfig.setSortedDuplicates(true); + Database db = env.openDatabase(null, DBNAME, dbConfig); + + /* Insert {0,0} and {0,1}. */ + DatabaseEntry key = new DatabaseEntry(); + DatabaseEntry data = new DatabaseEntry(); + IntegerBinding.intToEntry(9, key); + IntegerBinding.intToEntry(0, data); + assertSame(OperationStatus.SUCCESS, + db.putNoDupData(null, key, data)); + IntegerBinding.intToEntry(1, data); + assertSame(OperationStatus.SUCCESS, + db.putNoDupData(null, key, data)); + + /* Check that both exist. */ + Cursor c = db.openCursor(null, null); + try { + assertSame(OperationStatus.SUCCESS, + c.getNext(key, data, LockMode.DEFAULT)); + assertEquals(9, IntegerBinding.entryToInt(key)); + assertEquals(0, IntegerBinding.entryToInt(data)); + + assertSame(OperationStatus.SUCCESS, + c.getNext(key, data, LockMode.DEFAULT)); + assertEquals(9, IntegerBinding.entryToInt(key)); + assertEquals(1, IntegerBinding.entryToInt(data)); + + assertSame(OperationStatus.NOTFOUND, + c.getNext(key, data, LockMode.DEFAULT)); + } finally { + c.close(); + } + + /* Close without a checkpoint to redo the LNs during recovery. */ + db.sync(); + db.close(); + DbInternal.envGetEnvironmentImpl(env).close(false); + env = null; + + /* Recover and check again. */ + env = new Environment(envHome, envConfig); + db = env.openDatabase(null, DBNAME, dbConfig); + c = db.openCursor(null, null); + try { + assertSame(OperationStatus.SUCCESS, + c.getNext(key, data, LockMode.DEFAULT)); + + /* + * Before fixing the problem with deferred-write duplicate logging, + * the key read below was 0 instead of 9. The bug was that the + * data (0) was being logged as the main tree key. + */ + assertEquals(9, IntegerBinding.entryToInt(key)); + assertEquals(0, IntegerBinding.entryToInt(data)); + + assertSame(OperationStatus.SUCCESS, + c.getNext(key, data, LockMode.DEFAULT)); + assertEquals(9, IntegerBinding.entryToInt(key)); + assertEquals(1, IntegerBinding.entryToInt(data)); + + assertSame(OperationStatus.NOTFOUND, + c.getNext(key, data, LockMode.DEFAULT)); + } finally { + c.close(); + } + + db.close(); + env.close(); + env = null; + } + public void testPreloadNoSync() throws DatabaseException { Modified: trunk/contrib/bdb/test/com/sleepycat/je/test/SecondaryTest.java =================================================================== --- trunk/contrib/bdb/test/com/sleepycat/je/test/SecondaryTest.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/test/com/sleepycat/je/test/SecondaryTest.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: SecondaryTest.java,v 1.38.2.1 2007/02/01 14:50:20 cwl Exp $ + * $Id: SecondaryTest.java,v 1.38.2.2 2007/06/13 21:22:18 mark Exp $ */ package com.sleepycat.je.test; @@ -18,6 +18,7 @@ import com.sleepycat.je.DatabaseConfig; import com.sleepycat.je.DatabaseEntry; import com.sleepycat.je.DatabaseException; +import com.sleepycat.je.DeadlockException; import com.sleepycat.je.EnvironmentConfig; import com.sleepycat.je.LockMode; import com.sleepycat.je.OperationStatus; @@ -27,6 +28,7 @@ import com.sleepycat.je.SecondaryKeyCreator; import com.sleepycat.je.Transaction; import com.sleepycat.je.config.EnvironmentParams; +import com.sleepycat.je.junit.JUnitThread; import com.sleepycat.je.util.TestUtils; public class SecondaryTest extends MultiKeyTxnTestCase { @@ -34,6 +36,8 @@ private static final int NUM_RECS = 5; private static final int KEY_OFFSET = 100; + private JUnitThread junitThread; + private static EnvironmentConfig envConfig = TestUtils.initEnvConfig(); static { envConfig.setConfigParam(EnvironmentParams.ENV_CHECK_LEAKS.getName(), @@ -41,6 +45,7 @@ envConfig.setConfigParam(EnvironmentParams.NODE_MAX.getName(), "6"); envConfig.setTxnNoSync(Boolean.getBoolean(TestUtils.NO_SYNC)); + envConfig.setLockTimeout(1); // to speed up intentional deadlocks envConfig.setAllowCreate(true); } @@ -49,6 +54,19 @@ return multiKeyTxnTestSuite(SecondaryTest.class, envConfig, null); } + public void tearDown() + throws Exception { + + super.tearDown(); + if (junitThread != null) { + while (junitThread.isAlive()) { + junitThread.interrupt(); + Thread.yield(); + } + junitThread = null; + } + } + public void testPutAndDelete() throws DatabaseException { @@ -275,6 +293,7 @@ assertDataEquals(entry(i + KEY_OFFSET), secKey); assertDataEquals(entry(i), key); assertDataEquals(entry(i), data); + assertPriLocked(priDb, key); secKey.setData(null); key.setData(null); data.setData(null); @@ -291,6 +310,7 @@ assertDataEquals(entry(NUM_RECS - 1 + KEY_OFFSET), secKey); assertDataEquals(entry(NUM_RECS - 1), key); assertDataEquals(entry(NUM_RECS - 1), data); + assertPriLocked(priDb, key); /* SecondaryCursor.getLast()/getPrev() */ secKey.setData(null); @@ -302,6 +322,7 @@ assertDataEquals(entry(i + KEY_OFFSET), secKey); assertDataEquals(entry(i), key); assertDataEquals(entry(i), data); + assertPriLocked(priDb, key); secKey.setData(null); key.setData(null); data.setData(null); @@ -318,6 +339,7 @@ assertDataEquals(entry(0 + KEY_OFFSET), secKey); assertDataEquals(entry(0), key); assertDataEquals(entry(0), data); + assertPriLocked(priDb, key); /* SecondaryCursor.getSearchKey() */ key.setData(null); @@ -333,6 +355,7 @@ assertSame(OperationStatus.SUCCESS, status); assertDataEquals(entry(i), key); assertDataEquals(entry(i), data); + assertPriLocked(priDb, key); } key.setData(null); data.setData(null); @@ -351,6 +374,7 @@ data, LockMode.DEFAULT); assertSame(OperationStatus.SUCCESS, status); assertDataEquals(entry(i), data); + assertPriLocked(priDb, entry(i)); } data.setData(null); status = cursor.getSearchBoth(entry(NUM_RECS + KEY_OFFSET), @@ -366,6 +390,7 @@ assertSame(OperationStatus.SUCCESS, status); assertDataEquals(entry(0), key); assertDataEquals(entry(0), data); + assertPriLocked(priDb, key); for (int i = 0; i < NUM_RECS; i += 1) { key.setData(null); data.setData(null); @@ -374,6 +399,7 @@ assertSame(OperationStatus.SUCCESS, status); assertDataEquals(entry(i), key); assertDataEquals(entry(i), data); + assertPriLocked(priDb, key); } key.setData(null); data.setData(null); @@ -387,6 +413,7 @@ data, LockMode.DEFAULT); assertSame(OperationStatus.SUCCESS, status); assertDataEquals(entry(1), data); + assertPriLocked(priDb, entry(1)); for (int i = 0; i < NUM_RECS; i += 1) { data.setData(null); status = cursor.getSearchBothRange(entry(i + KEY_OFFSET), @@ -394,6 +421,7 @@ LockMode.DEFAULT); assertSame(OperationStatus.SUCCESS, status); assertDataEquals(entry(i), data); + assertPriLocked(priDb, entry(i)); } data.setData(null); status = cursor.getSearchBothRange(entry(NUM_RECS + KEY_OFFSET), @@ -422,6 +450,7 @@ assertDataEquals(entry(i + KEY_OFFSET), secKey); assertDataEquals(entry(i), key); assertDataEquals(entry(i), data); + assertPriLocked(priDb, key, data); secKey.setData(null); key.setData(null); data.setData(null); @@ -431,6 +460,7 @@ assertDataEquals(entry(i + KEY_OFFSET), secKey); assertDataEquals(entry(i + KEY_OFFSET), key); assertDataEquals(entry(i), data); + assertPriLocked(priDb, key, data); secKey.setData(null); key.setData(null); data.setData(null); @@ -454,6 +484,7 @@ assertDataEquals(entry(i + KEY_OFFSET), secKey); assertDataEquals(entry(i), key); assertDataEquals(entry(i), data); + assertPriLocked(priDb, key, data); secKey.setData(null); key.setData(null); data.setData(null); @@ -472,6 +503,7 @@ assertDataEquals(entry(i + KEY_OFFSET), secKey); assertDataEquals(entry(i + KEY_OFFSET), key); assertDataEquals(entry(i), data); + assertPriLocked(priDb, key, data); secKey.setData(null); key.setData(null); data.setData(null); @@ -481,6 +513,7 @@ assertDataEquals(entry(i + KEY_OFFSET), secKey); assertDataEquals(entry(i), key); assertDataEquals(entry(i), data); + assertPriLocked(priDb, key, data); secKey.setData(null); key.setData(null); data.setData(null); @@ -504,6 +537,7 @@ assertDataEquals(entry(i + KEY_OFFSET), secKey); assertDataEquals(entry(i + KEY_OFFSET), key); assertDataEquals(entry(i), data); + assertPriLocked(priDb, key, data); secKey.setData(null); key.setData(null); data.setData(null); @@ -1461,6 +1495,61 @@ assertTrue(e1.equals(e2)); } + private void assertPriLocked(Database priDb, DatabaseEntry key) { + assertPriLocked(priDb, key, null); + } + + /** + * Checks that the given key (or both key and data if data is non-null) is + * locked in the primary database. The primary record should be locked + * whenever a secondary cursor is positioned to point to that primary + * record. [#15573] + */ + private void assertPriLocked(final Database priDb, + final DatabaseEntry key, + final DatabaseEntry data) { + + /* + * Whether the record is locked transactionally or not in the current + * thread, we should not be able to write lock the record + * non-transactionally in another thread. + */ + final StringBuffer error = new StringBuffer(); + junitThread = new JUnitThread("primary-locker") { + public void testBody() + throws DatabaseException { + try { + if (data != null) { + priDb.getSearchBoth(null, key, data, LockMode.RMW); + } else { + DatabaseEntry myData = new DatabaseEntry(); + priDb.get(null, key, myData, LockMode.RMW); + } + error.append("Expected DeadlockException"); + } catch (DeadlockException expected) { + } + } + }; + + junitThread.start(); + Throwable t = null; + try { + junitThread.finishTest(); + } catch (Throwable e) { + t = e; + } finally { + junitThread = null; + } + + if (t != null) { + t.printStackTrace(); + fail(t.toString()); + } + if (error.length() > 0) { + fail(error.toString()); + } + } + private static class MyKeyCreator implements SecondaryKeyCreator { public boolean createSecondaryKey(SecondaryDatabase secondary, Modified: trunk/contrib/bdb/test/com/sleepycat/je/tree/MemorySizeTest.java =================================================================== --- trunk/contrib/bdb/test/com/sleepycat/je/tree/MemorySizeTest.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/test/com/sleepycat/je/tree/MemorySizeTest.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: MemorySizeTest.java,v 1.21.2.2 2007/03/07 01:24:42 mark Exp $ + * $Id: MemorySizeTest.java,v 1.21.2.3 2007/07/02 19:54:55 mark Exp $ */ package com.sleepycat.je.tree; @@ -27,6 +27,7 @@ import com.sleepycat.je.OperationStatus; import com.sleepycat.je.Transaction; import com.sleepycat.je.config.EnvironmentParams; +import com.sleepycat.je.dbi.DbTree; import com.sleepycat.je.dbi.EnvironmentImpl; import com.sleepycat.je.dbi.INList; import com.sleepycat.je.log.FileManager; @@ -411,7 +412,8 @@ Iterator iter = inList.iterator(); while (iter.hasNext()) { IN in = (IN) iter.next(); - if (in instanceof BIN) { + if (in instanceof BIN && + !in.getDatabase().getId().equals(DbTree.ID_DB_ID)) { BIN bin = (BIN) in; bin.latch(); assertTrue(bin.evictLNs() > 0); Modified: trunk/contrib/bdb/test/com/sleepycat/je/txn/LockManagerTest.java =================================================================== --- trunk/contrib/bdb/test/com/sleepycat/je/txn/LockManagerTest.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/test/com/sleepycat/je/txn/LockManagerTest.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: LockManagerTest.java,v 1.45.2.1 2007/02/01 14:50:21 cwl Exp $ + * $Id: LockManagerTest.java,v 1.45.2.2 2007/07/13 02:32:06 cwl Exp $ */ package com.sleepycat.je.txn; @@ -87,17 +87,20 @@ assertTrue(lockManager.nWaiters(new Long(2)) == -1); /* lock 2 doesn't exist, shouldn't affect any the existing lock */ - lockManager.release(2L, txn1); + lockManager.release(2L, txn1); + txn1.removeLock(2L); assertTrue(lockManager.isLocked(nid)); /* txn2 is not the owner, shouldn't release lock 1. */ lockManager.release(1L, txn2); + txn2.removeLock(1L); assertTrue(lockManager.isLocked(nid)); assertTrue(lockManager.isOwner(nid, txn1, LockType.READ)); assertTrue(lockManager.nOwners(nid) == 1); /* Now really release. */ lockManager.release(1L, txn1); + txn1.removeLock(1L); assertFalse(lockManager.isLocked(nid)); assertFalse(lockManager.isOwner(nid, txn1, LockType.READ)); assertFalse(lockManager.nOwners(nid) == 1); @@ -135,6 +138,7 @@ Thread.yield(); } lockManager.release(1L, txn1); + txn1.removeLock(1L); } catch (DatabaseException DBE) { DBE.printStackTrace(); fail("caught DatabaseException " + DBE); @@ -155,6 +159,7 @@ Thread.yield(); } lockManager.release(1L, txn2); + txn2.removeLock(1L); } catch (DatabaseException DBE) { DBE.printStackTrace(); fail("caught DatabaseException " + DBE); @@ -175,6 +180,7 @@ Thread.yield(); } lockManager.release(1L, txn3); + txn3.removeLock(1L); } catch (DatabaseException DBE) { DBE.printStackTrace(); fail("caught DatabaseException " + DBE); @@ -211,6 +217,7 @@ assertTrue(lockManager.isWaiter(nid, txn3)); assertFalse(lockManager.isWaiter(nid, txn1)); lockManager.release(1L, txn1); + txn1.removeLock(1L); assertFalse (lockManager.isOwner(nid, txn1, LockType.READ)); } catch (DatabaseException DBE) { @@ -233,6 +240,7 @@ } assertTrue(lockManager.isWaiter(nid, txn3)); lockManager.release(1L, txn2); + txn2.removeLock(1L); assertFalse (lockManager.isOwner(nid, txn2, LockType.READ)); } catch (DatabaseException DBE) { @@ -289,6 +297,7 @@ Thread.yield(); } lockManager.release(1L, txn1); + txn1.removeLock(1L); } catch (DatabaseException DBE) { DBE.printStackTrace(); fail("caught DatabaseException " + DBE); @@ -308,6 +317,7 @@ Thread.yield(); } lockManager.release(1L, txn2); + txn2.removeLock(1L); } catch (DatabaseException DBE) { DBE.printStackTrace(); fail("caught DatabaseException " + DBE); @@ -330,6 +340,7 @@ assertTrue (lockManager.isOwner(nid, txn3, LockType.WRITE)); lockManager.release(1L, txn3); + txn3.removeLock(1L); } catch (DatabaseException DBE) { DBE.printStackTrace(); fail("caught DatabaseException " + DBE); @@ -349,6 +360,7 @@ assertTrue (lockManager.isOwner(nid, txn4, LockType.READ)); lockManager.release(1L, txn4); + txn4.removeLock(1L); } catch (DatabaseException DBE) { DBE.printStackTrace(); fail("caught DatabaseException " + DBE); @@ -388,6 +400,7 @@ Thread.yield(); } lockManager.release(1L, txn1); + txn1.removeLock(1L); } catch (DatabaseException DBE) { DBE.printStackTrace(); fail("caught DatabaseException " + DBE); @@ -410,6 +423,7 @@ false, null); assertTrue(lockManager.nWaiters(nid) == 1); lockManager.release(1L, txn2); + txn2.removeLock(1L); } catch (DatabaseException DBE) { DBE.printStackTrace(); fail("caught DatabaseException " + DBE); @@ -429,6 +443,7 @@ assertTrue (lockManager.isOwner(nid, txn3, LockType.WRITE)); lockManager.release(1L, txn3); + txn3.removeLock(1L); } catch (DatabaseException DBE) { DBE.printStackTrace(); fail("caught DatabaseException " + DBE); @@ -463,6 +478,7 @@ Thread.yield(); } lockManager.release(1L, txn1); + txn1.removeLock(1L); } catch (DatabaseException DBE) { DBE.printStackTrace(); fail("caught DatabaseException " + DBE); @@ -503,6 +519,7 @@ assertTrue(lockManager.nWaiters(nid) == 0); assertTrue(lockManager.nOwners(nid) == 1); lockManager.release(1L, txn2); + txn2.removeLock(1L); } catch (DatabaseException DBE) { DBE.printStackTrace(); fail("caught DatabaseException " + DBE); @@ -536,6 +553,7 @@ Thread.yield(); } lockManager.release(1L, txn1); + txn1.removeLock(1L); } catch (DatabaseException DBE) { DBE.printStackTrace(); fail("caught DatabaseException " + DBE); @@ -576,6 +594,7 @@ assertTrue(lockManager.nWaiters(nid) == 0); assertTrue(lockManager.nOwners(nid) == 1); lockManager.release(1L, txn2); + txn2.removeLock(1L); } catch (DatabaseException DBE) { DBE.printStackTrace(); fail("caught DatabaseException " + DBE); @@ -609,6 +628,7 @@ Thread.yield(); } lockManager.release(1L, txn1); + txn1.removeLock(1L); } catch (DatabaseException DBE) { DBE.printStackTrace(); fail("caught DatabaseException " + DBE); @@ -653,6 +673,7 @@ assertTrue(lockManager.nWaiters(nid) == 0); assertTrue(lockManager.nOwners(nid) == 1); lockManager.release(1L, txn2); + txn2.removeLock(1L); } catch (DatabaseException DBE) { DBE.printStackTrace(); fail("caught DatabaseException " + DBE); @@ -691,7 +712,9 @@ Thread.sleep(5000); lockManager.release(1, txn1); + txn1.removeLock(1); lockManager.release(2, txn1); + txn1.removeLock(2); } catch (DatabaseException DBE) { DBE.printStackTrace(); fail("tester1 caught DatabaseException " + DBE); @@ -723,7 +746,9 @@ Thread.sleep(5000); lockManager.release(1, txn2); + txn2.removeLock(1); lockManager.release(2, txn1); + txn1.removeLock(2); } catch (DatabaseException DBE) { DBE.printStackTrace(); fail("tester2 caught DatabaseException " + DBE); Modified: trunk/contrib/bdb/test/com/sleepycat/je/txn/LockTest.java =================================================================== --- trunk/contrib/bdb/test/com/sleepycat/je/txn/LockTest.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/test/com/sleepycat/je/txn/LockTest.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: LockTest.java,v 1.44.2.1 2007/02/01 14:50:21 cwl Exp $ + * $Id: LockTest.java,v 1.44.2.2 2007/07/13 02:32:06 cwl Exp $ */ package com.sleepycat.je.txn; @@ -62,7 +62,7 @@ * should only be one owner. Then add multiple * would-be-writers as waiters. */ - Lock lock = new Lock(new Long(1)); + Lock lock = new Lock(); assertEquals(LockGrantType.NEW, lock.lock(LockType.READ, txn1, false, mb, 0)); assertEquals(LockGrantType.EXISTING, @@ -84,7 +84,7 @@ /* Start fresh. Get a write lock, then get a read lock. */ - lock = new Lock(new Long(1)); + lock = new Lock(); assertEquals(LockGrantType.NEW, lock.lock(LockType.WRITE, txn1, false, mb, 0)); @@ -94,7 +94,7 @@ assertEquals(0, lock.nWaiters()); /* Start fresh. Get a read lock, upgrade to a write lock. */ - lock = new Lock(new Long(1)); + lock = new Lock(); assertEquals(LockGrantType.NEW, lock.lock(LockType.READ, txn1, false, mb, 0)); assertEquals(LockGrantType.PROMOTION, @@ -106,7 +106,7 @@ * Start fresh. Get a read lock, then ask for a non-blocking * write lock. The latter should be denied. */ - lock = new Lock(new Long(1)); + lock = new Lock(); assertEquals(LockGrantType.NEW, lock.lock(LockType.READ, txn1, false, mb, 0)); @@ -116,7 +116,7 @@ assertEquals(0, lock.nWaiters()); /* Two write requsts, should be one owner. */ - lock = new Lock(new Long(1)); + lock = new Lock(); assertEquals(LockGrantType.NEW, lock.lock(LockType.WRITE, txn1, false, mb, 0)); assertEquals(LockGrantType.EXISTING, @@ -128,7 +128,7 @@ * Ensure that a read request behind a write request that waits * also waits. blocking requests. */ - lock = new Lock(new Long(1)); + lock = new Lock(); assertEquals(LockGrantType.NEW, lock.lock(LockType.READ, txn1, false, mb, 0)); @@ -143,7 +143,7 @@ assertEquals(2, lock.nWaiters()); /* Check non blocking requests */ - lock = new Lock(new Long(1)); + lock = new Lock(); assertEquals(LockGrantType.NEW, lock.lock(LockType.READ, txn1, false, mb, 0)); @@ -157,7 +157,7 @@ assertEquals(2, lock.nOwners()); assertEquals(0, lock.nWaiters()); - lock = new Lock(new Long(1)); + lock = new Lock(); assertEquals(LockGrantType.NEW, lock.lock(LockType.READ, txn1, false, mb, 0)); @@ -188,7 +188,7 @@ * Build up 3 owners and waiters for a lock, to test the * lazy initialization and optimization for single owner/waiter. */ - Lock lock = new Lock(new Long(1)); + Lock lock = new Lock(); /* should be no writer. */ assertTrue(lock.getWriteOwnerLocker() == null); @@ -202,9 +202,6 @@ /* should be no writer. */ assertTrue(lock.getWriteOwnerLocker() == null); - /* should be node 1 */ - assertEquals(1, lock.getNodeId().longValue()); - /* expect 3 owners, 0 waiters. */ Set expectedOwners = new HashSet(); expectedOwners.add(new LockInfo(txn1, LockType.READ)); @@ -283,7 +280,7 @@ * Build up 1 owners and 3waiters for a lock, to test the * lazy initialization and optimization for single owner/waiter. */ - Lock lock = new Lock(new Long(1)); + Lock lock = new Lock(); assertEquals(LockGrantType.NEW, lock.lock(LockType.READ, txn1, false, mb, 0)); assertEquals(LockGrantType.NEW, @@ -298,9 +295,6 @@ /* should be no writer. */ assertTrue(lock.getWriteOwnerLocker() == null); - /* should be node 1 */ - assertEquals(1, lock.getNodeId().longValue()); - /* expect 2 owners, 3 waiters. */ Set expectedOwners = new HashSet(); expectedOwners.add(new LockInfo(txn1, LockType.READ)); @@ -410,7 +404,7 @@ * Build up 1 owners and 3 read waiters for a lock. Then * check that all the waiters promote properly. */ - Lock lock = new Lock(new Long(1)); + Lock lock = new Lock(); assertEquals(LockGrantType.NEW, lock.lock(LockType.WRITE, txn1, false, mb, 0)); assertEquals(LockGrantType.WAIT_NEW, @@ -557,7 +551,7 @@ MemoryBudget mb = envImpl.getMemoryBudget(); try { - Lock lock = new Lock(new Long(1)); + Lock lock = new Lock(); if (firstRequest != null) { assertEquals(LockGrantType.NEW, @@ -727,7 +721,7 @@ MemoryBudget mb = envImpl.getMemoryBudget(); try { - Lock lock = new Lock(new Long(1)); + Lock lock = new Lock(); assertEquals(LockGrantType.NEW, lock.lock(firstRequest, txn1, false, mb, 0)); @@ -772,7 +766,7 @@ MemoryBudget mb = envImpl.getMemoryBudget(); try { - Lock lock = new Lock(new Long(1)); + Lock lock = new Lock(); assertEquals(LockGrantType.NEW, lock.lock(LockType.RANGE_READ, txn1, false, mb, 0)); assertEquals(LockGrantType.WAIT_NEW, @@ -838,8 +832,6 @@ assertFalse(lock.isOwner(info.getLocker(), info.getLockType())); assertTrue(lock.isWaiter(info.getLocker())); } - - } public void testTransfer() @@ -854,7 +846,8 @@ try { /* Transfer from one locker to another locker. */ - Lock lock = new Lock(new Long(1)); + Long nid = new Long(1); + Lock lock = new Lock(); assertEquals(LockGrantType.NEW, lock.lock(LockType.WRITE, txn1, false, mb, 0)); assertEquals(LockGrantType.WAIT_NEW, @@ -862,7 +855,7 @@ assertTrue(lock.isOwner(txn1, LockType.WRITE)); assertFalse(lock.isOwner(txn2, LockType.READ)); - lock.transfer(txn1, txn2, mb, 0); + lock.transfer(nid, txn1, txn2, mb, 0); assertFalse(lock.isOwner(txn1, LockType.WRITE)); assertFalse(lock.isOwner(txn1, LockType.READ)); assertTrue(lock.isOwnedWriteLock(txn2)); @@ -873,7 +866,7 @@ destLockers[1] = txn4; destLockers[2] = txn5; lock.demote(txn2); - lock.transferMultiple(txn2, destLockers, mb, 0); + lock.transferMultiple(nid, txn2, destLockers, mb, 0); assertFalse(lock.isOwner(txn2, LockType.WRITE)); assertFalse(lock.isOwner(txn2, LockType.READ)); @@ -882,7 +875,6 @@ assertFalse(lock.isOwner(destLockers[i], LockType.WRITE)); } - } finally { txn1.operationEnd(); txn2.operationEnd(); Modified: trunk/contrib/bdb/test/com/sleepycat/persist/test/EvolveClasses.java =================================================================== --- trunk/contrib/bdb/test/com/sleepycat/persist/test/EvolveClasses.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/test/com/sleepycat/persist/test/EvolveClasses.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2000,2007 Oracle. All rights reserved. * - * $Id: EvolveClasses.java,v 1.11.2.1 2007/02/01 14:50:24 cwl Exp $ + * $Id: EvolveClasses.java,v 1.11.2.2 2007/05/01 04:08:36 mark Exp $ */ package com.sleepycat.persist.test; @@ -326,7 +326,7 @@ @Override public String getStoreOpenException() { - return "com.sleepycat.persist.evolve.IncompatibleClassException: Mutation is missing to evolve class: com.sleepycat.persist.test.EvolveClasses$DeletedEntity3_AnnotRemoved_NoMutation version: 0 Error: java.lang.IllegalArgumentException: Class is not persistent: com.sleepycat.persist.test.EvolveClasses$DeletedEntity3_AnnotRemoved_NoMutation"; + return "com.sleepycat.persist.evolve.IncompatibleClassException: Mutation is missing to evolve class: com.sleepycat.persist.test.EvolveClasses$DeletedEntity3_AnnotRemoved_NoMutation version: 0 Error: java.lang.IllegalArgumentException: Class could not be loaded or is not persistent: com.sleepycat.persist.test.EvolveClasses$DeletedEntity3_AnnotRemoved_NoMutation"; } @Override @@ -391,7 +391,7 @@ TestCase.fail(); } catch (Exception e) { checkEquals - ("java.lang.IllegalArgumentException: Not an entity class: com.sleepycat.persist.test.EvolveClasses$DeletedEntity4_AnnotRemoved_WithDeleter", + ("java.lang.IllegalArgumentException: Class could not be loaded or is not an entity class: com.sleepycat.persist.test.EvolveClasses$DeletedEntity4_AnnotRemoved_WithDeleter", e.toString()); } } @@ -493,7 +493,7 @@ TestCase.fail(); } catch (Exception e) { checkEquals - ("java.lang.IllegalArgumentException: Not an entity class: com.sleepycat.persist.test.EvolveClasses$DeletedEntity6_EntityToPersist_WithDeleter", + ("java.lang.IllegalArgumentException: Class could not be loaded or is not an entity class: com.sleepycat.persist.test.EvolveClasses$DeletedEntity6_EntityToPersist_WithDeleter", e.toString()); } @@ -644,7 +644,7 @@ TestCase.fail(); } catch (Exception e) { checkEquals - ("java.lang.IllegalArgumentException: Not an entity class: com.sleepycat.persist.test.EvolveClasses$DeletedPersist2_ClassRemoved_WithDeleter", + ("java.lang.IllegalArgumentException: Class could not be loaded or is not an entity class: com.sleepycat.persist.test.EvolveClasses$DeletedPersist2_ClassRemoved_WithDeleter", e.toString()); } } @@ -696,7 +696,7 @@ @Override public String getStoreOpenException() { - return "com.sleepycat.persist.evolve.IncompatibleClassException: Mutation is missing to evolve class: com.sleepycat.persist.test.EvolveClasses$DeletedPersist3_AnnotRemoved version: 0 Error: java.lang.IllegalArgumentException: Class is not persistent: com.sleepycat.persist.test.EvolveClasses$DeletedPersist3_AnnotRemoved"; + return "com.sleepycat.persist.evolve.IncompatibleClassException: Mutation is missing to evolve class: com.sleepycat.persist.test.EvolveClasses$DeletedPersist3_AnnotRemoved version: 0 Error: java.lang.IllegalArgumentException: Class could not be loaded or is not persistent: com.sleepycat.persist.test.EvolveClasses$DeletedPersist3_AnnotRemoved"; } @Override @@ -782,7 +782,7 @@ TestCase.fail(); } catch (Exception e) { checkEquals - ("java.lang.IllegalArgumentException: Not an entity class: com.sleepycat.persist.test.EvolveClasses$DeletedPersist4_AnnotRemoved_WithDeleter", + ("java.lang.IllegalArgumentException: Class could not be loaded or is not an entity class: com.sleepycat.persist.test.EvolveClasses$DeletedPersist4_AnnotRemoved_WithDeleter", e.toString()); } } @@ -933,7 +933,7 @@ TestCase.fail(); } catch (Exception e) { checkEquals - ("java.lang.IllegalArgumentException: Not an entity class: com.sleepycat.persist.test.EvolveClasses$DeletedPersist6_PersistToEntity_WithDeleter", + ("java.lang.IllegalArgumentException: Class could not be loaded or is not an entity class: com.sleepycat.persist.test.EvolveClasses$DeletedPersist6_PersistToEntity_WithDeleter", e.toString()); } @@ -5490,7 +5490,7 @@ @Override public String getStoreOpenException() { - return "com.sleepycat.persist.evolve.IncompatibleClassException: Mutation is missing to evolve class: java.lang.StringBuffer version: 0 Error: java.lang.IllegalArgumentException: Class is not persistent: java.lang.StringBuffer"; + return "com.sleepycat.persist.evolve.IncompatibleClassException: Mutation is missing to evolve class: java.lang.StringBuffer version: 0 Error: java.lang.IllegalArgumentException: Class could not be loaded or is not persistent: java.lang.StringBuffer"; } } Modified: trunk/contrib/bdb/test/com/sleepycat/persist/test/OperationTest.java =================================================================== --- trunk/contrib/bdb/test/com/sleepycat/persist/test/OperationTest.java 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/test/com/sleepycat/persist/test/OperationTest.java 2007-08-17 14:14:29 UTC (rev 14749) @@ -3,7 +3,7 @@ * * Copyright (c) 2002,2007 Oracle. All rights reserved. * - * $Id: OperationTest.java,v 1.12.2.2 2007/02/10 02:58:19 mark Exp $ + * $Id: OperationTest.java,v 1.12.2.3 2007/06/14 13:06:05 mark Exp $ */ package com.sleepycat.persist.test; @@ -784,4 +784,77 @@ RelatedY() { } } + + public void testSecondaryBulkLoad1() + throws DatabaseException { + + doSecondaryBulkLoad(true); + } + + public void testSecondaryBulkLoad2() + throws DatabaseException { + + doSecondaryBulkLoad(false); + } + + private void doSecondaryBulkLoad(boolean closeAndOpenNormally) + throws DatabaseException { + + PrimaryIndex<Integer,RelatedX> priX; + PrimaryIndex<Integer,RelatedY> priY; + SecondaryIndex<Integer,Integer,RelatedX> secX; + + /* Open priX with SecondaryBulkLoad=true. */ + StoreConfig config = new StoreConfig(); + config.setAllowCreate(true); + config.setSecondaryBulkLoad(true); + open(config); + + /* Getting priX should not create the secondary index. */ + priX = store.getPrimaryIndex(Integer.class, RelatedX.class); + PersistTestUtils.assertDbExists + (false, env, STORE_NAME, RelatedX.class.getName(), "key2"); + + /* We can put records that violate the secondary key constraint. */ + priX.put(new RelatedX()); + + if (closeAndOpenNormally) { + /* Open normally and the secondary will be populated. */ + close(); + open(); + try { + /* Before adding the foreign key, constraint is violated. */ + priX = store.getPrimaryIndex(Integer.class, RelatedX.class); + } catch (DatabaseException e) { + assertTrue(e.toString(), + e.toString().contains("foreign key not allowed")); + } + /* Add the foreign key to avoid the constraint error. */ + priY = store.getPrimaryIndex(Integer.class, RelatedY.class); + priY.put(new RelatedY()); + priX = store.getPrimaryIndex(Integer.class, RelatedX.class); + PersistTestUtils.assertDbExists + (true, env, STORE_NAME, RelatedX.class.getName(), "key2"); + secX = store.getSecondaryIndex(priX, Integer.class, "key2"); + } else { + /* Get secondary index explicitly and it will be populated. */ + try { + /* Before adding the foreign key, constraint is violated. */ + secX = store.getSecondaryIndex(priX, Integer.class, "key2"); + } catch (DatabaseException e) { + assertTrue(e.toString(), + e.toString().contains("foreign key not allowed")); + } + /* Add the foreign key. */ + priY = store.getPrimaryIndex(Integer.class, RelatedY.class); + priY.put(new RelatedY()); + secX = store.getSecondaryIndex(priX, Integer.class, "key2"); + PersistTestUtils.assertDbExists + (true, env, STORE_NAME, RelatedX.class.getName(), "key2"); + } + + RelatedX x = secX.get(88); + assertNotNull(x); + close(); + } } Modified: trunk/contrib/bdb/test/je.properties =================================================================== --- trunk/contrib/bdb/test/je.properties 2007-08-17 14:09:41 UTC (rev 14748) +++ trunk/contrib/bdb/test/je.properties 2007-08-17 14:14:29 UTC (rev 14749) @@ -4,12 +4,13 @@ # It may be useful to use a property file when debugging. # This file should always be checked in with all properties # commented out. -# $Id: je.properties,v 1.43.2.1 2007/02/08 15:02:15 mark Exp $ +# $Id: je.properties,v 1.43.2.2 2007/07/02 19:54:54 mark Exp $ # Settings for permutations of unit testing: #je.evictor.lruOnly=false #je.evictor.forcedYield=false #je.env.forcedYield=true +#je.env.dbEviction=true #je.log.useNIO=true #je.log.directNIO=true #je.log.chunkedNIO=4096
