Attached is the patch to increase the max possible log files number from
2^22 -1 to
2^33 -1. Derby creates log file names in sequential order by
incrementing log file number
by one each time(like log1.dat ....logN.dat). id's are not reused, so
the last possible log file that
can be created currently is limited to 2^22 - 1 (4194303). Problem with
this limit is ,
it can be possible hit on systems that creates lot of log records very
fast with the default
log file size (1MB) , for example if 1MB of log is created every
second this limit can be hit
in (4194303 / (60 * 60 * 24)) 48 days.
Attached fix bumps the max possible log file number to 2^33 -1 by
borrowing some unused bits in the
log file number and log file size fields in the log instant. With the
same assumption as above if a
a 1MB of log file is created for every second; it will take
((8589934591 / (60 * 60 * 24))/365)
approximately 272 years; Which I think is a reasonable limit. :-)
I believe the new limit is a reasonable one , and not necessary to solve
this problem by
implementing a complicated solution that reuse the ids, at this moment.
Testing : ran derbyall ; all tests passed.
New tests : Added couple of tests to test the max log file ids using
debug flags.
svn status
M java\engine\org\apache\derby\impl\store\raw\log\LogCounter.java
M java\engine\org\apache\derby\impl\store\raw\log\LogToFile.java
M
java\testing\org\apache\derbyTesting\functionTests\tests\store\copyfiles.ant
A
java\testing\org\apache\derbyTesting\functionTests\tests\store\MaxLogNumberRecovery_derby.properties
A
java\testing\org\apache\derbyTesting\functionTests\tests\store\MaxLogNumber_app.properties
A
java\testing\org\apache\derbyTesting\functionTests\tests\store\MaxLogNumberRecovery_app.properties
A
java\testing\org\apache\derbyTesting\functionTests\tests\store\MaxLogNumber_derby.properties
A
java\testing\org\apache\derbyTesting\functionTests\tests\store\MaxLogNumber.java
A
java\testing\org\apache\derbyTesting\functionTests\tests\store\MaxLogNumberRecovery.java
A
java\testing\org\apache\derbyTesting\functionTests\master\MaxLogNumber.out
A
java\testing\org\apache\derbyTesting\functionTests\master\MaxLogNumberRecovery.out
M
java\testing\org\apache\derbyTesting\functionTests\suites\storerecovery.runall
$
Thanks
-suresht
------------------------------------------------------------------------
Index: java/engine/org/apache/derby/impl/store/raw/log/LogCounter.java
===================================================================
--- java/engine/org/apache/derby/impl/store/raw/log/LogCounter.java
(revision 170490)
+++ java/engine/org/apache/derby/impl/store/raw/log/LogCounter.java
(working copy)
@@ -62,13 +62,15 @@
public static final long INVALID_LOG_INSTANT = 0;
- // reserve top 10 bits in log file number for future use
- public static final long MAX_LOGFILE_NUMBER = (long)0x003FFFFFL; // 4194303
- private static final long FILE_NUMBER_SHIFT = 32;
+ // max possible log file number is 2^33 -1 (8589934591)
+ public static final long MAX_LOGFILE_NUMBER = (long)0x1FFFFFFFFL ;
+ // lower end of 30 bits in a long type are used to store the log file postion.
+ private static final long FILE_NUMBER_SHIFT = 30;
- // reserve top 4 bits in log file size for future use
+ // reserve top 2 bits in log file size for future use
public static final long MAX_LOGFILE_SIZE =
(long)0x0FFFFFFFL; // 268435455
- private static final long FILE_POSITION_MASK =
(long)0x7FFFFFFFL;
+ // 30 bits are used to store the log file postion
+ private static final long FILE_POSITION_MASK =
(long)0x3FFFFFFFL;
private long fileNumber;
private long filePosition;
Index: java/engine/org/apache/derby/impl/store/raw/log/LogToFile.java
===================================================================
--- java/engine/org/apache/derby/impl/store/raw/log/LogToFile.java (revision 170490)
+++ java/engine/org/apache/derby/impl/store/raw/log/LogToFile.java (working copy)
@@ -649,7 +649,7 @@
if (beginLogFileNumber != null)
{
logFileNumber =
- Integer.valueOf(beginLogFileNumber).intValue();
+ Long.valueOf(beginLogFileNumber).longValue();
}
else
{
@@ -2975,6 +2975,17 @@
{
firstLogFileNumber = 1;
logFileNumber = 1;
+ if (SanityManager.DEBUG)
+ {
+ if (SanityManager.DEBUG_ON(TEST_MAX_LOGFILE_NUMBER))
+ {
+ // set the value to be two less than max possible
+ // log number, test case will perform some ops to
+ // hit the max number case.
+ firstLogFileNumber = LogCounter.MAX_LOGFILE_NUMBER -2;
+ logFileNumber = LogCounter.MAX_LOGFILE_NUMBER -2;
+ }
+ }
logFile = getLogFileName(logFileNumber);
if (privExists(logFile))
@@ -4293,7 +4304,12 @@
*/
public static final String TEST_RECORD_TO_FILL_LOG = SanityManager.DEBUG ?
"db2j.unittest.recordToFillLog" : null;
+ /**
+ * Set to true if we want to simulate max possible log file number is
being used.
+ */
+ public static final String TEST_MAX_LOGFILE_NUMBER = SanityManager.DEBUG ?
"testMaxLogFileNumber" : null;
+
//enable the log archive mode
public void enableLogArchiveMode() throws StandardException
{
Index:
java/testing/org/apache/derbyTesting/functionTests/tests/store/copyfiles.ant
===================================================================
---
java/testing/org/apache/derbyTesting/functionTests/tests/store/copyfiles.ant
(revision 170490)
+++
java/testing/org/apache/derbyTesting/functionTests/tests/store/copyfiles.ant
(working copy)
@@ -131,3 +131,8 @@
LogChecksumRecovery_derby.properties
LogChecksumRecovery1_app.properties
LogChecksumRecovery1_derby.properties
+MaxLogNumber_app.properties
+MaxLogNumber_derby.properties
+MaxLogNumberRecovery_app.properties
+MaxLogNumberRecovery_derby.properties
+
Index:
java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery_derby.properties
===================================================================
---
java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery_derby.properties
(revision 0)
+++
java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery_derby.properties
(revision 0)
@@ -0,0 +1 @@
+derby.debug.true=testMaxLogFileNumber
Property changes on:
java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery_derby.properties
___________________________________________________________________
Name: svn:eol-style
+ native
Index:
java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber_app.properties
===================================================================
---
java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber_app.properties
(revision 0)
+++
java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber_app.properties
(revision 0)
@@ -0,0 +1,13 @@
+#
+# *** DO NOT PUT PROPERTIES FOR THE DERBY SYSTEM IN THIS FILE. THEY BELONG
+# *** IN the _derby.properties file.
+#
+# It will get handed to the test on the command line in a -p <filename>
+# argument.
+#
+# This causes ij (or the GUI on ij) to load the driver and make an
+# initial connection to the database.
+#
+#
+database=jdbc:derby:maxlogwombat;create=true
+
Property changes on:
java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber_app.properties
___________________________________________________________________
Name: svn:eol-style
+ native
Index:
java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery_app.properties
===================================================================
---
java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery_app.properties
(revision 0)
+++
java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery_app.properties
(revision 0)
@@ -0,0 +1,12 @@
+#
+# *** DO NOT PUT PROPERTIES FOR THE DERBY SYSTEM IN THIS FILE. THEY BELONG
+# *** IN the _derby.properties file.
+#
+# It will get handed to the test on the command line in a -p <filename>
+# argument.
+#
+# This causes ij (or the GUI on ij) to load the driver and make an
+# initial connection to the database.
+#
+#
+database=jdbc:derby:maxlogwombat
Property changes on:
java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery_app.properties
___________________________________________________________________
Name: svn:eol-style
+ native
Index:
java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber_derby.properties
===================================================================
---
java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber_derby.properties
(revision 0)
+++
java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber_derby.properties
(revision 0)
@@ -0,0 +1 @@
+derby.debug.true=testMaxLogFileNumber
Property changes on:
java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber_derby.properties
___________________________________________________________________
Name: svn:eol-style
+ native
Index: java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber.java
===================================================================
--- java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber.java (revision 0)
+++ java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber.java (revision 0)
@@ -0,0 +1,216 @@
+/*
+
+ Derby - Class org.apache.derbyTesting.functionTests.store.MaxLogNumber
+
+ Copyright 2005 The Apache Software Foundation or its licensors, as applicable.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ */
+
+package org.apache.derbyTesting.functionTests.tests.store;
+import java.sql.Connection;
+import java.sql.Statement;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import org.apache.derby.tools.ij;
+
+/*
+ * This class tests log writes to the transaction log files with large log file
+ * id's and do a setup to test recovery with large log file id's in
+ * MaxLogNumberRecovery.java test. large log file id's are simulated using
+ * a debug flag 'testMaxLogFileNumber' in the log factory, this is enabled
+ * by setting derby.debug.true=testMaxLogFileNumber in the properties file.
+ * In Non debug mode, this tests just acts as a plain log recovery test.
+ *
+ * @author <a href="mailto:[EMAIL PROTECTED]">Suresh Thalamati</a>
+ * @version 1.0
+ */
+
+public class MaxLogNumber{
+
+ MaxLogNumber() {
+ }
+
+
+ private void runTest(Connection conn) throws SQLException {
+ logMessage("Begin MaxLogNumber Test");
+ // perform a checkpoint otherwise recovery test will look at log1
+ // instead of the log number that gets by the testMaxLogFileNumber
+ // debug flags.
+ performCheckPoint(conn);
+ createTable(conn);
+ insert(conn, 100, COMMIT, 10);
+ insert(conn, 100, ROLLBACK, 10);
+ update(conn, 50, COMMIT, 10);
+ update(conn, 50, ROLLBACK, 10);
+ verifyData(conn, 100);
+ //do some inserts that will be rolled back by recovey
+ insert(conn, 2000, NOACTION, 2000);
+ logMessage("End MaxLogNumber Test");
+ }
+
+ void performCheckPoint(Connection conn) throws SQLException
+ {
+ Statement stmt = conn.createStatement();
+ //wait to make sure that checkpoint thread finished it's work
+ stmt.executeUpdate("CALL SYSCS_UTIL.SYSCS_CHECKPOINT_DATABASE()");
+ stmt.close();
+ }
+
+
+ /**
+ * Insert some rows into the table.
+ */
+ void insert(Connection conn, int rowCount,
+ int txStatus, int commitCount) throws SQLException {
+
+ PreparedStatement ps = conn.prepareStatement("INSERT INTO " +
+ "emp" +
+ " VALUES(?,?,?)");
+ for (int i = 0; i < rowCount; i++) {
+
+ ps.setInt(1, i); // ID
+ ps.setString(2 , "skywalker" + i);
+ ps.setFloat(3, (float)(i * 2000));
+ ps.executeUpdate();
+ if ((i % commitCount) == 0)
+ {
+ endTransaction(conn, txStatus);
+ }
+ }
+
+ endTransaction(conn, txStatus);
+ ps.close();
+ }
+
+
+ static final int COMMIT = 1;
+ static final int ROLLBACK = 2;
+ static final int NOACTION = 3;
+
+ void endTransaction(Connection conn, int txStatus) throws SQLException
+ {
+ switch(txStatus){
+ case COMMIT:
+ conn.commit();
+ break;
+ case ROLLBACK:
+ conn.rollback();
+ break;
+ case NOACTION:
+ //do nothing
+ break;
+ }
+ }
+
+ /**
+ * update some rows in the table.
+ */
+
+ void update(Connection conn, int rowCount,
+ int txStatus, int commitCount) throws SQLException
+ {
+
+ PreparedStatement ps = conn.prepareStatement("update " + "emp" +
+ " SET salary=? where id=?");
+
+ for (int i = 0; i < rowCount; i++) {
+
+ ps.setFloat(1, (float)(i * 2000 * 0.08));
+ ps.setInt(2, i); // ID
+ ps.executeUpdate();
+ if ((i % commitCount) == 0)
+ {
+ endTransaction(conn, txStatus);
+ }
+ }
+ endTransaction(conn, txStatus);
+ ps.close();
+ }
+
+
+ /*
+ * verify the rows in the table.
+ */
+ void verifyData(Connection conn, int expectedRowCount) throws SQLException {
+
+ Statement s = conn.createStatement();
+ ResultSet rs = s.executeQuery("SELECT ID, name from emp order by id" );
+ int count = 0;
+ int id = 0;
+ while(rs.next())
+ {
+ int tid = rs.getInt(1);
+ String name = rs.getString(2);
+ if(name.equals("skywalker" + id) && tid!= id)
+ {
+
+ logMessage("DATA IN THE TABLE IS NOT AS EXPECTED");
+ logMessage("Got :ID=" + tid + " Name=:" + name);
+ logMessage("Expected: ID=" + id + "Name=" + "skywalker" + id );
+ }
+
+ id++;
+ count++;
+ }
+
+ if(count != expectedRowCount)
+ {
+ logMessage("Expected Number Of Rows (" +
+ expectedRowCount + ")" + "!=" +
+ "No Of rows in the Table(" +
+ count + ")");
+ }
+ s.close();
+ }
+
+ /*
+ * create the tables that are used by this test.
+ */
+ void createTable(Connection conn) throws SQLException {
+
+ Statement s = conn.createStatement();
+ s.executeUpdate("CREATE TABLE " + "emp" +
+ "(id INT," +
+ "name CHAR(200),"+
+ "salary float)");
+ s.executeUpdate("create index emp_idx on emp(id) ");
+ conn.commit();
+ s.close();
+ }
+
+ void logMessage(String str)
+ {
+ System.out.println(str);
+ }
+
+
+ public static void main(String[] argv) throws Throwable {
+
+ MaxLogNumber test = new MaxLogNumber();
+ ij.getPropertyArg(argv);
+ Connection conn = ij.startJBMS();
+ conn.setAutoCommit(false);
+
+ try {
+ test.runTest(conn);
+ }
+ catch (SQLException sqle) {
+ org.apache.derby.tools.JDBCDisplayUtil.ShowSQLException(
+ System.out, sqle);
+ sqle.printStackTrace(System.out);
+ }
+ }
+}
Property changes on:
java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber.java
___________________________________________________________________
Name: svn:eol-style
+ native
Index: java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery.java
===================================================================
--- java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery.java (revision 0)
+++ java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery.java (revision 0)
@@ -0,0 +1,104 @@
+/*
+
+ Derby - Class org.apache.derbyTesting.functionTests.store.MaxLogNumber
+
+ Copyright 2005 The Apache Software Foundation or its licensors, as applicable.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ */
+
+package org.apache.derbyTesting.functionTests.tests.store;
+import java.sql.Connection;
+import java.sql.SQLException;
+import org.apache.derby.tools.ij;
+import org.apache.derby.iapi.services.sanity.SanityManager;
+
+/*
+ * This class tests recovery logic with large log file id's and the error
+ * handling logic when Max possible log file limit is reached. MaxLogNumber.java
+ * test does the setup, so it should be run before this test.
+ * In Non debug mode, this tests just acts as a plain log recovery test.
+ *
+ * @author <a href="mailto:[EMAIL PROTECTED]">Suresh Thalamati</a>
+ * @version 1.0
+ * @see MaxLogNumber
+ */
+
+public class MaxLogNumberRecovery extends MaxLogNumber {
+
+ MaxLogNumberRecovery() {
+ super();
+ }
+
+ private void runTest(Connection conn) throws SQLException {
+ logMessage("Begin MaxLogNumberRecovery Test");
+ verifyData(conn, 100);
+ boolean hitMaxLogLimitError = false;
+ try{
+ insert(conn, 110, COMMIT, 11);
+ update(conn, 110, ROLLBACK, 5);
+ update(conn, 110, NOACTION, 5);
+ verifyData(conn, 210);
+ if (SanityManager.DEBUG)
+ {
+ // do lot of inserts in debug mode ,
+ // so that actuall reach the max log file number
+ // limit
+ insert(conn, 11000, COMMIT, 5);
+ }
+ } catch(SQLException se) {
+
+ SQLException ose = se;
+ while (se != null) {
+ if ("XSLAK".equals(se.getSQLState())) {
+ hitMaxLogLimitError = true;
+ break;
+ }
+ se = se.getNextException();
+ }
+ if(!hitMaxLogLimitError)
+ throw ose;
+ }
+
+ if (SanityManager.DEBUG)
+ {
+ // In the debug build mode , this test should hit the max log limit while
+ // doing above DML.
+ if(!hitMaxLogLimitError)
+ logMessage("Expected: ERROR XSLAK:" +
+ "Database has exceeded largest log file" +
+ "number 8,589,934,591.");
+ }
+
+ logMessage("End MaxLogNumberRecovery Test");
+ }
+
+
+ public static void main(String[] argv) throws Throwable {
+
+ MaxLogNumberRecovery test = new MaxLogNumberRecovery();
+ ij.getPropertyArg(argv);
+ Connection conn = ij.startJBMS();
+ conn.setAutoCommit(false);
+
+ try {
+ test.runTest(conn);
+ }
+ catch (SQLException sqle) {
+ org.apache.derby.tools.JDBCDisplayUtil.ShowSQLException(
+ System.out, sqle);
+ sqle.printStackTrace(System.out);
+ }
+ }
+}
Property changes on:
java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery.java
___________________________________________________________________
Name: svn:eol-style
+ native
Index:
java/testing/org/apache/derbyTesting/functionTests/master/MaxLogNumber.out
===================================================================
--- java/testing/org/apache/derbyTesting/functionTests/master/MaxLogNumber.out
(revision 0)
+++ java/testing/org/apache/derbyTesting/functionTests/master/MaxLogNumber.out
(revision 0)
@@ -0,0 +1,2 @@
+Begin MaxLogNumber Test
+End MaxLogNumber Test
Property changes on:
java/testing/org/apache/derbyTesting/functionTests/master/MaxLogNumber.out
___________________________________________________________________
Name: svn:eol-style
+ native
Index:
java/testing/org/apache/derbyTesting/functionTests/master/MaxLogNumberRecovery.out
===================================================================
---
java/testing/org/apache/derbyTesting/functionTests/master/MaxLogNumberRecovery.out
(revision 0)
+++
java/testing/org/apache/derbyTesting/functionTests/master/MaxLogNumberRecovery.out
(revision 0)
@@ -0,0 +1,2 @@
+Begin MaxLogNumberRecovery Test
+End MaxLogNumberRecovery Test
Property changes on:
java/testing/org/apache/derbyTesting/functionTests/master/MaxLogNumberRecovery.out
___________________________________________________________________
Name: svn:eol-style
+ native
Index:
java/testing/org/apache/derbyTesting/functionTests/suites/storerecovery.runall
===================================================================
---
java/testing/org/apache/derbyTesting/functionTests/suites/storerecovery.runall
(revision 170490)
+++
java/testing/org/apache/derbyTesting/functionTests/suites/storerecovery.runall
(working copy)
@@ -1,3 +1,5 @@
store/LogChecksumSetup.java
store/LogChecksumRecovery.java
store/LogChecksumRecovery1.java
+store/MaxLogNumber.java
+store/MaxLogNumberRecovery.java