I committed this change with svn 170686.
Suresh Thalamati wrote:
> Attached is the patch to increase the max possible log files number from
> 2^22 -1 to
> 2^33 -1. Derby creates log file names in sequential order by
> incrementing log file number
> by one each time(like log1.dat ....logN.dat). id's are not reused, so
> the last possible log file that
> can be created currently is limited to 2^22 - 1 (4194303). Problem with
> this limit is ,
> it can be possible hit on systems that creates lot of log records very
> fast with the default
> log file size (1MB) , for example if 1MB of log is created every
> second this limit can be hit
> in (4194303 / (60 * 60 * 24)) 48 days.
>
> Attached fix bumps the max possible log file number to 2^33 -1 by
> borrowing some unused bits in the
> log file number and log file size fields in the log instant. With the
> same assumption as above if a
> a 1MB of log file is created for every second; it will take
> ((8589934591 / (60 * 60 * 24))/365)
> approximately 272 years; Which I think is a reasonable limit. :-)
>
> I believe the new limit is a reasonable one , and not necessary to solve
> this problem by
> implementing a complicated solution that reuse the ids, at this moment.
>
> Testing : ran derbyall ; all tests passed.
> New tests : Added couple of tests to test the max log file ids using
> debug flags.
>
> svn status
> M java\engine\org\apache\derby\impl\store\raw\log\LogCounter.java
> M java\engine\org\apache\derby\impl\store\raw\log\LogToFile.java
> M
> java\testing\org\apache\derbyTesting\functionTests\tests\store\copyfiles.ant
>
> A
> java\testing\org\apache\derbyTesting\functionTests\tests\store\MaxLogNumberRecovery_derby.properties
>
> A
> java\testing\org\apache\derbyTesting\functionTests\tests\store\MaxLogNumber_app.properties
>
> A
> java\testing\org\apache\derbyTesting\functionTests\tests\store\MaxLogNumberRecovery_app.properties
>
> A
> java\testing\org\apache\derbyTesting\functionTests\tests\store\MaxLogNumber_derby.properties
>
> A
> java\testing\org\apache\derbyTesting\functionTests\tests\store\MaxLogNumber.java
>
> A
> java\testing\org\apache\derbyTesting\functionTests\tests\store\MaxLogNumberRecovery.java
>
> A
> java\testing\org\apache\derbyTesting\functionTests\master\MaxLogNumber.out
> A
> java\testing\org\apache\derbyTesting\functionTests\master\MaxLogNumberRecovery.out
>
> M
> java\testing\org\apache\derbyTesting\functionTests\suites\storerecovery.runall
>
> $
> Thanks
> -suresht
>
>
> ------------------------------------------------------------------------
>
> Index: java/engine/org/apache/derby/impl/store/raw/log/LogCounter.java
> ===================================================================
> --- java/engine/org/apache/derby/impl/store/raw/log/LogCounter.java
> (revision 170490)
> +++ java/engine/org/apache/derby/impl/store/raw/log/LogCounter.java
> (working copy)
> @@ -62,13 +62,15 @@
> public static final long INVALID_LOG_INSTANT = 0;
>
>
> - // reserve top 10 bits in log file number for future use
> - public static final long MAX_LOGFILE_NUMBER =
> (long)0x003FFFFFL; // 4194303
> - private static final long FILE_NUMBER_SHIFT = 32;
> + // max possible log file number is 2^33 -1 (8589934591)
> + public static final long MAX_LOGFILE_NUMBER = (long)0x1FFFFFFFFL ;
> + // lower end of 30 bits in a long type are used to store the log file
> postion.
> + private static final long FILE_NUMBER_SHIFT = 30;
>
> - // reserve top 4 bits in log file size for future use
> + // reserve top 2 bits in log file size for future use
> public static final long MAX_LOGFILE_SIZE =
> (long)0x0FFFFFFFL; // 268435455
> - private static final long FILE_POSITION_MASK =
> (long)0x7FFFFFFFL;
> + // 30 bits are used to store the log file postion
> + private static final long FILE_POSITION_MASK =
> (long)0x3FFFFFFFL;
>
> private long fileNumber;
> private long filePosition;
> Index: java/engine/org/apache/derby/impl/store/raw/log/LogToFile.java
> ===================================================================
> --- java/engine/org/apache/derby/impl/store/raw/log/LogToFile.java
> (revision 170490)
> +++ java/engine/org/apache/derby/impl/store/raw/log/LogToFile.java
> (working copy)
> @@ -649,7 +649,7 @@
> if (beginLogFileNumber != null)
> {
> logFileNumber =
> -
> Integer.valueOf(beginLogFileNumber).intValue();
> + Long.valueOf(beginLogFileNumber).longValue();
> }
> else
> {
> @@ -2975,6 +2975,17 @@
> {
> firstLogFileNumber = 1;
> logFileNumber = 1;
> + if (SanityManager.DEBUG)
> + {
> + if
> (SanityManager.DEBUG_ON(TEST_MAX_LOGFILE_NUMBER))
> + {
> + // set the value to be
> two less than max possible
> + // log number, test
> case will perform some ops to
> + // hit the max number
> case.
> + firstLogFileNumber =
> LogCounter.MAX_LOGFILE_NUMBER -2;
> + logFileNumber =
> LogCounter.MAX_LOGFILE_NUMBER -2;
> + }
> + }
> logFile = getLogFileName(logFileNumber);
>
> if (privExists(logFile))
> @@ -4293,7 +4304,12 @@
> */
> public static final String TEST_RECORD_TO_FILL_LOG =
> SanityManager.DEBUG ? "db2j.unittest.recordToFillLog" : null;
>
> + /**
> + * Set to true if we want to simulate max possible log file number is
> being used.
> + */
> + public static final String TEST_MAX_LOGFILE_NUMBER =
> SanityManager.DEBUG ? "testMaxLogFileNumber" : null;
>
> +
> //enable the log archive mode
> public void enableLogArchiveMode() throws StandardException
> {
> Index:
> java/testing/org/apache/derbyTesting/functionTests/tests/store/copyfiles.ant
> ===================================================================
> ---
> java/testing/org/apache/derbyTesting/functionTests/tests/store/copyfiles.ant
> (revision 170490)
> +++
> java/testing/org/apache/derbyTesting/functionTests/tests/store/copyfiles.ant
> (working copy)
> @@ -131,3 +131,8 @@
> LogChecksumRecovery_derby.properties
> LogChecksumRecovery1_app.properties
> LogChecksumRecovery1_derby.properties
> +MaxLogNumber_app.properties
> +MaxLogNumber_derby.properties
> +MaxLogNumberRecovery_app.properties
> +MaxLogNumberRecovery_derby.properties
> +
> Index:
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery_derby.properties
> ===================================================================
> ---
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery_derby.properties
> (revision 0)
> +++
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery_derby.properties
> (revision 0)
> @@ -0,0 +1 @@
> +derby.debug.true=testMaxLogFileNumber
>
> Property changes on:
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery_derby.properties
> ___________________________________________________________________
> Name: svn:eol-style
> + native
>
> Index:
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber_app.properties
> ===================================================================
> ---
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber_app.properties
> (revision 0)
> +++
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber_app.properties
> (revision 0)
> @@ -0,0 +1,13 @@
> +#
> +# *** DO NOT PUT PROPERTIES FOR THE DERBY SYSTEM IN THIS FILE. THEY BELONG
> +# *** IN the _derby.properties file.
> +#
> +# It will get handed to the test on the command line in a -p <filename>
> +# argument.
> +#
> +# This causes ij (or the GUI on ij) to load the driver and make an
> +# initial connection to the database.
> +#
> +#
> +database=jdbc:derby:maxlogwombat;create=true
> +
>
> Property changes on:
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber_app.properties
> ___________________________________________________________________
> Name: svn:eol-style
> + native
>
> Index:
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery_app.properties
> ===================================================================
> ---
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery_app.properties
> (revision 0)
> +++
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery_app.properties
> (revision 0)
> @@ -0,0 +1,12 @@
> +#
> +# *** DO NOT PUT PROPERTIES FOR THE DERBY SYSTEM IN THIS FILE. THEY BELONG
> +# *** IN the _derby.properties file.
> +#
> +# It will get handed to the test on the command line in a -p <filename>
> +# argument.
> +#
> +# This causes ij (or the GUI on ij) to load the driver and make an
> +# initial connection to the database.
> +#
> +#
> +database=jdbc:derby:maxlogwombat
>
> Property changes on:
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery_app.properties
> ___________________________________________________________________
> Name: svn:eol-style
> + native
>
> Index:
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber_derby.properties
> ===================================================================
> ---
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber_derby.properties
> (revision 0)
> +++
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber_derby.properties
> (revision 0)
> @@ -0,0 +1 @@
> +derby.debug.true=testMaxLogFileNumber
>
> Property changes on:
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber_derby.properties
> ___________________________________________________________________
> Name: svn:eol-style
> + native
>
> Index:
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber.java
> ===================================================================
> ---
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber.java
> (revision 0)
> +++
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber.java
> (revision 0)
> @@ -0,0 +1,216 @@
> +/*
> +
> + Derby - Class org.apache.derbyTesting.functionTests.store.MaxLogNumber
> +
> + Copyright 2005 The Apache Software Foundation or its licensors, as
> applicable.
> +
> + Licensed under the Apache License, Version 2.0 (the "License");
> + you may not use this file except in compliance with the License.
> + You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> +
> + */
> +
> +package org.apache.derbyTesting.functionTests.tests.store;
> +import java.sql.Connection;
> +import java.sql.Statement;
> +import java.sql.PreparedStatement;
> +import java.sql.ResultSet;
> +import java.sql.SQLException;
> +import org.apache.derby.tools.ij;
> +
> +/*
> + * This class tests log writes to the transaction log files with large log
> file
> + * id's and do a setup to test recovery with large log file id's in
> + * MaxLogNumberRecovery.java test. large log file id's are simulated using
> + * a debug flag 'testMaxLogFileNumber' in the log factory, this is enabled
> + * by setting derby.debug.true=testMaxLogFileNumber in the properties file.
> + * In Non debug mode, this tests just acts as a plain log recovery test.
> + *
> + * @author <a href="mailto:[EMAIL PROTECTED]">Suresh Thalamati</a>
> + * @version 1.0
> + */
> +
> +public class MaxLogNumber{
> +
> + MaxLogNumber() {
> + }
> +
> +
> + private void runTest(Connection conn) throws SQLException {
> + logMessage("Begin MaxLogNumber Test");
> + // perform a checkpoint otherwise recovery test will look at
> log1
> + // instead of the log number that gets by the
> testMaxLogFileNumber
> + // debug flags.
> + performCheckPoint(conn);
> + createTable(conn);
> + insert(conn, 100, COMMIT, 10);
> + insert(conn, 100, ROLLBACK, 10);
> + update(conn, 50, COMMIT, 10);
> + update(conn, 50, ROLLBACK, 10);
> + verifyData(conn, 100);
> + //do some inserts that will be rolled back by recovey
> + insert(conn, 2000, NOACTION, 2000);
> + logMessage("End MaxLogNumber Test");
> + }
> +
> + void performCheckPoint(Connection conn) throws SQLException
> + {
> + Statement stmt = conn.createStatement();
> + //wait to make sure that checkpoint thread finished it's work
> + stmt.executeUpdate("CALL
> SYSCS_UTIL.SYSCS_CHECKPOINT_DATABASE()");
> + stmt.close();
> + }
> +
> +
> + /**
> + * Insert some rows into the table.
> + */
> + void insert(Connection conn, int rowCount,
> + int txStatus, int commitCount) throws
> SQLException {
> +
> + PreparedStatement ps = conn.prepareStatement("INSERT INTO " +
> +
> "emp" +
> +
> " VALUES(?,?,?)");
> + for (int i = 0; i < rowCount; i++) {
> +
> + ps.setInt(1, i); // ID
> + ps.setString(2 , "skywalker" + i);
> + ps.setFloat(3, (float)(i * 2000));
> + ps.executeUpdate();
> + if ((i % commitCount) == 0)
> + {
> + endTransaction(conn, txStatus);
> + }
> + }
> +
> + endTransaction(conn, txStatus);
> + ps.close();
> + }
> +
> +
> + static final int COMMIT = 1;
> + static final int ROLLBACK = 2;
> + static final int NOACTION = 3;
> +
> + void endTransaction(Connection conn, int txStatus) throws SQLException
> + {
> + switch(txStatus){
> + case COMMIT:
> + conn.commit();
> + break;
> + case ROLLBACK:
> + conn.rollback();
> + break;
> + case NOACTION:
> + //do nothing
> + break;
> + }
> + }
> +
> + /**
> + * update some rows in the table.
> + */
> +
> + void update(Connection conn, int rowCount,
> + int txStatus, int commitCount) throws
> SQLException
> + {
> +
> + PreparedStatement ps = conn.prepareStatement("update " + "emp"
> +
> +
> " SET salary=? where id=?");
> +
> + for (int i = 0; i < rowCount; i++) {
> +
> + ps.setFloat(1, (float)(i * 2000 * 0.08));
> + ps.setInt(2, i); // ID
> + ps.executeUpdate();
> + if ((i % commitCount) == 0)
> + {
> + endTransaction(conn, txStatus);
> + }
> + }
> + endTransaction(conn, txStatus);
> + ps.close();
> + }
> +
> +
> + /*
> + * verify the rows in the table.
> + */
> + void verifyData(Connection conn, int expectedRowCount) throws
> SQLException {
> +
> + Statement s = conn.createStatement();
> + ResultSet rs = s.executeQuery("SELECT ID, name from emp order
> by id" );
> + int count = 0;
> + int id = 0;
> + while(rs.next())
> + {
> + int tid = rs.getInt(1);
> + String name = rs.getString(2);
> + if(name.equals("skywalker" + id) && tid!= id)
> + {
> +
> + logMessage("DATA IN THE TABLE IS NOT AS
> EXPECTED");
> + logMessage("Got :ID=" + tid + " Name=:" +
> name);
> + logMessage("Expected: ID=" + id + "Name=" +
> "skywalker" + id );
> + }
> +
> + id++;
> + count++;
> + }
> +
> + if(count != expectedRowCount)
> + {
> + logMessage("Expected Number Of Rows (" +
> + expectedRowCount + ")" + "!=" +
> + "No Of rows in the Table(" +
> + count + ")");
> + }
> + s.close();
> + }
> +
> + /*
> + * create the tables that are used by this test.
> + */
> + void createTable(Connection conn) throws SQLException {
> +
> + Statement s = conn.createStatement();
> + s.executeUpdate("CREATE TABLE " + "emp" +
> + "(id INT," +
> + "name CHAR(200),"+
> + "salary float)");
> + s.executeUpdate("create index emp_idx on emp(id) ");
> + conn.commit();
> + s.close();
> + }
> +
> + void logMessage(String str)
> + {
> + System.out.println(str);
> + }
> +
> +
> + public static void main(String[] argv) throws Throwable {
> +
> + MaxLogNumber test = new MaxLogNumber();
> + ij.getPropertyArg(argv);
> + Connection conn = ij.startJBMS();
> + conn.setAutoCommit(false);
> +
> + try {
> + test.runTest(conn);
> + }
> + catch (SQLException sqle) {
> + org.apache.derby.tools.JDBCDisplayUtil.ShowSQLException(
> + System.out, sqle);
> + sqle.printStackTrace(System.out);
> + }
> + }
> +}
>
> Property changes on:
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumber.java
> ___________________________________________________________________
> Name: svn:eol-style
> + native
>
> Index:
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery.java
> ===================================================================
> ---
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery.java
> (revision 0)
> +++
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery.java
> (revision 0)
> @@ -0,0 +1,104 @@
> +/*
> +
> + Derby - Class org.apache.derbyTesting.functionTests.store.MaxLogNumber
> +
> + Copyright 2005 The Apache Software Foundation or its licensors, as
> applicable.
> +
> + Licensed under the Apache License, Version 2.0 (the "License");
> + you may not use this file except in compliance with the License.
> + You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> +
> + */
> +
> +package org.apache.derbyTesting.functionTests.tests.store;
> +import java.sql.Connection;
> +import java.sql.SQLException;
> +import org.apache.derby.tools.ij;
> +import org.apache.derby.iapi.services.sanity.SanityManager;
> +
> +/*
> + * This class tests recovery logic with large log file id's and the error
> + * handling logic when Max possible log file limit is reached.
> MaxLogNumber.java
> + * test does the setup, so it should be run before this test.
> + * In Non debug mode, this tests just acts as a plain log recovery test.
> + *
> + * @author <a href="mailto:[EMAIL PROTECTED]">Suresh Thalamati</a>
> + * @version 1.0
> + * @see MaxLogNumber
> + */
> +
> +public class MaxLogNumberRecovery extends MaxLogNumber {
> +
> + MaxLogNumberRecovery() {
> + super();
> + }
> +
> + private void runTest(Connection conn) throws SQLException {
> + logMessage("Begin MaxLogNumberRecovery Test");
> + verifyData(conn, 100);
> + boolean hitMaxLogLimitError = false;
> + try{
> + insert(conn, 110, COMMIT, 11);
> + update(conn, 110, ROLLBACK, 5);
> + update(conn, 110, NOACTION, 5);
> + verifyData(conn, 210);
> + if (SanityManager.DEBUG)
> + {
> + // do lot of inserts in debug mode ,
> + // so that actuall reach the max log file
> number
> + // limit
> + insert(conn, 11000, COMMIT, 5);
> + }
> + } catch(SQLException se) {
> +
> + SQLException ose = se;
> + while (se != null) {
> + if ("XSLAK".equals(se.getSQLState())) {
> + hitMaxLogLimitError = true;
> + break;
> + }
> + se = se.getNextException();
> + }
> + if(!hitMaxLogLimitError)
> + throw ose;
> + }
> +
> + if (SanityManager.DEBUG)
> + {
> + // In the debug build mode , this test should hit the
> max log limit while
> + // doing above DML.
> + if(!hitMaxLogLimitError)
> + logMessage("Expected: ERROR XSLAK:" +
> + "Database has exceeded
> largest log file" +
> + "number 8,589,934,591.");
> + }
> +
> + logMessage("End MaxLogNumberRecovery Test");
> + }
> +
> +
> + public static void main(String[] argv) throws Throwable {
> +
> + MaxLogNumberRecovery test = new MaxLogNumberRecovery();
> + ij.getPropertyArg(argv);
> + Connection conn = ij.startJBMS();
> + conn.setAutoCommit(false);
> +
> + try {
> + test.runTest(conn);
> + }
> + catch (SQLException sqle) {
> + org.apache.derby.tools.JDBCDisplayUtil.ShowSQLException(
> + System.out, sqle);
> + sqle.printStackTrace(System.out);
> + }
> + }
> +}
>
> Property changes on:
> java/testing/org/apache/derbyTesting/functionTests/tests/store/MaxLogNumberRecovery.java
> ___________________________________________________________________
> Name: svn:eol-style
> + native
>
> Index:
> java/testing/org/apache/derbyTesting/functionTests/master/MaxLogNumber.out
> ===================================================================
> ---
> java/testing/org/apache/derbyTesting/functionTests/master/MaxLogNumber.out
> (revision 0)
> +++
> java/testing/org/apache/derbyTesting/functionTests/master/MaxLogNumber.out
> (revision 0)
> @@ -0,0 +1,2 @@
> +Begin MaxLogNumber Test
> +End MaxLogNumber Test
>
> Property changes on:
> java/testing/org/apache/derbyTesting/functionTests/master/MaxLogNumber.out
> ___________________________________________________________________
> Name: svn:eol-style
> + native
>
> Index:
> java/testing/org/apache/derbyTesting/functionTests/master/MaxLogNumberRecovery.out
> ===================================================================
> ---
> java/testing/org/apache/derbyTesting/functionTests/master/MaxLogNumberRecovery.out
> (revision 0)
> +++
> java/testing/org/apache/derbyTesting/functionTests/master/MaxLogNumberRecovery.out
> (revision 0)
> @@ -0,0 +1,2 @@
> +Begin MaxLogNumberRecovery Test
> +End MaxLogNumberRecovery Test
>
> Property changes on:
> java/testing/org/apache/derbyTesting/functionTests/master/MaxLogNumberRecovery.out
> ___________________________________________________________________
> Name: svn:eol-style
> + native
>
> Index:
> java/testing/org/apache/derbyTesting/functionTests/suites/storerecovery.runall
> ===================================================================
> ---
> java/testing/org/apache/derbyTesting/functionTests/suites/storerecovery.runall
> (revision 170490)
> +++
> java/testing/org/apache/derbyTesting/functionTests/suites/storerecovery.runall
> (working copy)
> @@ -1,3 +1,5 @@
> store/LogChecksumSetup.java
> store/LogChecksumRecovery.java
> store/LogChecksumRecovery1.java
> +store/MaxLogNumber.java
> +store/MaxLogNumberRecovery.java
