This is an automated email from the ASF dual-hosted git repository. tv pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/commons-jcs.git
commit b9b4477c4c284f972275b13c16e286cf2a8b1514 Author: Thomas Vandahl <[email protected]> AuthorDate: Mon Apr 13 13:57:45 2020 +0200 Reactivate tests. Make database caches more robust --- commons-jcs-core/pom.xml | 4 - .../jcs/auxiliary/disk/jdbc/JDBCDiskCache.java | 52 +++++---- .../disk/jdbc/hsql/HSQLDiskCacheFactory.java | 116 +++++++-------------- .../jdbc/hsql/HSQLDiskCacheConcurrentUnitTest.java | 30 ++---- .../disk/jdbc/hsql/HSQLDiskCacheUnitTest.java | 36 ++----- .../src/test/test-conf/TestHSQLDiskCache.ccf | 1 + .../test/test-conf/TestHSQLDiskCacheConcurrent.ccf | 1 + 7 files changed, 87 insertions(+), 153 deletions(-) diff --git a/commons-jcs-core/pom.xml b/commons-jcs-core/pom.xml index e7bc878..be95c6d 100644 --- a/commons-jcs-core/pom.xml +++ b/commons-jcs-core/pom.xml @@ -130,10 +130,6 @@ sensitive. --> <!-- You need to manually run these test cases. --> - <exclude>**/BlockDiskElementDescriptorUnitTest.java</exclude> - <exclude>**/HSQLDiskCacheConcurrentUnitTest.java</exclude> - <exclude>**/HSQLDiskCacheUnitTest.java</exclude> - <exclude>**/IndexedDiskCacheOptimizationUnitTest.java</exclude> <exclude>**/TestTCPLateralUnitTest.java</exclude> <exclude>**/UDPDiscoveryUnitTest.java</exclude> <!-- Causes hang in Continuum --> diff --git a/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/jdbc/JDBCDiskCache.java b/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/jdbc/JDBCDiskCache.java index 508eec2..a60b81d 100644 --- a/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/jdbc/JDBCDiskCache.java +++ b/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/jdbc/JDBCDiskCache.java @@ -21,6 +21,7 @@ package org.apache.commons.jcs.auxiliary.disk.jdbc; import java.io.IOException; import java.sql.Connection; +import java.sql.DatabaseMetaData; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -615,37 +616,44 @@ public class JDBCDiskCache<K, V> try (Connection con = getDataSource().getConnection()) { - getTableState().setState( TableState.DELETE_RUNNING ); + // The shrinker thread might kick in before the table is created + // So check if the table exists first + DatabaseMetaData dmd = con.getMetaData(); + ResultSet result = dmd.getTables(null, null, + getJdbcDiskCacheAttributes().getTableName(), null); - long now = System.currentTimeMillis() / 1000; + if (result.next()) + { + getTableState().setState( TableState.DELETE_RUNNING ); + long now = System.currentTimeMillis() / 1000; - // This is to slow when we push over a million records - // String sql = "delete from " + - // getJdbcDiskCacheAttributes().getTableName() + " where REGION = '" - // + this.getCacheName() + "' and IS_ETERNAL = 'F' and (" + now - // + " - UPDATE_TIME_SECONDS) > MAX_LIFE_SECONDS"; + String sql = "delete from " + getJdbcDiskCacheAttributes().getTableName() + + " where IS_ETERNAL = ? and REGION = ? and ? > SYSTEM_EXPIRE_TIME_SECONDS"; - String sql = "delete from " + getJdbcDiskCacheAttributes().getTableName() - + " where IS_ETERNAL = ? and REGION = ? and ? > SYSTEM_EXPIRE_TIME_SECONDS"; + try (PreparedStatement psDelete = con.prepareStatement( sql )) + { + psDelete.setString( 1, "F" ); + psDelete.setString( 2, this.getCacheName() ); + psDelete.setLong( 3, now ); - try (PreparedStatement psDelete = con.prepareStatement( sql )) - { - psDelete.setString( 1, "F" ); - psDelete.setString( 2, this.getCacheName() ); - psDelete.setLong( 3, now ); + setAlive(true); - setAlive(true); + deleted = psDelete.executeUpdate(); + } + catch ( SQLException e ) + { + log.error( "Problem creating statement.", e ); + setAlive(false); + } - deleted = psDelete.executeUpdate(); + logApplicationEvent( getAuxiliaryCacheAttributes().getName(), "deleteExpired", + "Deleted expired elements. URL: " + getDiskLocation() ); } - catch ( SQLException e ) + else { - log.error( "Problem creating statement.", e ); - setAlive(false); + log.warn( "Trying to shrink non-existing table [{0}]", + getJdbcDiskCacheAttributes().getTableName() ); } - - logApplicationEvent( getAuxiliaryCacheAttributes().getName(), "deleteExpired", - "Deleted expired elements. URL: " + getDiskLocation() ); } catch ( SQLException e ) { diff --git a/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/jdbc/hsql/HSQLDiskCacheFactory.java b/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/jdbc/hsql/HSQLDiskCacheFactory.java index 186e46b..1afb779 100644 --- a/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/jdbc/hsql/HSQLDiskCacheFactory.java +++ b/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/jdbc/hsql/HSQLDiskCacheFactory.java @@ -20,10 +20,12 @@ package org.apache.commons.jcs.auxiliary.disk.jdbc.hsql; */ import java.sql.Connection; -import java.sql.DriverManager; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.concurrent.CopyOnWriteArraySet; + +import javax.sql.DataSource; import org.apache.commons.jcs.auxiliary.AuxiliaryCacheAttributes; import org.apache.commons.jcs.auxiliary.disk.jdbc.JDBCDiskCache; @@ -46,9 +48,6 @@ public class HSQLDiskCacheFactory /** The logger */ private static final Log log = LogManager.getLog( HSQLDiskCacheFactory.class ); - /** The databases. */ - private CopyOnWriteArraySet<String> databases; - /** * This factory method should create an instance of the hsqlcache. * <p> @@ -66,70 +65,30 @@ public class HSQLDiskCacheFactory IElementSerializer elementSerializer ) throws SQLException { - setupDatabase( (JDBCDiskCacheAttributes) rawAttr ); - return super.createCache(rawAttr, compositeCacheManager, cacheEventLogger, elementSerializer); - } + // TODO get this from the attributes. + System.setProperty( "hsqldb.cache_scale", "8" ); - /** - * Initialize this factory - */ - @Override - public void initialize() - { - super.initialize(); - this.databases = new CopyOnWriteArraySet<>(); + JDBCDiskCache<K, V> cache = super.createCache(rawAttr, compositeCacheManager, + cacheEventLogger, elementSerializer); + setupDatabase( cache.getDataSource(), (JDBCDiskCacheAttributes) rawAttr ); + + return cache; } /** - * Creates the database if it doesn't exist, registers the driver class, etc. + * Creates the table if it doesn't exist * <p> - * @param attributes + * @param ds Data Source + * @param attributes Cache region configuration * @throws SQLException */ - protected void setupDatabase( JDBCDiskCacheAttributes attributes ) + protected void setupDatabase( DataSource ds, JDBCDiskCacheAttributes attributes ) throws SQLException { - if ( attributes == null ) - { - throw new SQLException( "The attributes are null." ); - } - - // url should start with "jdbc:hsqldb:" - String database = attributes.getUrl() + attributes.getDatabase(); - - if ( databases.contains( database ) ) - { - log.info("We already setup database [{0}]", database); - return; - } - - synchronized (databases) + try (Connection cConn = ds.getConnection()) { - // TODO get this from the attributes. - System.setProperty( "hsqldb.cache_scale", "8" ); - - // "org.hsqldb.jdbcDriver" - String driver = attributes.getDriverClassName(); - // "sa" - String user = attributes.getUserName(); - // "" - String password = attributes.getPassword(); - - try - { - Class.forName( driver ).newInstance(); - } - catch (Exception e) - { - throw new SQLException( "Could not initialize driver " + driver, e ); - } - - Connection cConn = DriverManager.getConnection( database, user, password ); setupTable( cConn, attributes.getTableName() ); - - log.info( "Finished setting up database [{0}]", database); - - databases.add( database ); + log.info( "Finished setting up table [{0}]", attributes.getTableName()); } } @@ -141,30 +100,29 @@ public class HSQLDiskCacheFactory */ protected void setupTable( Connection cConn, String tableName ) throws SQLException { - // TODO make the cached nature of the table configurable - StringBuilder createSql = new StringBuilder(); - createSql.append( "CREATE CACHED TABLE ").append( tableName ); - createSql.append( "( " ); - createSql.append( "CACHE_KEY VARCHAR(250) NOT NULL, " ); - createSql.append( "REGION VARCHAR(250) NOT NULL, " ); - createSql.append( "ELEMENT BINARY, " ); - createSql.append( "CREATE_TIME TIMESTAMP, " ); - createSql.append( "UPDATE_TIME_SECONDS BIGINT, " ); - createSql.append( "MAX_LIFE_SECONDS BIGINT, " ); - createSql.append( "SYSTEM_EXPIRE_TIME_SECONDS BIGINT, " ); - createSql.append( "IS_ETERNAL CHAR(1), " ); - createSql.append( "PRIMARY KEY (CACHE_KEY, REGION) " ); - createSql.append( ");" ); + DatabaseMetaData dmd = cConn.getMetaData(); + ResultSet result = dmd.getTables(null, null, tableName, null); - try (Statement sStatement = cConn.createStatement()) - { - sStatement.execute( createSql.toString() ); - } - catch ( SQLException e ) + if (!result.next()) { - if (!"23000".equals(e.getSQLState())) + // TODO make the cached nature of the table configurable + StringBuilder createSql = new StringBuilder(); + createSql.append( "CREATE CACHED TABLE ").append( tableName ); + createSql.append( "( " ); + createSql.append( "CACHE_KEY VARCHAR(250) NOT NULL, " ); + createSql.append( "REGION VARCHAR(250) NOT NULL, " ); + createSql.append( "ELEMENT BINARY, " ); + createSql.append( "CREATE_TIME TIMESTAMP, " ); + createSql.append( "UPDATE_TIME_SECONDS BIGINT, " ); + createSql.append( "MAX_LIFE_SECONDS BIGINT, " ); + createSql.append( "SYSTEM_EXPIRE_TIME_SECONDS BIGINT, " ); + createSql.append( "IS_ETERNAL CHAR(1), " ); + createSql.append( "PRIMARY KEY (CACHE_KEY, REGION) " ); + createSql.append( ");" ); + + try (Statement sStatement = cConn.createStatement()) { - throw e; + sStatement.execute( createSql.toString() ); } } } diff --git a/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/jdbc/hsql/HSQLDiskCacheConcurrentUnitTest.java b/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/jdbc/hsql/HSQLDiskCacheConcurrentUnitTest.java index 91a0811..4a666b8 100644 --- a/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/jdbc/hsql/HSQLDiskCacheConcurrentUnitTest.java +++ b/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/jdbc/hsql/HSQLDiskCacheConcurrentUnitTest.java @@ -1,5 +1,13 @@ package org.apache.commons.jcs.auxiliary.disk.jdbc.hsql; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.jcs.JCS; +import org.apache.commons.jcs.access.CacheAccess; +import org.apache.commons.jcs.engine.behavior.ICacheElement; + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -22,13 +30,6 @@ package org.apache.commons.jcs.auxiliary.disk.jdbc.hsql; import junit.extensions.ActiveTestSuite; import junit.framework.Test; import junit.framework.TestCase; -import org.apache.commons.jcs.JCS; -import org.apache.commons.jcs.access.CacheAccess; -import org.apache.commons.jcs.engine.behavior.ICacheElement; - -import java.util.HashSet; -import java.util.Map; -import java.util.Set; /** * Test which exercises the indexed disk cache. This one uses three different regions for thre @@ -52,17 +53,6 @@ public class HSQLDiskCacheConcurrentUnitTest } /** - * Main method passes this test to the text test runner. - * <p> - * @param args - */ - public static void main( String args[] ) - { - String[] testCaseName = { HSQLDiskCacheConcurrentUnitTest.class.getName() }; - junit.textui.TestRunner.main( testCaseName ); - } - - /** * A unit test suite for JUnit. Uses ActiveTestSuite to run multiple tests concurrently. * <p> * @return The test suite @@ -126,7 +116,6 @@ public class HSQLDiskCacheConcurrentUnitTest CacheAccess<String, String> jcs = JCS.getInstance( region ); // Add items to cache - for ( int i = 0; i <= items; i++ ) { jcs.put( i + ":key", region + " data " + i ); @@ -135,7 +124,6 @@ public class HSQLDiskCacheConcurrentUnitTest // System.out.println( jcs.getStats() ); // Test that all items are in cache - for ( int i = 0; i <= items; i++ ) { String value = jcs.get( i + ":key" ); @@ -159,14 +147,12 @@ public class HSQLDiskCacheConcurrentUnitTest } // Remove all the items - for ( int i = 0; i <= items; i++ ) { jcs.remove( i + ":key" ); } // Verify removal - for ( int i = 0; i <= items; i++ ) { assertNull( "Removed key should be null: " + i + ":key", jcs.get( i + ":key" ) ); diff --git a/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/jdbc/hsql/HSQLDiskCacheUnitTest.java b/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/jdbc/hsql/HSQLDiskCacheUnitTest.java index d6c0294..58d7e77 100644 --- a/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/jdbc/hsql/HSQLDiskCacheUnitTest.java +++ b/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/jdbc/hsql/HSQLDiskCacheUnitTest.java @@ -1,5 +1,14 @@ package org.apache.commons.jcs.auxiliary.disk.jdbc.hsql; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.jcs.JCS; +import org.apache.commons.jcs.access.CacheAccess; +import org.apache.commons.jcs.access.exception.CacheException; +import org.apache.commons.jcs.engine.behavior.ICacheElement; + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -20,18 +29,9 @@ package org.apache.commons.jcs.auxiliary.disk.jdbc.hsql; */ import junit.framework.TestCase; -import org.apache.commons.jcs.JCS; -import org.apache.commons.jcs.access.CacheAccess; -import org.apache.commons.jcs.access.exception.CacheException; -import org.apache.commons.jcs.engine.behavior.ICacheElement; - -import java.util.HashSet; -import java.util.Map; -import java.util.Set; /** - * Test which exercises the indexed disk cache. This one uses three different regions for thre - * threads. + * Test which exercises the HSQL cache. */ public class HSQLDiskCacheUnitTest extends TestCase @@ -61,22 +61,15 @@ public class HSQLDiskCacheUnitTest CacheAccess<String, String> jcs = JCS.getInstance( region ); // Add items to cache - for ( int i = 0; i <= items; i++ ) { jcs.put( i + ":key", region + " data " + i ); } - //SleepUtil.sleepAtLeast( 1000 ); - -// System.out.println( jcs.getStats() ); - // Test that all items are in cache - for ( int i = 0; i <= items; i++ ) { String value = jcs.get( i + ":key" ); - assertEquals( "key = [" + i + ":key] value = [" + value + "]", region + " data " + i, value ); } @@ -95,16 +88,13 @@ public class HSQLDiskCacheUnitTest assertEquals( "value " + i + ":key", region + " data " + i, element.getVal() ); } - // Remove all the items - for ( int i = 0; i <= items; i++ ) { jcs.remove( i + ":key" ); } // Verify removal - for ( int i = 0; i <= items; i++ ) { assertNull( "Removed key should be null: " + i + ":key", jcs.get( i + ":key" ) ); @@ -126,7 +116,6 @@ public class HSQLDiskCacheUnitTest int items = 20; // Add items to cache - for ( int i = 0; i <= items; i++ ) { jcs.put( i + ":key", region + " data " + i ); @@ -146,7 +135,6 @@ public class HSQLDiskCacheUnitTest for ( int i = 0; i <= items; i++ ) { String value = jcs.get( i + ":key" ); - assertNull( "value should be null key = [" + i + ":key] value = [" + value + "]", value ); } } @@ -166,7 +154,6 @@ public class HSQLDiskCacheUnitTest int items = 20; // Add items to cache - for ( int i = 0; i <= items; i++ ) { jcs.put( i + ":key", region + " data " + i ); @@ -175,14 +162,11 @@ public class HSQLDiskCacheUnitTest // a db thread could be updating the disk when // Thread.sleep( 500 ); -// System.out.println( jcs.getStats() ); - jcs.clear(); for ( int i = 0; i <= items; i++ ) { String value = jcs.get( i + ":key" ); - assertEquals( "key = [" + i + ":key] value = [" + value + "]", region + " data " + i, value ); } } diff --git a/commons-jcs-core/src/test/test-conf/TestHSQLDiskCache.ccf b/commons-jcs-core/src/test/test-conf/TestHSQLDiskCache.ccf index 5134103..e3fe8b9 100644 --- a/commons-jcs-core/src/test/test-conf/TestHSQLDiskCache.ccf +++ b/commons-jcs-core/src/test/test-conf/TestHSQLDiskCache.ccf @@ -67,6 +67,7 @@ jcs.auxiliary.HSQL_NORA.attributes.tableName=JCS_STORE4 jcs.auxiliary.HSQL_NORA.attributes.testBeforeInsert=false jcs.auxiliary.HSQL_NORA.attributes.maxTotal=15 jcs.auxiliary.HSQL_NORA.attributes.allowRemoveAll=false +jcs.auxiliary.HSQL_NORA.attributes.useDiskShrinker=false jcs.auxiliary.HSQL_NORA.attributes.MaxPurgatorySize=10000000 jcs.auxiliary.HSQL_NORA.attributes.EventQueueType=POOLED jcs.auxiliary.HSQL_NORA.attributes.EventQueuePoolName=disk_cache_event_queue diff --git a/commons-jcs-core/src/test/test-conf/TestHSQLDiskCacheConcurrent.ccf b/commons-jcs-core/src/test/test-conf/TestHSQLDiskCacheConcurrent.ccf index 908de0f..8f2eb68 100644 --- a/commons-jcs-core/src/test/test-conf/TestHSQLDiskCacheConcurrent.ccf +++ b/commons-jcs-core/src/test/test-conf/TestHSQLDiskCacheConcurrent.ccf @@ -68,6 +68,7 @@ jcs.auxiliary.HSQL_NORA.attributes.tableName=JCS_STORE4 jcs.auxiliary.HSQL_NORA.attributes.testBeforeInsert=false jcs.auxiliary.HSQL_NORA.attributes.maxTotal=15 jcs.auxiliary.HSQL_NORA.attributes.allowRemoveAll=false +jcs.auxiliary.HSQL_NORA.attributes.useDiskShrinker=false jcs.auxiliary.HSQL_NORA.attributes.MaxPurgatorySize=10000000 jcs.auxiliary.HSQL_NORA.attributes.EventQueueType=POOLED jcs.auxiliary.HSQL_NORA.attributes.EventQueuePoolName=disk_cache_event_queue
