This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 33ba50162ff HDDS-15268. Speed up
TestUnhealthyContainersDerbyPerformance (a bit) (#10492)
33ba50162ff is described below
commit 33ba50162ff4f96e6879c153461a1016bb88ffed
Author: Arafat2198 <[email protected]>
AuthorDate: Sun Jun 14 21:27:37 2026 +0530
HDDS-15268. Speed up TestUnhealthyContainersDerbyPerformance (a bit)
(#10492)
Co-authored-by: Cursor <[email protected]>
---
.../recon/persistence/DerbyDataSourceProvider.java | 3 +-
.../TestUnhealthyContainersDerbyPerformance.java | 123 +++++++++++++++------
2 files changed, 94 insertions(+), 32 deletions(-)
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/DerbyDataSourceProvider.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/DerbyDataSourceProvider.java
index adafe200086..e0b413df517 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/DerbyDataSourceProvider.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/persistence/DerbyDataSourceProvider.java
@@ -51,7 +51,8 @@ public DataSource get() {
LOG.error("Error creating Recon Derby DB.", e);
}
EmbeddedDataSource dataSource = new EmbeddedDataSource();
- dataSource.setDatabaseName(jdbcUrl.split(":")[2]);
+ String dbName = jdbcUrl.replaceFirst("^jdbc:derby:", "");
+ dataSource.setDatabaseName(dbName);
dataSource.setUser(RECON_SCHEMA_NAME);
return dataSource;
}
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestUnhealthyContainersDerbyPerformance.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestUnhealthyContainersDerbyPerformance.java
index f4134c40219..66c6b271432 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestUnhealthyContainersDerbyPerformance.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestUnhealthyContainersDerbyPerformance.java
@@ -46,6 +46,7 @@
import org.apache.ozone.recon.schema.ReconSchemaGenerationModule;
import
org.apache.ozone.recon.schema.generated.tables.daos.UnhealthyContainersDao;
import org.jooq.DSLContext;
+import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
@@ -74,9 +75,9 @@
*
* <h2>Performance settings applied in this test</h2>
* <ul>
- * <li><b>Page cache</b>: {@code derby.storage.pageCacheSize = 20000}
- * (~80 MB of 4-KB pages) keeps hot B-tree nodes in memory, reducing
- * filesystem reads even with the file-based Derby driver.</li>
+ * <li><b>In-memory database</b>: The test uses an in-memory Derby database
+ * ({@code jdbc:derby:memory:...}) to eliminate disk I/O and fsync
overhead
+ * during the 1-million-row benchmark.</li>
* <li><b>JDBC fetch size</b>: set to {@value #READ_PAGE_SIZE} on each query
* so Derby pre-buffers a full page of rows per JDBC round-trip instead
* of the default 1-row-at-a-time fetch.</li>
@@ -184,8 +185,8 @@ public class TestUnhealthyContainersDerbyPerformance {
private static final int READ_PAGE_SIZE = 5_000;
// -----------------------------------------------------------------------
- // Performance thresholds (CI-safe; expected run times are 5–10× faster
- // than the original file-based Derby baseline after the optimisations)
+ // Performance thresholds (CI-safe; expected run times are significantly
faster
+ // than the original file-based Derby baseline after switching to in-memory)
// -----------------------------------------------------------------------
/** Maximum acceptable time to insert all TOTAL_RECORDS into Derby. */
@@ -220,43 +221,89 @@ public class TestUnhealthyContainersDerbyPerformance {
// One-time setup: create Derby schema + insert 1 M records
// -----------------------------------------------------------------------
+ private String inMemoryDbName;
+
/**
- * Initialises the embedded Derby database and creates the Recon schema.
+ * Initialises the embedded in-memory Derby database and creates the Recon
schema.
* Data population is done in dedicated test methods.
*
- * <p>The {@code @TempDir} is injected as a <em>method parameter</em> rather
- * than a class field. With {@code @TestInstance(PER_CLASS)}, a field-level
- * {@code @TempDir} is populated by JUnit's {@code TempDirExtension} in its
- * own {@code beforeAll} callback, which may run <em>after</em> the user's
- * {@code @BeforeAll} — leaving it null when needed here. A method
- * parameter is resolved by JUnit before the method body executes.</p>
- *
- * <h3>Performance settings applied here</h3>
- * <ul>
- * <li><b>Page cache</b> ({@code derby.storage.pageCacheSize = 20000}):
- * ~80 MB of 4-KB B-tree pages resident in heap — covers the hot path
- * for index scans on a 1-M-row table even with the file-based
- * driver.</li>
- * </ul>
+ * <p>Reuses {@link DerbyDataSourceConfigurationProvider} for all connection
settings
+ * and overrides only {@code getJdbcUrl()} to use an in-memory Derby database
+ * ({@code jdbc:derby:memory:...}), eliminating disk I/O and fsync overhead
across
+ * the benchmark's insert, replace, and delete transactions.</p>
*/
@BeforeAll
public void setUpDatabase(@TempDir Path tempDir) throws Exception {
+ inMemoryDbName = "reconPerf_" + java.util.UUID.randomUUID().toString();
LOG.info("=== Derby Performance Benchmark — Setup ===");
LOG.info("Dataset: {} states × {} container IDs = {} total records",
TESTED_STATES.size(), CONTAINER_ID_RANGE, TOTAL_RECORDS);
- // Derby engine property — must be set before the first connection.
- //
- // pageCacheSize: number of 4-KB pages Derby keeps in its buffer pool.
- // Default = 1,000 pages (4 MB) — far too small for a 1-M-row table.
- // 20,000 pages = ~80 MB, enough to hold the full B-tree for both the
- // primary-key index and the composite (state, container_id) index.
- System.setProperty("derby.storage.pageCacheSize", "20000");
-
- // ----- Guice wiring (mirrors AbstractReconSqlDBTest) -----
File configDir = Files.createDirectory(tempDir.resolve("Config")).toFile();
- Provider<DataSourceConfiguration> configProvider =
- new DerbyDataSourceConfigurationProvider(configDir);
+ DataSourceConfiguration base = new
DerbyDataSourceConfigurationProvider(configDir).get();
+
+ // Reuse DerbyDataSourceConfigurationProvider settings but point to an
in-memory DB.
+ Provider<DataSourceConfiguration> configProvider = () -> new
DataSourceConfiguration() {
+ @Override
+ public String getDriverClass() {
+ return base.getDriverClass();
+ }
+
+ @Override
+ public String getJdbcUrl() {
+ return "jdbc:derby:memory:" + inMemoryDbName;
+ }
+
+ @Override
+ public String getUserName() {
+ return base.getUserName();
+ }
+
+ @Override
+ public String getPassword() {
+ return base.getPassword();
+ }
+
+ @Override
+ public boolean setAutoCommit() {
+ return base.setAutoCommit();
+ }
+
+ @Override
+ public long getConnectionTimeout() {
+ return base.getConnectionTimeout();
+ }
+
+ @Override
+ public String getSqlDialect() {
+ return base.getSqlDialect();
+ }
+
+ @Override
+ public Integer getMaxActiveConnections() {
+ return base.getMaxActiveConnections();
+ }
+
+ @Override
+ public long getMaxConnectionAge() {
+ return base.getMaxConnectionAge();
+ }
+
+ @Override
+ public long getMaxIdleConnectionAge() {
+ return base.getMaxIdleConnectionAge();
+ }
+
+ @Override
+ public String getConnectionTestStatement() {
+ return base.getConnectionTestStatement();
+ }
+
+ @Override
+ public long getIdleConnectionTestPeriod() {
+ return base.getIdleConnectionTestPeriod();
+ }
+ };
Injector injector = Guice.createInjector(
new JooqPersistenceModule(configProvider),
@@ -277,6 +324,20 @@ protected void configure() {
schemaManager = new ContainerHealthSchemaManager(schemaDefinition, new
OzoneConfiguration());
}
+ @AfterAll
+ public void tearDownDatabase() {
+ if (inMemoryDbName != null) {
+ try (java.sql.Connection conn = java.sql.DriverManager.getConnection(
+ "jdbc:derby:memory:" + inMemoryDbName + ";drop=true")) {
+ conn.isValid(1);
+ } catch (java.sql.SQLException e) {
+ if (!"08006".equals(e.getSQLState())) {
+ LOG.warn("Unexpected SQLState while dropping in-memory Derby DB:
{}", e.getSQLState(), e);
+ }
+ }
+ }
+ }
+
// -----------------------------------------------------------------------
// Test 1 — Batch INSERT performance for 1M records
// -----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
