This is an automated email from the ASF dual-hosted git repository.
andy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/jena.git
The following commit(s) were added to refs/heads/main by this push:
new b104143 JENA-2096 - Added ability to delete old DB after compaction
has completed successfully
new d144861 Merge pull request #988 from bsara/compact-delete-old
b104143 is described below
commit b10414354f8a16331e7c1a4598ca8279c0017f81
Author: Brandon Sara <[email protected]>
AuthorDate: Mon Apr 26 11:23:46 2021 -0600
JENA-2096 - Added ability to delete old DB after compaction has completed
successfully
---
jena-cmds/src/main/java/tdb2/tdbcompact.java | 19 ++++++-
.../java/org/apache/jena/tdb2/DatabaseMgr.java | 22 +++++++-
.../java/org/apache/jena/tdb2/sys/DatabaseOps.java | 64 +++++++++++++++-------
.../org/apache/jena/tdb2/sys/TestDatabaseOps.java | 41 ++++++++++++--
.../org/apache/jena/fuseki/ctl/ActionCompact.java | 11 +++-
.../apache/jena/fuseki/main/TestConfigFile.java | 50 ++++++++++++++---
.../apache/jena/fuseki/main/TestFusekiMainCmd.java | 18 +++++-
.../testing/Config/server-tdb2.ttl | 1 -
.../{server-tdb2.ttl => server-tdb2_compact0.ttl} | 5 +-
.../{server-tdb2.ttl => server-tdb2_compact1.ttl} | 5 +-
10 files changed, 186 insertions(+), 50 deletions(-)
diff --git a/jena-cmds/src/main/java/tdb2/tdbcompact.java
b/jena-cmds/src/main/java/tdb2/tdbcompact.java
index a762381..d2bca64 100644
--- a/jena-cmds/src/main/java/tdb2/tdbcompact.java
+++ b/jena-cmds/src/main/java/tdb2/tdbcompact.java
@@ -18,18 +18,33 @@
package tdb2;
+import org.apache.jena.cmd.ArgDecl;
import org.apache.jena.tdb2.store.DatasetGraphSwitchable;
import org.apache.jena.tdb2.sys.DatabaseOps;
import tdb2.cmdline.CmdTDB;
public class tdbcompact extends CmdTDB {
+ private static final ArgDecl argDeleteOld = new ArgDecl(ArgDecl.NoValue,
"deleteOld");
+
+ private boolean shouldDeleteOld = false;
+
+
static public void main(String... argv) {
CmdTDB.init() ;
new tdbcompact(argv).mainRun() ;
}
protected tdbcompact(String[] argv) {
- super(argv) ;
+ super(argv);
+
+ super.add(argDeleteOld, "--deleteOld", "Delete old database after
compaction");
+ }
+
+ @Override
+ protected void processModulesAndArgs() {
+ super.processModulesAndArgs();
+
+ shouldDeleteOld = contains(argDeleteOld);
}
@Override
@@ -41,7 +56,7 @@ public class tdbcompact extends CmdTDB {
protected void exec() {
DatasetGraphSwitchable dsg = getDatabaseContainer() ;
long start = System.currentTimeMillis();
- DatabaseOps.compact(dsg) ;
+ DatabaseOps.compact(dsg, shouldDeleteOld) ;
long finish = System.currentTimeMillis();
System.out.printf("Compacted in %.3fs\n", (finish-start)/1000.0);
}
diff --git
a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/DatabaseMgr.java
b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/DatabaseMgr.java
index 54351a1..479895f 100644
--- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/DatabaseMgr.java
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/DatabaseMgr.java
@@ -33,7 +33,7 @@ import org.apache.jena.tdb2.sys.TDBInternal;
* @see TDB2Factory
*/
public class DatabaseMgr {
-
+
private DatabaseMgr() {}
// All creation of DatasetGraph for TDB2 goes through this method.
@@ -55,11 +55,29 @@ public class DatabaseMgr {
* Compact a datasets which must be a switchable TDB database.
* This is the normal dataset type for on-disk TDB2 databases.
*
+ * Deletes old database after successful compaction if `shouldDeleteOld`
is `true`.
+ *
* @param container
+ *
+ * @deprecated Use `compact(container, false)` instead.
*/
+ @Deprecated
public static void compact(DatasetGraph container) {
+ compact(container, false);
+ }
+
+ /**
+ * Compact a datasets which must be a switchable TDB database.
+ * This is the normal dataset type for on-disk TDB2 databases.
+ *
+ * Deletes old database after successful compaction if `shouldDeleteOld`
is `true`.
+ *
+ * @param container
+ * @param shouldDeleteOld
+ */
+ public static void compact(DatasetGraph container, boolean
shouldDeleteOld) {
DatasetGraphSwitchable dsg = requireSwitchable(container);
- DatabaseOps.compact(dsg);
+ DatabaseOps.compact(dsg, shouldDeleteOld);
}
/**
diff --git
a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/sys/DatabaseOps.java
b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/sys/DatabaseOps.java
index 4745d10..1026f7d 100644
--- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/sys/DatabaseOps.java
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/sys/DatabaseOps.java
@@ -20,7 +20,9 @@ package org.apache.jena.tdb2.sys;
import java.io.*;
import java.nio.file.*;
+import java.util.Comparator;
import java.util.List;
+import java.util.stream.Stream;
import java.util.zip.GZIPOutputStream;
import org.apache.jena.atlas.RuntimeIOException;
@@ -83,19 +85,19 @@ public class DatabaseOps {
return new DatasetGraphSwitchable(null, location, dsg);
}
// Exists?
- if ( ! location.exists() )
- throw new TDBException("No such location: "+location);
- Path path = IOX.asPath(location);
- // Scan for DBs
- Path db = findLocation(path, dbPrefix);
- if ( db == null ) {
- db = path.resolve(dbPrefix+SEP+startCount);
- IOX.createDirectory(db);
- }
- Location loc2 = IOX.asLocation(db);
- DatasetGraphTDB dsg = StoreConnection.connectCreate(loc2,
params).getDatasetGraphTDB();
- DatasetGraphSwitchable appDSG = new DatasetGraphSwitchable(path,
location, dsg);
- return appDSG;
+ if ( ! location.exists() )
+ throw new TDBException("No such location: "+location);
+ Path path = IOX.asPath(location);
+ // Scan for DBs
+ Path db = findLocation(path, dbPrefix);
+ if ( db == null ) {
+ db = path.resolve(dbPrefix+SEP+startCount);
+ IOX.createDirectory(db);
+ }
+ Location loc2 = IOX.asLocation(db);
+ DatasetGraphTDB dsg = StoreConnection.connectCreate(loc2,
params).getDatasetGraphTDB();
+ DatasetGraphSwitchable appDSG = new DatasetGraphSwitchable(path,
location, dsg);
+ return appDSG;
}
public static String backup(DatasetGraphSwitchable container) {
@@ -107,13 +109,13 @@ public class DatabaseOps {
DatasetGraph dsg = container;
-// // Per backup source lock.
-// synchronized(activeBackups) {
-// // Atomically check-and-set
-// if ( activeBackups.contains(dsg) )
-// Log.warn(Fuseki.serverLog, "Backup already in progress");
-// activeBackups.add(dsg);
-// }
+ // // Per backup source lock.
+ // synchronized(activeBackups) {
+ // // Atomically check-and-set
+ // if ( activeBackups.contains(dsg) )
+ // Log.warn(Fuseki.serverLog, "Backup already in progress");
+ // activeBackups.add(dsg);
+ // }
Pair<OutputStream, Path> x = openUniqueFileForWriting(backupDir,
BACKUPS_FN, "nq.gz");
try (OutputStream out2 = x.getLeft();
@@ -165,7 +167,16 @@ public class DatabaseOps {
// JVM-wide :-(
private static Object compactionLock = new Object();
+
+ /**
+ * @deprecated Use `compact(container, false)` instead.
+ */
+ @Deprecated
public static void compact(DatasetGraphSwitchable container) {
+ compact(container, false);
+ }
+
+ public static void compact(DatasetGraphSwitchable container, boolean
shouldDeleteOld) {
checkSupportsAdmin(container);
synchronized(compactionLock) {
Path base = container.getContainerPath();
@@ -195,6 +206,19 @@ public class DatabaseOps {
LOG.debug(String.format("Compact %s -> %s\n", db1.getFileName(),
db2.getFileName()));
compact(container, loc1, loc2);
+
+ if ( shouldDeleteOld ) {
+ Path loc1Path = IOX.asPath(loc1);
+ LOG.debug("Deleting old database after successful compaction
(old db path='" + loc1Path + "')...");
+
+ try (Stream<Path> walk = Files.walk(loc1Path)){
+ walk.sorted(Comparator.reverseOrder())
+ .map(Path::toFile)
+ .forEach(File::delete);
+ } catch (IOException ex) {
+ throw IOX.exception(ex);
+ }
+ }
}
}
diff --git
a/jena-db/jena-tdb2/src/test/java/org/apache/jena/tdb2/sys/TestDatabaseOps.java
b/jena-db/jena-tdb2/src/test/java/org/apache/jena/tdb2/sys/TestDatabaseOps.java
index 7584d0f..4314f45 100644
---
a/jena-db/jena-tdb2/src/test/java/org/apache/jena/tdb2/sys/TestDatabaseOps.java
+++
b/jena-db/jena-tdb2/src/test/java/org/apache/jena/tdb2/sys/TestDatabaseOps.java
@@ -72,7 +72,7 @@ public class TestDatabaseOps
dsg.add(quad2);
dsg.add(quad1);
});
- DatabaseMgr.compact(dsg);
+ DatabaseMgr.compact(dsg, false);
assertFalse(StoreConnection.isSetup(loc1));
@@ -108,7 +108,7 @@ public class TestDatabaseOps
dsg.add(quad2);
dsg.add(quad1);
});
- DatabaseMgr.compact(dsg);
+ DatabaseMgr.compact(dsg, false);
Txn.executeRead(dsg, ()-> {
assertEquals(2, g.size());
assertTrue(g.contains(triple2));
@@ -143,9 +143,9 @@ public class TestDatabaseOps
compact_prefixes_3_test();
} catch (NullPointerException ex) {
ex.printStackTrace();
-// StackTraceElement[] x = ex.getStackTrace();
-// if ( x.length >= 0 &&
"java.nio.file.Files".equals(x[0].getClassName()) )
-// return ;
+ // StackTraceElement[] x = ex.getStackTrace();
+ // if ( x.length >= 0 &&
"java.nio.file.Files".equals(x[0].getClassName()) )
+ // return ;
throw ex;
}
}
@@ -169,7 +169,7 @@ public class TestDatabaseOps
int x1 = Txn.calculateRead(dsg, ()->dsg.prefixes().size());
assertTrue("Prefxies count", x1 > 0);
- DatabaseMgr.compact(dsgs); // HERE
+ DatabaseMgr.compact(dsgs, false); // HERE
// After
int x2 = Txn.calculateRead(dsg, ()->dsg.prefixes().size());
@@ -193,6 +193,35 @@ public class TestDatabaseOps
Txn.executeRead(dsgOld,
()->assertNull(dsgOld.getDefaultGraph().getPrefixMapping().getNsPrefixURI("ex")));
}
+ @Test public void compact_delete() {
+ DatasetGraph dsg = DatabaseMgr.connectDatasetGraph(dir);
+ DatasetGraphSwitchable dsgs = (DatasetGraphSwitchable)dsg;
+ DatasetGraph dsg1 = dsgs.get();
+ Location loc1 = ((DatasetGraphTDB)dsg1).getLocation();
+
+ Txn.executeWrite(dsg, ()-> {
+ dsg.add(quad2);
+ dsg.add(quad1);
+ });
+ DatabaseMgr.compact(dsg, true);
+
+ assertFalse(IOX.asFile(loc1).exists());
+
+ DatasetGraph dsg2 = dsgs.get();
+ Location loc2 = ((DatasetGraphTDB)dsg2).getLocation();
+
+ assertNotEquals(dsg1, dsg2);
+ assertNotEquals(loc1, loc2);
+
+ Txn.executeRead(dsg, ()-> {
+ assertTrue(dsg.contains(quad2));
+ assertTrue(dsg.contains(quad1));
+ });
+
+ Txn.executeRead(dsg, ()->assertTrue(dsg.contains(quad2)) );
+ Txn.executeRead(dsg2, ()->assertTrue(dsg2.contains(quad2)) );
+ }
+
@Test public void backup_1() {
DatasetGraph dsg = DatabaseMgr.connectDatasetGraph(dir);
Txn.executeWrite(dsg, ()-> {
diff --git
a/jena-fuseki2/jena-fuseki-core/src/main/java/org/apache/jena/fuseki/ctl/ActionCompact.java
b/jena-fuseki2/jena-fuseki-core/src/main/java/org/apache/jena/fuseki/ctl/ActionCompact.java
index c84b91e..a7234ca 100644
---
a/jena-fuseki2/jena-fuseki-core/src/main/java/org/apache/jena/fuseki/ctl/ActionCompact.java
+++
b/jena-fuseki2/jena-fuseki-core/src/main/java/org/apache/jena/fuseki/ctl/ActionCompact.java
@@ -88,10 +88,17 @@ public class ActionCompact extends ActionAsyncTask
}
static class CompactTask extends TaskBase {
- static private Logger log = Fuseki.compactLog;
+ static private final Logger log = Fuseki.compactLog;
+
+ private final boolean shouldDeleteOld;
public CompactTask(HttpAction action) {
super(action);
+
+ String deleteOldParam = action.request.getParameter("deleteOld");
+
+ this.shouldDeleteOld = ( deleteOldParam != null
+ && ( deleteOldParam.isEmpty() ||
deleteOldParam.equalsIgnoreCase("true") ) );
}
@Override
@@ -99,7 +106,7 @@ public class ActionCompact extends ActionAsyncTask
try {
DatasetGraph dsg = getTDB2(dataset);
log.info(format("[%d] >>>> Start compact %s", actionId,
datasetName));
- DatabaseMgr.compact(dsg);
+ DatabaseMgr.compact(dsg, this.shouldDeleteOld);
log.info(format("[%d] <<<< Finish compact %s", actionId,
datasetName));
} catch (Throwable ex) {
log.warn(format("[%d] **** Exception in compact", actionId),
ex);
diff --git
a/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/main/TestConfigFile.java
b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/main/TestConfigFile.java
index c2e7446..2932eac 100644
---
a/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/main/TestConfigFile.java
+++
b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/main/TestConfigFile.java
@@ -32,6 +32,7 @@ import org.apache.jena.atlas.io.IO;
import org.apache.jena.atlas.lib.StrUtils;
import org.apache.jena.atlas.web.TypedInputStream;
import org.apache.jena.atlas.web.WebLib;
+import org.apache.jena.base.Sys;
import org.apache.jena.graph.Graph;
import org.apache.jena.query.QueryExecution;
import org.apache.jena.rdfconnection.RDFConnection;
@@ -246,16 +247,51 @@ public class TestConfigFile {
assertNotNull(x2);
String x3 =
HttpOp.execHttpGetString("http://localhost:"+port+"/$/metrics";);
assertNotNull(x3);
- String x4 =
HttpOp.execHttpGetString("http://localhost:"+port+"/$/tasks";);
- assertNotNull(x4);
- try(TypedInputStream x5 =
HttpOp.execHttpPostStream("http://localhost:"+port+"/$/compact/ds";, null,
"application/json")) {
- assertNotNull(x5);
- assertNotEquals(0, x5.readAllBytes().length);
+ } finally {
+ server.stop();
+ }
+ }
+
+ @Test public void serverTDB2_compact0() {
+ int port = WebLib.choosePort();
+ FusekiServer server = server(port, "server-tdb2_compact0.ttl");
+ server.start();
+ try {
+ String x1=
HttpOp.execHttpGetString("http://localhost:"+port+"/$/tasks";);
+ assertNotNull(x1);
+ try(TypedInputStream x2 =
HttpOp.execHttpPostStream("http://localhost:"+port+"/$/compact/ds";, null,
"application/json")) {
+ assertNotNull(x2);
+ assertNotEquals(0, x2.readAllBytes().length);
} catch (IOException ex) {
IO.exception(ex);
}
- String x6 =
HttpOp.execHttpGetString("http://localhost:"+port+"/$/tasks/1";);
- assertNotNull(x6);
+ String x3 =
HttpOp.execHttpGetString("http://localhost:"+port+"/$/tasks/1";);
+ assertNotNull(x3);
+ } finally {
+ server.stop();
+ }
+ }
+
+ @Test public void serverTDB2_compact1() {
+ if ( Sys.isWindows ) {
+ // NOTE: Skipping deletion test for windows
+ return;
+ }
+
+ int port = WebLib.choosePort();
+ FusekiServer server = server(port, "server-tdb2_compact1.ttl");
+ server.start();
+ try {
+ String x1=
HttpOp.execHttpGetString("http://localhost:"+port+"/$/tasks";);
+ assertNotNull(x1);
+ try(TypedInputStream x2 =
HttpOp.execHttpPostStream("http://localhost:"+port+"/$/compact/ds?deleteOld";,
null, "application/json")) {
+ assertNotNull(x2);
+ assertNotEquals(0, x2.readAllBytes().length);
+ } catch (IOException ex) {
+ IO.exception(ex);
+ }
+ String x3 =
HttpOp.execHttpGetString("http://localhost:"+port+"/$/tasks/1";);
+ assertNotNull(x3);
} finally {
server.stop();
}
diff --git
a/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/main/TestFusekiMainCmd.java
b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/main/TestFusekiMainCmd.java
index 1ced4fa..f292a6e 100644
---
a/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/main/TestFusekiMainCmd.java
+++
b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/main/TestFusekiMainCmd.java
@@ -96,10 +96,10 @@ public class TestFusekiMainCmd {
}
@Test public void compact_01() throws IOException {
- String DB_DIR = DATABASES+"/DB-compact";
+ String DB_DIR = DATABASES+"/DB-compact1";
FileOps.ensureDir(DB_DIR);
FileOps.clearAll(DB_DIR);
- server("--loc="+DATABASES+"/DB-compact", "--tdb2", "--compact", "/ds");
+ server("--loc="+DATABASES+"/DB-compact1", "--tdb2", "--compact",
"/ds");
try(TypedInputStream x0 =
HttpOp.execHttpPostStream(serverURL+"/$/compact/ds", null, "application/json"))
{
assertNotNull(x0);
assertNotEquals(0, x0.readAllBytes().length);
@@ -110,4 +110,18 @@ public class TestFusekiMainCmd {
JSON.parseAny(x1);
// Leaves "DB-compact" behind.
}
+
+ @Test public void compact_02() throws IOException {
+ String DB_DIR = DATABASES+"/DB-compact2";
+ FileOps.ensureDir(DB_DIR);
+ FileOps.clearAll(DB_DIR);
+ server("--loc="+DATABASES+"/DB-compact2", "--tdb2", "--compact",
"/ds");
+ try(TypedInputStream x0 =
HttpOp.execHttpPostStream(serverURL+"/$/compact/ds?deleteOld", null,
"application/json")) {
+ assertNotNull(x0);
+ assertNotEquals(0, x0.readAllBytes().length);
+ }
+ String x1 = HttpOp.execHttpGetString(serverURL+"/$/tasks");
+ assertNotNull(x1);
+ // Leaves "DB-compact" behind.
+ }
}
diff --git a/jena-fuseki2/jena-fuseki-main/testing/Config/server-tdb2.ttl
b/jena-fuseki2/jena-fuseki-main/testing/Config/server-tdb2.ttl
index 18a873a..20afd12 100644
--- a/jena-fuseki2/jena-fuseki-main/testing/Config/server-tdb2.ttl
+++ b/jena-fuseki2/jena-fuseki-main/testing/Config/server-tdb2.ttl
@@ -13,7 +13,6 @@ PREFIX tdb2: <http://jena.apache.org/2016/tdb#>
fuseki:pingEP true ;
fuseki:statsEP true ;
fuseki:metricsEP true ;
- fuseki:compactEP true ;
.
<#service1> rdf:type fuseki:Service ;
diff --git a/jena-fuseki2/jena-fuseki-main/testing/Config/server-tdb2.ttl
b/jena-fuseki2/jena-fuseki-main/testing/Config/server-tdb2_compact0.ttl
similarity index 86%
copy from jena-fuseki2/jena-fuseki-main/testing/Config/server-tdb2.ttl
copy to jena-fuseki2/jena-fuseki-main/testing/Config/server-tdb2_compact0.ttl
index 18a873a..1de24ac 100644
--- a/jena-fuseki2/jena-fuseki-main/testing/Config/server-tdb2.ttl
+++ b/jena-fuseki2/jena-fuseki-main/testing/Config/server-tdb2_compact0.ttl
@@ -10,9 +10,6 @@ PREFIX tdb2: <http://jena.apache.org/2016/tdb#>
[] rdf:type fuseki:Server ;
fuseki:services ( <#service1> ) ;
- fuseki:pingEP true ;
- fuseki:statsEP true ;
- fuseki:metricsEP true ;
fuseki:compactEP true ;
.
@@ -26,5 +23,5 @@ PREFIX tdb2: <http://jena.apache.org/2016/tdb#>
.
<#emptyDataset> rdf:type tdb2:DatasetTDB ;
- tdb2:location "--mem--" ;
+ tdb2:location "target/Databases/DB-serverTDB2_compact0" ;
.
diff --git a/jena-fuseki2/jena-fuseki-main/testing/Config/server-tdb2.ttl
b/jena-fuseki2/jena-fuseki-main/testing/Config/server-tdb2_compact1.ttl
similarity index 86%
copy from jena-fuseki2/jena-fuseki-main/testing/Config/server-tdb2.ttl
copy to jena-fuseki2/jena-fuseki-main/testing/Config/server-tdb2_compact1.ttl
index 18a873a..9e8a40b 100644
--- a/jena-fuseki2/jena-fuseki-main/testing/Config/server-tdb2.ttl
+++ b/jena-fuseki2/jena-fuseki-main/testing/Config/server-tdb2_compact1.ttl
@@ -10,9 +10,6 @@ PREFIX tdb2: <http://jena.apache.org/2016/tdb#>
[] rdf:type fuseki:Server ;
fuseki:services ( <#service1> ) ;
- fuseki:pingEP true ;
- fuseki:statsEP true ;
- fuseki:metricsEP true ;
fuseki:compactEP true ;
.
@@ -26,5 +23,5 @@ PREFIX tdb2: <http://jena.apache.org/2016/tdb#>
.
<#emptyDataset> rdf:type tdb2:DatasetTDB ;
- tdb2:location "--mem--" ;
+ tdb2:location "target/Databases/DB-serverTDB2_compact1" ;
.
