DomGarguilo commented on code in PR #6435:
URL: https://github.com/apache/accumulo/pull/6435#discussion_r3460725764
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -988,4 +1003,106 @@ void moveTableProperties(ServerContext context) {
LOG.info(
"Moving table properties from system configuration to namespace
configurations complete.");
}
+
+ // visible for IT
+ public void deleteCompactionTempFiles(final ServerContext ctx, final
DeleteStats stats,
+ final Collection<Path> deletedFiles) {
+
+ final String pattern = "/tables/*/*/*";
+ final Collection<Volume> vols = ctx.getVolumeManager().getVolumes();
+ final ExecutorService svc = Executors.newFixedThreadPool(vols.size());
+ final List<Future<Void>> futures = new ArrayList<>(vols.size());
+ final Set<Path> oldCompactionTmpFiles = new HashSet<>();
+
+ for (Volume vol : vols) {
+ final Path volPattern = new Path(vol.getBasePath() + pattern);
+ LOG.info("Looking for old compaction tmp files that match pattern: {}",
volPattern);
+ futures.add(svc.submit(() -> {
+ try {
+ FileStatus[] files = vol.getFileSystem().globStatus(volPattern,
+ (p) -> (p.getName().startsWith("" +
FilePrefix.FULL_COMPACTION.getPrefix())
+ || p.getName().startsWith("" +
FilePrefix.COMPACTION.getPrefix()))
+ && p.getName().endsWith(".rf_tmp"));
+ Arrays.stream(files).forEach(fs ->
oldCompactionTmpFiles.add(fs.getPath()));
+ } catch (IOException e) {
+ LOG.error("Error looking for old compaction tmp files in volume:
{}", vol, e);
+ }
+ return null;
+ }));
+ }
+ svc.shutdown();
+
+ LOG.info("Waiting for tasks to complete finding files");
+ while (futures.size() > 0) {
+ Iterator<Future<Void>> iter = futures.iterator();
+ while (iter.hasNext()) {
+ Future<Void> future = iter.next();
+ if (future.isDone()) {
+ iter.remove();
+ try {
+ future.get();
+ } catch (InterruptedException | ExecutionException e) {
+ throw new RuntimeException("Error getting list of old compaction
tmp files", e);
+ }
+ }
+ }
+ int remaining = futures.size();
+ if (remaining > 0) {
+ LOG.debug("Waiting for {} tasks to complete", remaining);
+ UtilWaitThread.sleep(3_000);
+ }
+ }
+ LOG.info("Found {} old compaction tmp files:",
oldCompactionTmpFiles.size());
+ oldCompactionTmpFiles.forEach(p -> LOG.debug("{}", p));
+
+ LOG.info("Deleting old compaction tmp files...");
+ final ExecutorService delSvc = Executors.newFixedThreadPool(vols.size());
+ // use a linked list to make removal from the middle of the list quick
+ final List<Future<Boolean>> delFutures = new LinkedList<>();
+
+ oldCompactionTmpFiles.forEach(p -> {
+ delFutures.add(delSvc.submit(() -> {
+ if (ctx.getVolumeManager().exists(p)) {
+ boolean result = ctx.getVolumeManager().delete(p);
+ if (result) {
+ LOG.debug("Removed old temp file {}", p);
+ deletedFiles.add(p);
Review Comment:
Might want to put this in a sync block too. Or add to a temp concurrent set
then `.addAll()` into this passed-in collection at the end.
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -988,4 +1003,106 @@ void moveTableProperties(ServerContext context) {
LOG.info(
"Moving table properties from system configuration to namespace
configurations complete.");
}
+
+ // visible for IT
+ public void deleteCompactionTempFiles(final ServerContext ctx, final
DeleteStats stats,
+ final Collection<Path> deletedFiles) {
+
+ final String pattern = "/tables/*/*/*";
+ final Collection<Volume> vols = ctx.getVolumeManager().getVolumes();
+ final ExecutorService svc = Executors.newFixedThreadPool(vols.size());
+ final List<Future<Void>> futures = new ArrayList<>(vols.size());
+ final Set<Path> oldCompactionTmpFiles = new HashSet<>();
Review Comment:
```suggestion
final Set<Path> oldCompactionTmpFiles = new
ConcurrentHashMap.newKeySet();
```
Might be safer to use a concurrent set here instead.
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -988,4 +1003,106 @@ void moveTableProperties(ServerContext context) {
LOG.info(
"Moving table properties from system configuration to namespace
configurations complete.");
}
+
+ // visible for IT
+ public void deleteCompactionTempFiles(final ServerContext ctx, final
DeleteStats stats,
+ final Collection<Path> deletedFiles) {
+
+ final String pattern = "/tables/*/*/*";
+ final Collection<Volume> vols = ctx.getVolumeManager().getVolumes();
+ final ExecutorService svc = Executors.newFixedThreadPool(vols.size());
+ final List<Future<Void>> futures = new ArrayList<>(vols.size());
+ final Set<Path> oldCompactionTmpFiles = new HashSet<>();
+
+ for (Volume vol : vols) {
+ final Path volPattern = new Path(vol.getBasePath() + pattern);
+ LOG.info("Looking for old compaction tmp files that match pattern: {}",
volPattern);
+ futures.add(svc.submit(() -> {
+ try {
+ FileStatus[] files = vol.getFileSystem().globStatus(volPattern,
+ (p) -> (p.getName().startsWith("" +
FilePrefix.FULL_COMPACTION.getPrefix())
+ || p.getName().startsWith("" +
FilePrefix.COMPACTION.getPrefix()))
+ && p.getName().endsWith(".rf_tmp"));
+ Arrays.stream(files).forEach(fs ->
oldCompactionTmpFiles.add(fs.getPath()));
+ } catch (IOException e) {
+ LOG.error("Error looking for old compaction tmp files in volume:
{}", vol, e);
+ }
+ return null;
+ }));
+ }
+ svc.shutdown();
+
+ LOG.info("Waiting for tasks to complete finding files");
+ while (futures.size() > 0) {
+ Iterator<Future<Void>> iter = futures.iterator();
+ while (iter.hasNext()) {
+ Future<Void> future = iter.next();
+ if (future.isDone()) {
+ iter.remove();
+ try {
+ future.get();
+ } catch (InterruptedException | ExecutionException e) {
+ throw new RuntimeException("Error getting list of old compaction
tmp files", e);
+ }
+ }
+ }
+ int remaining = futures.size();
+ if (remaining > 0) {
+ LOG.debug("Waiting for {} tasks to complete", remaining);
+ UtilWaitThread.sleep(3_000);
+ }
+ }
+ LOG.info("Found {} old compaction tmp files:",
oldCompactionTmpFiles.size());
+ oldCompactionTmpFiles.forEach(p -> LOG.debug("{}", p));
+
+ LOG.info("Deleting old compaction tmp files...");
+ final ExecutorService delSvc = Executors.newFixedThreadPool(vols.size());
+ // use a linked list to make removal from the middle of the list quick
+ final List<Future<Boolean>> delFutures = new LinkedList<>();
+
+ oldCompactionTmpFiles.forEach(p -> {
+ delFutures.add(delSvc.submit(() -> {
+ if (ctx.getVolumeManager().exists(p)) {
+ boolean result = ctx.getVolumeManager().delete(p);
+ if (result) {
+ LOG.debug("Removed old temp file {}", p);
+ deletedFiles.add(p);
+ } else {
+ LOG.error(
+ "Unable to remove old temp file {}, operation returned false
with no exception", p);
+ }
+ return result;
+ }
+ return true;
+ }));
+ });
+ delSvc.shutdown();
+
+ while (delFutures.size() > 0) {
+ Iterator<Future<Boolean>> iter = delFutures.iterator();
+ while (iter.hasNext()) {
+ Future<Boolean> future = iter.next();
+ if (future.isDone()) {
+ iter.remove();
+ try {
+ if (future.get()) {
+ stats.success++;
+ } else {
+ stats.failure++;
+ }
+ } catch (InterruptedException | ExecutionException e) {
+ stats.error++;
+ LOG.error("Error deleting a compaction tmp file", e);
+ }
+ }
+ }
+ int remaining = oldCompactionTmpFiles.size();
Review Comment:
```suggestion
int remaining = delFutures.size();
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
