dlmarion commented on code in PR #6435:
URL: https://github.com/apache/accumulo/pull/6435#discussion_r3437404419
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -988,4 +1003,106 @@ void moveTableProperties(ServerContext context) {
LOG.info(
"Moving table properties from system configuration to namespace
configurations complete.");
}
+
+ // visible for IT
+ public void deleteCompactionTempFiles(final ServerContext ctx, final
DeleteStats stats,
+ final Collection<Path> deletedFiles) {
+
+ final String pattern = "/tables/*/*/*";
+ final Collection<Volume> vols = ctx.getVolumeManager().getVolumes();
+ final ExecutorService svc = Executors.newFixedThreadPool(vols.size());
+ final List<Future<Void>> futures = new ArrayList<>(vols.size());
+ final Set<Path> oldCompactionTmpFiles = new HashSet<>();
+
+ for (Volume vol : vols) {
+ final Path volPattern = new Path(vol.getBasePath() + pattern);
+ LOG.info("Looking for old compaction tmp files that match pattern: {}",
volPattern);
+ futures.add(svc.submit(() -> {
+ try {
+ FileStatus[] files = vol.getFileSystem().globStatus(volPattern,
+ (p) -> (p.getName().startsWith("" +
FilePrefix.FULL_COMPACTION.getPrefix())
+ || p.getName().startsWith("" +
FilePrefix.COMPACTION.getPrefix()))
+ && p.getName().endsWith(".rf_tmp"));
+ Arrays.stream(files).forEach(fs ->
oldCompactionTmpFiles.add(fs.getPath()));
+ } catch (IOException e) {
+ LOG.error("Error looking for old compaction tmp files in volume:
{}", vol, e);
+ }
+ return null;
+ }));
+ }
+ svc.shutdown();
+
+ LOG.info("Waiting for tasks to complete finding files");
+ while (futures.size() > 0) {
+ Iterator<Future<Void>> iter = futures.iterator();
+ while (iter.hasNext()) {
+ Future<Void> future = iter.next();
+ if (future.isDone()) {
+ iter.remove();
+ try {
+ future.get();
+ } catch (InterruptedException | ExecutionException e) {
+ throw new RuntimeException("Error getting list of old compaction
tmp files", e);
+ }
+ }
+ }
+ int remaining = futures.size();
+ if (remaining > 0) {
+ LOG.debug("Waiting for {} tasks to complete", remaining);
+ UtilWaitThread.sleep(3_000);
+ }
+ }
+ LOG.info("Found {} old compaction tmp files:",
oldCompactionTmpFiles.size());
+ oldCompactionTmpFiles.forEach(p -> LOG.debug("{}", p));
+
+ LOG.info("Deleting old compaction tmp files...");
+ final ExecutorService delSvc = Executors.newFixedThreadPool(vols.size());
+ // use a linked list to make removal from the middle of the list quick
+ final List<Future<Boolean>> delFutures = new LinkedList<>();
+
+ oldCompactionTmpFiles.forEach(p -> {
+ delFutures.add(delSvc.submit(() -> {
+ if (ctx.getVolumeManager().exists(p)) {
+ boolean result = ctx.getVolumeManager().delete(p);
+ if (result) {
+ LOG.debug("Removed old temp file {}", p);
+ deletedFiles.add(p);
+ } else {
+ LOG.error(
+ "Unable to remove old temp file {}, operation returned false
with no exception", p);
+ }
+ return result;
+ }
+ return true;
+ }));
+ });
+ delSvc.shutdown();
+
+ while (delFutures.size() > 0) {
+ Iterator<Future<Boolean>> iter = delFutures.iterator();
+ while (iter.hasNext()) {
+ Future<Boolean> future = iter.next();
+ if (future.isDone()) {
+ iter.remove();
+ try {
+ if (future.get()) {
+ stats.success++;
+ } else {
+ stats.failure++;
+ }
+ } catch (InterruptedException | ExecutionException e) {
+ stats.error++;
+ LOG.error("Error deleting a compaction tmp file", e);
+ }
+ }
+ }
+ int remaining = oldCompactionTmpFiles.size();
+ if (remaining > 0) {
+ LOG.debug("Waiting on {} background delete operations", remaining);
+ UtilWaitThread.sleep(3_000);
+ }
+ }
+ LOG.info("Deletion of compaction tmp files completed. Success:{},
Failure:{}, Error:{}",
+ stats.success, stats.failure, stats.error);
Review Comment:
I wonder if we should throw an error if failures or errors is non-zero. I
feel like we should not and treat this as a best effort.
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -988,4 +1003,106 @@ void moveTableProperties(ServerContext context) {
LOG.info(
"Moving table properties from system configuration to namespace
configurations complete.");
}
+
+ // visible for IT
+ public void deleteCompactionTempFiles(final ServerContext ctx, final
DeleteStats stats,
+ final Collection<Path> deletedFiles) {
+
+ final String pattern = "/tables/*/*/*";
+ final Collection<Volume> vols = ctx.getVolumeManager().getVolumes();
+ final ExecutorService svc = Executors.newFixedThreadPool(vols.size());
+ final List<Future<Void>> futures = new ArrayList<>(vols.size());
+ final Set<Path> oldCompactionTmpFiles = new HashSet<>();
+
+ for (Volume vol : vols) {
+ final Path volPattern = new Path(vol.getBasePath() + pattern);
+ LOG.info("Looking for old compaction tmp files that match pattern: {}",
volPattern);
+ futures.add(svc.submit(() -> {
+ try {
+ FileStatus[] files = vol.getFileSystem().globStatus(volPattern,
+ (p) -> (p.getName().startsWith("" +
FilePrefix.FULL_COMPACTION.getPrefix())
+ || p.getName().startsWith("" +
FilePrefix.COMPACTION.getPrefix()))
+ && p.getName().endsWith(".rf_tmp"));
+ Arrays.stream(files).forEach(fs ->
oldCompactionTmpFiles.add(fs.getPath()));
+ } catch (IOException e) {
+ LOG.error("Error looking for old compaction tmp files in volume:
{}", vol, e);
+ }
+ return null;
+ }));
+ }
+ svc.shutdown();
+
+ LOG.info("Waiting for tasks to complete finding files");
+ while (futures.size() > 0) {
+ Iterator<Future<Void>> iter = futures.iterator();
+ while (iter.hasNext()) {
+ Future<Void> future = iter.next();
+ if (future.isDone()) {
+ iter.remove();
+ try {
+ future.get();
+ } catch (InterruptedException | ExecutionException e) {
+ throw new RuntimeException("Error getting list of old compaction
tmp files", e);
+ }
+ }
+ }
+ int remaining = futures.size();
+ if (remaining > 0) {
+ LOG.debug("Waiting for {} tasks to complete", remaining);
+ UtilWaitThread.sleep(3_000);
+ }
+ }
+ LOG.info("Found {} old compaction tmp files:",
oldCompactionTmpFiles.size());
+ oldCompactionTmpFiles.forEach(p -> LOG.debug("{}", p));
+
+ LOG.info("Deleting old compaction tmp files...");
+ final ExecutorService delSvc = Executors.newFixedThreadPool(vols.size());
+ // use a linked list to make removal from the middle of the list quick
+ final List<Future<Boolean>> delFutures = new LinkedList<>();
+
+ oldCompactionTmpFiles.forEach(p -> {
+ delFutures.add(delSvc.submit(() -> {
+ if (ctx.getVolumeManager().exists(p)) {
+ boolean result = ctx.getVolumeManager().delete(p);
+ if (result) {
+ LOG.debug("Removed old temp file {}", p);
+ deletedFiles.add(p);
Review Comment:
Fixed in 81c35d3.
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -988,4 +1003,106 @@ void moveTableProperties(ServerContext context) {
LOG.info(
"Moving table properties from system configuration to namespace
configurations complete.");
}
+
+ // visible for IT
+ public void deleteCompactionTempFiles(final ServerContext ctx, final
DeleteStats stats,
+ final Collection<Path> deletedFiles) {
+
+ final String pattern = "/tables/*/*/*";
+ final Collection<Volume> vols = ctx.getVolumeManager().getVolumes();
+ final ExecutorService svc = Executors.newFixedThreadPool(vols.size());
+ final List<Future<Void>> futures = new ArrayList<>(vols.size());
+ final Set<Path> oldCompactionTmpFiles = new HashSet<>();
Review Comment:
Fixed in 81c35d3.
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -921,6 +935,7 @@ private void upgradeTabletsMetadata(@NonNull ServerContext
context, String metaN
}
}
+ // visible for IT
Review Comment:
Fixed in 81c35d3.
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -988,4 +1003,106 @@ void moveTableProperties(ServerContext context) {
LOG.info(
"Moving table properties from system configuration to namespace
configurations complete.");
}
+
+ // visible for IT
+ public void deleteCompactionTempFiles(final ServerContext ctx, final
DeleteStats stats,
+ final Collection<Path> deletedFiles) {
+
+ final String pattern = "/tables/*/*/*";
+ final Collection<Volume> vols = ctx.getVolumeManager().getVolumes();
+ final ExecutorService svc = Executors.newFixedThreadPool(vols.size());
+ final List<Future<Void>> futures = new ArrayList<>(vols.size());
+ final Set<Path> oldCompactionTmpFiles = new HashSet<>();
+
+ for (Volume vol : vols) {
+ final Path volPattern = new Path(vol.getBasePath() + pattern);
+ LOG.info("Looking for old compaction tmp files that match pattern: {}",
volPattern);
+ futures.add(svc.submit(() -> {
+ try {
+ FileStatus[] files = vol.getFileSystem().globStatus(volPattern,
+ (p) -> (p.getName().startsWith("" +
FilePrefix.FULL_COMPACTION.getPrefix())
+ || p.getName().startsWith("" +
FilePrefix.COMPACTION.getPrefix()))
+ && p.getName().endsWith(".rf_tmp"));
+ Arrays.stream(files).forEach(fs ->
oldCompactionTmpFiles.add(fs.getPath()));
+ } catch (IOException e) {
+ LOG.error("Error looking for old compaction tmp files in volume:
{}", vol, e);
+ }
+ return null;
+ }));
+ }
+ svc.shutdown();
+
+ LOG.info("Waiting for tasks to complete finding files");
+ while (futures.size() > 0) {
+ Iterator<Future<Void>> iter = futures.iterator();
+ while (iter.hasNext()) {
+ Future<Void> future = iter.next();
+ if (future.isDone()) {
+ iter.remove();
+ try {
+ future.get();
+ } catch (InterruptedException | ExecutionException e) {
+ throw new RuntimeException("Error getting list of old compaction
tmp files", e);
+ }
+ }
+ }
+ int remaining = futures.size();
+ if (remaining > 0) {
+ LOG.debug("Waiting for {} tasks to complete", remaining);
+ UtilWaitThread.sleep(3_000);
+ }
+ }
+ LOG.info("Found {} old compaction tmp files:",
oldCompactionTmpFiles.size());
+ oldCompactionTmpFiles.forEach(p -> LOG.debug("{}", p));
+
+ LOG.info("Deleting old compaction tmp files...");
+ final ExecutorService delSvc = Executors.newFixedThreadPool(vols.size());
+ // use a linked list to make removal from the middle of the list quick
+ final List<Future<Boolean>> delFutures = new LinkedList<>();
+
+ oldCompactionTmpFiles.forEach(p -> {
+ delFutures.add(delSvc.submit(() -> {
+ if (ctx.getVolumeManager().exists(p)) {
+ boolean result = ctx.getVolumeManager().delete(p);
+ if (result) {
+ LOG.debug("Removed old temp file {}", p);
+ deletedFiles.add(p);
+ } else {
+ LOG.error(
+ "Unable to remove old temp file {}, operation returned false
with no exception", p);
+ }
+ return result;
+ }
+ return true;
+ }));
+ });
+ delSvc.shutdown();
+
+ while (delFutures.size() > 0) {
+ Iterator<Future<Boolean>> iter = delFutures.iterator();
+ while (iter.hasNext()) {
+ Future<Boolean> future = iter.next();
+ if (future.isDone()) {
+ iter.remove();
+ try {
+ if (future.get()) {
+ stats.success++;
+ } else {
+ stats.failure++;
+ }
+ } catch (InterruptedException | ExecutionException e) {
+ stats.error++;
+ LOG.error("Error deleting a compaction tmp file", e);
+ }
+ }
+ }
+ int remaining = oldCompactionTmpFiles.size();
Review Comment:
Fixed in 81c35d3.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
