This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-classloaders.git
The following commit(s) were added to refs/heads/main by this push:
new 39afc71 Create init utility (#69)
39afc71 is described below
commit 39afc71d52458474c0ccdffee996f2cd2bee66c4
Author: Christopher Tubbs <[email protected]>
AuthorDate: Mon Feb 9 15:26:01 2026 -0500
Create init utility (#69)
* Move CLI commands into separate package
* Minor refactor to allow LocalStore to be initialized and to store
without creating a classloader to avoid utility from leaving behind
unnecessary working directories
* Ensure monitor threads are daemon threads (observed that they held up
process exit before refactoring LocalStore to be used without creating
a classloader; now unneeded, but still a good change)
---
.../classloader/ccl/CachingClassLoaderFactory.java | 47 ++++--------
.../accumulo/classloader/ccl/LocalStore.java | 39 ++++++++--
.../classloader/ccl/cli/CreateManifest.java | 73 +++++++++++++++++++
.../apache/accumulo/classloader/ccl/cli/Help.java | 53 ++++++++++++++
.../classloader/ccl/cli/InitializeCache.java | 80 +++++++++++++++++++++
.../classloader/ccl/manifest/Manifest.java | 83 +---------------------
.../accumulo/classloader/ccl/LocalStoreTest.java | 3 +-
7 files changed, 258 insertions(+), 120 deletions(-)
diff --git
a/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/CachingClassLoaderFactory.java
b/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/CachingClassLoaderFactory.java
index d5b199b..70db567 100644
---
a/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/CachingClassLoaderFactory.java
+++
b/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/CachingClassLoaderFactory.java
@@ -20,18 +20,14 @@ package org.apache.accumulo.classloader.ccl;
import static java.util.Objects.requireNonNull;
-import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.lang.ref.Cleaner;
import java.net.MalformedURLException;
-import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLClassLoader;
-import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.Map;
@@ -39,6 +35,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiConsumer;
import java.util.function.Supplier;
@@ -113,8 +110,15 @@ public class CachingClassLoaderFactory implements
ContextClassLoaderFactory {
private static final Logger LOG =
LoggerFactory.getLogger(CachingClassLoaderFactory.class);
private static final Cleaner CLEANER = Cleaner.create();
+ private static final AtomicLong monitorThreadCounter = new AtomicLong(0);
+
// executor for the monitor tasks
- private final ScheduledExecutorService executor =
Executors.newScheduledThreadPool(0);
+ private final ScheduledExecutorService executor =
Executors.newScheduledThreadPool(0, r -> {
+ var t = new Thread(r);
+ t.setName("url-context-monitor-thread-" +
monitorThreadCounter.getAndIncrement());
+ t.setDaemon(true);
+ return t;
+ });
// stores the latest seen manifest for a remote URL; String types are used
here for the key
// instead of URL because URL.hashCode could trigger network activity for
hostname lookups
@@ -157,7 +161,7 @@ public class CachingClassLoaderFactory implements
ContextClassLoaderFactory {
@Override
public void init(ContextClassLoaderEnvironment env) {
- String value = requireNonNull(env.getConfiguration().get(PROP_CACHE_DIR),
+ String baseDir = requireNonNull(env.getConfiguration().get(PROP_CACHE_DIR),
"Property " + PROP_CACHE_DIR + " not set, cannot create cache
directory.");
// these suppliers are used so we can update these config properties
without restarting,
@@ -190,25 +194,10 @@ public class CachingClassLoaderFactory implements
ContextClassLoaderFactory {
+ " No ClassLoader instances will be created until it is set.",
PROP_ALLOWED_URLS, env.getConfiguration().get(PROP_ALLOWED_URLS),
npe);
}
- final Path baseCacheDir;
- if (value.startsWith("file:")) {
- try {
- baseCacheDir = Path.of(new URL(value).toURI());
- } catch (IOException | URISyntaxException e) {
- throw new IllegalArgumentException(
- "Malformed file: URL specified for base directory: " + value, e);
- }
- } else if (value.startsWith("/")) {
- baseCacheDir = Path.of(value);
- } else {
- throw new IllegalArgumentException(
- "Base directory is neither a file URL nor an absolute file path: " +
value);
- }
try {
- localStore.set(new LocalStore(baseCacheDir, allowedUrlChecker));
+ localStore.set(new LocalStore(baseDir, allowedUrlChecker));
} catch (IOException e) {
- throw new UncheckedIOException("Unable to create the local storage area
at " + baseCacheDir,
- e);
+ throw new UncheckedIOException("Unable to create the local storage area
at " + baseDir, e);
}
}
@@ -358,7 +347,7 @@ public class CachingClassLoaderFactory implements
ContextClassLoaderFactory {
"Exception creating a hard link in {} due to missing resource {};
attempting re-download of context resources",
failedHardLinksDir, e.getMissingResource(), e);
try {
- recursiveDelete(failedHardLinksDir);
+ LocalStore.recursiveDelete(failedHardLinksDir);
} catch (IOException ioe) {
LOG.warn(
"Saw exception removing directory {} after hard link creation
failure; this should be cleaned up manually",
@@ -385,7 +374,7 @@ public class CachingClassLoaderFactory implements
ContextClassLoaderFactory {
final var cleanDir = hardLinksDir;
CLEANER.register(cl, () -> {
try {
- recursiveDelete(cleanDir);
+ LocalStore.recursiveDelete(cleanDir);
} catch (IOException e) {
LOG.warn("Saw exception when executing cleaner on directory {}",
cleanDir, e);
}
@@ -393,12 +382,4 @@ public class CachingClassLoaderFactory implements
ContextClassLoaderFactory {
return cl;
}
- private static void recursiveDelete(Path directory) throws IOException {
- if (Files.exists(directory)) {
- try (var walker = Files.walk(directory)) {
-
walker.map(Path::toFile).sorted(Comparator.reverseOrder()).forEach(File::delete);
- }
- }
- }
-
}
diff --git
a/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/LocalStore.java
b/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/LocalStore.java
index be65060..ec14ab2 100644
---
a/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/LocalStore.java
+++
b/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/LocalStore.java
@@ -27,14 +27,17 @@ import static java.util.Objects.requireNonNull;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.Closeable;
+import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
+import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
+import java.util.Comparator;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.FutureTask;
import java.util.concurrent.TimeUnit;
@@ -85,7 +88,7 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
* their checksums are verified before use, so that a {@link URLClassLoader}
can be constructed
* using the same resource ordering as in the {@link Manifest} file.
*/
-final class LocalStore {
+public final class LocalStore {
private static final Logger LOG = LoggerFactory.getLogger(LocalStore.class);
private static final String PID =
Long.toString(ProcessHandle.current().pid());
@@ -98,7 +101,27 @@ final class LocalStore {
public static final String RESOURCES_DIR = "resources";
public static final String WORKING_DIR = "working";
- public LocalStore(final Path baseDir, final BiConsumer<String,URL>
allowedUrlChecker)
+ public LocalStore(final String baseDir, final BiConsumer<String,URL>
allowedUrlChecker)
+ throws IOException {
+ this(baseDirStringToPath(baseDir), allowedUrlChecker);
+ }
+
+ private static Path baseDirStringToPath(final String value) {
+ if (value.startsWith("file:")) {
+ try {
+ return Path.of(new URL(value).toURI());
+ } catch (IOException | URISyntaxException e) {
+ throw new IllegalArgumentException(
+ "Malformed file: URL specified for base directory: " + value, e);
+ }
+ } else if (value.startsWith("/")) {
+ return Path.of(value);
+ }
+ throw new IllegalArgumentException(
+ "Base directory is neither a file URL nor an absolute file path: " +
value);
+ }
+
+ LocalStore(final Path baseDir, final BiConsumer<String,URL>
allowedUrlChecker)
throws IOException {
requireNonNull(baseDir);
this.allowedUrlChecker = requireNonNull(allowedUrlChecker);
@@ -174,7 +197,7 @@ final class LocalStore {
* Save the {@link Manifest} to the manifests directory, and all of its
resources to the resources
* directory.
*/
- void storeContext(final Manifest manifest) {
+ public void storeContext(final Manifest manifest) {
requireNonNull(manifest, "manifest must be supplied");
final String destinationName = checksumForFileName(manifest) + ".json";
try {
@@ -369,7 +392,7 @@ final class LocalStore {
}
}
- Path createWorkingHardLinks(final Manifest manifest, Consumer<Path>
forEachLink)
+ public Path createWorkingHardLinks(final Manifest manifest, Consumer<Path>
forEachLink)
throws HardLinkFailedException {
Path hardLinkDir = createTempDirectory("context-" +
checksumForFileName(manifest));
// create all hard links first
@@ -419,4 +442,12 @@ final class LocalStore {
}
+ public static void recursiveDelete(Path directory) throws IOException {
+ if (Files.exists(directory)) {
+ try (var walker = Files.walk(directory)) {
+
walker.map(Path::toFile).sorted(Comparator.reverseOrder()).forEach(File::delete);
+ }
+ }
+ }
+
}
diff --git
a/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/cli/CreateManifest.java
b/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/cli/CreateManifest.java
new file mode 100644
index 0000000..3441329
--- /dev/null
+++
b/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/cli/CreateManifest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.classloader.ccl.cli;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.accumulo.classloader.ccl.CachingClassLoaderFactory;
+import org.apache.accumulo.classloader.ccl.manifest.Manifest;
+import org.apache.accumulo.start.spi.KeywordExecutable;
+
+import com.beust.jcommander.Parameter;
+import com.google.auto.service.AutoService;
+
+@AutoService(KeywordExecutable.class)
+public class CreateManifest implements KeywordExecutable {
+
+ static class Opts extends Help {
+ @Parameter(names = {"-i", "--interval"}, required = true,
+ description = "monitor interval (in seconds)", arity = 1, order = 1)
+ int monitorInterval;
+
+ @Parameter(names = {"-a", "--algorithm"}, required = false,
+ description = "checksum algorithm to use (default: SHA-512)", arity =
1, order = 2)
+ String algorithm = "SHA-512";
+
+ @Parameter(required = true, description = "classpath element URL (<url>[
<url>...])",
+ arity = -1, order = 3)
+ public List<String> files = new ArrayList<>();
+ }
+
+ public CreateManifest() {}
+
+ @Override
+ public String keyword() {
+ return "create-classloader-manifest";
+ }
+
+ @Override
+ public String description() {
+ return "Creates and prints a class loader context manifest for the "
+ + CachingClassLoaderFactory.class.getSimpleName();
+ }
+
+ @Override
+ public void execute(String[] args) throws Exception {
+ var opts = new Opts();
+ opts.parseArgs(CreateManifest.class.getName(), args);
+ URL[] urls = new URL[opts.files.size()];
+ int count = 0;
+ for (String f : opts.files) {
+ urls[count++] = new URL(f);
+ }
+ System.out.print(Manifest.create(opts.monitorInterval, opts.algorithm,
urls).toJson());
+ }
+}
diff --git
a/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/cli/Help.java
b/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/cli/Help.java
new file mode 100644
index 0000000..85ed461
--- /dev/null
+++
b/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/cli/Help.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.classloader.ccl.cli;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParameterException;
+
+class Help {
+ @Parameter(names = {"-h", "-?", "--help", "-help"}, help = true)
+ public boolean help = false;
+
+ void parseArgs(String programName, String[] args) {
+ JCommander commander = new JCommander();
+ commander.addObject(this);
+ commander.setProgramName(programName);
+ try {
+ commander.parse(args);
+ } catch (ParameterException ex) {
+ commander.usage();
+ exitWithError(ex.getMessage(), 1);
+ }
+ if (help) {
+ commander.usage();
+ exit(0);
+ }
+ }
+
+ private void exit(int status) {
+ System.exit(status);
+ }
+
+ private void exitWithError(String message, int status) {
+ System.err.println(message);
+ exit(status);
+ }
+}
diff --git
a/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/cli/InitializeCache.java
b/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/cli/InitializeCache.java
new file mode 100644
index 0000000..903c502
--- /dev/null
+++
b/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/cli/InitializeCache.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.classloader.ccl.cli;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.accumulo.classloader.ccl.CachingClassLoaderFactory;
+import org.apache.accumulo.classloader.ccl.LocalStore;
+import org.apache.accumulo.classloader.ccl.manifest.Manifest;
+import org.apache.accumulo.start.spi.KeywordExecutable;
+
+import com.beust.jcommander.Parameter;
+import com.google.auto.service.AutoService;
+
+@AutoService(KeywordExecutable.class)
+public class InitializeCache implements KeywordExecutable {
+
+ static class Opts extends Help {
+ @Parameter(names = {"-d", "--directory"}, required = true,
+ description = "the local directory to initialize and stage", arity =
1, order = 1)
+ String directory;
+
+ @Parameter(names = {"-v", "--verify"}, required = false,
+ description = "also verify existing files if they were found already
present", arity = 0,
+ order = 2)
+ boolean verify;
+
+ @Parameter(required = false,
+ description = "URLs for context manifests to load (<url>[ <url>...])",
arity = -1,
+ order = 3)
+ public List<String> manifests = new ArrayList<>();
+ }
+
+ public InitializeCache() {}
+
+ @Override
+ public String keyword() {
+ return "init-classloader-cache-dir";
+ }
+
+ @Override
+ public String description() {
+ return "Initializes the specified directory for the "
+ + CachingClassLoaderFactory.class.getSimpleName()
+ + " and stages any resources for the specified context manifest
locations";
+ }
+
+ @Override
+ public void execute(String[] args) throws Exception {
+ var opts = new Opts();
+ opts.parseArgs(InitializeCache.class.getName(), args);
+ var localStore = new LocalStore(opts.directory, (a, b) -> {/* allow all
*/});
+ for (String manifestUrl : opts.manifests) {
+ var manifest = Manifest.download(new URL(manifestUrl));
+ localStore.storeContext(manifest);
+ if (opts.verify) {
+ var workingDir = localStore.createWorkingHardLinks(manifest, p -> {/*
do nothing */});
+ LocalStore.recursiveDelete(workingDir);
+ }
+ }
+ }
+}
diff --git
a/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/manifest/Manifest.java
b/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/manifest/Manifest.java
index e3d3672..80a3891 100644
---
a/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/manifest/Manifest.java
+++
b/modules/caching-class-loader/src/main/java/org/apache/accumulo/classloader/ccl/manifest/Manifest.java
@@ -28,22 +28,14 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashSet;
-import java.util.List;
import java.util.Objects;
import java.util.Set;
-import java.util.function.Consumer;
import java.util.function.Supplier;
-import org.apache.accumulo.start.spi.KeywordExecutable;
import org.apache.commons.codec.digest.DigestUtils;
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterException;
-import com.google.auto.service.AutoService;
import com.google.common.base.Preconditions;
import com.google.common.base.Suppliers;
import com.google.gson.Gson;
@@ -51,59 +43,7 @@ import com.google.gson.GsonBuilder;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-@AutoService(KeywordExecutable.class)
-public class Manifest implements KeywordExecutable {
-
- static class Opts {
- @Parameter(names = {"-i", "--interval"}, required = true,
- description = "monitor interval (in seconds)", arity = 1, order = 1)
- int monitorInterval;
-
- @Parameter(names = {"-a", "--algorithm"}, required = false,
- description = "checksum algorithm to use (default: " + SHA_512 + ")",
arity = 1, order = 2)
- String algorithm = SHA_512;
-
- @Parameter(required = true, description = "classpath element URL (<url>[
<url>...])",
- arity = -1, order = 3)
- public List<String> files = new ArrayList<>();
-
- @Parameter(names = {"-h", "-?", "--help", "-help"}, help = true)
- public boolean help = false;
-
- void parseArgs(Consumer<JCommander> jcConsumer, String programName,
String[] args,
- Object... others) {
- JCommander commander = new JCommander();
- jcConsumer.accept(commander);
- commander.addObject(this);
- for (Object other : others) {
- commander.addObject(other);
- }
- commander.setProgramName(programName);
- try {
- commander.parse(args);
- } catch (ParameterException ex) {
- commander.usage();
- exitWithError(ex.getMessage(), 1);
- }
- if (help) {
- commander.usage();
- exit(0);
- }
- }
-
- void parseArgs(String programName, String[] args, Object... others) {
- parseArgs(jCommander -> {}, programName, args, others);
- }
-
- void exit(int status) {
- System.exit(status);
- }
-
- void exitWithError(String message, int status) {
- System.err.println(message);
- exit(status);
- }
- }
+public class Manifest {
// pretty-print uses Unix newline
private static final Gson GSON =
@@ -200,25 +140,4 @@ public class Manifest implements KeywordExecutable {
return GSON.toJson(this).stripTrailing() + "\n";
}
- @Override
- public String keyword() {
- return "create-classloader-manifest";
- }
-
- @Override
- public String description() {
- return "Creates and prints a class loader context manifest for the
CachingClassLoaderFactory";
- }
-
- @Override
- public void execute(String[] args) throws Exception {
- Opts opts = new Opts();
- opts.parseArgs(Manifest.class.getName(), args);
- URL[] urls = new URL[opts.files.size()];
- int count = 0;
- for (String f : opts.files) {
- urls[count++] = new URL(f);
- }
- System.out.print(create(opts.monitorInterval, opts.algorithm,
urls).toJson());
- }
}
diff --git
a/modules/caching-class-loader/src/test/java/org/apache/accumulo/classloader/ccl/LocalStoreTest.java
b/modules/caching-class-loader/src/test/java/org/apache/accumulo/classloader/ccl/LocalStoreTest.java
index 22a1cb3..ce050ed 100644
---
a/modules/caching-class-loader/src/test/java/org/apache/accumulo/classloader/ccl/LocalStoreTest.java
+++
b/modules/caching-class-loader/src/test/java/org/apache/accumulo/classloader/ccl/LocalStoreTest.java
@@ -133,7 +133,8 @@ public class LocalStoreTest {
@Test
public void testPropertyNotSet() {
// test baseDir not set
- assertThrows(NullPointerException.class, () -> new LocalStore(null,
ALLOW_ALL_URLS));
+ assertThrows(NullPointerException.class, () -> new LocalStore((Path) null,
ALLOW_ALL_URLS));
+ assertThrows(NullPointerException.class, () -> new LocalStore((String)
null, ALLOW_ALL_URLS));
// test URL checker not set
assertThrows(NullPointerException.class, () -> new
LocalStore(baseCacheDir, null));
}
