dlmarion commented on code in PR #35: URL: https://github.com/apache/accumulo-classloaders/pull/35#discussion_r2635053121
########## modules/local-caching-classloader/src/main/java/org/apache/accumulo/classloader/lcc/util/LocalStore.java: ########## @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.classloader.lcc.util; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static java.util.Objects.requireNonNull; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +import org.apache.accumulo.classloader.lcc.Constants; +import org.apache.accumulo.classloader.lcc.definition.ContextDefinition; +import org.apache.accumulo.classloader.lcc.definition.Resource; +import org.apache.accumulo.classloader.lcc.resolvers.FileResolver; +import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory.ContextClassLoaderException; +import org.apache.accumulo.core.util.Retry; +import org.apache.accumulo.core.util.Retry.RetryFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class LocalStore { + private static final Logger LOG = LoggerFactory.getLogger(LocalStore.class); + private static final String PID = Long.toString(ProcessHandle.current().pid()); + + private final Path contextsDir; + private final Path resourcesDir; + + public LocalStore(final Path baseDir) throws IOException { + this.contextsDir = requireNonNull(baseDir).toAbsolutePath().resolve("contexts"); + this.resourcesDir = baseDir.resolve("resources"); + Files.createDirectories(contextsDir); + Files.createDirectories(resourcesDir); + } + + Path contextsDir() { + return contextsDir; + } + + Path resourcesDir() { + return resourcesDir; + } + + // pattern to match regular files that have at least one non-dot character preceding a dot and a + // non-zero suffix; these files can be easily converted so the local store retains the original + // file name extension, while non-matching files will not attempt to retain the original file name + // extension, and will instead just append the checksum to the original file name + private static Pattern fileNamesWithExtensionPattern = Pattern.compile("^(.*[^.].*)[.]([^.]+)$"); + + static String localName(String remoteFileName, String checksum) { + requireNonNull(remoteFileName); + requireNonNull(checksum); + var matcher = fileNamesWithExtensionPattern.matcher(remoteFileName); + if (matcher.matches()) { + return String.format("%s-%s.%s", matcher.group(1), checksum, matcher.group(2)); + } + return String.format("%s-%s", remoteFileName, checksum); + } + + static String tempName(String baseName) { + return "." + requireNonNull(baseName) + ".tmp_" + PID; + } + + public URLClassLoaderParams storeContextResources(final ContextDefinition contextDefinition) + throws IOException, ContextClassLoaderException, InterruptedException, URISyntaxException { + requireNonNull(contextDefinition, "definition must be supplied"); + final RetryFactory retryFactory = + Retry.builder().infiniteRetries().retryAfter(1, TimeUnit.SECONDS) + .incrementBy(1, TimeUnit.SECONDS).maxWait(5, TimeUnit.MINUTES).backOffFactor(2) + .logInterval(1, TimeUnit.SECONDS).createFactory(); + final String contextName = contextDefinition.getContextName(); + final Set<Path> localFiles = new LinkedHashSet<>(); + try { + storeContextDefinition(contextDefinition); + Retry retry = retryFactory.createRetry(); + boolean successful = false; + while (!successful) { + localFiles.clear(); + for (Resource resource : contextDefinition.getResources()) { + Path path = storeResource(resource); + if (path == null) { + LOG.debug("Skipped resource {} while another process or thread is downloading it", + resource.getLocation()); + continue; + } + localFiles.add(path); + LOG.trace("Added resource {} to classpath", path); + } + successful = localFiles.size() == contextDefinition.getResources().size(); + if (!successful) { + retry.logRetry(LOG, "Unable to store all resources for context " + contextName); + retry.waitForNextAttempt(LOG, "Store resources for context " + contextName); + retry.useRetry(); + } + retry.logCompletion(LOG, + "Resources for context " + contextName + " cached locally as " + localFiles); + } + + } catch (IOException | InterruptedException | RuntimeException e) { + LOG.error("Error initializing context: " + contextDefinition.getContextName(), e); Review Comment: ```suggestion LOG.error("Error initializing context: " + contextName, e); ``` ########## modules/local-caching-classloader/src/main/java/org/apache/accumulo/classloader/lcc/util/LocalStore.java: ########## @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.classloader.lcc.util; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static java.util.Objects.requireNonNull; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +import org.apache.accumulo.classloader.lcc.Constants; +import org.apache.accumulo.classloader.lcc.definition.ContextDefinition; +import org.apache.accumulo.classloader.lcc.definition.Resource; +import org.apache.accumulo.classloader.lcc.resolvers.FileResolver; +import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory.ContextClassLoaderException; +import org.apache.accumulo.core.util.Retry; +import org.apache.accumulo.core.util.Retry.RetryFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class LocalStore { + private static final Logger LOG = LoggerFactory.getLogger(LocalStore.class); + private static final String PID = Long.toString(ProcessHandle.current().pid()); + + private final Path contextsDir; + private final Path resourcesDir; + + public LocalStore(final Path baseDir) throws IOException { + this.contextsDir = requireNonNull(baseDir).toAbsolutePath().resolve("contexts"); + this.resourcesDir = baseDir.resolve("resources"); + Files.createDirectories(contextsDir); + Files.createDirectories(resourcesDir); + } + + Path contextsDir() { + return contextsDir; + } + + Path resourcesDir() { + return resourcesDir; + } + + // pattern to match regular files that have at least one non-dot character preceding a dot and a + // non-zero suffix; these files can be easily converted so the local store retains the original + // file name extension, while non-matching files will not attempt to retain the original file name + // extension, and will instead just append the checksum to the original file name + private static Pattern fileNamesWithExtensionPattern = Pattern.compile("^(.*[^.].*)[.]([^.]+)$"); + + static String localName(String remoteFileName, String checksum) { + requireNonNull(remoteFileName); + requireNonNull(checksum); + var matcher = fileNamesWithExtensionPattern.matcher(remoteFileName); + if (matcher.matches()) { + return String.format("%s-%s.%s", matcher.group(1), checksum, matcher.group(2)); + } + return String.format("%s-%s", remoteFileName, checksum); + } + + static String tempName(String baseName) { + return "." + requireNonNull(baseName) + ".tmp_" + PID; + } + + public URLClassLoaderParams storeContextResources(final ContextDefinition contextDefinition) + throws IOException, ContextClassLoaderException, InterruptedException, URISyntaxException { + requireNonNull(contextDefinition, "definition must be supplied"); + final RetryFactory retryFactory = + Retry.builder().infiniteRetries().retryAfter(1, TimeUnit.SECONDS) + .incrementBy(1, TimeUnit.SECONDS).maxWait(5, TimeUnit.MINUTES).backOffFactor(2) + .logInterval(1, TimeUnit.SECONDS).createFactory(); + final String contextName = contextDefinition.getContextName(); + final Set<Path> localFiles = new LinkedHashSet<>(); + try { + storeContextDefinition(contextDefinition); + Retry retry = retryFactory.createRetry(); + boolean successful = false; + while (!successful) { + localFiles.clear(); + for (Resource resource : contextDefinition.getResources()) { + Path path = storeResource(resource); + if (path == null) { + LOG.debug("Skipped resource {} while another process or thread is downloading it", + resource.getLocation()); + continue; + } + localFiles.add(path); + LOG.trace("Added resource {} to classpath", path); + } + successful = localFiles.size() == contextDefinition.getResources().size(); + if (!successful) { + retry.logRetry(LOG, "Unable to store all resources for context " + contextName); + retry.waitForNextAttempt(LOG, "Store resources for context " + contextName); + retry.useRetry(); + } + retry.logCompletion(LOG, + "Resources for context " + contextName + " cached locally as " + localFiles); + } + + } catch (IOException | InterruptedException | RuntimeException e) { + LOG.error("Error initializing context: " + contextDefinition.getContextName(), e); + throw e; + } + return new URLClassLoaderParams( + contextDefinition.getContextName() + "_" + contextDefinition.getChecksum(), + localFiles.stream().map(p -> { + try { + return p.toUri().toURL(); + } catch (MalformedURLException e) { + // this shouldn't happen since these are local file paths + throw new UncheckedIOException(e); + } + }).toArray(URL[]::new)); + } + + private void storeContextDefinition(final ContextDefinition contextDefinition) + throws IOException { + // context names could contain anything, so let's remove any path separators that would mess + // with the file names + String destinationName = + localName(contextDefinition.getContextName().replace(File.separatorChar, '_'), + contextDefinition.getChecksum()); + Path destinationPath = contextsDir.resolve(destinationName); + Path tempPath = contextsDir.resolve(tempName(destinationName)); + Files.write(tempPath, contextDefinition.toJson().getBytes(UTF_8)); + Files.move(tempPath, destinationPath, ATOMIC_MOVE); Review Comment: Could exit early if destinationName exists. No reason to write the temp file at that point. Also, `Files.move` may throw `FileAlreadyExistsException` if the file exists but `REPLACE_EXISTING` is not used. This could happen in the case of concurrent moves to the destinationPath, in which case I don't think you want to throw an error. ########## modules/local-caching-classloader/src/main/java/org/apache/accumulo/classloader/lcc/LocalCachingContextClassLoaderFactory.java: ########## @@ -266,10 +194,106 @@ public ClassLoader getClassLoader(final String contextLocation) } catch (RuntimeException e) { throw new ContextClassLoaderException(e.getMessage(), e); } - if (newlyCreated.get()) { - monitorContext(contextLocation, def.getMonitorIntervalSeconds()); + return classloader.get(); + } + + private ContextDefinition computeDefinitionAndClassLoader( + AtomicReference<URLClassLoader> resultHolder, String contextLocation, + ContextDefinition previousDefinition) { + ContextDefinition computedDefinition; + if (previousDefinition == null) { + try { + computedDefinition = parseContextDefinition(contextLocation); + monitorContext(contextLocation, computedDefinition.getMonitorIntervalSeconds()); + } catch (ContextClassLoaderException e) { + throw new WrappedException(e); + } + } else { + computedDefinition = previousDefinition; } - return cl.get(); + final URLClassLoader classloader = classloaders.computeIfAbsent( + computedDefinition.getContextName() + "-" + computedDefinition.getChecksum(), + (Supplier<URLClassLoaderParams>) () -> { + try { + return localStore.get().storeContextResources(computedDefinition); + } catch (Exception e) { + throw new WrappedException(e); + } + }); + resultHolder.set(classloader); + return computedDefinition; + } + + private void checkMonitoredLocation(String contextLocation, long interval) { + ContextDefinition currentDef = + contextDefs.compute(contextLocation, (contextLocationKey, previousDefinition) -> { + if (previousDefinition == null) { + return null; + } + if (!classloaders.anyMatch(k2 -> k2.substring(0, k2.lastIndexOf('-')) + .equals(previousDefinition.getContextName()))) { + // context has been removed from the map, no need to check for update + LOG.debug("ClassLoader for context {} not present, no longer monitoring for changes", + contextLocation); + return null; + } + return previousDefinition; + }); + if (currentDef == null) { + // context has been removed from the map, no need to check for update + LOG.debug("ContextDefinition for context {} not present, no longer monitoring for changes", + contextLocation); + return; + } + long nextInterval = interval; + try { + final ContextDefinition update = parseContextDefinition(contextLocation); + if (!currentDef.getChecksum().equals(update.getChecksum())) { + LOG.debug("Context definition for {} has changed", contextLocation); + if (!currentDef.getContextName().equals(update.getContextName())) { + LOG.warn( + "Context name changed for context {}, but context cache directory will remain {} (old={}, new={})", + contextLocation, currentDef.getContextName(), currentDef.getContextName(), + update.getContextName()); + } + localStore.get().storeContextResources(update); + contextDefs.put(contextLocation, update); + nextInterval = update.getMonitorIntervalSeconds(); + classloaderFailures.remove(contextLocation); + } else { + LOG.trace("Context definition for {} has not changed", contextLocation); + } + } catch (ContextClassLoaderException | InterruptedException | IOException | URISyntaxException + | RuntimeException e) { + LOG.error("Error parsing updated context definition at {}. Classloader NOT updated!", + contextLocation, e); + final Timer failureTimer = classloaderFailures.get(contextLocation); + if (updateFailureGracePeriodMins.isZero()) { + // failure monitoring is disabled + LOG.debug("Property {} not set, not tracking classloader failures for context {}", Review Comment: I'm thinking that this log statement, and the next two, should probably be at `warn`. ########## modules/local-caching-classloader/src/main/java/org/apache/accumulo/classloader/lcc/util/LocalStore.java: ########## @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.classloader.lcc.util; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static java.util.Objects.requireNonNull; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +import org.apache.accumulo.classloader.lcc.Constants; +import org.apache.accumulo.classloader.lcc.definition.ContextDefinition; +import org.apache.accumulo.classloader.lcc.definition.Resource; +import org.apache.accumulo.classloader.lcc.resolvers.FileResolver; +import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory.ContextClassLoaderException; +import org.apache.accumulo.core.util.Retry; +import org.apache.accumulo.core.util.Retry.RetryFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class LocalStore { + private static final Logger LOG = LoggerFactory.getLogger(LocalStore.class); + private static final String PID = Long.toString(ProcessHandle.current().pid()); + + private final Path contextsDir; + private final Path resourcesDir; + + public LocalStore(final Path baseDir) throws IOException { + this.contextsDir = requireNonNull(baseDir).toAbsolutePath().resolve("contexts"); + this.resourcesDir = baseDir.resolve("resources"); + Files.createDirectories(contextsDir); + Files.createDirectories(resourcesDir); + } + + Path contextsDir() { + return contextsDir; + } + + Path resourcesDir() { + return resourcesDir; + } + + // pattern to match regular files that have at least one non-dot character preceding a dot and a + // non-zero suffix; these files can be easily converted so the local store retains the original + // file name extension, while non-matching files will not attempt to retain the original file name + // extension, and will instead just append the checksum to the original file name + private static Pattern fileNamesWithExtensionPattern = Pattern.compile("^(.*[^.].*)[.]([^.]+)$"); + + static String localName(String remoteFileName, String checksum) { + requireNonNull(remoteFileName); + requireNonNull(checksum); + var matcher = fileNamesWithExtensionPattern.matcher(remoteFileName); + if (matcher.matches()) { + return String.format("%s-%s.%s", matcher.group(1), checksum, matcher.group(2)); + } + return String.format("%s-%s", remoteFileName, checksum); + } + + static String tempName(String baseName) { + return "." + requireNonNull(baseName) + ".tmp_" + PID; + } + + public URLClassLoaderParams storeContextResources(final ContextDefinition contextDefinition) + throws IOException, ContextClassLoaderException, InterruptedException, URISyntaxException { + requireNonNull(contextDefinition, "definition must be supplied"); + final RetryFactory retryFactory = + Retry.builder().infiniteRetries().retryAfter(1, TimeUnit.SECONDS) + .incrementBy(1, TimeUnit.SECONDS).maxWait(5, TimeUnit.MINUTES).backOffFactor(2) + .logInterval(1, TimeUnit.SECONDS).createFactory(); + final String contextName = contextDefinition.getContextName(); + final Set<Path> localFiles = new LinkedHashSet<>(); + try { + storeContextDefinition(contextDefinition); + Retry retry = retryFactory.createRetry(); + boolean successful = false; + while (!successful) { + localFiles.clear(); + for (Resource resource : contextDefinition.getResources()) { + Path path = storeResource(resource); + if (path == null) { + LOG.debug("Skipped resource {} while another process or thread is downloading it", + resource.getLocation()); + continue; + } + localFiles.add(path); + LOG.trace("Added resource {} to classpath", path); + } + successful = localFiles.size() == contextDefinition.getResources().size(); + if (!successful) { + retry.logRetry(LOG, "Unable to store all resources for context " + contextName); + retry.waitForNextAttempt(LOG, "Store resources for context " + contextName); + retry.useRetry(); + } + retry.logCompletion(LOG, + "Resources for context " + contextName + " cached locally as " + localFiles); + } + + } catch (IOException | InterruptedException | RuntimeException e) { + LOG.error("Error initializing context: " + contextDefinition.getContextName(), e); + throw e; + } + return new URLClassLoaderParams( + contextDefinition.getContextName() + "_" + contextDefinition.getChecksum(), + localFiles.stream().map(p -> { + try { + return p.toUri().toURL(); + } catch (MalformedURLException e) { + // this shouldn't happen since these are local file paths + throw new UncheckedIOException(e); + } + }).toArray(URL[]::new)); + } + + private void storeContextDefinition(final ContextDefinition contextDefinition) + throws IOException { + // context names could contain anything, so let's remove any path separators that would mess + // with the file names + String destinationName = + localName(contextDefinition.getContextName().replace(File.separatorChar, '_'), + contextDefinition.getChecksum()); + Path destinationPath = contextsDir.resolve(destinationName); + Path tempPath = contextsDir.resolve(tempName(destinationName)); + Files.write(tempPath, contextDefinition.toJson().getBytes(UTF_8)); + Files.move(tempPath, destinationPath, ATOMIC_MOVE); + } + + private Path storeResource(final Resource resource) { + final URL url = resource.getLocation(); + final FileResolver source; + try { + source = FileResolver.resolve(url); + } catch (IOException e) { + // there was an error getting the resolver for the resource location url Review Comment: Probably want to log an error here ########## modules/local-caching-classloader/src/test/java/org/apache/accumulo/classloader/lcc/util/LocalStoreTest.java: ########## @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.classloader.lcc.util; + +import static org.apache.accumulo.classloader.lcc.TestUtils.testClassFailsToLoad; +import static org.apache.accumulo.classloader.lcc.TestUtils.testClassLoads; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.net.URL; +import java.nio.file.Files; +import java.util.Comparator; +import java.util.LinkedHashSet; +import java.util.stream.Collectors; + +import org.apache.accumulo.classloader.lcc.TestUtils; +import org.apache.accumulo.classloader.lcc.TestUtils.TestClassInfo; +import org.apache.accumulo.classloader.lcc.definition.ContextDefinition; +import org.apache.accumulo.classloader.lcc.definition.Resource; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsUrlStreamHandlerFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.eclipse.jetty.server.Server; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class LocalStoreTest { + + @TempDir + private static java.nio.file.Path tempDir; + + private static final String CONTEXT_NAME = "TEST_CONTEXT"; + private static final int MONITOR_INTERVAL_SECS = 5; + private static MiniDFSCluster hdfs; + private static Server jetty; + private static ContextDefinition def; + private static TestClassInfo classA; + private static TestClassInfo classB; + private static TestClassInfo classC; + private static TestClassInfo classD; + private static java.nio.file.Path baseCacheDir = null; + + @BeforeAll + public static void beforeAll() throws Exception { + baseCacheDir = tempDir.resolve("base"); + + // Find the Test jar files + final URL jarAOrigLocation = LocalStoreTest.class.getResource("/ClassLoaderTestA/TestA.jar"); + assertNotNull(jarAOrigLocation); + final URL jarBOrigLocation = LocalStoreTest.class.getResource("/ClassLoaderTestB/TestB.jar"); + assertNotNull(jarBOrigLocation); + final URL jarCOrigLocation = LocalStoreTest.class.getResource("/ClassLoaderTestC/TestC.jar"); + assertNotNull(jarCOrigLocation); + + // Put B into HDFS + hdfs = TestUtils.getMiniCluster(); + final FileSystem fs = hdfs.getFileSystem(); + assertTrue(fs.mkdirs(new Path("/contextB"))); + final Path dst = new Path("/contextB/TestB.jar"); + fs.copyFromLocalFile(new Path(jarBOrigLocation.toURI()), dst); + assertTrue(fs.exists(dst)); + URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory(hdfs.getConfiguration(0))); + final URL jarBNewLocation = new URL(fs.getUri().toString() + dst.toUri().toString()); + + // Put C into Jetty + java.nio.file.Path jarCParentDirectory = + java.nio.file.Path.of(jarCOrigLocation.toURI()).getParent(); + jetty = TestUtils.getJetty(jarCParentDirectory); + final URL jarCNewLocation = jetty.getURI().resolve("TestC.jar").toURL(); + + // Create ContextDefinition with all three resources + final LinkedHashSet<Resource> resources = new LinkedHashSet<>(); + resources + .add(new Resource(jarAOrigLocation, TestUtils.computeResourceChecksum(jarAOrigLocation))); + resources + .add(new Resource(jarBNewLocation, TestUtils.computeResourceChecksum(jarBOrigLocation))); + resources + .add(new Resource(jarCNewLocation, TestUtils.computeResourceChecksum(jarCOrigLocation))); + + def = new ContextDefinition(CONTEXT_NAME, MONITOR_INTERVAL_SECS, resources); + classA = new TestClassInfo("test.TestObjectA", "Hello from A"); + classB = new TestClassInfo("test.TestObjectB", "Hello from B"); + classC = new TestClassInfo("test.TestObjectC", "Hello from C"); + classD = new TestClassInfo("test.TestObjectD", "Hello from D"); + } + + @AfterAll + public static void afterAll() throws Exception { + if (jetty != null) { + jetty.stop(); + jetty.join(); + } + if (hdfs != null) { + hdfs.shutdown(); + } + } + + @AfterEach + public void cleanBaseDir() throws Exception { + if (Files.exists(baseCacheDir)) { + try (var walker = Files.walk(baseCacheDir)) { + walker.map(java.nio.file.Path::toFile).sorted(Comparator.reverseOrder()) + .forEach(File::delete); + } + } + } + + @Test + public void testPropertyNotSet() { + assertThrows(NullPointerException.class, () -> new LocalStore(null)); + } + + @Test + public void testCreateBaseDirs() throws Exception { + assertFalse(Files.exists(baseCacheDir)); + var localStore = new LocalStore(baseCacheDir); + assertTrue(Files.exists(baseCacheDir)); + assertTrue(Files.exists(baseCacheDir.resolve("contexts"))); + assertTrue(Files.exists(baseCacheDir.resolve("resources"))); + assertEquals(baseCacheDir.resolve("contexts"), localStore.contextsDir()); + assertEquals(baseCacheDir.resolve("resources"), localStore.resourcesDir()); + } + + @Test + public void testCreateBaseDirsMultipleTimes() throws Exception { + assertFalse(Files.exists(baseCacheDir)); + assertNotNull(new LocalStore(baseCacheDir)); + assertNotNull(new LocalStore(baseCacheDir)); + assertNotNull(new LocalStore(baseCacheDir)); + assertNotNull(new LocalStore(baseCacheDir)); + assertTrue(Files.exists(baseCacheDir)); + } + + @Test + public void testLocalFileName() { + // regular json + assertEquals("f1-chk1.json", LocalStore.localName("f1.json", "chk1")); + // dotfile json + assertEquals(".f1-chk1.json", LocalStore.localName(".f1.json", "chk1")); + // regular jar (has multiple dots) + assertEquals("f2-1.0-chk2.jar", LocalStore.localName("f2-1.0.jar", "chk2")); + // dotfile jar (has multiple dots) + assertEquals(".f2-1.0-chk2.jar", LocalStore.localName(".f2-1.0.jar", "chk2")); + // regular file with no suffix + assertEquals("f3-chk3", LocalStore.localName("f3", "chk3")); + + // weird files with trailing dots and no file suffix + assertEquals("f4.-chk4", LocalStore.localName("f4.", "chk4")); + assertEquals("f4..-chk4", LocalStore.localName("f4..", "chk4")); + assertEquals("f4...-chk4", LocalStore.localName("f4...", "chk4")); + // weird dotfiles that don't really have a suffix + assertEquals(".f5-chk5", LocalStore.localName(".f5", "chk5")); + assertEquals("..f5-chk5", LocalStore.localName("..f5", "chk5")); + // weird files with weird dots, but do have a valid suffix + assertEquals("f6.-chk6.jar", LocalStore.localName("f6..jar", "chk6")); + assertEquals("f6..-chk6.jar", LocalStore.localName("f6...jar", "chk6")); + assertEquals(".f6-chk6.jar", LocalStore.localName(".f6.jar", "chk6")); + assertEquals("..f6-chk6.jar", LocalStore.localName("..f6.jar", "chk6")); + assertEquals(".f6.-chk6.jar", LocalStore.localName(".f6..jar", "chk6")); + assertEquals("..f6.-chk6.jar", LocalStore.localName("..f6..jar", "chk6")); + } + + @Test + public void testStoreContextResources() throws Exception { + var localStore = new LocalStore(baseCacheDir); + localStore.storeContextResources(def); + + // Confirm the 3 jars are cached locally + assertTrue(Files.exists(baseCacheDir)); + assertTrue(Files.exists(baseCacheDir.resolve("contexts") + .resolve(CONTEXT_NAME + "_" + def.getChecksum() + ".json"))); + for (Resource r : def.getResources()) { + String filename = TestUtils.getFileName(r.getLocation()); + String checksum = r.getChecksum(); + assertTrue( + Files.exists(baseCacheDir.resolve("resources").resolve(filename + "_" + checksum))); + } + } + + @Test + public void testClassLoader() throws Exception { + var helper = new LocalStore(baseCacheDir).storeContextResources(def); + ClassLoader contextClassLoader = helper.createClassLoader(); + + testClassLoads(contextClassLoader, classA); + testClassLoads(contextClassLoader, classB); + testClassLoads(contextClassLoader, classC); + } + + @Test + public void testUpdate() throws Exception { + var localStore = new LocalStore(baseCacheDir); + var helper = localStore.storeContextResources(def); + final ClassLoader contextClassLoader = helper.createClassLoader(); + + testClassLoads(contextClassLoader, classA); + testClassLoads(contextClassLoader, classB); + testClassLoads(contextClassLoader, classC); + + // keep all but C + var updatedResources = def.getResources().stream().limit(def.getResources().size() - 1) + .collect(Collectors.toCollection(LinkedHashSet::new)); + assertEquals(def.getResources().size() - 1, updatedResources.size()); + + // Add D + final URL jarDOrigLocation = LocalStoreTest.class.getResource("/ClassLoaderTestD/TestD.jar"); + assertNotNull(jarDOrigLocation); + updatedResources + .add(new Resource(jarDOrigLocation, TestUtils.computeResourceChecksum(jarDOrigLocation))); + + var updatedDef = new ContextDefinition(CONTEXT_NAME, MONITOR_INTERVAL_SECS, updatedResources); + helper = localStore.storeContextResources(updatedDef); + + // Confirm the 3 jars are cached locally + assertTrue(Files.exists(baseCacheDir.resolve("contexts") + .resolve(CONTEXT_NAME + "_" + updatedDef.getChecksum() + ".json"))); + for (Resource r : updatedDef.getResources()) { + String filename = TestUtils.getFileName(r.getLocation()); + assertFalse(filename.contains("C")); + String checksum = r.getChecksum(); + assertTrue( + Files.exists(baseCacheDir.resolve("resources").resolve(filename + "_" + checksum))); + } + Review Comment: Should we confirm that jar C is still in the local resource directory? ########## modules/local-caching-classloader/src/main/java/org/apache/accumulo/classloader/lcc/LocalCachingContextClassLoaderFactory.java: ########## @@ -266,10 +194,106 @@ public ClassLoader getClassLoader(final String contextLocation) } catch (RuntimeException e) { throw new ContextClassLoaderException(e.getMessage(), e); } - if (newlyCreated.get()) { - monitorContext(contextLocation, def.getMonitorIntervalSeconds()); + return classloader.get(); + } + + private ContextDefinition computeDefinitionAndClassLoader( + AtomicReference<URLClassLoader> resultHolder, String contextLocation, + ContextDefinition previousDefinition) { + ContextDefinition computedDefinition; + if (previousDefinition == null) { + try { + computedDefinition = parseContextDefinition(contextLocation); + monitorContext(contextLocation, computedDefinition.getMonitorIntervalSeconds()); + } catch (ContextClassLoaderException e) { + throw new WrappedException(e); + } + } else { + computedDefinition = previousDefinition; } - return cl.get(); + final URLClassLoader classloader = classloaders.computeIfAbsent( + computedDefinition.getContextName() + "-" + computedDefinition.getChecksum(), + (Supplier<URLClassLoaderParams>) () -> { + try { + return localStore.get().storeContextResources(computedDefinition); + } catch (Exception e) { + throw new WrappedException(e); + } + }); + resultHolder.set(classloader); + return computedDefinition; + } + + private void checkMonitoredLocation(String contextLocation, long interval) { + ContextDefinition currentDef = + contextDefs.compute(contextLocation, (contextLocationKey, previousDefinition) -> { + if (previousDefinition == null) { + return null; + } + if (!classloaders.anyMatch(k2 -> k2.substring(0, k2.lastIndexOf('-')) + .equals(previousDefinition.getContextName()))) { + // context has been removed from the map, no need to check for update + LOG.debug("ClassLoader for context {} not present, no longer monitoring for changes", + contextLocation); + return null; + } + return previousDefinition; + }); + if (currentDef == null) { + // context has been removed from the map, no need to check for update + LOG.debug("ContextDefinition for context {} not present, no longer monitoring for changes", + contextLocation); + return; + } + long nextInterval = interval; + try { + final ContextDefinition update = parseContextDefinition(contextLocation); + if (!currentDef.getChecksum().equals(update.getChecksum())) { + LOG.debug("Context definition for {} has changed", contextLocation); + if (!currentDef.getContextName().equals(update.getContextName())) { + LOG.warn( + "Context name changed for context {}, but context cache directory will remain {} (old={}, new={})", + contextLocation, currentDef.getContextName(), currentDef.getContextName(), Review Comment: I think the first instance of `currentDef.getContextName()` may not be correct for logging the cache directory. In fact, the notion of a per-context cache directory is going away. ########## modules/local-caching-classloader/src/main/java/org/apache/accumulo/classloader/lcc/util/LocalStore.java: ########## @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.classloader.lcc.util; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static java.util.Objects.requireNonNull; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +import org.apache.accumulo.classloader.lcc.Constants; +import org.apache.accumulo.classloader.lcc.definition.ContextDefinition; +import org.apache.accumulo.classloader.lcc.definition.Resource; +import org.apache.accumulo.classloader.lcc.resolvers.FileResolver; +import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory.ContextClassLoaderException; +import org.apache.accumulo.core.util.Retry; +import org.apache.accumulo.core.util.Retry.RetryFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class LocalStore { + private static final Logger LOG = LoggerFactory.getLogger(LocalStore.class); + private static final String PID = Long.toString(ProcessHandle.current().pid()); + + private final Path contextsDir; + private final Path resourcesDir; + + public LocalStore(final Path baseDir) throws IOException { + this.contextsDir = requireNonNull(baseDir).toAbsolutePath().resolve("contexts"); + this.resourcesDir = baseDir.resolve("resources"); + Files.createDirectories(contextsDir); + Files.createDirectories(resourcesDir); + } + + Path contextsDir() { + return contextsDir; + } + + Path resourcesDir() { + return resourcesDir; + } + + // pattern to match regular files that have at least one non-dot character preceding a dot and a + // non-zero suffix; these files can be easily converted so the local store retains the original + // file name extension, while non-matching files will not attempt to retain the original file name + // extension, and will instead just append the checksum to the original file name + private static Pattern fileNamesWithExtensionPattern = Pattern.compile("^(.*[^.].*)[.]([^.]+)$"); + + static String localName(String remoteFileName, String checksum) { + requireNonNull(remoteFileName); + requireNonNull(checksum); + var matcher = fileNamesWithExtensionPattern.matcher(remoteFileName); + if (matcher.matches()) { + return String.format("%s-%s.%s", matcher.group(1), checksum, matcher.group(2)); + } + return String.format("%s-%s", remoteFileName, checksum); + } + + static String tempName(String baseName) { + return "." + requireNonNull(baseName) + ".tmp_" + PID; + } + + public URLClassLoaderParams storeContextResources(final ContextDefinition contextDefinition) + throws IOException, ContextClassLoaderException, InterruptedException, URISyntaxException { + requireNonNull(contextDefinition, "definition must be supplied"); + final RetryFactory retryFactory = + Retry.builder().infiniteRetries().retryAfter(1, TimeUnit.SECONDS) + .incrementBy(1, TimeUnit.SECONDS).maxWait(5, TimeUnit.MINUTES).backOffFactor(2) + .logInterval(1, TimeUnit.SECONDS).createFactory(); + final String contextName = contextDefinition.getContextName(); + final Set<Path> localFiles = new LinkedHashSet<>(); + try { + storeContextDefinition(contextDefinition); + Retry retry = retryFactory.createRetry(); + boolean successful = false; + while (!successful) { + localFiles.clear(); + for (Resource resource : contextDefinition.getResources()) { + Path path = storeResource(resource); + if (path == null) { + LOG.debug("Skipped resource {} while another process or thread is downloading it", + resource.getLocation()); + continue; + } + localFiles.add(path); + LOG.trace("Added resource {} to classpath", path); + } + successful = localFiles.size() == contextDefinition.getResources().size(); + if (!successful) { + retry.logRetry(LOG, "Unable to store all resources for context " + contextName); + retry.waitForNextAttempt(LOG, "Store resources for context " + contextName); + retry.useRetry(); + } + retry.logCompletion(LOG, + "Resources for context " + contextName + " cached locally as " + localFiles); + } + + } catch (IOException | InterruptedException | RuntimeException e) { + LOG.error("Error initializing context: " + contextDefinition.getContextName(), e); + throw e; + } + return new URLClassLoaderParams( + contextDefinition.getContextName() + "_" + contextDefinition.getChecksum(), + localFiles.stream().map(p -> { + try { + return p.toUri().toURL(); + } catch (MalformedURLException e) { + // this shouldn't happen since these are local file paths + throw new UncheckedIOException(e); + } + }).toArray(URL[]::new)); + } + + private void storeContextDefinition(final ContextDefinition contextDefinition) + throws IOException { + // context names could contain anything, so let's remove any path separators that would mess + // with the file names + String destinationName = + localName(contextDefinition.getContextName().replace(File.separatorChar, '_'), + contextDefinition.getChecksum()); + Path destinationPath = contextsDir.resolve(destinationName); + Path tempPath = contextsDir.resolve(tempName(destinationName)); + Files.write(tempPath, contextDefinition.toJson().getBytes(UTF_8)); + Files.move(tempPath, destinationPath, ATOMIC_MOVE); + } + + private Path storeResource(final Resource resource) { + final URL url = resource.getLocation(); + final FileResolver source; + try { + source = FileResolver.resolve(url); + } catch (IOException e) { + // there was an error getting the resolver for the resource location url + return null; + } + final String baseName = localName(source.getFileName(), resource.getChecksum()); + final Path destinationPath = resourcesDir.resolve(baseName); + final Path tempPath = resourcesDir.resolve(tempName(baseName)); + final Path inProgressPath = resourcesDir.resolve("." + baseName + ".downloading"); + + if (Files.exists(destinationPath)) { + LOG.trace("Resource {} is already cached at {}", url, destinationPath); + return destinationPath; + } + + try { + Files.write(inProgressPath, PID.getBytes(UTF_8), StandardOpenOption.CREATE_NEW); + } catch (FileAlreadyExistsException e) { + // TODO try this, and check the timestamp to see if it has made recent progress + // if no recent progress, delete the file and return null + // for now, return null and assume it will be finished by the other process later + return null; + } catch (IOException e) { + // some other exception occurred that we don't know how to handle; will attempt retry + return null; + } + + LOG.trace("Storing remote resource {} locally at {}", url, destinationPath); + try (InputStream is = source.getInputStream()) { + // TODO update the in progress file as we make progress during the copy; maybe a background + // thread, or maybe X number of bytes written + Files.copy(is, tempPath, REPLACE_EXISTING); + final String checksum = Constants.getChecksummer().digestAsHex(tempPath); + if (!resource.getChecksum().equals(checksum)) { + LOG.error( + "Checksum {} for resource {} does not match checksum in context definition {}, removing cached copy.", + checksum, url, resource.getChecksum()); + Files.delete(destinationPath); Review Comment: Should this be tempPath? ########## modules/local-caching-classloader/src/main/java/org/apache/accumulo/classloader/lcc/util/LocalStore.java: ########## @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.classloader.lcc.util; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static java.util.Objects.requireNonNull; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +import org.apache.accumulo.classloader.lcc.Constants; +import org.apache.accumulo.classloader.lcc.definition.ContextDefinition; +import org.apache.accumulo.classloader.lcc.definition.Resource; +import org.apache.accumulo.classloader.lcc.resolvers.FileResolver; +import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory.ContextClassLoaderException; +import org.apache.accumulo.core.util.Retry; +import org.apache.accumulo.core.util.Retry.RetryFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class LocalStore { + private static final Logger LOG = LoggerFactory.getLogger(LocalStore.class); + private static final String PID = Long.toString(ProcessHandle.current().pid()); + + private final Path contextsDir; + private final Path resourcesDir; + + public LocalStore(final Path baseDir) throws IOException { + this.contextsDir = requireNonNull(baseDir).toAbsolutePath().resolve("contexts"); + this.resourcesDir = baseDir.resolve("resources"); + Files.createDirectories(contextsDir); + Files.createDirectories(resourcesDir); + } + + Path contextsDir() { + return contextsDir; + } + + Path resourcesDir() { + return resourcesDir; + } + + // pattern to match regular files that have at least one non-dot character preceding a dot and a + // non-zero suffix; these files can be easily converted so the local store retains the original + // file name extension, while non-matching files will not attempt to retain the original file name + // extension, and will instead just append the checksum to the original file name + private static Pattern fileNamesWithExtensionPattern = Pattern.compile("^(.*[^.].*)[.]([^.]+)$"); + + static String localName(String remoteFileName, String checksum) { + requireNonNull(remoteFileName); + requireNonNull(checksum); + var matcher = fileNamesWithExtensionPattern.matcher(remoteFileName); + if (matcher.matches()) { + return String.format("%s-%s.%s", matcher.group(1), checksum, matcher.group(2)); + } + return String.format("%s-%s", remoteFileName, checksum); + } + + static String tempName(String baseName) { + return "." + requireNonNull(baseName) + ".tmp_" + PID; + } + + public URLClassLoaderParams storeContextResources(final ContextDefinition contextDefinition) + throws IOException, ContextClassLoaderException, InterruptedException, URISyntaxException { + requireNonNull(contextDefinition, "definition must be supplied"); + final RetryFactory retryFactory = + Retry.builder().infiniteRetries().retryAfter(1, TimeUnit.SECONDS) + .incrementBy(1, TimeUnit.SECONDS).maxWait(5, TimeUnit.MINUTES).backOffFactor(2) + .logInterval(1, TimeUnit.SECONDS).createFactory(); + final String contextName = contextDefinition.getContextName(); + final Set<Path> localFiles = new LinkedHashSet<>(); + try { + storeContextDefinition(contextDefinition); + Retry retry = retryFactory.createRetry(); + boolean successful = false; + while (!successful) { + localFiles.clear(); + for (Resource resource : contextDefinition.getResources()) { + Path path = storeResource(resource); + if (path == null) { + LOG.debug("Skipped resource {} while another process or thread is downloading it", + resource.getLocation()); + continue; + } + localFiles.add(path); + LOG.trace("Added resource {} to classpath", path); + } + successful = localFiles.size() == contextDefinition.getResources().size(); + if (!successful) { + retry.logRetry(LOG, "Unable to store all resources for context " + contextName); + retry.waitForNextAttempt(LOG, "Store resources for context " + contextName); + retry.useRetry(); + } + retry.logCompletion(LOG, + "Resources for context " + contextName + " cached locally as " + localFiles); + } + + } catch (IOException | InterruptedException | RuntimeException e) { + LOG.error("Error initializing context: " + contextDefinition.getContextName(), e); + throw e; + } + return new URLClassLoaderParams( + contextDefinition.getContextName() + "_" + contextDefinition.getChecksum(), Review Comment: ```suggestion contextName + "_" + contextDefinition.getChecksum(), ``` ########## modules/local-caching-classloader/src/main/java/org/apache/accumulo/classloader/lcc/util/LocalStore.java: ########## @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.classloader.lcc.util; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static java.util.Objects.requireNonNull; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +import org.apache.accumulo.classloader.lcc.Constants; +import org.apache.accumulo.classloader.lcc.definition.ContextDefinition; +import org.apache.accumulo.classloader.lcc.definition.Resource; +import org.apache.accumulo.classloader.lcc.resolvers.FileResolver; +import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory.ContextClassLoaderException; +import org.apache.accumulo.core.util.Retry; +import org.apache.accumulo.core.util.Retry.RetryFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class LocalStore { + private static final Logger LOG = LoggerFactory.getLogger(LocalStore.class); + private static final String PID = Long.toString(ProcessHandle.current().pid()); + + private final Path contextsDir; + private final Path resourcesDir; + + public LocalStore(final Path baseDir) throws IOException { + this.contextsDir = requireNonNull(baseDir).toAbsolutePath().resolve("contexts"); + this.resourcesDir = baseDir.resolve("resources"); + Files.createDirectories(contextsDir); + Files.createDirectories(resourcesDir); + } + + Path contextsDir() { + return contextsDir; + } + + Path resourcesDir() { + return resourcesDir; + } + + // pattern to match regular files that have at least one non-dot character preceding a dot and a + // non-zero suffix; these files can be easily converted so the local store retains the original + // file name extension, while non-matching files will not attempt to retain the original file name + // extension, and will instead just append the checksum to the original file name + private static Pattern fileNamesWithExtensionPattern = Pattern.compile("^(.*[^.].*)[.]([^.]+)$"); + + static String localName(String remoteFileName, String checksum) { + requireNonNull(remoteFileName); + requireNonNull(checksum); + var matcher = fileNamesWithExtensionPattern.matcher(remoteFileName); + if (matcher.matches()) { + return String.format("%s-%s.%s", matcher.group(1), checksum, matcher.group(2)); + } + return String.format("%s-%s", remoteFileName, checksum); + } + + static String tempName(String baseName) { + return "." + requireNonNull(baseName) + ".tmp_" + PID; + } + + public URLClassLoaderParams storeContextResources(final ContextDefinition contextDefinition) + throws IOException, ContextClassLoaderException, InterruptedException, URISyntaxException { + requireNonNull(contextDefinition, "definition must be supplied"); + final RetryFactory retryFactory = + Retry.builder().infiniteRetries().retryAfter(1, TimeUnit.SECONDS) + .incrementBy(1, TimeUnit.SECONDS).maxWait(5, TimeUnit.MINUTES).backOffFactor(2) + .logInterval(1, TimeUnit.SECONDS).createFactory(); + final String contextName = contextDefinition.getContextName(); + final Set<Path> localFiles = new LinkedHashSet<>(); + try { + storeContextDefinition(contextDefinition); + Retry retry = retryFactory.createRetry(); + boolean successful = false; + while (!successful) { + localFiles.clear(); + for (Resource resource : contextDefinition.getResources()) { + Path path = storeResource(resource); + if (path == null) { + LOG.debug("Skipped resource {} while another process or thread is downloading it", + resource.getLocation()); + continue; + } + localFiles.add(path); + LOG.trace("Added resource {} to classpath", path); + } + successful = localFiles.size() == contextDefinition.getResources().size(); + if (!successful) { + retry.logRetry(LOG, "Unable to store all resources for context " + contextName); + retry.waitForNextAttempt(LOG, "Store resources for context " + contextName); + retry.useRetry(); + } + retry.logCompletion(LOG, + "Resources for context " + contextName + " cached locally as " + localFiles); + } + + } catch (IOException | InterruptedException | RuntimeException e) { + LOG.error("Error initializing context: " + contextDefinition.getContextName(), e); + throw e; + } + return new URLClassLoaderParams( + contextDefinition.getContextName() + "_" + contextDefinition.getChecksum(), + localFiles.stream().map(p -> { + try { + return p.toUri().toURL(); + } catch (MalformedURLException e) { + // this shouldn't happen since these are local file paths + throw new UncheckedIOException(e); + } + }).toArray(URL[]::new)); + } + + private void storeContextDefinition(final ContextDefinition contextDefinition) + throws IOException { + // context names could contain anything, so let's remove any path separators that would mess + // with the file names + String destinationName = + localName(contextDefinition.getContextName().replace(File.separatorChar, '_'), + contextDefinition.getChecksum()); + Path destinationPath = contextsDir.resolve(destinationName); + Path tempPath = contextsDir.resolve(tempName(destinationName)); + Files.write(tempPath, contextDefinition.toJson().getBytes(UTF_8)); + Files.move(tempPath, destinationPath, ATOMIC_MOVE); + } + + private Path storeResource(final Resource resource) { + final URL url = resource.getLocation(); + final FileResolver source; + try { + source = FileResolver.resolve(url); + } catch (IOException e) { + // there was an error getting the resolver for the resource location url + return null; + } + final String baseName = localName(source.getFileName(), resource.getChecksum()); + final Path destinationPath = resourcesDir.resolve(baseName); + final Path tempPath = resourcesDir.resolve(tempName(baseName)); + final Path inProgressPath = resourcesDir.resolve("." + baseName + ".downloading"); + + if (Files.exists(destinationPath)) { + LOG.trace("Resource {} is already cached at {}", url, destinationPath); + return destinationPath; + } + + try { + Files.write(inProgressPath, PID.getBytes(UTF_8), StandardOpenOption.CREATE_NEW); + } catch (FileAlreadyExistsException e) { + // TODO try this, and check the timestamp to see if it has made recent progress + // if no recent progress, delete the file and return null + // for now, return null and assume it will be finished by the other process later + return null; + } catch (IOException e) { + // some other exception occurred that we don't know how to handle; will attempt retry + return null; Review Comment: Probably want to log the error here ########## modules/local-caching-classloader/src/test/java/org/apache/accumulo/classloader/lcc/util/LocalStoreTest.java: ########## @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.classloader.lcc.util; + +import static org.apache.accumulo.classloader.lcc.TestUtils.testClassFailsToLoad; +import static org.apache.accumulo.classloader.lcc.TestUtils.testClassLoads; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.net.URL; +import java.nio.file.Files; +import java.util.Comparator; +import java.util.LinkedHashSet; +import java.util.stream.Collectors; + +import org.apache.accumulo.classloader.lcc.TestUtils; +import org.apache.accumulo.classloader.lcc.TestUtils.TestClassInfo; +import org.apache.accumulo.classloader.lcc.definition.ContextDefinition; +import org.apache.accumulo.classloader.lcc.definition.Resource; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsUrlStreamHandlerFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.eclipse.jetty.server.Server; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class LocalStoreTest { + + @TempDir + private static java.nio.file.Path tempDir; + + private static final String CONTEXT_NAME = "TEST_CONTEXT"; + private static final int MONITOR_INTERVAL_SECS = 5; + private static MiniDFSCluster hdfs; + private static Server jetty; + private static ContextDefinition def; + private static TestClassInfo classA; + private static TestClassInfo classB; + private static TestClassInfo classC; + private static TestClassInfo classD; + private static java.nio.file.Path baseCacheDir = null; + + @BeforeAll + public static void beforeAll() throws Exception { + baseCacheDir = tempDir.resolve("base"); + + // Find the Test jar files + final URL jarAOrigLocation = LocalStoreTest.class.getResource("/ClassLoaderTestA/TestA.jar"); + assertNotNull(jarAOrigLocation); + final URL jarBOrigLocation = LocalStoreTest.class.getResource("/ClassLoaderTestB/TestB.jar"); + assertNotNull(jarBOrigLocation); + final URL jarCOrigLocation = LocalStoreTest.class.getResource("/ClassLoaderTestC/TestC.jar"); + assertNotNull(jarCOrigLocation); + + // Put B into HDFS + hdfs = TestUtils.getMiniCluster(); + final FileSystem fs = hdfs.getFileSystem(); + assertTrue(fs.mkdirs(new Path("/contextB"))); + final Path dst = new Path("/contextB/TestB.jar"); + fs.copyFromLocalFile(new Path(jarBOrigLocation.toURI()), dst); + assertTrue(fs.exists(dst)); + URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory(hdfs.getConfiguration(0))); + final URL jarBNewLocation = new URL(fs.getUri().toString() + dst.toUri().toString()); + + // Put C into Jetty + java.nio.file.Path jarCParentDirectory = + java.nio.file.Path.of(jarCOrigLocation.toURI()).getParent(); + jetty = TestUtils.getJetty(jarCParentDirectory); + final URL jarCNewLocation = jetty.getURI().resolve("TestC.jar").toURL(); + + // Create ContextDefinition with all three resources + final LinkedHashSet<Resource> resources = new LinkedHashSet<>(); + resources + .add(new Resource(jarAOrigLocation, TestUtils.computeResourceChecksum(jarAOrigLocation))); + resources + .add(new Resource(jarBNewLocation, TestUtils.computeResourceChecksum(jarBOrigLocation))); + resources + .add(new Resource(jarCNewLocation, TestUtils.computeResourceChecksum(jarCOrigLocation))); + + def = new ContextDefinition(CONTEXT_NAME, MONITOR_INTERVAL_SECS, resources); + classA = new TestClassInfo("test.TestObjectA", "Hello from A"); + classB = new TestClassInfo("test.TestObjectB", "Hello from B"); + classC = new TestClassInfo("test.TestObjectC", "Hello from C"); + classD = new TestClassInfo("test.TestObjectD", "Hello from D"); + } + + @AfterAll + public static void afterAll() throws Exception { + if (jetty != null) { + jetty.stop(); + jetty.join(); + } + if (hdfs != null) { + hdfs.shutdown(); + } + } + + @AfterEach + public void cleanBaseDir() throws Exception { + if (Files.exists(baseCacheDir)) { + try (var walker = Files.walk(baseCacheDir)) { + walker.map(java.nio.file.Path::toFile).sorted(Comparator.reverseOrder()) + .forEach(File::delete); + } + } + } + + @Test + public void testPropertyNotSet() { + assertThrows(NullPointerException.class, () -> new LocalStore(null)); + } + + @Test + public void testCreateBaseDirs() throws Exception { + assertFalse(Files.exists(baseCacheDir)); + var localStore = new LocalStore(baseCacheDir); + assertTrue(Files.exists(baseCacheDir)); + assertTrue(Files.exists(baseCacheDir.resolve("contexts"))); + assertTrue(Files.exists(baseCacheDir.resolve("resources"))); + assertEquals(baseCacheDir.resolve("contexts"), localStore.contextsDir()); + assertEquals(baseCacheDir.resolve("resources"), localStore.resourcesDir()); + } + + @Test + public void testCreateBaseDirsMultipleTimes() throws Exception { + assertFalse(Files.exists(baseCacheDir)); + assertNotNull(new LocalStore(baseCacheDir)); + assertNotNull(new LocalStore(baseCacheDir)); + assertNotNull(new LocalStore(baseCacheDir)); + assertNotNull(new LocalStore(baseCacheDir)); + assertTrue(Files.exists(baseCacheDir)); + } + + @Test + public void testLocalFileName() { + // regular json + assertEquals("f1-chk1.json", LocalStore.localName("f1.json", "chk1")); + // dotfile json + assertEquals(".f1-chk1.json", LocalStore.localName(".f1.json", "chk1")); + // regular jar (has multiple dots) + assertEquals("f2-1.0-chk2.jar", LocalStore.localName("f2-1.0.jar", "chk2")); + // dotfile jar (has multiple dots) + assertEquals(".f2-1.0-chk2.jar", LocalStore.localName(".f2-1.0.jar", "chk2")); + // regular file with no suffix + assertEquals("f3-chk3", LocalStore.localName("f3", "chk3")); + + // weird files with trailing dots and no file suffix + assertEquals("f4.-chk4", LocalStore.localName("f4.", "chk4")); + assertEquals("f4..-chk4", LocalStore.localName("f4..", "chk4")); + assertEquals("f4...-chk4", LocalStore.localName("f4...", "chk4")); + // weird dotfiles that don't really have a suffix + assertEquals(".f5-chk5", LocalStore.localName(".f5", "chk5")); + assertEquals("..f5-chk5", LocalStore.localName("..f5", "chk5")); + // weird files with weird dots, but do have a valid suffix + assertEquals("f6.-chk6.jar", LocalStore.localName("f6..jar", "chk6")); + assertEquals("f6..-chk6.jar", LocalStore.localName("f6...jar", "chk6")); + assertEquals(".f6-chk6.jar", LocalStore.localName(".f6.jar", "chk6")); + assertEquals("..f6-chk6.jar", LocalStore.localName("..f6.jar", "chk6")); + assertEquals(".f6.-chk6.jar", LocalStore.localName(".f6..jar", "chk6")); + assertEquals("..f6.-chk6.jar", LocalStore.localName("..f6..jar", "chk6")); Review Comment: There is code in `LocalStore.storeContextDefinition` that replaces the File separator character with an underscore. I wonder if that replacement should move into localFileName and add a test for it here. ########## modules/local-caching-classloader/src/test/java/org/apache/accumulo/classloader/lcc/util/LocalStoreTest.java: ########## @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.classloader.lcc.util; + +import static org.apache.accumulo.classloader.lcc.TestUtils.testClassFailsToLoad; +import static org.apache.accumulo.classloader.lcc.TestUtils.testClassLoads; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.net.URL; +import java.nio.file.Files; +import java.util.Comparator; +import java.util.LinkedHashSet; +import java.util.stream.Collectors; + +import org.apache.accumulo.classloader.lcc.TestUtils; +import org.apache.accumulo.classloader.lcc.TestUtils.TestClassInfo; +import org.apache.accumulo.classloader.lcc.definition.ContextDefinition; +import org.apache.accumulo.classloader.lcc.definition.Resource; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsUrlStreamHandlerFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.eclipse.jetty.server.Server; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class LocalStoreTest { + + @TempDir + private static java.nio.file.Path tempDir; + + private static final String CONTEXT_NAME = "TEST_CONTEXT"; + private static final int MONITOR_INTERVAL_SECS = 5; + private static MiniDFSCluster hdfs; + private static Server jetty; + private static ContextDefinition def; + private static TestClassInfo classA; + private static TestClassInfo classB; + private static TestClassInfo classC; + private static TestClassInfo classD; + private static java.nio.file.Path baseCacheDir = null; + + @BeforeAll + public static void beforeAll() throws Exception { + baseCacheDir = tempDir.resolve("base"); + + // Find the Test jar files + final URL jarAOrigLocation = LocalStoreTest.class.getResource("/ClassLoaderTestA/TestA.jar"); + assertNotNull(jarAOrigLocation); + final URL jarBOrigLocation = LocalStoreTest.class.getResource("/ClassLoaderTestB/TestB.jar"); + assertNotNull(jarBOrigLocation); + final URL jarCOrigLocation = LocalStoreTest.class.getResource("/ClassLoaderTestC/TestC.jar"); + assertNotNull(jarCOrigLocation); + + // Put B into HDFS + hdfs = TestUtils.getMiniCluster(); + final FileSystem fs = hdfs.getFileSystem(); + assertTrue(fs.mkdirs(new Path("/contextB"))); + final Path dst = new Path("/contextB/TestB.jar"); + fs.copyFromLocalFile(new Path(jarBOrigLocation.toURI()), dst); + assertTrue(fs.exists(dst)); + URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory(hdfs.getConfiguration(0))); + final URL jarBNewLocation = new URL(fs.getUri().toString() + dst.toUri().toString()); + + // Put C into Jetty + java.nio.file.Path jarCParentDirectory = + java.nio.file.Path.of(jarCOrigLocation.toURI()).getParent(); + jetty = TestUtils.getJetty(jarCParentDirectory); + final URL jarCNewLocation = jetty.getURI().resolve("TestC.jar").toURL(); + + // Create ContextDefinition with all three resources + final LinkedHashSet<Resource> resources = new LinkedHashSet<>(); + resources + .add(new Resource(jarAOrigLocation, TestUtils.computeResourceChecksum(jarAOrigLocation))); + resources + .add(new Resource(jarBNewLocation, TestUtils.computeResourceChecksum(jarBOrigLocation))); + resources + .add(new Resource(jarCNewLocation, TestUtils.computeResourceChecksum(jarCOrigLocation))); + + def = new ContextDefinition(CONTEXT_NAME, MONITOR_INTERVAL_SECS, resources); + classA = new TestClassInfo("test.TestObjectA", "Hello from A"); + classB = new TestClassInfo("test.TestObjectB", "Hello from B"); + classC = new TestClassInfo("test.TestObjectC", "Hello from C"); + classD = new TestClassInfo("test.TestObjectD", "Hello from D"); + } + + @AfterAll + public static void afterAll() throws Exception { + if (jetty != null) { + jetty.stop(); + jetty.join(); + } + if (hdfs != null) { + hdfs.shutdown(); + } + } + + @AfterEach + public void cleanBaseDir() throws Exception { + if (Files.exists(baseCacheDir)) { + try (var walker = Files.walk(baseCacheDir)) { + walker.map(java.nio.file.Path::toFile).sorted(Comparator.reverseOrder()) + .forEach(File::delete); + } + } + } + + @Test + public void testPropertyNotSet() { + assertThrows(NullPointerException.class, () -> new LocalStore(null)); + } + + @Test + public void testCreateBaseDirs() throws Exception { + assertFalse(Files.exists(baseCacheDir)); + var localStore = new LocalStore(baseCacheDir); + assertTrue(Files.exists(baseCacheDir)); + assertTrue(Files.exists(baseCacheDir.resolve("contexts"))); + assertTrue(Files.exists(baseCacheDir.resolve("resources"))); + assertEquals(baseCacheDir.resolve("contexts"), localStore.contextsDir()); + assertEquals(baseCacheDir.resolve("resources"), localStore.resourcesDir()); + } + + @Test + public void testCreateBaseDirsMultipleTimes() throws Exception { + assertFalse(Files.exists(baseCacheDir)); + assertNotNull(new LocalStore(baseCacheDir)); + assertNotNull(new LocalStore(baseCacheDir)); + assertNotNull(new LocalStore(baseCacheDir)); + assertNotNull(new LocalStore(baseCacheDir)); + assertTrue(Files.exists(baseCacheDir)); + } + + @Test + public void testLocalFileName() { + // regular json + assertEquals("f1-chk1.json", LocalStore.localName("f1.json", "chk1")); + // dotfile json + assertEquals(".f1-chk1.json", LocalStore.localName(".f1.json", "chk1")); + // regular jar (has multiple dots) + assertEquals("f2-1.0-chk2.jar", LocalStore.localName("f2-1.0.jar", "chk2")); + // dotfile jar (has multiple dots) + assertEquals(".f2-1.0-chk2.jar", LocalStore.localName(".f2-1.0.jar", "chk2")); + // regular file with no suffix + assertEquals("f3-chk3", LocalStore.localName("f3", "chk3")); + + // weird files with trailing dots and no file suffix + assertEquals("f4.-chk4", LocalStore.localName("f4.", "chk4")); + assertEquals("f4..-chk4", LocalStore.localName("f4..", "chk4")); + assertEquals("f4...-chk4", LocalStore.localName("f4...", "chk4")); + // weird dotfiles that don't really have a suffix + assertEquals(".f5-chk5", LocalStore.localName(".f5", "chk5")); + assertEquals("..f5-chk5", LocalStore.localName("..f5", "chk5")); + // weird files with weird dots, but do have a valid suffix + assertEquals("f6.-chk6.jar", LocalStore.localName("f6..jar", "chk6")); + assertEquals("f6..-chk6.jar", LocalStore.localName("f6...jar", "chk6")); + assertEquals(".f6-chk6.jar", LocalStore.localName(".f6.jar", "chk6")); + assertEquals("..f6-chk6.jar", LocalStore.localName("..f6.jar", "chk6")); + assertEquals(".f6.-chk6.jar", LocalStore.localName(".f6..jar", "chk6")); + assertEquals("..f6.-chk6.jar", LocalStore.localName("..f6..jar", "chk6")); + } + + @Test + public void testStoreContextResources() throws Exception { + var localStore = new LocalStore(baseCacheDir); + localStore.storeContextResources(def); + + // Confirm the 3 jars are cached locally + assertTrue(Files.exists(baseCacheDir)); + assertTrue(Files.exists(baseCacheDir.resolve("contexts") + .resolve(CONTEXT_NAME + "_" + def.getChecksum() + ".json"))); + for (Resource r : def.getResources()) { + String filename = TestUtils.getFileName(r.getLocation()); + String checksum = r.getChecksum(); + assertTrue( + Files.exists(baseCacheDir.resolve("resources").resolve(filename + "_" + checksum))); + } + } + + @Test + public void testClassLoader() throws Exception { + var helper = new LocalStore(baseCacheDir).storeContextResources(def); + ClassLoader contextClassLoader = helper.createClassLoader(); + + testClassLoads(contextClassLoader, classA); + testClassLoads(contextClassLoader, classB); + testClassLoads(contextClassLoader, classC); + } + + @Test + public void testUpdate() throws Exception { + var localStore = new LocalStore(baseCacheDir); + var helper = localStore.storeContextResources(def); + final ClassLoader contextClassLoader = helper.createClassLoader(); + + testClassLoads(contextClassLoader, classA); + testClassLoads(contextClassLoader, classB); + testClassLoads(contextClassLoader, classC); + + // keep all but C + var updatedResources = def.getResources().stream().limit(def.getResources().size() - 1) + .collect(Collectors.toCollection(LinkedHashSet::new)); + assertEquals(def.getResources().size() - 1, updatedResources.size()); + + // Add D + final URL jarDOrigLocation = LocalStoreTest.class.getResource("/ClassLoaderTestD/TestD.jar"); + assertNotNull(jarDOrigLocation); + updatedResources + .add(new Resource(jarDOrigLocation, TestUtils.computeResourceChecksum(jarDOrigLocation))); + + var updatedDef = new ContextDefinition(CONTEXT_NAME, MONITOR_INTERVAL_SECS, updatedResources); + helper = localStore.storeContextResources(updatedDef); + + // Confirm the 3 jars are cached locally + assertTrue(Files.exists(baseCacheDir.resolve("contexts") + .resolve(CONTEXT_NAME + "_" + updatedDef.getChecksum() + ".json"))); + for (Resource r : updatedDef.getResources()) { + String filename = TestUtils.getFileName(r.getLocation()); + assertFalse(filename.contains("C")); + String checksum = r.getChecksum(); + assertTrue( + Files.exists(baseCacheDir.resolve("resources").resolve(filename + "_" + checksum))); + } + + final ClassLoader updatedContextClassLoader = helper.createClassLoader(); + + assertNotEquals(contextClassLoader, updatedContextClassLoader); + testClassLoads(updatedContextClassLoader, classA); + testClassLoads(updatedContextClassLoader, classB); + testClassFailsToLoad(updatedContextClassLoader, classC); + testClassLoads(updatedContextClassLoader, classD); + } + Review Comment: We should probably have a test that calls `localStore.storeContextResources(def);` concurrently from 2 or 3 threads. I think the test above calls it twice serially with overlapping files, so handling an existing file is covered. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
