This is an automated email from the ASF dual-hosted git repository. ishan pushed a commit to branch jira/solr-15951 in repository https://gitbox.apache.org/repos/asf/solr.git
commit 23b7443c2a89f6c5cffa9421f22a41a1862aabb9 Author: Ishan Chattopadhyaya <[email protected]> AuthorDate: Wed Feb 2 17:04:52 2022 +0530 Local packages --- gradle/solr/packaging.gradle | 2 + solr/bin/solr | 1 + .../solr/client/solrj/embedded/JettyConfig.java | 15 ++- .../src/java/org/apache/solr/pkg/PackageAPI.java | 8 +- .../java/org/apache/solr/pkg/PackageLoader.java | 123 ++++++++++++++--- .../apache/solr/cloud/PackageManagerCLITest.java | 5 +- .../cluster/events/ClusterEventProducerTest.java | 5 +- .../solr/filestore/TestDistribPackageStore.java | 5 +- .../apache/solr/handler/TestContainerPlugin.java | 5 +- .../org/apache/solr/pkg/TestLocalPackages.java | 146 +++++++++++++++++++++ .../src/test/org/apache/solr/pkg/TestPackages.java | 8 +- solr/modules/build.gradle | 71 ++++++++++ solr/modules/extraction/manifest.json | 22 ++++ solr/modules/scripting/manifest.json | 23 ++++ .../update/ScriptUpdateProcessorFactory.java | 7 +- .../apache/solr/cloud/MiniSolrCloudCluster.java | 1 + 16 files changed, 411 insertions(+), 36 deletions(-) diff --git a/gradle/solr/packaging.gradle b/gradle/solr/packaging.gradle index e0bd891..7a33582 100644 --- a/gradle/solr/packaging.gradle +++ b/gradle/solr/packaging.gradle @@ -65,6 +65,8 @@ configure(allprojects.findAll {project -> project.path.startsWith(":solr:modules task assemblePackaging(type: Sync) { from "README.md" + from "manifest.json" + from (tasks.jar, { into "lib" }) diff --git a/solr/bin/solr b/solr/bin/solr index 3a6f0e4..3eb5c87 100755 --- a/solr/bin/solr +++ b/solr/bin/solr @@ -2267,6 +2267,7 @@ function start_solr() { # users who don't care about useful error msgs can override in SOLR_OPTS with +OmitStackTraceInFastThrow "${SOLR_HOST_ARG[@]}" "-Duser.timezone=$SOLR_TIMEZONE" "-XX:-OmitStackTraceInFastThrow" \ "-XX:OnOutOfMemoryError=$SOLR_TIP/bin/oom_solr.sh $SOLR_PORT $SOLR_LOGS_DIR" \ + "-Dsolr.packages.local.dir=$SOLR_TIP/modules" \ "-Djetty.home=$SOLR_SERVER_DIR" "-Dsolr.solr.home=$SOLR_HOME" "-Dsolr.data.home=$SOLR_DATA_HOME" "-Dsolr.install.dir=$SOLR_TIP" \ "-Dsolr.default.confdir=$DEFAULT_CONFDIR" "${LOG4J_CONFIG[@]}" "${SOLR_OPTS[@]}" "${SECURITY_MANAGER_OPTS[@]}" "${SOLR_ADMIN_UI}") diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettyConfig.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettyConfig.java index e4a0547..0f97f18 100644 --- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettyConfig.java +++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettyConfig.java @@ -22,6 +22,7 @@ import javax.servlet.Filter; import java.util.LinkedHashMap; import java.util.Map; import java.util.TreeMap; +import java.util.function.Consumer; public class JettyConfig { @@ -47,9 +48,12 @@ public class JettyConfig { public final int portRetryTime; + public final Consumer<JettySolrRunner> preStartupHook; + private JettyConfig(boolean onlyHttp1, int port, int portRetryTime , String context, boolean stopAtShutdown, Long waitForLoadingCoresToFinishMs, Map<ServletHolder, String> extraServlets, - Map<Class<? extends Filter>, String> extraFilters, SSLConfig sslConfig, boolean enableV2) { + Map<Class<? extends Filter>, String> extraFilters, SSLConfig sslConfig, boolean enableV2, + Consumer<JettySolrRunner> preStartupHook) { this.onlyHttp1 = onlyHttp1; this.port = port; this.context = context; @@ -60,6 +64,7 @@ public class JettyConfig { this.sslConfig = sslConfig; this.portRetryTime = portRetryTime; this.enableV2 = enableV2; + this.preStartupHook = preStartupHook; } public static Builder builder() { @@ -89,6 +94,7 @@ public class JettyConfig { Map<Class<? extends Filter>, String> extraFilters = new LinkedHashMap<>(); SSLConfig sslConfig = null; int portRetryTime = 60; + Consumer<JettySolrRunner> preStartupHook; public Builder useOnlyHttp1(boolean useOnlyHttp1) { this.onlyHttp1 = useOnlyHttp1; @@ -151,10 +157,15 @@ public class JettyConfig { return this; } + public Builder withPreStartupHook(Consumer<JettySolrRunner> preStartupHook) { + this.preStartupHook = preStartupHook; + return this; + } + public JettyConfig build() { return new JettyConfig(onlyHttp1, port, portRetryTime, context, stopAtShutdown, - waitForLoadingCoresToFinishMs, extraServlets, extraFilters, sslConfig, enableV2); + waitForLoadingCoresToFinishMs, extraServlets, extraFilters, sslConfig, enableV2, preStartupHook); } } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index 3a01c1b..1c197b4 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -60,13 +60,12 @@ import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_ * */ public class PackageAPI { - public final boolean enablePackages = Boolean.parseBoolean(System.getProperty("enable.packages", "false")); private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static final String ERR_MSG = "Package loading is not enabled , Start your nodes with -Denable.packages=true"; final CoreContainer coreContainer; - private final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); + public static final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); private final PackageLoader packageLoader; Packages pkgs; @@ -81,6 +80,7 @@ public class PackageAPI { this.coreContainer = coreContainer; this.packageLoader = loader; pkgs = new Packages(); + if(!loader.repoPackagesEnabled) return; SolrZkClient zkClient = coreContainer.getZkController().getZkClient(); try { pkgs = readPkgsFromZk(null, null); @@ -360,11 +360,11 @@ public class PackageAPI { } public boolean isEnabled() { - return enablePackages; + return packageLoader.repoPackagesEnabled; } private boolean checkEnabled(CommandOperation payload) { - if (!enablePackages) { + if (!packageLoader.repoPackagesEnabled) { payload.addError(ERR_MSG); return false; } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java index b73be8c..bb85eb8 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java @@ -17,10 +17,9 @@ package org.apache.solr.pkg; -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.lang.invoke.MethodHandles; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -37,7 +36,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import org.apache.solr.common.MapWriter; +import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrResourceLoader; @@ -52,14 +53,24 @@ import static org.apache.lucene.util.IOUtils.closeWhileHandlingException; public class PackageLoader implements Closeable { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static final String LATEST = "$LATEST"; + public static final String LOCAL_PKGS_DIR_PROP = "solr.packages.local.dir"; + public static final String ENABLED_LOCAL_PKGS_PROP = "solr.enabled.local.pkgs"; + public static final String LOCAL_PACKAGES_JSON = "local_packages.json"; + public static final String ENABLE_PACKAGES_REPO_PROP = "enable.packages"; + public static final String ENABLE_PACKAGES_REPO_PROP_NEW = "solr.enable.pkgs.repo"; + public final String localPkgsDir = System.getProperty(LOCAL_PKGS_DIR_PROP); + public final String enabledLocalPkgsList = System.getProperty(ENABLED_LOCAL_PKGS_PROP, ""); + public final boolean repoPackagesEnabled = Boolean.parseBoolean(System.getProperty(ENABLE_PACKAGES_REPO_PROP, + System.getProperty(ENABLE_PACKAGES_REPO_PROP_NEW, "false"))); private final CoreContainer coreContainer; private final Map<String, Package> packageClassLoaders = new ConcurrentHashMap<>(); + public PackageAPI.Packages localPackages; private PackageAPI.Packages myCopy = new PackageAPI.Packages(); - private PackageAPI packageAPI; + private final PackageAPI packageAPI; public Optional<Package.Version> getPackageVersion(String pkg, String version) { @@ -70,9 +81,80 @@ public class PackageLoader implements Closeable { public PackageLoader(CoreContainer coreContainer) { this.coreContainer = coreContainer; + + List<String> enabledPackages = StrUtils.splitSmart(enabledLocalPkgsList, ','); packageAPI = new PackageAPI(coreContainer, this); - refreshPackageConf(); + if (localPkgsDir != null && !enabledPackages.isEmpty()) { + loadLocalPackages(localPkgsDir, enabledPackages); + } + if (repoPackagesEnabled) { + refreshPackageConf(); + } + } + + private void loadLocalPackages(String localPkgsDir, List<String> enabledPackages) { + final Path packagesPath = localPkgsDir.charAt(0) == File.separatorChar ? + Paths.get(localPkgsDir) : + Paths.get(coreContainer.getSolrHome()).resolve(localPkgsDir); + log.info("Packages to be loaded from directory: {}", packagesPath); + + if (!Files.exists(packagesPath)) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No such directory: " + packagesPath); + } + Path packagesJsonPath = packagesPath.resolve(LOCAL_PACKAGES_JSON); + if(Files.exists(packagesJsonPath)) { + try { + + try (InputStream in = Files.newInputStream(packagesJsonPath)) { + localPackages = PackageAPI.mapper.readValue(in, PackageAPI.Packages.class); + } + } catch (IOException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error reading local_packages.json", e); + } + } else { + //the no local_packages.json + //we will look for each subdirectory and consider them as a package + localPackages = readDirAsPackages(packagesPath); + } + + for (Map.Entry<String, List<PackageAPI.PkgVersion>> e : localPackages.packages.entrySet()) { + if(!enabledPackages.contains(e.getKey())) continue; + Package p = new Package(e.getKey()); + p.updateVersions(e.getValue(), packagesPath); + packageClassLoaders.put(e.getKey(), p); + } + } + + /** + * If a directory with no local_packages.json is provided, + * each sub directory that contains one or more jar files + * will be treated as a package and each jar file in that + * directory is added to the package classpath + */ + private PackageAPI.Packages readDirAsPackages(Path packagesPath) { + PackageAPI.Packages localDirAsPackages = new PackageAPI.Packages(); + packagesPath.toFile().list((dir, pkgName) -> { + File subDir = new File(dir, pkgName); + if(subDir.isDirectory()) { + PackageAPI.PkgVersion version = new PackageAPI.PkgVersion(); + version.pkg = pkgName; + version.version = "0"; + version.files = new ArrayList<>(); + subDir.list((dir1, jarName) -> { + if(jarName.endsWith(".jar")){ + version.files.add(pkgName+File.separator+jarName); + } + return false; + }); + if(!version.files.isEmpty()) { + //there are jar files in the dir. So, this is a package + localDirAsPackages.packages.put(pkgName, Collections.singletonList(version)); + } + } + return false; + }); + return localDirAsPackages; } public PackageAPI getPackageAPI() { @@ -99,7 +181,7 @@ public class PackageLoader implements Closeable { if (e.getValue() != null && p == null) { packageClassLoaders.put(e.getKey(), p = new Package(e.getKey())); } - p.updateVersions(e.getValue()); + p.updateVersions(e.getValue(), null); updated.add(p); } else { Package p = packageClassLoaders.remove(e.getKey()); @@ -181,14 +263,14 @@ public class PackageLoader implements Closeable { } - private synchronized void updateVersions(List<PackageAPI.PkgVersion> modified) { + private synchronized void updateVersions(List<PackageAPI.PkgVersion> modified, Path localpkgDir) { for (PackageAPI.PkgVersion v : modified) { Version version = myVersions.get(v.version); if (version == null) { log.info("A new version: {} added for package: {} with artifacts {}", v.version, this.name, v.files); Version ver = null; try { - ver = new Version(this, v); + ver = new Version(this, v, localpkgDir); } catch (Exception e) { log.error("package could not be loaded {}", ver, e); continue; @@ -274,20 +356,25 @@ public class PackageLoader implements Closeable { version.writeMap(ew); } - Version(Package parent, PackageAPI.PkgVersion v) { + Version(Package parent, PackageAPI.PkgVersion v, Path localPkgDir) { this.parent = parent; this.version = v; List<Path> paths = new ArrayList<>(); - - List<String> errs = new ArrayList<>(); - coreContainer.getPackageStoreAPI().validateFiles(version.files, true, s -> errs.add(s)); - if(!errs.isEmpty()) { - throw new RuntimeException("Cannot load package: " +errs); - } - for (String file : version.files) { - paths.add(coreContainer.getPackageStoreAPI().getPackageStore().getRealpath(file)); + if(localPkgDir != null) { + for (String file : v.files) { + if(file.charAt(0)== '/') file =file.substring(1); + paths.add( localPkgDir.resolve(file).toAbsolutePath()) ; + } + } else { + List<String> errs = new ArrayList<>(); + coreContainer.getPackageStoreAPI().validateFiles(version.files, true, s -> errs.add(s)); + if(!errs.isEmpty()) { + throw new RuntimeException("Cannot load package: " +errs); + } + for (String file : version.files) { + paths.add(coreContainer.getPackageStoreAPI().getPackageStore().getRealpath(file)); + } } - loader = new PackageResourceLoader( "PACKAGE_LOADER: " + parent.name() + ":" + version, paths, diff --git a/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java b/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java index aad9286..cf737a7 100644 --- a/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java +++ b/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java @@ -22,6 +22,7 @@ import java.util.Arrays; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.core.TestSolrConfigHandler; +import org.apache.solr.pkg.PackageLoader; import org.apache.solr.util.LogLevel; import org.apache.solr.util.PackageTool; import org.apache.solr.util.SolrCLI; @@ -51,7 +52,7 @@ public class PackageManagerCLITest extends SolrCloudTestCase { @BeforeClass public static void setupCluster() throws Exception { - System.setProperty("enable.packages", "true"); + System.setProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP, "true"); configureCluster(1) .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) @@ -65,7 +66,7 @@ public class PackageManagerCLITest extends SolrCloudTestCase { @AfterClass public static void teardown() throws Exception { repositoryServer.stop(); - System.clearProperty("enable.packages"); + System.clearProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP); } @Test diff --git a/solr/core/src/test/org/apache/solr/cluster/events/ClusterEventProducerTest.java b/solr/core/src/test/org/apache/solr/cluster/events/ClusterEventProducerTest.java index a96e209..4426806 100644 --- a/solr/core/src/test/org/apache/solr/cluster/events/ClusterEventProducerTest.java +++ b/solr/core/src/test/org/apache/solr/cluster/events/ClusterEventProducerTest.java @@ -29,6 +29,7 @@ import org.apache.solr.cluster.events.impl.DefaultClusterEventProducer; import org.apache.solr.cluster.events.impl.DelegatingClusterEventProducer; import org.apache.solr.common.cloud.ClusterProperties; import org.apache.solr.common.util.Utils; +import org.apache.solr.pkg.PackageLoader; import org.apache.solr.util.LogLevel; import org.junit.After; import org.junit.Before; @@ -70,7 +71,7 @@ public class ClusterEventProducerTest extends SolrCloudTestCase { @Before public void setUp() throws Exception { - System.setProperty("enable.packages", "true"); + System.setProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP, "true"); super.setUp(); cluster.deleteAllCollections(); eventsListener = new AllEventsListener(); @@ -85,7 +86,7 @@ public class ClusterEventProducerTest extends SolrCloudTestCase { @After public void teardown() throws Exception { - System.clearProperty("enable.packages"); + System.clearProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP); if (eventsListener != null) { cluster.getOpenOverseer().getCoreContainer().getClusterEventProducer().unregisterListener(eventsListener); eventsListener.events.clear(); diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java index e25be64..711cb52 100644 --- a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java +++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java @@ -36,6 +36,7 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; import org.apache.solr.packagemanager.PackageUtils; +import org.apache.solr.pkg.PackageLoader; import org.apache.solr.util.LogLevel; import org.apache.zookeeper.server.ByteBufferInputStream; import org.junit.After; @@ -63,12 +64,12 @@ public class TestDistribPackageStore extends SolrCloudTestCase { @Before public void setup() { - System.setProperty("enable.packages", "true"); + System.setProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP, "true"); } @After public void teardown() { - System.clearProperty("enable.packages"); + System.clearProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP); } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java index 7254b0f..a64de90 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java +++ b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java @@ -52,6 +52,7 @@ import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.filestore.PackageStoreAPI; import org.apache.solr.filestore.TestDistribPackageStore; import org.apache.solr.filestore.TestDistribPackageStore.Fetcher; +import org.apache.solr.pkg.PackageLoader; import org.apache.solr.pkg.TestPackages; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -71,13 +72,13 @@ public class TestContainerPlugin extends SolrCloudTestCase { @Before public void setup() { - System.setProperty("enable.packages", "true"); + System.setProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP, "true"); phaser = new Phaser(); } @After public void teardown() { - System.clearProperty("enable.packages"); + System.clearProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP); } @Test diff --git a/solr/core/src/test/org/apache/solr/pkg/TestLocalPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestLocalPackages.java new file mode 100644 index 0000000..1dfe51f --- /dev/null +++ b/solr/core/src/test/org/apache/solr/pkg/TestLocalPackages.java @@ -0,0 +1,146 @@ +package org.apache.solr.pkg; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.lang.invoke.MethodHandles; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.cloud.MiniSolrCloudCluster; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.util.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.pkg.PackageLoader.ENABLED_LOCAL_PKGS_PROP; +import static org.apache.solr.pkg.PackageLoader.LOCAL_PKGS_DIR_PROP; + +public class TestLocalPackages extends SolrCloudTestCase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public void testLocalPackagesAsDir() throws Exception { + String PKG_NAME = "mypkg"; + String jarName = "mypkg1.jar"; + String COLLECTION_NAME = "testLocalPkgsColl"; + String localPackagesDir = "testpkgdir"; + System.setProperty(ENABLED_LOCAL_PKGS_PROP, PKG_NAME); + System.setProperty(LOCAL_PKGS_DIR_PROP, localPackagesDir); + MiniSolrCloudCluster cluster = + configureCluster(4) + .withJettyConfig(builder -> builder.enableV2(true)) + .withJettyConfig(it -> it.withPreStartupHook(jsr -> { + try { + File pkgDir = new File(jsr.getSolrHome() + File.separator + localPackagesDir); + if (!pkgDir.exists()) { + pkgDir.mkdir(); + } + File subDir = new File(pkgDir, PKG_NAME); + if (!subDir.exists()) { + subDir.mkdir(); + } + try (FileInputStream fis = new FileInputStream(getFile("runtimecode/runtimelibs.jar.bin"))) { + byte[] buf = new byte[fis.available()]; + + fis.read(buf); + try (FileOutputStream fos = new FileOutputStream(new File(subDir, jarName))) { + fos.write(buf, 0, buf.length); + } + } + + } catch (Exception e) { + throw new RuntimeException("Unable to create files", e); + } + })) + .addConfig("conf", configset("conf2")) + .configure(); + + System.clearProperty(ENABLED_LOCAL_PKGS_PROP); + System.clearProperty(LOCAL_PKGS_DIR_PROP); + try { + for (JettySolrRunner jsr : cluster.getJettySolrRunners()) { + List<String> packageFiles = Arrays.asList(new File(jsr.getSolrHome() + File.separator + localPackagesDir + File.separator + PKG_NAME).list()); + assertTrue(packageFiles.contains(jarName)); + } + CollectionAdminRequest + .createCollection(COLLECTION_NAME, "conf", 2, 2) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); + + log.info("Collection created successfully"); + + TestPackages.verifyComponent(cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", PKG_NAME, "0"); + } finally { + cluster.shutdown(); + } + } + + public void testLocalPackages() throws Exception { + String PKG_NAME = "mypkg"; + String jarName = "mypkg1.jar"; + String COLLECTION_NAME = "testLocalPkgsColl"; + String localPackagesDir = "local_packages"; + PackageAPI.Packages p = new PackageAPI.Packages(); + PackageAPI.PkgVersion pkgVersion = new PackageAPI.PkgVersion(); + pkgVersion.files = Collections.singletonList(jarName); + pkgVersion.version = "0.1"; + pkgVersion.pkg = PKG_NAME; + p.packages.put(PKG_NAME, Collections.singletonList(pkgVersion)); + + log.info("local_packages.json: {}" , Utils.toJSONString(p)); + log.info("Local packages dir: {}" , localPackagesDir); + System.setProperty(ENABLED_LOCAL_PKGS_PROP, PKG_NAME); + System.setProperty(LOCAL_PKGS_DIR_PROP, localPackagesDir); + MiniSolrCloudCluster cluster = + configureCluster(4) + .withJettyConfig(builder -> builder.enableV2(true)) + .withJettyConfig(it -> it.withPreStartupHook(jsr -> { + try { + File pkgDir = new File(jsr.getSolrHome() + File.separator + localPackagesDir); + if (!pkgDir.exists()) { + pkgDir.mkdir(); + } + try (FileInputStream fis = new FileInputStream(getFile("runtimecode/runtimelibs.jar.bin"))) { + byte[] buf = new byte[fis.available()]; + + fis.read(buf); + try (FileOutputStream fos = new FileOutputStream(new File(pkgDir, jarName))) { + fos.write(buf, 0, buf.length); + } + } + + try( FileOutputStream fos = new FileOutputStream( new File(pkgDir, PackageLoader.LOCAL_PACKAGES_JSON) )) { + fos.write(Utils.toJSON(p)); + } + } catch (Exception e) { + throw new RuntimeException("Unable to create files", e); + } + })) + .addConfig("conf", configset("conf2")) + .configure(); + + System.clearProperty(ENABLED_LOCAL_PKGS_PROP); + System.clearProperty(LOCAL_PKGS_DIR_PROP); + try { + for (JettySolrRunner jsr : cluster.getJettySolrRunners()) { + List<String> packageFiles = Arrays.asList(new File(jsr.getSolrHome() + File.separator + localPackagesDir).list()); + assertTrue(packageFiles.contains(PackageLoader.LOCAL_PACKAGES_JSON)); + assertTrue(packageFiles.contains(jarName)); + } + CollectionAdminRequest + .createCollection(COLLECTION_NAME, "conf", 2, 2) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); + + log.info("Collection created successfully"); + + TestPackages.verifyComponent(cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", PKG_NAME, pkgVersion.version); + + + } finally { + cluster.shutdown(); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index 4c91daa..b1d3b3b 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -82,11 +82,12 @@ import static org.apache.solr.filestore.TestDistribPackageStore.*; @LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG") public class TestPackages extends SolrCloudTestCase { + @Before @Override public void setUp() throws Exception { super.setUp(); - System.setProperty("enable.packages", "true"); + System.setProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP, "true"); configureCluster(4) .withJettyConfig(jetty -> jetty.enableV2(true)) .addConfig("conf", configset("conf2")) @@ -100,7 +101,7 @@ public class TestPackages extends SolrCloudTestCase { if (cluster != null) { cluster.shutdown(); } - System.clearProperty("enable.packages"); + System.clearProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP); super.tearDown(); } @@ -114,6 +115,7 @@ public class TestPackages extends SolrCloudTestCase { } + @Test public void testCoreReloadingPlugin() throws Exception { String FILE1 = "/mypkg/runtimelibs.jar"; @@ -517,7 +519,7 @@ public class TestPackages extends SolrCloudTestCase { } } - private void verifyComponent(SolrClient client, String COLLECTION_NAME, + static void verifyComponent(SolrClient client, String COLLECTION_NAME, String componentType, String componentName, String pkg, String version) throws Exception { SolrParams params = new MapSolrParams(Map.of("collection", COLLECTION_NAME, WT, JAVABIN, diff --git a/solr/modules/build.gradle b/solr/modules/build.gradle new file mode 100644 index 0000000..c387f6b --- /dev/null +++ b/solr/modules/build.gradle @@ -0,0 +1,71 @@ +import groovy.json.JsonOutput + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +description = 'Solr Modules' + +ext { + packagingDir = file("$buildDir/packaging") + localPackagesFile = "$packagingDir/local_packages.json" +} + +configurations { + modulePackages + packaging +} + +dependencies { + [":solr:modules:extraction", + ":solr:modules:scripting" + ].each { moduleName -> + modulePackages project(path: moduleName, configuration: "packaging") + } + + packaging files(packagingDir) { + builtBy 'createLocalPackagesJson' + } +} + +task createLocalPackagesJson { + dependsOn configurations.modulePackages + + doLast { + def packagesJson = [ + packages: [:] + ] + configurations.modulePackages.each { depPath -> + def moduleName = depPath.parentFile.parentFile.name + def module = [ + version: version, + moduleFiles: [], + manifest: "", + ] + fileTree(depPath).filter { it.isFile() }.files.each {pkgFile -> + def filePath = depPath.toPath().relativize(pkgFile.toPath()).toString() + if (filePath.endsWith("manifest.json")) { + module.manifest = filePath + } else if (filePath.contains("/lib/")) { + module.moduleFiles.add(filePath) + } + } + packagesJson.packages[moduleName] = module + } + mkdir packagingDir + new File(localPackagesFile).text = JsonOutput.prettyPrint(JsonOutput.toJson(packagesJson)) + } +} diff --git a/solr/modules/extraction/manifest.json b/solr/modules/extraction/manifest.json new file mode 100644 index 0000000..6b82948 --- /dev/null +++ b/solr/modules/extraction/manifest.json @@ -0,0 +1,22 @@ +{ + "version-constraint": "9", + "plugins": [ + { + "name": "update-extraction", + "setup-command": { + "path": "/api/collections/${collection}/config", + "payload": {"add-requesthandler": {"name": "${NAME}", "class": "solr-extraction:org.apache.solr.handler.extraction.ExtractingRequestHandler"}}, + "method": "POST" + }, + "uninstall-command": { + "path": "/api/collections/${collection}/config", + "payload": {"delete-requesthandler": "${NAME}"}, + "method": "POST" + } + } + ], + "parameter-defaults": { + "NAME": "/update/extract" + } + } + diff --git a/solr/modules/scripting/manifest.json b/solr/modules/scripting/manifest.json new file mode 100644 index 0000000..cd3b28d --- /dev/null +++ b/solr/modules/scripting/manifest.json @@ -0,0 +1,23 @@ +{ + "version-constraint": "10", + "plugins": [ + { + "name": "scripting", + "setup-command": { + "path": "/api/collections/${collection}/config", + "payload": {"add-updateprocessor": {"name": "${NAME}", "class": "solr-scripting:org.apache.solr.scripting.update.ScriptUpdateProcessorFactory", "script": "${SCRIPT}"}}, + "method": "POST" + }, + "uninstall-command": { + "path": "/api/collections/${collection}/config", + "payload": {"delete-updateprocessor": "${NAME}"}, + "method": "POST" + } + } + ], + "parameter-defaults": { + "NAME": "scripting", + "SCRIPT": "update-script.js" + } + } + diff --git a/solr/modules/scripting/src/java/org/apache/solr/scripting/update/ScriptUpdateProcessorFactory.java b/solr/modules/scripting/src/java/org/apache/solr/scripting/update/ScriptUpdateProcessorFactory.java index 4543b97..804cca7 100644 --- a/solr/modules/scripting/src/java/org/apache/solr/scripting/update/ScriptUpdateProcessorFactory.java +++ b/solr/modules/scripting/src/java/org/apache/solr/scripting/update/ScriptUpdateProcessorFactory.java @@ -224,6 +224,7 @@ public class ScriptUpdateProcessorFactory extends UpdateRequestProcessorFactory } @Override + @SuppressWarnings({"unchecked", "rawtypes"}) public void inform(SolrCore core) { if (!core.getCoreDescriptor().isConfigSetTrusted()) { throw new SolrException(ErrorCode.UNAUTHORIZED, "The configset for this collection was uploaded without any authentication in place," @@ -232,8 +233,12 @@ public class ScriptUpdateProcessorFactory extends UpdateRequestProcessorFactory } resourceLoader = core.getResourceLoader(); - // test that our engines & scripts are valid + // Hack needed to preload some Lucene classes that otherwise don't load via Nashorn + // when this scripting module is loaded as a package. + // TODO: Preload all Lucene classes from lucene-core*jar and Solr classes from solr-core*jar + core.getResourceLoader().findClass("org.apache.lucene.index.Term", Object.class); + // test that our engines & scripts are valid SolrQueryResponse rsp = new SolrQueryResponse(); SolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams()); try { diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java index aa9e533..f3eed5c 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java @@ -471,6 +471,7 @@ public class MiniSolrCloudCluster { JettySolrRunner jetty = !trackJettyMetrics ? new JettySolrRunner(runnerPath.toString(), nodeProps, newConfig) : new JettySolrRunnerWithMetrics(runnerPath.toString(), nodeProps, newConfig); + if(config.preStartupHook != null) config.preStartupHook.accept(jetty); jetty.start(); jettys.add(jetty); synchronized (startupWait) {
