janhoy commented on a change in pull request #994: SOLR-13662: Package Manager (CLI) URL: https://github.com/apache/lucene-solr/pull/994#discussion_r346426522
########## File path: solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java ########## @@ -0,0 +1,328 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.packagemanager; + +import static org.apache.solr.packagemanager.PackageUtils.getMapper; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.lang.invoke.MethodHandles; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.lucene.util.Version; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.V2Request; +import org.apache.solr.client.solrj.request.beans.Package; +import org.apache.solr.client.solrj.response.V2Response; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.core.BlobRepository; +import org.apache.solr.packagemanager.SolrPackage.Artifact; +import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease; +import org.apache.solr.pkg.PackageAPI; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +/** + * Handles most of the management of repositories and packages present in external repositories. + */ +public class RepositoryManager { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + final private PackageManager packageManager; + + public static final String systemVersion = Version.LATEST.toString(); + + final HttpSolrClient solrClient; + + public RepositoryManager(HttpSolrClient solrClient, PackageManager packageManager) { + this.packageManager = packageManager; + this.solrClient = solrClient; + } + + public List<SolrPackage> getPackages() { + List<SolrPackage> list = new ArrayList<>(getPackagesMap().values()); + Collections.sort(list); + return list; + } + + /** + * Get a map of package name to {@link SolrPackage} objects + */ + public Map<String, SolrPackage> getPackagesMap() { + Map<String, SolrPackage> packagesMap = new HashMap<>(); + for (PackageRepository repository: getRepositories()) { + packagesMap.putAll(repository.getPackages()); + } + + return packagesMap; + } + + /** + * List of added repositories + */ + public List<PackageRepository> getRepositories() { + // TODO: Instead of fetching again and again, we should look for caching this + PackageRepository items[]; + try { + items = getMapper().readValue(getRepositoriesJson(packageManager.zkClient), DefaultPackageRepository[].class); + } catch (IOException | KeeperException | InterruptedException e) { + throw new SolrException(ErrorCode.SERVER_ERROR, e); + } + List<PackageRepository> repositories = Arrays.asList(items); + + for (PackageRepository updateRepository: repositories) { + updateRepository.refresh(); + } + + return repositories; + } + + /** + * Add a repository to Solr + */ + public void addRepository(String name, String uri) throws KeeperException, InterruptedException, MalformedURLException, IOException { + String existingRepositoriesJson = getRepositoriesJson(packageManager.zkClient); + log.info(existingRepositoriesJson); + + List repos = getMapper().readValue(existingRepositoriesJson, List.class); + repos.add(new DefaultPackageRepository(name, uri)); + if (packageManager.zkClient.exists("/repositories.json", true) == false) { + packageManager.zkClient.create("/repositories.json", getMapper().writeValueAsString(repos).getBytes("UTF-8"), CreateMode.PERSISTENT, true); + } else { + packageManager.zkClient.setData("/repositories.json", getMapper().writeValueAsString(repos).getBytes("UTF-8"), true); + } + + if (packageManager.zkClient.exists("/keys", true)==false) packageManager.zkClient.create("/keys", new byte[0], CreateMode.PERSISTENT, true); + if (packageManager.zkClient.exists("/keys/exe", true)==false) packageManager.zkClient.create("/keys/exe", new byte[0], CreateMode.PERSISTENT, true); + if (packageManager.zkClient.exists("/keys/exe/"+name+".der", true)==false) { + packageManager.zkClient.create("/keys/exe/"+name+".der", new byte[0], CreateMode.PERSISTENT, true); + } + packageManager.zkClient.setData("/keys/exe/"+name+".der", IOUtils.toByteArray(new URL(uri+"/publickey.der").openStream()), true); + + PackageUtils.printGreen("Added repository: "+name); + PackageUtils.printGreen(getRepositoriesJson(packageManager.zkClient)); + } + + private String getRepositoriesJson(SolrZkClient zkClient) throws UnsupportedEncodingException, KeeperException, InterruptedException { + if (zkClient.exists("/repositories.json", true)) { + return new String(zkClient.getData("/repositories.json", null, null, true), "UTF-8"); + } + return "[]"; + } + + /** + * Install a given package and version from the available repositories to Solr. + * The various steps for doing so are, briefly, a) find upload a manifest to package store, + * b) download the artifacts and upload to package store, c) call {@link PackageAPI} to register + * the package. + */ + private boolean installPackage(String packageName, String version) throws SolrException { + SolrPackageInstance existingPlugin = packageManager.getPackageInstance(packageName, version); + if (existingPlugin != null && existingPlugin.version.equals(version)) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Plugin already installed."); + } + + SolrPackageRelease release = getPackageRelease(packageName, version); + List<Path> downloaded = downloadPackageArtifacts(packageName, version); + // TODO: Should we introduce a checksum to validate the downloading? + // Currently, not a big problem since signature based checking happens anyway + + try { + // post the manifest + PackageUtils.printGreen("Posting manifest"); + + if (release.manifest == null) { + String manifestJson = PackageUtils.getFileFromJarsAsString(downloaded, "manifest.json"); + if (manifestJson == null) { + throw new SolrException(ErrorCode.BAD_REQUEST, "No manifest found for package: " + packageName + ", version: " + version); Review comment: You resolved my review comment without a reply or resolution? ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org For additional commands, e-mail: issues-h...@lucene.apache.org
