Github user hgadre commented on a diff in the pull request:
https://github.com/apache/lucene-solr/pull/52#discussion_r71745628
--- Diff:
solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotMetaDataManager.java
---
@@ -0,0 +1,419 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.core.snapshots;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexCommit;
+import org.apache.lucene.index.IndexDeletionPolicy;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.IOUtils;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.core.DirectoryFactory;
+import org.apache.solr.core.IndexDeletionPolicyWrapper;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.DirectoryFactory.DirContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * This class is responsible to manage the persistent snapshots meta-data
for the Solr indexes. The
+ * persistent snapshots are implemented by relying on Lucene {@linkplain
IndexDeletionPolicy}
+ * abstraction to configure a specific {@linkplain IndexCommit} to be
retained. The
+ * {@linkplain IndexDeletionPolicyWrapper} in Solr uses this class to
create/delete the Solr index
+ * snapshots.
+ */
+public class SolrSnapshotMetaDataManager {
+ private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final String SNAPSHOT_METADATA_DIR = "snapshot_metadata";
+
+ /**
+ * A class defining the meta-data for a specific snapshot.
+ */
+ public static class SnapshotMetaData {
+ private String name;
+ private String indexDirPath;
+ private long generationNumber;
+
+ public SnapshotMetaData(String name, String indexDirPath, long
generationNumber) {
+ super();
+ this.name = name;
+ this.indexDirPath = indexDirPath;
+ this.generationNumber = generationNumber;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getIndexDirPath() {
+ return indexDirPath;
+ }
+
+ public long getGenerationNumber() {
+ return generationNumber;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("SnapshotMetaData[name=");
+ builder.append(name);
+ builder.append(", indexDirPath=");
+ builder.append(indexDirPath);
+ builder.append(", generation=");
+ builder.append(generationNumber);
+ builder.append("]");
+ return builder.toString();
+ }
+ }
+
+ /** Prefix used for the save file. */
+ public static final String SNAPSHOTS_PREFIX = "snapshots_";
+ private static final int VERSION_START = 0;
+ private static final int VERSION_CURRENT = VERSION_START;
+ private static final String CODEC_NAME = "solr-snapshots";
+
+ // The index writer which maintains the snapshots metadata
+ private long nextWriteGen;
+
+ private final Directory dir;
+
+ /** Used to map snapshot name to snapshot meta-data. */
+ protected final Map<String,SnapshotMetaData> nameToDetailsMapping = new
HashMap<>();
+ /** Used to figure out the *current* index data directory path */
+ private final SolrCore solrCore;
+
+ /**
+ * A constructor.
+ *
+ * @param dir The directory where the snapshot meta-data should be
stored. Enables updating
+ * the existing meta-data.
+ * @throws IOException in case of errors.
+ */
+ public SolrSnapshotMetaDataManager(SolrCore solrCore, Directory dir)
throws IOException {
+ this(solrCore, dir, OpenMode.CREATE_OR_APPEND);
+ }
+
+ /**
+ * A constructor.
+ *
+ * @param dir The directory where the snapshot meta-data is stored.
+ * @param mode CREATE If previous meta-data should be erased.
+ * APPEND If previous meta-data should be read and updated.
+ * CREATE_OR_APPEND Creates a new meta-data structure if one
does not exist
+ * Updates the existing structure if one
exists.
+ * @throws IOException in case of errors.
+ */
+ public SolrSnapshotMetaDataManager(SolrCore solrCore, Directory dir,
OpenMode mode) throws IOException {
+ this.solrCore = solrCore;
+ this.dir = dir;
+
+ if (mode == OpenMode.CREATE) {
+ deleteSnapshotMetadataFiles();
+ }
+
+ loadFromSnapshotMetadataFile();
+
+ if (mode == OpenMode.APPEND && nextWriteGen == 0) {
+ throw new IllegalStateException("no snapshots stored in this
directory");
+ }
+ }
+
+ /**
+ * @return The snapshot meta-data directory
+ */
+ public Directory getSnapshotsDir() {
+ return dir;
+ }
+
+ /**
+ * This method creates a new snapshot meta-data entry.
+ *
+ * @param name The name of the snapshot.
+ * @param indexDirPath The directory path where the index files are
stored.
+ * @param gen The generation number for the {@linkplain IndexCommit}
being snapshotted.
+ * @throws IOException in case of I/O errors.
+ */
+ public synchronized void snapshot(String name, String indexDirPath, long
gen) throws IOException {
+ Preconditions.checkNotNull(name);
+
+ log.info("Creating the snapshot named {} for core {} associated with
index commit with generation {} in directory {}"
+ , name, solrCore.getName(), gen, indexDirPath);
+
+ if(nameToDetailsMapping.containsKey(name)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "A snapshot with name
" + name + " already exists");
+ }
+
+ SnapshotMetaData d = new SnapshotMetaData(name, indexDirPath, gen);
+ nameToDetailsMapping.put(name, d);
+
+ boolean success = false;
+ try {
+ persist();
+ success = true;
+ } finally {
+ if (!success) {
+ try {
+ release(name);
+ } catch (Exception e) {
+ // Suppress so we keep throwing original exception
+ }
+ }
+ }
+ }
+
+ /**
+ * This method deletes a previously created snapshot (if any).
+ *
+ * @param name The name of the snapshot to be deleted.
+ * @return The snapshot meta-data if the snapshot with the snapshot name
exists.
+ * @throws IOException in case of I/O error
+ */
+ public synchronized Optional<SnapshotMetaData> release(String name)
throws IOException {
+ log.info("Deleting the snapshot named {} for core {}", name,
solrCore.getName());
+ SnapshotMetaData result =
nameToDetailsMapping.remove(Preconditions.checkNotNull(name));
+ if(result != null) {
+ boolean success = false;
+ try {
+ persist();
+ success = true;
+ } finally {
+ if (!success) {
+ nameToDetailsMapping.put(name, result);
+ }
+ }
+ }
+ return Optional.ofNullable(result);
+ }
+
+ /**
+ * This method returns if snapshot is created for the specified
generation number in
+ * the *current* index directory.
+ *
+ * @param genNumber The generation number for the {@linkplain
IndexCommit} to be checked.
+ * @return true if the snapshot is created.
+ * false otherwise.
+ */
+ public synchronized boolean isSnapshotted(long genNumber) {
+ return isSnapshotted(solrCore.getIndexDir(), genNumber);
+ }
+
+ /**
+ * This method returns if snapshot is created for the specified
generation number in
+ * the specified index directory.
+ *
+ * @param genNumber The generation number for the {@linkplain
IndexCommit} to be checked.
+ * @return true if the snapshot is created.
+ * false otherwise.
+ */
+ public synchronized boolean isSnapshotted(String indexDirPath, long
genNumber) {
+ return nameToDetailsMapping.values().stream()
+ .filter(entry -> entry.getIndexDirPath().equals(indexDirPath) &&
entry.getGenerationNumber() == genNumber)
+ .findFirst()
+ .isPresent();
+ }
+
+ /**
+ * This method returns the snapshot meta-data for the specified name (if
it exists).
+ *
+ * @param name The name of the snapshot
+ * @return The snapshot meta-data if exists.
+ */
+ public synchronized Optional<SnapshotMetaData>
getSnapshotMetaData(String name) {
+ return Optional.ofNullable(nameToDetailsMapping.get(name));
+ }
+
+ /**
+ * @return A list of snapshots created so far.
+ */
+ public synchronized Collection<String> listSnapshots() {
+ return new HashSet<>(nameToDetailsMapping.keySet());
+ }
+
+ /**
+ * This method returns a list of snapshots created in a specified index
directory.
+ *
+ * @param indexDirPath The index directory path.
+ * @return a list snapshots stored in the specified directory.
+ */
+ public synchronized Collection<SnapshotMetaData>
listSnapshotsInIndexDir(String indexDirPath) {
+ return nameToDetailsMapping.entrySet().stream()
+ .filter(entry ->
indexDirPath.equals(entry.getValue().getIndexDirPath()))
+ .map(entry -> entry.getValue())
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * This method returns the {@linkplain IndexCommit} associated with the
specified
+ * <code>commitName</code>. A snapshot with specified
<code>commitName</code> must
+ * be created before invoking this method.
+ *
+ * @param commitName The name of persisted commit
+ * @return the {@linkplain IndexCommit}
+ * @throws IOException in case of I/O error.
+ */
+ public Optional<IndexCommit> getIndexCommitByName(String commitName)
throws IOException {
+ Optional<IndexCommit> result = Optional.empty();
+ Optional<SnapshotMetaData> metaData = getSnapshotMetaData(commitName);
+ if (metaData.isPresent()) {
+ String indexDirPath = metaData.get().getIndexDirPath();
+ long gen = metaData.get().getGenerationNumber();
+
+ Directory d = solrCore.getDirectoryFactory().get(indexDirPath,
DirContext.DEFAULT, DirectoryFactory.LOCK_TYPE_NONE);
+ try {
+ result = DirectoryReader.listCommits(d)
+ .stream()
+ .filter(ic -> ic.getGeneration() == gen)
+ .findAny();
+
+ if (!result.isPresent()) {
+ log.warn("Unable to find commit with generation {} in the
directory {}", gen, indexDirPath);
+ }
+
+ } finally {
+ solrCore.getDirectoryFactory().release(d);
+ }
+ } else {
+ log.warn("Commit with name {} is not persisted for core {}",
commitName, solrCore.getName());
+ }
+
+ return result;
+ }
+
+ synchronized private void persist() throws IOException {
--- End diff --
Done.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
