Hello,
At Community Over Code in Minneapolis, Rahul Goswami gave a presentation
on upgrading a Solr/Lucene index in-place with a clever blocking index
merge with a background process that would, in tandem, cause all index
segments on-disk to be migrated from the previous Lucene version (e.g.
7.x) to the current version (e.g. 8.x).
There seemed to be consensus that this alone would be a great addition
to Solr even if the next step was not (yet?) possible. This is
represented by https://issues.apache.org/jira/browse/SOLR-17725
The final step required in order to push an on-disk index from Lucene
X-1 to Lucene X is to re-write the indexCreatedVersionMajor field in the
segment info structure.
A short conversation during the session indicated that a command-line
tool might be helpful for intrepid users who were willing to force a
change to the index created version major.
I've written (with some help from ChatGPT) a short CLI utility to check
the segments and segment info and optionally force a change to the major
version. It should only work if the requested version is *higher* than
the current version AND if all the segments are already compatible. You
also have to use a command-line argument to enable WRITE mode.
So, you can run it on an index and get info, and verify beforehand that
it SHOULD work before asking it to actually write to the segment info
and "forge" the new indexCreatedVersionMajor field.
The code is below. Please let me know if you all think this would be
something worth exploring, expanding, and possibly maintaining.
-chris
import org.apache.lucene.index.*;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import java.io.IOException;
import java.lang.reflect.Field;
import java.nio.file.Path;
import java.nio.file.Paths;
public class LuceneIndexUpgrader {
public static void main(String[] args) {
if (args.length < 1) {
System.err.println("Usage: java LuceneIndexUpgrader
<path-to-index> [newMajorVersion] [--write]");
System.exit(1);
}
Path indexPath = null;
Integer newMajorVersion = null;
boolean shouldWrite = false;
// Parse args
for (String arg : args) {
if (arg.equals("--write")) {
shouldWrite = true;
} else if (arg.matches("\\d+")) {
newMajorVersion = Integer.parseInt(arg);
} else if (indexPath == null) {
indexPath = Paths.get(arg);
} else {
System.err.println("Unrecognized argument: " + arg);
System.exit(1);
}
}
if (indexPath == null) {
System.err.println("Error: Index path is required.");
System.exit(1);
}
if (newMajorVersion != null && (newMajorVersion < 1 ||
newMajorVersion > 99)) {
System.err.println("Invalid Lucene major version: " +
newMajorVersion);
System.exit(1);
}
try (FSDirectory directory = FSDirectory.open(indexPath)) {
SegmentInfos segmentInfos =
SegmentInfos.readLatestCommit(directory);
int currentMajorVersion =
segmentInfos.getIndexCreatedVersionMajor();
// Print index-level version info
System.out.println("Lucene Index Metadata:");
System.out.println("--------------------------------------------------");
System.out.println("Index Created Version Major: " +
currentMajorVersion);
System.out.println("Min Segment Lucene Version: " +
segmentInfos.getMinSegmentLuceneVersion());
System.out.println("\nSegment Details:");
boolean compatible = true;
for (SegmentCommitInfo sci : segmentInfos) {
SegmentInfo si = sci.info;
Version version = si.getVersion();
Version minVersion = si.getMinVersion();
System.out.println("--------------------------------------------------");
System.out.println("Segment name: " + si.name);
System.out.println(" Lucene version: " + version);
System.out.println(" Min Lucene ver: " + minVersion);
if (newMajorVersion != null) {
int actualVersion = minVersion != null ?
minVersion.major :
(version != null ? version.major : -1);
if (actualVersion == -1 || actualVersion <
newMajorVersion) {
System.err.printf(" Incompatible: segment
minVersion %d < requested indexCreatedVersionMajor %d%n",
actualVersion, newMajorVersion);
compatible = false;
}
}
}
if (newMajorVersion != null) {
System.out.printf("\nRequested new
indexCreatedVersionMajor: %d%n", newMajorVersion);
// Check that new version is not less than current
if (newMajorVersion < currentMajorVersion) {
System.err.printf("Aborting: new major version %d
is less than current index version %d%n",
newMajorVersion, currentMajorVersion);
System.exit(1);
}
if (!compatible) {
System.err.println("Aborting: Not all segments are
compatible with the requested version.");
System.exit(1);
} else if (shouldWrite) {
// Proceed with update
Field field =
SegmentInfos.class.getDeclaredField("indexCreatedVersionMajor");
field.setAccessible(true);
Field modifiersField =
Field.class.getDeclaredField("modifiers");
modifiersField.setAccessible(true);
modifiersField.setInt(field, field.getModifiers() &
~java.lang.reflect.Modifier.FINAL);
field.setInt(segmentInfos, newMajorVersion);
segmentInfos.commit(directory);
System.out.printf("Updated indexCreatedVersionMajor
to %d and saved to disk.%n", newMajorVersion);
} else {
System.out.println("Dry run:
indexCreatedVersionMajor would be updated, but --write was not set.");
}
} else {
System.out.println("\n(No version change requested.)");
}
} catch (IOException | NoSuchFieldException |
IllegalAccessException e) {
System.err.println("Error while processing Lucene index: "
+ e.getMessage());
e.printStackTrace();
}
}
}
