This is an automated email from the ASF dual-hosted git repository. gerlowskija pushed a commit to branch branch_9x in repository https://gitbox.apache.org/repos/asf/solr.git
commit bfe93b509f1aa0e09b0f1073f83059b86f697b45 Author: Matthew Biscocho <[email protected]> AuthorDate: Wed May 17 14:22:45 2023 -0400 SOLR-16470: Add v2 API for /replication?command=indexversion (#1620) No v2 equivalent existed prior to this commit. The new v2 API is `GET /api/cores/cName/replication/indexversion`. --------- Co-authored-by: Jason Gerlowski <[email protected]> --- solr/CHANGES.txt | 3 + .../apache/solr/handler/ReplicationHandler.java | 75 +++++++++++++-------- .../solr/handler/admin/api/CoreReplicationAPI.java | 75 +++++++++++++++++++++ .../solr/handler/admin/api/ReplicationAPIBase.java | 47 +++++++++++++ .../handler/admin/api/CoreReplicationAPITest.java | 76 ++++++++++++++++++++++ .../pages/user-managed-index-replication.adoc | 17 +++++ 6 files changed, 266 insertions(+), 27 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 4c385d3271d..e739184bf34 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -104,6 +104,9 @@ Improvements recency now uses `DELETE /api/backups/bName/versions?retainLatest=3`, and index-file "garbage collection" now uses `PUT /api/backups/backupName/purgeUnused` (Jason Gerlowski) +* SOLR-16470: `/coreName/replication?commit=indexversion` now has a v2 equivalent, available at + `GET /api/cores/coreName/replication/indexversion` (Matthew Biscocho via Jason Gerlowski) + Optimizations --------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java index ae0cd65b6d4..3e358fd8fb7 100644 --- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java @@ -68,6 +68,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.RateLimiter; +import org.apache.solr.api.JerseyResource; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.params.CommonParams; @@ -91,6 +92,9 @@ import org.apache.solr.core.SolrEventListener; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.core.backup.repository.LocalFileSystemRepository; import org.apache.solr.handler.IndexFetcher.IndexFetchResult; +import org.apache.solr.handler.admin.api.CoreReplicationAPI; +import org.apache.solr.handler.api.V2ApiUtils; +import org.apache.solr.jersey.SolrJerseyResponse; import org.apache.solr.metrics.MetricsMap; import org.apache.solr.metrics.SolrMetricsContext; import org.apache.solr.request.SolrQueryRequest; @@ -263,33 +267,8 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw // This command does not give the current index version of the leader // It gives the current 'replicateable' index version if (command.equals(CMD_INDEX_VERSION)) { - IndexCommit commitPoint = indexCommitPoint; // make a copy so it won't change - - if (commitPoint == null) { - // if this handler is 'lazy', we may not have tracked the last commit - // because our commit listener is registered on inform - commitPoint = core.getDeletionPolicy().getLatestCommit(); - } - - if (commitPoint != null && replicationEnabled.get()) { - // - // There is a race condition here. The commit point may be changed / deleted by the time - // we get around to reserving it. This is a very small window though, and should not result - // in a catastrophic failure, but will result in the client getting an empty file list for - // the CMD_GET_FILE_LIST command. - // - core.getDeletionPolicy() - .setReserveDuration(commitPoint.getGeneration(), reserveCommitDuration); - rsp.add(CMD_INDEX_VERSION, IndexDeletionPolicyWrapper.getCommitTimestamp(commitPoint)); - rsp.add(GENERATION, commitPoint.getGeneration()); - rsp.add(STATUS, OK_STATUS); - } else { - // This happens when replication is not configured to happen after startup and no - // commit/optimize has happened yet. - rsp.add(CMD_INDEX_VERSION, 0L); - rsp.add(GENERATION, 0L); - rsp.add(STATUS, OK_STATUS); - } + final SolrJerseyResponse indexVersionResponse = getIndexVersionResponse(); + V2ApiUtils.squashIntoSolrResponseWithoutHeader(rsp, indexVersionResponse); } else if (command.equals(CMD_GET_FILE)) { getFileStream(solrParams, rsp); } else if (command.equals(CMD_GET_FILE_LIST)) { @@ -798,6 +777,38 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw } } + public CoreReplicationAPI.IndexVersionResponse getIndexVersionResponse() throws IOException { + + IndexCommit commitPoint = indexCommitPoint; // make a copy so it won't change + CoreReplicationAPI.IndexVersionResponse rsp = new CoreReplicationAPI.IndexVersionResponse(); + if (commitPoint == null) { + // if this handler is 'lazy', we may not have tracked the last commit + // because our commit listener is registered on inform + commitPoint = core.getDeletionPolicy().getLatestCommit(); + } + + if (commitPoint != null && replicationEnabled.get()) { + // + // There is a race condition here. The commit point may be changed / deleted by the time + // we get around to reserving it. This is a very small window though, and should not result + // in a catastrophic failure, but will result in the client getting an empty file list for + // the CMD_GET_FILE_LIST command. + // + core.getDeletionPolicy() + .setReserveDuration(commitPoint.getGeneration(), reserveCommitDuration); + rsp.indexVersion = IndexDeletionPolicyWrapper.getCommitTimestamp(commitPoint); + rsp.generation = commitPoint.getGeneration(); + } else { + // This happens when replication is not configured to happen after startup and no + // commit/optimize has happened yet. + rsp.indexVersion = 0L; + rsp.generation = 0L; + } + rsp.status = OK_STATUS; + + return rsp; + } + /** * Retrieves the maximum version number from an index commit. NOTE: The commit <b>MUST</b> be * reserved before calling this method @@ -1464,6 +1475,16 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw log.info("Commits will be reserved for {} ms", reserveCommitDuration); } + @Override + public Collection<Class<? extends JerseyResource>> getJerseyResources() { + return List.of(CoreReplicationAPI.class); + } + + @Override + public Boolean registerV2() { + return Boolean.TRUE; + } + // check leader or follower is enabled private boolean isEnabled(NamedList<?> params) { if (params == null) return false; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CoreReplicationAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CoreReplicationAPI.java new file mode 100644 index 00000000000..29ead1bd519 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CoreReplicationAPI.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2; +import static org.apache.solr.security.PermissionNameProvider.Name.CORE_READ_PERM; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.IOException; +import javax.inject.Inject; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import org.apache.solr.core.SolrCore; +import org.apache.solr.jersey.PermissionName; +import org.apache.solr.jersey.SolrJerseyResponse; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; + +/** + * V2 APIs for inspecting and replicating indices + * + * <p>These APIs are analogous to the v1 /coreName/replication APIs. + */ +@Path("/cores/{coreName}/replication") +public class CoreReplicationAPI extends ReplicationAPIBase { + + @Inject + public CoreReplicationAPI(SolrCore solrCore, SolrQueryRequest req, SolrQueryResponse rsp) { + super(solrCore, req, rsp); + } + + @GET + @Path("/indexversion") + @Produces({"application/json", "application/xml", BINARY_CONTENT_TYPE_V2}) + @PermissionName(CORE_READ_PERM) + public IndexVersionResponse fetchIndexVersion() throws IOException { + return doFetchIndexVersion(); + } + + /** Response for {@link CoreReplicationAPI#fetchIndexVersion()}. */ + public static class IndexVersionResponse extends SolrJerseyResponse { + + @JsonProperty("indexversion") + public Long indexVersion; + + @JsonProperty("generation") + public Long generation; + + @JsonProperty("status") + public String status; + + public IndexVersionResponse() {} + + public IndexVersionResponse(Long indexVersion, Long generation, String status) { + this.indexVersion = indexVersion; + this.generation = generation; + this.status = status; + } + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ReplicationAPIBase.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ReplicationAPIBase.java new file mode 100644 index 00000000000..e53261350ff --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ReplicationAPIBase.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import java.io.IOException; +import org.apache.solr.api.JerseyResource; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.ReplicationHandler; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; + +/** A common parent for "replication" (i.e. replication-level) APIs. */ +public abstract class ReplicationAPIBase extends JerseyResource { + + protected final SolrCore solrCore; + protected final SolrQueryRequest solrQueryRequest; + protected final SolrQueryResponse solrQueryResponse; + + public ReplicationAPIBase( + SolrCore solrCore, SolrQueryRequest solrQueryRequest, SolrQueryResponse solrQueryResponse) { + this.solrCore = solrCore; + this.solrQueryRequest = solrQueryRequest; + this.solrQueryResponse = solrQueryResponse; + } + + protected CoreReplicationAPI.IndexVersionResponse doFetchIndexVersion() throws IOException { + + ReplicationHandler replicationHandler = + (ReplicationHandler) solrCore.getRequestHandler(ReplicationHandler.PATH); + + return replicationHandler.getIndexVersionResponse(); + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CoreReplicationAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CoreReplicationAPITest.java new file mode 100644 index 00000000000..98961280fa1 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CoreReplicationAPITest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.admin.api; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.opentracing.noop.NoopSpan; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.ReplicationHandler; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Unit tests for {@link CoreReplicationAPI} */ +public class CoreReplicationAPITest extends SolrTestCaseJ4 { + + private CoreReplicationAPI coreReplicationAPI; + private SolrCore mockCore; + private ReplicationHandler mockReplicationHandler; + private static final String coreName = "test"; + private SolrQueryRequest mockQueryRequest; + private SolrQueryResponse queryResponse; + + @BeforeClass + public static void ensureWorkingMockito() { + assumeWorkingMockito(); + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + setUpMocks(); + mockQueryRequest = mock(SolrQueryRequest.class); + when(mockQueryRequest.getSpan()).thenReturn(NoopSpan.INSTANCE); + queryResponse = new SolrQueryResponse(); + coreReplicationAPI = new CoreReplicationAPI(mockCore, mockQueryRequest, queryResponse); + } + + @Test + public void testGetIndexVersion() throws Exception { + CoreReplicationAPI.IndexVersionResponse expected = + new CoreReplicationAPI.IndexVersionResponse(123L, 123L, "testGeneration"); + when(mockReplicationHandler.getIndexVersionResponse()).thenReturn(expected); + + CoreReplicationAPI.IndexVersionResponse response = coreReplicationAPI.doFetchIndexVersion(); + assertEquals(expected.indexVersion, response.indexVersion); + assertEquals(expected.generation, response.generation); + assertEquals(expected.status, response.status); + } + + private void setUpMocks() { + mockCore = mock(SolrCore.class); + mockReplicationHandler = mock(ReplicationHandler.class); + when(mockCore.getRequestHandler(ReplicationHandler.PATH)).thenReturn(mockReplicationHandler); + } +} diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/user-managed-index-replication.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/user-managed-index-replication.adoc index b09be99ca8c..4def27755a2 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/user-managed-index-replication.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/user-managed-index-replication.adoc @@ -378,9 +378,26 @@ http://_leader_host:port_/solr/_core_name_/replication?command=disablereplicatio `indexversion`:: Return the version of the latest replicatable index on the specified leader or follower. + +==== +[.tab-label]*V1 API* + [source,bash] +---- http://_host:port_/solr/_core_name_/replication?command=indexversion +---- +==== ++ +==== +[.tab-label]*V2 API* + +[source,bash] +---- +http://_host:port_/api/cores/_core_name_/replication/indexversion + +---- +==== + `fetchindex`:: Force the specified follower to fetch a copy of the index from its leader. +
