Repository: asterixdb Updated Branches: refs/heads/master 2f934e312 -> dd4cae8e5
[NO ISSUE][REPL] Add API to Release Partition - user model changes: no - storage format changes: no - interface changes: yes Details: - Add API to allow an NC to release a partition by flushing all of its resources and unregistering all of its replicas. This API can be used to allow graceful failover of a cluster partition. - Add test case. Change-Id: I7c8bbfc123904f13aae9f640dbefc7d17df69a42 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2349 Sonar-Qube: Jenkins <[email protected]> Reviewed-by: Michael Blow <[email protected]> Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/dd4cae8e Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/dd4cae8e Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/dd4cae8e Branch: refs/heads/master Commit: dd4cae8e59ce0d2594be062d4088eff2cbe75016 Parents: 2f934e3 Author: Murtadha Hubail <[email protected]> Authored: Sat Feb 3 06:11:27 2018 +0300 Committer: Murtadha Hubail <[email protected]> Committed: Sat Feb 3 13:54:37 2018 -0800 ---------------------------------------------------------------------- .../api/http/server/StorageApiServlet.java | 13 ++++++++++ .../apache/asterix/app/nc/ReplicaManager.java | 27 ++++++++++++++++++++ .../release_partition.1.sto.cmd | 19 ++++++++++++++ .../release_partition.2.pollget.http | 21 +++++++++++++++ .../release_partition.3.post.http | 19 ++++++++++++++ .../release_partition.4.get.http | 19 ++++++++++++++ .../test/resources/runtimets/replication.xml | 5 ++++ .../release_partition/release_partition.2.adm | 7 +++++ .../release_partition/release_partition.3.adm | 0 .../release_partition/release_partition.4.adm | 1 + .../asterix/common/storage/IReplicaManager.java | 9 +++++++ 11 files changed, 140 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dd4cae8e/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java index e770bb3..142b99b 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java @@ -96,6 +96,9 @@ public class StorageApiServlet extends AbstractServlet { case "/promote": processPromote(request, response); break; + case "/release": + processRelease(request, response); + break; default: sendError(response, HttpResponseStatus.NOT_FOUND); break; @@ -175,4 +178,14 @@ public class StorageApiServlet extends AbstractServlet { appCtx.getReplicaManager().promote(Integer.valueOf(partition)); response.setStatus(HttpResponseStatus.OK); } + + private void processRelease(IServletRequest request, IServletResponse response) throws HyracksDataException { + final String partition = request.getParameter("partition"); + if (partition == null) { + response.setStatus(HttpResponseStatus.BAD_REQUEST); + return; + } + appCtx.getReplicaManager().release(Integer.valueOf(partition)); + response.setStatus(HttpResponseStatus.OK); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dd4cae8e/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java index 4edae69..8c733fb 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java @@ -27,6 +27,7 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.asterix.common.api.IDatasetLifecycleManager; import org.apache.asterix.common.api.INcApplicationContext; import org.apache.asterix.common.replication.IPartitionReplica; import org.apache.asterix.common.storage.IReplicaManager; @@ -35,6 +36,7 @@ import org.apache.asterix.common.transactions.IRecoveryManager; import org.apache.asterix.replication.api.PartitionReplica; import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.storage.common.LocalResource; public class ReplicaManager implements IReplicaManager { @@ -93,4 +95,29 @@ public class ReplicaManager implements IReplicaManager { recoveryManager.replayReplicaPartitionLogs(Stream.of(partition).collect(Collectors.toSet()), true); partitions.add(partition); } + + @Override + public void release(int partition) throws HyracksDataException { + if (!partitions.contains(partition)) { + return; + } + final IDatasetLifecycleManager datasetLifecycleManager = appCtx.getDatasetLifecycleManager(); + datasetLifecycleManager.flushDataset(appCtx.getReplicationManager().getReplicationStrategy()); + closePartitionResources(partition); + final List<IPartitionReplica> partitionReplicas = getReplicas(partition); + for (IPartitionReplica replica : partitionReplicas) { + appCtx.getReplicationManager().unregister(replica); + } + partitions.remove(partition); + } + + private void closePartitionResources(int partition) throws HyracksDataException { + final PersistentLocalResourceRepository resourceRepository = + (PersistentLocalResourceRepository) appCtx.getLocalResourceRepository(); + final Map<Long, LocalResource> partitionResources = resourceRepository.getPartitionResources(partition); + final IDatasetLifecycleManager datasetLifecycleManager = appCtx.getDatasetLifecycleManager(); + for (LocalResource resource : partitionResources.values()) { + datasetLifecycleManager.close(resource.getPath()); + } + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dd4cae8e/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.1.sto.cmd ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.1.sto.cmd b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.1.sto.cmd new file mode 100644 index 0000000..7ddaa20 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.1.sto.cmd @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +nc:asterix_nc1 /addReplica 0 asterix_nc2 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dd4cae8e/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.2.pollget.http ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.2.pollget.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.2.pollget.http new file mode 100644 index 0000000..6867a5d --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.2.pollget.http @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +//polltimeoutsecs=30 + +nc:asterix_nc1 /admin/storage/partition/0 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dd4cae8e/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.3.post.http ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.3.post.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.3.post.http new file mode 100644 index 0000000..bb942d8 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.3.post.http @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +nc:asterix_nc1 /admin/storage/release?partition=0 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dd4cae8e/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.4.get.http ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.4.get.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.4.get.http new file mode 100644 index 0000000..d5cf952 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.4.get.http @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +nc:asterix_nc1 /admin/storage/partition/0 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dd4cae8e/asterixdb/asterix-app/src/test/resources/runtimets/replication.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/replication.xml b/asterixdb/asterix-app/src/test/resources/runtimets/replication.xml index 79c9f5c..469446d 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/replication.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/replication.xml @@ -44,5 +44,10 @@ <output-dir compare="Text">flushed_component</output-dir> </compilation-unit> </test-case> + <test-case FilePath="replication"> + <compilation-unit name="release_partition"> + <output-dir compare="Text">release_partition</output-dir> + </compilation-unit> + </test-case> </test-group> </test-suite> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dd4cae8e/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.2.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.2.adm new file mode 100644 index 0000000..e48cbcf --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.2.adm @@ -0,0 +1,7 @@ +[ { + "partition" : 0, + "replicas" : [ { + "location" : "127.0.0.1:2002", + "status" : "IN_SYNC" + } ] +} ] http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dd4cae8e/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.3.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.3.adm new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dd4cae8e/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.4.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.4.adm new file mode 100644 index 0000000..8878e54 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.4.adm @@ -0,0 +1 @@ +[ ] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dd4cae8e/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java index 19eee8f..b2deb1e 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java @@ -62,4 +62,13 @@ public interface IReplicaManager { * @param partition */ void promote(int partition) throws HyracksDataException; + + /** + * Releases a partition by flushing all its resources to disk + * then removing all partition replicas. + * + * @param partition + * @throws HyracksDataException + */ + void release(int partition) throws HyracksDataException; }
