Michael Blow has submitted this change and it was merged. Change subject: [NO ISSUE] Metadata partition bootstrap, replication ......................................................................
[NO ISSUE] Metadata partition bootstrap, replication - Send metadata partition id as part of metadata bootstrap - Ignore requests to add replicas to myself - Avoid hard checkpoint at shutdown Change-Id: I08c91de570bbc90f5532329f72db21a424993fed Reviewed-on: https://asterix-gerrit.ics.uci.edu/2369 Reviewed-by: Murtadha Hubail <mhub...@apache.org> Integration-Tests: Murtadha Hubail <mhub...@apache.org> Tested-by: Murtadha Hubail <mhub...@apache.org> Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu> --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MetadataBootstrapTask.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeRequestMessage.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java 10 files changed, 53 insertions(+), 23 deletions(-) Approvals: Anon. E. Moose #1000171: Jenkins: Murtadha Hubail: Looks good to me, approved; Verified; Verified diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java index 1de6938..d42db39 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java @@ -416,10 +416,10 @@ } @Override - public void initializeMetadata(boolean newUniverse) throws Exception { + public void initializeMetadata(boolean newUniverse, int partitionId) throws Exception { LOGGER.info("Bootstrapping metadata"); MetadataNode.INSTANCE.initialize(this, ncExtensionManager.getMetadataTupleTranslatorProvider(), - ncExtensionManager.getMetadataExtensions()); + ncExtensionManager.getMetadataExtensions(), partitionId); //noinspection unchecked ConcurrentHashMap<CcId, IAsterixStateProxy> proxyMap = diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java index 3717673..de3c691 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java @@ -668,8 +668,6 @@ @Override public void stop(boolean dumpState, OutputStream os) throws IOException { - // Shutdown checkpoint - checkpointManager.doSharpCheckpoint(); } @Override diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java index 8c733fb..30416a3 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java @@ -18,6 +18,7 @@ */ package org.apache.asterix.app.nc; +import java.net.InetSocketAddress; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -35,10 +36,15 @@ import org.apache.asterix.common.transactions.IRecoveryManager; import org.apache.asterix.replication.api.PartitionReplica; import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository; +import org.apache.hyracks.api.config.IApplicationConfig; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.control.common.controllers.NCConfig; import org.apache.hyracks.storage.common.LocalResource; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; public class ReplicaManager implements IReplicaManager { + private static final Logger LOGGER = LogManager.getLogger(); private final INcApplicationContext appCtx; /** @@ -60,6 +66,10 @@ if (!partitions.contains(id.getPartition())) { throw new IllegalStateException( "This node is not the current master of partition(" + id.getPartition() + ")"); + } + if (isSelf(id)) { + LOGGER.info("ignoring request to add replica to ourselves"); + return; } replicas.computeIfAbsent(id, k -> new PartitionReplica(k, appCtx)); replicas.get(id).sync(); @@ -88,6 +98,9 @@ @Override public synchronized void promote(int partition) throws HyracksDataException { + if (partitions.contains(partition)) { + return; + } final PersistentLocalResourceRepository localResourceRepository = (PersistentLocalResourceRepository) appCtx.getLocalResourceRepository(); localResourceRepository.cleanup(partition); @@ -120,4 +133,13 @@ datasetLifecycleManager.close(resource.getPath()); } } + + private boolean isSelf(ReplicaIdentifier id) { + IApplicationConfig appConfig = appCtx.getServiceContext().getAppConfig(); + String host = appConfig.getString(NCConfig.Option.REPLICATION_LISTEN_ADDRESS); + int port = appConfig.getInt(NCConfig.Option.REPLICATION_LISTEN_PORT); + + final InetSocketAddress replicaAddress = new InetSocketAddress(host, port); + return id.equals(ReplicaIdentifier.of(id.getPartition(), replicaAddress)); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MetadataBootstrapTask.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MetadataBootstrapTask.java index 001af23..c18b967 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MetadataBootstrapTask.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MetadataBootstrapTask.java @@ -28,13 +28,19 @@ public class MetadataBootstrapTask implements INCLifecycleTask { private static final long serialVersionUID = 1L; + private final int partitionId; + + public MetadataBootstrapTask(int partitionId) { + this.partitionId = partitionId; + } @Override public void perform(CcId ccId, IControllerService cs) throws HyracksDataException { INcApplicationContext appContext = (INcApplicationContext) cs.getApplicationContext(); try { + appContext.getReplicaManager().promote(partitionId); SystemState state = appContext.getTransactionSubsystem().getRecoveryManager().getSystemState(); - appContext.initializeMetadata(state == SystemState.PERMANENT_DATA_LOSS); + appContext.initializeMetadata(state == SystemState.PERMANENT_DATA_LOSS, partitionId); } catch (Exception e) { throw HyracksDataException.create(e); } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java index 8939059..25e768d 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java @@ -155,7 +155,7 @@ tasks.add(new StartReplicationServiceTask()); } if (isMetadataNode) { - tasks.add(new MetadataBootstrapTask()); + tasks.add(new MetadataBootstrapTask(clusterManager.getMetadataPartition().getPartitionId())); } tasks.add(new ExternalLibrarySetupTask(isMetadataNode)); tasks.add(new CheckpointTask()); @@ -184,7 +184,8 @@ } // if current metadata node is active, we need to unbind its metadata proxy objects if (clusterManager.isMetadataNodeActive()) { - MetadataNodeRequestMessage msg = new MetadataNodeRequestMessage(false); + MetadataNodeRequestMessage msg = + new MetadataNodeRequestMessage(false, clusterManager.getMetadataPartition().getPartitionId()); try { messageBroker.sendApplicationMessageToNC(msg, metadataNodeId); // when the current node responses, we will bind to the new one @@ -207,7 +208,8 @@ } private void requestMetadataNodeTakeover(String node) throws HyracksDataException { - MetadataNodeRequestMessage msg = new MetadataNodeRequestMessage(true); + MetadataNodeRequestMessage msg = + new MetadataNodeRequestMessage(true, clusterManager.getMetadataPartition().getPartitionId()); try { messageBroker.sendApplicationMessageToNC(msg, node); } catch (Exception e) { diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeRequestMessage.java index 4443825..817bbe6 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeRequestMessage.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeRequestMessage.java @@ -34,9 +34,11 @@ private static final long serialVersionUID = 1L; private static final Logger LOGGER = LogManager.getLogger(); private final boolean export; + private final int partitionId; - public MetadataNodeRequestMessage(boolean export) { + public MetadataNodeRequestMessage(boolean export, int partitionId) { this.export = export; + this.partitionId = partitionId; } @Override @@ -45,7 +47,7 @@ HyracksDataException hde = null; try { if (export) { - appContext.initializeMetadata(false); + appContext.initializeMetadata(false, partitionId); appContext.exportMetadataNodeStub(); appContext.bindMetadataNodeStub(getCcId()); } else { diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java index a02bda5..82936b3 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java @@ -94,9 +94,10 @@ * Initializes the metadata node and bootstraps the metadata. * * @param newUniverse + * @param partitionId * @throws Exception */ - void initializeMetadata(boolean newUniverse) throws Exception; + void initializeMetadata(boolean newUniverse, int partitionId) throws Exception; /** * Unexports the metadata node from the RMI registry diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java index 948bdad..0b18f98 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java @@ -105,11 +105,6 @@ return accessor.getString(Option.METADATA_NODE); } - public ClusterPartition getMetadataPartition() { - // metadata partition is always the first partition on the metadata node - return accessor.getNodePartitions().get(getMetadataNodeName())[0]; - } - public Map<String, String[]> getStores() { return accessor.getStores(); } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java index 72d5cf5..fd21941 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java @@ -145,11 +145,12 @@ } public void initialize(INcApplicationContext runtimeContext, - MetadataTupleTranslatorProvider tupleTranslatorProvider, List<IMetadataExtension> metadataExtensions) { + MetadataTupleTranslatorProvider tupleTranslatorProvider, List<IMetadataExtension> metadataExtensions, + int partitionId) { this.tupleTranslatorProvider = tupleTranslatorProvider; this.transactionSubsystem = runtimeContext.getTransactionSubsystem(); this.datasetLifecycleManager = runtimeContext.getDatasetLifecycleManager(); - this.metadataStoragePartition = runtimeContext.getMetadataProperties().getMetadataPartition().getPartitionId(); + this.metadataStoragePartition = partitionId; if (metadataExtensions != null) { extensionDatasets = new HashMap<>(); for (IMetadataExtension metadataExtension : metadataExtensions) { @@ -161,6 +162,10 @@ this.txnIdFactory = new BulkTxnIdFactory(); } + public int getMetadataStoragePartition() { + return metadataStoragePartition; + } + @Override public void ensureMinimumTxnId(long maxId) throws ACIDException, RemoteException { txnIdFactory.ensureMinimumId(maxId); diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java index 9ebd21b..23806bd 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java @@ -25,7 +25,6 @@ import java.util.List; import org.apache.asterix.common.api.INcApplicationContext; -import org.apache.asterix.common.cluster.ClusterPartition; import org.apache.asterix.common.config.DatasetConfig.DatasetType; import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.common.config.MetadataProperties; @@ -44,6 +43,7 @@ import org.apache.asterix.external.indexing.ExternalFile; import org.apache.asterix.metadata.IDatasetDetails; import org.apache.asterix.metadata.MetadataManager; +import org.apache.asterix.metadata.MetadataNode; import org.apache.asterix.metadata.MetadataTransactionContext; import org.apache.asterix.metadata.api.IMetadataIndex; import org.apache.asterix.metadata.entities.BuiltinTypeMap; @@ -311,11 +311,10 @@ if (!appContext.getDatasetMemoryManager().reserve(index.getDatasetId().getId())) { throw new IllegalStateException("Failed to reserve memory for metadata dataset (" + datasetId + ")"); } - ClusterPartition metadataPartition = appContext.getMetadataProperties().getMetadataPartition(); - int metadataDeviceId = metadataPartition.getIODeviceNum(); - String metadataPartitionPath = StoragePathUtil.prepareStoragePartitionPath(metadataPartition.getPartitionId()); + String metadataPartitionPath = + StoragePathUtil.prepareStoragePartitionPath(MetadataNode.INSTANCE.getMetadataStoragePartition()); String resourceName = metadataPartitionPath + File.separator + index.getFileNameRelativePath(); - FileReference file = ioManager.getFileReference(metadataDeviceId, resourceName); + FileReference file = ioManager.resolve(resourceName); index.setFile(file); ITypeTraits[] typeTraits = index.getTypeTraits(); IBinaryComparatorFactory[] cmpFactories = index.getKeyBinaryComparatorFactory(); -- To view, visit https://asterix-gerrit.ics.uci.edu/2369 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I08c91de570bbc90f5532329f72db21a424993fed Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Michael Blow <mb...@apache.org> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Gerrit-Reviewer: Michael Blow <mb...@apache.org> Gerrit-Reviewer: Murtadha Hubail <mhub...@apache.org>
