Michael Blow has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/2389

Change subject: [NO ISSUE] Add support for NodeGroup upsert, etc.
......................................................................

[NO ISSUE] Add support for NodeGroup upsert, etc.

Also, enable resource id generation when only pending removal nodes are absent
from the cluster

Change-Id: I15cfb74bc345680102cedafa99f7ff4f144860bc
---
M 
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
M 
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataTxnTest.java
M 
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java
M 
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
M 
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
M 
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
M 
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
M 
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
M 
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
M 
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
M 
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
M 
asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/message/ResourceIdRequestMessage.java
M 
asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/transaction/ResourceIdManager.java
13 files changed, 37 insertions(+), 31 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb 
refs/changes/89/2389/1

diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 7bf65c7..ecf0ecc 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -2539,7 +2539,7 @@
                 for (Identifier id : ncIdentifiers) {
                     ncNames.add(id.getValue());
                 }
-                MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, new 
NodeGroup(ngName, ncNames));
+                MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, new 
NodeGroup(ngName, ncNames), false);
             }
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
         } catch (Exception e) {
diff --git 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataTxnTest.java
 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataTxnTest.java
index 70e5f6e..5d48299 100644
--- 
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataTxnTest.java
+++ 
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataTxnTest.java
@@ -76,7 +76,7 @@
         final String nodeGroupName = "ng";
         try {
             final List<String> ngNodes = Arrays.asList("asterix_nc1");
-            MetadataManager.INSTANCE.addNodegroup(mdTxn, new 
NodeGroup(nodeGroupName, ngNodes));
+            MetadataManager.INSTANCE.addNodegroup(mdTxn, new 
NodeGroup(nodeGroupName, ngNodes), false);
             MetadataManager.INSTANCE.abortTransaction(mdTxn);
         } finally {
             metadataProvider.getLocks().unlock();
@@ -224,7 +224,7 @@
         Thread transactor = new Thread(() -> {
             final List<String> ngNodes = Arrays.asList("asterix_nc1");
             try {
-                MetadataManager.INSTANCE.addNodegroup(mdTxn, new 
NodeGroup(nodeGroupName, ngNodes));
+                MetadataManager.INSTANCE.addNodegroup(mdTxn, new 
NodeGroup(nodeGroupName, ngNodes), false);
                 Thread.currentThread().interrupt();
                 MetadataManager.INSTANCE.commitTransaction(mdTxn);
             } catch (Exception e) {
diff --git 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java
 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java
index a12b079..b994c50 100644
--- 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java
+++ 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java
@@ -21,7 +21,6 @@
 
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
@@ -191,12 +190,9 @@
         }
     }
 
-    public NodeGroup addNodeGroupIfNotExists(NodeGroup nodeGroup) {
+    public NodeGroup addOrUpdateNodeGroup(NodeGroup nodeGroup) {
         synchronized (nodeGroups) {
-            if (!nodeGroups.containsKey(nodeGroup.getNodeGroupName())) {
-                return nodeGroups.put(nodeGroup.getNodeGroupName(), nodeGroup);
-            }
-            return null;
+            return nodeGroups.put(nodeGroup.getNodeGroupName(), nodeGroup);
         }
     }
 
diff --git 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
index b2757f2..3eb9a98 100644
--- 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
+++ 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
@@ -479,9 +479,10 @@
     }
 
     @Override
-    public void addNodegroup(MetadataTransactionContext ctx, NodeGroup 
nodeGroup) throws AlgebricksException {
+    public void addNodegroup(MetadataTransactionContext ctx, NodeGroup 
nodeGroup, boolean upsert)
+            throws AlgebricksException {
         try {
-            metadataNode.addNodeGroup(ctx.getTxnId(), nodeGroup);
+            metadataNode.addNodeGroup(ctx.getTxnId(), nodeGroup, upsert);
         } catch (RemoteException e) {
             throw new 
MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e);
         }
diff --git 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index 966c99a..c0d9441 100644
--- 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -404,11 +404,16 @@
     }
 
     @Override
-    public void addNodeGroup(TxnId txnId, NodeGroup nodeGroup) throws 
AlgebricksException, RemoteException {
+    public void addNodeGroup(TxnId txnId, NodeGroup nodeGroup, boolean force)
+            throws AlgebricksException, RemoteException {
         try {
             NodeGroupTupleTranslator tupleReaderWriter = 
tupleTranslatorProvider.getNodeGroupTupleTranslator(true);
             ITupleReference tuple = 
tupleReaderWriter.getTupleFromMetadataEntity(nodeGroup);
-            insertTupleIntoIndex(txnId, 
MetadataPrimaryIndexes.NODEGROUP_DATASET, tuple);
+            if (force) {
+                upsertTupleIntoIndex(txnId, 
MetadataPrimaryIndexes.NODEGROUP_DATASET, tuple);
+            } else {
+                insertTupleIntoIndex(txnId, 
MetadataPrimaryIndexes.NODEGROUP_DATASET, tuple);
+            }
         } catch (HyracksDataException e) {
             if (e.getComponent().equals(ErrorCode.HYRACKS) && e.getErrorCode() 
== ErrorCode.DUPLICATE_KEY) {
                 throw new AlgebricksException(
diff --git 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
index cb67ee5..961a1ee 100644
--- 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
+++ 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
@@ -152,7 +152,7 @@
 
     public void dropNodeGroup(String nodeGroupName) {
         NodeGroup nodeGroup = new NodeGroup(nodeGroupName, null);
-        droppedCache.addNodeGroupIfNotExists(nodeGroup);
+        droppedCache.addOrUpdateNodeGroup(nodeGroup);
         logAndApply(new MetadataLogicalOperation(nodeGroup, false));
     }
 
diff --git 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
index e030db3..b1f9a94 100644
--- 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
+++ 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
@@ -320,16 +320,18 @@
             throws AlgebricksException;
 
     /**
-     * Inserts a node group.
+     * Inserts or upserts a node group.
      *
      * @param ctx
      *            MetadataTransactionContext of an active metadata transaction.
      * @param nodeGroup
      *            Node group instance to insert.
+     * @param upsert
+     *            Whether to upsert rather than insert the node group
      * @throws AlgebricksException
-     *             For example, if the node group already exists.
+     *             For example, if the node group already exists, and not an 
upsert
      */
-    void addNodegroup(MetadataTransactionContext ctx, NodeGroup nodeGroup) 
throws AlgebricksException;
+    void addNodegroup(MetadataTransactionContext ctx, NodeGroup nodeGroup, 
boolean upsert) throws AlgebricksException;
 
     /**
      * Retrieves a node group.
diff --git 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
index f6abc53..09c5cf3 100644
--- 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
+++ 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
@@ -340,11 +340,12 @@
      *            A globally unique id for an active metadata transaction.
      * @param nodeGroup
      *            Node group instance to insert.
+     * @param force
      * @throws AlgebricksException
      *             For example, if the node group already exists.
      * @throws RemoteException
      */
-    void addNodeGroup(TxnId txnId, NodeGroup nodeGroup) throws 
AlgebricksException, RemoteException;
+    void addNodeGroup(TxnId txnId, NodeGroup nodeGroup, boolean force) throws 
AlgebricksException, RemoteException;
 
     /**
      * Retrieves a node group, acquiring local locks on behalf of the given
diff --git 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
index 7d5558d..ab02b5d 100644
--- 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -249,7 +249,7 @@
         List<String> metadataGroupNodeNames = new ArrayList<>();
         metadataGroupNodeNames.add(metadataNodeName);
         NodeGroup groupRecord = new 
NodeGroup(MetadataConstants.METADATA_NODEGROUP_NAME, metadataGroupNodeNames);
-        MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, groupRecord);
+        MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, groupRecord, false);
     }
 
     private static void insertInitialAdapters(MetadataTransactionContext 
mdTxnCtx) throws AlgebricksException {
diff --git 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
index e5088aa..8a7d5ec 100644
--- 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
+++ 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
@@ -50,7 +50,7 @@
 
     @Override
     public NodeGroup addToCache(MetadataCache cache) {
-        return cache.addNodeGroupIfNotExists(this);
+        return cache.addOrUpdateNodeGroup(this);
     }
 
     @Override
diff --git 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 6d81145..80de46d 100644
--- 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -544,7 +544,7 @@
             nodeGroup = nodeGroup + "_" + UUID.randomUUID().toString();
             
appCtx.getMetadataLockManager().acquireNodeGroupWriteLock(metadataProvider.getLocks(),
 nodeGroup);
         }
-        MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, new 
NodeGroup(nodeGroup, new ArrayList<>(ncNames)));
+        MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, new 
NodeGroup(nodeGroup, new ArrayList<>(ncNames)), false);
         return nodeGroup;
     }
 
diff --git 
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/message/ResourceIdRequestMessage.java
 
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/message/ResourceIdRequestMessage.java
index 3e172c7..087913f 100644
--- 
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/message/ResourceIdRequestMessage.java
+++ 
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/message/ResourceIdRequestMessage.java
@@ -41,16 +41,16 @@
             ICCMessageBroker broker = (ICCMessageBroker) 
appCtx.getServiceContext().getMessageBroker();
             ResourceIdRequestResponseMessage response = new 
ResourceIdRequestResponseMessage();
             IClusterStateManager clusterStateManager = 
appCtx.getClusterStateManager();
-            if (!clusterStateManager.isClusterActive()) {
-                response.setResourceId(-1);
-                response.setException(new Exception("Cannot generate global 
resource id when cluster is not active."));
-            } else {
-                IResourceIdManager resourceIdManager = 
appCtx.getResourceIdManager();
-                response.setResourceId(resourceIdManager.createResourceId());
-                if (response.getResourceId() < 0) {
+            IResourceIdManager resourceIdManager = 
appCtx.getResourceIdManager();
+            response.setResourceId(resourceIdManager.createResourceId());
+            if (response.getResourceId() < 0) {
+                if (!(clusterStateManager.isClusterActive())) {
+                    response.setException(
+                            new Exception("Cannot generate global resource id 
when cluster is not active."));
+                } else {
                     response.setException(new Exception("One or more nodes has 
not reported max resource id."));
+                    requestMaxResourceID(clusterStateManager, 
resourceIdManager, broker);
                 }
-                requestMaxResourceID(clusterStateManager, resourceIdManager, 
broker);
             }
             broker.sendApplicationMessageToNC(response, src);
         } catch (Exception e) {
@@ -60,7 +60,7 @@
 
     private void requestMaxResourceID(IClusterStateManager 
clusterStateManager, IResourceIdManager resourceIdManager,
             ICCMessageBroker broker) throws Exception {
-        Set<String> getParticipantNodes = 
clusterStateManager.getParticipantNodes();
+        Set<String> getParticipantNodes = 
clusterStateManager.getParticipantNodes(true);
         ReportLocalCountersRequestMessage msg = new 
ReportLocalCountersRequestMessage();
         for (String nodeId : getParticipantNodes) {
             if (!resourceIdManager.reported(nodeId)) {
diff --git 
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/transaction/ResourceIdManager.java
 
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/transaction/ResourceIdManager.java
index 0bb862d..6d3077e 100644
--- 
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/transaction/ResourceIdManager.java
+++ 
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/transaction/ResourceIdManager.java
@@ -38,7 +38,8 @@
 
     @Override
     public long createResourceId() {
-        return csm.isClusterActive() ? globalResourceId.incrementAndGet() : -1;
+        return csm.isClusterActive() || 
reportedNodes.containsAll(csm.getParticipantNodes(true))
+                ? globalResourceId.incrementAndGet() : -1;
     }
 
     @Override

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/2389
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I15cfb74bc345680102cedafa99f7ff4f144860bc
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>

Reply via email to