abdullah alamoudi has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1174
Change subject: Add Default Dataverse and AnyObject Datatype
......................................................................
Add Default Dataverse and AnyObject Datatype
Change-Id: Ie02c74c8b005dce73888afd4dfe19ede2c60a8e8
---
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
M
asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta01/meta01.1.adm
M
asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm
M
asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_dataverse/metadata_dataverse.1.adm
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
A
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBuiltinEntities.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java
7 files changed, 196 insertions(+), 107 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/74/1174/1
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
index c11b875..a8637cc 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
@@ -37,16 +37,15 @@
import org.apache.asterix.common.config.MessagingProperties;
import org.apache.asterix.common.replication.IRemoteRecoveryManager;
import org.apache.asterix.common.transactions.IRecoveryManager;
-import
org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.event.schema.cluster.Cluster;
import org.apache.asterix.event.schema.cluster.Node;
import org.apache.asterix.messaging.MessagingChannelInterfaceFactory;
import org.apache.asterix.messaging.NCMessageBroker;
-import org.apache.asterix.metadata.bootstrap.MetadataBootstrap;
import org.apache.asterix.runtime.message.ReportMaxResourceIdMessage;
import org.apache.asterix.runtime.util.AsterixClusterProperties;
+import
org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
import
org.apache.asterix.transaction.management.service.recovery.RecoveryManager;
import org.apache.commons.io.FileUtils;
import org.apache.hyracks.api.application.INCApplicationContext;
@@ -190,11 +189,6 @@
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Stopping Asterix node controller: " + nodeId);
}
-
- if (isMetadataNode) {
- MetadataBootstrap.stopUniverse();
- }
-
//Clean any temporary files
performLocalCleanUp();
diff --git
a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta01/meta01.1.adm
b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta01/meta01.1.adm
index e341727..313cefe 100644
---
a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta01/meta01.1.adm
+++
b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta01/meta01.1.adm
@@ -1,2 +1,3 @@
+{ "DataverseName": "Default", "DataFormat":
"org.apache.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Wed Sep
14 23:26:12 EEST 2016", "PendingOp": 0 }
{ "DataverseName": "Metadata", "DataFormat":
"org.apache.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Tue Jun
21 15:54:20 PDT 2016", "PendingOp": 0 }
{ "DataverseName": "testdv", "DataFormat":
"org.apache.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Tue Jun
21 15:54:27 PDT 2016", "PendingOp": 0 }
diff --git
a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm
b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm
index 1920bab..91639a8 100644
---
a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm
+++
b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm
@@ -1,3 +1,4 @@
+{ "DataverseName": "Metadata", "DatatypeName": "AnyObject", "Derived": {
"Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [
] } }, "Timestamp": "Wed Sep 14 23:26:12 EEST 2016" }
{ "DataverseName": "Metadata", "DatatypeName": "CompactionPolicyRecordType",
"Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true,
"Fields": [ { "FieldName": "DataverseName", "FieldType": "string",
"IsNullable": false }, { "FieldName": "CompactionPolicy", "FieldType":
"string", "IsNullable": false }, { "FieldName": "Classname", "FieldType":
"string", "IsNullable": false } ] } }, "Timestamp": "Wed Jul 27 00:27:47 AST
2016" }
{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType", "Derived":
{ "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields":
[ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false },
{ "FieldName": "DatasetName", "FieldType": "string", "IsNullable": false }, {
"FieldName": "DatatypeDataverseName", "FieldType": "string", "IsNullable":
false }, { "FieldName": "DatatypeName", "FieldType": "string", "IsNullable":
false }, { "FieldName": "DatasetType", "FieldType": "string", "IsNullable":
false }, { "FieldName": "GroupName", "FieldType": "string", "IsNullable": false
}, { "FieldName": "CompactionPolicy", "FieldType": "string", "IsNullable":
false }, { "FieldName": "CompactionPolicyProperties", "FieldType":
"DatasetRecordType_CompactionPolicyProperties", "IsNullable": false }, {
"FieldName": "InternalDetails", "FieldType":
"DatasetRecordType_InternalDetails", "IsNullable": true }, { "FieldName":
"ExternalDetails", "FieldType":
"DatasetRecordType_ExternalDetails", "IsNullable": true }, { "FieldName":
"Hints", "FieldType": "DatasetRecordType_Hints", "IsNullable": false }, {
"FieldName": "Timestamp", "FieldType": "string", "IsNullable": false }, {
"FieldName": "DatasetId", "FieldType": "int32", "IsNullable": false }, {
"FieldName": "PendingOp", "FieldType": "int32", "IsNullable": false } ] } },
"Timestamp": "Wed Jul 27 00:27:47 AST 2016" }
{ "DataverseName": "Metadata", "DatatypeName":
"DatasetRecordType_CompactionPolicyProperties", "Derived": { "Tag":
"ORDEREDLIST", "IsAnonymous": true, "OrderedList":
"DatasetRecordType_CompactionPolicyProperties_Item" }, "Timestamp": "Wed Jul 27
00:27:47 AST 2016" }
diff --git
a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_dataverse/metadata_dataverse.1.adm
b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_dataverse/metadata_dataverse.1.adm
index b301d59..519c800 100644
---
a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_dataverse/metadata_dataverse.1.adm
+++
b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_dataverse/metadata_dataverse.1.adm
@@ -1,3 +1,4 @@
+{ "DataverseName": "Default", "DataFormat":
"org.apache.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Wed Sep
14 23:26:12 EEST 2016", "PendingOp": 0 }
{ "DataverseName": "Metadata", "DataFormat":
"org.apache.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Tue Jun
21 15:54:20 PDT 2016", "PendingOp": 0 }
{ "DataverseName": "test", "DataFormat":
"org.apache.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Tue Jun
21 15:54:27 PDT 2016", "PendingOp": 0 }
{ "DataverseName": "testdv", "DataFormat":
"org.apache.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Tue Jun
21 15:54:28 PDT 2016", "PendingOp": 0 }
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
index 55b1045..cf215e0 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -19,6 +19,7 @@
package org.apache.asterix.metadata.bootstrap;
import java.io.File;
+import java.rmi.RemoteException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -37,6 +38,7 @@
import org.apache.asterix.common.config.IAsterixPropertiesProvider;
import org.apache.asterix.common.context.BaseOperationTracker;
import org.apache.asterix.common.context.CorrelatedPrefixMergePolicyFactory;
+import org.apache.asterix.common.exceptions.ACIDException;
import
org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.external.adapter.factory.GenericAdapterFactory;
@@ -104,7 +106,7 @@
* stopUniverse() should be called upon application undeployment.
*/
public class MetadataBootstrap {
- public static final boolean IS_DEBUG_MODE = false;// true
+ public static final boolean IS_DEBUG_MODE = false;
private static final Logger LOGGER =
Logger.getLogger(MetadataBootstrap.class.getName());
private static IAsterixAppRuntimeContext runtimeContext;
private static IBufferCache bufferCache;
@@ -116,7 +118,6 @@
private static List<String> nodeNames;
private static String outputDir;
private static boolean isNewUniverse;
-
private static final IMetadataIndex[] PRIMARY_INDEXES =
new IMetadataIndex[] { MetadataPrimaryIndexes.DATAVERSE_DATASET,
MetadataPrimaryIndexes.DATASET_DATASET,
MetadataPrimaryIndexes.DATATYPE_DATASET,
MetadataPrimaryIndexes.INDEX_DATASET,
@@ -128,8 +129,23 @@
private static IAsterixPropertiesProvider propertiesProvider;
+ private MetadataBootstrap() {
+ }
+
+ /**
+ * bootstrap metadata
+ *
+ * @param asterixPropertiesProvider
+ * @param ncApplicationContext
+ * @param isNewUniverse
+ * @throws ACIDException
+ * @throws RemoteException
+ * @throws MetadataException
+ * @throws Exception
+ */
public static void startUniverse(IAsterixPropertiesProvider
asterixPropertiesProvider,
- INCApplicationContext ncApplicationContext, boolean isNewUniverse)
throws Exception {
+ INCApplicationContext ncApplicationContext, boolean isNewUniverse)
+ throws RemoteException, ACIDException, MetadataException {
MetadataBootstrap.setNewUniverse(isNewUniverse);
runtimeContext = (IAsterixAppRuntimeContext)
ncApplicationContext.getApplicationObject();
propertiesProvider = asterixPropertiesProvider;
@@ -175,33 +191,35 @@
} catch (Exception e) {
try {
if (IS_DEBUG_MODE) {
- e.printStackTrace();
+ LOGGER.log(Level.SEVERE, "Failure during metadata
bootstrap", e);
}
MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
} catch (Exception e2) {
e.addSuppressed(e2);
- // TODO
- // change the exception type to AbortFailureException
+ // TODO change the exception type to AbortFailureException
throw new MetadataException(e);
}
- throw e;
+ throw new MetadataException(e);
}
}
- public static void stopUniverse() {
- // Close all BTree files in BufferCache.
- // metadata datasets will be closed when the dataset life cycle manger
is closed
- }
-
- private static void insertInitialDataverses(MetadataTransactionContext
mdTxnCtx) throws Exception {
- String dataverseName =
MetadataPrimaryIndexes.DATAVERSE_DATASET.getDataverseName();
+ private static void insertInitialDataverses(MetadataTransactionContext
mdTxnCtx) throws MetadataException {
String dataFormat = NonTaggedDataFormat.NON_TAGGED_DATA_FORMAT;
MetadataManager.INSTANCE.addDataverse(mdTxnCtx,
- new Dataverse(dataverseName, dataFormat,
IMetadataEntity.PENDING_NO_OP));
+ new Dataverse(MetadataConstants.METADATA_DATAVERSE_NAME,
dataFormat, IMetadataEntity.PENDING_NO_OP));
+ MetadataManager.INSTANCE.addDataverse(mdTxnCtx,
MetadataBuiltinEntities.DEFAULT_DATAVERSE);
}
+ /**
+ * Inserts a metadata dataset to the physical dataset index
+ * Should be performed on a bootstrap of a new universe
+ *
+ * @param mdTxnCtx
+ * @param indexes
+ * @throws MetadataException
+ */
public static void insertMetadataDatasets(MetadataTransactionContext
mdTxnCtx, IMetadataIndex[] indexes)
- throws Exception {
+ throws MetadataException {
for (int i = 0; i < indexes.length; i++) {
IDatasetDetails id = new
InternalDatasetDetails(FileStructure.BTREE, PartitioningStrategy.HASH,
indexes[i].getPartitioningExpr(),
indexes[i].getPartitioningExpr(), null,
@@ -219,7 +237,7 @@
}
}
- public static void getBuiltinTypes(ArrayList<IAType> types) throws
Exception {
+ private static void getBuiltinTypes(List<IAType> types) {
Collection<BuiltinType> builtinTypes =
AsterixBuiltinTypeMap.getBuiltinTypes().values();
Iterator<BuiltinType> iter = builtinTypes.iterator();
while (iter.hasNext()) {
@@ -227,47 +245,47 @@
}
}
- public static void getMetadataTypes(ArrayList<IAType> types) throws
Exception {
+ private static void getMetadataTypes(ArrayList<IAType> types) {
for (int i = 0; i < PRIMARY_INDEXES.length; i++) {
types.add(PRIMARY_INDEXES[i].getPayloadRecordType());
}
}
- public static void insertMetadataDatatypes(MetadataTransactionContext
mdTxnCtx) throws Exception {
- String dataverseName =
MetadataPrimaryIndexes.DATAVERSE_DATASET.getDataverseName();
- ArrayList<IAType> types = new ArrayList<IAType>();
+ private static void insertMetadataDatatypes(MetadataTransactionContext
mdTxnCtx) throws MetadataException {
+ ArrayList<IAType> types = new ArrayList<>();
getBuiltinTypes(types);
getMetadataTypes(types);
for (int i = 0; i < types.size(); i++) {
MetadataManager.INSTANCE.addDatatype(mdTxnCtx,
- new Datatype(dataverseName, types.get(i).getTypeName(),
types.get(i), false));
+ new Datatype(MetadataConstants.METADATA_DATAVERSE_NAME,
types.get(i).getTypeName(), types.get(i),
+ false));
}
+ MetadataManager.INSTANCE.addDatatype(mdTxnCtx,
+ MetadataBuiltinEntities.ANY_OBJECT_DATATYPE);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Finished inserting initial datatypes.");
}
}
- public static void insertNodes(MetadataTransactionContext mdTxnCtx) throws
Exception {
+ private static void insertNodes(MetadataTransactionContext mdTxnCtx)
throws MetadataException {
for (String nodesName : nodeNames) {
MetadataManager.INSTANCE.addNode(mdTxnCtx, new Node(nodesName, 0,
0));
}
}
- public static void insertInitialGroups(MetadataTransactionContext
mdTxnCtx) throws Exception {
- String groupName =
MetadataPrimaryIndexes.DATAVERSE_DATASET.getNodeGroupName();
- List<String> metadataGroupNodeNames = new ArrayList<String>();
+ private static void insertInitialGroups(MetadataTransactionContext
mdTxnCtx) throws MetadataException {
+ List<String> metadataGroupNodeNames = new ArrayList<>();
metadataGroupNodeNames.add(metadataNodeName);
- NodeGroup groupRecord = new NodeGroup(groupName,
metadataGroupNodeNames);
+ NodeGroup groupRecord = new
NodeGroup(MetadataConstants.METADATA_NODEGROUP_NAME, metadataGroupNodeNames);
MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, groupRecord);
-
- List<String> nodes = new ArrayList<String>();
+ List<String> nodes = new ArrayList<>();
nodes.addAll(nodeNames);
NodeGroup defaultGroup = new
NodeGroup(MetadataConstants.METADATA_DEFAULT_NODEGROUP_NAME, nodes);
MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, defaultGroup);
-
}
- private static void insertInitialAdapters(MetadataTransactionContext
mdTxnCtx) throws Exception {
+ private static void insertInitialAdapters(MetadataTransactionContext
mdTxnCtx)
+ throws MetadataException {
String[] builtInAdapterClassNames = new String[] {
GenericAdapterFactory.class.getName() };
DatasourceAdapter adapter;
for (String adapterClassName : builtInAdapterClassNames) {
@@ -279,7 +297,7 @@
}
}
- private static void insertInitialFeedPolicies(MetadataTransactionContext
mdTxnCtx) throws Exception {
+ private static void insertInitialFeedPolicies(MetadataTransactionContext
mdTxnCtx) throws MetadataException {
for (FeedPolicyEntity feedPolicy : BuiltinFeedPolicies.policies) {
MetadataManager.INSTANCE.addFeedPolicy(mdTxnCtx, feedPolicy);
}
@@ -288,29 +306,46 @@
}
}
- private static void
insertInitialCompactionPolicies(MetadataTransactionContext mdTxnCtx) throws
Exception {
+ private static void
insertInitialCompactionPolicies(MetadataTransactionContext mdTxnCtx) throws
MetadataException {
String[] builtInCompactionPolicyClassNames =
new String[] { ConstantMergePolicyFactory.class.getName(),
PrefixMergePolicyFactory.class.getName(),
NoMergePolicyFactory.class.getName(),
CorrelatedPrefixMergePolicyFactory.class.getName() };
- CompactionPolicy compactionPolicy;
for (String policyClassName : builtInCompactionPolicyClassNames) {
- compactionPolicy = getCompactionPolicyEntity(policyClassName);
+ CompactionPolicy compactionPolicy =
getCompactionPolicyEntity(policyClassName);
MetadataManager.INSTANCE.addCompactionPolicy(mdTxnCtx,
compactionPolicy);
}
}
- private static DatasourceAdapter getAdapter(String
adapterFactoryClassName) throws Exception {
- String adapterName = ((IAdapterFactory)
(Class.forName(adapterFactoryClassName).newInstance())).getAlias();
- return new DatasourceAdapter(new
AdapterIdentifier(MetadataConstants.METADATA_DATAVERSE_NAME, adapterName),
- adapterFactoryClassName,
IDataSourceAdapter.AdapterType.INTERNAL);
+ private static DatasourceAdapter getAdapter(String adapterFactoryClassName)
+ throws MetadataException {
+ try {
+ String adapterName = ((IAdapterFactory)
(Class.forName(adapterFactoryClassName).newInstance())).getAlias();
+ return new DatasourceAdapter(new
AdapterIdentifier(MetadataConstants.METADATA_DATAVERSE_NAME, adapterName),
+ adapterFactoryClassName,
IDataSourceAdapter.AdapterType.INTERNAL);
+ } catch (InstantiationException | IllegalAccessException |
ClassNotFoundException e) {
+ throw new MetadataException("Unable to instantiate builtin
Adapter", e);
+ }
}
- private static CompactionPolicy getCompactionPolicyEntity(String
compactionPolicyClassName) throws Exception {
- String policyName =
- ((ILSMMergePolicyFactory)
(Class.forName(compactionPolicyClassName).newInstance())).getName();
- return new CompactionPolicy(MetadataConstants.METADATA_DATAVERSE_NAME,
policyName, compactionPolicyClassName);
+ private static CompactionPolicy getCompactionPolicyEntity(String
compactionPolicyClassName)
+ throws MetadataException {
+ try {
+ String policyName =
+ ((ILSMMergePolicyFactory)
(Class.forName(compactionPolicyClassName).newInstance())).getName();
+ return new
CompactionPolicy(MetadataConstants.METADATA_DATAVERSE_NAME, policyName,
+ compactionPolicyClassName);
+ } catch (InstantiationException | IllegalAccessException |
ClassNotFoundException e) {
+ throw new MetadataException("Unable to instantiate builtin Merge
Policy Factory", e);
+ }
}
+ /**
+ * Enlist a metadata index so it is available for metadata operations
+ * should be performed upon bootstrapping
+ *
+ * @param index
+ * @throws HyracksDataException
+ */
public static void enlistMetadataDataset(IMetadataIndex index) throws
HyracksDataException {
ClusterPartition metadataPartition =
propertiesProvider.getMetadataProperties().getMetadataPartition();
int metadataDeviceId = metadataPartition.getIODeviceNum();
@@ -387,12 +422,12 @@
return metadataNodeName;
}
+ /**
+ * Perform recovery of DDL operations metadata records
+ */
public static void startDDLRecovery() throws MetadataException {
// #. clean up any record which has pendingAdd/DelOp flag
// as traversing all records from DATAVERSE_DATASET to
DATASET_DATASET, and then to INDEX_DATASET.
- String dataverseName = null;
- String datasetName = null;
- String indexName = null;
MetadataTransactionContext mdTxnCtx = null;
MetadataManager.INSTANCE.acquireWriteLatch();
if (LOGGER.isLoggable(Level.INFO)) {
@@ -401,59 +436,9 @@
try {
mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
-
List<Dataverse> dataverses =
MetadataManager.INSTANCE.getDataverses(mdTxnCtx);
for (Dataverse dataverse : dataverses) {
- dataverseName = dataverse.getDataverseName();
- if (dataverse.getPendingOp() != IMetadataEntity.PENDING_NO_OP)
{
- // drop pending dataverse
- MetadataManager.INSTANCE.dropDataverse(mdTxnCtx,
dataverseName);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Dropped a pending dataverse: " +
dataverseName);
- }
- } else {
- List<Dataset> datasets =
MetadataManager.INSTANCE.getDataverseDatasets(mdTxnCtx, dataverseName);
- for (Dataset dataset : datasets) {
- datasetName = dataset.getDatasetName();
- if (dataset.getPendingOp() !=
IMetadataEntity.PENDING_NO_OP) {
- // drop pending dataset
- MetadataManager.INSTANCE.dropDataset(mdTxnCtx,
dataverseName, datasetName);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Dropped a pending dataset: " +
dataverseName + "." + datasetName);
- }
- } else {
- List<Index> indexes =
-
MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName,
datasetName);
- for (Index index : indexes) {
- indexName = index.getIndexName();
- if (index.getPendingOp() !=
IMetadataEntity.PENDING_NO_OP) {
- // drop pending index
-
MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName,
indexName);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Dropped a pending index:
" + dataverseName + "." + datasetName
- + "." + indexName);
- }
- }
- }
- }
- if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
- // if the dataset has no indexes, delete all its
files
- List<Index> indexes =
-
MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName,
datasetName);
- if (indexes.size() == 0) {
- List<ExternalFile> files =
-
MetadataManager.INSTANCE.getDatasetExternalFiles(mdTxnCtx, dataset);
- for (ExternalFile file : files) {
-
MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Dropped an external file:
" + dataverseName + "." + datasetName
- + "." + file.getFileNumber());
- }
- }
- }
- }
- }
- }
+ recoverDataverse(mdTxnCtx, dataverse);
}
// the commit wasn't there before. yet, everything was working
correctly!!!!!!!!!!!
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -463,7 +448,7 @@
} catch (Exception e) {
try {
if (IS_DEBUG_MODE) {
- e.printStackTrace();
+ LOGGER.log(Level.SEVERE, "Failure during DDL recovery", e);
}
MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
} catch (Exception e2) {
@@ -475,6 +460,68 @@
}
}
+ private static void recoverDataverse(MetadataTransactionContext mdTxnCtx,
Dataverse dataverse)
+ throws MetadataException {
+ if (dataverse.getPendingOp() != IMetadataEntity.PENDING_NO_OP) {
+ // drop pending dataverse
+ MetadataManager.INSTANCE.dropDataverse(mdTxnCtx,
dataverse.getDataverseName());
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Dropped a pending dataverse: " +
dataverse.getDataverseName());
+ }
+ } else {
+ List<Dataset> datasets =
+ MetadataManager.INSTANCE.getDataverseDatasets(mdTxnCtx,
dataverse.getDataverseName());
+ for (Dataset dataset : datasets) {
+ recoverDataset(mdTxnCtx, dataset);
+ }
+ }
+ }
+
+ private static void recoverDataset(MetadataTransactionContext mdTxnCtx,
Dataset dataset) throws MetadataException {
+ if (dataset.getPendingOp() != IMetadataEntity.PENDING_NO_OP) {
+ // drop pending dataset
+ MetadataManager.INSTANCE.dropDataset(mdTxnCtx,
dataset.getDataverseName(), dataset.getDatasetName());
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(
+ "Dropped a pending dataset: " +
dataset.getDataverseName() + "." + dataset.getDatasetName());
+ }
+ } else {
+ List<Index> indexes =
+ MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx,
dataset.getDataverseName(),
+ dataset.getDatasetName());
+ for (Index index : indexes) {
+ if (index.getPendingOp() != IMetadataEntity.PENDING_NO_OP) {
+ // drop pending index
+ MetadataManager.INSTANCE.dropIndex(mdTxnCtx,
dataset.getDataverseName(), dataset.getDatasetName(),
+ index.getIndexName());
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Dropped a pending index: " +
dataset.getDataverseName() + "."
+ + dataset.getDatasetName()
+ + "." + index.getIndexName());
+ }
+ }
+ }
+ }
+ if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
+ // if the dataset has no indexes, delete all its files
+ List<Index> indexes =
+ MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx,
dataset.getDataverseName(),
+ dataset.getDatasetName());
+ if (indexes.isEmpty()) {
+ List<ExternalFile> files =
+
MetadataManager.INSTANCE.getDatasetExternalFiles(mdTxnCtx, dataset);
+ for (ExternalFile file : files) {
+ MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Dropped an external file: " +
dataset.getDataverseName() + "."
+ + dataset.getDatasetName()
+ + "." + file.getFileNumber());
+ }
+ }
+ }
+ }
+ }
+
public static boolean isNewUniverse() {
return isNewUniverse;
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBuiltinEntities.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBuiltinEntities.java
new file mode 100644
index 0000000..eae94c1
--- /dev/null
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBuiltinEntities.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.bootstrap;
+
+import org.apache.asterix.metadata.api.IMetadataEntity;
+import org.apache.asterix.metadata.entities.Datatype;
+import org.apache.asterix.metadata.entities.Dataverse;
+import org.apache.asterix.metadata.utils.MetadataConstants;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
+
+public class MetadataBuiltinEntities {
+ //--------------------------------------- Dataverses
----------------------------------------//
+ public static final String DEFAULT_DATAVERSE_NAME = "Default";
+ public static final Dataverse DEFAULT_DATAVERSE =
+ new Dataverse(DEFAULT_DATAVERSE_NAME,
NonTaggedDataFormat.class.getName(),
+ IMetadataEntity.PENDING_NO_OP);
+ //--------------------------------------- Datatypes
-----------------------------------------//
+ public static final ARecordType ANY_OBJECT_RECORD_TYPE =
+ new ARecordType("AnyObject", new String[0], new IAType[0], true);
+ public static final Datatype ANY_OBJECT_DATATYPE =
+ new Datatype(MetadataConstants.METADATA_DATAVERSE_NAME,
ANY_OBJECT_RECORD_TYPE.getTypeName(),
+ ARecordType.FULLY_OPEN_RECORD_TYPE, false);
+}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java
index fbd2380..1dc6657 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java
@@ -19,9 +19,10 @@
package org.apache.asterix.metadata.utils;
+/**
+ * Contains metadata constants
+ */
public class MetadataConstants {
- private MetadataConstants() {
- }
// Name of the dataverse the metadata lives in.
public static final String METADATA_DATAVERSE_NAME = "Metadata";
@@ -32,4 +33,7 @@
// Name of the default nodegroup where internal/feed datasets will be
partitioned
// if an explicit nodegroup is not specified at the time of creation of a
dataset
public static final String METADATA_DEFAULT_NODEGROUP_NAME =
"DEFAULT_NG_ALL_NODES";
+
+ private MetadataConstants() {
+ }
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/1174
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie02c74c8b005dce73888afd4dfe19ede2c60a8e8
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <bamou...@gmail.com>
