This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 24e35b21e0 [ASTERIXDB-3259][MTD] Add namespace path resolver
24e35b21e0 is described below
commit 24e35b21e0180430f6412381b672dac10eabfd68
Author: Ali Alsuliman <[email protected]>
AuthorDate: Thu Oct 5 10:12:42 2023 -0700
[ASTERIXDB-3259][MTD] Add namespace path resolver
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
Add namespace path resolver which given a database name
and a dataverse name will resolve the storage path to
be used depending on whether database is enabled or not.
- Pass the namespace path resolver to the data partitioning
provider.
Change-Id: I285be236fd634a8e036447dbecfd0e3039613907
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17846
Integration-Tests: Jenkins <[email protected]>
Tested-by: Ali Alsuliman <[email protected]>
Reviewed-by: Ali Alsuliman <[email protected]>
Reviewed-by: Murtadha Hubail <[email protected]>
---
.../asterix/app/cc/CcApplicationContext.java | 20 ++++++-
.../apache/asterix/app/nc/NCAppRuntimeContext.java | 18 +++++++
.../org/apache/asterix/utils/DataverseUtil.java | 27 +++++-----
.../asterix/app/bootstrap/TestNodeController.java | 9 +++-
.../org/apache/asterix/common/TestDataUtil.java | 5 +-
.../storage/IndexDropOperatorNodePushableTest.java | 5 +-
.../asterix/common/api/IApplicationContext.java | 4 ++
.../asterix/common/api/INamespacePathResolver.java | 30 +++++++++++
.../asterix/common/api/INamespaceResolver.java | 28 ++++++++++
.../apache/asterix/common/metadata/Namespace.java | 61 ++++++++++++++++++++++
.../common/metadata/NamespacePathResolver.java | 42 +++++++++++++++
.../asterix/common/metadata/NamespaceResolver.java | 39 ++++++++++++++
.../asterix/common/utils/StoragePathUtil.java | 23 ++++----
.../adapter/factory/GenericAdapterFactory.java | 9 +++-
.../external/library/ExternalLibraryManager.java | 7 ++-
.../apache/asterix/external/util/FeedUtils.java | 9 ++--
.../asterix/metadata/api/IMetadataIndex.java | 3 +-
.../metadata/bootstrap/MetadataBootstrap.java | 6 ++-
.../asterix/metadata/bootstrap/MetadataIndex.java | 7 +--
.../metadata/declared/MetadataProvider.java | 12 ++++-
.../metadata/utils/DataPartitioningProvider.java | 12 +++--
.../utils/DynamicDataPartitioningProvider.java | 10 ++--
.../metadata/utils/SplitsAndConstraintsUtil.java | 36 ++++++-------
.../utils/StaticDataPartitioningProvider.java | 13 ++---
24 files changed, 357 insertions(+), 78 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
index bb7be73e31..a79eb0d9dd 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
@@ -29,6 +29,8 @@ import org.apache.asterix.common.api.IConfigValidator;
import org.apache.asterix.common.api.IConfigValidatorFactory;
import org.apache.asterix.common.api.ICoordinationService;
import org.apache.asterix.common.api.IMetadataLockManager;
+import org.apache.asterix.common.api.INamespacePathResolver;
+import org.apache.asterix.common.api.INamespaceResolver;
import org.apache.asterix.common.api.INodeJobTracker;
import org.apache.asterix.common.api.IReceptionist;
import org.apache.asterix.common.api.IReceptionistFactory;
@@ -55,6 +57,8 @@ import
org.apache.asterix.common.dataflow.IDataPartitioningProvider;
import org.apache.asterix.common.external.IAdapterFactoryService;
import org.apache.asterix.common.metadata.IMetadataBootstrap;
import org.apache.asterix.common.metadata.IMetadataLockUtil;
+import org.apache.asterix.common.metadata.NamespacePathResolver;
+import org.apache.asterix.common.metadata.NamespaceResolver;
import org.apache.asterix.common.replication.INcLifecycleCoordinator;
import org.apache.asterix.common.storage.ICompressionManager;
import org.apache.asterix.common.transactions.IResourceIdManager;
@@ -123,6 +127,8 @@ public class CcApplicationContext implements
ICcApplicationContext {
private final IDataPartitioningProvider dataPartitioningProvider;
private final IGlobalTxManager globalTxManager;
private final IOManager ioManager;
+ private final NamespacePathResolver namespacePathResolver;
+ private final NamespaceResolver namespaceResolver;
public CcApplicationContext(ICCServiceContext ccServiceCtx,
HyracksConnection hcc,
Supplier<IMetadataBootstrap> metadataBootstrapSupplier,
IGlobalRecoveryManager globalRecoveryManager,
@@ -167,9 +173,11 @@ public class CcApplicationContext implements
ICcApplicationContext {
requestTracker = new RequestTracker(this);
configValidator = configValidatorFactory.create();
this.adapterFactoryService = adapterFactoryService;
- dataPartitioningProvider = DataPartitioningProvider.create(this);
+ this.namespacePathResolver = new
NamespacePathResolver(isCloudDeployment());
+ this.namespaceResolver = new NamespaceResolver(isCloudDeployment());
this.globalTxManager = globalTxManager;
this.ioManager = ioManager;
+ dataPartitioningProvider = DataPartitioningProvider.create(this);
}
@Override
@@ -379,6 +387,16 @@ public class CcApplicationContext implements
ICcApplicationContext {
return dataPartitioningProvider;
}
+ @Override
+ public INamespaceResolver getNamespaceResolver() {
+ return namespaceResolver;
+ }
+
+ @Override
+ public INamespacePathResolver getNamespacePathResolver() {
+ return namespacePathResolver;
+ }
+
@Override
public boolean isCloudDeployment() {
return ccServiceCtx.getAppConfig().getBoolean(CLOUD_DEPLOYMENT);
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index 9e796f2651..2b869fd75b 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -38,6 +38,8 @@ import org.apache.asterix.common.api.IConfigValidatorFactory;
import org.apache.asterix.common.api.ICoordinationService;
import org.apache.asterix.common.api.IDatasetLifecycleManager;
import org.apache.asterix.common.api.IDiskWriteRateLimiterProvider;
+import org.apache.asterix.common.api.INamespacePathResolver;
+import org.apache.asterix.common.api.INamespaceResolver;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.api.IPropertiesFactory;
import org.apache.asterix.common.api.IReceptionist;
@@ -59,6 +61,8 @@ import
org.apache.asterix.common.context.DiskWriteRateLimiterProvider;
import org.apache.asterix.common.context.GlobalVirtualBufferCache;
import org.apache.asterix.common.context.IStorageComponentProvider;
import org.apache.asterix.common.library.ILibraryManager;
+import org.apache.asterix.common.metadata.NamespacePathResolver;
+import org.apache.asterix.common.metadata.NamespaceResolver;
import org.apache.asterix.common.replication.IReplicationChannel;
import org.apache.asterix.common.replication.IReplicationManager;
import org.apache.asterix.common.replication.IReplicationStrategyFactory;
@@ -169,6 +173,8 @@ public class NCAppRuntimeContext implements
INcApplicationContext {
private IDiskWriteRateLimiterProvider diskWriteRateLimiterProvider;
private final CloudProperties cloudProperties;
private IPartitionBootstrapper partitionBootstrapper;
+ private final NamespacePathResolver namespacePathResolver;
+ private final NamespaceResolver namespaceResolver;
public NCAppRuntimeContext(INCServiceContext ncServiceContext,
NCExtensionManager extensionManager,
IPropertiesFactory propertiesFactory) {
@@ -190,6 +196,8 @@ public class NCAppRuntimeContext implements
INcApplicationContext {
.createResourceIdFactory();
persistedResourceRegistry =
ncServiceContext.getPersistedResourceRegistry();
cacheManager = new CacheManager();
+ namespacePathResolver = new NamespacePathResolver(isCloudDeployment());
+ namespaceResolver = new NamespaceResolver(isCloudDeployment());
}
@Override
@@ -465,6 +473,16 @@ public class NCAppRuntimeContext implements
INcApplicationContext {
return libraryManager;
}
+ @Override
+ public INamespaceResolver getNamespaceResolver() {
+ return namespaceResolver;
+ }
+
+ @Override
+ public INamespacePathResolver getNamespacePathResolver() {
+ return namespacePathResolver;
+ }
+
@Override
public void initializeMetadata(boolean newUniverse, int partitionId)
throws Exception {
LOGGER.info("Bootstrapping ({}) metadata in partition {}", newUniverse
? "new" : "existing", partitionId);
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/DataverseUtil.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/DataverseUtil.java
index 3b7cb00202..d132b6b295 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/DataverseUtil.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/DataverseUtil.java
@@ -31,24 +31,21 @@ public class DataverseUtil {
private DataverseUtil() {
}
- public static JobSpecification dropDataverseJobSpec(Dataverse dataverse,
MetadataProvider metadata) {
- JobSpecification jobSpec =
RuntimeUtils.createJobSpecification(metadata.getApplicationContext());
- PartitioningProperties partitioningProperties =
metadata.splitAndConstraints(dataverse.getDataverseName());
- FileRemoveOperatorDescriptor frod = new
FileRemoveOperatorDescriptor(jobSpec,
- partitioningProperties.getSplitsProvider(), false,
partitioningProperties.getComputeStorageMap());
-
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec,
frod,
- partitioningProperties.getConstraints());
- jobSpec.addRoot(frod);
- return jobSpec;
+ public static JobSpecification dropDataverseJobSpec(Dataverse dataverse,
MetadataProvider md) {
+ PartitioningProperties pp =
md.splitAndConstraints(dataverse.getDatabaseName(),
dataverse.getDataverseName());
+ return dropJobSpec(md, pp);
+ }
+
+ public static JobSpecification dropDatabaseJobSpec(String database,
MetadataProvider md) {
+ PartitioningProperties pp = md.splitAndConstraints(database);
+ return dropJobSpec(md, pp);
}
- public static JobSpecification dropDatabaseJobSpec(String database,
MetadataProvider metadata) {
+ private static JobSpecification dropJobSpec(MetadataProvider metadata,
PartitioningProperties pp) {
JobSpecification jobSpec =
RuntimeUtils.createJobSpecification(metadata.getApplicationContext());
- PartitioningProperties partitioningProperties =
metadata.splitAndConstraints(database);
- FileRemoveOperatorDescriptor frod = new
FileRemoveOperatorDescriptor(jobSpec,
- partitioningProperties.getSplitsProvider(), false,
partitioningProperties.getComputeStorageMap());
-
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec,
frod,
- partitioningProperties.getConstraints());
+ FileRemoveOperatorDescriptor frod =
+ new FileRemoveOperatorDescriptor(jobSpec,
pp.getSplitsProvider(), false, pp.getComputeStorageMap());
+
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec,
frod, pp.getConstraints());
jobSpec.addRoot(frod);
return jobSpec;
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
index 63958ce2df..f9296772eb 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
@@ -37,6 +37,7 @@ import
org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.metadata.MetadataUtil;
+import org.apache.asterix.common.metadata.NamespacePathResolver;
import org.apache.asterix.common.transactions.ITransactionManager;
import org.apache.asterix.common.transactions.TxnId;
import org.apache.asterix.dataflow.data.nontagged.MissingWriterFactory;
@@ -673,8 +674,10 @@ public class TestNodeController {
List<String> nodes =
Collections.singletonList(ExecutionTestUtil.integrationUtil.ncs[0].getId());
CcApplicationContext appCtx =
(CcApplicationContext)
ExecutionTestUtil.integrationUtil.cc.getApplicationContext();
+ String dvPath = new
NamespacePathResolver(false).resolve(primaryIndexInfo.dataset.getDatabaseName(),
+ primaryIndexInfo.dataset.getDataverseName());
FileSplit[] splits =
SplitsAndConstraintsUtil.getIndexSplits(appCtx.getClusterStateManager(),
- primaryIndexInfo.dataset, secondaryIndex.getIndexName(),
nodes);
+ primaryIndexInfo.dataset, secondaryIndex.getIndexName(),
nodes, dvPath);
fileSplitProvider = new ConstantFileSplitProvider(splits);
secondaryIndexTypeTraits =
createSecondaryIndexTypeTraits(primaryIndexInfo.recordType,
primaryIndexInfo.metaType,
primaryIndexInfo.primaryKeyTypes, secondaryIndexDetails
@@ -760,8 +763,10 @@ public class TestNodeController {
List<String> nodes =
Collections.singletonList(ExecutionTestUtil.integrationUtil.ncs[0].getId());
CcApplicationContext appCtx =
(CcApplicationContext)
ExecutionTestUtil.integrationUtil.cc.getApplicationContext();
+ String dvPath =
+ new
NamespacePathResolver(false).resolve(dataset.getDatabaseName(),
dataset.getDataverseName());
FileSplit[] splits =
SplitsAndConstraintsUtil.getIndexSplits(appCtx.getClusterStateManager(),
dataset,
- index.getIndexName(), nodes);
+ index.getIndexName(), nodes, dvPath);
fileSplitProvider = new ConstantFileSplitProvider(splits);
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java
index bbdbe827e7..ea34f482d8 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java
@@ -32,6 +32,7 @@ import org.apache.asterix.common.api.IMetadataLockManager;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.metadata.MetadataConstants;
+import org.apache.asterix.common.metadata.NamespacePathResolver;
import org.apache.asterix.common.utils.Servlets;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
@@ -251,8 +252,10 @@ public class TestDataUtil {
(ICcApplicationContext)
integrationUtil.getClusterControllerService().getApplicationContext();
final MetadataTransactionContext mdTxnCtx =
MetadataManager.INSTANCE.beginTransaction();
try {
+ String dvPath =
+ new
NamespacePathResolver(false).resolve(dataset.getDatabaseName(),
dataset.getDataverseName());
return SplitsAndConstraintsUtil.getIndexSplits(dataset,
dataset.getDatasetName(), mdTxnCtx,
- ccAppCtx.getClusterStateManager());
+ ccAppCtx.getClusterStateManager(), dvPath);
} finally {
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
index d5c84659f4..bdbdfab931 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
@@ -32,6 +32,7 @@ import
org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.metadata.MetadataConstants;
import org.apache.asterix.common.metadata.MetadataUtil;
+import org.apache.asterix.common.metadata.NamespacePathResolver;
import org.apache.asterix.file.StorageComponentProvider;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
@@ -158,8 +159,10 @@ public class IndexDropOperatorNodePushableTest {
final Dataset dataset = MetadataManager.INSTANCE.getDataset(mdTxn,
MetadataConstants.DEFAULT_DATABASE,
defaultDv, datasetName);
MetadataManager.INSTANCE.commitTransaction(mdTxn);
+ String dvPath =
+ new
NamespacePathResolver(false).resolve(dataset.getDatabaseName(),
dataset.getDataverseName());
FileSplit[] splits =
SplitsAndConstraintsUtil.getIndexSplits(appCtx.getClusterStateManager(),
dataset,
- indexName, Arrays.asList("asterix_nc1"));
+ indexName, Arrays.asList("asterix_nc1"), dvPath);
final ConstantFileSplitProvider constantFileSplitProvider =
new ConstantFileSplitProvider(Arrays.copyOfRange(splits,
0, 1));
IndexDataflowHelperFactory helperFactory =
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java
index 812e2e55f1..eabebf7b3a 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java
@@ -102,4 +102,8 @@ public interface IApplicationContext {
* @return the cloud properties
*/
CloudProperties getCloudProperties();
+
+ INamespaceResolver getNamespaceResolver();
+
+ INamespacePathResolver getNamespacePathResolver();
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespacePathResolver.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespacePathResolver.java
new file mode 100644
index 0000000000..6f33d6f29a
--- /dev/null
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespacePathResolver.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.api;
+
+import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
+
+public interface INamespacePathResolver {
+
+ String resolve(Namespace namespace);
+
+ String resolve(String databaseName, DataverseName dataverseName);
+
+}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespaceResolver.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespaceResolver.java
new file mode 100644
index 0000000000..f281280fde
--- /dev/null
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespaceResolver.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.api;
+
+import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
+
+public interface INamespaceResolver {
+
+ Namespace resolve(DataverseName dataverseName);
+
+}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/Namespace.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/Namespace.java
new file mode 100644
index 0000000000..26b549ae81
--- /dev/null
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/Namespace.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.common.metadata;
+
+import java.util.Objects;
+
+public final class Namespace {
+
+ private final String databaseName;
+ private final DataverseName dataverseName;
+
+ public Namespace(String databaseName, DataverseName dataverseName) {
+ this.databaseName = Objects.requireNonNull(databaseName);
+ this.dataverseName = Objects.requireNonNull(dataverseName);
+ }
+
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ public DataverseName getDataverseName() {
+ return dataverseName;
+ }
+
+ @Override
+ public String toString() {
+ return databaseName + "." + dataverseName;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(databaseName, dataverseName);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof Namespace)) {
+ return false;
+ }
+ Namespace that = (Namespace) obj;
+ return Objects.equals(databaseName, that.databaseName) &&
Objects.equals(dataverseName, that.dataverseName);
+ }
+
+}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespacePathResolver.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespacePathResolver.java
new file mode 100644
index 0000000000..26d1ce06f9
--- /dev/null
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespacePathResolver.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.common.metadata;
+
+import org.apache.asterix.common.api.INamespacePathResolver;
+import org.apache.asterix.common.utils.StoragePathUtil;
+
+public class NamespacePathResolver implements INamespacePathResolver {
+
+ private final boolean usingDatabase;
+
+ public NamespacePathResolver(boolean usingDatabase) {
+ this.usingDatabase = false;
+ }
+
+ @Override
+ public String resolve(Namespace namespace) {
+ return
StoragePathUtil.prepareDataverseName(namespace.getDataverseName());
+ }
+
+ @Override
+ public String resolve(String databaseName, DataverseName dataverseName) {
+ return StoragePathUtil.prepareDataverseName(dataverseName);
+ }
+}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespaceResolver.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespaceResolver.java
new file mode 100644
index 0000000000..14bba84694
--- /dev/null
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespaceResolver.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.common.metadata;
+
+import org.apache.asterix.common.api.INamespaceResolver;
+
+public class NamespaceResolver implements INamespaceResolver {
+
+ private final boolean usingDatabase;
+
+ public NamespaceResolver(boolean usingDatabase) {
+ this.usingDatabase = false;
+ }
+
+ @Override
+ public Namespace resolve(DataverseName dataverseName) {
+ if (dataverseName == null) {
+ return null;
+ }
+ return new Namespace(MetadataUtil.databaseFor(dataverseName),
dataverseName);
+ }
+}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
index 418e1717aa..9862be7826 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
@@ -24,6 +24,7 @@ import static
org.apache.asterix.common.utils.StorageConstants.STORAGE_ROOT_DIR_
import java.io.File;
import java.nio.file.Paths;
import java.util.Iterator;
+import java.util.List;
import org.apache.asterix.common.cluster.ClusterPartition;
import org.apache.asterix.common.metadata.DataverseName;
@@ -46,7 +47,7 @@ public class StoragePathUtil {
private static final Logger LOGGER = LogManager.getLogger();
public static final char DATAVERSE_CONTINUATION_MARKER = '^';
- private static String PARTITION_PATH = STORAGE_ROOT_DIR_NAME +
File.separator + PARTITION_DIR_PREFIX;
+ private static final String PARTITION_PATH = STORAGE_ROOT_DIR_NAME +
File.separator + PARTITION_DIR_PREFIX;
private StoragePathUtil() {
}
@@ -74,21 +75,25 @@ public class StoragePathUtil {
return new DefaultIoDeviceFileSplit(nodeId, relativePath);
}
- public static String prepareStoragePartitionPath(int partitonId) {
- return Paths.get(StorageConstants.STORAGE_ROOT_DIR_NAME,
PARTITION_DIR_PREFIX + partitonId).toString();
+ public static String prepareStoragePartitionPath(int partitionId) {
+ return Paths.get(StorageConstants.STORAGE_ROOT_DIR_NAME,
PARTITION_DIR_PREFIX + partitionId).toString();
}
public static String prepareIngestionLogPath() {
return Paths.get(StorageConstants.INGESTION_LOGS_DIR_NAME).toString();
}
- public static String prepareDataverseIndexName(DataverseName
dataverseName, String datasetName, String idxName,
- long rebalanceCount) {
- return prepareDataverseComponentName(dataverseName,
prepareFullIndexName(datasetName, idxName, rebalanceCount));
+ public static String prepareNamespaceIndexName(String datasetName, String
idxName, long rebalanceCount,
+ String namespacePath) {
+ return prepareNamespaceComponentName(namespacePath,
prepareFullIndexName(datasetName, idxName, rebalanceCount));
}
public static String prepareDataverseName(DataverseName dataverseName) {
- Iterator<String> dvParts = dataverseName.getParts().iterator();
+ List<String> parts = dataverseName.getParts();
+ if (parts.size() < 2) {
+ return parts.get(0);
+ }
+ Iterator<String> dvParts = parts.iterator();
StringBuilder builder = new StringBuilder();
builder.append(dvParts.next());
while (dvParts.hasNext()) {
@@ -97,8 +102,8 @@ public class StoragePathUtil {
return builder.toString();
}
- public static String prepareDataverseComponentName(DataverseName
dataverseName, String component) {
- return prepareDataverseName(dataverseName) + File.separatorChar +
component;
+ public static String prepareNamespaceComponentName(String namespacePath,
String component) {
+ return namespacePath + File.separatorChar + component;
}
private static String prepareFullIndexName(String datasetName, String
idxName, long rebalanceCount) {
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
index 7398d0284c..759d86ee5f 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
@@ -27,6 +27,8 @@ import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.external.IDataSourceAdapter;
import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
import org.apache.asterix.common.library.ILibraryManager;
+import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.MetadataUtil;
import org.apache.asterix.external.api.IDataFlowController;
import org.apache.asterix.external.api.IDataParserFactory;
import org.apache.asterix.external.api.IExternalDataSourceFactory;
@@ -156,9 +158,12 @@ public class GenericAdapterFactory implements
ITypedAdapterFactory {
this.isFeed = ExternalDataUtils.isFeed(configuration);
this.logIngestionEvents =
ExternalDataUtils.isLogIngestionEvents(configuration);
if (logIngestionEvents) {
+ DataverseName dataverseName =
ExternalDataUtils.getDatasetDataverse(configuration);
+ String databaseName = MetadataUtil.databaseFor(dataverseName);
+ String namespacePath =
appCtx.getNamespacePathResolver().resolve(databaseName, dataverseName);
//TODO(partitioning) make this code reuse DataPartitioningProvider
- feedLogFileSplits =
FeedUtils.splitsForAdapter(ExternalDataUtils.getDatasetDataverse(configuration),
- ExternalDataUtils.getFeedName(configuration),
dataSourceFactory.getPartitionConstraint());
+ feedLogFileSplits = FeedUtils.splitsForAdapter(namespacePath,
ExternalDataUtils.getFeedName(configuration),
+ dataSourceFactory.getPartitionConstraint());
}
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
index 050d9f0071..59b1a02cb7 100755
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
@@ -55,6 +55,8 @@ import java.util.zip.ZipFile;
import javax.net.ssl.SSLContext;
+import org.apache.asterix.common.api.INamespacePathResolver;
+import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.functions.ExternalFunctionLanguage;
@@ -136,17 +138,20 @@ public class ExternalLibraryManager implements
ILibraryManager, ILifeCycleCompon
private final FileReference trashDir;
private final FileReference distDir;
private final Path trashDirPath;
+ //TODO(DB): change for database
private final Map<Pair<DataverseName, String>, ILibrary> libraries = new
HashMap<>();
private IPCSystem pythonIPC;
private final ExternalFunctionResultRouter router;
private final IIOManager ioManager;
- private boolean sslEnabled;
+ private final INamespacePathResolver namespacePathResolver;
+ private final boolean sslEnabled;
private Function<ILibraryManager, CloseableHttpClient> uploadClientSupp;
public ExternalLibraryManager(NodeControllerService ncs,
IPersistedResourceRegistry reg, FileReference appDir,
IIOManager ioManager) {
this.ncs = ncs;
this.reg = reg;
+ namespacePathResolver = ((INcApplicationContext)
ncs.getApplicationContext()).getNamespacePathResolver();
baseDir = appDir.getChild(LIBRARY_MANAGER_BASE_DIR_NAME);
storageDir = baseDir.getChild(STORAGE_DIR_NAME);
storageDirPath = storageDir.getFile().toPath();
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
index 5baefcba47..bf924fc079 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
@@ -32,7 +32,6 @@ import
org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
-import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.runtime.utils.RuntimeUtils;
import org.apache.commons.lang3.StringUtils;
@@ -80,8 +79,8 @@ public class FeedUtils {
private FeedUtils() {
}
- private static FileSplit splitsForAdapter(DataverseName dataverseName,
String feedName, String nodeName) {
- String relPathFile =
StoragePathUtil.prepareDataverseComponentName(dataverseName, feedName);
+ private static FileSplit splitsForAdapter(String namespacePath, String
feedName, String nodeName) {
+ String relPathFile =
StoragePathUtil.prepareNamespaceComponentName(namespacePath, feedName);
String storagePartitionPath =
StoragePathUtil.prepareIngestionLogPath();
// Note: feed adapter instances in a single node share the feed logger
// format: 'ingestion logs dir
name'/dataverse_part1[^dataverse_part2[...]]/feed/node
@@ -89,7 +88,7 @@ public class FeedUtils {
return StoragePathUtil.getDefaultIoDeviceFileSpiltForNode(nodeName,
f.getPath());
}
- public static FileSplit[] splitsForAdapter(DataverseName dataverseName,
String feedName,
+ public static FileSplit[] splitsForAdapter(String namespacePath, String
feedName,
AlgebricksPartitionConstraint partitionConstraints) throws
AsterixException {
if (partitionConstraints.getPartitionConstraintType() ==
PartitionConstraintType.COUNT) {
throw new AsterixException("Can't create file splits for adapter
with count partitioning constraints");
@@ -97,7 +96,7 @@ public class FeedUtils {
String[] locations = ((AlgebricksAbsolutePartitionConstraint)
partitionConstraints).getLocations();
List<FileSplit> splits = new ArrayList<>();
for (String nd : locations) {
- splits.add(splitsForAdapter(dataverseName, feedName, nd));
+ splits.add(splitsForAdapter(namespacePath, feedName, nd));
}
return splits.toArray(new FileSplit[] {});
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataIndex.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataIndex.java
index 21329accd9..082eb2f6ac 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataIndex.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataIndex.java
@@ -22,6 +22,7 @@ package org.apache.asterix.metadata.api;
import java.io.Serializable;
import java.util.List;
+import org.apache.asterix.common.api.INamespacePathResolver;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.transactions.DatasetId;
import org.apache.asterix.om.types.ARecordType;
@@ -67,7 +68,7 @@ public interface IMetadataIndex extends Serializable {
public int[] getFieldPermutation();
- public String getFileNameRelativePath();
+ public String getFileNameRelativePath(INamespacePathResolver
namespacePathResolver);
public ARecordType getPayloadRecordType();
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
index 4bab4c9ed9..b154287168 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.List;
import org.apache.asterix.common.api.ILSMComponentIdGeneratorFactory;
+import org.apache.asterix.common.api.INamespacePathResolver;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.MetadataProperties;
@@ -392,9 +393,12 @@ public class MetadataBootstrap {
public static void enlistMetadataDataset(INCServiceContext ncServiceCtx,
IMetadataIndex index,
MetadataIndexesProvider mdIndexesProvider) throws
HyracksDataException {
final int datasetId = index.getDatasetId().getId();
+ INamespacePathResolver namespacePathResolver =
+ ((INcApplicationContext)
ncServiceCtx.getApplicationContext()).getNamespacePathResolver();
String metadataPartitionPath =
StoragePathUtil.prepareStoragePartitionPath(MetadataNode.INSTANCE.getMetadataStoragePartition());
- String resourceName = metadataPartitionPath + File.separator +
index.getFileNameRelativePath();
+ String resourceName =
+ metadataPartitionPath + File.separator +
index.getFileNameRelativePath(namespacePathResolver);
FileReference file = ioManager.resolve(resourceName);
index.setFile(file);
ITypeTraits[] typeTraits = index.getTypeTraits();
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndex.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndex.java
index d259643fc6..c06b878cfb 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndex.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndex.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import org.apache.asterix.common.api.INamespacePathResolver;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.metadata.MetadataConstants;
import org.apache.asterix.common.metadata.MetadataIndexImmutableProperties;
@@ -237,10 +238,10 @@ public class MetadataIndex implements IMetadataIndex {
}
@Override
- public String getFileNameRelativePath() {
+ public String getFileNameRelativePath(INamespacePathResolver
namespacePathResolver) {
// The rebalance count for metadata dataset is always 0.
- return StoragePathUtil.prepareDataverseIndexName(getDataverseName(),
getIndexedDatasetName(), getIndexName(),
- 0);
+ String namespacePath =
namespacePathResolver.resolve(getDatabaseName(), getDataverseName());
+ return
StoragePathUtil.prepareNamespaceIndexName(getIndexedDatasetName(),
getIndexName(), 0, namespacePath);
}
@Override
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 0ecef7d012..d1b07c5812 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -32,6 +32,7 @@ import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
+import org.apache.asterix.common.api.INamespaceResolver;
import org.apache.asterix.common.cluster.PartitioningProperties;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
@@ -49,6 +50,7 @@ import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.metadata.LockList;
import org.apache.asterix.common.metadata.MetadataConstants;
import org.apache.asterix.common.metadata.MetadataUtil;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.common.storage.ICompressionManager;
import org.apache.asterix.common.transactions.ITxnIdFactory;
import org.apache.asterix.common.transactions.TxnId;
@@ -189,6 +191,7 @@ public class MetadataProvider implements
IMetadataProvider<DataSourceId, String>
private boolean blockingOperatorDisabled = false;
private final DataPartitioningProvider dataPartitioningProvider;
+ private final INamespaceResolver namespaceResolver;
private IDataFormat dataFormat = FormatUtils.getDefaultFormat();
public static MetadataProvider create(ICcApplicationContext appCtx,
Dataverse defaultDataverse) {
@@ -202,6 +205,7 @@ public class MetadataProvider implements
IMetadataProvider<DataSourceId, String>
protected MetadataProvider(ICcApplicationContext appCtx) {
this.appCtx = appCtx;
this.storageComponentProvider = appCtx.getStorageComponentProvider();
+ namespaceResolver = appCtx.getNamespaceResolver();
storageProperties = appCtx.getStorageProperties();
functionManager = ((IFunctionExtensionManager)
appCtx.getExtensionManager()).getFunctionManager();
dataPartitioningProvider = (DataPartitioningProvider)
appCtx.getDataPartitioningProvider();
@@ -959,8 +963,8 @@ public class MetadataProvider implements
IMetadataProvider<DataSourceId, String>
return
dataPartitioningProvider.getPartitioningProperties(databaseName);
}
- public PartitioningProperties splitAndConstraints(DataverseName
dataverseName) {
- return
dataPartitioningProvider.getPartitioningProperties(dataverseName);
+ public PartitioningProperties splitAndConstraints(String databaseName,
DataverseName dataverseName) {
+ return
dataPartitioningProvider.getPartitioningProperties(databaseName, dataverseName);
}
public FileSplit[] splitsForIndex(MetadataTransactionContext mdTxnCtx,
Dataset dataset, String indexName)
@@ -1823,6 +1827,10 @@ public class MetadataProvider implements
IMetadataProvider<DataSourceId, String>
return storageComponentProvider;
}
+ public Namespace resolve(DataverseName dataverseName) {
+ return namespaceResolver.resolve(dataverseName);
+ }
+
public PartitioningProperties getPartitioningProperties(Index idx) throws
AlgebricksException {
Dataset ds = findDataset(idx.getDatabaseName(),
idx.getDataverseName(), idx.getDatasetName());
return getPartitioningProperties(ds, idx.getIndexName());
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DataPartitioningProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DataPartitioningProvider.java
index d99aa137cf..53c5d444bc 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DataPartitioningProvider.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DataPartitioningProvider.java
@@ -22,6 +22,8 @@ import java.util.Arrays;
import java.util.Set;
import java.util.TreeSet;
+import org.apache.asterix.common.api.INamespacePathResolver;
+import org.apache.asterix.common.api.INamespaceResolver;
import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.common.cluster.PartitioningProperties;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
@@ -47,11 +49,15 @@ import
org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
public abstract class DataPartitioningProvider implements
IDataPartitioningProvider {
protected final ICcApplicationContext appCtx;
+ protected final INamespacePathResolver namespacePathResolver;
+ protected final INamespaceResolver namespaceResolver;
protected final ClusterStateManager clusterStateManager;
protected final int storagePartitionsCounts;
DataPartitioningProvider(ICcApplicationContext appCtx) {
this.appCtx = appCtx;
+ this.namespacePathResolver = appCtx.getNamespacePathResolver();
+ this.namespaceResolver = appCtx.getNamespaceResolver();
this.clusterStateManager = (ClusterStateManager)
appCtx.getClusterStateManager();
this.storagePartitionsCounts =
clusterStateManager.getStoragePartitionsCount();
}
@@ -70,7 +76,7 @@ public abstract class DataPartitioningProvider implements
IDataPartitioningProvi
public abstract PartitioningProperties getPartitioningProperties(String
databaseName);
- public abstract PartitioningProperties
getPartitioningProperties(DataverseName dataverseName);
+ public abstract PartitioningProperties getPartitioningProperties(String
databaseName, DataverseName dataverseName);
public abstract PartitioningProperties
getPartitioningProperties(MetadataTransactionContext mdTxnCtx, Dataset ds,
String indexName) throws AlgebricksException;
@@ -81,8 +87,8 @@ public abstract class DataPartitioningProvider implements
IDataPartitioningProvi
Set<String> nodes = new
TreeSet<>(Arrays.asList(allCluster.getLocations()));
AlgebricksAbsolutePartitionConstraint locations =
new AlgebricksAbsolutePartitionConstraint(nodes.toArray(new
String[0]));
- FileSplit[] feedLogFileSplits =
- FeedUtils.splitsForAdapter(feed.getDataverseName(),
feed.getFeedName(), locations);
+ String namespacePath =
namespacePathResolver.resolve(feed.getDatabaseName(), feed.getDataverseName());
+ FileSplit[] feedLogFileSplits =
FeedUtils.splitsForAdapter(namespacePath, feed.getFeedName(), locations);
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> spC =
StoragePathUtil.splitProviderAndPartitionConstraints(feedLogFileSplits);
int[][] partitionsMap =
getOneToOnePartitionsMap(getLocationsCount(spC.second));
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DynamicDataPartitioningProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DynamicDataPartitioningProvider.java
index 803a53b7d9..57392db4f0 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DynamicDataPartitioningProvider.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DynamicDataPartitioningProvider.java
@@ -45,9 +45,10 @@ public class DynamicDataPartitioningProvider extends
DataPartitioningProvider {
}
@Override
- public PartitioningProperties getPartitioningProperties(DataverseName
dataverseName) {
+ public PartitioningProperties getPartitioningProperties(String
databaseName, DataverseName dataverseName) {
+ String namespacePath = namespacePathResolver.resolve(databaseName,
dataverseName);
Pair<IFileSplitProvider, AlgebricksPartitionConstraint>
splitsAndConstraints = SplitsAndConstraintsUtil
-
.getDataverseSplitProviderAndConstraints(appCtx.getClusterStateManager(),
dataverseName);
+
.getDataverseSplitProviderAndConstraints(appCtx.getClusterStateManager(),
namespacePath);
int[][] partitionsMap =
getOneToOnePartitionsMap(getLocationsCount(splitsAndConstraints.second));
return PartitioningProperties.of(splitsAndConstraints.first,
splitsAndConstraints.second, partitionsMap);
}
@@ -55,8 +56,9 @@ public class DynamicDataPartitioningProvider extends
DataPartitioningProvider {
@Override
public PartitioningProperties
getPartitioningProperties(MetadataTransactionContext mdTxnCtx, Dataset ds,
String indexName) throws AlgebricksException {
- FileSplit[] splits =
- SplitsAndConstraintsUtil.getIndexSplits(ds, indexName,
mdTxnCtx, appCtx.getClusterStateManager());
+ String namespacePath =
namespacePathResolver.resolve(ds.getDatabaseName(), ds.getDataverseName());
+ FileSplit[] splits = SplitsAndConstraintsUtil.getIndexSplits(ds,
indexName, mdTxnCtx,
+ appCtx.getClusterStateManager(), namespacePath);
Pair<IFileSplitProvider, AlgebricksPartitionConstraint>
splitsAndConstraints =
StoragePathUtil.splitProviderAndPartitionConstraints(splits);
int[][] partitionsMap =
getOneToOnePartitionsMap(getLocationsCount(splitsAndConstraints.second));
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
index 2abb4f6ad5..da7cd0d742 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
@@ -27,7 +27,6 @@ import java.util.List;
import org.apache.asterix.common.cluster.ClusterPartition;
import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.common.exceptions.MetadataException;
-import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.metadata.MetadataConstants;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.metadata.MetadataManager;
@@ -45,47 +44,42 @@ public class SplitsAndConstraintsUtil {
private SplitsAndConstraintsUtil() {
}
- private static FileSplit[] getDatabaseSplits(IClusterStateManager
clusterStateManager, String databaseName) {
- List<FileSplit> splits = new ArrayList<>();
- // get all partitions
- for (ClusterPartition clusterPartition :
clusterStateManager.getClusterPartitons()) {
- File f = new
File(StoragePathUtil.prepareStoragePartitionPath(clusterPartition.getPartitionId()),
- databaseName);
-
splits.add(StoragePathUtil.getFileSplitForClusterPartition(clusterPartition,
f.getPath()));
- }
- return splits.toArray(new FileSplit[] {});
+ private static FileSplit[] getDatabaseSplits(IClusterStateManager
clusterStateManager, String databasePath) {
+ return getSplits(clusterStateManager, databasePath);
+ }
+
+ private static FileSplit[] getDataverseSplits(IClusterStateManager
clusterStateManager, String dataversePath) {
+ return getSplits(clusterStateManager, dataversePath);
}
- private static FileSplit[] getDataverseSplits(IClusterStateManager
clusterStateManager,
- DataverseName dataverseName) {
+ private static FileSplit[] getSplits(IClusterStateManager
clusterStateManager, String path) {
List<FileSplit> splits = new ArrayList<>();
// get all partitions
for (ClusterPartition clusterPartition :
clusterStateManager.getClusterPartitons()) {
- File f = new
File(StoragePathUtil.prepareStoragePartitionPath(clusterPartition.getPartitionId()),
- StoragePathUtil.prepareDataverseName(dataverseName));
+ File f = new
File(StoragePathUtil.prepareStoragePartitionPath(clusterPartition.getPartitionId()),
path);
splits.add(StoragePathUtil.getFileSplitForClusterPartition(clusterPartition,
f.getPath()));
}
return splits.toArray(new FileSplit[] {});
}
public static FileSplit[] getIndexSplits(Dataset dataset, String
indexName, MetadataTransactionContext mdTxnCtx,
- IClusterStateManager csm) throws AlgebricksException {
+ IClusterStateManager csm, String namespacePath) throws
AlgebricksException {
try {
NodeGroup nodeGroup =
MetadataManager.INSTANCE.getNodegroup(mdTxnCtx, dataset.getNodeGroupName());
if (nodeGroup == null) {
throw new AlgebricksException("Couldn't find node group " +
dataset.getNodeGroupName());
}
List<String> nodeList = nodeGroup.getNodeNames();
- return getIndexSplits(csm, dataset, indexName, nodeList);
+ return getIndexSplits(csm, dataset, indexName, nodeList,
namespacePath);
} catch (MetadataException me) {
throw new AlgebricksException(me);
}
}
public static FileSplit[] getIndexSplits(IClusterStateManager
clusterStateManager, Dataset dataset,
- String indexName, List<String> nodes) {
- final String relPath =
StoragePathUtil.prepareDataverseIndexName(dataset.getDataverseName(),
- dataset.getDatasetName(), indexName,
dataset.getRebalanceCount());
+ String indexName, List<String> nodes, String namespacePath) {
+ final String relPath =
StoragePathUtil.prepareNamespaceIndexName(dataset.getDatasetName(), indexName,
+ dataset.getRebalanceCount(), namespacePath);
final List<ClusterPartition> datasetPartitions =
getDatasetPartitions(clusterStateManager, dataset, nodes);
final List<FileSplit> splits = new ArrayList<>();
for (ClusterPartition partition : datasetPartitions) {
@@ -102,8 +96,8 @@ public class SplitsAndConstraintsUtil {
}
public static Pair<IFileSplitProvider, AlgebricksPartitionConstraint>
getDataverseSplitProviderAndConstraints(
- IClusterStateManager clusterStateManager, DataverseName
dataverseName) {
- FileSplit[] splits = getDataverseSplits(clusterStateManager,
dataverseName);
+ IClusterStateManager clusterStateManager, String dataversePath) {
+ FileSplit[] splits = getDataverseSplits(clusterStateManager,
dataversePath);
return StoragePathUtil.splitProviderAndPartitionConstraints(splits);
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java
index f8f967ad97..7206a5084a 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java
@@ -58,8 +58,8 @@ public class StaticDataPartitioningProvider extends
DataPartitioningProvider {
}
@Override
- public PartitioningProperties getPartitioningProperties(DataverseName
dataverseName) {
- SplitComputeLocations dataverseSplits =
getDataverseSplits(dataverseName);
+ public PartitioningProperties getPartitioningProperties(String
databaseName, DataverseName dataverseName) {
+ SplitComputeLocations dataverseSplits =
getDataverseSplits(databaseName, dataverseName);
StorageComputePartitionsMap partitionMap =
clusterStateManager.getStorageComputeMap();
int[][] partitionsMap = partitionMap.getComputeToStorageMap(false);
return PartitioningProperties.of(dataverseSplits.getSplitsProvider(),
dataverseSplits.getConstraints(),
@@ -77,8 +77,8 @@ public class StaticDataPartitioningProvider extends
DataPartitioningProvider {
partitionsMap);
}
- private SplitComputeLocations getDataverseSplits(DataverseName
dataverseName) {
- return getSplits(StoragePathUtil.prepareDataverseName(dataverseName));
+ private SplitComputeLocations getDataverseSplits(String databaseName,
DataverseName dataverseName) {
+ return getSplits(namespacePathResolver.resolve(databaseName,
dataverseName));
}
private SplitComputeLocations getSplits(String subPath) {
@@ -105,13 +105,14 @@ public class StaticDataPartitioningProvider extends
DataPartitioningProvider {
List<FileSplit> splits = new ArrayList<>();
List<String> locations = new ArrayList<>();
Set<Integer> uniqueLocations = new HashSet<>();
+ String namespacePath =
namespacePathResolver.resolve(dataset.getDatabaseName(),
dataset.getDataverseName());
StorageComputePartitionsMap partitionMap =
clusterStateManager.getStorageComputeMap();
final int datasetPartitions = getNumberOfPartitions(dataset);
boolean metadataDataset =
MetadataIndexImmutableProperties.isMetadataDataset(dataset.getDatasetId());
for (int i = 0; i < datasetPartitions; i++) {
int storagePartition = metadataDataset ?
StorageConstants.METADATA_PARTITION : i;
- final String relPath =
StoragePathUtil.prepareDataverseIndexName(dataset.getDataverseName(),
- dataset.getDatasetName(), indexName,
dataset.getRebalanceCount());
+ final String relPath =
StoragePathUtil.prepareNamespaceIndexName(dataset.getDatasetName(), indexName,
+ dataset.getRebalanceCount(), namespacePath);
File f = new
File(StoragePathUtil.prepareStoragePartitionPath(storagePartition), relPath);
ComputePartition computePartition =
partitionMap.getComputePartition(storagePartition);
splits.add(new MappedFileSplit(computePartition.getNodeId(),
f.getPath(), 0));
