This is an automated email from the ASF dual-hosted git repository.
peeyush pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new e01d6aa6d7 [NO ISSUE][CONF][TX] Make path to store atomic txn logs
configurable
e01d6aa6d7 is described below
commit e01d6aa6d7a038d38607464376a59e755a882502
Author: Peeyush Gupta <[email protected]>
AuthorDate: Thu Jul 20 11:42:03 2023 -0700
[NO ISSUE][CONF][TX] Make path to store atomic txn logs configurable
- user model changes: no
- storage format changes: no
- interface changes: no
Change-Id: I0579cc7630bc718e678234e1d7147fcdb679b3f6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17653
Reviewed-by: Peeyush Gupta <[email protected]>
Reviewed-by: Murtadha Al Hubail <[email protected]>
Integration-Tests: Jenkins <[email protected]>
Tested-by: Murtadha Al Hubail <[email protected]>
---
.../java/org/apache/asterix/app/cc/GlobalTxManager.java | 3 +--
.../org/apache/asterix/hyracks/bootstrap/CCApplication.java | 5 ++---
.../org/apache/asterix/common/utils/StorageConstants.java | 3 ---
.../service/transaction/GlobalTransactionContext.java | 7 ++-----
.../apache/hyracks/control/common/controllers/CCConfig.java | 13 ++++++++++++-
5 files changed, 17 insertions(+), 14 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/GlobalTxManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/GlobalTxManager.java
index 4dbf38bb3a..d69376aabf 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/GlobalTxManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/GlobalTxManager.java
@@ -30,7 +30,6 @@ import org.apache.asterix.common.cluster.IGlobalTxManager;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.messaging.api.ICCMessageBroker;
import org.apache.asterix.common.transactions.IGlobalTransactionContext;
-import org.apache.asterix.common.utils.StorageConstants;
import
org.apache.asterix.transaction.management.service.transaction.GlobalTransactionContext;
import
org.apache.asterix.transaction.management.service.transaction.GlobalTxInfo;
import org.apache.hyracks.api.application.ICCServiceContext;
@@ -178,7 +177,7 @@ public class GlobalTxManager implements IGlobalTxManager {
@Override
public void rollback() throws Exception {
- Set<FileReference> txnLogFileRefs =
ioManager.list(ioManager.resolve(StorageConstants.CC_TX_LOG_DIR));
+ Set<FileReference> txnLogFileRefs =
ioManager.list(ioManager.resolve("."));
for (FileReference txnLogFileRef : txnLogFileRefs) {
IGlobalTransactionContext context = new
GlobalTransactionContext(txnLogFileRef, ioManager);
txnContextRepository.put(context.getJobId(), context);
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
index 0e740f73e0..e64b72f6df 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
@@ -82,7 +82,6 @@ import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.metadata.IMetadataLockUtil;
import org.apache.asterix.common.replication.INcLifecycleCoordinator;
import org.apache.asterix.common.utils.Servlets;
-import org.apache.asterix.common.utils.StorageConstants;
import org.apache.asterix.external.adapter.factory.AdapterFactoryService;
import org.apache.asterix.file.StorageComponentProvider;
import org.apache.asterix.messaging.CCMessageBroker;
@@ -167,9 +166,10 @@ public class CCApplication extends BaseCCApplication {
componentProvider = new StorageComponentProvider();
ccExtensionManager = new CCExtensionManager(new
ArrayList<>(getExtensions()));
IGlobalRecoveryManager globalRecoveryManager =
createGlobalRecoveryManager();
+ final CCConfig ccConfig = controllerService.getCCConfig();
List<IODeviceHandle> devices = new ArrayList<>();
- devices.add(new IODeviceHandle(new
File(StorageConstants.CC_STORAGE_ROOT_DIR), "."));
+ devices.add(new IODeviceHandle(new File(ccConfig.getGlobalTxLogDir()),
"."));
IOManager ioManager = new IOManager(devices, new
DefaultDeviceResolver(), 1, 10);
CloudProperties cloudProperties = null;
if (ccServiceCtx.getAppConfig().getBoolean(CLOUD_DEPLOYMENT)) {
@@ -180,7 +180,6 @@ public class CCApplication extends BaseCCApplication {
appCtx = createApplicationContext(null, globalRecoveryManager,
lifecycleCoordinator, Receptionist::new,
ConfigValidator::new, ccExtensionManager, new
AdapterFactoryService(), globalTxManager, ioManager,
cloudProperties);
- final CCConfig ccConfig = controllerService.getCCConfig();
if (System.getProperty("java.rmi.server.hostname") == null) {
System.setProperty("java.rmi.server.hostname",
ccConfig.getClusterPublicAddress());
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
index 7b6ed0d72f..1321c96306 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.common.utils;
-import java.nio.file.Paths;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -31,8 +30,6 @@ import
org.apache.hyracks.storage.am.lsm.common.impls.ConcurrentMergePolicyFacto
*/
public class StorageConstants {
- public static final String CC_STORAGE_ROOT_DIR = "/tmp/";
- public static final String CC_TX_LOG_DIR = Paths.get("cc",
"txnlogs").toString();
public static final String STORAGE_ROOT_DIR_NAME = "storage";
public static final String INGESTION_LOGS_DIR_NAME = "ingestion_logs";
public static final String PARTITION_DIR_PREFIX = "partition_";
diff --git
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/GlobalTransactionContext.java
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/GlobalTransactionContext.java
index 9d1dd1d60b..4cac9d8d13 100644
---
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/GlobalTransactionContext.java
+++
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/GlobalTransactionContext.java
@@ -27,7 +27,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.asterix.common.cluster.IGlobalTxManager.TransactionStatus;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.transactions.IGlobalTransactionContext;
-import org.apache.asterix.common.utils.StorageConstants;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.job.JobId;
@@ -124,8 +123,7 @@ public class GlobalTransactionContext implements
IGlobalTransactionContext {
@Override
public void persist(IOManager ioManager) {
try {
- FileReference fref = ioManager
- .resolve(Paths.get(StorageConstants.CC_TX_LOG_DIR,
String.format("%s.log", jobId)).toString());
+ FileReference fref =
ioManager.resolve(Paths.get(String.format("%s.log", jobId)).toString());
AtomicTransactionLog txnLog = new AtomicTransactionLog(jobId,
datasetIds, nodeResourceMap.keySet(),
nodeResourceMap, numPartitions);
ioManager.overwrite(fref,
@@ -138,8 +136,7 @@ public class GlobalTransactionContext implements
IGlobalTransactionContext {
@Override
public void delete(IOManager ioManager) {
try {
- FileReference fref = ioManager
- .resolve(Paths.get(StorageConstants.CC_TX_LOG_DIR,
String.format("%s.log", jobId)).toString());
+ FileReference fref =
ioManager.resolve(Paths.get(String.format("%s.log", jobId)).toString());
ioManager.delete(fref);
} catch (HyracksDataException e) {
throw new RuntimeException(e);
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java
index d148145e57..6904e3c9fc 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java
@@ -84,7 +84,12 @@ public class CCConfig extends ControllerConfig {
OptionTypes.STRING,
(Function<IApplicationConfig, String>) appConfig -> FileUtil
.joinPath(appConfig.getString(ControllerConfig.Option.DEFAULT_DIR), "passwd"),
- ControllerConfig.Option.DEFAULT_DIR.cmdline() + "/passwd");
+ ControllerConfig.Option.DEFAULT_DIR.cmdline() + "/passwd"),
+ GLOBAL_TXN_LOG_DIR(
+ STRING,
+ appConfig ->
FileUtil.joinPath(appConfig.getString(ControllerConfig.Option.DEFAULT_DIR),
+ "global-txn-log"),
+ ControllerConfig.Option.DEFAULT_DIR.cmdline() +
"/global-txn-log");
private final IOptionType parser;
private Object defaultValue;
@@ -206,6 +211,8 @@ public class CCConfig extends ControllerConfig {
return "The password to the provided key store";
case CREDENTIAL_FILE:
return "Path to HTTP basic credentials";
+ case GLOBAL_TXN_LOG_DIR:
+ return "Directory to store global transaction logs";
default:
throw new IllegalStateException("NYI: " + this);
}
@@ -481,4 +488,8 @@ public class CCConfig extends ControllerConfig {
return getAppConfig().getString(Option.CREDENTIAL_FILE);
}
+ public String getGlobalTxLogDir() {
+ return getAppConfig().getString(Option.GLOBAL_TXN_LOG_DIR);
+ }
+
}
