Murtadha Hubail has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/469
Change subject: ASTERIXDB-1152: Delete storage data of old instances
......................................................................
ASTERIXDB-1152: Delete storage data of old instances
This change deletes any existing data of old insatances if the system state is
NEW_UNIVERSE.
Change-Id: Ibb6c6949bdf2ed6c3e491fa66a23491ff34fc830
---
M
asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
M
asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
M asterix-common/src/test/java/org/apache/asterix/test/aql/TestsUtils.java
M
asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
M
asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java
5 files changed, 64 insertions(+), 30 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/69/469/1
diff --git
a/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
b/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
index 35afd8b..c8c6c5b 100644
---
a/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
+++
b/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
@@ -38,6 +38,8 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.feeds.api.IFeedManager;
import
org.apache.asterix.common.transactions.IAsterixAppRuntimeContextProvider;
+import org.apache.asterix.common.transactions.IRecoveryManager;
+import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
import org.apache.asterix.common.transactions.ITransactionSubsystem;
import org.apache.asterix.feeds.FeedManager;
import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes;
@@ -138,17 +140,23 @@
metadataMergePolicyFactory = new PrefixMergePolicyFactory();
- ILocalResourceRepositoryFactory
persistentLocalResourceRepositoryFactory = new
PersistentLocalResourceRepositoryFactory(
- ioManager, ncApplicationContext.getNodeId());
- localResourceRepository = (PersistentLocalResourceRepository)
persistentLocalResourceRepositoryFactory
- .createRepository();
- resourceIdFactory = (new
ResourceIdFactoryProvider(localResourceRepository)).createResourceIdFactory();
+
IAsterixAppRuntimeContextProvider asterixAppRuntimeContextProvider =
new AsterixAppRuntimeContextProdiverForRecovery(
this);
txnSubsystem = new
TransactionSubsystem(ncApplicationContext.getNodeId(),
asterixAppRuntimeContextProvider,
txnProperties);
+ IRecoveryManager recoveryMgr = txnSubsystem.getRecoveryManager();
+ SystemState systemState = recoveryMgr.getSystemState();
+
+ ILocalResourceRepositoryFactory
persistentLocalResourceRepositoryFactory = new
PersistentLocalResourceRepositoryFactory(
+ ioManager, ncApplicationContext.getNodeId(), systemState);
+ localResourceRepository = (PersistentLocalResourceRepository)
persistentLocalResourceRepositoryFactory
+ .createRepository();
+
+ resourceIdFactory = (new
ResourceIdFactoryProvider(localResourceRepository)).createResourceIdFactory();
+
indexLifecycleManager = new DatasetLifecycleManager(storageProperties,
localResourceRepository,
MetadataPrimaryIndexes.FIRST_AVAILABLE_USER_DATASET_ID,
(LogManager) txnSubsystem.getLogManager());
diff --git
a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
index 63f862c..fd5a416 100644
---
a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
+++
b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
@@ -25,10 +25,6 @@
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.kohsuke.args4j.CmdLineException;
-import org.kohsuke.args4j.CmdLineParser;
-import org.kohsuke.args4j.Option;
-
import org.apache.asterix.api.common.AsterixAppRuntimeContext;
import org.apache.asterix.common.api.AsterixThreadFactory;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
@@ -50,6 +46,9 @@
import org.apache.hyracks.api.application.INCApplicationEntryPoint;
import org.apache.hyracks.api.lifecycle.ILifeCycleComponentManager;
import org.apache.hyracks.api.lifecycle.LifeCycleComponentManager;
+import org.kohsuke.args4j.CmdLineException;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
public class NCApplicationEntryPoint implements INCApplicationEntryPoint {
private static final Logger LOGGER =
Logger.getLogger(NCApplicationEntryPoint.class.getName());
@@ -111,12 +110,10 @@
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("System is in a state: " + systemState);
}
-
if (systemState == SystemState.CORRUPTED) {
recoveryMgr.startRecovery(true);
}
}
-
}
@Override
@@ -159,7 +156,7 @@
PersistentLocalResourceRepository localResourceRepository =
(PersistentLocalResourceRepository) runtimeContext
.getLocalResourceRepository();
- localResourceRepository.initialize(nodeId,
metadataProperties.getStores().get(nodeId)[0]);
+ localResourceRepository.initializeNewUniverse(nodeId,
metadataProperties.getStores().get(nodeId)[0]);
}
IAsterixStateProxy proxy = null;
diff --git
a/asterix-common/src/test/java/org/apache/asterix/test/aql/TestsUtils.java
b/asterix-common/src/test/java/org/apache/asterix/test/aql/TestsUtils.java
index 8ff524e..83634dd 100644
--- a/asterix-common/src/test/java/org/apache/asterix/test/aql/TestsUtils.java
+++ b/asterix-common/src/test/java/org/apache/asterix/test/aql/TestsUtils.java
@@ -42,7 +42,6 @@
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.io.IOUtils;
import org.json.JSONObject;
-
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.testframework.context.TestCaseContext;
import org.apache.asterix.testframework.context.TestCaseContext.OutputFormat;
@@ -205,12 +204,20 @@
// In future this may be changed depending on the requested
// output format sent to the servlet.
String errorBody = method.getResponseBodyAsString();
- JSONObject result = new JSONObject(errorBody);
- String[] errors = {
result.getJSONArray("error-code").getString(0), result.getString("summary"),
- result.getString("stacktrace") };
- GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, errors[2]);
- throw new Exception("HTTP operation failed: " + errors[0] +
"\nSTATUS LINE: " + method.getStatusLine()
- + "\nSUMMARY: " + errors[1] + "\nSTACKTRACE: " +
errors[2]);
+ JSONObject result = null;
+ try {
+ result = new JSONObject(errorBody);
+ } catch (Exception e) {
+ throw new Exception(errorBody);
+ }
+ if (result != null) {
+ String[] errors = {
result.getJSONArray("error-code").getString(0), result.getString("summary"),
+ result.getString("stacktrace") };
+ GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, errors[2]);
+
+ throw new Exception("HTTP operation failed: " + errors[0] +
"\nSTATUS LINE: " + method.getStatusLine()
+ + "\nSUMMARY: " + errors[1] + "\nSTACKTRACE: " +
errors[2]);
+ }
}
return statusCode;
}
@@ -220,15 +227,14 @@
final String url = "http://localhost:19002/query";;
HttpMethodBase method = null;
- if(str.length() + url.length() < MAX_URL_LENGTH ){
+ if (str.length() + url.length() < MAX_URL_LENGTH) {
//Use GET for small-ish queries
method = new GetMethod(url);
method.setQueryString(new NameValuePair[] { new
NameValuePair("query", str) });
- }
- else{
+ } else {
//Use POST for bigger ones to avoid 413 FULL_HEAD
method = new PostMethod(url);
- ((PostMethod)method).setRequestEntity(new
StringRequestEntity(str));
+ ((PostMethod) method).setRequestEntity(new
StringRequestEntity(str));
}
//Set accepted output response type
@@ -436,7 +442,9 @@
resultStream = executeAnyAQLAsync(statement,
true, fmt);
if (queryCount >= expectedResultFileCtxs.size()) {
- throw new IllegalStateException("no result
file for " + testFile.toString() + "; queryCount: " + queryCount + ",
filectxs.size: " + expectedResultFileCtxs.size());
+ throw new IllegalStateException("no result
file for " + testFile.toString()
+ + "; queryCount: " + queryCount + ",
filectxs.size: "
+ + expectedResultFileCtxs.size());
}
expectedResultFile =
expectedResultFileCtxs.get(queryCount).getFile();
diff --git
a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index 1eda9cc..e8a9b6c 100644
---
a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++
b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -30,6 +30,8 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
+import org.apache.commons.io.FileUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.storage.common.file.ILocalResourceRepository;
@@ -50,7 +52,8 @@
private final String nodeId;
private static final int MAX_CACHED_RESOURCES = 1000;
- public PersistentLocalResourceRepository(List<IODeviceHandle> devices,
String nodeId) throws HyracksDataException {
+ public PersistentLocalResourceRepository(List<IODeviceHandle> devices,
String nodeId, SystemState systemState)
+ throws HyracksDataException {
mountPoints = new String[devices.size()];
this.nodeId = nodeId;
for (int i = 0; i < mountPoints.length; i++) {
@@ -66,6 +69,14 @@
}
}
+ if (systemState == SystemState.NEW_UNIVERSE) {
+ //delete any existing storage data from old instances
+ try {
+ deleteAsterixStorageData();
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
resourceCache =
CacheBuilder.newBuilder().maximumSize(MAX_CACHED_RESOURCES).build();
}
@@ -73,7 +84,7 @@
return mountPoint + ROOT_METADATA_DIRECTORY + File.separator + nodeId
+ "_" + "iodevice" + ioDeviceId;
}
- public void initialize(String nodeId, String rootDir) throws
HyracksDataException {
+ public void initializeNewUniverse(String nodeId, String rootDir) throws
HyracksDataException {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Initializing local resource repository ... ");
}
@@ -134,7 +145,6 @@
@Override
public synchronized void insert(LocalResource resource) throws
HyracksDataException {
File resourceFile = new File(getFileName(resource.getResourceName(),
resource.getResourceId()));
-
if (resourceFile.exists()) {
throw new HyracksDataException("Duplicate resource");
}
@@ -339,6 +349,15 @@
}
}
+ public void deleteAsterixStorageData() throws IOException {
+ for (int i = 0; i < mountPoints.length; i++) {
+ File mountingPoint = new File(mountPoints[i]);
+ if (mountingPoint.exists() && mountingPoint.isDirectory()) {
+ FileUtils.deleteDirectory(mountingPoint);
+ }
+ }
+ }
+
private static final FilenameFilter METADATA_FILES_FILTER = new
FilenameFilter() {
public boolean accept(File dir, String name) {
if (name.equalsIgnoreCase(METADATA_FILE_NAME)) {
diff --git
a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java
b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java
index b6bb7dc..d378ba5 100644
---
a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java
+++
b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.transaction.management.resource;
+import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.common.file.ILocalResourceRepository;
@@ -26,14 +27,15 @@
public class PersistentLocalResourceRepositoryFactory implements
ILocalResourceRepositoryFactory {
private final IIOManager ioManager;
private final String nodeId;
-
- public PersistentLocalResourceRepositoryFactory(IIOManager ioManager,
String nodeId) {
+ private final SystemState systemState;
+ public PersistentLocalResourceRepositoryFactory(IIOManager ioManager,
String nodeId, SystemState systemState) {
this.ioManager = ioManager;
this.nodeId = nodeId;
+ this.systemState = systemState;
}
@Override
public ILocalResourceRepository createRepository() throws
HyracksDataException {
- return new PersistentLocalResourceRepository(ioManager.getIODevices(),
nodeId);
+ return new PersistentLocalResourceRepository(ioManager.getIODevices(),
nodeId, systemState);
}
}
\ No newline at end of file
--
To view, visit https://asterix-gerrit.ics.uci.edu/469
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ibb6c6949bdf2ed6c3e491fa66a23491ff34fc830
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <[email protected]>
