Murtadha Hubail has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1270
Change subject: Add Checkpoint Test
......................................................................
Add Checkpoint Test
This change adds a unit test case which validates that
checkpoints do not delete log files that are still required
for recovery, and delete those that are no longer needed.
Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
---
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestHelper.java
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java
A
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java
M
asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
6 files changed, 282 insertions(+), 44 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/70/1270/1
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
index a548b3a..bce556d 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
@@ -32,11 +32,8 @@
import org.apache.asterix.common.context.TransactionSubsystemProvider;
import
org.apache.asterix.common.dataflow.AsterixLSMInsertDeleteOperatorNodePushable;
import
org.apache.asterix.common.dataflow.AsterixLSMTreeInsertDeleteOperatorDescriptor;
-import org.apache.asterix.common.exceptions.ACIDException;
-import org.apache.asterix.common.exceptions.AsterixException;
import
org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
import org.apache.asterix.common.transactions.IRecoveryManager.ResourceType;
-import org.apache.asterix.runtime.util.AsterixRuntimeComponentsProvider;
import org.apache.asterix.common.transactions.ITransactionManager;
import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
@@ -46,6 +43,7 @@
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
+import org.apache.asterix.runtime.util.AsterixRuntimeComponentsProvider;
import org.apache.asterix.test.runtime.ExecutionTestUtil;
import
org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexModificationOperationCallbackFactory;
import
org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexOperationTrackerProvider;
@@ -67,7 +65,6 @@
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.api.util.HyracksConstants;
@@ -115,8 +112,12 @@
private JobId jobId;
private long jobCounter = 0L;
private IHyracksJobletContext jobletCtx;
+ private final String testConfigFileName;
+ private final boolean runHDFS;
- public TestNodeController() throws AsterixException, HyracksException,
ACIDException {
+ public TestNodeController(String testConfigFileName, boolean runHDFS) {
+ this.testConfigFileName = testConfigFileName;
+ this.runHDFS = runHDFS;
}
public void init() throws Exception {
@@ -125,7 +126,9 @@
outdir.mkdirs();
// remove library directory
TestLibrarian.removeLibraryDir();
- ExecutionTestUtil.setUp(cleanupOnStart);
+ ExecutionTestUtil.setUp(cleanupOnStart,
+ testConfigFileName == null ? TEST_CONFIG_FILE_NAME :
testConfigFileName,
+ ExecutionTestUtil.integrationUtil, runHDFS);
} catch (Throwable th) {
th.printStackTrace();
throw th;
@@ -299,7 +302,7 @@
PrimaryIndexInfo primaryIndexInfo = new PrimaryIndexInfo(dataset,
primaryKeyTypes, recordType, metaType,
mergePolicyFactory, mergePolicyProperties, filterFields);
TreeIndexCreateOperatorDescriptor indexOpDesc =
getIndexCreateOpDesc(primaryIndexInfo);
- return getPrimaryIndexDataflowHelper(createTestContext(),
primaryIndexInfo, indexOpDesc);
+ return getPrimaryIndexDataflowHelper(createTestContext(true),
primaryIndexInfo, indexOpDesc);
}
public void createPrimaryIndex(Dataset dataset, IAType[] primaryKeyTypes,
ARecordType recordType,
@@ -308,7 +311,7 @@
PrimaryIndexInfo primaryIndexInfo = new PrimaryIndexInfo(dataset,
primaryKeyTypes, recordType, metaType,
mergePolicyFactory, mergePolicyProperties, filterFields);
TreeIndexCreateOperatorDescriptor indexOpDesc =
getIndexCreateOpDesc(primaryIndexInfo);
- LSMBTreeDataflowHelper dataflowHelper =
getPrimaryIndexDataflowHelper(createTestContext(), primaryIndexInfo,
+ LSMBTreeDataflowHelper dataflowHelper =
getPrimaryIndexDataflowHelper(createTestContext(true), primaryIndexInfo,
indexOpDesc);
dataflowHelper.create();
}
@@ -359,9 +362,11 @@
return primaryIndexTypeTraits;
}
- public IHyracksTaskContext createTestContext() throws HyracksDataException
{
+ public IHyracksTaskContext createTestContext(boolean withMessaging) throws
HyracksDataException {
IHyracksTaskContext ctx = TestUtils.create(KB32);
- TaskUtils.putInSharedMap(HyracksConstants.KEY_MESSAGE, new
VSizeFrame(ctx), ctx);
+ if (withMessaging) {
+ TaskUtils.putInSharedMap(HyracksConstants.KEY_MESSAGE, new
VSizeFrame(ctx), ctx);
+ }
ctx = Mockito.spy(ctx);
Mockito.when(ctx.getJobletContext()).thenReturn(jobletCtx);
Mockito.when(ctx.getIOManager())
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestHelper.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestHelper.java
index 5661258..f86f6ab 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestHelper.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestHelper.java
@@ -19,6 +19,7 @@
package org.apache.asterix.test.common;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -28,6 +29,13 @@
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.Unmarshaller;
+
+import org.apache.asterix.common.configuration.AsterixConfiguration;
+import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.commons.compress.utils.IOUtils;
import org.apache.commons.lang3.StringUtils;
@@ -52,11 +60,11 @@
Enumeration<? extends ZipEntry> entries = zipFile.entries();
while (entries.hasMoreElements()) {
ZipEntry entry = entries.nextElement();
- File entryDestination = new File(outputDir,
entry.getName());
+ File entryDestination = new File(outputDir,
entry.getName());
if (!entry.isDirectory()) {
entryDestination.getParentFile().mkdirs();
try (InputStream in = zipFile.getInputStream(entry);
- OutputStream out = new
FileOutputStream(entryDestination)) {
+ OutputStream out = new
FileOutputStream(entryDestination)) {
IOUtils.copy(in, out);
}
}
@@ -72,4 +80,34 @@
}
}
}
-}
+
+ public static AsterixConfiguration getConfigurations(String fileName)
+ throws IOException, JAXBException, AsterixException {
+ try (InputStream is =
TestHelper.class.getClassLoader().getResourceAsStream(fileName)) {
+ if (is != null) {
+ JAXBContext ctx =
JAXBContext.newInstance(AsterixConfiguration.class);
+ Unmarshaller unmarshaller = ctx.createUnmarshaller();
+ return (AsterixConfiguration) unmarshaller.unmarshal(is);
+ } else {
+ throw new AsterixException("Could not find configuration file
" + fileName);
+ }
+ }
+ }
+
+ public static void writeConfigurations(AsterixConfiguration ac, String
fileName)
+ throws FileNotFoundException, IOException, JAXBException {
+ File configFile = new File(fileName);
+ if (!configFile.exists()) {
+ configFile.getParentFile().mkdirs();
+ configFile.createNewFile();
+ } else {
+ configFile.delete();
+ }
+ try (FileOutputStream os = new FileOutputStream(fileName)) {
+ JAXBContext ctx =
JAXBContext.newInstance(AsterixConfiguration.class);
+ Marshaller marshaller = ctx.createMarshaller();
+ marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
+ marshaller.marshal(ac, os);
+ }
+ }
+}
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java
index a0ef31e..ea71ab8 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java
@@ -27,7 +27,6 @@
import org.apache.asterix.app.data.gen.TupleGenerator;
import org.apache.asterix.app.data.gen.TupleGenerator.GenerationFunction;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
-import org.apache.asterix.common.config.GlobalConfig;
import
org.apache.asterix.common.dataflow.AsterixLSMInsertDeleteOperatorNodePushable;
import org.apache.asterix.common.transactions.DatasetId;
import org.apache.asterix.common.transactions.ILogRecord;
@@ -63,12 +62,11 @@
public class LogMarkerTest {
- private static final String TEST_CONFIG_FILE_NAME =
"asterix-build-configuration.xml";
private static final IAType[] KEY_TYPES = { BuiltinType.AINT32 };
private static final ARecordType RECORD_TYPE = new
ARecordType("TestRecordType", new String[] { "key", "value" },
new IAType[] { BuiltinType.AINT32, BuiltinType.AINT64 }, false);
- private static final GenerationFunction[] RECORD_GEN_FUNCTION =
- { GenerationFunction.DETERMINISTIC,
GenerationFunction.DETERMINISTIC };
+ private static final GenerationFunction[] RECORD_GEN_FUNCTION = {
GenerationFunction.DETERMINISTIC,
+ GenerationFunction.DETERMINISTIC };
private static final boolean[] UNIQUE_RECORD_FIELDS = { true, false };
private static final ARecordType META_TYPE = null;
private static final GenerationFunction[] META_GEN_FUNCTION = null;
@@ -86,37 +84,20 @@
@Before
public void setUp() throws Exception {
- System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY,
TEST_CONFIG_FILE_NAME);
System.out.println("SetUp: ");
- File f = new File(System.getProperty("user.dir") + File.separator +
"target" + File.separator + "txnLogDir");
- FileUtils.deleteQuietly(f);
- System.out.println("Dir " + f.getName() + " deleted");
- f = new File(System.getProperty("user.dir") + File.separator +
"target" + File.separator + "IODevice");
- FileUtils.deleteQuietly(f);
- System.out.println("Dir " + f.getName() + " deleted");
- f = new File(System.getProperty("user.dir") + File.separator +
SPILL_AREA);
- FileUtils.deleteQuietly(f);
- System.out.println("Dir " + f.getName() + " deleted");
+ cleanUp();
}
@After
public void tearDown() throws Exception {
System.out.println("TearDown");
- File f = new File(System.getProperty("user.dir") + File.separator +
"target" + File.separator + "txnLogDir");
- FileUtils.deleteQuietly(f);
- System.out.println("Dir " + f.getName() + " deleted");
- f = new File(System.getProperty("user.dir") + File.separator +
"target" + File.separator + "IODevice");
- FileUtils.deleteQuietly(f);
- System.out.println("Dir " + f.getName() + " deleted");
- f = new File(System.getProperty("user.dir") + File.separator +
SPILL_AREA);
- FileUtils.deleteQuietly(f);
- System.out.println("Dir " + f.getName() + " deleted");
+ cleanUp();
}
@Test
public void testInsertWithSnapshot() {
try {
- TestNodeController nc = new TestNodeController();
+ TestNodeController nc = new TestNodeController(null, false);
nc.init();
Dataset dataset = new Dataset(DATAVERSE_NAME, DATASET_NAME,
DATAVERSE_NAME, DATA_TYPE_NAME,
NODE_GROUP_NAME, null, null, new
InternalDatasetDetails(null, PartitioningStrategy.HASH,
@@ -125,7 +106,7 @@
try {
nc.createPrimaryIndex(dataset, KEY_TYPES, RECORD_TYPE,
META_TYPE, new NoMergePolicyFactory(), null,
null);
- IHyracksTaskContext ctx = nc.createTestContext();
+ IHyracksTaskContext ctx = nc.createTestContext(true);
nc.newJobId();
ITransactionContext txnCtx =
nc.getTransactionManager().getTransactionContext(nc.getTxnJobId(), true);
AsterixLSMInsertDeleteOperatorNodePushable insertOp =
nc.getInsertPipeline(ctx, dataset, KEY_TYPES,
@@ -208,4 +189,16 @@
return new TestTupleCounterFrameWriter(recordDescriptor, openAnswer,
nextAnswer, flushAnswer, failAnswer,
closeAnswer, deepCopyInputFrames);
}
+
+ private void cleanUp() {
+ File f = new File(System.getProperty("user.dir") + File.separator +
"target" + File.separator + "txnLogDir");
+ FileUtils.deleteQuietly(f);
+ System.out.println("Dir " + f.getName() + " deleted");
+ f = new File(System.getProperty("user.dir") + File.separator +
"target" + File.separator + "IODevice");
+ FileUtils.deleteQuietly(f);
+ System.out.println("Dir " + f.getName() + " deleted");
+ f = new File(System.getProperty("user.dir") + File.separator +
SPILL_AREA);
+ FileUtils.deleteQuietly(f);
+ System.out.println("Dir " + f.getName() + " deleted");
+ }
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java
new file mode 100644
index 0000000..4bd295d
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.logging;
+
+import java.io.File;
+import java.util.Collections;
+
+import org.apache.asterix.app.bootstrap.TestNodeController;
+import org.apache.asterix.app.data.gen.TupleGenerator;
+import org.apache.asterix.app.data.gen.TupleGenerator.GenerationFunction;
+import org.apache.asterix.common.config.AsterixTransactionProperties;
+import org.apache.asterix.common.config.DatasetConfig.DatasetType;
+import org.apache.asterix.common.configuration.AsterixConfiguration;
+import org.apache.asterix.common.configuration.Property;
+import
org.apache.asterix.common.dataflow.AsterixLSMInsertDeleteOperatorNodePushable;
+import org.apache.asterix.common.transactions.DatasetId;
+import org.apache.asterix.common.transactions.ITransactionContext;
+import org.apache.asterix.external.util.DataflowUtils;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.InternalDatasetDetails;
+import
org.apache.asterix.metadata.entities.InternalDatasetDetails.PartitioningStrategy;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.test.common.TestHelper;
+import org.apache.asterix.transaction.management.service.logging.LogManager;
+import org.apache.commons.io.FileUtils;
+import org.apache.hyracks.api.comm.VSizeFrame;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory;
+import org.apache.hyracks.util.StorageUtil;
+import org.apache.hyracks.util.StorageUtil.StorageUnit;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class CheckpointingTest {
+
+ private static final String DEFAULT_TEST_CONFIG_FILE_NAME =
"asterix-build-configuration.xml";
+ private static final String TEST_CONFIG_FILE_NAME =
"asterix-test-configuration.xml";
+ private static final String TEST_CONFIG_PATH =
System.getProperty("user.dir") + File.separator + "target"
+ + File.separator + "config";
+ private static final String TEST_CONFIG_FILE_PATH = TEST_CONFIG_PATH +
File.separator + TEST_CONFIG_FILE_NAME;
+ private static final IAType[] KEY_TYPES = { BuiltinType.AINT32 };
+ private static final ARecordType RECORD_TYPE = new
ARecordType("TestRecordType", new String[] { "key", "value" },
+ new IAType[] { BuiltinType.AINT32, BuiltinType.AINT64 }, false);
+ private static final GenerationFunction[] RECORD_GEN_FUNCTION = {
GenerationFunction.DETERMINISTIC,
+ GenerationFunction.DETERMINISTIC };
+ private static final boolean[] UNIQUE_RECORD_FIELDS = { true, false };
+ private static final ARecordType META_TYPE = null;
+ private static final GenerationFunction[] META_GEN_FUNCTION = null;
+ private static final boolean[] UNIQUE_META_FIELDS = null;
+ private static final int[] KEY_INDEXES = { 0 };
+ private static final int[] KEY_INDICATORS = { 0 };
+ private static final int DATASET_ID = 101;
+ private static final String DATAVERSE_NAME = "TestDV";
+ private static final String DATASET_NAME = "TestDS";
+ private static final String DATA_TYPE_NAME = "DUMMY";
+ private static final String NODE_GROUP_NAME = "DEFAULT";
+ private static final int TXN_LOG_PARTITION_SIZE =
StorageUtil.getSizeInBytes(2, StorageUnit.MEGABYTE);
+
+ @Before
+ public void setUp() throws Exception {
+ System.out.println("SetUp: ");
+ cleanUp();
+ // Read default test configurations
+ AsterixConfiguration ac =
TestHelper.getConfigurations(DEFAULT_TEST_CONFIG_FILE_NAME);
+ // Set log file size to 2MB
+ ac.getProperty().add(new
Property(AsterixTransactionProperties.TXN_LOG_PARTITIONSIZE_KEY,
+ String.valueOf(TXN_LOG_PARTITION_SIZE), ""));
+ // Disable checkpointing by making checkpoint thread wait max wait time
+ ac.getProperty().add(new
Property(AsterixTransactionProperties.TXN_LOG_CHECKPOINT_POLLFREQUENCY_KEY,
+ String.valueOf(Integer.MAX_VALUE), ""));
+ // Write test config file
+ TestHelper.writeConfigurations(ac, TEST_CONFIG_FILE_PATH);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ System.out.println("TearDown");
+ cleanUp();
+ }
+
+ @Test
+ public void testDeleteOldLogFiles() {
+ try {
+ TestNodeController nc = new TestNodeController(new
File(TEST_CONFIG_FILE_PATH).getAbsolutePath(), false);
+ nc.init();
+ Dataset dataset = new Dataset(DATAVERSE_NAME, DATASET_NAME,
DATAVERSE_NAME, DATA_TYPE_NAME,
+ NODE_GROUP_NAME, null, null, new
InternalDatasetDetails(null, PartitioningStrategy.HASH,
+ Collections.emptyList(), null, null, null, false,
null, false),
+ null, DatasetType.INTERNAL, DATASET_ID, 0);
+ try {
+ nc.createPrimaryIndex(dataset, KEY_TYPES, RECORD_TYPE,
META_TYPE, new NoMergePolicyFactory(), null,
+ null);
+ IHyracksTaskContext ctx = nc.createTestContext(false);
+ nc.newJobId();
+ ITransactionContext txnCtx =
nc.getTransactionManager().getTransactionContext(nc.getTxnJobId(), true);
+ // Prepare insert operation
+ AsterixLSMInsertDeleteOperatorNodePushable insertOp =
nc.getInsertPipeline(ctx, dataset, KEY_TYPES,
+ RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(),
null, null);
+ insertOp.open();
+ TupleGenerator tupleGenerator = new
TupleGenerator(RECORD_TYPE, META_TYPE, KEY_INDEXES, KEY_INDICATORS,
+ RECORD_GEN_FUNCTION, UNIQUE_RECORD_FIELDS,
META_GEN_FUNCTION, UNIQUE_META_FIELDS);
+ VSizeFrame frame = new VSizeFrame(ctx);
+ FrameTupleAppender tupleAppender = new
FrameTupleAppender(frame);
+
+ LogManager logManager = (LogManager)
nc.getTransactionSubsystem().getLogManager();
+ // Number of log files after node startup should be one
+ int numberOfLogFiles = logManager.getLogFileIds().size();
+ Assert.assertEquals(1, numberOfLogFiles);
+
+ // Low-water mark LSN
+ long lowWaterMarkLSN =
nc.getTransactionSubsystem().getRecoveryManager().getMinFirstLSN();
+ // Low-water mark log file id
+ long initialLowWaterMarkFileId =
logManager.getLogFileId(lowWaterMarkLSN);
+ // Initial Low-water mark should be in the only available log
file
+ Assert.assertEquals(initialLowWaterMarkFileId,
logManager.getLogFileIds().get(0).longValue());
+
+ // Insert records until a new log file is created
+ while (logManager.getLogFileIds().size() != 2) {
+ ITupleReference tuple = tupleGenerator.next();
+ DataflowUtils.addTupleToFrame(tupleAppender, tuple,
insertOp);
+ }
+
+ // Check if the new low-water mark is still in the initial
low-water mark log file
+ lowWaterMarkLSN =
nc.getTransactionSubsystem().getRecoveryManager().getMinFirstLSN();
+ long currentLowWaterMarkLogFileId =
logManager.getLogFileId(lowWaterMarkLSN);
+
+ if (initialLowWaterMarkFileId == currentLowWaterMarkLogFileId)
{
+ // Checkpoint should not delete any log files
+
nc.getTransactionSubsystem().getRecoveryManager().checkpoint(false,
lowWaterMarkLSN);
+ // Make sure checkpoint did not delete any log files
+ numberOfLogFiles = logManager.getLogFileIds().size();
+ Assert.assertEquals(2, numberOfLogFiles);
+ }
+
+ // Insert records until the low-water mark is not in the
initial log file id
+ while (currentLowWaterMarkLogFileId ==
initialLowWaterMarkFileId) {
+ ITupleReference tuple = tupleGenerator.next();
+ DataflowUtils.addTupleToFrame(tupleAppender, tuple,
insertOp);
+ lowWaterMarkLSN =
nc.getTransactionSubsystem().getRecoveryManager().getMinFirstLSN();
+ currentLowWaterMarkLogFileId =
logManager.getLogFileId(lowWaterMarkLSN);
+ }
+
+ // Checkpoint should delete initialLowWaterMarkFileId
+
nc.getTransactionSubsystem().getRecoveryManager().checkpoint(false,
lowWaterMarkLSN);
+
+ // Validate initialLowWaterMarkFileId was deleted
+ for (Long fileId : logManager.getLogFileIds()) {
+ Assert.assertNotEquals(initialLowWaterMarkFileId,
fileId.longValue());
+ }
+
+ // Validate only a single log file exists
+ numberOfLogFiles = logManager.getLogFileIds().size();
+ Assert.assertEquals(1, numberOfLogFiles);
+
+ if (tupleAppender.getTupleCount() > 0) {
+ tupleAppender.write(insertOp, true);
+ }
+ insertOp.close();
+ nc.getTransactionManager().completedTransaction(txnCtx, new
DatasetId(-1), -1, true);
+ } finally {
+ nc.deInit();
+ }
+ } catch (Throwable e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ private void cleanUp() {
+ File f = new File(System.getProperty("user.dir") + File.separator +
"target" + File.separator + "txnLogDir");
+ FileUtils.deleteQuietly(f);
+ System.out.println("Dir " + f.getName() + " deleted");
+ f = new File(System.getProperty("user.dir") + File.separator +
"target" + File.separator + "IODevice");
+ FileUtils.deleteQuietly(f);
+ System.out.println("Dir " + f.getName() + " deleted");
+ f = new File(TEST_CONFIG_PATH);
+ FileUtils.deleteQuietly(f);
+ System.out.println("Dir " + f.getName() + " deleted");
+ }
+}
\ No newline at end of file
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java
index afc9e4f..0480887 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java
@@ -18,12 +18,12 @@
*/
package org.apache.asterix.common.config;
+import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
+import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE;
+
import java.util.Map;
import org.apache.hyracks.util.StorageUtil;
-
-import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
-import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE;
public class AsterixTransactionProperties extends AbstractAsterixProperties {
@@ -33,13 +33,13 @@
private static final String TXN_LOG_BUFFER_PAGESIZE_KEY =
"txn.log.buffer.pagesize";
private static final int TXN_LOG_BUFFER_PAGESIZE_DEFAULT =
StorageUtil.getSizeInBytes(128, KILOBYTE);
- private static final String TXN_LOG_PARTITIONSIZE_KEY =
"txn.log.partitionsize";
+ public static final String TXN_LOG_PARTITIONSIZE_KEY =
"txn.log.partitionsize";
private static final long TXN_LOG_PARTITIONSIZE_DEFAULT =
StorageUtil.getSizeInBytes(256L, MEGABYTE);
private static final String TXN_LOG_CHECKPOINT_LSNTHRESHOLD_KEY =
"txn.log.checkpoint.lsnthreshold";
private static final int TXN_LOG_CHECKPOINT_LSNTHRESHOLD_DEFAULT =
StorageUtil.getSizeInBytes(64, MEGABYTE);
- private static final String TXN_LOG_CHECKPOINT_POLLFREQUENCY_KEY =
"txn.log.checkpoint.pollfrequency";
+ public static final String TXN_LOG_CHECKPOINT_POLLFREQUENCY_KEY =
"txn.log.checkpoint.pollfrequency";
private static final int TXN_LOG_CHECKPOINT_POLLFREQUENCY_DEFAULT = 120;
// 120s
private static final String TXN_LOG_CHECKPOINT_HISTORY_KEY =
"txn.log.checkpoint.history";
diff --git
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
index 9a66aa5..c234475 100644
---
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
+++
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogManager.java
@@ -421,7 +421,7 @@
}
}
- private List<Long> getLogFileIds() {
+ public List<Long> getLogFileIds() {
File fileLogDir = new File(logDir);
String[] logFileNames = null;
List<Long> logFileIds = null;
--
To view, visit https://asterix-gerrit.ics.uci.edu/1270
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I4cb4743fe488deb5ad10f65604adc2231948795e
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <[email protected]>
