This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new dc3704f [NO ISSUE][STO] Ensure component seq is always increasing
dc3704f is described below
commit dc3704f65b6290234b0296ad0b7cfe8c94900a59
Author: Murtadha Hubail <[email protected]>
AuthorDate: Mon Aug 23 22:24:12 2021 +0300
[NO ISSUE][STO] Ensure component seq is always increasing
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Initialize last used component sequence number
from the index checkpoint. This way even in the
case of a component deletion, the next written
component will use a sequence number that was
not used before.
Change-Id: I48babb9bf251c86520788942ade82ca6ded5b377
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12944
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Murtadha Hubail <[email protected]>
Reviewed-by: Ali Alsuliman <[email protected]>
---
.../org/apache/asterix/app/nc/IndexCheckpointManager.java | 6 +++++-
.../asterix/common/ioopcallbacks/LSMIOOperationCallback.java | 6 ++++++
.../messaging/CheckpointPartitionIndexesTask.java | 4 +++-
.../asterix/replication/messaging/ReplicateFileTask.java | 4 +++-
.../resource/PersistentLocalResourceRepository.java | 5 +++--
.../storage/am/lsm/common/api/ILSMIOOperationCallback.java | 5 +++++
.../storage/am/lsm/common/api/ILSMIndexFileManager.java | 7 +++++++
.../storage/am/lsm/common/impls/AbstractLSMIndex.java | 2 ++
.../am/lsm/common/impls/AbstractLSMIndexFileManager.java | 12 ++++++++----
.../storage/am/lsm/btree/EncapsulatingIoCallback.java | 5 +++++
10 files changed, 47 insertions(+), 9 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
index cbc4d94..4acc6d3 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
@@ -34,6 +34,7 @@ import
org.apache.asterix.common.storage.IIndexCheckpointManager;
import org.apache.asterix.common.storage.IndexCheckpoint;
import org.apache.asterix.common.utils.StorageConstants;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
import org.apache.hyracks.util.annotations.ThreadSafe;
import org.apache.hyracks.util.file.FileUtil;
import org.apache.logging.log4j.LogManager;
@@ -119,7 +120,10 @@ public class IndexCheckpointManager implements
IIndexCheckpointManager {
@Override
public long getValidComponentSequence() throws HyracksDataException {
- return getLatest().getValidComponentSequence();
+ if (getCheckpointCount() > 0) {
+ return getLatest().getValidComponentSequence();
+ }
+ return AbstractLSMIndexFileManager.UNINITIALIZED_COMPONENT_SEQ;
}
@Override
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationCallback.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationCallback.java
index ce6d253..1189b51 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationCallback.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMIOOperationCallback.java
@@ -294,6 +294,12 @@ public class LSMIOOperationCallback implements
ILSMIOOperationCallback {
// no op
}
+ @Override
+ public long getLastValidSequence() throws HyracksDataException {
+ ResourceReference resourceReference =
ResourceReference.ofIndex(lsmIndex.getIndexIdentifier());
+ return
indexCheckpointManagerProvider.get(resourceReference).getValidComponentSequence();
+ }
+
private boolean isMerge(ILSMIOOperation operation) {
return operation.getIOOpertionType() == LSMIOOperationType.MERGE
&& operation.getAccessor().getOpContext().getOperation() !=
IndexOperation.DELETE_COMPONENTS;
diff --git
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/CheckpointPartitionIndexesTask.java
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/CheckpointPartitionIndexesTask.java
index 767eb76..dac4a70 100644
---
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/CheckpointPartitionIndexesTask.java
+++
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/CheckpointPartitionIndexesTask.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.replication.messaging;
+import static
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager.UNINITIALIZED_COMPONENT_SEQ;
+
import java.io.DataInput;
import java.io.DataOutputStream;
import java.io.IOException;
@@ -72,7 +74,7 @@ public class CheckpointPartitionIndexesTask implements
IReplicaTask {
throw HyracksDataException
.create(new IOException(indexPath + " is not a
directory or an IO Error occurred"));
}
- long maxComponentSequence = Long.MIN_VALUE;
+ long maxComponentSequence = UNINITIALIZED_COMPONENT_SEQ;
for (String file : files) {
maxComponentSequence =
Math.max(maxComponentSequence,
IndexComponentFileReference.of(file).getSequenceEnd());
diff --git
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
index 5f7bee6..44c7bea 100644
---
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
+++
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.replication.messaging;
+import static
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager.UNINITIALIZED_COMPONENT_SEQ;
+
import java.io.DataInput;
import java.io.DataOutputStream;
import java.io.IOException;
@@ -100,7 +102,7 @@ public class ReplicateFileTask implements IReplicaTask {
final IIndexCheckpointManager indexCheckpointManager =
checkpointManagerProvider.get(indexRef);
final long currentLSN =
appCtx.getTransactionSubsystem().getLogManager().getAppendLSN();
indexCheckpointManager.delete();
- indexCheckpointManager.init(Long.MIN_VALUE, currentLSN,
+ indexCheckpointManager.init(UNINITIALIZED_COMPONENT_SEQ, currentLSN,
LSMComponentId.EMPTY_INDEX_LAST_COMPONENT_ID.getMaxId());
LOGGER.info(() -> "Checkpoint index: " + indexRef);
}
diff --git
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index f753868..a73b71a 100644
---
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -23,6 +23,7 @@ import static
org.apache.asterix.common.utils.StorageConstants.INDEX_CHECKPOINT_
import static
org.apache.asterix.common.utils.StorageConstants.METADATA_FILE_NAME;
import static org.apache.hyracks.api.exceptions.ErrorCode.CANNOT_CREATE_FILE;
import static
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager.COMPONENT_FILES_FILTER;
+import static
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager.UNINITIALIZED_COMPONENT_SEQ;
import java.io.File;
import java.io.FilenameFilter;
@@ -198,8 +199,8 @@ public class PersistentLocalResourceRepository implements
ILocalResourceReposito
createResourceFileMask(resourceFile);
byte[] bytes =
OBJECT_MAPPER.writeValueAsBytes(resource.toJson(persistedResourceRegistry));
FileUtil.writeAndForce(Paths.get(resourceFile.getAbsolutePath()),
bytes);
-
indexCheckpointManagerProvider.get(DatasetResourceReference.of(resource)).init(Long.MIN_VALUE,
0,
- LSMComponentId.EMPTY_INDEX_LAST_COMPONENT_ID.getMaxId());
+
indexCheckpointManagerProvider.get(DatasetResourceReference.of(resource)).init(UNINITIALIZED_COMPONENT_SEQ,
+ 0,
LSMComponentId.EMPTY_INDEX_LAST_COMPONENT_ID.getMaxId());
deleteResourceFileMask(resourceFile);
} catch (Exception e) {
cleanup(resourceFile);
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationCallback.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationCallback.java
index e448ae0..a778a4c 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationCallback.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperationCallback.java
@@ -19,6 +19,7 @@
package org.apache.hyracks.storage.am.lsm.common.api;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
public interface ILSMIOOperationCallback {
@@ -80,4 +81,8 @@ public interface ILSMIOOperationCallback {
* the allocated component
*/
void allocated(ILSMMemoryComponent component) throws HyracksDataException;
+
+ default long getLastValidSequence() throws HyracksDataException {
+ return AbstractLSMIndexFileManager.UNINITIALIZED_COMPONENT_SEQ;
+ }
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
index 5580387..2d38758 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
@@ -78,4 +78,11 @@ public interface ILSMIndexFileManager {
* @throws IOException
*/
LSMComponentFileReferences getNewTransactionFileReference() throws
IOException;
+
+ /**
+ * Initializes the last used sequence
+ *
+ * @param lastUsedSeq
+ */
+ void initLastUsedSeq(long lastUsedSeq);
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index a5ff2e6..bf93dc0 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -139,6 +139,7 @@ public abstract class AbstractLSMIndex implements ILSMIndex
{
this.inactiveMemoryComponents = new ArrayList<>();
this.durable = durable;
this.tracer = tracer;
+ fileManager.initLastUsedSeq(ioOpCallback.getLastValidSequence());
lsmHarness = new LSMHarness(this, ioScheduler, mergePolicy, opTracker,
diskBufferCache.isReplicationEnabled(),
tracer);
isActive = false;
@@ -184,6 +185,7 @@ public abstract class AbstractLSMIndex implements ILSMIndex
{
filterManager = null;
treeFields = null;
filterFields = null;
+ fileManager.initLastUsedSeq(ioOpCallback.getLastValidSequence());
}
@Override
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
index 35a11ff..5d70337 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
@@ -83,20 +83,19 @@ public abstract class AbstractLSMIndexFileManager
implements ILSMIndexFileManage
* Hides transaction components until they are either committed by
removing this file or deleted along with the file
*/
public static final String TXN_PREFIX = ".T";
-
+ public static final long UNINITIALIZED_COMPONENT_SEQ = -1;
public static final FilenameFilter COMPONENT_FILES_FILTER = (dir, name) ->
!name.startsWith(".");
protected static final FilenameFilter txnFileNameFilter = (dir, name) ->
name.startsWith(TXN_PREFIX);
protected static FilenameFilter bloomFilterFilter =
(dir, name) -> !name.startsWith(".") &&
name.endsWith(BLOOM_FILTER_SUFFIX);
protected static final Comparator<String> cmp = new FileNameComparator();
private static final FilenameFilter dummyFilter = (dir, name) -> true;
- private static final long UNINITALIZED_COMPONENT_SEQ = -1;
protected final IIOManager ioManager;
// baseDir should reflect dataset name and partition name and be absolute
protected final FileReference baseDir;
protected final Comparator<IndexComponentFileReference> recencyCmp = new
RecencyComparator();
protected final TreeIndexFactory<? extends ITreeIndex> treeFactory;
- private long lastUsedComponentSeq = UNINITALIZED_COMPONENT_SEQ;
+ private long lastUsedComponentSeq = UNINITIALIZED_COMPONENT_SEQ;
private final ICompressorDecompressorFactory compressorDecompressorFactory;
public AbstractLSMIndexFileManager(IIOManager ioManager, FileReference
file,
@@ -348,6 +347,11 @@ public abstract class AbstractLSMIndexFileManager
implements ILSMIndexFileManage
return null;
}
+ @Override
+ public void initLastUsedSeq(long lastUsedSeq) {
+ lastUsedComponentSeq = lastUsedSeq;
+ }
+
private static FilenameFilter createTransactionFilter(String
transactionFileName, final boolean inclusive) {
final String timeStamp =
transactionFileName.substring(transactionFileName.indexOf(TXN_PREFIX) +
TXN_PREFIX.length());
@@ -372,7 +376,7 @@ public abstract class AbstractLSMIndexFileManager
implements ILSMIndexFileManage
}
protected String getNextComponentSequence(FilenameFilter filenameFilter)
throws HyracksDataException {
- if (lastUsedComponentSeq == UNINITALIZED_COMPONENT_SEQ) {
+ if (lastUsedComponentSeq == UNINITIALIZED_COMPONENT_SEQ) {
lastUsedComponentSeq =
getOnDiskLastUsedComponentSequence(filenameFilter);
}
return
IndexComponentFileReference.getFlushSequence(++lastUsedComponentSeq);
diff --git
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/EncapsulatingIoCallback.java
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/EncapsulatingIoCallback.java
index cb791c5..7e0ee35 100644
---
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/EncapsulatingIoCallback.java
+++
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/EncapsulatingIoCallback.java
@@ -100,6 +100,11 @@ public class EncapsulatingIoCallback implements
ILSMIOOperationCallback {
encapsulated.allocated(component);
}
+ @Override
+ public long getLastValidSequence() throws HyracksDataException {
+ return encapsulated.getLastValidSequence();
+ }
+
public ILSMIOOperationCallback getEncapsulated() {
return encapsulated;
}
