This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit b7ba0f64f068dd524aae994bfbc01f6b99d1f003 Author: Ritik Raj <[email protected]> AuthorDate: Wed Nov 19 12:11:37 2025 +0530 [ASTERIXDB-3652][STO] Restore the old schema on abort - user model changes: no - storage format changes: no - interface changes: no Details: After abort, restore the previous schema for the index. Ext-ref: MB-69414 Change-Id: I97ba7dbc39241ffba2554ea92ee49be608b28cde Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20588 Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> --- ...query.sqlpp => ASTERIXDB-3652.004.update.sqlpp} | 3 +- ....query.sqlpp => ASTERIXDB-3652.005.query.sqlpp} | 0 .../ASTERIXDB-3652.004.update.sqlpp} | 3 +- ....query.sqlpp => ASTERIXDB-3652.005.query.sqlpp} | 0 ...TERIXDB-3652.004.adm => ASTERIXDB-3652.005.adm} | 0 ...TERIXDB-3652.004.adm => ASTERIXDB-3652.005.adm} | 0 .../metadata/AbstractColumnImmutableMetadata.java | 5 ++++ .../column/metadata/AbstractColumnMetadata.java | 2 +- .../column/metadata/schema/ObjectSchemaNode.java | 14 ++++----- .../operation/lsm/flush/ColumnTransformer.java | 2 +- .../operation/lsm/flush/FlushColumnMetadata.java | 33 ++++++++++++++++++---- .../lsm/flush/NoWriteColumnTransformer.java | 2 +- .../data/std/util/ArrayBackedValueStorage.java | 9 +++++- .../am/lsm/btree/column/api/IColumnMetadata.java | 6 ++++ .../column/impls/lsm/LSMColumnIndexBulkloader.java | 1 + 15 files changed, 58 insertions(+), 22 deletions(-) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-4/ASTERIXDB-3652.004.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-4/ASTERIXDB-3652.004.update.sqlpp similarity index 95% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-4/ASTERIXDB-3652.004.query.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-4/ASTERIXDB-3652.004.update.sqlpp index c368f8ae80..075d204e2d 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-4/ASTERIXDB-3652.004.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-4/ASTERIXDB-3652.004.update.sqlpp @@ -19,5 +19,4 @@ USE test; -SELECT VALUE c -FROM ColumnDataset c; \ No newline at end of file +COMPACT DATASET ColumnDataset; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-4/ASTERIXDB-3652.004.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-4/ASTERIXDB-3652.005.query.sqlpp similarity index 100% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-4/ASTERIXDB-3652.004.query.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-4/ASTERIXDB-3652.005.query.sqlpp diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-4/ASTERIXDB-3652.004.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-5/ASTERIXDB-3652.004.update.sqlpp similarity index 95% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-4/ASTERIXDB-3652.004.query.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-5/ASTERIXDB-3652.004.update.sqlpp index c368f8ae80..075d204e2d 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-4/ASTERIXDB-3652.004.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-5/ASTERIXDB-3652.004.update.sqlpp @@ -19,5 +19,4 @@ USE test; -SELECT VALUE c -FROM ColumnDataset c; \ No newline at end of file +COMPACT DATASET ColumnDataset; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-5/ASTERIXDB-3652.004.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-5/ASTERIXDB-3652.005.query.sqlpp similarity index 100% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-5/ASTERIXDB-3652.004.query.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/ASTERIXDB-3652-5/ASTERIXDB-3652.005.query.sqlpp diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/assembly/ASTERIXDB-3652-4/ASTERIXDB-3652.004.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/assembly/ASTERIXDB-3652-4/ASTERIXDB-3652.005.adm similarity index 100% rename from asterixdb/asterix-app/src/test/resources/runtimets/results/column/assembly/ASTERIXDB-3652-4/ASTERIXDB-3652.004.adm rename to asterixdb/asterix-app/src/test/resources/runtimets/results/column/assembly/ASTERIXDB-3652-4/ASTERIXDB-3652.005.adm diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/assembly/ASTERIXDB-3652-5/ASTERIXDB-3652.004.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/assembly/ASTERIXDB-3652-5/ASTERIXDB-3652.005.adm similarity index 100% rename from asterixdb/asterix-app/src/test/resources/runtimets/results/column/assembly/ASTERIXDB-3652-5/ASTERIXDB-3652.004.adm rename to asterixdb/asterix-app/src/test/resources/runtimets/results/column/assembly/ASTERIXDB-3652-5/ASTERIXDB-3652.005.adm diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/AbstractColumnImmutableMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/AbstractColumnImmutableMetadata.java index c7b4651bdb..9754c7556d 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/AbstractColumnImmutableMetadata.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/AbstractColumnImmutableMetadata.java @@ -38,6 +38,11 @@ public abstract class AbstractColumnImmutableMetadata extends AbstractColumnMeta return serializedMetadata; } + @Override + public final void swapSerializedColumnsMetadata() { + // since the metadata is immutable + } + @Override public final void abort() throws HyracksDataException { //NoOp as the metadata is immutable diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/AbstractColumnMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/AbstractColumnMetadata.java index 2e95384e5b..3637a29b62 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/AbstractColumnMetadata.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/AbstractColumnMetadata.java @@ -63,4 +63,4 @@ public abstract class AbstractColumnMetadata implements IColumnMetadata { @Override public abstract int getNumberOfColumns(); -} \ No newline at end of file +} diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/ObjectSchemaNode.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/ObjectSchemaNode.java index 59a3edde4c..6b90ade961 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/ObjectSchemaNode.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/ObjectSchemaNode.java @@ -49,7 +49,6 @@ public final class ObjectSchemaNode extends AbstractSchemaNestedNode { private final Int2IntMap fieldNameIndexToChildIndexMap; private final List<AbstractSchemaNode> children; private IntUnaryOperator nextIndex; - private boolean isEmptyObject = false; public ObjectSchemaNode() { fieldNameIndexToChildIndexMap = new Int2IntOpenHashMap(); @@ -81,7 +80,7 @@ public final class ObjectSchemaNode extends AbstractSchemaNestedNode { FlushColumnMetadata columnMetadata) throws HyracksDataException { int numberOfChildren = children.size(); int fieldNameIndex = columnMetadata.getFieldNamesDictionary().getOrCreateFieldNameIndex(fieldName); - boolean previouslyMissing = isEmptyObject; + boolean previouslyMissing = isEmptyMissingObject(); int childIndex = fieldNameIndexToChildIndexMap.getOrDefault(fieldNameIndex, nextIndex.apply(fieldNameIndex)); AbstractSchemaNode currentChild = childIndex == numberOfChildren ? null : children.get(childIndex); AbstractSchemaNode newChild = columnMetadata.getOrCreateChild(currentChild, childTypeTag, previouslyMissing); @@ -111,13 +110,17 @@ public final class ObjectSchemaNode extends AbstractSchemaNestedNode { if (!children.isEmpty()) { return null; } - isEmptyObject = true; + AbstractSchemaNode emptyChild = columnMetadata.getOrCreateChild(null, ATypeTag.MISSING, false); addChild(DUMMY_FIELD_NAME_INDEX, emptyChild); nextIndex = this::emptyColumnIndex; return emptyChild; } + public boolean isEmptyMissingObject() { + return children.size() == 1 && fieldNameIndexToChildIndexMap.containsKey(DUMMY_FIELD_NAME_INDEX); + } + public AbstractSchemaNode getChild(int fieldNameIndex) { if (fieldNameIndexToChildIndexMap.containsKey(fieldNameIndex)) { return children.get(fieldNameIndexToChildIndexMap.get(fieldNameIndex)); @@ -217,11 +220,6 @@ public final class ObjectSchemaNode extends AbstractSchemaNestedNode { nextIndex = this::nextIndex; fieldNameIndexToChildIndexMap.remove(DUMMY_FIELD_NAME_INDEX); fieldNameIndexToChildIndexMap.put(fieldNameIndex, 0); - isEmptyObject = false; return 0; } - - public boolean isEmptyObject() { - return isEmptyObject; - } } diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/ColumnTransformer.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/ColumnTransformer.java index 7b1a85bb05..d146872fa5 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/ColumnTransformer.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/ColumnTransformer.java @@ -130,7 +130,7 @@ public class ColumnTransformer implements ILazyVisitablePointableVisitor<Abstrac if (pointable.getNumberOfChildren() == 0) { // Set as empty object objectNode.setEmptyObject(columnMetadata); - if (!objectNode.isMissingInitiallyInBatch() && objectNode.isEmptyObject()) { + if (!objectNode.isMissingInitiallyInBatch() && objectNode.isEmptyMissingObject()) { objectNode.needAllColumns(true); objectNode.setMissingInitiallyInBatch(true); PrimitiveSchemaNode missingNode = (PrimitiveSchemaNode) objectNode.getChildren().get(0); diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java index 23032350e3..17a0607bac 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java @@ -82,10 +82,12 @@ public class FlushColumnMetadata extends AbstractColumnMetadata { private final IColumnValuesWriterFactory columnWriterFactory; protected final List<IColumnValuesWriter> columnWriters; private final ArrayBackedValueStorage serializedMetadata; + private final ArrayBackedValueStorage tempSerializedMetadata; private final PathInfoSerializer pathInfoSerializer; protected final IntArrayList nullWriterIndexes; private final boolean metaContainsKeys; private boolean changed; + private boolean schemaEvolved; protected int level; protected int repeated; protected int requiredTemporaryBuffersCount; @@ -125,8 +127,11 @@ public class FlushColumnMetadata extends AbstractColumnMetadata { } serializedMetadata = new ArrayBackedValueStorage(); + tempSerializedMetadata = new ArrayBackedValueStorage(); + schemaEvolved = false; changed = true; serializeColumnsMetadata(); + swapSerializedColumnsMetadata(); } public FlushColumnMetadata(ARecordType datasetType, ARecordType metaType, int numPrimaryKeys, @@ -153,6 +158,8 @@ public class FlushColumnMetadata extends AbstractColumnMetadata { //Add definition levels for the root addDefinitionLevelsAndGet(root); this.serializedMetadata = serializedMetadata; + this.tempSerializedMetadata = new ArrayBackedValueStorage(); + this.schemaEvolved = false; changed = false; } @@ -178,17 +185,20 @@ public class FlushColumnMetadata extends AbstractColumnMetadata { try { serializeChanges(); logSchema(root, metaRoot, fieldNamesDictionary); + schemaEvolved = true; changed = false; } catch (IOException e) { throw HyracksDataException.create(e); } + } else if (!schemaEvolved) { + return serializedMetadata; } - return serializedMetadata; + return tempSerializedMetadata; } private void serializeChanges() throws IOException { - serializedMetadata.reset(); - DataOutput output = serializedMetadata.getDataOutput(); + tempSerializedMetadata.reset(); + DataOutput output = tempSerializedMetadata.getDataOutput(); int writersOffsetPointer = reserveInt(output); int fieldNamesOffsetPointer = reserveInt(output); @@ -222,15 +232,24 @@ public class FlushColumnMetadata extends AbstractColumnMetadata { pathInfoSerializer.serialize(output, getNumberOfColumns()); } + @Override + public void swapSerializedColumnsMetadata() { + if (schemaEvolved) { + ArrayBackedValueStorage.swap(serializedMetadata, tempSerializedMetadata); + tempSerializedMetadata.reset(); + schemaEvolved = false; + } + } + private int reserveInt(DataOutput output) throws IOException { - int offset = serializedMetadata.getLength(); + int offset = tempSerializedMetadata.getLength(); output.writeInt(-1); return offset; } private void setOffset(int pointer) { - int offset = serializedMetadata.getLength(); - IntegerPointable.setInteger(serializedMetadata.getByteArray(), pointer, offset); + int offset = tempSerializedMetadata.getLength(); + IntegerPointable.setInteger(tempSerializedMetadata.getByteArray(), pointer, offset); } public static FlushColumnMetadata create(ARecordType datasetType, ARecordType metaType, int numPrimaryKeys, @@ -283,6 +302,8 @@ public class FlushColumnMetadata extends AbstractColumnMetadata { serializedMetadata.getStartOffset(), serializedMetadata.getLength())); try { abort(input); + schemaEvolved = false; + changed = false; } catch (IOException e) { throw HyracksDataException.create(e); } diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/NoWriteColumnTransformer.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/NoWriteColumnTransformer.java index 89d812b316..f67ce1c0a8 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/NoWriteColumnTransformer.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/NoWriteColumnTransformer.java @@ -110,7 +110,7 @@ public class NoWriteColumnTransformer if (pointable.getNumberOfChildren() == 0) { // Set as empty object AbstractSchemaNode missingChild = objectNode.setEmptyObject(columnMetadata); - if (!objectNode.isMissingInitiallyInBatch() && objectNode.isEmptyObject()) { + if (!objectNode.isMissingInitiallyInBatch() && objectNode.isEmptyMissingObject()) { objectNode.needAllColumns(true); // to include the missing column, while finalizing the batch. objectNode.setMissingInitiallyInBatch(true); if (missingChild != null) { diff --git a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ArrayBackedValueStorage.java b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ArrayBackedValueStorage.java index d4feff60a6..02af029b3a 100644 --- a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ArrayBackedValueStorage.java +++ b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ArrayBackedValueStorage.java @@ -29,7 +29,7 @@ import org.apache.hyracks.data.std.api.IValueReference; public class ArrayBackedValueStorage implements IMutableValueStorage, IPointable { - private final GrowableArray data; + private GrowableArray data; public ArrayBackedValueStorage(int size) { data = new GrowableArray(size); @@ -39,6 +39,13 @@ public class ArrayBackedValueStorage implements IMutableValueStorage, IPointable data = new GrowableArray(); } + public static void swap(ArrayBackedValueStorage serializedMetadata, + ArrayBackedValueStorage tempSerializedMetadata) { + GrowableArray temp = serializedMetadata.data; + serializedMetadata.data = tempSerializedMetadata.data; + tempSerializedMetadata.data = temp; + } + @Override public void reset() { data.reset(); diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnMetadata.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnMetadata.java index f11ef9af57..0e970b6413 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnMetadata.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/IColumnMetadata.java @@ -42,4 +42,10 @@ public interface IColumnMetadata { * abort in case of an error. This should clean up any artifact */ void abort() throws HyracksDataException; + + /** + * make the temporary schema the persistent schema + * This happens after the flush is successful. + */ + void swapSerializedColumnsMetadata(); } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnIndexBulkloader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnIndexBulkloader.java index ba41227626..ebd9b63ceb 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnIndexBulkloader.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnIndexBulkloader.java @@ -40,6 +40,7 @@ public class LSMColumnIndexBulkloader extends LSMIndexBulkLoader { public void end() throws HyracksDataException { ColumnUtil.putColumnsMetadataValue(columnMetadata.serializeColumnsMetadata(), componentMetadata); super.end(); + columnMetadata.swapSerializedColumnsMetadata(); } @Override
