This is an automated email from the ASF dual-hosted git repository.
rpuch pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new 267ac43992 IGNITE-23473 Do not use ByteUtils#toBytes to persist
IndexMeta (#4583)
267ac43992 is described below
commit 267ac439925df886d1a885fc346355aca467ce35
Author: Roman Puchkovskiy <[email protected]>
AuthorDate: Fri Oct 18 16:51:38 2024 +0400
IGNITE-23473 Do not use ByteUtils#toBytes to persist IndexMeta (#4583)
---
.../table/distributed/index/IndexMeta.java | 11 +--
.../distributed/index/IndexMetaSerializer.java | 91 ++++++++++++++++++++++
.../table/distributed/index/IndexMetaStorage.java | 7 +-
.../table/distributed/index/MetaIndexStatus.java | 61 +++++++++++++--
.../distributed/index/MetaIndexStatusChange.java | 5 +-
.../index/BaseIndexMetaStorageTest.java | 3 +-
.../distributed/index/IndexMetaSerializerTest.java | 79 +++++++++++++++++++
.../distributed/index/MetaIndexStatusTest.java | 35 +++++++++
8 files changed, 270 insertions(+), 22 deletions(-)
diff --git
a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/index/IndexMeta.java
b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/index/IndexMeta.java
index d973b35e8d..ffd3f0cf0e 100644
---
a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/index/IndexMeta.java
+++
b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/index/IndexMeta.java
@@ -19,7 +19,6 @@ package org.apache.ignite.internal.table.distributed.index;
import static java.util.Collections.unmodifiableMap;
-import java.io.Serializable;
import java.util.EnumMap;
import java.util.Map;
import org.apache.ignite.internal.catalog.Catalog;
@@ -30,21 +29,19 @@ import org.apache.ignite.internal.tostring.S;
import org.jetbrains.annotations.Nullable;
/** Immutable index meta, based on the {@link CatalogIndexDescriptor}. */
-public class IndexMeta implements Serializable {
- private static final long serialVersionUID = 1044129530453957897L;
-
+public class IndexMeta {
private final int catalogVersion;
private final int indexId;
private final int tableId;
+ private final int tableVersionOnIndexCreation;
+
private final String indexName;
private final MetaIndexStatus currentStatus;
- private final int tableVersionOnIndexCreation;
-
@IgniteToStringInclude
private final Map<MetaIndexStatus, MetaIndexStatusChange> statusChanges;
@@ -59,7 +56,7 @@ public class IndexMeta implements Serializable {
* @param currentStatus Current status of the index
* @param statusChanges <b>Immutable</b> map of index statuses with change
info (for example catalog version) in which they appeared.
*/
- private IndexMeta(
+ IndexMeta(
int catalogVersion,
int indexId,
int tableId,
diff --git
a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/index/IndexMetaSerializer.java
b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/index/IndexMetaSerializer.java
new file mode 100644
index 0000000000..0462bcfb94
--- /dev/null
+++
b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/index/IndexMetaSerializer.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.table.distributed.index;
+
+import java.io.IOException;
+import java.util.EnumMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import org.apache.ignite.internal.util.io.IgniteDataInput;
+import org.apache.ignite.internal.util.io.IgniteDataOutput;
+import org.apache.ignite.internal.versioned.VersionedSerializer;
+
+/**
+ * {@link VersionedSerializer} for {@link IndexMeta} instances.
+ */
+class IndexMetaSerializer extends VersionedSerializer<IndexMeta> {
+ static final IndexMetaSerializer INSTANCE = new IndexMetaSerializer();
+
+ @Override
+ protected void writeExternalData(IndexMeta meta, IgniteDataOutput out)
throws IOException {
+ out.writeVarInt(meta.catalogVersion());
+ out.writeVarInt(meta.indexId());
+ out.writeVarInt(meta.tableId());
+ out.writeVarInt(meta.tableVersion());
+ out.writeUTF(meta.indexName());
+ out.writeVarInt(meta.status().code());
+
+ out.writeVarInt(meta.statusChanges().size());
+ for (Entry<MetaIndexStatus, MetaIndexStatusChange> entry :
meta.statusChanges().entrySet()) {
+ out.writeVarInt(entry.getKey().code());
+
+ MetaIndexStatusChange change = entry.getValue();
+ out.writeVarInt(change.catalogVersion());
+ // Writing long and not varlong as the latter requires 9 bytes.
+ out.writeLong(change.activationTimestamp());
+ }
+ }
+
+ @Override
+ protected IndexMeta readExternalData(byte protoVer, IgniteDataInput in)
throws IOException {
+ int catalogVersion = in.readVarIntAsInt();
+ int indexId = in.readVarIntAsInt();
+ int tableId = in.readVarIntAsInt();
+ int tableVersion = in.readVarIntAsInt();
+ String indexName = in.readUTF();
+ MetaIndexStatus status =
MetaIndexStatus.findByCode(in.readVarIntAsInt());
+ Map<MetaIndexStatus, MetaIndexStatusChange> statusChanges =
readStatusChanges(in);
+
+ return new IndexMeta(
+ catalogVersion,
+ indexId,
+ tableId,
+ tableVersion,
+ indexName,
+ status,
+ statusChanges
+ );
+ }
+
+ private static Map<MetaIndexStatus, MetaIndexStatusChange>
readStatusChanges(IgniteDataInput in) throws IOException {
+ int length = in.readVarIntAsInt();
+
+ Map<MetaIndexStatus, MetaIndexStatusChange> map = new
EnumMap<>(MetaIndexStatus.class);
+ for (int i = 0; i < length; i++) {
+ MetaIndexStatus status =
MetaIndexStatus.findByCode(in.readVarIntAsInt());
+
+ int catalogVersion = in.readVarIntAsInt();
+ long activationTimestamp = in.readLong();
+ MetaIndexStatusChange change = new
MetaIndexStatusChange(catalogVersion, activationTimestamp);
+
+ map.put(status, change);
+ }
+
+ return map;
+ }
+}
diff --git
a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/index/IndexMetaStorage.java
b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/index/IndexMetaStorage.java
index 34feaed67e..afaf721fef 100644
---
a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/index/IndexMetaStorage.java
+++
b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/index/IndexMetaStorage.java
@@ -40,7 +40,6 @@ import static
org.apache.ignite.internal.table.distributed.index.MetaIndexStatus
import static
org.apache.ignite.internal.table.distributed.index.MetaIndexStatus.REMOVED;
import static
org.apache.ignite.internal.table.distributed.index.MetaIndexStatus.statusOnRemoveIndex;
import static org.apache.ignite.internal.util.ByteUtils.intToBytesKeepingOrder;
-import static org.apache.ignite.internal.util.ByteUtils.toBytes;
import static org.apache.ignite.internal.util.CollectionUtils.difference;
import static
org.apache.ignite.internal.util.CompletableFutures.falseCompletedFuture;
import static
org.apache.ignite.internal.util.CompletableFutures.nullCompletedFuture;
@@ -80,8 +79,8 @@ import org.apache.ignite.internal.manager.ComponentContext;
import org.apache.ignite.internal.manager.IgniteComponent;
import org.apache.ignite.internal.metastorage.Entry;
import org.apache.ignite.internal.metastorage.MetaStorageManager;
-import org.apache.ignite.internal.util.ByteUtils;
import org.apache.ignite.internal.util.Cursor;
+import org.apache.ignite.internal.versioned.VersionedSerialization;
import org.jetbrains.annotations.Nullable;
/**
@@ -409,7 +408,7 @@ public class IndexMetaStorage implements IgniteComponent {
return cursor.stream()
.map(Entry::value)
.filter(Objects::nonNull)
- .map(entryBytes -> (IndexMeta)
ByteUtils.fromBytes(entryBytes))
+ .map(entryBytes ->
VersionedSerialization.fromBytes(entryBytes, IndexMetaSerializer.INSTANCE))
.collect(toMap(IndexMeta::indexId, Function.identity()));
}
}
@@ -446,7 +445,7 @@ public class IndexMetaStorage implements IgniteComponent {
value(versionKey).lt(versionValue),
List.of(
put(versionKey, versionValue),
- put(indexMetaValueKey(newMeta), toBytes(newMeta))
+ put(indexMetaValueKey(newMeta),
VersionedSerialization.toBytes(newMeta, IndexMetaSerializer.INSTANCE))
),
List.of(noop())
);
diff --git
a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/index/MetaIndexStatus.java
b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/index/MetaIndexStatus.java
index c58ed21c96..9b3c162bd5 100644
---
a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/index/MetaIndexStatus.java
+++
b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/index/MetaIndexStatus.java
@@ -17,6 +17,8 @@
package org.apache.ignite.internal.table.distributed.index;
+import java.util.HashSet;
+import java.util.Set;
import org.apache.ignite.internal.catalog.descriptors.CatalogIndexStatus;
/** Index status as stored in the {@link IndexMeta}. */
@@ -26,35 +28,35 @@ public enum MetaIndexStatus {
*
* @see CatalogIndexStatus#REGISTERED
*/
- REGISTERED,
+ REGISTERED(0),
/**
* Index was in the process of being built.
*
* @see CatalogIndexStatus#BUILDING
*/
- BUILDING,
+ BUILDING(1),
/**
* Index has been successfully built and is available for reading from it.
*
* @see CatalogIndexStatus#AVAILABLE
*/
- AVAILABLE,
+ AVAILABLE(2),
/**
* Available index has started preparing to be removed from the catalog.
*
* @see CatalogIndexStatus#STOPPING
*/
- STOPPING,
+ STOPPING(3),
/**
* Index has been removed from the catalog, with {@link
CatalogIndexStatus#REGISTERED}/{@link CatalogIndexStatus#BUILDING} statuses, but
* has not yet been destroyed due to a low watermark.
*/
// TODO: IGNITE-20934 Get rid of such indexes immediately
- REMOVED,
+ REMOVED(4),
/**
* Index has been removed from the catalog, with statuses {@link
CatalogIndexStatus#AVAILABLE} (on table destruction) or
@@ -62,7 +64,54 @@ public enum MetaIndexStatus {
*
* <p>Such an index cannot be used by RW transactions, only by RO
transactions with the correct readTimestamp.</p>
*/
- READ_ONLY;
+ READ_ONLY(5);
+
+ /** Status code. It's persisted and should not be changed for existing
statuses. */
+ private final int code;
+
+ private static final MetaIndexStatus[] VALUES_INDEXED_BY_CODE;
+
+ static {
+ int maxCode = -1;
+ Set<Integer> seenCodes = new HashSet<>();
+ for (MetaIndexStatus status : values()) {
+ assert status.code >= 0 : status + " has a negative code";
+ maxCode = Math.max(maxCode, status.code);
+
+ boolean added = seenCodes.add(status.code);
+ assert added : "Duplicate status code for: " + status;
+ }
+
+ VALUES_INDEXED_BY_CODE = new MetaIndexStatus[maxCode + 1];
+ for (MetaIndexStatus status : values()) {
+ VALUES_INDEXED_BY_CODE[status.code] = status;
+ }
+ }
+
+ MetaIndexStatus(int code) {
+ this.code = code;
+ }
+
+ /**
+ * Finds status by code.
+ *
+ * @param code Code to use when searching.
+ * @throws IllegalArgumentException If there is no status with the
provided code.
+ */
+ static MetaIndexStatus findByCode(int code) {
+ MetaIndexStatus status = VALUES_INDEXED_BY_CODE[code];
+
+ if (status == null) {
+ throw new IllegalArgumentException("Unknown code [" + code + "]");
+ }
+
+ return status;
+ }
+
+ /** Returns status code (the code is persisted and should not be changed
for existing statuses). */
+ public int code() {
+ return code;
+ }
/** Converts {@link CatalogIndexStatus} to {@link MetaIndexStatus}. */
public static MetaIndexStatus convert(CatalogIndexStatus
catalogIndexStatus) {
diff --git
a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/index/MetaIndexStatusChange.java
b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/index/MetaIndexStatusChange.java
index fdfdf1a069..72040807de 100644
---
a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/index/MetaIndexStatusChange.java
+++
b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/index/MetaIndexStatusChange.java
@@ -17,14 +17,11 @@
package org.apache.ignite.internal.table.distributed.index;
-import java.io.Serializable;
import org.apache.ignite.internal.catalog.Catalog;
import org.apache.ignite.internal.tostring.S;
/** Information about {@link MetaIndexStatus index status} changes. */
-public class MetaIndexStatusChange implements Serializable {
- private static final long serialVersionUID = -2837971110813775936L;
-
+public class MetaIndexStatusChange {
private final int catalogVersion;
private final long activationTs;
diff --git
a/modules/table/src/test/java/org/apache/ignite/internal/table/distributed/index/BaseIndexMetaStorageTest.java
b/modules/table/src/test/java/org/apache/ignite/internal/table/distributed/index/BaseIndexMetaStorageTest.java
index ea97cbba65..d3f49fd88f 100644
---
a/modules/table/src/test/java/org/apache/ignite/internal/table/distributed/index/BaseIndexMetaStorageTest.java
+++
b/modules/table/src/test/java/org/apache/ignite/internal/table/distributed/index/BaseIndexMetaStorageTest.java
@@ -52,6 +52,7 @@ import
org.apache.ignite.internal.metastorage.MetaStorageManager;
import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
import org.apache.ignite.internal.util.ByteUtils;
import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.versioned.VersionedSerialization;
import org.jetbrains.annotations.Nullable;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -139,7 +140,7 @@ abstract class BaseIndexMetaStorageTest extends
BaseIgniteAbstractTest {
assertNotNull(versionBytes, "indexId=" + indexId);
int catalogVersion = ByteUtils.bytesToIntKeepingOrder(versionBytes);
- IndexMeta indexMeta = ByteUtils.fromBytes(valueBytes);
+ IndexMeta indexMeta = VersionedSerialization.fromBytes(valueBytes,
IndexMetaSerializer.INSTANCE);
assertEquals(indexMeta.catalogVersion(), catalogVersion, "indexId=" +
indexId);
diff --git
a/modules/table/src/test/java/org/apache/ignite/internal/table/distributed/index/IndexMetaSerializerTest.java
b/modules/table/src/test/java/org/apache/ignite/internal/table/distributed/index/IndexMetaSerializerTest.java
new file mode 100644
index 0000000000..c15c2be7ae
--- /dev/null
+++
b/modules/table/src/test/java/org/apache/ignite/internal/table/distributed/index/IndexMetaSerializerTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.table.distributed.index;
+
+import static
org.apache.ignite.internal.table.distributed.index.MetaIndexStatus.AVAILABLE;
+import static
org.apache.ignite.internal.table.distributed.index.MetaIndexStatus.BUILDING;
+import static
org.apache.ignite.internal.table.distributed.index.MetaIndexStatus.READ_ONLY;
+import static
org.apache.ignite.internal.table.distributed.index.MetaIndexStatus.REGISTERED;
+import static
org.apache.ignite.internal.table.distributed.index.MetaIndexStatus.REMOVED;
+import static
org.apache.ignite.internal.table.distributed.index.MetaIndexStatus.STOPPING;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+import java.util.Base64;
+import java.util.Map;
+import org.apache.ignite.internal.versioned.VersionedSerialization;
+import org.junit.jupiter.api.Test;
+
+class IndexMetaSerializerTest {
+ private final IndexMetaSerializer serializer = new IndexMetaSerializer();
+
+ @Test
+ void serializationAndDeserialization() {
+ IndexMeta originalMeta = new IndexMeta(1000, 2000, 3000, 4000,
"index", READ_ONLY, originalStatusChanges());
+
+ byte[] bytes = VersionedSerialization.toBytes(originalMeta,
serializer);
+ IndexMeta restoredMeta = VersionedSerialization.fromBytes(bytes,
serializer);
+
+ assertThat(restoredMeta.catalogVersion(), is(1000));
+ assertThat(restoredMeta.indexId(), is(2000));
+ assertThat(restoredMeta.tableId(), is(3000));
+ assertThat(restoredMeta.tableVersion(), is(4000));
+ assertThat(restoredMeta.indexName(), is("index"));
+ assertThat(restoredMeta.status(), is(READ_ONLY));
+ assertThat(restoredMeta.statusChanges(),
equalTo(originalStatusChanges()));
+ }
+
+ private static Map<MetaIndexStatus, MetaIndexStatusChange>
originalStatusChanges() {
+ return Map.of(
+ REGISTERED, new MetaIndexStatusChange(1, 100),
+ BUILDING, new MetaIndexStatusChange(2, 200),
+ AVAILABLE, new MetaIndexStatusChange(3, 300),
+ STOPPING, new MetaIndexStatusChange(4, 400),
+ REMOVED, new MetaIndexStatusChange(5, 500),
+ READ_ONLY, new MetaIndexStatusChange(6, 600)
+ );
+ }
+
+ @Test
+ void v1CanBeDeserialized() {
+ byte[] bytes =
Base64.getDecoder().decode("Ae++Q+kH0Q+5F6EfBmluZGV4BgcCA8gAAAAAAAAAAwQsAQAAAAAAAAECZAAAAAAAAAAFBvQBAAAAAAAABAWQ"
+ + "AQAAAAAAAAYHWAIAAAAAAAA=");
+ IndexMeta restoredMeta = VersionedSerialization.fromBytes(bytes,
serializer);
+
+ assertThat(restoredMeta.catalogVersion(), is(1000));
+ assertThat(restoredMeta.indexId(), is(2000));
+ assertThat(restoredMeta.tableId(), is(3000));
+ assertThat(restoredMeta.tableVersion(), is(4000));
+ assertThat(restoredMeta.indexName(), is("index"));
+ assertThat(restoredMeta.status(), is(READ_ONLY));
+ assertThat(restoredMeta.statusChanges(),
equalTo(originalStatusChanges()));
+ }
+}
diff --git
a/modules/table/src/test/java/org/apache/ignite/internal/table/distributed/index/MetaIndexStatusTest.java
b/modules/table/src/test/java/org/apache/ignite/internal/table/distributed/index/MetaIndexStatusTest.java
new file mode 100644
index 0000000000..733cd167b0
--- /dev/null
+++
b/modules/table/src/test/java/org/apache/ignite/internal/table/distributed/index/MetaIndexStatusTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.table.distributed.index;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+import org.junit.jupiter.api.Test;
+
+class MetaIndexStatusTest {
+ @Test
+ void codesAreStable() {
+ assertThat(MetaIndexStatus.REGISTERED.code(), is(0));
+ assertThat(MetaIndexStatus.BUILDING.code(), is(1));
+ assertThat(MetaIndexStatus.AVAILABLE.code(), is(2));
+ assertThat(MetaIndexStatus.STOPPING.code(), is(3));
+ assertThat(MetaIndexStatus.REMOVED.code(), is(4));
+ assertThat(MetaIndexStatus.READ_ONLY.code(), is(5));
+ }
+}
