This is an automated email from the ASF dual-hosted git repository.
tkalkirill pushed a commit to branch catalog-feature
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit 96a9ed49c214af3b6df22032b04572559f48c3da
Merge: 81cefdd6a4 6bc603deb6
Author: Kirill Tkalenko <tkalkir...@yandex.ru>
AuthorDate: Thu Sep 14 09:50:11 2023 +0300
Merge branch 'ai-main' into catalog-feature
.../internal/catalog/CatalogManagerImpl.java | 17 +-
.../catalog/CatalogParamsValidationUtils.java | 6 +-
.../ignite/internal/catalog/CatalogService.java | 2 +
.../catalog/CatalogValidationException.java | 2 +-
.../commands/AbstractCreateIndexCommand.java | 5 +-
.../commands/AlterTableAddColumnCommand.java | 2 +-
.../commands/AlterTableAlterColumnCommand.java | 2 +-
.../commands/AlterTableDropColumnCommand.java | 2 +-
.../internal/catalog/commands/CatalogUtils.java | 20 ++
.../internal/catalog/commands/ColumnParams.java | 20 ++
.../catalog/commands/CreateSystemViewCommand.java | 206 +++++++++++
.../commands/CreateSystemViewCommandBuilder.java | 62 ++++
.../catalog/commands/CreateTableCommand.java | 10 +-
.../catalog/commands/DropIndexCommand.java | 2 +-
.../catalog/commands/DropTableCommand.java | 4 +-
.../descriptors/CatalogObjectDescriptor.java | 5 +-
.../descriptors/CatalogSchemaDescriptor.java | 28 +-
.../descriptors/CatalogSystemViewDescriptor.java | 105 ++++++
.../internal/catalog/events/CatalogEvent.java | 5 +-
.../events/CreateSystemViewEventParameters.java | 48 +++
.../internal/catalog/storage/AlterColumnEntry.java | 18 +-
.../internal/catalog/storage/DropColumnsEntry.java | 17 +-
.../internal/catalog/storage/DropIndexEntry.java | 19 +-
.../internal/catalog/storage/DropTableEntry.java | 19 +-
.../internal/catalog/storage/NewColumnsEntry.java | 16 +-
.../internal/catalog/storage/NewIndexEntry.java | 19 +-
.../catalog/storage/NewSystemViewEntry.java | 99 ++++++
.../internal/catalog/storage/NewTableEntry.java | 26 +-
.../internal/catalog/CatalogManagerSelfTest.java | 43 ++-
.../internal/catalog/CatalogSystemViewTest.java | 381 +++++++++++++++++++++
.../commands/AbstractCommandValidationTest.java | 10 +-
.../CreateSystemViewCommandValidationTest.java | 141 ++++++++
.../internal/catalog/BaseCatalogManagerTest.java | 12 +-
.../processor/ConfigurationProcessor.java | 3 +-
.../configuration/ConfigurationTreeGenerator.java | 9 +-
.../ConfigurationNotificationUtils.java | 42 ++-
.../notifications/ConfigurationNotifier.java | 44 +--
.../configuration/util/ConfigurationUtil.java | 138 +++++++-
.../configuration/util/ConfigurationUtilTest.java | 106 ++++++
.../ignite/internal/util/CollectionUtils.java | 184 ++--------
.../ignite/internal/util/CollectionUtilsTest.java | 88 +----
.../persistence/compaction/Compactor.java | 2 +-
.../persistence/store/FilePageStoreManager.java | 5 +-
.../persistence/store/GroupPageStoresMap.java | 7 +-
.../store/FilePageStoreManagerTest.java | 8 +-
.../persistence/store/GroupPageStoresMapTest.java | 9 +-
.../placementdriver/PlacementDriverManager.java | 2 +
.../internal/benchmark/SqlOneNodeBenchmark.java | 147 ++++++++
.../internal/runner/app/ItTablesApiTest.java | 1 +
.../ignite/internal/schema/BinaryTupleSchema.java | 3 +-
.../ignite/internal/schema/SchemaTestUtils.java | 5 +-
modules/sql-engine/build.gradle | 1 +
.../internal/sql/api/AsyncResultSetImpl.java | 5 +
.../engine/exec/ExecutableTableRegistryImpl.java | 9 +-
.../sql/engine/exec/ScannableTableImpl.java | 18 +-
.../sql/engine/exec/TableRowConverter.java | 15 +-
...onverter.java => TableRowConverterFactory.java} | 24 +-
.../sql/engine/exec/TableRowConverterImpl.java | 85 +++--
.../sql/engine/exec/UpdatableTableImpl.java | 2 +-
.../sql/engine/schema/TableDescriptor.java | 2 +-
.../sql/engine/schema/TableDescriptorImpl.java | 7 +
.../sql/engine/util/AbstractProjectedTuple.java | 227 ++++++++++++
.../util/FieldDeserializingProjectedTuple.java | 75 ++++
.../sql/engine/util/FormatAwareProjectedTuple.java | 93 +++++
.../exec/ExecutableTableRegistrySelfTest.java | 3 +
.../engine/exec/rel/ScannableTableSelfTest.java | 31 +-
.../exec/rel/TableScanNodeExecutionTest.java | 5 +-
.../sql/engine/planner/AbstractPlannerTest.java | 6 +
.../engine/schema/CatalogSqlSchemaManagerTest.java | 5 +-
.../sql/engine/util/ProjectedTupleTest.java | 179 ++++++++++
.../internal/table/distributed/TableManager.java | 3 +-
.../replicator/PartitionReplicaListener.java | 1 +
.../apache/ignite/internal/table/TxLocalTest.java | 13 +-
.../table/impl/DummyInternalTableImpl.java | 15 +-
74 files changed, 2464 insertions(+), 536 deletions(-)
diff --cc
modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/AbstractCreateIndexCommand.java
index 70c2225baf,471c3332e1..bee55a4c18
---
a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/AbstractCreateIndexCommand.java
+++
b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/AbstractCreateIndexCommand.java
@@@ -17,7 -17,7 +17,8 @@@
package org.apache.ignite.internal.catalog.commands;
+import static
org.apache.ignite.internal.catalog.CatalogParamsValidationUtils.ensureNoTableOrIndexExistsWithGivenName;
+ import static
org.apache.ignite.internal.catalog.CatalogParamsValidationUtils.ensureNoTableIndexOrSysViewExistsWithGivenName;
import static
org.apache.ignite.internal.catalog.CatalogParamsValidationUtils.validateIdentifier;
import static
org.apache.ignite.internal.catalog.commands.CatalogUtils.schemaOrThrow;
import static
org.apache.ignite.internal.catalog.commands.CatalogUtils.tableOrThrow;
diff --cc
modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/CatalogUtils.java
index f8a6553284,fae6e961b5..d34b705ed9
---
a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/CatalogUtils.java
+++
b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/CatalogUtils.java
@@@ -17,10 -17,13 +17,11 @@@
package org.apache.ignite.internal.catalog.commands;
-import static java.util.stream.Collectors.toList;
import static org.apache.ignite.lang.IgniteStringFormatter.format;
+ import java.util.Collection;
import java.util.EnumMap;
import java.util.EnumSet;
-import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
diff --cc
modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItTablesApiTest.java
index ac1c41e638,0acac49c2c..9947f25bd8
---
a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItTablesApiTest.java
+++
b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItTablesApiTest.java
@@@ -39,17 -44,23 +39,18 @@@ import java.util.stream.IntStream
import org.apache.ignite.Ignite;
import org.apache.ignite.IgnitionManager;
import org.apache.ignite.InitParameters;
+ import org.apache.ignite.internal.catalog.IndexExistsValidationException;
-import org.apache.ignite.internal.schema.testutils.builder.SchemaBuilders;
-import
org.apache.ignite.internal.schema.testutils.definition.ColumnDefinition;
-import org.apache.ignite.internal.schema.testutils.definition.ColumnType;
+import org.apache.ignite.internal.catalog.CatalogValidationException;
+import org.apache.ignite.internal.catalog.IndexExistsValidationException;
+import org.apache.ignite.internal.catalog.TableExistsValidationException;
+import org.apache.ignite.internal.catalog.TableNotFoundValidationException;
import org.apache.ignite.internal.table.IgniteTablesInternal;
import org.apache.ignite.internal.table.TableImpl;
-import org.apache.ignite.internal.table.distributed.TableManager;
import org.apache.ignite.internal.test.WatchListenerInhibitor;
import org.apache.ignite.internal.testframework.IgniteAbstractTest;
-import org.apache.ignite.internal.testframework.IgniteTestUtils;
import org.apache.ignite.internal.testframework.TestIgnitionManager;
import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.lang.ColumnAlreadyExistsException;
-import org.apache.ignite.lang.IndexAlreadyExistsException;
import org.apache.ignite.lang.NodeStoppingException;
-import org.apache.ignite.lang.TableAlreadyExistsException;
-import org.apache.ignite.lang.TableNotFoundException;
import org.apache.ignite.sql.Session;
import org.apache.ignite.table.Table;
import org.apache.ignite.table.Tuple;
diff --cc
modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterImpl.java
index 7fe1f5f3e0,355a0b42c4..8801dc69a0
---
a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterImpl.java
+++
b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterImpl.java
@@@ -18,17 -18,16 +18,19 @@@
package org.apache.ignite.internal.sql.engine.exec;
import java.util.BitSet;
- import org.apache.ignite.internal.binarytuple.BinaryTupleBuilder;
import org.apache.ignite.internal.schema.BinaryRow;
- import org.apache.ignite.internal.schema.BinaryRowConverter;
import org.apache.ignite.internal.schema.BinaryTuple;
import org.apache.ignite.internal.schema.BinaryTupleSchema;
+import org.apache.ignite.internal.schema.Column;
import org.apache.ignite.internal.schema.SchemaDescriptor;
import org.apache.ignite.internal.schema.SchemaRegistry;
+ import org.apache.ignite.internal.schema.row.InternalTuple;
+ import org.apache.ignite.internal.sql.engine.schema.ColumnDescriptor;
+import org.apache.ignite.internal.schema.row.Row;
+import org.apache.ignite.internal.sql.engine.schema.ColumnDescriptor;
import org.apache.ignite.internal.sql.engine.schema.TableDescriptor;
+ import
org.apache.ignite.internal.sql.engine.util.FieldDeserializingProjectedTuple;
+ import org.apache.ignite.internal.sql.engine.util.FormatAwareProjectedTuple;
import org.jetbrains.annotations.Nullable;
/**
diff --cc
modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistrySelfTest.java
index 937cc182ff,6d99328dce..99b56ab6a8
---
a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistrySelfTest.java
+++
b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistrySelfTest.java
@@@ -29,7 -29,9 +30,8 @@@ import org.apache.ignite.internal.schem
import org.apache.ignite.internal.schema.Column;
import org.apache.ignite.internal.schema.NativeTypes;
import org.apache.ignite.internal.schema.SchemaDescriptor;
-import org.apache.ignite.internal.schema.SchemaManager;
import org.apache.ignite.internal.schema.SchemaRegistry;
+ import org.apache.ignite.internal.sql.engine.schema.ColumnDescriptor;
import org.apache.ignite.internal.sql.engine.schema.TableDescriptor;
import org.apache.ignite.internal.table.InternalTable;
import org.apache.ignite.internal.table.TableImpl;
@@@ -151,8 -152,9 +153,9 @@@ public class ExecutableTableRegistrySel
when(tableManager.tableAsync(tableId)).thenReturn(CompletableFuture.completedFuture(table));
when(schemaManager.schemaRegistry(tableId)).thenReturn(schemaRegistry);
when(schemaRegistry.schema()).thenReturn(schemaDescriptor);
+
when(descriptor.iterator()).thenReturn(List.<ColumnDescriptor>of().iterator());
- return registry.getTable(tableId, descriptor);
+ return registry.getTable(tableId, tableVersion, descriptor);
}
}
}
diff --cc
modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
index 66af7c86cf,1bbb0b1e45..c83f5d6666
---
a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
+++
b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
@@@ -148,7 -167,7 +148,8 @@@ import org.apache.ignite.internal.table
import
org.apache.ignite.internal.table.distributed.raft.snapshot.PartitionSnapshotStorageFactory;
import
org.apache.ignite.internal.table.distributed.raft.snapshot.outgoing.OutgoingSnapshotsManager;
import
org.apache.ignite.internal.table.distributed.raft.snapshot.outgoing.SnapshotAwarePartitionDataStorage;
+import
org.apache.ignite.internal.table.distributed.replicator.DirectCatalogTables;
+ import
org.apache.ignite.internal.table.distributed.replicator.CatalogTablesWithIdConversion;
import
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener;
import
org.apache.ignite.internal.table.distributed.replicator.TransactionStateResolver;
import org.apache.ignite.internal.table.distributed.schema.NonHistoricSchemas;
@@@ -491,48 -538,80 +492,48 @@@ public class TableManager extends Produ
@Override
public void start() {
- mvGc.start();
+ inBusyLock(busyLock, () -> {
+ mvGc.start();
- lowWatermark.start();
+ lowWatermark.start();
- try {
- metaStorageMgr.recoveryFinishedFuture()
- .thenComposeAsync(this::performRebalanceOnRecovery,
ioExecutor)
- .get();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
-
- throw new IgniteInternalException(INTERNAL_ERR, e);
- } catch (ExecutionException e) {
- throw new IgniteInternalException(INTERNAL_ERR, e);
- }
-
-
metaStorageMgr.registerPrefixWatch(ByteArray.fromString(PENDING_ASSIGNMENTS_PREFIX),
pendingAssignmentsRebalanceListener);
-
metaStorageMgr.registerPrefixWatch(ByteArray.fromString(STABLE_ASSIGNMENTS_PREFIX),
stableAssignmentsRebalanceListener);
-
metaStorageMgr.registerPrefixWatch(ByteArray.fromString(ASSIGNMENTS_SWITCH_REDUCE_PREFIX),
assignmentsSwitchRebalanceListener);
+ startTables();
- tablesCfg.tables().listenElements(new
ConfigurationNamedListListener<>() {
- @Override
- public CompletableFuture<?>
onCreate(ConfigurationNotificationEvent<TableView> ctx) {
- return onTableCreate(ctx);
- }
-
- @Override
- public CompletableFuture<?>
onRename(ConfigurationNotificationEvent<TableView> ctx) {
- // TODO: IGNITE-15485 Support table rename operation.
-
- return completedFuture(null);
- }
-
- @Override
- public CompletableFuture<?>
onDelete(ConfigurationNotificationEvent<TableView> ctx) {
- return onTableDelete(ctx);
- }
- });
-
- schemaManager.listen(SchemaEvent.CREATE, new EventListener<>() {
- @Override
- public CompletableFuture<Boolean> notify(SchemaEventParameters
parameters, @Nullable Throwable exception) {
- var eventParameters = new
TableEventParameters(parameters.causalityToken(), parameters.tableId());
-
- return fireEvent(TableEvent.ALTER,
eventParameters).thenApply(v -> false);
+ try {
+ metaStorageMgr.recoveryFinishedFuture()
+ .thenComposeAsync(this::performRebalanceOnRecovery,
ioExecutor)
+ .get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ throw new IgniteInternalException(INTERNAL_ERR, e);
+ } catch (ExecutionException e) {
+ throw new IgniteInternalException(INTERNAL_ERR, e);
}
- });
- addMessageHandler(clusterService.messagingService());
+
metaStorageMgr.registerPrefixWatch(ByteArray.fromString(PENDING_ASSIGNMENTS_PREFIX),
pendingAssignmentsRebalanceListener);
+
metaStorageMgr.registerPrefixWatch(ByteArray.fromString(STABLE_ASSIGNMENTS_PREFIX),
stableAssignmentsRebalanceListener);
+
metaStorageMgr.registerPrefixWatch(ByteArray.fromString(ASSIGNMENTS_SWITCH_REDUCE_PREFIX),
assignmentsSwitchRebalanceListener);
- // TODO: IGNITE-19499 - remove when switched to the Catalog.
- recoverTableIdsMapping();
+ catalogService.listen(CatalogEvent.TABLE_CREATE, (parameters,
exception) -> {
+ assert exception == null : parameters;
- // TODO: IGNITE-19499 - remove when switched to the Catalog.
- catalogService.listen(CatalogEvent.TABLE_CREATE, (parameters,
exception) -> {
- CreateTableEventParameters event = (CreateTableEventParameters)
parameters;
-
- TableView tableConfig =
tablesCfg.tables().value().get(event.tableDescriptor().name());
-
- assert tableConfig != null : "No table config found by name " +
event.tableDescriptor().name();
+ return onTableCreate((CreateTableEventParameters)
parameters).thenApply(unused -> false);
+ });
- tableIdTranslator.registerMapping(tableConfig.id(),
event.tableId());
+ catalogService.listen(CatalogEvent.TABLE_DROP, (parameters,
exception) -> {
+ assert exception == null : parameters;
- return completedFuture(false);
- });
- }
+ return onTableDelete(((DropTableEventParameters)
parameters)).thenApply(unused -> false);
+ });
- private void recoverTableIdsMapping() {
- tablesCfg.tables().value().forEach(tableView -> {
- CatalogTableDescriptor tableDescriptor =
catalogService.table(tableView.name(), Long.MAX_VALUE);
+ schemaManager.listen(SchemaEvent.CREATE, (parameters, exception)
-> inBusyLockAsync(busyLock, () -> {
+ var eventParameters = new
TableEventParameters(parameters.causalityToken(), parameters.tableId());
- assert tableDescriptor != null : "No table in the Catalog with
name " + tableView.name();
+ return fireEvent(TableEvent.ALTER,
eventParameters).thenApply(v -> false);
+ }));
- tableIdTranslator.registerMapping(tableView.id(),
tableDescriptor.id());
+ addMessageHandler(clusterService.messagingService());
});
}
diff --cc
modules/table/src/main/java/org/apache/ignite/internal/table/distributed/replicator/PartitionReplicaListener.java
index fba6bec0e7,8b785381c4..a649d0ad6c
---
a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/replicator/PartitionReplicaListener.java
+++
b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/replicator/PartitionReplicaListener.java
@@@ -279,8 -281,8 +279,9 @@@ public class PartitionReplicaListener i
* @param localNode Instance of the local node.
* @param mvTableStorage Table storage.
* @param indexBuilder Index builder.
- * @param tablesConfig Tables configuration.
+ * @param catalogService Catalog service.
+ * @param placementDriver Placement driver.
+ * @param placementDriver Placement driver.
*/
public PartitionReplicaListener(
MvPartitionStorage mvDataStorage,
diff --cc
modules/table/src/testFixtures/java/org/apache/ignite/internal/table/impl/DummyInternalTableImpl.java
index bcc7c184b6,69476824e4..89ca256b7e
---
a/modules/table/src/testFixtures/java/org/apache/ignite/internal/table/impl/DummyInternalTableImpl.java
+++
b/modules/table/src/testFixtures/java/org/apache/ignite/internal/table/impl/DummyInternalTableImpl.java
@@@ -113,12 -113,12 +112,14 @@@ public class DummyInternalTableImpl ext
public static final ClusterNode LOCAL_NODE = new ClusterNodeImpl("id",
"node", ADDR);
- public static final HybridClock CLOCK = new TestHybridClock(new
LongSupplier() {
- @Override
- public long getAsLong() {
- return 0;
- }
- });
++ public static final ClusterNode LOCAL_NODE = new ClusterNodeImpl("id",
"node", ADDR);
++
+ // 2000 was picked to avoid negative time that we get when building read
timestamp
+ // in TxManagerImpl.currentReadTimestamp.
+ // We subtract
(ReplicaManager.IDLE_SAFE_TIME_PROPAGATION_PERIOD_MILLISECONDS +
HybridTimestamp.CLOCK_SKEW) = (1000 + 7) = 1007
+ // from the current time.
+ // Any value greater than that will work, hence 2000.
+ public static final HybridClock CLOCK = new TestHybridClock(() -> 2000);
private static final int PART_ID = 0;
