[ignite] branch master updated: IGNITE-17316 Add pluggable affinity mapper to the thin client partition awareness feature (#10140)
This is an automated email from the ASF dual-hosted git repository. mmuzaf pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/master by this push: new daa397ccb09 IGNITE-17316 Add pluggable affinity mapper to the thin client partition awareness feature (#10140) daa397ccb09 is described below commit daa397ccb09a82fbf41367bc6725cf8fd0e841fd Author: Maxim Muzafarov AuthorDate: Wed Aug 10 01:56:04 2022 +0300 IGNITE-17316 Add pluggable affinity mapper to the thin client partition awareness feature (#10140) --- .../org/apache/ignite/snippets/JavaThinClient.java | 24 +++ docs/_docs/thin-clients/java-thin-client.adoc | 8 + ...ientPartitionAwarenessMapperAPITestWrapper.java | 91 + .../clients/JavaThinCompatibilityTest.java | 28 ++- .../client/ClientPartitionAwarenessMapper.java | 43 + .../ClientPartitionAwarenessMapperFactory.java | 48 + .../ignite/configuration/ClientConfiguration.java | 28 +++ .../client/thin/ClientCacheAffinityContext.java| 208 + .../client/thin/ClientCacheAffinityMapping.java| 92 +++-- .../client/thin/ProtocolBitmaskFeature.java| 5 +- .../internal/client/thin/ReliableChannel.java | 29 ++- .../internal/client/thin/TcpClientCache.java | 2 + .../internal/client/thin/TcpIgniteClient.java | 13 +- .../platform/client/ClientBitmaskFeature.java | 5 +- .../platform/client/ClientMessageParser.java | 2 +- .../cache/ClientCachePartitionAwarenessGroup.java | 87 + .../client/cache/ClientCachePartitionMapping.java | 27 ++- .../client/cache/ClientCachePartitionsRequest.java | 164 .../cache/ClientCachePartitionsResponse.java | 8 +- ...lientPartitionAwarenessResourceReleaseTest.java | 80 ...ClientPartitionAwarenessStableTopologyTest.java | 28 ++- 21 files changed, 817 insertions(+), 203 deletions(-) diff --git a/docs/_docs/code-snippets/java/src/main/java/org/apache/ignite/snippets/JavaThinClient.java b/docs/_docs/code-snippets/java/src/main/java/org/apache/ignite/snippets/JavaThinClient.java index d3ba1c510e7..c11241a4ae8 100644 --- a/docs/_docs/code-snippets/java/src/main/java/org/apache/ignite/snippets/JavaThinClient.java +++ b/docs/_docs/code-snippets/java/src/main/java/org/apache/ignite/snippets/JavaThinClient.java @@ -365,6 +365,30 @@ public class JavaThinClient { //end::partition-awareness[] } +void partitionAwarenessWithCustomMapper() throws Exception { +//tag::partition-awareness-with-mapper[] +// Partition awarenes is enabled by default since Apache Ignite 2.11 release. +ClientConfiguration cfg = new ClientConfiguration() +.setAddresses("node1_address:10800", "node2_address:10800", "node3_address:10800") +.setPartitionAwarenessMapperFactory(new ClientPartitionAwarenessMapperFactory() { +/** {@inheritDoc} */ +@Override public ClientPartitionAwarenessMapper create(String cacheName, int partitions) { +AffinityFunction aff = new RendezvousAffinityFunction(false, partitions); + +return aff::partition; +} +}) + +try (IgniteClient client = Ignition.startClient(cfg)) { +ClientCache cache = client.cache(PART_CUSTOM_AFFINITY_CACHE_NAME); +// Put, get or remove data from the cache, partition awarenes will be enabled. +} +catch (ClientException e) { +System.err.println(e.getMessage()); +} +//end::partition-awareness-with-mapper[] +} + private String[] fetchServerAddresses() { return null; } diff --git a/docs/_docs/thin-clients/java-thin-client.adoc b/docs/_docs/thin-clients/java-thin-client.adoc index f430c301c34..1469e446645 100644 --- a/docs/_docs/thin-clients/java-thin-client.adoc +++ b/docs/_docs/thin-clients/java-thin-client.adoc @@ -86,6 +86,14 @@ The following code sample illustrates how to use the partition awareness feature include::{sourceCodeFile}[tag=partition-awareness,indent=0] +The code sample below shows how to use a custom cache key to partition mapping function to enable affinity awareness on +a thin client side if the cache already exists in a cluster or/and was created with custom AffinityFunction or AffinityKeyMapper. + +[source, java] + +include::{sourceCodeFile}[tag=partition-awareness-with-mapper,indent=0] + + If a list of server nodes is dynamically changing or scaling, then it is possible to configure the connection with custom implementation of `ClientAddressFinder`. It should provide a number of current server addresses every time a client asks for them. The following code sample illustrates how to use it. diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/
[ignite-3] branch main updated: IGNITE-17233 Clarify cluster URL parameter name. Fixes #974
This is an automated email from the ASF dual-hosted git repository. sk0x50 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/ignite-3.git The following commit(s) were added to refs/heads/main by this push: new 10e977eb6 IGNITE-17233 Clarify cluster URL parameter name. Fixes #974 10e977eb6 is described below commit 10e977eb67aa5a702afce9b864b383c6d2660cd6 Author: Vadim Pakhnushev <8614891+valep...@users.noreply.github.com> AuthorDate: Tue Aug 9 13:20:06 2022 +0300 IGNITE-17233 Clarify cluster URL parameter name. Fixes #974 Signed-off-by: Slava Koptilin --- .../ItClusterStatusReplCommandInitializedTest.java | 2 +- ...ClusterStatusReplCommandNotInitializedTest.java | 2 +- .../ItTopologyCommandInitializedClusterTest.java | 4 +- ...ItTopologyCommandNotInitializedClusterTest.java | 4 +- .../configuration/ItConfigCommandTest.java | 16 +++ .../cli/deprecated/ItClusterCommandTest.java | 2 +- .../ignite/cli/commands/OptionsConstants.java | 50 ++ .../config/ClusterConfigShowReplSubCommand.java| 8 +++- .../config/ClusterConfigShowSubCommand.java| 8 +++- .../config/ClusterConfigUpdateReplSubCommand.java | 8 +++- .../config/ClusterConfigUpdateSubCommand.java | 8 +++- .../cluster/init/ClusterInitReplSubCommand.java| 12 -- .../cluster/init/ClusterInitSubCommand.java| 10 +++-- .../status/ClusterStatusReplSubCommand.java| 11 +++-- .../cluster/status/ClusterStatusSubCommand.java| 8 +++- .../node/config/NodeConfigShowReplSubCommand.java | 10 +++-- .../node/config/NodeConfigShowSubCommand.java | 8 +++- .../config/NodeConfigUpdateReplSubCommand.java | 8 +++- .../node/config/NodeConfigUpdateSubCommand.java| 8 +++- .../node/status/NodeStatusReplSubCommand.java | 13 +++--- .../commands/node/status/NodeStatusSubCommand.java | 8 +++- .../questions/ConnectToClusterQuestion.java| 3 +- .../topology/LogicalTopologyReplSubCommand.java| 11 +++-- .../topology/LogicalTopologySubCommand.java| 8 +++- .../topology/PhysicalTopologyReplSubCommand.java | 11 +++-- .../topology/PhysicalTopologySubCommand.java | 8 +++- .../apache/ignite/cli/config/ConfigConstants.java | 2 +- .../repl/completer/DynamicCompleterFilter.java | 5 ++- .../cli/core/repl/completer/NodeUrlProvider.java | 6 ++- .../cli/commands/UrlOptionsNegativeTest.java | 24 +-- .../cliconfig/CliConfigGetSubCommandTest.java | 2 +- .../cliconfig/CliConfigSetSubCommandTest.java | 10 ++--- .../cli/commands/cliconfig/ConfigManagerTest.java | 4 +- .../repl/completer/DynamicCompleterFilterTest.java | 16 +++ .../repl/completer/HoconDynamicCompleterTest.java | 8 ++-- .../cli/deprecated/IgniteCliInterfaceTest.java | 31 +++--- .../cli/src/test/resources/integration_tests.ini | 2 +- 37 files changed, 244 insertions(+), 115 deletions(-) diff --git a/modules/cli/src/integrationTest/java/org/apache/ignite/cli/commands/cluster/status/ItClusterStatusReplCommandInitializedTest.java b/modules/cli/src/integrationTest/java/org/apache/ignite/cli/commands/cluster/status/ItClusterStatusReplCommandInitializedTest.java index 3891d663a..ccba62994 100644 --- a/modules/cli/src/integrationTest/java/org/apache/ignite/cli/commands/cluster/status/ItClusterStatusReplCommandInitializedTest.java +++ b/modules/cli/src/integrationTest/java/org/apache/ignite/cli/commands/cluster/status/ItClusterStatusReplCommandInitializedTest.java @@ -40,7 +40,7 @@ class ItClusterStatusReplCommandInitializedTest extends CliCommandTestInitialize @Test @DisplayName("Should print status when valid cluster url is given but cluster is initialized") void printStatus() { -execute("cluster", "status", "--cluster-url", NODE_URL); +execute("cluster", "status", "--cluster-endpoint-url", NODE_URL); assertAll( this::assertExitCodeIsZero, diff --git a/modules/cli/src/integrationTest/java/org/apache/ignite/cli/commands/cluster/status/ItClusterStatusReplCommandNotInitializedTest.java b/modules/cli/src/integrationTest/java/org/apache/ignite/cli/commands/cluster/status/ItClusterStatusReplCommandNotInitializedTest.java index c1f4ab7f0..229ba2db5 100644 --- a/modules/cli/src/integrationTest/java/org/apache/ignite/cli/commands/cluster/status/ItClusterStatusReplCommandNotInitializedTest.java +++ b/modules/cli/src/integrationTest/java/org/apache/ignite/cli/commands/cluster/status/ItClusterStatusReplCommandNotInitializedTest.java @@ -31,7 +31,7 @@ class ItClusterStatusReplCommandNotInitializedTest extends CliCommandTestNotInit @Test @DisplayName("Should print status when valid cluster url is given but cluster is not initialized") void printStatus() { -execute("cluster", "status", "--cluster-url", NODE_URL); +execute("cluster", "status", "--cluster-endpoint-url", N
[ignite-python-thin-client] branch master updated: IGNITE-17494 use_ssl is not set when auth used (#55)
This is an automated email from the ASF dual-hosted git repository. isapego pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite-python-thin-client.git The following commit(s) were added to refs/heads/master by this push: new bc5a52c IGNITE-17494 use_ssl is not set when auth used (#55) bc5a52c is described below commit bc5a52c1c9b70136a63240da9749cabcb5c14c0e Author: Igor Sapego AuthorDate: Tue Aug 9 11:15:51 2022 +0300 IGNITE-17494 use_ssl is not set when auth used (#55) --- pyignite/connection/connection.py | 3 --- tests/security/test_auth.py | 29 + 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/pyignite/connection/connection.py b/pyignite/connection/connection.py index 3d86f01..4596e23 100644 --- a/pyignite/connection/connection.py +++ b/pyignite/connection/connection.py @@ -48,9 +48,6 @@ class BaseConnection: check_ssl_params(ssl_params) -if self.username and self.password and 'use_ssl' not in ssl_params: -ssl_params['use_ssl'] = True - self.ssl_params = ssl_params self._failed = False diff --git a/tests/security/test_auth.py b/tests/security/test_auth.py index 83ac780..f4ca29b 100644 --- a/tests/security/test_auth.py +++ b/tests/security/test_auth.py @@ -46,8 +46,7 @@ def cleanup(): clear_ignite_work_dir() -def test_auth_success(with_ssl, ssl_params, caplog): -ssl_params['use_ssl'] = with_ssl +def check_auth_success(ssl_params, caplog): listener = AccumulatingConnectionListener() client = Client(username=DEFAULT_IGNITE_USERNAME, password=DEFAULT_IGNITE_PASSWORD, event_listeners=[listener], **ssl_params) @@ -60,9 +59,18 @@ def test_auth_success(with_ssl, ssl_params, caplog): __assert_successful_connect_events(conn, listener) -@pytest.mark.asyncio -async def test_auth_success_async(with_ssl, ssl_params, caplog): +def test_auth_success_no_explicit_ssl(with_ssl, ssl_params, caplog): +if with_ssl: +ssl_params['use_ssl'] = with_ssl +check_auth_success(ssl_params, caplog) + + +def test_auth_success(with_ssl, ssl_params, caplog): ssl_params['use_ssl'] = with_ssl +check_auth_success(ssl_params, caplog) + + +async def check_auth_success_async(ssl_params, caplog): listener = AccumulatingConnectionListener() client = AioClient(username=DEFAULT_IGNITE_USERNAME, password=DEFAULT_IGNITE_PASSWORD, event_listeners=[listener], **ssl_params) @@ -75,6 +83,19 @@ async def test_auth_success_async(with_ssl, ssl_params, caplog): __assert_successful_connect_events(conn, listener) +@pytest.mark.asyncio +async def test_auth_success_no_explicit_ssl_async(with_ssl, ssl_params, caplog): +if with_ssl: +ssl_params['use_ssl'] = with_ssl +await check_auth_success_async(ssl_params, caplog) + + +@pytest.mark.asyncio +async def test_auth_success_async(with_ssl, ssl_params, caplog): +ssl_params['use_ssl'] = with_ssl +await check_auth_success_async(ssl_params, caplog) + + def __assert_successful_connect_log(conn, caplog): assert any(re.match(rf'Connecting to node\(address={conn.host},\s+port={conn.port}', r.message) for r in caplog.records)
[ignite-3] branch main updated: IGNITE-17348 Added quit/exit command to Ignite3 CLI sql REPL. Fixes #971
This is an automated email from the ASF dual-hosted git repository. sk0x50 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/ignite-3.git The following commit(s) were added to refs/heads/main by this push: new 8e25fce53 IGNITE-17348 Added quit/exit command to Ignite3 CLI sql REPL. Fixes #971 8e25fce53 is described below commit 8e25fce53bcc0079008efaf3fc0bab9da28b96a8 Author: Vadim Pakhnushev <8614891+valep...@users.noreply.github.com> AuthorDate: Tue Aug 9 11:06:39 2022 +0300 IGNITE-17348 Added quit/exit command to Ignite3 CLI sql REPL. Fixes #971 Signed-off-by: Slava Koptilin --- .../ignite/cli/commands/sql/SqlReplCommand.java | 8 +++- .../core/repl/executor/RegistryCommandExecutor.java | 21 - .../ignite/cli/core/repl/executor/ReplExecutor.java | 5 +++-- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/modules/cli/src/main/java/org/apache/ignite/cli/commands/sql/SqlReplCommand.java b/modules/cli/src/main/java/org/apache/ignite/cli/commands/sql/SqlReplCommand.java index 5f6aa702b..1647916c8 100644 --- a/modules/cli/src/main/java/org/apache/ignite/cli/commands/sql/SqlReplCommand.java +++ b/modules/cli/src/main/java/org/apache/ignite/cli/commands/sql/SqlReplCommand.java @@ -48,8 +48,6 @@ import picocli.CommandLine.Parameters; */ @Command(name = "sql", description = "Executes SQL query.") public class SqlReplCommand extends BaseCommand implements Runnable { -private static final String INTERNAL_COMMAND_PREFIX = "!"; - @Option(names = {"-u", "--jdbc-url"}, required = true, descriptionKey = "ignite.jdbc-url", description = "JDBC url to ignite cluster") private String jdbc; @@ -100,8 +98,8 @@ public class SqlReplCommand extends BaseCommand implements Runnable { } private CallExecutionPipelineProvider provider(SqlManager sqlManager) { -return (call, exceptionHandlers, line) -> line.startsWith(INTERNAL_COMMAND_PREFIX) -? createInternalCommandPipeline(call, exceptionHandlers, line) +return (executor, exceptionHandlers, line) -> executor.hasCommand(line) +? createInternalCommandPipeline(executor, exceptionHandlers, line) : createSqlExecPipeline(sqlManager, line); } @@ -118,7 +116,7 @@ public class SqlReplCommand extends BaseCommand implements Runnable { ExceptionHandlers exceptionHandlers, String line) { return CallExecutionPipeline.builder(call) -.inputProvider(() -> new StringCallInput(line.substring(INTERNAL_COMMAND_PREFIX.length( +.inputProvider(() -> new StringCallInput(line)) .output(spec.commandLine().getOut()) .errOutput(spec.commandLine().getErr()) .exceptionHandlers(exceptionHandlers) diff --git a/modules/cli/src/main/java/org/apache/ignite/cli/core/repl/executor/RegistryCommandExecutor.java b/modules/cli/src/main/java/org/apache/ignite/cli/core/repl/executor/RegistryCommandExecutor.java index 5077fde67..e2cd681d6 100644 --- a/modules/cli/src/main/java/org/apache/ignite/cli/core/repl/executor/RegistryCommandExecutor.java +++ b/modules/cli/src/main/java/org/apache/ignite/cli/core/repl/executor/RegistryCommandExecutor.java @@ -23,6 +23,9 @@ import org.apache.ignite.cli.core.call.CallOutput; import org.apache.ignite.cli.core.call.DefaultCallOutput; import org.apache.ignite.cli.core.call.StringCallInput; import org.jline.console.SystemRegistry; +import org.jline.reader.ParsedLine; +import org.jline.reader.Parser; +import org.jline.reader.Parser.ParseContext; /** * Command executor based on {@link SystemRegistry}. @@ -30,13 +33,17 @@ import org.jline.console.SystemRegistry; public class RegistryCommandExecutor implements Call { private final SystemRegistry systemRegistry; +private final Parser parser; + /** * Constructor. * * @param systemRegistry {@link SystemRegistry} instance. + * @param parser A {@link Parser} used to create {@code systemRegistry}. */ -public RegistryCommandExecutor(SystemRegistry systemRegistry) { +public RegistryCommandExecutor(SystemRegistry systemRegistry, Parser parser) { this.systemRegistry = systemRegistry; +this.parser = parser; } /** @@ -65,4 +72,16 @@ public class RegistryCommandExecutor implements Call { public void cleanUp() { systemRegistry.cleanUp(); } + +/** + * Determines whether the {@link SystemRegistry} has a command with this name. + * + * @param line command name to check. + * @return true if the registry has the command. + */ +public boolean hasCommand(String line) { +ParsedLine pl = parser.parse(line, 0, ParseContext.SPLIT_LINE); + +return !pl.words().isEmpty() && systemRegistry.hasCommand(parser.getCommand(pl.words().get(0))); +} } diff --git a/modules/cli
[ignite-3] branch main updated: IGNITE-14937 Introduce index management (#980)
This is an automated email from the ASF dual-hosted git repository. korlov pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/ignite-3.git The following commit(s) were added to refs/heads/main by this push: new a5662cc26 IGNITE-14937 Introduce index management (#980) a5662cc26 is described below commit a5662cc269dfb85c5dd01ef8f0e4ef754f6f0d4d Author: korlov42 AuthorDate: Tue Aug 9 10:00:42 2022 +0300 IGNITE-14937 Introduce index management (#980) --- .../table/HashIndexConfigurationSchema.java| 2 +- .../table/PartialIndexConfigurationSchema.java | 38 --- .../table/TableIndexConfigurationSchema.java | 13 +- .../builder/PartialIndexDefinitionBuilder.java | 72 .../java/org/apache/ignite/lang/ErrorGroups.java | 9 + modules/index/pom.xml | 95 ++ .../org/apache/ignite/internal/index/Index.java} | 31 +- .../apache/ignite/internal/index/IndexManager.java | 261 +++ .../ignite/internal/index/IndexManagerTest.java| 363 + modules/runner/pom.xml | 5 + .../org/apache/ignite/internal/app/IgniteImpl.java | 13 + .../CoreDistributedConfigurationModule.java| 2 - .../CoreDistributedConfigurationModuleTest.java| 6 - .../SchemaConfigurationConverter.java | 33 +- .../builder/PartialIndexDefinitionBuilderImpl.java | 121 --- .../index/PartialIndexDefinitionImpl.java | 61 .../org/apache/ignite/schema/SchemaBuilders.java | 12 - .../internal/schema/SchemaConfigurationTest.java | 9 - .../builder/PartialIndexDefinitionBuilderTest.java | 46 --- .../SchemaConfigurationConverterTest.java | 27 -- .../configuration/TableValidatorImplTest.java | 4 +- .../internal/sql/engine/session/Session.java | 2 +- .../sql/engine/exec/MockedStructuresTest.java | 2 - .../internal/table/distributed/TableManager.java | 2 +- .../table/distributed/TableManagerTest.java| 2 - parent/pom.xml | 6 + pom.xml| 1 + 27 files changed, 782 insertions(+), 456 deletions(-) diff --git a/modules/api/src/main/java/org/apache/ignite/configuration/schemas/table/HashIndexConfigurationSchema.java b/modules/api/src/main/java/org/apache/ignite/configuration/schemas/table/HashIndexConfigurationSchema.java index 7fa3564e4..ce1a1e5c2 100644 --- a/modules/api/src/main/java/org/apache/ignite/configuration/schemas/table/HashIndexConfigurationSchema.java +++ b/modules/api/src/main/java/org/apache/ignite/configuration/schemas/table/HashIndexConfigurationSchema.java @@ -29,5 +29,5 @@ import org.apache.ignite.configuration.annotation.Value; public class HashIndexConfigurationSchema extends TableIndexConfigurationSchema { /** Columns names for hash indexes. */ @Value -public String[] colNames; +public String[] columnNames; } diff --git a/modules/api/src/main/java/org/apache/ignite/configuration/schemas/table/PartialIndexConfigurationSchema.java b/modules/api/src/main/java/org/apache/ignite/configuration/schemas/table/PartialIndexConfigurationSchema.java deleted file mode 100644 index 952bb4347..0 --- a/modules/api/src/main/java/org/apache/ignite/configuration/schemas/table/PartialIndexConfigurationSchema.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.configuration.schemas.table; - -import static org.apache.ignite.configuration.schemas.table.TableIndexConfigurationSchema.PARTIAL_INDEX_TYPE; - -import org.apache.ignite.configuration.annotation.NamedConfigValue; -import org.apache.ignite.configuration.annotation.PolymorphicConfigInstance; -import org.apache.ignite.configuration.annotation.Value; - -/** - * Schema for the Partial Index configuration. - */ -@PolymorphicConfigInstance(PARTIAL_INDEX_TYPE) -public class PartialIndexConfigurationSchema extends TableIndexConfigurationSchema { -/** Expression for PartialIndex: PARTIAL indexes. */ -@Value -public String expr; - -/** Columns configuration for sorted indexes. */ -@Nam