This is an automated email from the ASF dual-hosted git repository. maedhroz pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit f95c1b5bb3d6f8728a00e13ca81993e12a9b14cd Merge: 32755cabfa 7903ce2727 Author: Caleb Rackliffe <[email protected]> AuthorDate: Wed Jul 31 16:24:49 2024 -0500 Merge branch 'cassandra-5.0' into trunk * cassandra-5.0: Deprecate and ignore use_deterministic_table_id CHANGES.txt | 1 + NEWS.txt | 2 ++ src/java/org/apache/cassandra/config/Config.java | 1 + .../apache/cassandra/config/DatabaseDescriptor.java | 12 ++---------- .../cql3/statements/schema/CreateTableStatement.java | 3 +-- .../cql3/statements/schema/CreateViewStatement.java | 2 +- .../org/apache/cassandra/schema/TableMetadata.java | 2 +- .../cql3/validation/operations/CreateTest.java | 19 +++---------------- 8 files changed, 12 insertions(+), 30 deletions(-) diff --cc CHANGES.txt index 24b68b6a03,74cbbf9b00..7e397c1f4c --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,71 -1,13 +1,72 @@@ -5.0.1 +5.1 + * Update legacy peers tables during node replacement (CASSANDRA-19782) + * Refactor ColumnCondition (CASSANDRA-19620) + * Allow configuring log format for Audit Logs (CASSANDRA-19792) + * Support for noboolean rpm (centos7 compatible) packages removed (CASSANDRA-19787) + * Allow threads waiting for the metadata log follower to be interrupted (CASSANDRA-19761) + * Support dictionary lookup for CassandraPasswordValidator (CASSANDRA-19762) + * Disallow denylisting keys in system_cluster_metadata (CASSANDRA-19713) + * Fix gossip status after replacement (CASSANDRA-19712) + * Ignore repair requests for system_cluster_metadata (CASSANDRA-19711) + * Avoid ClassCastException when verifying tables with reversed partitioner (CASSANDRA-19710) + * Always repair the full range when repairing system_cluster_metadata (CASSANDRA-19709) + * Use table-specific partitioners during Paxos repair (CASSANDRA-19714) + * Expose current compaction throughput in nodetool (CASSANDRA-13890) + * CEP-24 Password validation / generation (CASSANDRA-17457) + * Reconfigure CMS after replacement, bootstrap and move operations (CASSANDRA-19705) + * Support querying LocalStrategy tables with any partitioner (CASSANDRA-19692) + * Relax slow_query_log_timeout for MultiNodeSAITest (CASSANDRA-19693) + * Audit Log entries are missing identity for mTLS connections (CASSANDRA-19669) + * Add support for the BETWEEN operator in WHERE clauses (CASSANDRA-19604) + * Replace Stream iteration with for-loop for SimpleRestriction::bindAndGetClusteringElements (CASSANDRA-19679) + * Consolidate logging on trace level (CASSANDRA-19632) + * Expand DDL statements on coordinator before submission to the CMS (CASSANDRA-19592) + * Fix number of arguments of String.format() in various classes (CASSANDRA-19645) + * Remove unused fields from config (CASSANDRA-19599) + * Refactor Relation and Restriction hierarchies (CASSANDRA-19341) + * Raise priority of TCM internode messages during critical operations (CASSANDRA-19517) + * Add nodetool command to unregister LEFT nodes (CASSANDRA-19581) + * Add cluster metadata id to gossip syn messages (CASSANDRA-19613) + * Reduce heap usage occupied by the metrics (CASSANDRA-19567) + * Improve handling of transient replicas during range movements (CASSANDRA-19344) + * Enable debounced internode log requests to be cancelled at shutdown (CASSANDRA-19514) + * Correctly set last modified epoch when combining multistep operations into a single step (CASSANDRA-19538) + * Add new TriggersPolicy configuration to allow operators to disable triggers (CASSANDRA-19532) + * Use Transformation.Kind.id in local and distributed log tables (CASSANDRA-19516) + * Remove period field from ClusterMetadata and metadata log tables (CASSANDRA-19482) + * Enrich system_views.pending_hints vtable with hints sizes (CASSANDRA-19486) + * Expose all dropwizard metrics in virtual tables (CASSANDRA-14572) + * Ensured that PropertyFileSnitchTest do not overwrite cassandra-toploogy.properties (CASSANDRA-19502) + * Add option for MutualTlsAuthenticator to restrict the certificate validity period (CASSANDRA-18951) + * Fix StorageService::constructRangeToEndpointMap for non-distributed keyspaces (CASSANDRA-19255) + * Group nodetool cms commands into single command group (CASSANDRA-19393) + * Register the measurements of the bootstrap process as Dropwizard metrics (CASSANDRA-19447) + * Add LIST SUPERUSERS CQL statement (CASSANDRA-19417) + * Modernize CQLSH datetime conversions (CASSANDRA-18879) + * Harry model and in-JVM tests for partition-restricted 2i queries (CASSANDRA-18275) + * Refactor cqlshmain global constants (CASSANDRA-19201) + * Remove native_transport_port_ssl (CASSANDRA-19397) + * Make nodetool reconfigurecms sync by default and add --cancel to be able to cancel ongoing reconfigurations (CASSANDRA-19216) + * Expose auth mode in system_views.clients, nodetool clientstats, metrics (CASSANDRA-19366) + * Remove sealed_periods and last_sealed_period tables (CASSANDRA-19189) + * Improve setup and initialisation of LocalLog/LogSpec (CASSANDRA-19271) + * Refactor structure of caching metrics and expose auth cache metrics via JMX (CASSANDRA-17062) + * Allow CQL client certificate authentication to work without sending an AUTHENTICATE request (CASSANDRA-18857) + * Extend nodetool tpstats and system_views.thread_pools with detailed pool parameters (CASSANDRA-19289) + * Remove dependency on Sigar in favor of OSHI (CASSANDRA-16565) + * Simplify the bind marker and Term logic (CASSANDRA-18813) + * Limit cassandra startup to supported JDKs, allow higher JDKs by setting CASSANDRA_JDK_UNSUPPORTED (CASSANDRA-18688) + * Standardize nodetool tablestats formatting of data units (CASSANDRA-19104) + * Make nodetool tablestats use number of significant digits for time and average values consistently (CASSANDRA-19015) + * Upgrade jackson to 2.15.3 and snakeyaml to 2.1 (CASSANDRA-18875) + * Transactional Cluster Metadata [CEP-21] (CASSANDRA-18330) + * Add ELAPSED command to cqlsh (CASSANDRA-18861) + * Add the ability to disable bulk loading of SSTables (CASSANDRA-18781) + * Clean up obsolete functions and simplify cql_version handling in cqlsh (CASSANDRA-18787) +Merged from 5.0: + * Deprecate and ignore use_deterministic_table_id (CASSANDRA-19809) * Prioritize built indexes in IndexStatusManager (CASSANDRA-19400) * Add java.base/java.lang.reflect among opens for jvm11-client.options (CASSANDRA-19780) - - -5.0-rc2 - - -5.0-rc1 * Move bcpkix-jdk18on to build dependencies, update to 1.78 and explicitly enumerate transitive dependencies (CASSANDRA-19739) * Avoid streams in the common case for UpdateTransaction creation (CASSANDRA-19675) * Only wait until native_transport_timeout for dispatcher to finish (CASSANDRA-19697) diff --cc NEWS.txt index 202290c574,ae5f36601a..8d9e620801 --- a/NEWS.txt +++ b/NEWS.txt @@@ -66,95 -66,6 +66,96 @@@ restore snapshots created with the prev using the provided 'sstableupgrade' tool. +5.1 +=== + +New features +------------ + [The following is a placeholder, to be revised asap] + - CEP-21 Transactional Cluster Metadata introduces a distributed log for linearizing modifications to cluster + metadata. In the first instance, this encompasses cluster membership, token ownership and schema metadata. See + https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-21%3A+Transactional+Cluster+Metadata for more detail on + the motivation and design, and see "Upgrading" below for specific instructions on migrating clusters to this system. + More updates and documentation to follow. + - New Guardrails added: + - Whether bulk loading of SSTables is allowed. + - nodetool tpstats can display core pool size, max pool size and max tasks queued if --verbose / -v flag is specified. + system_views.thread_pools adds core_pool_size, max_pool_size and max_tasks_queued columns. + - Authentication mode is exposed in system_views.clients table, nodetool clientstats and ClientMetrics + to help operators identify which authentication modes are being used. nodetool clientstats introduces --verbose flag + behind which this information is visible. + - CEP-24 - Password validation / generation. When built-in 'password_validator' guardrail is enabled, it will + generate a password of configured password strength policy upon role creation or alteration + when 'GENERATED PASSWORD' clause is used. Character sets supported are: English, Cyrillic, modern Cyrillic, + German, Polish and Czech. + + +Upgrading +--------- + [The following is a placeholder, to be revised asap] + - Upgrading to 5.1 is currently only supported from 4.0, 4.1 or 5.0. Clusters running earlier releases must first + upgrade to at least the latest 4.0 release. When a cluster is upgraded to 5.1 there is an additional step for + operators to perform. The upgrade is not considered complete until: + 1. All UP nodes are running 5.1 + 2. The Cluster Metadata Service (CMS) has been enabled + The first step is just a regular upgrade, there are no changes to external APIs, SSTable formats to consider. + Step 2 requires an operator to run a new nodetool subcommand, intializecms, on one node in the cluster. + - > nodetool initializecms + Doing so creates the CMS with the local node as its only member. The initializecms command cannot be executed until + step 1 is complete and all nodes are running 5.1 as migrating metadata management over to the new TCM system + requires all nodes to be in agreement over the current state of the cluster. Essentially this means agreement on + schema and topology. Once the upgrade has started, but before initializecms is run, metadata-changing operations are + not permitted and if attempted from an upgraded node will be rejected. + Prohibited operations include: + - schema changes + - node replacement + - bootstrap + - decommission + - move + - assasinate + For the time being there is no mechanism in place to prevent these operations being executed by a node still running + a previous version, though this is planned before release. Any automation which can trigger these operations should + be disabled for the cluster prior to starting the upgrade. + Should any of the prohibited operations be executed (i.e. on a node that is still running a pre-5.1 version) before + the CMS migration, nodes which are DOWN or which have been upgraded will not process the metadata changes. However, + nodes still UP and running the old version will. This will eventually cause the migration to fail, as the cluster + will not be in agreement. + - > nodetool initializecms + Got mismatching cluster metadatas from [/x.x.x.x:7000] aborting migration + See 'nodetool help' or 'nodetool help <command>'. + If the initializecms command fails, it will indicate which nodes’ current metadata does not agree with the node + where the command was executed. To mitigate this situation, bring any mismatching nodes DOWN and rerun the + initializecms command with the additional —ignore flag. + - nodetool intializecms -ignore x.x.x.x + Once the command has run successfully the ignored nodes can be restarted but any metadata changes that they + accepted/and or applied whilst the cluster was in mixed mode will be lost. We plan to improve this before beta, but + in the meantime operators should ensure no schema or membership/ownership changes are performed during the upgrade. + Although the restrictions on metadata-changing operations will be removed as soon as the initial CMS migration is + complete, at that point the CMS will only contain a single member, which is not suitable for real clusters. To + modify the membership of the CMS a second nodetool subcommand, reconfigurecms, should be run. This command allows + the number of required number of CMS members to be specified for each datacenter. Consensus for metadata operations + by the CMS is obtained via Paxos, operating at SERIAL/QUORUM consistency, so the minimum safe size for the CMS is 3 + nodes. In multi-DC deployments, the CMS members may be distributed across DCs to ensure resilience in the case of a + DC outage. It is not recommended to make every node a member of the CMS. Redundancy is the primary concern here, not + locality or latency, so between 3 and 7 nodes per DC depending on the size of the cluster should be the goal. + Deploying to a fresh cluster is more straightforward. As the cluster comes up, at least one node will automatically + nominate itself as the first CMS node. A simple election mechanism ensures that only one node will be chosen if + multiple peers attempt to nominate themselves. As soon as the election is complete, metadata modifications are + supported. Typically, this takes only a few seconds and is likely to complete before all nodes are fully UP. Nodes + which come up during or after an election will learn of the elected first CMS node and direct metadata updates to + it. It is important to remember that at the completion of the election, the CMS still only comprises a single + member. Just as in the upgrade case, operators should add further members as soon as possible. + - native_transport_port_ssl property was removed. Please transition to using one port only. Encrypted communication + may be optional by setting `optional` flag in `client_encryption_options` to `true` and it should be set only + while in unencrypted or transitional operation. Please consult `client_encryption_options` in cassandra.yaml + for more information. + + +Deprecation +----------- ++ - `use_deterministic_table_id` is no longer supported and should be removed from cassandra.yaml. Table IDs may still be supplied explicitly on CREATE. + + 5.0 === diff --cc src/java/org/apache/cassandra/config/DatabaseDescriptor.java index fa1b65c8bf,61d4a44adb..69d46da2fc --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@@ -1009,6 -1001,8 +1009,8 @@@ public class DatabaseDescripto conf.native_transport_min_backoff_on_queue_overload, conf.native_transport_max_backoff_on_queue_overload)); + if (conf.use_deterministic_table_id) - logger.warn("use_deterministic_table_id is deprecated and will be ignored in a future release."); ++ logger.warn("use_deterministic_table_id is no longer supported and should be removed from cassandra.yaml."); } @VisibleForTesting @@@ -3477,16 -3468,16 +3479,6 @@@ return conf.hinted_handoff_disabled_datacenters; } -- public static boolean useDeterministicTableID() -- { -- return conf != null && conf.use_deterministic_table_id; -- } -- -- public static void useDeterministicTableID(boolean value) -- { -- conf.use_deterministic_table_id = value; -- } -- public static void enableHintsForDC(String dc) { conf.hinted_handoff_disabled_datacenters.remove(dc); diff --cc src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java index 59c7046223,747837f305..7c705f7fee --- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java @@@ -31,9 -34,9 +31,8 @@@ import org.apache.cassandra.audit.Audit import org.apache.cassandra.auth.DataResource; import org.apache.cassandra.auth.IResource; import org.apache.cassandra.auth.Permission; - import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.*; import org.apache.cassandra.cql3.functions.masking.ColumnMask; -import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.guardrails.Guardrails; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.exceptions.AlreadyExistsException; @@@ -116,23 -108,10 +115,23 @@@ public final class CreateTableStatemen throw new AlreadyExistsException(keyspaceName, tableName); } - TableMetadata table = builder(keyspace.types).build(); + // add all user functions to be able to give a good error message to the user if the alter references + // a function from another keyspace + UserFunctions.Builder ufBuilder = UserFunctions.builder().add(); + for (KeyspaceMetadata ksm : schema) + ufBuilder.add(ksm.userFunctions); + + TableMetadata.Builder builder = builder(keyspace.types, ufBuilder.build()).epoch(metadata.nextEpoch()); + + // We do not want to set table ID here just yet, since we are using CQL for serialising a fully expanded CREATE TABLE statement. + this.expandedCql = builder.build().toCqlString(false, attrs.hasProperty(TableAttributes.ID), ifNotExists); + - if (!attrs.hasProperty(TableAttributes.ID) && !DatabaseDescriptor.useDeterministicTableID()) ++ if (!attrs.hasProperty(TableAttributes.ID)) + builder.id(TableId.get(metadata)); + TableMetadata table = builder.build(); table.validate(); - if (keyspace.createReplicationStrategy().hasTransientReplicas() + if (keyspace.replicationStrategy.hasTransientReplicas() && table.params.readRepair != ReadRepairStrategy.NONE) { throw ire("read_repair must be set to 'NONE' for transiently replicated keyspaces"); diff --cc src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java index a4a8ddcfbe,05629e00fd..f6c8135bb1 --- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java @@@ -327,8 -324,6 +327,8 @@@ public final class CreateViewStatement if (attrs.hasProperty(TableAttributes.ID)) builder.id(attrs.getId()); - else if (!builder.hasId() && !DatabaseDescriptor.useDeterministicTableID()) ++ else if (!builder.hasId()) + builder.id(TableId.get(metadata)); builder.params(attrs.asNewTableParams()) .kind(TableMetadata.Kind.VIEW); diff --cc src/java/org/apache/cassandra/schema/TableMetadata.java index 5e2ab4b79c,93cdc35482..889264d43f --- a/src/java/org/apache/cassandra/schema/TableMetadata.java +++ b/src/java/org/apache/cassandra/schema/TableMetadata.java @@@ -827,12 -800,10 +827,12 @@@ public class TableMetadata implements S if (id == null) { - // make sure vtables use determiniestic ids so they can be referenced in calls cross-nodes + // make sure vtables use deteriminstic ids so they can be referenced in calls cross-nodes // see CASSANDRA-17295 - if (DatabaseDescriptor.useDeterministicTableID() || kind == Kind.VIRTUAL) - if (DatabaseDescriptor.useDeterministicTableID() || kind == Kind.VIRTUAL) id = TableId.unsafeDeterministic(keyspace, name); - else id = TableId.generate(); ++ if (kind == Kind.VIRTUAL) + id = TableId.unsafeDeterministic(keyspace, name); + else + id = TableId.generate(); } if (Flag.isCQLTable(flags)) diff --cc test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java index 0e881dac2d,f671dd9159..82a7d18910 --- a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java @@@ -17,17 -17,7 +17,16 @@@ */ package org.apache.cassandra.cql3.validation.operations; -import org.apache.cassandra.config.DatabaseDescriptor; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.UUID; + +import org.junit.Assert; +import org.junit.Test; + - import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.Duration; import org.apache.cassandra.db.Mutation; @@@ -40,31 -30,27 +39,32 @@@ import org.apache.cassandra.exceptions. import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.exceptions.SyntaxException; import org.apache.cassandra.io.util.DataOutputBuffer; -import org.apache.cassandra.locator.AbstractEndpointSnitch; -import org.apache.cassandra.locator.IEndpointSnitch; -import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.schema.*; -import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.schema.MemtableParams; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.SchemaKeyspaceTables; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.membership.Location; +import org.apache.cassandra.tcm.membership.NodeAddresses; +import org.apache.cassandra.tcm.membership.NodeVersion; +import org.apache.cassandra.tcm.transformations.Register; import org.apache.cassandra.triggers.ITrigger; import org.apache.cassandra.utils.ByteBufferUtil; -import org.junit.Assert; -import org.junit.Test; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.Collections; -import java.util.Map; -import java.util.UUID; import static java.lang.String.format; -import static org.apache.cassandra.cql3.Duration.*; -import static org.junit.Assert.*; +import static org.apache.cassandra.cql3.Duration.NANOS_PER_HOUR; +import static org.apache.cassandra.cql3.Duration.NANOS_PER_MICRO; +import static org.apache.cassandra.cql3.Duration.NANOS_PER_MILLI; +import static org.apache.cassandra.cql3.Duration.NANOS_PER_MINUTE; +import static org.apache.cassandra.tcm.membership.MembershipUtils.endpoint; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; ++import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class CreateTest extends CQLTester { @@@ -724,27 -729,6 +724,14 @@@ + " WITH compression = { 'class' : 'SnappyCompressor', 'unknownOption' : 32 };"); } + @Test - public void testUsingDeterministicTableID() ++ public void testNotUsingDeterministicTableIDOnCreate() + { - DatabaseDescriptor.useDeterministicTableID(true); - - createTable("CREATE TABLE %s (id text PRIMARY KEY);"); - TableMetadata tmd = currentTableMetadata(); - assertEquals(TableId.unsafeDeterministic(tmd.keyspace, tmd.name), tmd.id); - - } - - @Test - public void testNotUsingDeterministicTableIDWhenDisabled() - { - DatabaseDescriptor.useDeterministicTableID(false); - + createTable("CREATE TABLE %s (id text PRIMARY KEY);"); + TableMetadata tmd = currentTableMetadata(); - assertFalse(TableId.unsafeDeterministic(tmd.keyspace, tmd.name).equals(tmd.id)); ++ assertNotEquals(TableId.unsafeDeterministic(tmd.keyspace, tmd.name), tmd.id); + } + private void assertThrowsConfigurationException(String errorMsg, String createStmt) { try --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
