This is an automated email from the ASF dual-hosted git repository. asf-gitbox-commits pushed a commit to branch PHOENIX-7998 in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit 1cec6d6288b50d720306714186ef5073b2d3ac25 Author: Andrew Purtell <[email protected]> AuthorDate: Tue Oct 6 00:32:54 2026 -0700 PHOENIX-7998 Phase 14: Compatibility and version validation Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> --- .../phoenix/compile/CreateIndexCompiler.java | 10 ++ .../java/org/apache/phoenix/schema/PTable.java | 3 +- .../java/org/apache/phoenix/util/MetaDataUtil.java | 2 +- .../phoenix/compile/VectorIndexCompilerTest.java | 33 +++++++ .../expression/ExpressionTypeOrdinalTest.java | 106 +++++++++++++++++++++ .../apache/phoenix/schema/VectorIndexTypeTest.java | 2 + .../phoenix/schema/types/VectorDataTypeTest.java | 18 ++++ .../org/apache/phoenix/util/MetaDataUtilTest.java | 22 +++++ 8 files changed, 194 insertions(+), 2 deletions(-) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java index e59852cc15..02870ab007 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java @@ -48,6 +48,7 @@ import org.apache.phoenix.parse.ParseNode; import org.apache.phoenix.parse.StatelessTraverseAllParseNodeVisitor; import org.apache.phoenix.parse.SubqueryParseNode; import org.apache.phoenix.parse.TableName; +import org.apache.phoenix.query.ConnectionQueryServices; import org.apache.phoenix.schema.MetaDataClient; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PTable; @@ -304,6 +305,15 @@ public class CreateIndexCompiler { */ private void verifyVectorIndex(CreateIndexStatement create, StatementContext context) throws SQLException { + if ( + !context.getConnection().getQueryServices() + .supportsFeature(ConnectionQueryServices.Feature.VECTOR_INDEX) + ) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.INCOMPATIBLE_CLIENT_SERVER_JAR) + .setMessage("Cannot create a vector index: at least one server predates vector index " + + "support. A server upgrade is required.") + .build().buildException(); + } PTable dataTable = context.getResolver().getTables().get(0).getTable(); if (dataTable.getType() == PTableType.VIEW) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java index a4b7f520af..86f01821d7 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java @@ -137,7 +137,8 @@ public interface PTable extends PMetaDataEntity { public static IndexType fromSerializedValue(byte serializedValue) { if (serializedValue < 1 || serializedValue > IndexType.values().length) { - throw new IllegalArgumentException("Invalid IndexType " + serializedValue); + throw new IllegalArgumentException("Unsupported IndexType " + serializedValue + + ": the index was created by a newer Phoenix version. Upgrade this client to read it."); } return IndexType.values()[serializedValue - 1]; } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core-client/src/main/java/org/apache/phoenix/util/MetaDataUtil.java index 3bbaac572c..486aa977a4 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/util/MetaDataUtil.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/util/MetaDataUtil.java @@ -264,7 +264,7 @@ public class MetaDataUtil { } public static boolean decodeHasIndexWALCodec(long version) { - return (version & 0xF) == 0; + return (version & 0x1) == 0; } // Given the encoded integer representing the client hbase version in the encoded version value. diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/VectorIndexCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/VectorIndexCompilerTest.java index ab69629485..7a58a39a07 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/VectorIndexCompilerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/VectorIndexCompilerTest.java @@ -20,6 +20,7 @@ package org.apache.phoenix.compile; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.sql.DriverManager; @@ -30,6 +31,8 @@ import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.query.BaseConnectionlessQueryTest; +import org.apache.phoenix.query.ConnectionQueryServices; +import org.apache.phoenix.query.DelegateConnectionQueryServices; import org.apache.phoenix.util.PropertiesUtil; import org.junit.BeforeClass; import org.junit.Test; @@ -50,6 +53,36 @@ public class VectorIndexCompilerTest extends BaseConnectionlessQueryTest { } } + /** + * A vector index cannot be created while any server predates vector index support, and the error + * directs the operator to upgrade the server. + */ + @Test + public void testVectorIndexRequiresServerSupport() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + ConnectionQueryServices oldServer = + new DelegateConnectionQueryServices(conn.getQueryServices()) { + @Override + public boolean supportsFeature(Feature feature) { + return feature != Feature.VECTOR_INDEX && super.supportsFeature(feature); + } + }; + try (PhoenixConnection oldConn = new PhoenixConnection(conn, oldServer, props)) { + PhoenixStatement stmt = oldConn.createStatement().unwrap(PhoenixStatement.class); + stmt.compileMutation("CREATE VECTOR INDEX idx ON T_VEC (V1) WITH (algorithm='IVF', " + + "metric='L2', lists=16, sample_size=500)"); + fail("Expected CREATE VECTOR INDEX to be refused without server vector support"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.INCOMPATIBLE_CLIENT_SERVER_JAR.getErrorCode(), + e.getErrorCode()); + String msg = e.getMessage().toLowerCase(); + assertTrue(e.getMessage(), msg.contains("server") && msg.contains("upgrade")); + } + } + } + @Test public void testInvalidDistanceMetricRejection() throws Exception { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); diff --git a/phoenix-core/src/test/java/org/apache/phoenix/expression/ExpressionTypeOrdinalTest.java b/phoenix-core/src/test/java/org/apache/phoenix/expression/ExpressionTypeOrdinalTest.java new file mode 100644 index 0000000000..71be31a86c --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/expression/ExpressionTypeOrdinalTest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/** + * Guards the ordinal stability of {@link ExpressionType}. Expressions are serialized by ordinal + * between clients and servers of different versions, so entries may only be appended. + */ +public class ExpressionTypeOrdinalTest { + + /** Every entry present before vector support, in its released order. */ + private static final String[] RELEASED = { "ReverseFunction", "RowKey", "KeyValue", + "LiteralValue", "RoundFunction", "FloorFunction", "CeilFunction", "RoundDateExpression", + "FloorDateExpression", "CeilDateExpression", "RoundTimestampExpression", + "CeilTimestampExpression", "RoundDecimalExpression", "FloorDecimalExpression", + "CeilDecimalExpression", "TruncFunction", "ToDateFunction", "ToCharFunction", + "ToNumberFunction", "CoerceFunction", "SubstrFunction", "AndExpression", "OrExpression", + "ComparisonExpression", "CountAggregateFunction", "SumAggregateFunction", + "MinAggregateFunction", "MaxAggregateFunction", "StringBasedLikeExpression", "NotExpression", + "CaseExpression", "InListExpression", "IsNullExpression", "LongSubtractExpression", + "DateSubtractExpression", "DecimalSubtractExpression", "LongAddExpression", + "DecimalAddExpression", "DateAddExpression", "LongMultiplyExpression", + "DecimalMultiplyExpression", "LongDivideExpression", "DecimalDivideExpression", + "CoalesceFunction", "StringBasedRegexpReplaceFunction", "SQLTypeNameFunction", + "StringBasedRegexpSubstrFunction", "StringConcatExpression", "LengthFunction", "LTrimFunction", + "RTrimFunction", "UpperFunction", "LowerFunction", "TrimFunction", + "DistinctCountAggregateFunction", "PercentileContAggregateFunction", + "PercentRankAggregateFunction", "StddevPopFunction", "StddevSampFunction", + "PercentileDiscAggregateFunction", "DoubleAddExpression", "DoubleSubtractExpression", + "DoubleMultiplyExpression", "DoubleDivideExpression", "RowValueConstructorExpression", + "MD5Function", "SQLTableTypeFunction", "IndexStateName", "InvertFunction", + "ProjectedColumnExpression", "TimestampAddExpression", "TimestampSubtractExpression", + "ArrayIndexFunction", "ArrayLengthFunction", "ArrayConstructorExpression", + "SQLViewTypeFunction", "ExternalSqlTypeIdFunction", "ConvertTimezoneFunction", "DecodeFunction", + "TimezoneOffsetFunction", "EncodeFunction", "LpadFunction", "NthValueFunction", + "FirstValueFunction", "LastValueFunction", "ArrayAnyComparisonExpression", + "ArrayAllComparisonExpression", "InlineArrayElemRefExpression", "SQLIndexTypeFunction", + "ModulusExpression", "DistinctValueAggregateFunction", "StringBasedRegexpSplitFunction", + "RandomFunction", "ToTimeFunction", "ToTimestampFunction", "ByteBasedLikeExpression", + "ByteBasedRegexpReplaceFunction", "ByteBasedRegexpSubstrFunction", + "ByteBasedRegexpSplitFunction", "LikeExpression", "RegexpReplaceFunction", + "RegexpSubstrFunction", "RegexpSplitFunction", "SignFunction", "YearFunction", "MonthFunction", + "SecondFunction", "WeekFunction", "HourFunction", "NowFunction", "InstrFunction", + "MinuteFunction", "DayOfMonthFunction", "ArrayAppendFunction", "UDFExpression", + "ArrayPrependFunction", "SqrtFunction", "AbsFunction", "CbrtFunction", "LnFunction", + "LogFunction", "ExpFunction", "PowerFunction", "ArrayConcatFunction", "ArrayFillFunction", + "ArrayToStringFunction", "StringToArrayFunction", "GetByteFunction", "SetByteFunction", + "GetBitFunction", "SetBitFunction", "OctetLengthFunction", "RoundWeekExpression", + "RoundMonthExpression", "RoundYearExpression", "FloorWeekExpression", "FloorMonthExpression", + "FloorYearExpression", "CeilWeekExpression", "CeilMonthExpression", "CeilYearExpression", + "DayOfWeekFunction", "DayOfYearFunction", "DefaultValueExpression", "ArrayColumnExpression", + "FirstValuesFunction", "LastValuesFunction", "DistinctCountHyperLogLogAggregateFunction", + "CollationKeyFunction", "ArrayRemoveFunction", "TransactionProviderNameFunction", + "MathPIFunction", "SinFunction", "CosFunction", "TanFunction", "RowKeyBytesStringFunction", + "PhoenixRowTimestampFunction", "JsonValueFunction", "JsonQueryFunction", "JsonExistsFunction", + "JsonModifyFunction", "BsonConditionExpressionFunction", "BsonUpdateExpressionFunction", + "BsonValueFunction", "BsonValueTypeFunction", "PartitionIdFunction", "DecodeBinaryFunction", + "EncodeBinaryFunction", "DecodeViewIdFunction", "SubBinaryFunction", "ScanStartKeyFunction", + "ScanEndKeyFunction", "TotalSegmentsFunction", "RowSizeFunction", "RawRowSizeFunction", + "RegexpLikeFunction", "ByteBasedRegexpLikeFunction", "StringBasedRegexpLikeFunction", }; + + @Test + public void testReleasedOrdinalsAreUnchanged() { + for (int i = 0; i < RELEASED.length; i++) { + assertEquals("ordinal of " + RELEASED[i], i, ExpressionType.valueOf(RELEASED[i]).ordinal()); + } + } + + @Test + public void testVectorExpressionsAreAppended() { + String[] vectorTypes = { "L2DistanceFunction", "L2DistanceSquaredFunction", + "CosineDistanceFunction", "InnerProductDistanceFunction", "BsonVectorValueFunction" }; + for (String name : vectorTypes) { + assertTrue(name + " must follow every released entry", + ExpressionType.valueOf(name).ordinal() >= RELEASED.length); + } + } + + @Test + public void testOrdinalsAreContiguous() { + ExpressionType[] values = ExpressionType.values(); + for (int i = 0; i < values.length; i++) { + assertEquals(i, values[i].ordinal()); + } + } +} diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/VectorIndexTypeTest.java b/phoenix-core/src/test/java/org/apache/phoenix/schema/VectorIndexTypeTest.java index 8675d02f1c..12fff88a31 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/schema/VectorIndexTypeTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/VectorIndexTypeTest.java @@ -67,6 +67,8 @@ public class VectorIndexTypeTest { fail("Expected IllegalArgumentException for unknown serialized value 99"); } catch (IllegalArgumentException e) { assertTrue("Exception message should mention invalid value", e.getMessage().contains("99")); + assertTrue("Exception message should direct an upgrade", + e.getMessage().toLowerCase().contains("upgrade")); } } diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/types/VectorDataTypeTest.java b/phoenix-core/src/test/java/org/apache/phoenix/schema/types/VectorDataTypeTest.java index 9ed8b4d29d..03b9d6bd28 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/schema/types/VectorDataTypeTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/types/VectorDataTypeTest.java @@ -55,6 +55,24 @@ public class VectorDataTypeTest { assertTrue(factory.getTypes().contains(PVectorDouble.INSTANCE)); } + /** + * SQL type ids and serialization ordinals identify a type across versions, so the vector types + * must hold fixed values that collide with no other registered type. + */ + @Test + public void testTypeIdentifiersAreUniqueAndStable() { + assertEquals(51, PVectorFloat.INSTANCE.ordinal()); + assertEquals(52, PVectorDouble.INSTANCE.ordinal()); + java.util.Set<Integer> sqlTypes = new java.util.HashSet<>(); + java.util.Set<Integer> ordinals = new java.util.HashSet<>(); + for (PDataType<?> type : PDataTypeFactory.getInstance().getTypes()) { + assertTrue("duplicate sql type " + type.getSqlType() + " for " + type, + sqlTypes.add(type.getSqlType())); + assertTrue("duplicate ordinal " + type.ordinal() + " for " + type, + ordinals.add(type.ordinal())); + } + } + @Test public void testSqlTypeNameResolution() { PDataTypeFactory factory = PDataTypeFactory.getInstance(); diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java index 9589d06ecf..6f3bcd76e6 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java @@ -51,6 +51,7 @@ import org.apache.phoenix.coprocessorclient.MetaDataProtocol; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; +import org.apache.phoenix.hbase.index.util.IndexManagementUtil; import org.apache.phoenix.hbase.index.util.KeyValueBuilder; import org.apache.phoenix.hbase.index.util.VersionUtil; import org.apache.phoenix.query.HBaseFactoryProvider; @@ -293,6 +294,27 @@ public class MetaDataUtilTest { assertEquals(expectedPhoenixVersion, phoenixVersion); } + /** + * The WAL codec flag is bit 0 of the encoded server version. A server with the indexing WAL codec + * configured must decode as having it, whatever the other low bits hold, and the version this + * server encodes must also decode that way under the mask released clients apply (low nibble). + */ + @Test + public void testDecodeHasIndexWALCodec() { + Configuration config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration(); + config.set(IndexManagementUtil.WAL_CELL_CODEC_CLASS_KEY, + IndexManagementUtil.INDEX_WAL_EDIT_CODEC_CLASS_NAME); + long withCodec = MetaDataUtil.encodeVersion("2.5.0", config); + assertTrue(MetaDataUtil.decodeHasIndexWALCodec(withCodec)); + assertEquals("released clients decode with a low-nibble mask", 0, withCodec & 0xF); + assertTrue(MetaDataUtil.decodeHasIndexWALCodec(withCodec | 0xE)); + + config.unset(IndexManagementUtil.WAL_CELL_CODEC_CLASS_KEY); + long withoutCodec = MetaDataUtil.encodeVersion("2.5.0", config); + assertFalse(MetaDataUtil.decodeHasIndexWALCodec(withoutCodec)); + assertFalse(MetaDataUtil.decodeHasIndexWALCodec(withoutCodec | 0xE)); + } + private Put generateOriginalPut() { String version = VersionInfo.getVersion(); KeyValueBuilder builder = KeyValueBuilder.get(version);
