This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/phoenix-adapters.git
The following commit(s) were added to refs/heads/main by this push:
new 7259bed Use uncovered index
7259bed is described below
commit 7259bedafb78af36d5770d49a1c819d31ed0037c
Author: Palash Chauhan <[email protected]>
AuthorDate: Wed Nov 12 21:44:32 2025 -0800
Use uncovered index
---
.../phoenix/ddb/service/CreateTableService.java | 4 +-
.../apache/phoenix/ddb/ConditionalPutItemIT.java | 7 -
.../java/org/apache/phoenix/ddb/PutItemIT.java | 4 -
.../java/org/apache/phoenix/ddb/QueryIndex4IT.java | 352 +++++++++++++++++++++
.../java/org/apache/phoenix/ddb/ScanIndex3IT.java | 330 +++++++++++++++++++
.../java/org/apache/phoenix/ddb/TestUtils.java | 2 +-
6 files changed, 685 insertions(+), 14 deletions(-)
diff --git
a/phoenix-ddb-rest/src/main/java/org/apache/phoenix/ddb/service/CreateTableService.java
b/phoenix-ddb-rest/src/main/java/org/apache/phoenix/ddb/service/CreateTableService.java
index b6c4bcc..290f558 100644
---
a/phoenix-ddb-rest/src/main/java/org/apache/phoenix/ddb/service/CreateTableService.java
+++
b/phoenix-ddb-rest/src/main/java/org/apache/phoenix/ddb/service/CreateTableService.java
@@ -158,9 +158,9 @@ public class CreateTableService {
}
final String finalIndexName =
PhoenixUtils.getInternalIndexName(tableName, indexName);
- indexDDLs.add("CREATE INDEX IF NOT EXISTS \"" + finalIndexName + "\"
ON "
+ indexDDLs.add("CREATE UNCOVERED INDEX IF NOT EXISTS \"" +
finalIndexName + "\" ON "
+ PhoenixUtils.getFullTableName(tableName, true) + " (" +
indexOn
- + ") INCLUDE (COL) WHERE " + indexHashKey + " IS NOT " + "NULL
" + ((indexSortKey
+ + ") WHERE " + indexHashKey + " IS NOT " + "NULL " +
((indexSortKey
!= null) ? " AND " + indexSortKey + " IS NOT " + "NULL " : "")
+ (isAsync ?
" ASYNC " :
"") + TableOptionsConfig.getIndexOptions());
diff --git
a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ConditionalPutItemIT.java
b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ConditionalPutItemIT.java
index 5c24c14..04d48a1 100644
---
a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ConditionalPutItemIT.java
+++
b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ConditionalPutItemIT.java
@@ -264,10 +264,6 @@ public class ConditionalPutItemIT {
Assert.assertTrue(rs.next());
Assert.assertEquals(rs.getLong(1),
Long.parseLong(item1.get("attr_1").n()));
Assert.assertEquals(rs.getString(2), item1.get("attr_0").s());
- Map<String, AttributeValue> indexRowItem =
- BsonDocumentToDdbAttributes.getFullItem((BsonDocument)
rs.getObject(3));
- Assert.assertEquals(indexRowItem, item1);
- Assert.assertEquals(indexRowItem, dynamoItem);
}
}
@@ -634,9 +630,6 @@ public class ConditionalPutItemIT {
Assert.assertTrue(rs.next());
Assert.assertEquals(rs.getDouble(1),
Double.parseDouble(item2.get("Id2").n()), 0.0);
Assert.assertEquals(rs.getString(2), item2.get("attr_0").s());
- Map<String, AttributeValue> indexRowItem =
- BsonDocumentToDdbAttributes.getFullItem((BsonDocument)
rs.getObject(3));
- Assert.assertEquals(indexRowItem, item2);
//TODO: uncomment when we have utility to compare sets
//Assert.assertEquals(dynamoItem, phoenixItem);
diff --git
a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/PutItemIT.java
b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/PutItemIT.java
index 1dedb24..bdfc408 100644
--- a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/PutItemIT.java
+++ b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/PutItemIT.java
@@ -256,10 +256,6 @@ public class PutItemIT {
Assert.assertEquals(rs.getString(1), item.get("Title").s());
Assert.assertEquals(rs.getString(2), item.get("attr_0").s());
Assert.assertEquals(rs.getDouble(3),
Double.parseDouble(item.get("attr_1").n()), 0.0);
- bsonDoc = (BsonDocument) rs.getObject(4);
- Map<String, AttributeValue> indexItem =
- BsonDocumentToDdbAttributes.getFullItem(bsonDoc);
- Assert.assertEquals(item, indexItem);
//TODO: uncomment when we have utility to compare sets
//Assert.assertEquals(dynamoItem, indexItem);
diff --git
a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/QueryIndex4IT.java
b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/QueryIndex4IT.java
new file mode 100644
index 0000000..2073a99
--- /dev/null
+++ b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/QueryIndex4IT.java
@@ -0,0 +1,352 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.ddb;
+
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.phoenix.ddb.rest.RESTServer;
+import org.apache.phoenix.end2end.ServerMetadataCacheTestImpl;
+import org.apache.phoenix.jdbc.PhoenixDriver;
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.ServerUtil;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
+import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
+import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
+import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
+import software.amazon.awssdk.services.dynamodb.model.QueryRequest;
+import software.amazon.awssdk.services.dynamodb.model.QueryResponse;
+import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
+
+import static org.apache.phoenix.query.BaseTest.setUpConfigForMiniCluster;
+
+public class QueryIndex4IT {
+
+ private static final String TABLE_NAME = "QueryIndex4IT_Table";
+ private static final String INDEX_NAME = "gsi_QueryIndex4IT";
+ private static final int NUM_RECORDS = 20000;
+
+ private static DynamoDbClient dynamoDbClient;
+ private static DynamoDbClient phoenixDBClientV2;
+ private static String url;
+ private static HBaseTestingUtility utility = null;
+ private static String tmpDir;
+ private static RESTServer restServer = null;
+
+ @Rule
+ public final TestName testName = new TestName();
+
+ @BeforeClass
+ public static void initialize() throws Exception {
+ tmpDir = System.getProperty("java.io.tmpdir");
+ LocalDynamoDbTestBase.localDynamoDb().start();
+ Configuration conf = HBaseConfiguration.create();
+ utility = new HBaseTestingUtility(conf);
+ setUpConfigForMiniCluster(conf);
+
+ utility.startMiniCluster();
+ String zkQuorum = "localhost:" +
utility.getZkCluster().getClientPort();
+ url = PhoenixRuntime.JDBC_PROTOCOL +
PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkQuorum;
+
+ restServer = new RESTServer(utility.getConfiguration());
+ restServer.run();
+
+ phoenixDBClientV2 = LocalDynamoDB.createV2Client("http://"; +
restServer.getServerAddress());
+ dynamoDbClient =
LocalDynamoDbTestBase.localDynamoDb().createV2Client();
+
+ createTableAndInsertData();
+ }
+
+ private static void createTableAndInsertData() {
+ CreateTableRequest createTableRequest =
DDLTestUtils.getCreateTableRequest(
+ TABLE_NAME, "pk", ScalarAttributeType.S, "sk",
ScalarAttributeType.N);
+ createTableRequest = DDLTestUtils.addIndexToRequest(true,
createTableRequest, INDEX_NAME,
+ "category", ScalarAttributeType.S, "score",
ScalarAttributeType.N);
+
+ phoenixDBClientV2.createTable(createTableRequest);
+ dynamoDbClient.createTable(createTableRequest);
+
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ Map<String, AttributeValue> item = new HashMap<>();
+ item.put("pk", AttributeValue.builder().s("pk_" + (i %
100)).build());
+ item.put("sk",
AttributeValue.builder().n(String.valueOf(i)).build());
+ item.put("category", AttributeValue.builder().s("category_" + (i %
10)).build());
+ item.put("score",
AttributeValue.builder().n(String.valueOf(i)).build());
+ item.put("price", AttributeValue.builder().n(String.valueOf((i %
1000) + 1)).build());
+ item.put("active", AttributeValue.builder().bool(i % 2 ==
0).build());
+ item.put("name", AttributeValue.builder().s("item_" + i).build());
+ item.put("tags", AttributeValue.builder().ss("tag_" + (i % 5),
"common_tag").build());
+ item.put("counts", AttributeValue.builder().ns(String.valueOf(i %
10), String.valueOf((i % 20) + 100)).build());
+ item.put("bytes", AttributeValue.builder().bs(
+ software.amazon.awssdk.core.SdkBytes.fromByteArray(new
byte[]{(byte) (i % 128), 0x01}),
+ software.amazon.awssdk.core.SdkBytes.fromByteArray(new
byte[]{0x02, 0x03})).build());
+
+ Map<String, AttributeValue> nestedMap = new HashMap<>();
+ nestedMap.put("field1", AttributeValue.builder().s("nested_" + (i
% 3)).build());
+ nestedMap.put("field2",
AttributeValue.builder().n(String.valueOf(i % 100)).build());
+ item.put("metadata",
AttributeValue.builder().m(nestedMap).build());
+
+ List<AttributeValue> nestedList = new ArrayList<>();
+ nestedList.add(AttributeValue.builder().s("item_" + (i %
7)).build());
+ nestedList.add(AttributeValue.builder().n(String.valueOf(i %
50)).build());
+ nestedList.add(AttributeValue.builder().ss("nested_tag_" + (i %
4), "nested_common").build());
+ item.put("attributes",
AttributeValue.builder().l(nestedList).build());
+
+ PutItemRequest putRequest =
PutItemRequest.builder().tableName(TABLE_NAME).item(item).build();
+ phoenixDBClientV2.putItem(putRequest);
+ dynamoDbClient.putItem(putRequest);
+ }
+ }
+
+ @AfterClass
+ public static void stopLocalDynamoDb() throws Exception {
+ LocalDynamoDbTestBase.localDynamoDb().stop();
+ if (restServer != null) {
+ restServer.stop();
+ }
+ ServerUtil.ConnectionFactory.shutdown();
+ try {
+ DriverManager.deregisterDriver(PhoenixDriver.INSTANCE);
+ } finally {
+ if (utility != null) {
+ utility.shutdownMiniCluster();
+ }
+ ServerMetadataCacheTestImpl.resetCache();
+ }
+ System.setProperty("java.io.tmpdir", tmpDir);
+ }
+
+ @Test(timeout = 300000)
+ public void testEqualityKeyCondition() throws Exception {
+ QueryRequest.Builder qr =
QueryRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ qr.keyConditionExpression("category = :cat");
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":cat",
AttributeValue.builder().s("category_5").build());
+ qr.expressionAttributeValues(exprAttrVal);
+
+ TestUtils.compareQueryOutputs(qr, phoenixDBClientV2, dynamoDbClient);
+ TestUtils.validateIndexUsed(qr.build(), url);
+ }
+
+ @Test(timeout = 300000)
+ public void testRangeKeyCondition() throws SQLException {
+ QueryRequest.Builder qr =
QueryRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ qr.keyConditionExpression("category = :cat AND score > :minScore");
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":cat",
AttributeValue.builder().s("category_3").build());
+ exprAttrVal.put(":minScore",
AttributeValue.builder().n("5000").build());
+ qr.expressionAttributeValues(exprAttrVal);
+
+ TestUtils.compareQueryOutputs(qr, phoenixDBClientV2, dynamoDbClient);
+ TestUtils.validateIndexUsed(qr.build(), url);
+ }
+
+ @Test(timeout = 300000)
+ public void testBetweenKeyCondition() throws SQLException {
+ QueryRequest.Builder qr =
QueryRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ qr.keyConditionExpression("category = :cat AND score BETWEEN :low AND
:high");
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":cat",
AttributeValue.builder().s("category_7").build());
+ exprAttrVal.put(":low", AttributeValue.builder().n("2000").build());
+ exprAttrVal.put(":high", AttributeValue.builder().n("8000").build());
+ qr.expressionAttributeValues(exprAttrVal);
+
+ TestUtils.compareQueryOutputs(qr, phoenixDBClientV2, dynamoDbClient);
+ TestUtils.validateIndexUsed(qr.build(), url);
+ }
+
+ @Test(timeout = 300000)
+ public void testLimit() throws SQLException {
+ QueryRequest.Builder qr =
QueryRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ qr.keyConditionExpression("category = :cat");
+ qr.limit(50);
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":cat",
AttributeValue.builder().s("category_4").build());
+ qr.expressionAttributeValues(exprAttrVal);
+
+ QueryResponse phoenixResult = phoenixDBClientV2.query(qr.build());
+ QueryResponse dynamoResult = dynamoDbClient.query(qr.build());
+
+ Assert.assertEquals(dynamoResult.count(), phoenixResult.count());
+ Assert.assertTrue(ItemComparator.areItemsEqual(dynamoResult.items(),
phoenixResult.items()));
+ TestUtils.validateIndexUsed(qr.build(), url);
+ }
+
+ @Test(timeout = 300000)
+ public void testPagination() throws SQLException {
+ QueryRequest.Builder qr =
QueryRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ qr.keyConditionExpression("category = :cat AND score >= :minScore");
+ qr.limit(100);
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":cat",
AttributeValue.builder().s("category_1").build());
+ exprAttrVal.put(":minScore", AttributeValue.builder().n("0").build());
+ qr.expressionAttributeValues(exprAttrVal);
+
+ TestUtils.compareQueryOutputs(qr, phoenixDBClientV2, dynamoDbClient);
+ TestUtils.validateIndexUsed(qr.build(), url);
+ }
+
+ @Test(timeout = 300000)
+ public void testScanIndexForward() throws SQLException {
+ QueryRequest.Builder qr =
QueryRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ qr.keyConditionExpression("category = :cat");
+ qr.scanIndexForward(false);
+ qr.limit(100);
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":cat",
AttributeValue.builder().s("category_6").build());
+ qr.expressionAttributeValues(exprAttrVal);
+
+ QueryResponse phoenixResult = phoenixDBClientV2.query(qr.build());
+ QueryResponse dynamoResult = dynamoDbClient.query(qr.build());
+
+ Assert.assertEquals(dynamoResult.count(), phoenixResult.count());
+ Assert.assertTrue(ItemComparator.areItemsEqual(dynamoResult.items(),
phoenixResult.items()));
+ TestUtils.validateIndexUsed(qr.build(), url);
+ }
+
+ @Test(timeout = 300000)
+ public void testProjection() throws SQLException {
+ QueryRequest.Builder qr =
QueryRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ qr.keyConditionExpression("category = :cat");
+ qr.projectionExpression("pk, score, price");
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":cat",
AttributeValue.builder().s("category_8").build());
+ qr.expressionAttributeValues(exprAttrVal);
+
+ TestUtils.compareQueryOutputs(qr, phoenixDBClientV2, dynamoDbClient);
+ TestUtils.validateIndexUsed(qr.build(), url);
+ }
+
+ @Test(timeout = 300000)
+ public void testComplexFilterExpression() throws SQLException {
+ QueryRequest.Builder qr =
QueryRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ qr.keyConditionExpression("category = :cat AND score < :maxScore");
+ qr.filterExpression("(price BETWEEN :minPrice AND :maxPrice) OR
(active = :isActive AND #itemName <> :excludeName)");
+ Map<String, String> exprAttrNames = new HashMap<>();
+ exprAttrNames.put("#itemName", "name");
+ qr.expressionAttributeNames(exprAttrNames);
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":cat",
AttributeValue.builder().s("category_9").build());
+ exprAttrVal.put(":maxScore",
AttributeValue.builder().n("7000").build());
+ exprAttrVal.put(":minPrice",
AttributeValue.builder().n("100").build());
+ exprAttrVal.put(":maxPrice",
AttributeValue.builder().n("900").build());
+ exprAttrVal.put(":isActive",
AttributeValue.builder().bool(false).build());
+ exprAttrVal.put(":excludeName",
AttributeValue.builder().s("item_100").build());
+ qr.expressionAttributeValues(exprAttrVal);
+
+ TestUtils.compareQueryOutputs(qr, phoenixDBClientV2, dynamoDbClient);
+ TestUtils.validateIndexUsed(qr.build(), url);
+ }
+
+ @Test(timeout = 300000)
+ public void testLessThanEqual() throws SQLException {
+ QueryRequest.Builder qr =
QueryRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ qr.keyConditionExpression("category = :cat AND score <= :maxScore");
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":cat",
AttributeValue.builder().s("category_0").build());
+ exprAttrVal.put(":maxScore",
AttributeValue.builder().n("5000").build());
+ qr.expressionAttributeValues(exprAttrVal);
+
+ TestUtils.compareQueryOutputs(qr, phoenixDBClientV2, dynamoDbClient);
+ TestUtils.validateIndexUsed(qr.build(), url);
+ }
+
+ @Test(timeout = 300000)
+ public void testFilterExpression1() throws SQLException {
+ QueryRequest.Builder qr =
QueryRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ qr.keyConditionExpression("category = :cat");
+ qr.filterExpression("price > :minPrice AND active = :isActive");
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":cat",
AttributeValue.builder().s("category_2").build());
+ exprAttrVal.put(":minPrice",
AttributeValue.builder().n("500").build());
+ exprAttrVal.put(":isActive",
AttributeValue.builder().bool(true).build());
+ qr.expressionAttributeValues(exprAttrVal);
+
+ TestUtils.compareQueryOutputs(qr, phoenixDBClientV2, dynamoDbClient);
+ TestUtils.validateIndexUsed(qr.build(), url);
+ }
+
+ @Test(timeout = 300000)
+ public void testFilterExpression2() throws SQLException {
+ QueryRequest.Builder qr =
QueryRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ qr.keyConditionExpression("category = :cat AND score > :minScore");
+ qr.filterExpression("price < :maxPrice AND active = :isActive");
+ qr.limit(200);
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":cat",
AttributeValue.builder().s("category_3").build());
+ exprAttrVal.put(":minScore",
AttributeValue.builder().n("3000").build());
+ exprAttrVal.put(":maxPrice",
AttributeValue.builder().n("600").build());
+ exprAttrVal.put(":isActive",
AttributeValue.builder().bool(true).build());
+ qr.expressionAttributeValues(exprAttrVal);
+
+ QueryResponse phoenixResult = phoenixDBClientV2.query(qr.build());
+ QueryResponse dynamoResult = dynamoDbClient.query(qr.build());
+
+ Assert.assertEquals(dynamoResult.count(), phoenixResult.count());
+ Assert.assertTrue(ItemComparator.areItemsEqual(dynamoResult.items(),
phoenixResult.items()));
+ TestUtils.validateIndexUsed(qr.build(), url);
+ }
+
+ @Test(timeout = 300000)
+ public void testFilterOnSetsAndNestedMap() throws SQLException {
+ QueryRequest.Builder qr =
QueryRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ qr.keyConditionExpression("category = :cat");
+ qr.filterExpression("contains(tags, :tag) AND metadata.field1 =
:nestedVal");
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":cat",
AttributeValue.builder().s("category_2").build());
+ exprAttrVal.put(":tag", AttributeValue.builder().s("tag_1").build());
+ exprAttrVal.put(":nestedVal",
AttributeValue.builder().s("nested_1").build());
+ qr.expressionAttributeValues(exprAttrVal);
+
+ TestUtils.compareQueryOutputs(qr, phoenixDBClientV2, dynamoDbClient);
+ TestUtils.validateIndexUsed(qr.build(), url);
+ }
+
+ @Test(timeout = 300000)
+ public void testFilterOnNestedListAndSets() throws SQLException {
+ QueryRequest.Builder qr =
QueryRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ qr.keyConditionExpression("category = :cat AND score < :maxScore");
+ qr.filterExpression("contains(counts, :count) AND size(#attributes) >=
:minSize");
+ Map<String, String> exprAttrNames = new HashMap<>();
+ exprAttrNames.put("#attributes", "attributes");
+ qr.expressionAttributeNames(exprAttrNames);
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":cat",
AttributeValue.builder().s("category_7").build());
+ exprAttrVal.put(":maxScore",
AttributeValue.builder().n("15000").build());
+ exprAttrVal.put(":count", AttributeValue.builder().n("105").build());
+ exprAttrVal.put(":minSize", AttributeValue.builder().n("2").build());
+ qr.expressionAttributeValues(exprAttrVal);
+
+ TestUtils.compareQueryOutputs(qr, phoenixDBClientV2, dynamoDbClient);
+ TestUtils.validateIndexUsed(qr.build(), url);
+ }
+}
+
diff --git
a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ScanIndex3IT.java
b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ScanIndex3IT.java
new file mode 100644
index 0000000..a57200a
--- /dev/null
+++ b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ScanIndex3IT.java
@@ -0,0 +1,330 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.ddb;
+
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.phoenix.ddb.rest.RESTServer;
+import org.apache.phoenix.end2end.ServerMetadataCacheTestImpl;
+import org.apache.phoenix.jdbc.PhoenixDriver;
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.ServerUtil;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
+import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
+import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
+import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
+import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
+import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
+import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
+
+import static org.apache.phoenix.query.BaseTest.setUpConfigForMiniCluster;
+
+public class ScanIndex3IT {
+
+ private static final String TABLE_NAME = "ScanIndex3IT_Table";
+ private static final String INDEX_NAME = "gsi_ScanIndex3IT";
+ private static final int NUM_RECORDS = 18000;
+
+ private static DynamoDbClient dynamoDbClient;
+ private static DynamoDbClient phoenixDBClientV2;
+ private static String url;
+ private static HBaseTestingUtility utility = null;
+ private static String tmpDir;
+ private static RESTServer restServer = null;
+
+ @Rule
+ public final TestName testName = new TestName();
+
+ @BeforeClass
+ public static void initialize() throws Exception {
+ tmpDir = System.getProperty("java.io.tmpdir");
+ LocalDynamoDbTestBase.localDynamoDb().start();
+ Configuration conf = HBaseConfiguration.create();
+ utility = new HBaseTestingUtility(conf);
+ setUpConfigForMiniCluster(conf);
+
+ utility.startMiniCluster();
+ String zkQuorum = "localhost:" +
utility.getZkCluster().getClientPort();
+ url = PhoenixRuntime.JDBC_PROTOCOL +
PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkQuorum;
+
+ restServer = new RESTServer(utility.getConfiguration());
+ restServer.run();
+
+ phoenixDBClientV2 = LocalDynamoDB.createV2Client("http://"; +
restServer.getServerAddress());
+ dynamoDbClient =
LocalDynamoDbTestBase.localDynamoDb().createV2Client();
+
+ createTableAndInsertData();
+ }
+
+ private static void createTableAndInsertData() {
+ CreateTableRequest createTableRequest =
DDLTestUtils.getCreateTableRequest(
+ TABLE_NAME, "pk", ScalarAttributeType.S, "sk",
ScalarAttributeType.N);
+ createTableRequest = DDLTestUtils.addIndexToRequest(true,
createTableRequest, INDEX_NAME,
+ "status", ScalarAttributeType.S, "timestamp",
ScalarAttributeType.N);
+
+ phoenixDBClientV2.createTable(createTableRequest);
+ dynamoDbClient.createTable(createTableRequest);
+
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ Map<String, AttributeValue> item = new HashMap<>();
+ item.put("pk", AttributeValue.builder().s("pk_" + (i %
100)).build());
+ item.put("sk",
AttributeValue.builder().n(String.valueOf(i)).build());
+ item.put("status", AttributeValue.builder().s("status_" + (i %
10)).build());
+ item.put("timestamp",
AttributeValue.builder().n(String.valueOf(i)).build());
+ item.put("amount", AttributeValue.builder().n(String.valueOf((i %
1000) + 1)).build());
+ item.put("priority", AttributeValue.builder().n(String.valueOf(i %
10)).build());
+ item.put("active", AttributeValue.builder().bool(i % 2 ==
0).build());
+ item.put("description", AttributeValue.builder().s("description_"
+ i).build());
+ item.put("labels", AttributeValue.builder().ss("label_" + (i % 6),
"shared_label").build());
+ item.put("scores", AttributeValue.builder().ns(String.valueOf(i %
15), String.valueOf((i % 25) + 100)).build());
+ item.put("data", AttributeValue.builder().bs(
+ software.amazon.awssdk.core.SdkBytes.fromByteArray(new
byte[]{(byte) (i % 64), 0x01}),
+ software.amazon.awssdk.core.SdkBytes.fromByteArray(new
byte[]{0x0A, 0x0B})).build());
+
+ if (i % 5 == 0) {
+ item.put("optional_field",
AttributeValue.builder().s("optional_" + i).build());
+ }
+
+ Map<String, AttributeValue> nestedMap = new HashMap<>();
+ nestedMap.put("key1", AttributeValue.builder().s("value_" + (i %
4)).build());
+ nestedMap.put("key2", AttributeValue.builder().n(String.valueOf(i
% 80)).build());
+ item.put("config", AttributeValue.builder().m(nestedMap).build());
+
+ List<AttributeValue> nestedList = new ArrayList<>();
+ nestedList.add(AttributeValue.builder().s("list_item_" + (i %
8)).build());
+ nestedList.add(AttributeValue.builder().n(String.valueOf(i %
40)).build());
+ nestedList.add(AttributeValue.builder().ss("set_in_list_" + (i %
3), "list_common").build());
+ item.put("itms", AttributeValue.builder().l(nestedList).build());
+
+ PutItemRequest putRequest =
PutItemRequest.builder().tableName(TABLE_NAME).item(item).build();
+ phoenixDBClientV2.putItem(putRequest);
+ dynamoDbClient.putItem(putRequest);
+ }
+ }
+
+ @AfterClass
+ public static void stopLocalDynamoDb() throws Exception {
+ LocalDynamoDbTestBase.localDynamoDb().stop();
+ if (restServer != null) {
+ restServer.stop();
+ }
+ ServerUtil.ConnectionFactory.shutdown();
+ try {
+ DriverManager.deregisterDriver(PhoenixDriver.INSTANCE);
+ } finally {
+ if (utility != null) {
+ utility.shutdownMiniCluster();
+ }
+ ServerMetadataCacheTestImpl.resetCache();
+ }
+ System.setProperty("java.io.tmpdir", tmpDir);
+ }
+
+ @Test(timeout = 300000)
+ public void testFullTableScan() throws SQLException {
+ ScanRequest.Builder sr =
ScanRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+
+ TestUtils.compareScanOutputs(sr, phoenixDBClientV2, dynamoDbClient,
"pk", "sk",
+ ScalarAttributeType.S, ScalarAttributeType.N);
+ TestUtils.validateIndexUsed(sr.build(), url, "FULL SCAN ");
+ }
+
+ @Test(timeout = 300000)
+ public void testFilterExpression() throws SQLException {
+ ScanRequest.Builder sr =
ScanRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ sr.filterExpression("amount > :minAmount AND active = :isActive");
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":minAmount",
AttributeValue.builder().n("500").build());
+ exprAttrVal.put(":isActive",
AttributeValue.builder().bool(true).build());
+ sr.expressionAttributeValues(exprAttrVal);
+
+ TestUtils.compareScanOutputs(sr, phoenixDBClientV2, dynamoDbClient,
"pk", "sk",
+ ScalarAttributeType.S, ScalarAttributeType.N);
+ TestUtils.validateIndexUsed(sr.build(), url, "FULL SCAN ");
+ }
+
+ @Test(timeout = 300000)
+ public void testLimit() throws SQLException {
+ ScanRequest.Builder sr =
ScanRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ sr.limit(75);
+
+ ScanResponse phoenixResult = phoenixDBClientV2.scan(sr.build());
+ ScanResponse dynamoResult = dynamoDbClient.scan(sr.build());
+
+ Assert.assertEquals(dynamoResult.count(), phoenixResult.count());
+ Assert.assertTrue(ItemComparator.areItemsEqual(
+ TestUtils.sortItemsByPartitionAndSortKey(dynamoResult.items(),
"pk", "sk"),
+
TestUtils.sortItemsByPartitionAndSortKey(phoenixResult.items(), "pk", "sk")));
+ TestUtils.validateIndexUsed(sr.build(), url, "FULL SCAN ");
+ }
+
+ @Test(timeout = 300000)
+ public void testPagination() throws SQLException {
+ ScanRequest.Builder sr =
ScanRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ sr.limit(150);
+
+ TestUtils.compareScanOutputs(sr, phoenixDBClientV2, dynamoDbClient,
"pk", "sk",
+ ScalarAttributeType.S, ScalarAttributeType.N);
+ TestUtils.validateIndexUsed(sr.build(), url, "FULL SCAN ");
+ }
+
+ @Test(timeout = 300000)
+ public void testProjection() throws SQLException {
+ ScanRequest.Builder sr =
ScanRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ sr.projectionExpression("pk, #status, amount");
+ Map<String, String> exprAttrNames = new HashMap<>();
+ exprAttrNames.put("#status", "status");
+ sr.expressionAttributeNames(exprAttrNames);
+ sr.limit(50);
+
+ ScanResponse phoenixResult = phoenixDBClientV2.scan(sr.build());
+ ScanResponse dynamoResult = dynamoDbClient.scan(sr.build());
+
+ Assert.assertEquals(dynamoResult.count(), phoenixResult.count());
+ TestUtils.validateIndexUsed(sr.build(), url, "FULL SCAN ");
+ }
+
+ @Test(timeout = 300000)
+ public void testComplexFilterExpression1() throws SQLException {
+ ScanRequest.Builder sr =
ScanRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ sr.filterExpression("(amount BETWEEN :minAmt AND :maxAmt) OR (active =
:isActive AND priority > :minPriority)");
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":minAmt", AttributeValue.builder().n("200").build());
+ exprAttrVal.put(":maxAmt", AttributeValue.builder().n("800").build());
+ exprAttrVal.put(":isActive",
AttributeValue.builder().bool(false).build());
+ exprAttrVal.put(":minPriority",
AttributeValue.builder().n("7").build());
+ sr.expressionAttributeValues(exprAttrVal);
+
+ TestUtils.compareScanOutputs(sr, phoenixDBClientV2, dynamoDbClient,
"pk", "sk",
+ ScalarAttributeType.S, ScalarAttributeType.N);
+ TestUtils.validateIndexUsed(sr.build(), url, "FULL SCAN ");
+ }
+
+ @Test(timeout = 300000)
+ public void testAttributeExists() throws SQLException {
+ ScanRequest.Builder sr =
ScanRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ sr.filterExpression("attribute_exists(optional_field)");
+
+ TestUtils.compareScanOutputs(sr, phoenixDBClientV2, dynamoDbClient,
"pk", "sk",
+ ScalarAttributeType.S, ScalarAttributeType.N);
+ TestUtils.validateIndexUsed(sr.build(), url, "FULL SCAN ");
+ }
+
+ @Test(timeout = 300000)
+ public void testAttributeNotExists() throws SQLException {
+ ScanRequest.Builder sr =
ScanRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ sr.filterExpression("attribute_not_exists(optional_field)");
+
+ TestUtils.compareScanOutputs(sr, phoenixDBClientV2, dynamoDbClient,
"pk", "sk",
+ ScalarAttributeType.S, ScalarAttributeType.N);
+ TestUtils.validateIndexUsed(sr.build(), url, "FULL SCAN ");
+ }
+
+ @Test(timeout = 300000)
+ public void testComplexFilterExpression2() throws SQLException {
+ ScanRequest.Builder sr =
ScanRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ sr.filterExpression("amount > :minAmt AND priority >= :minPri AND
active = :isActive");
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":minAmt", AttributeValue.builder().n("600").build());
+ exprAttrVal.put(":minPri", AttributeValue.builder().n("5").build());
+ exprAttrVal.put(":isActive",
AttributeValue.builder().bool(true).build());
+ sr.expressionAttributeValues(exprAttrVal);
+
+ TestUtils.compareScanOutputs(sr, phoenixDBClientV2, dynamoDbClient,
"pk", "sk",
+ ScalarAttributeType.S, ScalarAttributeType.N);
+ TestUtils.validateIndexUsed(sr.build(), url, "FULL SCAN ");
+ }
+
+ @Test(timeout = 300000)
+ public void testNotCondition() throws SQLException {
+ ScanRequest.Builder sr =
ScanRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ sr.filterExpression("NOT (#status = :excludeStatus)");
+ Map<String, String> exprAttrNames = new HashMap<>();
+ exprAttrNames.put("#status", "status");
+ sr.expressionAttributeNames(exprAttrNames);
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":excludeStatus",
AttributeValue.builder().s("status_0").build());
+ sr.expressionAttributeValues(exprAttrVal);
+
+ TestUtils.compareScanOutputs(sr, phoenixDBClientV2, dynamoDbClient,
"pk", "sk",
+ ScalarAttributeType.S, ScalarAttributeType.N);
+ TestUtils.validateIndexUsed(sr.build(), url, "FULL SCAN ");
+ }
+
+ @Test(timeout = 300000)
+ public void testInCondition() throws SQLException {
+ ScanRequest.Builder sr =
ScanRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ sr.filterExpression("#status IN (:status1, :status2, :status3)");
+ Map<String, String> exprAttrNames = new HashMap<>();
+ exprAttrNames.put("#status", "status");
+ sr.expressionAttributeNames(exprAttrNames);
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":status1",
AttributeValue.builder().s("status_1").build());
+ exprAttrVal.put(":status2",
AttributeValue.builder().s("status_3").build());
+ exprAttrVal.put(":status3",
AttributeValue.builder().s("status_7").build());
+ sr.expressionAttributeValues(exprAttrVal);
+
+ TestUtils.compareScanOutputs(sr, phoenixDBClientV2, dynamoDbClient,
"pk", "sk",
+ ScalarAttributeType.S, ScalarAttributeType.N);
+ TestUtils.validateIndexUsed(sr.build(), url, "FULL SCAN ");
+ }
+
+ @Test(timeout = 300000)
+ public void testFilterOnSetsAndNestedMap() throws SQLException {
+ ScanRequest.Builder sr =
ScanRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ sr.filterExpression("contains(labels, :label) AND config.key1 =
:configVal");
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":label",
AttributeValue.builder().s("label_3").build());
+ exprAttrVal.put(":configVal",
AttributeValue.builder().s("value_2").build());
+ sr.expressionAttributeValues(exprAttrVal);
+
+ TestUtils.compareScanOutputs(sr, phoenixDBClientV2, dynamoDbClient,
"pk", "sk",
+ ScalarAttributeType.S, ScalarAttributeType.N);
+ TestUtils.validateIndexUsed(sr.build(), url, "FULL SCAN ");
+ }
+
+ @Test(timeout = 300000)
+ public void testFilterOnNestedListAndMultipleSets() throws SQLException {
+ ScanRequest.Builder sr =
ScanRequest.builder().tableName(TABLE_NAME).indexName(INDEX_NAME);
+ sr.filterExpression("contains(scores, :score) AND size(itms) =
:listSize AND contains(labels, :label)");
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":score", AttributeValue.builder().n("110").build());
+ exprAttrVal.put(":listSize", AttributeValue.builder().n("3").build());
+ exprAttrVal.put(":label",
AttributeValue.builder().s("shared_label").build());
+ sr.expressionAttributeValues(exprAttrVal);
+
+ TestUtils.compareScanOutputs(sr, phoenixDBClientV2, dynamoDbClient,
"pk", "sk",
+ ScalarAttributeType.S, ScalarAttributeType.N);
+ TestUtils.validateIndexUsed(sr.build(), url, "FULL SCAN ");
+ }
+}
+
diff --git
a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/TestUtils.java
b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/TestUtils.java
index 4a51347..6b1c9d2 100644
--- a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/TestUtils.java
+++ b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/TestUtils.java
@@ -387,7 +387,7 @@ public class TestUtils {
qr.exclusiveStartKey(ddbResponse.lastEvaluatedKey());
} while (ddbResponse.hasLastEvaluatedKey());
Assert.assertEquals(ddbResult.size(), phoenixResult.size());
- Assert.assertEquals(ddbResult, phoenixResult);
+ Assert.assertTrue(ItemComparator.areItemsEqual(ddbResult,
phoenixResult));
}
public static void compareScanOutputs(ScanRequest.Builder sr,
