This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/phoenix-adapters.git
The following commit(s) were added to refs/heads/main by this push:
new 64fb2db Additional tests for scan and query with indexes
64fb2db is described below
commit 64fb2dbdc428c1e43d61e6e6e69c6482b0c48b7a
Author: Viraj Jasani <[email protected]>
AuthorDate: Sat Nov 15 04:23:59 2025 -0800
Additional tests for scan and query with indexes
---
.../java/org/apache/phoenix/ddb/Misc1Util.java | 64 ++++++++++++++++++++++
.../java/org/apache/phoenix/ddb/ScanIndex3IT.java | 13 +++++
.../java/org/apache/phoenix/ddb/TestUtils.java | 2 -
3 files changed, 77 insertions(+), 2 deletions(-)
diff --git
a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/Misc1Util.java
b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/Misc1Util.java
index cbb1e0f..398b6ac 100644
--- a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/Misc1Util.java
+++ b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/Misc1Util.java
@@ -860,6 +860,14 @@ public class Misc1Util {
batchWriteItems(dynamoDbClient, phoenixDBClientV2, table1Name,
table1Items);
batchWriteItems(dynamoDbClient, phoenixDBClientV2, table2Name,
table2Items);
+ List<Map<String, AttributeValue>> table1ItemsToDelete =
+ selectItemsToDelete(table1Items, numPartitions, 0.22);
+ List<Map<String, AttributeValue>> table2ItemsToDelete =
+ selectItemsToDelete(table2Items, numPartitions, 0.27);
+
+ batchDeleteItems(dynamoDbClient, phoenixDBClientV2, table1Name,
table1ItemsToDelete);
+ batchDeleteItems(dynamoDbClient, phoenixDBClientV2, table2Name,
table2ItemsToDelete);
+
for (int partitionId = 0; partitionId < numPartitions; partitionId++) {
String pkValueTable1 = "pk_" + partitionId + "_t1";
String pkValueTable2 = "pk_" + partitionId + "_t2";
@@ -1104,6 +1112,62 @@ public class Misc1Util {
}
}
+ private static void batchDeleteItems(DynamoDbClient dynamoDbClient,
+ DynamoDbClient phoenixDBClientV2, String tableName,
+ List<Map<String, AttributeValue>> itemKeys) {
+ int batchSize = 25;
+ int totalBatches = (int) Math.ceil((double) itemKeys.size() /
batchSize);
+
+ for (int batchNum = 0; batchNum < totalBatches; batchNum++) {
+ int startIdx = batchNum * batchSize;
+ int endIdx = Math.min(startIdx + batchSize, itemKeys.size());
+ List<Map<String, AttributeValue>> batchKeys =
itemKeys.subList(startIdx, endIdx);
+
+ List<WriteRequest> deleteRequests = batchKeys.stream().map(key ->
WriteRequest.builder()
+ .deleteRequest(DeleteRequest.builder()
+
.key(key).build()).build()).collect(Collectors.toList());
+
+ Map<String, List<WriteRequest>> requestItems =
+ Collections.singletonMap(tableName, deleteRequests);
+
+ BatchWriteItemRequest batchDeleteRequest =
+
BatchWriteItemRequest.builder().requestItems(requestItems).build();
+
+ dynamoDbClient.batchWriteItem(batchDeleteRequest);
+ phoenixDBClientV2.batchWriteItem(batchDeleteRequest);
+ }
+ }
+
+ private static List<Map<String, AttributeValue>> selectItemsToDelete(
+ List<Map<String, AttributeValue>> items, int numPartitions, double
deletePercentage) {
+ List<Map<String, AttributeValue>> itemsToDelete = new ArrayList<>();
+ int itemsPerPartition = (int) Math.ceil((double) items.size() /
numPartitions);
+
+ for (int partitionId = 0; partitionId < numPartitions; partitionId++) {
+ int startIdx = partitionId * itemsPerPartition;
+ int endIdx = Math.min(startIdx + itemsPerPartition, items.size());
+ int itemsInThisPartition = endIdx - startIdx;
+ int itemsToDeleteFromPartition =
+ (int) Math.ceil(itemsInThisPartition * deletePercentage);
+
+ for (int i = 0; i < itemsToDeleteFromPartition && (startIdx + i) <
endIdx; i++) {
+ int itemIdx = startIdx + (i * itemsInThisPartition /
itemsToDeleteFromPartition);
+ if (itemIdx < endIdx) {
+ Map<String, AttributeValue> item = items.get(itemIdx);
+ Map<String, AttributeValue> key = new HashMap<>();
+ key.put("pk", item.get("pk"));
+ key.put("sk", item.get("sk"));
+ itemsToDelete.add(key);
+ }
+ }
+ }
+
+ itemsToDelete.sort(
+ Comparator.comparing((Map<String, AttributeValue> item) ->
item.get("pk").s())
+ .thenComparing(item ->
Long.parseLong(item.get("sk").n())));
+ return itemsToDelete;
+ }
+
private static void queryGSIsWithoutConditions(DynamoDbClient
dynamoDbClient,
DynamoDbClient phoenixDBClientV2, String tableName, String
tableSuffix) {
diff --git
a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ScanIndex3IT.java
b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ScanIndex3IT.java
index a57200a..ad584a4 100644
--- a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ScanIndex3IT.java
+++ b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ScanIndex3IT.java
@@ -41,6 +41,7 @@ import org.junit.rules.TestName;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
+import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest;
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
@@ -129,6 +130,18 @@ public class ScanIndex3IT {
phoenixDBClientV2.putItem(putRequest);
dynamoDbClient.putItem(putRequest);
}
+
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ if (i % 4 == 0 || i % 11 == 0) {
+ Map<String, AttributeValue> key = new HashMap<>();
+ key.put("pk", AttributeValue.builder().s("pk_" + (i %
100)).build());
+ key.put("sk",
AttributeValue.builder().n(String.valueOf(i)).build());
+ DeleteItemRequest deleteRequest =
+
DeleteItemRequest.builder().tableName(TABLE_NAME).key(key).build();
+ phoenixDBClientV2.deleteItem(deleteRequest);
+ dynamoDbClient.deleteItem(deleteRequest);
+ }
+ }
}
@AfterClass
diff --git
a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/TestUtils.java
b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/TestUtils.java
index 6b1c9d2..7afecc9 100644
--- a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/TestUtils.java
+++ b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/TestUtils.java
@@ -341,7 +341,6 @@ public class TestUtils {
phoenixRecordsResponse =
phoenixDBStreamsClientV2.getRecords(phoenixRecordsRequest);
allPhoenixRecords.addAll(phoenixRecordsResponse.records());
phoenixShardIterator = phoenixRecordsResponse.nextShardIterator();
- LOGGER.info("Phoenix shard iterator: {}", phoenixShardIterator);
} while (phoenixShardIterator != null &&
!phoenixRecordsResponse.records().isEmpty());
do {
@@ -350,7 +349,6 @@ public class TestUtils {
dynamoRecordsResponse =
dynamoDbStreamsClient.getRecords(dynamoRecordsRequest);
allDynamoRecords.addAll(dynamoRecordsResponse.records());
dynamoShardIterator = dynamoRecordsResponse.nextShardIterator();
- LOGGER.info("Dynamo shard iterator: {}", dynamoShardIterator);
} while (dynamoShardIterator != null &&
!dynamoRecordsResponse.records().isEmpty());
LOGGER.info("Phoenix stream records count: {}",
allPhoenixRecords.size());
