huaxingao commented on code in PR #15096:
URL: https://github.com/apache/iceberg/pull/15096#discussion_r2718330422
##########
core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java:
##########
@@ -1040,4 +1118,135 @@ public void serverSupportsPlanningButNotCancellation()
throws IOException {
// Verify no exception was thrown - cancelPlan returns false when endpoint
not supported
assertThat(cancelled).isFalse();
}
+
+ @Test
+ public void planTableScanReplaysResponseForSameIdempotencyKey() {
+ configurePlanningBehavior(TestPlanningBehavior.Builder::asynchronous);
+
+ TableIdentifier ident = TableIdentifier.of(NS, "idempotent_plan_scan");
+ Table table = createTableWithScanPlanning(scanPlanningCatalog(), ident);
+ setParserContext(table);
+
+ Map<String, String> headers =
idempotencyHeader("test-idempotency-key-planTableScan");
+ PlanTableScanRequest request = defaultPlanRequest();
+
+ List<PlanTableScanResponse> responses =
+ executeTwice(
+ HTTPRequest.HTTPMethod.POST,
+ planPath(ident),
+ headers,
+ request,
+ PlanTableScanResponse.class,
+ ErrorHandlers.tableErrorHandler());
+ PlanTableScanResponse first = responses.get(0);
+ PlanTableScanResponse second = responses.get(1);
+
+ assertThat(first.planStatus()).isEqualTo(PlanStatus.SUBMITTED);
+ assertThat(second.planId()).isEqualTo(first.planId());
+ }
+
+ @Test
+ public void fetchScanTasksReplaysResponseForSameIdempotencyKey() {
+
configurePlanningBehavior(TestPlanningBehavior.Builder::synchronousWithPagination);
+
+ TableIdentifier ident = TableIdentifier.of(NS,
"idempotent_fetch_scan_tasks");
+ Table table = createTableWithScanPlanning(scanPlanningCatalog(), ident);
+ // Ensure 2 data files so tasksPerPage=1 produces a next plan task that
must be fetched via
+ // fetchScanTasks.
+ table.newAppend().appendFile(FILE_A).appendFile(FILE_B).commit();
+ setParserContext(table);
+
+ PlanTableScanResponse plan =
+ execute(
+ HTTPRequest.HTTPMethod.POST,
+ planPath(ident),
+ Map.of(),
+ defaultPlanRequest(),
+ PlanTableScanResponse.class,
+ ErrorHandlers.tableErrorHandler());
+
+ assertThat(plan.planTasks()).isNotNull();
+ assertThat(plan.planTasks()).isNotEmpty();
+ String nextPlanTask = plan.planTasks().get(0);
+ FetchScanTasksRequest tasksRequest = new
FetchScanTasksRequest(nextPlanTask);
+
+ Map<String, String> headers =
idempotencyHeader("test-idempotency-key-fetchScanTasks");
+ FetchScanTasksResponse first =
+ execute(
+ HTTPRequest.HTTPMethod.POST,
+ tasksPath(ident),
+ headers,
+ tasksRequest,
+ FetchScanTasksResponse.class,
+ ErrorHandlers.planTaskHandler());
+
+ CatalogHandlers.cancelPlanTableScan(plan.planId());
+
+ FetchScanTasksResponse second =
+ execute(
+ HTTPRequest.HTTPMethod.POST,
+ tasksPath(ident),
+ headers,
+ tasksRequest,
+ FetchScanTasksResponse.class,
+ ErrorHandlers.planTaskHandler());
+
+ // We cancel the planning state before this call, so a *fresh execution*
of fetchScanTasks would
Review Comment:
Fixed
##########
core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java:
##########
@@ -1040,4 +1118,135 @@ public void serverSupportsPlanningButNotCancellation()
throws IOException {
// Verify no exception was thrown - cancelPlan returns false when endpoint
not supported
assertThat(cancelled).isFalse();
}
+
+ @Test
+ public void planTableScanReplaysResponseForSameIdempotencyKey() {
+ configurePlanningBehavior(TestPlanningBehavior.Builder::asynchronous);
+
+ TableIdentifier ident = TableIdentifier.of(NS, "idempotent_plan_scan");
+ Table table = createTableWithScanPlanning(scanPlanningCatalog(), ident);
+ setParserContext(table);
+
+ Map<String, String> headers =
idempotencyHeader("test-idempotency-key-planTableScan");
+ PlanTableScanRequest request = defaultPlanRequest();
+
+ List<PlanTableScanResponse> responses =
+ executeTwice(
+ HTTPRequest.HTTPMethod.POST,
+ planPath(ident),
+ headers,
+ request,
+ PlanTableScanResponse.class,
+ ErrorHandlers.tableErrorHandler());
+ PlanTableScanResponse first = responses.get(0);
+ PlanTableScanResponse second = responses.get(1);
+
+ assertThat(first.planStatus()).isEqualTo(PlanStatus.SUBMITTED);
+ assertThat(second.planId()).isEqualTo(first.planId());
+ }
+
+ @Test
+ public void fetchScanTasksReplaysResponseForSameIdempotencyKey() {
+
configurePlanningBehavior(TestPlanningBehavior.Builder::synchronousWithPagination);
+
+ TableIdentifier ident = TableIdentifier.of(NS,
"idempotent_fetch_scan_tasks");
+ Table table = createTableWithScanPlanning(scanPlanningCatalog(), ident);
+ // Ensure 2 data files so tasksPerPage=1 produces a next plan task that
must be fetched via
+ // fetchScanTasks.
+ table.newAppend().appendFile(FILE_A).appendFile(FILE_B).commit();
+ setParserContext(table);
+
+ PlanTableScanResponse plan =
+ execute(
+ HTTPRequest.HTTPMethod.POST,
+ planPath(ident),
+ Map.of(),
+ defaultPlanRequest(),
+ PlanTableScanResponse.class,
+ ErrorHandlers.tableErrorHandler());
+
+ assertThat(plan.planTasks()).isNotNull();
+ assertThat(plan.planTasks()).isNotEmpty();
+ String nextPlanTask = plan.planTasks().get(0);
+ FetchScanTasksRequest tasksRequest = new
FetchScanTasksRequest(nextPlanTask);
+
+ Map<String, String> headers =
idempotencyHeader("test-idempotency-key-fetchScanTasks");
+ FetchScanTasksResponse first =
+ execute(
+ HTTPRequest.HTTPMethod.POST,
+ tasksPath(ident),
+ headers,
+ tasksRequest,
+ FetchScanTasksResponse.class,
+ ErrorHandlers.planTaskHandler());
+
+ CatalogHandlers.cancelPlanTableScan(plan.planId());
+
+ FetchScanTasksResponse second =
+ execute(
+ HTTPRequest.HTTPMethod.POST,
+ tasksPath(ident),
+ headers,
+ tasksRequest,
+ FetchScanTasksResponse.class,
+ ErrorHandlers.planTaskHandler());
+
+ // We cancel the planning state before this call, so a *fresh execution*
of fetchScanTasks would
+ // fail. The only way this second call can succeed is if the server
replays the cached response
+ // for the Idempotency-Key. We validate that replay by comparing the
returned payload's stable
+ // identifiers (data file locations) to the first response. Use
order-insensitive comparison
+ // since task ordering isn't guaranteed.
Review Comment:
In `CatalogHandlers.planFilesFor`, we iterate `scan.planFiles()` and
directly `Iterables.partition(planTasks, tasksPerPlanTask)` without sorting, so
task order is whatever `planFiles()` yields and isn’t guaranteed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
