singhpk234 commented on code in PR #15096:
URL: https://github.com/apache/iceberg/pull/15096#discussion_r2715100071
##########
core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java:
##########
@@ -1040,4 +1118,135 @@ public void serverSupportsPlanningButNotCancellation()
throws IOException {
// Verify no exception was thrown - cancelPlan returns false when endpoint
not supported
assertThat(cancelled).isFalse();
}
+
+ @Test
+ public void planTableScanReplaysResponseForSameIdempotencyKey() {
+ configurePlanningBehavior(TestPlanningBehavior.Builder::asynchronous);
+
+ TableIdentifier ident = TableIdentifier.of(NS, "idempotent_plan_scan");
+ Table table = createTableWithScanPlanning(scanPlanningCatalog(), ident);
+ setParserContext(table);
+
+ Map<String, String> headers =
idempotencyHeader("test-idempotency-key-planTableScan");
+ PlanTableScanRequest request = defaultPlanRequest();
+
+ List<PlanTableScanResponse> responses =
+ executeTwice(
+ HTTPRequest.HTTPMethod.POST,
+ planPath(ident),
+ headers,
+ request,
+ PlanTableScanResponse.class,
+ ErrorHandlers.tableErrorHandler());
+ PlanTableScanResponse first = responses.get(0);
+ PlanTableScanResponse second = responses.get(1);
Review Comment:
i am not sure if execute twice is getting something here we any ways are
writing 2 loc to get the first and the second invocation ?
##########
core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java:
##########
@@ -229,6 +239,74 @@ private void configurePlanningBehavior(
adapterForRESTServer.setPlanningBehavior(configurator.apply(builder).build());
}
+ private String planPath(TableIdentifier ident) {
+ return String.format(
+ "v1/namespaces/%s/tables/%s/plan",
+ RESTUtil.encodeNamespace(ident.namespace(),
RESTUtil.NAMESPACE_SEPARATOR_URLENCODED_UTF_8),
+ RESTUtil.encodeString(ident.name()));
+ }
+
+ private String tasksPath(TableIdentifier ident) {
+ return String.format(
+ "v1/namespaces/%s/tables/%s/tasks",
+ RESTUtil.encodeNamespace(ident.namespace(),
RESTUtil.NAMESPACE_SEPARATOR_URLENCODED_UTF_8),
+ RESTUtil.encodeString(ident.name()));
+ }
+
+ private String cancelPath(TableIdentifier ident, String planId) {
+ return String.format(
+ "v1/namespaces/%s/tables/%s/plan/%s",
+ RESTUtil.encodeNamespace(ident.namespace(),
RESTUtil.NAMESPACE_SEPARATOR_URLENCODED_UTF_8),
+ RESTUtil.encodeString(ident.name()),
+ RESTUtil.encodeString(planId));
+ }
Review Comment:
can reuse this
https://github.com/apache/iceberg/blob/main/core/src/main/java/org/apache/iceberg/rest/ResourcePaths.java#L154
##########
core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java:
##########
@@ -1040,4 +1118,135 @@ public void serverSupportsPlanningButNotCancellation()
throws IOException {
// Verify no exception was thrown - cancelPlan returns false when endpoint
not supported
assertThat(cancelled).isFalse();
}
+
+ @Test
+ public void planTableScanReplaysResponseForSameIdempotencyKey() {
+ configurePlanningBehavior(TestPlanningBehavior.Builder::asynchronous);
+
+ TableIdentifier ident = TableIdentifier.of(NS, "idempotent_plan_scan");
+ Table table = createTableWithScanPlanning(scanPlanningCatalog(), ident);
+ setParserContext(table);
+
+ Map<String, String> headers =
idempotencyHeader("test-idempotency-key-planTableScan");
+ PlanTableScanRequest request = defaultPlanRequest();
+
+ List<PlanTableScanResponse> responses =
+ executeTwice(
+ HTTPRequest.HTTPMethod.POST,
+ planPath(ident),
+ headers,
+ request,
+ PlanTableScanResponse.class,
+ ErrorHandlers.tableErrorHandler());
+ PlanTableScanResponse first = responses.get(0);
+ PlanTableScanResponse second = responses.get(1);
+
+ assertThat(first.planStatus()).isEqualTo(PlanStatus.SUBMITTED);
+ assertThat(second.planId()).isEqualTo(first.planId());
+ }
+
+ @Test
+ public void fetchScanTasksReplaysResponseForSameIdempotencyKey() {
+
configurePlanningBehavior(TestPlanningBehavior.Builder::synchronousWithPagination);
+
+ TableIdentifier ident = TableIdentifier.of(NS,
"idempotent_fetch_scan_tasks");
+ Table table = createTableWithScanPlanning(scanPlanningCatalog(), ident);
+ // Ensure 2 data files so tasksPerPage=1 produces a next plan task that
must be fetched via
+ // fetchScanTasks.
+ table.newAppend().appendFile(FILE_A).appendFile(FILE_B).commit();
+ setParserContext(table);
+
+ PlanTableScanResponse plan =
+ execute(
+ HTTPRequest.HTTPMethod.POST,
+ planPath(ident),
+ Map.of(),
+ defaultPlanRequest(),
+ PlanTableScanResponse.class,
+ ErrorHandlers.tableErrorHandler());
+
+ assertThat(plan.planTasks()).isNotNull();
+ assertThat(plan.planTasks()).isNotEmpty();
+ String nextPlanTask = plan.planTasks().get(0);
+ FetchScanTasksRequest tasksRequest = new
FetchScanTasksRequest(nextPlanTask);
+
+ Map<String, String> headers =
idempotencyHeader("test-idempotency-key-fetchScanTasks");
+ FetchScanTasksResponse first =
+ execute(
+ HTTPRequest.HTTPMethod.POST,
+ tasksPath(ident),
+ headers,
+ tasksRequest,
+ FetchScanTasksResponse.class,
+ ErrorHandlers.planTaskHandler());
+
+ CatalogHandlers.cancelPlanTableScan(plan.planId());
+
+ FetchScanTasksResponse second =
+ execute(
+ HTTPRequest.HTTPMethod.POST,
+ tasksPath(ident),
+ headers,
+ tasksRequest,
+ FetchScanTasksResponse.class,
+ ErrorHandlers.planTaskHandler());
+
+ // We cancel the planning state before this call, so a *fresh execution*
of fetchScanTasks would
Review Comment:
```suggestion
// We cancel the planning state before this call, so a fresh execution
of fetchScanTasks would
```
??
##########
core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java:
##########
@@ -1040,4 +1118,135 @@ public void serverSupportsPlanningButNotCancellation()
throws IOException {
// Verify no exception was thrown - cancelPlan returns false when endpoint
not supported
assertThat(cancelled).isFalse();
}
+
+ @Test
+ public void planTableScanReplaysResponseForSameIdempotencyKey() {
+ configurePlanningBehavior(TestPlanningBehavior.Builder::asynchronous);
+
+ TableIdentifier ident = TableIdentifier.of(NS, "idempotent_plan_scan");
+ Table table = createTableWithScanPlanning(scanPlanningCatalog(), ident);
+ setParserContext(table);
+
+ Map<String, String> headers =
idempotencyHeader("test-idempotency-key-planTableScan");
+ PlanTableScanRequest request = defaultPlanRequest();
+
+ List<PlanTableScanResponse> responses =
+ executeTwice(
+ HTTPRequest.HTTPMethod.POST,
+ planPath(ident),
+ headers,
+ request,
+ PlanTableScanResponse.class,
+ ErrorHandlers.tableErrorHandler());
+ PlanTableScanResponse first = responses.get(0);
+ PlanTableScanResponse second = responses.get(1);
+
+ assertThat(first.planStatus()).isEqualTo(PlanStatus.SUBMITTED);
+ assertThat(second.planId()).isEqualTo(first.planId());
+ }
+
+ @Test
+ public void fetchScanTasksReplaysResponseForSameIdempotencyKey() {
+
configurePlanningBehavior(TestPlanningBehavior.Builder::synchronousWithPagination);
+
+ TableIdentifier ident = TableIdentifier.of(NS,
"idempotent_fetch_scan_tasks");
+ Table table = createTableWithScanPlanning(scanPlanningCatalog(), ident);
+ // Ensure 2 data files so tasksPerPage=1 produces a next plan task that
must be fetched via
+ // fetchScanTasks.
+ table.newAppend().appendFile(FILE_A).appendFile(FILE_B).commit();
+ setParserContext(table);
+
+ PlanTableScanResponse plan =
+ execute(
+ HTTPRequest.HTTPMethod.POST,
+ planPath(ident),
+ Map.of(),
+ defaultPlanRequest(),
+ PlanTableScanResponse.class,
+ ErrorHandlers.tableErrorHandler());
+
+ assertThat(plan.planTasks()).isNotNull();
+ assertThat(plan.planTasks()).isNotEmpty();
+ String nextPlanTask = plan.planTasks().get(0);
+ FetchScanTasksRequest tasksRequest = new
FetchScanTasksRequest(nextPlanTask);
+
+ Map<String, String> headers =
idempotencyHeader("test-idempotency-key-fetchScanTasks");
+ FetchScanTasksResponse first =
+ execute(
+ HTTPRequest.HTTPMethod.POST,
+ tasksPath(ident),
+ headers,
+ tasksRequest,
+ FetchScanTasksResponse.class,
+ ErrorHandlers.planTaskHandler());
+
+ CatalogHandlers.cancelPlanTableScan(plan.planId());
+
+ FetchScanTasksResponse second =
+ execute(
+ HTTPRequest.HTTPMethod.POST,
+ tasksPath(ident),
+ headers,
+ tasksRequest,
+ FetchScanTasksResponse.class,
+ ErrorHandlers.planTaskHandler());
+
+ // We cancel the planning state before this call, so a *fresh execution*
of fetchScanTasks would
+ // fail. The only way this second call can succeed is if the server
replays the cached response
+ // for the Idempotency-Key. We validate that replay by comparing the
returned payload's stable
+ // identifiers (data file locations) to the first response. Use
order-insensitive comparison
+ // since task ordering isn't guaranteed.
Review Comment:
why isn't the task ordering gauranteed ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
