This is an automated email from the ASF dual-hosted git repository.
yashmayya pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new b54479b78c1 Fix ColocatedJoinEmptyPartitionTest flake from deleting a
table's uploaded segment tars (#19304)
b54479b78c1 is described below
commit b54479b78c128b6dc4c2bae01b20f04d4098d0b0
Author: Yash Mayya <[email protected]>
AuthorDate: Wed Aug 19 00:16:55 2026 -0400
Fix ColocatedJoinEmptyPartitionTest flake from deleting a table's uploaded
segment tars (#19304)
---
.../custom/ColocatedJoinEmptyPartitionTest.java | 29 ++++++++++++++++++----
1 file changed, 24 insertions(+), 5 deletions(-)
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/ColocatedJoinEmptyPartitionTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/ColocatedJoinEmptyPartitionTest.java
index 3c75ccae35b..797ff14d35f 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/ColocatedJoinEmptyPartitionTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/ColocatedJoinEmptyPartitionTest.java
@@ -131,6 +131,20 @@ public class ColocatedJoinEmptyPartitionTest extends
CustomDataQueryClusterInteg
throws Exception {
setUpTable(LEFT_TABLE_NAME, LEFT_POPULATED_PARTITIONS,
LEFT_METRIC_MULTIPLIER);
setUpTable(RIGHT_TABLE_NAME, RIGHT_POPULATED_PARTITIONS,
RIGHT_METRIC_MULTIPLIER);
+ // Setting up the second table must not have removed the first one's tar
files, see setUpTable(String, List, int).
+ assertTarFilesRetained(LEFT_TABLE_NAME, LEFT_POPULATED_PARTITIONS.size());
+ assertTarFilesRetained(RIGHT_TABLE_NAME,
RIGHT_POPULATED_PARTITIONS.size());
+ }
+
+ /// Asserts that the segment tar files uploaded for the given table are
still on disk. A metadata-only push, which
+ /// `ClusterTest#uploadSegments` selects at random, makes the tar file the
only deep store copy, so deleting one
+ /// leaves its segment stuck in ERROR -- and only for a table whose segments
no server had fetched yet, which is a
+ /// race that fails rarely and far from its cause.
+ private void assertTarFilesRetained(String tableName, int
expectedNumSegments) {
+ File[] tarFiles = new File(_tarDir, tableName).listFiles();
+ assertNotNull(tarFiles, "Missing tar directory for table: " + tableName);
+ assertEquals(tarFiles.length, expectedNumSegments,
+ "Unexpected number of segment tar files for table: " + tableName);
}
@Override
@@ -358,17 +372,22 @@ public class ColocatedJoinEmptyPartitionTest extends
CustomDataQueryClusterInteg
TableConfig tableConfig = createTableConfigForTable(tableName);
addTableConfig(tableConfig);
- // The segment directories are shared across tables, and uploadSegments
pushes everything it finds in the tar one.
- TestUtils.ensureDirectoriesExistAndEmpty(_segmentDir, _tarDir);
+ // Give each table its own directories, and never empty a directory
another table already uploaded from. A
+ // metadata-only push records a file:// download URI that points at the
tar file, and the servers read it after the
+ // upload call has returned, so deleting that file makes the segment
unloadable. Separate directories also keep
+ // uploadSegments, which pushes every tar it finds, from picking up the
other table's segments.
+ File segmentDir = new File(_segmentDir, tableName);
+ File tarDir = new File(_tarDir, tableName);
+ TestUtils.ensureDirectoriesExistAndEmpty(segmentDir, tarDir);
int segmentIndex = 0;
for (int partition : populatedPartitions) {
// One segment per partition, so that every segment holds exactly one
partition id (a segment spanning several has
// no usable partition metadata) and every partition has a fully
replicated server.
File avroFile = createAvroFile(tableName, partition, metricMultiplier);
- ClusterIntegrationTestUtils.buildSegmentFromAvro(avroFile, tableConfig,
schema, segmentIndex++, _segmentDir,
- _tarDir);
+ ClusterIntegrationTestUtils.buildSegmentFromAvro(avroFile, tableConfig,
schema, segmentIndex++, segmentDir,
+ tarDir);
}
- uploadSegments(tableName, _tarDir);
+ uploadSegments(tableName, tarDir);
}
private static Schema createSchemaForTable(String tableName) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
