hililiwei commented on code in PR #7171:
URL: https://github.com/apache/iceberg/pull/7171#discussion_r1189289880
##########
flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkManifestUtil.java:
##########
@@ -73,21 +78,43 @@ static ManifestOutputFileFactory createOutputFileFactory(
attemptNumber);
}
+ /**
+ * Write the {@link WriteResult} to temporary manifest files.
+ *
+ * @param result all those DataFiles/DeleteFiles in this WriteResult should
be written with same
+ * partition spec
+ */
static DeltaManifests writeCompletedFiles(
- WriteResult result, Supplier<OutputFile> outputFileSupplier,
PartitionSpec spec)
+ WriteResult result,
+ Supplier<OutputFile> outputFileSupplier,
+ Map<Integer, PartitionSpec> specsById)
throws IOException {
ManifestFile dataManifest = null;
ManifestFile deleteManifest = null;
+ PartitionSpec spec = null;
// Write the completed data files into a newly created data manifest file.
if (result.dataFiles() != null && result.dataFiles().length > 0) {
+ int specId = result.dataFiles()[0].specId();
+ spec = specsById.get(specId);
+ Preconditions.checkState(
+ Arrays.stream(result.dataFiles()).allMatch(file -> file.specId() ==
specId),
+ "All data files should have same partition spec");
dataManifest =
writeDataFiles(outputFileSupplier.get(), spec,
Lists.newArrayList(result.dataFiles()));
}
// Write the completed delete files into a newly created delete manifest
file.
if (result.deleteFiles() != null && result.deleteFiles().length > 0) {
+ if (spec == null) {
+ spec = specsById.get(result.deleteFiles()[0].specId());
+ }
+ int specId = spec.specId();
Review Comment:
nit: new empty line.
##########
flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkManifestUtil.java:
##########
@@ -73,21 +78,43 @@ static ManifestOutputFileFactory createOutputFileFactory(
attemptNumber);
}
+ /**
+ * Write the {@link WriteResult} to temporary manifest files.
+ *
+ * @param result all those DataFiles/DeleteFiles in this WriteResult should
be written with same
+ * partition spec
+ */
static DeltaManifests writeCompletedFiles(
- WriteResult result, Supplier<OutputFile> outputFileSupplier,
PartitionSpec spec)
+ WriteResult result,
+ Supplier<OutputFile> outputFileSupplier,
+ Map<Integer, PartitionSpec> specsById)
throws IOException {
ManifestFile dataManifest = null;
ManifestFile deleteManifest = null;
+ PartitionSpec spec = null;
// Write the completed data files into a newly created data manifest file.
if (result.dataFiles() != null && result.dataFiles().length > 0) {
+ int specId = result.dataFiles()[0].specId();
+ spec = specsById.get(specId);
+ Preconditions.checkState(
+ Arrays.stream(result.dataFiles()).allMatch(file -> file.specId() ==
specId),
Review Comment:
use
`Arrays.stream(result.deleteFiles()).map(ContentFile::specId).distinct().count()
> 1` ?
##########
flink/v1.16/flink/src/test/java/org/apache/iceberg/flink/SimpleDataUtil.java:
##########
@@ -134,6 +135,19 @@ public static DataFile writeFile(
String filename,
List<RowData> rows)
throws IOException {
+ return writeFile(table, schema, spec, conf, location, filename, rows,
null);
+ }
+
+ public static DataFile writeFile(
Review Comment:
nit: add doc?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
