MaxNevermind commented on code in PR #1273:
URL: https://github.com/apache/parquet-mr/pull/1273#discussion_r1524234166
##########
parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java:
##########
@@ -137,16 +160,31 @@ public ParquetRewriter(RewriteOptions options) throws
IOException {
getPaths(schema, paths, null);
for (String col : pruneColumns) {
if (!paths.contains(col)) {
- LOG.warn("Input column name {} doesn't show up in the schema of file
{}", col, reader.getFile());
+ LOG.warn("Input column name {} doesn't show up in the schema", col);
}
}
Set<ColumnPath> prunePaths = convertToColumnPaths(pruneColumns);
schema = pruneColumnsInSchema(schema, prunePaths);
}
- this.descriptorsMap =
- schema.getColumns().stream().collect(Collectors.toMap(x ->
ColumnPath.get(x.getPath()), x -> x));
+ if (inputFilesR.isEmpty()) { // TODO: find a more suitable solution
+ this.descriptorsMap =
+ schema.getColumns().stream().collect(Collectors.toMap(x ->
ColumnPath.get(x.getPath()), x -> x));
+ } else {
+ this.descriptorsMap = schemaL.getColumns().stream()
+ .filter(x -> x.getPath().length == 0 ||
!fieldNamesR.containsKey(x.getPath()[0]))
+ .collect(Collectors.toMap(x -> ColumnPath.get(x.getPath()), x -> x));
+ }
+
+ long rowCountL =
inputFiles.stream().mapToLong(ParquetFileReader::getRecordCount).sum();
+ inputFilesR.stream()
+ .map(x ->
x.stream().mapToLong(ParquetFileReader::getRecordCount).sum())
+ .forEach(rowCountR -> {
+ if (rowCountL != rowCountR) {
+ throw new IllegalArgumentException("The number of records on the
left and on the right don't match!");
Review Comment:
done
##########
parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java:
##########
@@ -813,7 +841,7 @@ private Type extractField(GroupType candidate, Type
targetField) {
} else {
Type tempField = extractField(field.asGroupType(), targetField);
if (tempField != null) {
- return tempField;
+ return new GroupType(candidate.getRepetition(), candidate.getName(),
tempField);
Review Comment:
yeah, that's the issue I mentioned in email communication, it become a
problem when I started a using a different for the right side file group
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
