szehon-ho edited a comment on issue #2783:
URL: https://github.com/apache/iceberg/issues/2783#issuecomment-873869272
Adding a reproduction test (can run in spark3-extensions test case, for
example):
```
package org.apache.iceberg.spark.extensions;
import com.google.common.collect.Lists;
import org.apache.iceberg.spark.SparkCatalog;
import org.apache.iceberg.spark.source.SimpleRecord;
import org.apache.iceberg.spark.source.SparkTable;
import org.apache.spark.sql.Column;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.connector.catalog.Identifier;
import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
import scala.Some;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
public class TestSparkMetadataTable extends SparkExtensionsTestBase {
public TestSparkMetadataTable(String catalogName, String implementation,
Map<String, String> config) {
super(catalogName, implementation, config);
}
@Test
public void testDataFileProjectionError() throws Exception {
// init load
List<SimpleRecord> records = Lists.newArrayList(new SimpleRecord(1,
"1"));
Dataset<Row> inputDf = spark.createDataFrame(records,
SimpleRecord.class);
inputDf.writeTo(tableName).create();
Dataset<Row> stringDs = spark.createDataset(Arrays.asList("my_path"),
Encoders.STRING())
.toDF("file_path");
SparkCatalog catalog = (SparkCatalog)
spark.sessionState().catalogManager().catalog(catalogName);
String[] tableIdentifiers = tableName.split("\\.");
Identifier metaId = Identifier.of(
new String[]{tableIdentifiers[1], tableIdentifiers[2]}, "entries");
SparkTable metaTable = catalog.loadTable(metaId);
Dataset<Row> entriesDs = Dataset.ofRows(spark,
DataSourceV2Relation.create(metaTable, Some.apply(catalog), Some.apply(
metaId)));
Column joinCond =
entriesDs.col("data_file.file_path").equalTo(stringDs.col("file_path"));
Dataset<Row> res = entriesDs.join(stringDs, joinCond);
boolean empty = res.isEmpty();
Assert.assertEquals(true, empty);
}
@After
public void dropTables() {
sql("DROP TABLE IF EXISTS %s", tableName);
}
}
```
Side note: I could not use a simple aggregation to reproduce it due to the
other error (two issues filed of the same error):
https://github.com/apache/iceberg/issues/1378 and
https://github.com/apache/iceberg/issues/1735
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
