chinnaraolalam commented on code in PR #6927:
URL: https://github.com/apache/hive/pull/6927#discussion_r4618280328


##########
iceberg/iceberg-handler/src/test/queries/positive/unknown_type.q:
##########
@@ -52,3 +53,190 @@ CREATE TABLE unknown_test_ctas STORED BY ICEBERG 
tblproperties('format-version'=
 DESC FORMATTED unknown_test_ctas;
 
 SELECT id, placeholder, extra FROM unknown_test_ctas ORDER BY id;
+
+-- Partial column insert: omitted UNKNOWN column is implicit NULL
+CREATE EXTERNAL TABLE unknown_test_partial (
+    name STRING,
+    id UNKNOWN
+) STORED BY ICEBERG tblproperties('format-version'='3');
+
+INSERT INTO unknown_test_partial (name) VALUES ('aqc');
+
+SELECT name, id FROM unknown_test_partial;
+
+-- Explicit NULL in a column list
+INSERT INTO unknown_test_partial (name, id) VALUES ('explicit', NULL);
+
+-- Multi-row partial column insert
+INSERT INTO unknown_test_partial (name) VALUES ('multi1'), ('multi2');
+
+-- Partial column insert from a SELECT
+INSERT INTO unknown_test_partial (name) SELECT CAST(id AS STRING) FROM 
unknown_test_basic;
+
+SELECT name, id FROM unknown_test_partial ORDER BY name;
+
+-- Rewriting an UNKNOWN column with its own values
+INSERT OVERWRITE TABLE unknown_test_partial SELECT name, id FROM 
unknown_test_partial WHERE name LIKE 'multi%';
+
+SELECT name, id FROM unknown_test_partial ORDER BY name;
+
+-- UNKNOWN column between other columns, omitted from the column list
+CREATE EXTERNAL TABLE unknown_test_middle (
+    a INT,
+    b UNKNOWN,
+    c INT
+) STORED BY ICEBERG tblproperties('format-version'='3');
+
+INSERT INTO unknown_test_middle (a, c) VALUES (1, 10);
+INSERT INTO unknown_test_middle (c, a) VALUES (20, 2);
+INSERT INTO unknown_test_middle VALUES (3, NULL, 30);
+
+SELECT a, b, c FROM unknown_test_middle ORDER BY a;
+
+-- Filters and aggregates on an UNKNOWN column
+SELECT count(*) FROM unknown_test_middle WHERE b IS NULL;
+SELECT count(*) FROM unknown_test_middle WHERE b IS NOT NULL;
+SELECT count(b), count(*) FROM unknown_test_middle;
+
+-- GROUP BY, DISTINCT, windowing and join with an UNKNOWN column
+SELECT b, count(*) FROM unknown_test_middle GROUP BY b;
+SELECT DISTINCT b FROM unknown_test_middle;
+SELECT a, b, row_number() OVER (PARTITION BY b ORDER BY a) FROM 
unknown_test_middle;
+SELECT m.a, m.b, p.placeholder FROM unknown_test_middle m JOIN 
unknown_test_basic p ON m.a = p.id;
+
+-- Row-mode execution and non-converted fetch
+set hive.vectorized.execution.enabled=false;
+SELECT a, b, c FROM unknown_test_middle ORDER BY a;
+SELECT b, count(*) FROM unknown_test_middle GROUP BY b;
+set hive.vectorized.execution.enabled=true;
+
+set hive.fetch.task.conversion=none;
+SELECT a, b, c FROM unknown_test_middle;
+set hive.fetch.task.conversion=more;
+
+-- Views over UNKNOWN columns
+CREATE VIEW unknown_test_view AS SELECT a, b FROM unknown_test_middle;
+SELECT a, b FROM unknown_test_view ORDER BY a;
+
+-- UPDATE, DELETE and MERGE on a table with an UNKNOWN column
+UPDATE unknown_test_middle SET c = 11 WHERE a = 1;
+UPDATE unknown_test_middle SET b = NULL WHERE a = 3;
+DELETE FROM unknown_test_middle WHERE a = 2;
+MERGE INTO unknown_test_middle t
+USING (SELECT 3 AS a, 33 AS c UNION ALL SELECT 4 AS a, 40 AS c) s
+ON t.a = s.a
+WHEN MATCHED THEN UPDATE SET c = s.c
+WHEN NOT MATCHED THEN INSERT (a, c) VALUES (s.a, s.c);
+
+SELECT a, b, c FROM unknown_test_middle ORDER BY a;
+
+-- Column statistics on a table with an UNKNOWN column
+ANALYZE TABLE unknown_test_middle COMPUTE STATISTICS FOR COLUMNS;
+
+SELECT a, b, c FROM unknown_test_middle ORDER BY a;
+
+-- UNKNOWN field inside a struct
+CREATE EXTERNAL TABLE unknown_test_nested (
+    id INT,
+    s STRUCT<x:INT, u:UNKNOWN>
+) STORED BY ICEBERG tblproperties('format-version'='3');
+
+INSERT INTO unknown_test_nested (id) VALUES (1);
+INSERT INTO unknown_test_nested SELECT 2, named_struct('x', 5, 'u', NULL);
+
+SELECT id, s, s.u FROM unknown_test_nested ORDER BY id;
+
+-- NULL is the only default an UNKNOWN column accepts
+CREATE EXTERNAL TABLE unknown_test_show (
+    id INT,
+    u UNKNOWN DEFAULT NULL,
+    s STRUCT<x:INT, u:UNKNOWN>
+) STORED BY ICEBERG tblproperties('format-version'='3');
+
+SHOW CREATE TABLE unknown_test_show;
+
+INSERT INTO unknown_test_show (id) VALUES (1);
+
+SELECT id, u, s FROM unknown_test_show;
+
+-- ORC and Avro data files
+CREATE EXTERNAL TABLE unknown_test_orc (
+    a INT,
+    b UNKNOWN,
+    c INT,
+    s STRUCT<x:INT, u:UNKNOWN>
+) STORED BY ICEBERG tblproperties('format-version'='3', 
'write.format.default'='orc');
+
+INSERT INTO unknown_test_orc (a, c) VALUES (1, 10), (2, 20);
+INSERT INTO unknown_test_orc SELECT 3, NULL, 30, named_struct('x', 5, 'u', 
NULL);
+UPDATE unknown_test_orc SET c = 11 WHERE a = 1;
+DELETE FROM unknown_test_orc WHERE a = 2;
+
+SELECT a, b, c, s, s.u FROM unknown_test_orc ORDER BY a;
+SELECT count(*) FROM unknown_test_orc WHERE b IS NULL AND c > 10;
+
+set hive.vectorized.execution.enabled=false;
+SELECT a, b, c, s, s.u FROM unknown_test_orc ORDER BY a;
+set hive.vectorized.execution.enabled=true;
+
+CREATE EXTERNAL TABLE unknown_test_avro (
+    a INT,
+    b UNKNOWN,
+    c INT,
+    s STRUCT<x:INT, u:UNKNOWN>
+) STORED BY ICEBERG tblproperties('format-version'='3', 
'write.format.default'='avro');
+
+INSERT INTO unknown_test_avro (a, c) VALUES (1, 10), (2, 20);
+INSERT INTO unknown_test_avro SELECT 3, NULL, 30, named_struct('x', 5, 'u', 
NULL);
+UPDATE unknown_test_avro SET c = 11 WHERE a = 1;
+DELETE FROM unknown_test_avro WHERE a = 2;
+
+SELECT a, b, c, s, s.u FROM unknown_test_avro ORDER BY a;
+SELECT count(*) FROM unknown_test_avro WHERE b IS NULL AND c > 10;
+
+-- Promote an UNKNOWN column to a concrete type; existing rows read as NULL
+ALTER TABLE unknown_test_middle CREATE TAG before_promotion;
+
+ALTER TABLE unknown_test_middle CHANGE COLUMN b b INT;
+

Review Comment:
   after changing the column b INT how the view looks (SELECT * FROM 
unknown_test_view)



##########
iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java:
##########
@@ -539,6 +544,7 @@ public void 
commitAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable
           if (transaction != null) {
             transaction.commitTransaction();
           }
+          commitUnknownTypePromotion();

Review Comment:
   I can see its 2 commits operations, after first one success and second one 
fails. How to looks, is it tested?



##########
iceberg/iceberg-handler/src/main/java/org/apache/iceberg/orc/VectorizedReadUtils.java:
##########
@@ -142,25 +145,29 @@ public static void handleIcebergProjection(FileScanTask 
task, JobConf job,
     PartitionSpec spec = task.spec();
     Schema currentSchema = spec.schema();
 
+    Schema readSchema = currentSchema;
     TypeDescription readOrcSchema;
     if (ORCSchemaUtil.hasIds(fileSchema)) {
       readOrcSchema = ORCSchemaUtil.buildOrcProjection(currentSchema, 
fileSchema);
     } else {
-      Schema readSchemaForOriginalFile = currentSchema;
       // In case of migrated, originally partitioned tables, partition values 
are not present in the file
       if (spec.isPartitioned()) {
-        readSchemaForOriginalFile = 
currentSchema.select(currentSchema.columns().stream()
+        readSchema = currentSchema.select(currentSchema.columns().stream()
             .filter(c -> !spec.identitySourceIds().contains(c.fieldId()))
             .map(c -> c.name())
             .collect(Collectors.toList()));
       }
 
       TypeDescription typeWithIds =
           ORCSchemaUtil.applyNameMapping(fileSchema, 
MappingUtil.create(currentSchema));
-      readOrcSchema = 
ORCSchemaUtil.buildOrcProjection(readSchemaForOriginalFile, typeWithIds);
+      readOrcSchema = ORCSchemaUtil.buildOrcProjection(readSchema, 
typeWithIds);
     }
 
-    job.set(ColumnProjectionUtils.ORC_SCHEMA_STRING, readOrcSchema.toString());
+    boolean hasUnknownType = 
TypeUtil.indexById(readSchema.asStruct()).values().stream()
+        .anyMatch(field -> field.type().typeId() == Type.TypeID.UNKNOWN);

Review Comment:
   is this simple "TypeUtil.find(readSchema, t -> t.typeId() == UNKNOWN) != 
null"



##########
iceberg/iceberg-handler/src/test/results/negative/unknown_type_struct_all_unknown.q.out:
##########
@@ -0,0 +1,8 @@
+PREHOOK: query: CREATE EXTERNAL TABLE unknown_test_struct (
+  id INT,
+  s STRUCT<u:UNKNOWN>
+) STORED BY ICEBERG tblproperties('format-version'='3')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unknown_test_struct
+FAILED: Execution Error, return code 40000 from 
org.apache.hadoop.hive.ql.ddl.DDLTask. java.lang.IllegalArgumentException: 
Column 's' of type struct<3: u: optional unknown> is not supported: a struct 
must have at least one field that is not of type unknown

Review Comment:
   struct<3: u: optional unknown> this is struct<3: u: unknown> correct?



##########
iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java:
##########
@@ -453,6 +456,45 @@ private Schema getSchemaWithIdentifierFields(Schema 
schema, Set<String> identifi
     return new Schema(cols, identifierFieldIds);
   }
 
+  /**
+   * UNKNOWN values are omitted from data files, so Iceberg can only write 
them as top-level columns or as struct
+   * fields that have at least one sibling of another type. Lists, maps and 
all-UNKNOWN structs would have no
+   * physical representation left.
+   */
+  static void validateUnknownType(String columnPath, Type type) {
+    switch (type.typeId()) {
+      case STRUCT -> {
+        List<Types.NestedField> fields = type.asStructType().fields();
+        Preconditions.checkArgument(fields.stream().anyMatch(field -> 
!isUnknown(field.type())),
+            "Column '%s' of type %s is not supported: a struct must have at 
least one field that is not of type " +

Review Comment:
   CREATE TABLE t (u UNKNOWN) will this work



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to