This is an automated email from the ASF dual-hosted git repository.

philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 01e71ec06b [GLUTEN-6625][VL] Support null elements in complex-type 
literals (#13035)
01e71ec06b is described below

commit 01e71ec06b814c7b63af0306c24d9974bd2a30e8
Author: Kai Jiang <[email protected]>
AuthorDate: Sat Nov 7 17:39:04 2026 -0800

    [GLUTEN-6625][VL] Support null elements in complex-type literals (#13035)
---
 .../gluten/execution/VeloxLiteralSuite.scala       |  27 +++-
 cpp/velox/substrait/SubstraitToVeloxExpr.cc        | 115 ++++++++++++----
 .../gluten/utils/velox/VeloxTestSettings.scala     |   8 ++
 .../sql/GlutenRuntimeNullChecksV2Writes.scala      | 146 ++++++++++++++++++---
 .../gluten/utils/velox/VeloxTestSettings.scala     |   8 ++
 .../sql/GlutenRuntimeNullChecksV2Writes.scala      | 144 +++++++++++++++++---
 .../gluten/utils/velox/VeloxTestSettings.scala     |   8 ++
 .../sql/GlutenRuntimeNullChecksV2Writes.scala      | 144 +++++++++++++++++---
 .../gluten/utils/velox/VeloxTestSettings.scala     |   8 ++
 .../sql/GlutenRuntimeNullChecksV2Writes.scala      | 144 +++++++++++++++++---
 10 files changed, 642 insertions(+), 110 deletions(-)

diff --git 
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxLiteralSuite.scala
 
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxLiteralSuite.scala
index cf2e7257f5..63a90abef3 100644
--- 
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxLiteralSuite.scala
+++ 
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxLiteralSuite.scala
@@ -111,14 +111,27 @@ class VeloxLiteralSuite extends 
VeloxWholeStageTransformerSuite {
   test("Null Literal") {
     validateOffloadResult("SELECT cast(null as int)")
     validateOffloadResult("SELECT cast(null as decimal)")
+    validateOffloadResult("SELECT CAST(null as struct<u:integer,v:string>)")
     validateOffloadResult("SELECT array(5, 1, null)")
     validateOffloadResult("SELECT array(5.321E2BD, 0.1, null)")
     validateOffloadResult("SELECT struct('Spark', cast(null as int))")
     validateOffloadResult("SELECT struct(cast(null as decimal))")
     validateOffloadResult("SELECT map('b', 'a', 'e', null)")
-    validateOffloadResult("SELECT array(null)")
     validateOffloadResult("SELECT array(cast(null as int))")
     validateOffloadResult("SELECT map(1, null)")
+    validateOffloadResult("SELECT array(null)")
+
+    validateOffloadResult("SELECT array(array(1,2), null, array(1), null)")
+    validateOffloadResult("SELECT array(array(1.0,2), null, array(1), null)")
+    validateOffloadResult("SELECT array(array(1, 2), null)")
+
+    validateOffloadResult("SELECT array(null, 2, 3.0)")
+    validateOffloadResult("SELECT array(1, 0, null, 2, 3)")
+    validateOffloadResult("SELECT array(null, array(null, array(1,2), 
array(2)), null)")
+    validateOffloadResult("SELECT array(null, array(1, 2))")
+    validateOffloadResult("SELECT array(null, array(null, 1, 2))")
+    validateOffloadResult("SELECT array(null, array(1, 2, null), null)")
+    validateOffloadResult("SELECT array(null, array(null, array(1, 2), null, 
array(1)))")
   }
 
   test("Scalar Type Literal") {
@@ -133,10 +146,14 @@ class VeloxLiteralSuite extends 
VeloxWholeStageTransformerSuite {
     validateOffloadResult("SELECT TIMESTAMP'2020-12-31', 
TIMESTAMP'2020-12-30'")
     validateOffloadResult("SELECT X'1234', X'a'")
     validateOffloadResult("SELECT DATE'2020-12-31', DATE'2020-12-30'")
-  }
+    validateOffloadResult("SELECT CAST(null as struct<u:integer,v:string>)")
+    validateOffloadResult("SELECT array(struct(1, 'a'), null, struct(1, 'a'))")
+    validateOffloadResult(
+      "SELECT array(struct(1, 'a'), " +
+        "CAST(null AS struct<col1:integer,col2:string>))"
+    )
 
-  test("Literal Fallback") {
-    validateFallbackResult("SELECT struct(cast(null as struct<a: string>))")
-    validateFallbackResult("SELECT array(struct(1, 'a'), null)")
+    validateOffloadResult("SELECT struct(cast(null as struct<a: string>))")
+    validateOffloadResult("SELECT array(null, array('str', 'a'))")
   }
 }
diff --git a/cpp/velox/substrait/SubstraitToVeloxExpr.cc 
b/cpp/velox/substrait/SubstraitToVeloxExpr.cc
index 1fe143597a..518e97fa88 100755
--- a/cpp/velox/substrait/SubstraitToVeloxExpr.cc
+++ b/cpp/velox/substrait/SubstraitToVeloxExpr.cc
@@ -16,6 +16,7 @@
  */
 
 #include "SubstraitToVeloxExpr.h"
+#include <glog/logging.h>
 #include "TypeUtils.h"
 #include "velox/functions/sparksql/specialforms/SparkCastExpr.h"
 #include "velox/type/Timestamp.h"
@@ -72,6 +73,14 @@ ArrayVectorPtr makeEmptyArrayVector(memory::MemoryPool* 
pool, const TypePtr& ele
   return std::make_shared<ArrayVector>(pool, ARRAY(elementType), nullptr, 1, 
offsets, sizes, nullptr);
 }
 
+ArrayVectorPtr makeNullArrayVector(memory::MemoryPool* pool, const TypePtr& 
elementType, size_t child_size) {
+  BufferPtr nulls = allocateNulls(1, pool, bits::kNull);
+  BufferPtr offsets = allocateOffsets(1, pool);
+  BufferPtr sizes = allocateOffsets(1, pool);
+
+  return std::make_shared<ArrayVector>(pool, elementType, nulls, child_size, 
offsets, sizes, nullptr);
+}
+
 MapVectorPtr makeEmptyMapVector(memory::MemoryPool* pool, const TypePtr& 
keyType, const TypePtr& valueType) {
   BufferPtr offsets = allocateOffsets(1, pool);
   BufferPtr sizes = allocateOffsets(1, pool);
@@ -82,6 +91,17 @@ RowVectorPtr makeEmptyRowVector(memory::MemoryPool* pool) {
   return makeRowVector({}, {}, 0, pool);
 }
 
+RowVectorPtr makeNullRowVector(memory::MemoryPool* pool, const TypePtr& 
rowType) {
+  auto rowTyped = asRowType(rowType);
+  std::vector<VectorPtr> children;
+  children.reserve(rowTyped->size());
+  for (int i = 0; i < rowTyped->size(); i++) {
+    children.push_back(BaseVector::createNullConstant(rowTyped->childAt(i), 1, 
pool));
+  }
+  BufferPtr nulls = allocateNulls(1, pool, bits::kNull);
+  return std::make_shared<RowVector>(pool, rowType, nulls, 1, children);
+}
+
 template <typename T>
 void setLiteralValue(const ::substrait::Expression::Literal& literal, 
FlatVector<T>* vector, vector_size_t index) {
   if (literal.has_null()) {
@@ -112,6 +132,8 @@ VectorPtr constructFlatVector(
 TypePtr getScalarType(const ::substrait::Expression::Literal& literal) {
   auto typeCase = literal.literal_type_case();
   switch (typeCase) {
+    case ::substrait::Expression_Literal::LiteralTypeCase::kNull:
+      return gluten::SubstraitParser::parseType(literal.null());
     case ::substrait::Expression_Literal::LiteralTypeCase::kBoolean:
       return BOOLEAN();
     case ::substrait::Expression_Literal::LiteralTypeCase::kI8:
@@ -474,7 +496,8 @@ std::shared_ptr<const core::ConstantTypedExpr> 
SubstraitVeloxExprConverter::toVe
 
 ArrayVectorPtr SubstraitVeloxExprConverter::literalsToArrayVector(const 
::substrait::Expression::Literal& literal) {
   auto childSize = literal.list().values().size();
-  VELOX_CHECK_GT(childSize, 0, "there should be at least 1 value in list 
literal.");
+  auto childType = literal.literal_type_case();
+
   auto childLiteral = literal.list().values(0);
   auto elementAtFunc = [&](vector_size_t idx) { return 
literal.list().values(idx); };
   auto childVector = literalsToVector(childLiteral, childSize, elementAtFunc);
@@ -498,37 +521,51 @@ VectorPtr SubstraitVeloxExprConverter::literalsToVector(
     vector_size_t childSize,
     std::function<::substrait::Expression::Literal(vector_size_t /* idx */)> 
elementAtFunc) {
   auto childTypeCase = childLiteral.literal_type_case();
-  switch (childTypeCase) {
-    case ::substrait::Expression_Literal::LiteralTypeCase::kNull: {
-      auto veloxType = SubstraitParser::parseType(childLiteral.null());
-      auto kind = veloxType->kind();
-      return VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH_ALL(
-          constructFlatVector, kind, elementAtFunc, childSize, veloxType, 
pool_);
+  if (childLiteral.has_null()) {
+    auto nullType = SubstraitParser::parseType(childLiteral.null());
+    if (nullType) {
+      switch (nullType->kind()) {
+        case TypeKind::ARRAY:
+          childTypeCase = 
::substrait::Expression_Literal::LiteralTypeCase::kList;
+          break;
+        case TypeKind::ROW:
+          childTypeCase = 
::substrait::Expression_Literal::LiteralTypeCase::kStruct;
+          break;
+        case TypeKind::MAP:
+          childTypeCase = 
::substrait::Expression_Literal::LiteralTypeCase::kMap;
+          break;
+        default:
+          break;
+      }
     }
+  }
+  switch (childTypeCase) {
     case 
::substrait::Expression_Literal::LiteralTypeCase::kIntervalDayToSecond:
       return constructFlatVector<TypeKind::BIGINT>(elementAtFunc, childSize, 
INTERVAL_DAY_TIME(), pool_);
     // Handle EmptyList and List together since the children could be either 
case.
     case ::substrait::Expression_Literal::LiteralTypeCase::kEmptyList:
     case ::substrait::Expression_Literal::LiteralTypeCase::kList: {
-      ArrayVectorPtr elements;
+      ArrayVectorPtr arrayVector;
       for (int i = 0; i < childSize; i++) {
         auto child = elementAtFunc(i);
         auto childType = child.literal_type_case();
         ArrayVectorPtr grandVector;
-
-        if (childType == 
::substrait::Expression_Literal::LiteralTypeCase::kEmptyList) {
+        if (childType == 
::substrait::Expression_Literal::LiteralTypeCase::kNull) {
+          auto veloxType = SubstraitParser::parseType(child.null());
+          grandVector = makeNullArrayVector(pool_, veloxType, 1);
+        } else if (childType == 
::substrait::Expression_Literal::LiteralTypeCase::kEmptyList) {
           auto elementType = 
SubstraitParser::parseType(child.empty_list().type());
           grandVector = makeEmptyArrayVector(pool_, elementType);
         } else {
           grandVector = literalsToArrayVector(child);
         }
-        if (!elements) {
-          elements = grandVector;
+        if (!arrayVector) {
+          arrayVector = grandVector;
         } else {
-          elements->append(grandVector.get());
+          arrayVector->append(grandVector.get());
         }
       }
-      return elements;
+      return arrayVector;
     }
     // Handle EmptyMap and Map together since the children could be either 
case.
     case ::substrait::Expression_Literal::LiteralTypeCase::kEmptyMap:
@@ -558,7 +595,13 @@ VectorPtr SubstraitVeloxExprConverter::literalsToVector(
       RowVectorPtr rowVector;
       for (int i = 0; i < childSize; i++) {
         auto element = elementAtFunc(i);
-        RowVectorPtr grandVector = literalsToRowVector(element);
+        RowVectorPtr grandVector;
+        if (element.literal_type_case() == 
::substrait::Expression_Literal::LiteralTypeCase::kNull) {
+          auto veloxType = SubstraitParser::parseType(element.null());
+          grandVector = makeNullRowVector(pool_, veloxType);
+        } else {
+          grandVector = literalsToRowVector(element);
+        }
         if (!rowVector) {
           rowVector = grandVector;
         } else {
@@ -567,24 +610,37 @@ VectorPtr SubstraitVeloxExprConverter::literalsToVector(
       }
       return rowVector;
     }
-    default:
+    default: {
+      // if null, get null type instead of next element type
       auto veloxType = getScalarType(elementAtFunc(0));
       if (veloxType) {
-        auto kind = veloxType->kind();
-        return VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH(
-            constructFlatVector, kind, elementAtFunc, childSize, veloxType, 
pool_);
+        // Handle UNKNOWN elements (mapped from Spark NullType, as in 
array(null))
+        // by creating an all-null vector.
+        if (veloxType->kind() == TypeKind::UNKNOWN) {
+          return BaseVector::createNullConstant(UNKNOWN(), childSize, pool_);
+        }
+        // if veloxType is scalar type
+        if (veloxType->isPrimitiveType()) {
+          auto kind = veloxType->kind();
+          return VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH(
+              constructFlatVector, kind, elementAtFunc, childSize, veloxType, 
pool_);
+        }
       }
       VELOX_NYI("literals not supported for type case '{}'", 
std::to_string(childTypeCase));
+    }
   }
 }
 
 RowVectorPtr SubstraitVeloxExprConverter::literalsToRowVector(const 
::substrait::Expression::Literal& structLiteral) {
-  if (structLiteral.has_null()) {
-    VELOX_NYI("NULL for struct type is not supported.");
-  }
   auto numFields = structLiteral.struct_().fields().size();
   if (numFields == 0) {
-    return makeEmptyRowVector(pool_);
+    auto is_null = structLiteral.literal_type_case() == 
::substrait::Expression_Literal::LiteralTypeCase::kNull;
+    if (!is_null) {
+      return makeEmptyRowVector(pool_);
+    } else {
+      auto veloxType = SubstraitParser::parseType(structLiteral.null());
+      return makeNullRowVector(pool_, veloxType);
+    }
   }
   std::vector<VectorPtr> vectors;
   std::vector<std::string> names;
@@ -603,9 +659,15 @@ RowVectorPtr 
SubstraitVeloxExprConverter::literalsToRowVector(const ::substrait:
       case ::substrait::Expression_Literal::LiteralTypeCase::kNull: {
         auto veloxType = SubstraitParser::parseType(child.null());
         auto kind = veloxType->kind();
-        auto vecPtr =
-            VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH(constructFlatVectorForStruct, 
kind, child, 1, veloxType, pool_);
-        vectors.emplace_back(vecPtr);
+        if (kind == TypeKind::ROW) {
+          vectors.emplace_back(makeNullRowVector(pool_, veloxType));
+        } else if (kind == TypeKind::ARRAY || kind == TypeKind::MAP || kind == 
TypeKind::UNKNOWN) {
+          vectors.emplace_back(BaseVector::createNullConstant(veloxType, 1, 
pool_));
+        } else {
+          auto vecPtr =
+              VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH(constructFlatVectorForStruct, 
kind, child, 1, veloxType, pool_);
+          vectors.emplace_back(vecPtr);
+        }
         break;
       }
       case ::substrait::Expression_Literal::LiteralTypeCase::kList: {
@@ -726,5 +788,4 @@ std::unordered_map<std::string, std::string> 
SubstraitVeloxExprConverter::extrac
     {"YEAR", "year"},
     {"WEEK_OF_YEAR", "week_of_year"},
     {"YEAR_OF_WEEK", "year_of_week"}};
-
 } // namespace gluten
diff --git 
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
 
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index c97b6adaa0..f65652d522 100644
--- 
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++ 
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -945,13 +945,21 @@ class VeloxTestSettings extends BackendTestSettings {
     .exclude("NOT NULL checks for nested struct fields (byPosition)")
     .exclude("NOT NULL checks for nested structs, arrays, maps (byName)")
     .exclude("NOT NULL checks for nested structs, arrays, maps (byPosition)")
+    // Overridden.
+    .exclude("NOT NULL checks for nullable array with required element 
(byName)")
+    // Overridden.
     .exclude("NOT NULL checks for nullable array with required element 
(byPosition)")
+    // Overridden.
+    .exclude("NOT NULL checks for fields inside nullable array (byName)")
+    // Overridden.
     .exclude("not null checks for fields inside nullable array (byPosition)")
     // Overridden.
     .exclude("NOT NULL checks for nullable map with required values (byName)")
     // Overridden.
     .exclude("NOT NULL checks for nullable map with required values 
(byPosition)")
     // Overridden.
+    .exclude("NOT NULL checks for fields inside nullable maps (byName)")
+    // Overridden.
     .exclude("NOT NULL checks for fields inside nullable maps (byPosition)")
   enableSuite[GlutenTableOptionsConstantFoldingSuite]
   enableSuite[GlutenDeltaBasedMergeIntoTableSuite]
diff --git 
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
 
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
index 0ef7eb2da9..fc98d77b98 100644
--- 
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
+++ 
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
@@ -16,10 +16,12 @@
  */
 package org.apache.spark.sql
 
+import org.apache.gluten.exception.GlutenException
+
 import org.apache.spark.SparkException
 import org.apache.spark.sql.connector.catalog.{Column => ColumnV2, Identifier}
 import org.apache.spark.sql.connector.expressions.Transform
-import org.apache.spark.sql.types.{IntegerType, MapType, StructType}
+import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StructType}
 
 import java.util.Collections
 
@@ -28,24 +30,129 @@ class GlutenRuntimeNullChecksV2Writes extends 
RuntimeNullChecksV2Writes with Glu
   /**
    * Shadows Spark's `assertNotNullException`, which is private and so cannot 
be reused.
    *
-   * Spark requires the cause to be a `NullPointerException` and matches the 
offending column path
-   * against `colPath.mkString("\n", "\n", "\n")`. Velox raises a 
`VeloxUserError` that Gluten
-   * surfaces as a `SparkException`, carrying the same reason text but not 
Spark's column path
-   * formatting, so `colPath` is reported on failure rather than asserted. The 
reason match is case
-   * insensitive because Spark 3.5 words it "Null value appeared ..." and 
Spark 4.x "NULL value
-   * appeared ...".
+   * Spark expects the cause of the `SparkException` to be a 
`NullPointerException` thrown by its
+   * `AssertNotNull`. When the projection is offloaded, Velox 
`assert_not_null` raises a
+   * `VeloxUserError` instead, which Gluten surfaces as a `GlutenException` 
cause carrying the same
+   * reason text but not Spark's column path formatting, so `colPath` is only 
checked when the plan
+   * fell back to vanilla Spark.
    */
   private def assertNotNullException(e: SparkException, colPath: Seq[String]): 
Unit = {
-    val messages = Iterator
-      .iterate[Throwable](e)(_.getCause)
-      .takeWhile(_ != null)
-      .flatMap(t => Option(t.getMessage))
-      .mkString("\n")
-
-    assert(
-      messages.toLowerCase(java.util.Locale.ROOT).contains("value appeared in 
non-nullable field"),
-      s"expected a not-null violation for ${colPath.mkString(".")}, 
got:\n$messages"
-    )
+    e.getCause match {
+      case ge: GlutenException =>
+        assert(
+          ge.getMessage.contains("Null value appeared in non-nullable field"),
+          s"expected a not-null violation for ${colPath.mkString(".")}, 
got:\n${ge.getMessage}"
+        )
+      case npe: NullPointerException =>
+        assert(npe.getMessage.contains("Null value appeared in non-nullable 
field"))
+        assert(npe.getMessage.contains(colPath.mkString("\n", "\n", "\n")))
+      case other =>
+        fail(s"Unexpected exception cause: $other")
+    }
+  }
+
+  testGluten("NOT NULL checks for nullable array with required element 
(byName)") {
+    checkNullableArrayWithNotNullElement(byName = true)
+  }
+
+  testGluten("NOT NULL checks for nullable array with required element 
(byPosition)") {
+    checkNullableArrayWithNotNullElement(byName = false)
+  }
+
+  private def checkNullableArrayWithNotNullElement(byName: Boolean): Unit = {
+    withTable("t") {
+      val structType = new StructType().add("x", "int").add("y", "int")
+      catalog.createTable(
+        ident = Identifier.of(Array(), "t"),
+        columns = Array(
+          ColumnV2.create("i", IntegerType),
+          ColumnV2.create("arr", ArrayType(structType, containsNull = false))),
+        partitions = Array.empty[Transform],
+        properties = Collections.emptyMap[String, String]
+      )
+
+      if (byName) {
+        val inputDF = sql("SELECT 1 AS i, null AS arr")
+        inputDF.writeTo("t").append()
+      } else {
+        sql("INSERT INTO t VALUES (1 AS i, null AS arr)")
+      }
+      checkAnswer(spark.table("t"), Row(1, null))
+
+      val e = intercept[SparkException] {
+        if (byName) {
+          val inputDF = sql(
+            s"""SELECT
+               | 1 AS i,
+               | array(null, named_struct('x', 1, 'y', 1)) AS arr
+             """.stripMargin)
+          inputDF.writeTo("t").append()
+        } else {
+          sql(
+            s"""INSERT INTO t VALUES (
+               | 1 AS i,
+               | array(null, named_struct('x', 1, 'y', 1)) AS arr)
+             """.stripMargin)
+        }
+      }
+      assertNotNullException(e, Seq("arr", "element"))
+    }
+  }
+
+  testGluten("NOT NULL checks for fields inside nullable array (byName)") {
+    checkNotNullFieldsInsideNullableArray(byName = true)
+  }
+
+  testGluten("not null checks for fields inside nullable array (byPosition)") {
+    checkNotNullFieldsInsideNullableArray(byName = false)
+  }
+
+  private def checkNotNullFieldsInsideNullableArray(byName: Boolean): Unit = {
+    withTable("t") {
+      val structType = new StructType().add("x", "int", nullable = 
false).add("y", "int")
+      catalog.createTable(
+        ident = Identifier.of(Array(), "t"),
+        columns = Array(
+          ColumnV2.create("i", IntegerType),
+          ColumnV2.create("arr", ArrayType(structType, containsNull = true))),
+        partitions = Array.empty[Transform],
+        properties = Collections.emptyMap[String, String]
+      )
+
+      if (byName) {
+        val inputDF = sql(
+          s"""SELECT
+             | 1 AS i,
+             | array(null, named_struct('x', 1, 'y', 1)) AS arr
+           """.stripMargin)
+        inputDF.writeTo("t").append()
+      } else {
+        sql(
+          s"""INSERT INTO t VALUES (
+             | 1 AS i,
+             | array(null, named_struct('x', 1, 'y', 1)) AS arr)
+           """.stripMargin)
+      }
+      checkAnswer(spark.table("t"), Row(1, List(null, Row(1, 1))))
+
+      val e = intercept[SparkException] {
+        if (byName) {
+          val inputDF = sql(
+            s"""SELECT
+               | 1 AS i,
+               | array(null, named_struct('x', null, 'y', 1)) AS arr
+             """.stripMargin)
+          inputDF.writeTo("t").append()
+        } else {
+          sql(
+            s"""INSERT INTO t VALUES (
+               | 1 AS i,
+               | array(null, named_struct('x', null, 'y', 1)) AS arr)
+             """.stripMargin)
+        }
+      }
+      assertNotNullException(e, Seq("arr", "element", "x"))
+    }
   }
 
   testGluten("NOT NULL checks for nullable map with required values (byName)") 
{
@@ -87,7 +194,10 @@ class GlutenRuntimeNullChecksV2Writes extends 
RuntimeNullChecksV2Writes with Glu
     }
   }
 
-  /** Only the byPosition case is overridden. */
+  testGluten("NOT NULL checks for fields inside nullable maps (byName)") {
+    checkNotNullFieldsInsideNullableMap(byName = true)
+  }
+
   testGluten("NOT NULL checks for fields inside nullable maps (byPosition)") {
     checkNotNullFieldsInsideNullableMap(byName = false)
   }
diff --git 
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
 
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 50198c58ab..5e754ecd77 100644
--- 
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++ 
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1219,13 +1219,21 @@ class VeloxTestSettings extends BackendTestSettings {
     .exclude("NOT NULL checks for nested struct fields (byPosition)")
     .exclude("NOT NULL checks for nested structs, arrays, maps (byName)")
     .exclude("NOT NULL checks for nested structs, arrays, maps (byPosition)")
+    // Overridden.
+    .exclude("NOT NULL checks for nullable array with required element 
(byName)")
+    // Overridden.
     .exclude("NOT NULL checks for nullable array with required element 
(byPosition)")
+    // Overridden.
+    .exclude("NOT NULL checks for fields inside nullable array (byName)")
+    // Overridden.
     .exclude("not null checks for fields inside nullable array (byPosition)")
     // Overridden.
     .exclude("NOT NULL checks for nullable map with required values (byName)")
     // Overridden.
     .exclude("NOT NULL checks for nullable map with required values 
(byPosition)")
     // Overridden.
+    .exclude("NOT NULL checks for fields inside nullable maps (byName)")
+    // Overridden.
     .exclude("NOT NULL checks for fields inside nullable maps (byPosition)")
   enableSuite[GlutenTableOptionsConstantFoldingSuite]
   enableSuite[GlutenDeltaBasedMergeIntoTableSuite]
diff --git 
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
 
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
index d0cd89b643..d8e83175c1 100644
--- 
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
+++ 
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
@@ -16,10 +16,12 @@
  */
 package org.apache.spark.sql
 
+import org.apache.gluten.exception.GlutenException
+
 import org.apache.spark.SparkException
 import org.apache.spark.sql.connector.catalog.{Column => ColumnV2, Identifier}
 import org.apache.spark.sql.connector.expressions.Transform
-import org.apache.spark.sql.types.{IntegerType, MapType, StructType}
+import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StructType}
 
 import java.util.Collections
 
@@ -28,23 +30,125 @@ class GlutenRuntimeNullChecksV2Writes extends 
RuntimeNullChecksV2Writes with Glu
   /**
    * Shadows Spark's `assertNotNullException`, which is private and so cannot 
be reused.
    *
-   * Spark expects a `SparkRuntimeException` carrying 
`NOT_NULL_ASSERT_VIOLATION`, asserts nothing
-   * else, and never reads `colPath`. Velox raises a `VeloxUserError` that 
Gluten surfaces as a
-   * `SparkException`, which carries the same reason text but not the error 
condition, so this
-   * asserts the reason and reports `colPath` on failure. The reason match is 
case insensitive
-   * because Spark 3.5 words it "Null value appeared ..." and Spark 4.x "NULL 
value appeared ...".
+   * Spark expects a `SparkRuntimeException` carrying 
`NOT_NULL_ASSERT_VIOLATION` and never reads
+   * `colPath`. When the projection is offloaded, Velox `assert_not_null` 
raises a `VeloxUserError`
+   * instead, which reaches the test as a `SparkException` whose cause is a 
`GlutenException`
+   * carrying the reason text but not the error condition.
    */
   private def assertNotNullException(e: SparkException, colPath: Seq[String]): 
Unit = {
-    val messages = Iterator
-      .iterate[Throwable](e)(_.getCause)
-      .takeWhile(_ != null)
-      .flatMap(t => Option(t.getMessage))
-      .mkString("\n")
-
-    assert(
-      messages.toLowerCase(java.util.Locale.ROOT).contains("value appeared in 
non-nullable field"),
-      s"expected a not-null violation for ${colPath.mkString(".")}, 
got:\n$messages"
-    )
+    e.getCause match {
+      case ge: GlutenException =>
+        assert(
+          ge.getMessage.contains("Null value appeared in non-nullable field"),
+          s"expected a not-null violation for ${colPath.mkString(".")}, 
got:\n${ge.getMessage}"
+        )
+      case other =>
+        fail(s"Unexpected exception cause: $other")
+    }
+  }
+
+  testGluten("NOT NULL checks for nullable array with required element 
(byName)") {
+    checkNullableArrayWithNotNullElement(byName = true)
+  }
+
+  testGluten("NOT NULL checks for nullable array with required element 
(byPosition)") {
+    checkNullableArrayWithNotNullElement(byName = false)
+  }
+
+  private def checkNullableArrayWithNotNullElement(byName: Boolean): Unit = {
+    withTable("t") {
+      val structType = new StructType().add("x", "int").add("y", "int")
+      catalog.createTable(
+        ident = Identifier.of(Array(), "t"),
+        columns = Array(
+          ColumnV2.create("i", IntegerType),
+          ColumnV2.create("arr", ArrayType(structType, containsNull = false))),
+        partitions = Array.empty[Transform],
+        properties = Collections.emptyMap[String, String]
+      )
+
+      if (byName) {
+        val inputDF = sql("SELECT 1 AS i, null AS arr")
+        inputDF.writeTo("t").append()
+      } else {
+        sql("INSERT INTO t VALUES (1 AS i, null AS arr)")
+      }
+      checkAnswer(spark.table("t"), Row(1, null))
+
+      val e = intercept[SparkException] {
+        if (byName) {
+          val inputDF = sql(
+            s"""SELECT
+               | 1 AS i,
+               | array(null, named_struct('x', 1, 'y', 1)) AS arr
+             """.stripMargin)
+          inputDF.writeTo("t").append()
+        } else {
+          sql(
+            s"""INSERT INTO t VALUES (
+               | 1 AS i,
+               | array(null, named_struct('x', 1, 'y', 1)) AS arr)
+             """.stripMargin)
+        }
+      }
+      assertNotNullException(e, Seq("arr", "element"))
+    }
+  }
+
+  testGluten("NOT NULL checks for fields inside nullable array (byName)") {
+    checkNotNullFieldsInsideNullableArray(byName = true)
+  }
+
+  testGluten("not null checks for fields inside nullable array (byPosition)") {
+    checkNotNullFieldsInsideNullableArray(byName = false)
+  }
+
+  private def checkNotNullFieldsInsideNullableArray(byName: Boolean): Unit = {
+    withTable("t") {
+      val structType = new StructType().add("x", "int", nullable = 
false).add("y", "int")
+      catalog.createTable(
+        ident = Identifier.of(Array(), "t"),
+        columns = Array(
+          ColumnV2.create("i", IntegerType),
+          ColumnV2.create("arr", ArrayType(structType, containsNull = true))),
+        partitions = Array.empty[Transform],
+        properties = Collections.emptyMap[String, String]
+      )
+
+      if (byName) {
+        val inputDF = sql(
+          s"""SELECT
+             | 1 AS i,
+             | array(null, named_struct('x', 1, 'y', 1)) AS arr
+           """.stripMargin)
+        inputDF.writeTo("t").append()
+      } else {
+        sql(
+          s"""INSERT INTO t VALUES (
+             | 1 AS i,
+             | array(null, named_struct('x', 1, 'y', 1)) AS arr)
+           """.stripMargin)
+      }
+      checkAnswer(spark.table("t"), Row(1, List(null, Row(1, 1))))
+
+      val e = intercept[SparkException] {
+        if (byName) {
+          val inputDF = sql(
+            s"""SELECT
+               | 1 AS i,
+               | array(null, named_struct('x', null, 'y', 1)) AS arr
+             """.stripMargin)
+          inputDF.writeTo("t").append()
+        } else {
+          sql(
+            s"""INSERT INTO t VALUES (
+               | 1 AS i,
+               | array(null, named_struct('x', null, 'y', 1)) AS arr)
+             """.stripMargin)
+        }
+      }
+      assertNotNullException(e, Seq("arr", "element", "x"))
+    }
   }
 
   testGluten("NOT NULL checks for nullable map with required values (byName)") 
{
@@ -74,7 +178,6 @@ class GlutenRuntimeNullChecksV2Writes extends 
RuntimeNullChecksV2Writes with Glu
       }
       checkAnswer(spark.table("t"), Row(1, null))
 
-      // Gluten exception differs from Spark
       val e = intercept[SparkException] {
         if (byName) {
           val inputDF = sql("SELECT 1 AS i, map(1, null) AS m")
@@ -87,7 +190,10 @@ class GlutenRuntimeNullChecksV2Writes extends 
RuntimeNullChecksV2Writes with Glu
     }
   }
 
-  /** Only the byPosition case is overridden. */
+  testGluten("NOT NULL checks for fields inside nullable maps (byName)") {
+    checkNotNullFieldsInsideNullableMap(byName = true)
+  }
+
   testGluten("NOT NULL checks for fields inside nullable maps (byPosition)") {
     checkNotNullFieldsInsideNullableMap(byName = false)
   }
@@ -112,7 +218,6 @@ class GlutenRuntimeNullChecksV2Writes extends 
RuntimeNullChecksV2Writes with Glu
       }
       checkAnswer(spark.table("t"), Row(1, Map(Row(1, 1) -> null)))
 
-      // Gluten exception differs from Spark
       val e1 = intercept[SparkException] {
         if (byName) {
           val inputDF = sql(
@@ -131,7 +236,6 @@ class GlutenRuntimeNullChecksV2Writes extends 
RuntimeNullChecksV2Writes with Glu
       }
       assertNotNullException(e1, Seq("m", "key", "x"))
 
-      // Gluten exception differs from Spark
       val e2 = intercept[SparkException] {
         if (byName) {
           val inputDF = sql(
diff --git 
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
 
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 1e29452c78..58a2f37ab9 100644
--- 
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++ 
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1221,13 +1221,21 @@ class VeloxTestSettings extends BackendTestSettings {
     .exclude("NOT NULL checks for nested struct fields (byPosition)")
     .exclude("NOT NULL checks for nested structs, arrays, maps (byName)")
     .exclude("NOT NULL checks for nested structs, arrays, maps (byPosition)")
+    // Overridden.
+    .exclude("NOT NULL checks for nullable array with required element 
(byName)")
+    // Overridden.
     .exclude("NOT NULL checks for nullable array with required element 
(byPosition)")
+    // Overridden.
+    .exclude("NOT NULL checks for fields inside nullable array (byName)")
+    // Overridden.
     .exclude("not null checks for fields inside nullable array (byPosition)")
     // Overridden.
     .exclude("NOT NULL checks for nullable map with required values (byName)")
     // Overridden.
     .exclude("NOT NULL checks for nullable map with required values 
(byPosition)")
     // Overridden.
+    .exclude("NOT NULL checks for fields inside nullable maps (byName)")
+    // Overridden.
     .exclude("NOT NULL checks for fields inside nullable maps (byPosition)")
   enableSuite[GlutenTableOptionsConstantFoldingSuite]
   enableSuite[GlutenDeltaBasedMergeIntoTableSuite]
diff --git 
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
 
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
index 642ad786fb..e677abdce6 100644
--- 
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
+++ 
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
@@ -16,32 +16,136 @@
  */
 package org.apache.spark.sql
 
+import org.apache.gluten.exception.GlutenException
+
 import org.apache.spark.SparkException
 import org.apache.spark.sql.connector.catalog.{Column => ColumnV2, Identifier, 
TableInfo}
-import org.apache.spark.sql.types.{IntegerType, MapType, StructType}
+import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StructType}
 
 class GlutenRuntimeNullChecksV2Writes extends RuntimeNullChecksV2Writes with 
GlutenSQLTestsTrait {
 
   /**
    * Shadows Spark's `assertNotNullException`, which is private and so cannot 
be reused.
    *
-   * Spark expects a `SparkRuntimeException` carrying 
`NOT_NULL_ASSERT_VIOLATION`, asserts nothing
-   * else, and never reads `colPath`. Velox raises a `VeloxUserError` that 
Gluten surfaces as a
-   * `SparkException`, which carries the same reason text but not the error 
condition, so this
-   * asserts the reason and reports `colPath` on failure. The reason match is 
case insensitive
-   * because Spark 3.5 words it "Null value appeared ..." and Spark 4.x "NULL 
value appeared ...".
+   * Spark expects a `SparkRuntimeException` carrying 
`NOT_NULL_ASSERT_VIOLATION` and never reads
+   * `colPath`. When the projection is offloaded, Velox `assert_not_null` 
raises a `VeloxUserError`
+   * instead, which reaches the test as a `SparkException` whose cause is a 
`GlutenException`
+   * carrying the reason text but not the error condition.
    */
   private def assertNotNullException(e: SparkException, colPath: Seq[String]): 
Unit = {
-    val messages = Iterator
-      .iterate[Throwable](e)(_.getCause)
-      .takeWhile(_ != null)
-      .flatMap(t => Option(t.getMessage))
-      .mkString("\n")
-
-    assert(
-      messages.toLowerCase(java.util.Locale.ROOT).contains("value appeared in 
non-nullable field"),
-      s"expected a not-null violation for ${colPath.mkString(".")}, 
got:\n$messages"
-    )
+    e.getCause match {
+      case ge: GlutenException =>
+        assert(
+          ge.getMessage.contains("Null value appeared in non-nullable field"),
+          s"expected a not-null violation for ${colPath.mkString(".")}, 
got:\n${ge.getMessage}"
+        )
+      case other =>
+        fail(s"Unexpected exception cause: $other")
+    }
+  }
+
+  testGluten("NOT NULL checks for nullable array with required element 
(byName)") {
+    checkNullableArrayWithNotNullElement(byName = true)
+  }
+
+  testGluten("NOT NULL checks for nullable array with required element 
(byPosition)") {
+    checkNullableArrayWithNotNullElement(byName = false)
+  }
+
+  private def checkNullableArrayWithNotNullElement(byName: Boolean): Unit = {
+    withTable("t") {
+      val structType = new StructType().add("x", "int").add("y", "int")
+      val tableInfo = new TableInfo.Builder()
+        .withColumns(Array(
+          ColumnV2.create("i", IntegerType),
+          ColumnV2.create("arr", ArrayType(structType, containsNull = false))))
+        .build()
+      catalog.createTable(
+        ident = Identifier.of(Array(), "t"),
+        tableInfo = tableInfo)
+
+      if (byName) {
+        val inputDF = sql("SELECT 1 AS i, null AS arr")
+        inputDF.writeTo("t").append()
+      } else {
+        sql("INSERT INTO t VALUES (1 AS i, null AS arr)")
+      }
+      checkAnswer(spark.table("t"), Row(1, null))
+
+      val e = intercept[SparkException] {
+        if (byName) {
+          val inputDF = sql(
+            s"""SELECT
+               | 1 AS i,
+               | array(null, named_struct('x', 1, 'y', 1)) AS arr
+             """.stripMargin)
+          inputDF.writeTo("t").append()
+        } else {
+          sql(
+            s"""INSERT INTO t VALUES (
+               | 1 AS i,
+               | array(null, named_struct('x', 1, 'y', 1)) AS arr)
+             """.stripMargin)
+        }
+      }
+      assertNotNullException(e, Seq("arr", "element"))
+    }
+  }
+
+  testGluten("NOT NULL checks for fields inside nullable array (byName)") {
+    checkNotNullFieldsInsideNullableArray(byName = true)
+  }
+
+  testGluten("not null checks for fields inside nullable array (byPosition)") {
+    checkNotNullFieldsInsideNullableArray(byName = false)
+  }
+
+  private def checkNotNullFieldsInsideNullableArray(byName: Boolean): Unit = {
+    withTable("t") {
+      val structType = new StructType().add("x", "int", nullable = 
false).add("y", "int")
+      val tableInfo = new TableInfo.Builder()
+        .withColumns(Array(
+          ColumnV2.create("i", IntegerType),
+          ColumnV2.create("arr", ArrayType(structType, containsNull = true))))
+        .build()
+      catalog.createTable(
+        ident = Identifier.of(Array(), "t"),
+        tableInfo = tableInfo)
+
+      if (byName) {
+        val inputDF = sql(
+          s"""SELECT
+             | 1 AS i,
+             | array(null, named_struct('x', 1, 'y', 1)) AS arr
+           """.stripMargin)
+        inputDF.writeTo("t").append()
+      } else {
+        sql(
+          s"""INSERT INTO t VALUES (
+             | 1 AS i,
+             | array(null, named_struct('x', 1, 'y', 1)) AS arr)
+           """.stripMargin)
+      }
+      checkAnswer(spark.table("t"), Row(1, List(null, Row(1, 1))))
+
+      val e = intercept[SparkException] {
+        if (byName) {
+          val inputDF = sql(
+            s"""SELECT
+               | 1 AS i,
+               | array(null, named_struct('x', null, 'y', 1)) AS arr
+             """.stripMargin)
+          inputDF.writeTo("t").append()
+        } else {
+          sql(
+            s"""INSERT INTO t VALUES (
+               | 1 AS i,
+               | array(null, named_struct('x', null, 'y', 1)) AS arr)
+             """.stripMargin)
+        }
+      }
+      assertNotNullException(e, Seq("arr", "element", "x"))
+    }
   }
 
   testGluten("NOT NULL checks for nullable map with required values (byName)") 
{
@@ -71,7 +175,6 @@ class GlutenRuntimeNullChecksV2Writes extends 
RuntimeNullChecksV2Writes with Glu
       }
       checkAnswer(spark.table("t"), Row(1, null))
 
-      // Gluten exception differs from Spark
       val e = intercept[SparkException] {
         if (byName) {
           val inputDF = sql("SELECT 1 AS i, map(1, null) AS m")
@@ -84,7 +187,10 @@ class GlutenRuntimeNullChecksV2Writes extends 
RuntimeNullChecksV2Writes with Glu
     }
   }
 
-  /** Only the byPosition case is overridden. */
+  testGluten("NOT NULL checks for fields inside nullable maps (byName)") {
+    checkNotNullFieldsInsideNullableMap(byName = true)
+  }
+
   testGluten("NOT NULL checks for fields inside nullable maps (byPosition)") {
     checkNotNullFieldsInsideNullableMap(byName = false)
   }
@@ -109,7 +215,6 @@ class GlutenRuntimeNullChecksV2Writes extends 
RuntimeNullChecksV2Writes with Glu
       }
       checkAnswer(spark.table("t"), Row(1, Map(Row(1, 1) -> null)))
 
-      // Gluten exception differs from Spark
       val e1 = intercept[SparkException] {
         if (byName) {
           val inputDF = sql(
@@ -128,7 +233,6 @@ class GlutenRuntimeNullChecksV2Writes extends 
RuntimeNullChecksV2Writes with Glu
       }
       assertNotNullException(e1, Seq("m", "key", "x"))
 
-      // Gluten exception differs from Spark
       val e2 = intercept[SparkException] {
         if (byName) {
           val inputDF = sql(
diff --git 
a/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
 
b/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 36c9d7c327..cbe607088d 100644
--- 
a/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++ 
b/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1315,13 +1315,21 @@ class VeloxTestSettings extends BackendTestSettings {
     .exclude("NOT NULL checks for nested struct fields (byPosition)")
     .exclude("NOT NULL checks for nested structs, arrays, maps (byName)")
     .exclude("NOT NULL checks for nested structs, arrays, maps (byPosition)")
+    // Overridden.
+    .exclude("NOT NULL checks for nullable array with required element 
(byName)")
+    // Overridden.
     .exclude("NOT NULL checks for nullable array with required element 
(byPosition)")
+    // Overridden.
+    .exclude("NOT NULL checks for fields inside nullable array (byName)")
+    // Overridden.
     .exclude("not null checks for fields inside nullable array (byPosition)")
     // Overridden.
     .exclude("NOT NULL checks for nullable map with required values (byName)")
     // Overridden.
     .exclude("NOT NULL checks for nullable map with required values 
(byPosition)")
     // Overridden.
+    .exclude("NOT NULL checks for fields inside nullable maps (byName)")
+    // Overridden.
     .exclude("NOT NULL checks for fields inside nullable maps (byPosition)")
   enableSuite[GlutenTableOptionsConstantFoldingSuite]
   enableSuite[GlutenDeltaBasedMergeIntoTableSuite]
diff --git 
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
 
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
index 642ad786fb..e677abdce6 100644
--- 
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
+++ 
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala
@@ -16,32 +16,136 @@
  */
 package org.apache.spark.sql
 
+import org.apache.gluten.exception.GlutenException
+
 import org.apache.spark.SparkException
 import org.apache.spark.sql.connector.catalog.{Column => ColumnV2, Identifier, 
TableInfo}
-import org.apache.spark.sql.types.{IntegerType, MapType, StructType}
+import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StructType}
 
 class GlutenRuntimeNullChecksV2Writes extends RuntimeNullChecksV2Writes with 
GlutenSQLTestsTrait {
 
   /**
    * Shadows Spark's `assertNotNullException`, which is private and so cannot 
be reused.
    *
-   * Spark expects a `SparkRuntimeException` carrying 
`NOT_NULL_ASSERT_VIOLATION`, asserts nothing
-   * else, and never reads `colPath`. Velox raises a `VeloxUserError` that 
Gluten surfaces as a
-   * `SparkException`, which carries the same reason text but not the error 
condition, so this
-   * asserts the reason and reports `colPath` on failure. The reason match is 
case insensitive
-   * because Spark 3.5 words it "Null value appeared ..." and Spark 4.x "NULL 
value appeared ...".
+   * Spark expects a `SparkRuntimeException` carrying 
`NOT_NULL_ASSERT_VIOLATION` and never reads
+   * `colPath`. When the projection is offloaded, Velox `assert_not_null` 
raises a `VeloxUserError`
+   * instead, which reaches the test as a `SparkException` whose cause is a 
`GlutenException`
+   * carrying the reason text but not the error condition.
    */
   private def assertNotNullException(e: SparkException, colPath: Seq[String]): 
Unit = {
-    val messages = Iterator
-      .iterate[Throwable](e)(_.getCause)
-      .takeWhile(_ != null)
-      .flatMap(t => Option(t.getMessage))
-      .mkString("\n")
-
-    assert(
-      messages.toLowerCase(java.util.Locale.ROOT).contains("value appeared in 
non-nullable field"),
-      s"expected a not-null violation for ${colPath.mkString(".")}, 
got:\n$messages"
-    )
+    e.getCause match {
+      case ge: GlutenException =>
+        assert(
+          ge.getMessage.contains("Null value appeared in non-nullable field"),
+          s"expected a not-null violation for ${colPath.mkString(".")}, 
got:\n${ge.getMessage}"
+        )
+      case other =>
+        fail(s"Unexpected exception cause: $other")
+    }
+  }
+
+  testGluten("NOT NULL checks for nullable array with required element 
(byName)") {
+    checkNullableArrayWithNotNullElement(byName = true)
+  }
+
+  testGluten("NOT NULL checks for nullable array with required element 
(byPosition)") {
+    checkNullableArrayWithNotNullElement(byName = false)
+  }
+
+  private def checkNullableArrayWithNotNullElement(byName: Boolean): Unit = {
+    withTable("t") {
+      val structType = new StructType().add("x", "int").add("y", "int")
+      val tableInfo = new TableInfo.Builder()
+        .withColumns(Array(
+          ColumnV2.create("i", IntegerType),
+          ColumnV2.create("arr", ArrayType(structType, containsNull = false))))
+        .build()
+      catalog.createTable(
+        ident = Identifier.of(Array(), "t"),
+        tableInfo = tableInfo)
+
+      if (byName) {
+        val inputDF = sql("SELECT 1 AS i, null AS arr")
+        inputDF.writeTo("t").append()
+      } else {
+        sql("INSERT INTO t VALUES (1 AS i, null AS arr)")
+      }
+      checkAnswer(spark.table("t"), Row(1, null))
+
+      val e = intercept[SparkException] {
+        if (byName) {
+          val inputDF = sql(
+            s"""SELECT
+               | 1 AS i,
+               | array(null, named_struct('x', 1, 'y', 1)) AS arr
+             """.stripMargin)
+          inputDF.writeTo("t").append()
+        } else {
+          sql(
+            s"""INSERT INTO t VALUES (
+               | 1 AS i,
+               | array(null, named_struct('x', 1, 'y', 1)) AS arr)
+             """.stripMargin)
+        }
+      }
+      assertNotNullException(e, Seq("arr", "element"))
+    }
+  }
+
+  testGluten("NOT NULL checks for fields inside nullable array (byName)") {
+    checkNotNullFieldsInsideNullableArray(byName = true)
+  }
+
+  testGluten("not null checks for fields inside nullable array (byPosition)") {
+    checkNotNullFieldsInsideNullableArray(byName = false)
+  }
+
+  private def checkNotNullFieldsInsideNullableArray(byName: Boolean): Unit = {
+    withTable("t") {
+      val structType = new StructType().add("x", "int", nullable = 
false).add("y", "int")
+      val tableInfo = new TableInfo.Builder()
+        .withColumns(Array(
+          ColumnV2.create("i", IntegerType),
+          ColumnV2.create("arr", ArrayType(structType, containsNull = true))))
+        .build()
+      catalog.createTable(
+        ident = Identifier.of(Array(), "t"),
+        tableInfo = tableInfo)
+
+      if (byName) {
+        val inputDF = sql(
+          s"""SELECT
+             | 1 AS i,
+             | array(null, named_struct('x', 1, 'y', 1)) AS arr
+           """.stripMargin)
+        inputDF.writeTo("t").append()
+      } else {
+        sql(
+          s"""INSERT INTO t VALUES (
+             | 1 AS i,
+             | array(null, named_struct('x', 1, 'y', 1)) AS arr)
+           """.stripMargin)
+      }
+      checkAnswer(spark.table("t"), Row(1, List(null, Row(1, 1))))
+
+      val e = intercept[SparkException] {
+        if (byName) {
+          val inputDF = sql(
+            s"""SELECT
+               | 1 AS i,
+               | array(null, named_struct('x', null, 'y', 1)) AS arr
+             """.stripMargin)
+          inputDF.writeTo("t").append()
+        } else {
+          sql(
+            s"""INSERT INTO t VALUES (
+               | 1 AS i,
+               | array(null, named_struct('x', null, 'y', 1)) AS arr)
+             """.stripMargin)
+        }
+      }
+      assertNotNullException(e, Seq("arr", "element", "x"))
+    }
   }
 
   testGluten("NOT NULL checks for nullable map with required values (byName)") 
{
@@ -71,7 +175,6 @@ class GlutenRuntimeNullChecksV2Writes extends 
RuntimeNullChecksV2Writes with Glu
       }
       checkAnswer(spark.table("t"), Row(1, null))
 
-      // Gluten exception differs from Spark
       val e = intercept[SparkException] {
         if (byName) {
           val inputDF = sql("SELECT 1 AS i, map(1, null) AS m")
@@ -84,7 +187,10 @@ class GlutenRuntimeNullChecksV2Writes extends 
RuntimeNullChecksV2Writes with Glu
     }
   }
 
-  /** Only the byPosition case is overridden. */
+  testGluten("NOT NULL checks for fields inside nullable maps (byName)") {
+    checkNotNullFieldsInsideNullableMap(byName = true)
+  }
+
   testGluten("NOT NULL checks for fields inside nullable maps (byPosition)") {
     checkNotNullFieldsInsideNullableMap(byName = false)
   }
@@ -109,7 +215,6 @@ class GlutenRuntimeNullChecksV2Writes extends 
RuntimeNullChecksV2Writes with Glu
       }
       checkAnswer(spark.table("t"), Row(1, Map(Row(1, 1) -> null)))
 
-      // Gluten exception differs from Spark
       val e1 = intercept[SparkException] {
         if (byName) {
           val inputDF = sql(
@@ -128,7 +233,6 @@ class GlutenRuntimeNullChecksV2Writes extends 
RuntimeNullChecksV2Writes with Glu
       }
       assertNotNullException(e1, Seq("m", "key", "x"))
 
-      // Gluten exception differs from Spark
       val e2 = intercept[SparkException] {
         if (byName) {
           val inputDF = sql(


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to