This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5efa83ee269 [fix](load) Load a literal \N path value as NULL (#69064)
5efa83ee269 is described below
commit 5efa83ee269758b6490f65eea3dc892aa5e8790f
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Thu Oct 15 14:48:49 2026 +0800
[fix](load) Load a literal \N path value as NULL (#69064)
### What problem does this PR solve?
Issue Number: None
Related PR: #66012, #62821
Problem Summary:
**In short.** A broker load that reads a column from the file path
(`COLUMNS FROM PATH AS (p)`) stores a path value of `\N` as the
two-character string `\N` on master. Before #62821, and on branch-4.1
(#66012), the load stores NULL. This PR ports the load-specific parsing
of branch-4.1 #66012 so that loads store NULL again; queries are not
changed.
**Background.** For `COLUMNS FROM PATH AS`, the FE parses every file
path (for example `s3://bucket/t/p=\N/data.csv`) and sends the values to
the BE in the scan range, together with a null flag per value
(`columns_from_path_is_null`). #62821 added the flag and sets it only
for the Hive default partition, `__HIVE_DEFAULT_PARTITION__`. Before
that, the FE wrote a NULL as `\N` and the BE read the values with its
nullable parser, so a literal `\N` directory value became NULL as well.
branch-4.1 got the same flag in #66012, which also added a load-specific
parser that keeps the old meaning of `\N` for loads.
**The problem, and what it cost.**
| Path value | master | before #62821, and branch-4.1 | with this PR |
|---|---|---|---|
| `p=\N`, string column | the string `\N` | NULL | NULL |
| `p=\N`, INT column | NULL from the load's cast; the row is filtered in
strict mode | NULL | NULL |
| `p=__HIVE_DEFAULT_PARTITION__` | NULL | NULL | NULL |
| Queries (Hive catalog, TVF, `INSERT ... SELECT FROM s3()`) | `\N` is
data | `\N` is data | unchanged |
**How this PR fixes it.**
`FilePartitionUtils.parseColumnsFromPathWithNullInfoForLoad` parses the
path as before and then turns a literal `\N` into an empty value with
the null flag set, like branch-4.1's
`BrokerUtil.parseColumnsFromPathWithNullInfoForLoad`. Both
range-building paths of `NereidsFileGroupInfo` (splittable and
unsplittable files) use it, and so does the legacy `FileGroupInfo`,
which master no longer instantiates. Stream load does not send path
columns and is not affected.
**Results.**
`BrokerUtilTest#parseColumnsFromPathForLoadKeepsLegacyNullSemantics`
(from branch-4.1, plus a case with a regular value and the Hive default
partition) checks the parser.
`NereidsFileGroupInfoTest#loadRangesCarryLiteralNullPathValueAsNull`
builds the scan ranges of a load for a splittable and an unsplittable
CSV file under `p=\N` and checks that the range carries an empty value
with the null flag set. Switching either call site, or the parser, back
to the query behavior makes the corresponding test fail.
`BrokerUtilTest`, `NereidsFileGroupInfoTest` and
`NereidsLoadScanProviderTest` pass (12 tests). There is no regression
test: `test_load_data_from_path` reads fixed objects from the shared S3
test bucket, which has no `p=\N` object.
Co-authored-by: Socrates <[email protected]>
---
.../doris/datasource/scan/FileGroupInfo.java | 4 +-
.../doris/datasource/scan/FilePartitionUtils.java | 25 +++++++
.../doris/nereids/load/NereidsFileGroupInfo.java | 4 +-
.../apache/doris/common/util/BrokerUtilTest.java | 15 ++++
.../nereids/load/NereidsFileGroupInfoTest.java | 81 ++++++++++++++++++++++
5 files changed, 125 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileGroupInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileGroupInfo.java
index b1d70b467fc..1608a896b99 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileGroupInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FileGroupInfo.java
@@ -265,7 +265,7 @@ public class FileGroupInfo {
fileStatus.path);
context.params.setCompressType(compressType);
FilePartitionUtils.ParsedColumnsFromPath columnsFromPath =
-
FilePartitionUtils.parseColumnsFromPathWithNullInfo(fileStatus.path,
+
FilePartitionUtils.parseColumnsFromPathWithNullInfoForLoad(fileStatus.path,
context.fileGroup.getColumnNamesFromPath(),
true, false);
List<String> columnsFromPathKeys =
context.fileGroup.getColumnNamesFromPath();
TFileRangeDesc rangeDesc = createFileRangeDesc(0, fileStatus,
fileStatus.size,
@@ -312,7 +312,7 @@ public class FileGroupInfo {
fileStatus.path);
context.params.setCompressType(compressType);
FilePartitionUtils.ParsedColumnsFromPath columnsFromPath =
-
FilePartitionUtils.parseColumnsFromPathWithNullInfo(fileStatus.path,
+
FilePartitionUtils.parseColumnsFromPathWithNullInfoForLoad(fileStatus.path,
context.fileGroup.getColumnNamesFromPath(), true,
false);
List<String> columnsFromPathKeys =
context.fileGroup.getColumnNamesFromPath();
// Assign scan range locations only for broker load.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FilePartitionUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FilePartitionUtils.java
index 096eb074f71..0b6e73ec4f6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FilePartitionUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/FilePartitionUtils.java
@@ -17,6 +17,7 @@
package org.apache.doris.datasource.scan;
+import org.apache.doris.common.FeConstants;
import org.apache.doris.common.UserException;
import org.apache.doris.connector.spi.scan.ConnectorPartitionValues;
@@ -155,6 +156,30 @@ public final class FilePartitionUtils {
return new ParsedColumnsFromPath(Lists.newArrayList(columns),
Lists.newArrayList(columnValueIsNull));
}
+ /**
+ * Parses partition column values from a load source path. Unlike the
query path, a load also
+ * treats a literal {@code \N} directory value as NULL, the semantics
loads had before the
+ * null flag existed.
+ */
+ public static ParsedColumnsFromPath
parseColumnsFromPathWithNullInfoForLoad(
+ String filePath,
+ List<String> columnsFromPath,
+ boolean caseSensitive,
+ boolean isACID)
+ throws UserException {
+ ParsedColumnsFromPath parsed = parseColumnsFromPathWithNullInfo(
+ filePath, columnsFromPath, caseSensitive, isACID);
+ List<String> values = new ArrayList<>(parsed.getValues());
+ List<Boolean> isNull = new ArrayList<>(parsed.getIsNull());
+ for (int i = 0; i < values.size(); i++) {
+ if (FeConstants.null_string.equals(values.get(i))) {
+ values.set(i, "");
+ isNull.set(i, true);
+ }
+ }
+ return new ParsedColumnsFromPath(values, isNull);
+ }
+
public static ParsedColumnsFromPath normalizeColumnsFromPath(List<String>
columnsFromPath) {
if (columnsFromPath == null || columnsFromPath.isEmpty()) {
return new ParsedColumnsFromPath(Collections.emptyList(),
Collections.emptyList());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsFileGroupInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsFileGroupInfo.java
index 2713365cf28..841965f63aa 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsFileGroupInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsFileGroupInfo.java
@@ -276,7 +276,7 @@ public class NereidsFileGroupInfo {
fileStatus.path);
context.params.setCompressType(compressType);
FilePartitionUtils.ParsedColumnsFromPath columnsFromPath =
-
FilePartitionUtils.parseColumnsFromPathWithNullInfo(fileStatus.path,
+
FilePartitionUtils.parseColumnsFromPathWithNullInfoForLoad(fileStatus.path,
context.fileGroup.getColumnNamesFromPath(),
true, false);
List<String> columnsFromPathKeys =
context.fileGroup.getColumnNamesFromPath();
TFileRangeDesc rangeDesc = createFileRangeDesc(0, fileStatus,
fileStatus.size,
@@ -329,7 +329,7 @@ public class NereidsFileGroupInfo {
fileStatus.path);
context.params.setCompressType(compressType);
FilePartitionUtils.ParsedColumnsFromPath columnsFromPath =
-
FilePartitionUtils.parseColumnsFromPathWithNullInfo(fileStatus.path,
+
FilePartitionUtils.parseColumnsFromPathWithNullInfoForLoad(fileStatus.path,
context.fileGroup.getColumnNamesFromPath(), true,
false);
List<String> columnsFromPathKeys =
context.fileGroup.getColumnNamesFromPath();
// Assign scan range locations only for broker load.
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java
index 5e1d286fc84..bbe5e4340ed 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/BrokerUtilTest.java
@@ -151,4 +151,19 @@ public class BrokerUtilTest {
Assertions.assertEquals(Collections.singletonList(false),
parsed.getIsNull());
}
+ @Test
+ public void parseColumnsFromPathForLoadKeepsLegacyNullSemantics() throws
Exception {
+ FilePartitionUtils.ParsedColumnsFromPath parsed =
FilePartitionUtils.parseColumnsFromPathWithNullInfoForLoad(
+ "hdfs://host/table/p=\\N/data.orc",
Collections.singletonList("p"), true, false);
+ Assertions.assertEquals(Collections.singletonList(""),
parsed.getValues());
+ Assertions.assertEquals(Collections.singletonList(true),
parsed.getIsNull());
+
+ // Other values, including the Hive default partition, parse as on the
query path.
+ parsed = FilePartitionUtils.parseColumnsFromPathWithNullInfoForLoad(
+ "hdfs://host/table/p=a/q=" +
ConnectorPartitionValues.NULL_PARTITION_NAME + "/data.orc",
+ Lists.newArrayList("p", "q"), true, false);
+ Assertions.assertEquals(Lists.newArrayList("a", ""),
parsed.getValues());
+ Assertions.assertEquals(Lists.newArrayList(false, true),
parsed.getIsNull());
+ }
+
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/load/NereidsFileGroupInfoTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/load/NereidsFileGroupInfoTest.java
new file mode 100644
index 00000000000..f6d19ff9b40
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/load/NereidsFileGroupInfoTest.java
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.load;
+
+import org.apache.doris.analysis.BrokerDesc;
+import org.apache.doris.analysis.StorageBackend;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties;
+import org.apache.doris.datasource.scan.FederationBackendPolicy;
+import org.apache.doris.load.loadv2.LoadTask;
+import org.apache.doris.system.Backend;
+import org.apache.doris.thrift.TBrokerFileStatus;
+import org.apache.doris.thrift.TFileRangeDesc;
+import org.apache.doris.thrift.TFileScanRangeParams;
+import org.apache.doris.thrift.TFileType;
+import org.apache.doris.thrift.TScanRangeLocations;
+
+import com.google.common.collect.Lists;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+
+class NereidsFileGroupInfoTest {
+ @Test
+ void loadRangesCarryLiteralNullPathValueAsNull() throws Exception {
+ // Splittable and unsplittable files build their ranges in different
methods.
+ for (boolean splittable : new boolean[] {false, true}) {
+ CsvFileFormatProperties format = new
CsvFileFormatProperties("csv");
+ format.analyzeFileFormatProperties(new HashMap<>(), true);
+ NereidsBrokerFileGroup fileGroup = new NereidsBrokerFileGroup(1L,
false, null,
+ Lists.newArrayList("s3://bucket/table/*/data.csv"), null,
null, Lists.newArrayList("p"),
+ Lists.newArrayList(), null, null, null, null,
LoadTask.MergeType.APPEND, null, -1L,
+ false, false, format);
+ BrokerDesc brokerDesc = Mockito.mock(BrokerDesc.class);
+
Mockito.when(brokerDesc.getFileType()).thenReturn(TFileType.FILE_S3);
+
Mockito.when(brokerDesc.getStorageType()).thenReturn(StorageBackend.StorageType.S3);
+ FederationBackendPolicy backendPolicy =
Mockito.mock(FederationBackendPolicy.class);
+ Mockito.when(backendPolicy.getNextBe()).thenReturn(new Backend(1L,
"127.0.0.1", 9050));
+ Mockito.when(backendPolicy.numBackends()).thenReturn(1);
+ TBrokerFileStatus fileStatus = new
TBrokerFileStatus("s3://bucket/table/p=\\N/data.csv", false, 10,
+ splittable);
+ NereidsFileGroupInfo fileGroupInfo = new NereidsFileGroupInfo(1L,
2L, Mockito.mock(OlapTable.class),
+ brokerDesc, fileGroup, Lists.newArrayList(fileStatus), 1,
false, 1);
+ fileGroupInfo.getFileStatusAndCalcInstance(backendPolicy);
+ NereidsParamCreateContext context = new
NereidsParamCreateContext();
+ context.fileGroup = fileGroup;
+ context.params = new TFileScanRangeParams();
+
+ List<TScanRangeLocations> locations = new ArrayList<>();
+ fileGroupInfo.createScanRangeLocations(context, backendPolicy,
locations);
+
+ Assertions.assertEquals(1, locations.size());
+ List<TFileRangeDesc> ranges =
locations.get(0).getScanRange().getExtScanRange().getFileScanRange()
+ .getRanges();
+ Assertions.assertEquals(1, ranges.size());
+ // A load keeps the meaning a literal \N had before the null flag
existed: SQL NULL.
+ Assertions.assertEquals(Collections.singletonList(""),
ranges.get(0).getColumnsFromPath());
+ Assertions.assertEquals(Collections.singletonList(true),
ranges.get(0).getColumnsFromPathIsNull());
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
