[
https://issues.apache.org/jira/browse/HADOOP-19941?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=18107345#comment-18107345
]
ASF GitHub Bot commented on HADOOP-19941:
-----------------------------------------
anmolanmol1234 commented on code in PR #8611:
URL: https://github.com/apache/hadoop/pull/8611#discussion_r3842952895
##########
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsPhotonListStatus.java:
##########
@@ -0,0 +1,249 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+
+import static
org.apache.hadoop.fs.azurebfs.AbfsStatistic.PHOTON_FALLBACK_COUNT;
+import static
org.apache.hadoop.fs.azurebfs.AbfsStatistic.PHOTON_LISTING_LATENCY;
+import static
org.apache.hadoop.fs.azurebfs.AbfsStatistic.PHOTON_PARSE_FAILURE_COUNT;
+import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.PHOTON_REQUEST_COUNT;
+import static
org.apache.hadoop.fs.azurebfs.AbfsStatistic.PHOTON_RESPONSE_COUNT;
+import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_LIST_MAX_RESULTS;
+import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_PHOTON;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Live integration tests for the Photon (Apache Arrow based ListBlob) listing
+ * path against a real Blob-endpoint account. These cover the integration
+ * scenarios from the design document that are not verifiable with unit tests:
+ * <ul>
+ * <li>XML-vs-Photon {@link FileStatus} parity (identical results regardless
of
+ * the wire format).</li>
+ * <li>Graceful XML fallback when Arrow is requested but the account returns
+ * XML (asserted via the Photon telemetry classification).</li>
+ * <li>Pagination across multiple Photon responses.</li>
+ * <li>Photon telemetry counters and listing-latency tracker are
emitted.</li>
+ * </ul>
+ *
+ * <p>Photon is only offered on the Blob endpoint, so every test asserts a Blob
+ * service type before running.</p>
+ */
+public class ITestAbfsPhotonListStatus extends AbstractAbfsIntegrationTest {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestAbfsPhotonListStatus.class);
+
+ private static final String[] CHILD_FILES = {
+ "file-a.txt",
+ "file-b.txt",
+ "name with space.txt",
+ "\u6587\u4ef6-unicode.txt",
+ };
+
+ private static final String CHILD_DIR = "subdir";
+
+ public ITestAbfsPhotonListStatus() throws Exception {
+ }
+
+ /**
+ * Create a filesystem instance with Photon explicitly toggled and,
optionally,
+ * a reduced ListBlobs page size to force multi-page pagination.
+ */
+ private AzureBlobFileSystem newFileSystem(final boolean photonEnabled,
+ final int listMaxResults) throws IOException {
+ Configuration conf = new Configuration(getRawConfiguration());
+ conf.setBoolean(FS_AZURE_ENABLE_PHOTON, photonEnabled);
+ if (listMaxResults > 0) {
+ conf.setInt(AZURE_LIST_MAX_RESULTS, listMaxResults);
+ }
+ return (AzureBlobFileSystem) FileSystem.newInstance(conf);
+ }
+
+ /**
+ * Populate a directory with a mix of files (including special-character and
+ * Unicode names) and a subdirectory, so a listing exercises files and
+ * directories together.
+ */
+ private void createTree(final AzureBlobFileSystem fs, final Path baseDir)
+ throws IOException {
+ fs.mkdirs(baseDir);
+ for (int i = 0; i < CHILD_FILES.length; i++) {
+ try (FSDataOutputStream out = fs.create(new Path(baseDir,
CHILD_FILES[i]))) {
+ // Give the files distinct, non-zero sizes to make parity meaningful.
+ out.write(new byte[i + 1]);
+ }
+ }
+ fs.mkdirs(new Path(baseDir, CHILD_DIR));
+ }
+
+ private static List<FileStatus> sortedByName(final FileStatus[] statuses) {
+ return Arrays.stream(statuses)
+ .sorted(Comparator.comparing(s -> s.getPath().getName()))
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Verify that listing the same directory over XML (Photon disabled) and over
+ * Arrow (Photon enabled) yields identical {@link FileStatus} results,
covering
+ * the doc's XML/Photon parity and identical-FileStatus scenarios.
+ */
+ @Test
+ public void testPhotonAndXmlListingParity() throws Exception {
Review Comment:
We compare path, isDirectory and length, but not getModificationTime().
Timestamp normalization is the largest and riskiest part of this change — the
hand-written fastIsoUtcToRfc1123() with the Sakamoto weekday math — and this is
the only place we could catch a real end-to-end mismatch against XML. Please
add mtime parity (allow a second of tolerance if the formats differ in
precision).
> ABFS: Support Photon (Apache Arrow) based ListBlobs on Blob endpoint with XML
> fallback
> --------------------------------------------------------------------------------------
>
> Key: HADOOP-19941
> URL: https://issues.apache.org/jira/browse/HADOOP-19941
> Project: Hadoop Common
> Issue Type: New Feature
> Components: fs/azure
> Reporter: Manish Bhatt
> Assignee: Manish Bhatt
> Priority: Major
> Labels: pull-request-available
>
> Add config-gated support (fs.azure.photon.enabled, default off) for consuming
> ListBlobs responses in the Apache Arrow (Photon) format on the ABFS Blob
> endpoint. When enabled, ABFS advertises Arrow via an Accept header; the
> response Content-Type selects an Arrow or the existing XML parser, both
> producing identical FileStatus results, so downstream behaviour is unchanged.
> Includes automatic, transparent fallback to XML, full parsing parity
> (metadata, directory markers, implicit directories, copy properties, native
> timestamp/length vectors), interrupt-safe Arrow parsing, and Photon telemetry
> (request, response, fallback, parse-failure counts and listing latency). No
> public API changes. Covered by unit and integration tests.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
