[
https://issues.apache.org/jira/browse/HADOOP-19778?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=18055137#comment-18055137
]
ASF GitHub Bot commented on HADOOP-19778:
-----------------------------------------
anujmodi2021 commented on code in PR #8207:
URL: https://github.com/apache/hadoop/pull/8207#discussion_r2741203659
##########
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java:
##########
@@ -18,4039 +18,130 @@
package org.apache.hadoop.fs.azure;
-import java.io.DataOutputStream;
-import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.charset.StandardCharsets;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.EnumSet;
-import java.util.TimeZone;
-import java.util.UUID;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Optional;
-import java.util.Stack;
-import java.util.HashMap;
-
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectReader;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.BufferedFSInputStream;
-import org.apache.hadoop.fs.CommonPathCapabilities;
-import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.hadoop.fs.FSInputStream;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PositionedReadable;
-import org.apache.hadoop.fs.Seekable;
-import org.apache.hadoop.fs.StreamCapabilities;
-import org.apache.hadoop.fs.Syncable;
-import org.apache.hadoop.fs.XAttrSetFlag;
-import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
-import org.apache.hadoop.fs.azure.metrics.AzureFileSystemMetricsSystem;
-import org.apache.hadoop.fs.azure.security.Constants;
-import org.apache.hadoop.fs.azure.security.RemoteWasbDelegationTokenManager;
-import org.apache.hadoop.fs.azure.security.WasbDelegationTokenManager;
-import org.apache.hadoop.fs.impl.AbstractFSBuilderImpl;
-import org.apache.hadoop.fs.impl.OpenFileParameters;
-import org.apache.hadoop.fs.impl.StoreImplementationUtils;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.permission.PermissionStatus;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
-import org.apache.hadoop.util.LambdaUtils;
-import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.util.Time;
-
-import org.apache.hadoop.util.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static
org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_STANDARD_OPTIONS;
-import static org.apache.hadoop.fs.azure.NativeAzureFileSystemHelper.*;
-import static
org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.hadoop.classification.VisibleForTesting;
-import com.microsoft.azure.storage.StorageException;
-
-/**
- * A {@link FileSystem} for reading and writing files stored on <a
- * href="http://store.azure.com/";>Windows Azure</a>. This implementation is
- * blob-based and stores files on Azure in their native form so they can be
read
- * by other Azure tools.
- */
[email protected]
[email protected]
-public class NativeAzureFileSystem extends FileSystem {
- private static final int USER_WX_PERMISION = 0300;
- private static final String USER_HOME_DIR_PREFIX_DEFAULT = "/user";
- /**
- * A description of a folder rename operation, including the source and
- * destination keys, and descriptions of the files in the source folder.
- */
-
- public static class FolderRenamePending {
- private SelfRenewingLease folderLease;
- private String srcKey;
- private String dstKey;
- private FileMetadata[] fileMetadata = null; // descriptions of source
files
- private ArrayList<String> fileStrings = null;
- private NativeAzureFileSystem fs;
- private static final int MAX_RENAME_PENDING_FILE_SIZE = 10000000;
- private static final int FORMATTING_BUFFER = 10000;
- private boolean committed;
- public static final String SUFFIX = "-RenamePending.json";
- private static final ObjectReader READER = new ObjectMapper()
- .configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true)
- .readerFor(JsonNode.class);
-
- // Prepare in-memory information needed to do or redo a folder rename.
- public FolderRenamePending(String srcKey, String dstKey, SelfRenewingLease
lease,
- NativeAzureFileSystem fs) throws IOException {
- this.srcKey = srcKey;
- this.dstKey = dstKey;
- this.folderLease = lease;
- this.fs = fs;
-
- // List all the files in the folder.
- long start = Time.monotonicNow();
- fileMetadata = fs.getStoreInterface().list(srcKey, AZURE_LIST_ALL,
- AZURE_UNBOUNDED_DEPTH);
-
- long end = Time.monotonicNow();
- LOG.debug("Time taken to list {} blobs for rename operation is: {} ms",
fileMetadata.length, (end - start));
-
- this.committed = true;
- }
-
- // Prepare in-memory information needed to do or redo folder rename from
- // a -RenamePending.json file read from storage. This constructor is to
use during
- // redo processing.
- public FolderRenamePending(Path redoFile, NativeAzureFileSystem fs)
- throws IllegalArgumentException, IOException {
-
- this.fs = fs;
-
- // open redo file
- Path f = redoFile;
- int l;
- byte[] bytes;
- try (FSDataInputStream input = fs.open(f)) {
- bytes = new byte[MAX_RENAME_PENDING_FILE_SIZE];
- l = input.read(bytes);
- }
- if (l <= 0) {
- // Jira HADOOP-12678 -Handle empty rename pending metadata file during
- // atomic rename in redo path. If during renamepending file is created
- // but not written yet, then this means that rename operation
- // has not started yet. So we should delete rename pending metadata
file.
- LOG.error("Deleting empty rename pending file "
- + redoFile + "
> Remove Deprecated WASB Code from Hadoop
> ---------------------------------------
>
> Key: HADOOP-19778
> URL: https://issues.apache.org/jira/browse/HADOOP-19778
> Project: Hadoop Common
> Issue Type: Sub-task
> Components: fs/azure
> Affects Versions: 3.4.2
> Reporter: Anuj Modi
> Assignee: Anuj Modi
> Priority: Major
> Labels: pull-request-available
>
> WASB Driver has been deprecated for a while now:
> https://issues.apache.org/jira/browse/HADOOP-19178
> Starting with hadoop-3.5.0 we will no longer ship WASB Driver as a official
> hadoop-connector for Azure.
> Moving ahead ABFS Driver will be the only official Hadoop connector for both
> types of Accounts that is "Heirarchichal Namespace Enabled (HNS)" and
> "Heirarchichal Namespace Disabled (FNS)"
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
