steveloughran commented on a change in pull request #2971: URL: https://github.com/apache/hadoop/pull/2971#discussion_r826234036
########## File path: hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestStoreOperationsThroughFileSystem.java ########## @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.CommonPathCapabilities; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathIOException; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.Trash; +import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData; +import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest; +import org.apache.hadoop.util.JsonSerialization; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; + +/** + * Implementation of manifest store operations through the filesystem API. + * {@link #moveToTrash(String, Path)} uses the normal Trash classes. + * resilient commit is not available. + * This class is subclassed in the ABFS module, which does add the resilient + * commit method. + */ [email protected]("mapreduce, object-stores") [email protected] +public class ManifestStoreOperationsThroughFileSystem extends ManifestStoreOperations { + + private static final Logger LOG = LoggerFactory.getLogger( + ManifestStoreOperationsThroughFileSystem.class); + + /** + * Trash.moveToTrash() returned false. + */ + public static final String E_TRASH_FALSE = "Failed to rename to trash" + + " -check trash interval in " + FS_TRASH_INTERVAL_KEY +": "; + + /** + * Filesystem; set in {@link #bindToFileSystem(FileSystem, Path)}. + */ + private FileSystem fileSystem; + + /** + * Has a call to msync failed as unsupported? + */ + private boolean msyncUnsupported = false; + + /** + * Direct Constructor. + * @param fileSystem filesystem to write through. + */ + public ManifestStoreOperationsThroughFileSystem(final FileSystem fileSystem) { + this.fileSystem = fileSystem; + } + + /** + * Constructor used for introspection-based binding. + */ + public ManifestStoreOperationsThroughFileSystem() { + } + + @Override + public void close() throws IOException { + /* no-op; FS is assumed to be shared. */ + + } + + /** + * Get the filesystem. + * @return the filesystem; null until bound. + */ + public FileSystem getFileSystem() { + return fileSystem; + } + + @Override + public void bindToFileSystem(FileSystem filesystem, Path path) throws IOException { + fileSystem = filesystem; + } + + @Override + public FileStatus getFileStatus(Path path) throws IOException { + return fileSystem.getFileStatus(path); + } + + /** + * Using FileSystem.isFile to offer stores the option to optimize their probes. + * @param path path to probe + * @return true if the path resolves to a file. + * @throws IOException IO failure. + */ + @SuppressWarnings("deprecation") + @Override + public boolean isFile(Path path) throws IOException { + return fileSystem.isFile(path); + } + + @Override + public boolean delete(Path path, boolean recursive) + throws IOException { + return fileSystem.delete(path, recursive); + } + + @Override + public boolean mkdirs(Path path) + throws IOException { + return fileSystem.mkdirs(path); + } + + @Override + public boolean renameFile(Path source, Path dest) + throws IOException { + return fileSystem.rename(source, dest); + } + + @Override + public RemoteIterator<FileStatus> listStatusIterator(Path path) + throws IOException { + return fileSystem.listStatusIterator(path); + } + + @Override + public TaskManifest loadTaskManifest( + JsonSerialization<TaskManifest> serializer, + FileStatus st) throws IOException { + return TaskManifest.load(serializer, fileSystem, st.getPath(), st); + } + + @Override + public <T extends AbstractManifestData<T>> void save( + final T manifestData, + final Path path, + final boolean overwrite) throws IOException { + manifestData.save(fileSystem, path, overwrite); + } + + @Override + public boolean isTrashEnabled(Path path) { + try { + return fileSystem.getServerDefaults(path).getTrashInterval() > 0; Review comment: there's standard policies here. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
