[
https://issues.apache.org/jira/browse/HADOOP-19254?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17912051#comment-17912051
]
ASF GitHub Bot commented on HADOOP-19254:
-----------------------------------------
mukund-thakur commented on code in PR #7197:
URL: https://github.com/apache/hadoop/pull/7197#discussion_r1910703916
##########
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestBulkDeleteCommand.java:
##########
@@ -27,103 +27,134 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.HadoopTestBase;
import org.assertj.core.api.Assertions;
import org.junit.BeforeClass;
import org.junit.Test;
-public class TestBulkDeleteCommand extends HadoopTestBase {
- private static Configuration conf;
- private static FsShell shell;
- private static LocalFileSystem lfs;
- private static Path testRootDir;
+public class TestBulkDeleteCommand extends HadoopTestBase {
+ private static Configuration conf;
+ private static FsShell shell;
+ private static LocalFileSystem lfs;
+ private static Path testRootDir;
- @BeforeClass
- public static void setup() throws IOException {
- conf = new Configuration();
- shell = new FsShell(conf);
- lfs = FileSystem.getLocal(conf);
- testRootDir = lfs.makeQualified(new Path(GenericTestUtils.getTempPath(
- "testFsShellBulkDelete")));
- lfs.delete(testRootDir, true);
- lfs.mkdirs(testRootDir);
- lfs.setWorkingDirectory(testRootDir);
- }
+ @BeforeClass
+ public static void setup() throws IOException {
+ conf = new Configuration();
+ shell = new FsShell(conf);
+ lfs = FileSystem.getLocal(conf);
+ testRootDir = lfs.makeQualified(new Path(GenericTestUtils.getTempPath(
+ "testFsShellBulkDelete")));
+ lfs.delete(testRootDir, true);
+ lfs.mkdirs(testRootDir);
+ lfs.setWorkingDirectory(testRootDir);
+ }
- @Test
- public void testDefaults() throws IOException {
- LinkedList<String> options = new LinkedList<>();
- BulkDeleteCommand bulkDeleteCommand = new BulkDeleteCommand();
- bulkDeleteCommand.processOptions(options);
- assertTrue(bulkDeleteCommand.childArgs.isEmpty());
- }
+ @Test
+ public void testDefaults() throws IOException {
+ LinkedList<String> options = new LinkedList<>();
+ BulkDeleteCommand bulkDeleteCommand = new BulkDeleteCommand(conf);
+ bulkDeleteCommand.processOptions(options);
+ assertTrue(bulkDeleteCommand.childArgs.isEmpty());
+ }
- @Test
- public void testArguments() throws IOException, URISyntaxException {
- BulkDeleteCommand bulkDeleteCommand = new BulkDeleteCommand(conf);
- LinkedList<String> arguments = new LinkedList<>();
- String arg1 = "file:///file/name/1";
- String arg2 = "file:///file/name/2";
- arguments.add(arg1);
- arguments.add(arg2);
- LinkedList<PathData> pathData =
bulkDeleteCommand.expandArguments(arguments);
- Assertions.assertThat(pathData.size()).
- describedAs("Only one root path must be present").isEqualTo(1);
- Assertions.assertThat(pathData.get(0).path.toUri().getPath()).
- describedAs("Base path of the command should
match").isEqualTo(new URI(arg1).getPath());
- Assertions.assertThat(bulkDeleteCommand.childArgs.size()).
- describedAs("Only one other argument was passed to the
command").
- isEqualTo(1);
- Assertions.assertThat(bulkDeleteCommand.childArgs.get(0)).
- describedAs("Children arguments must match").isEqualTo(arg2);
- }
+ @Test
+ public void testArguments() throws IOException, URISyntaxException {
+ BulkDeleteCommand bulkDeleteCommand = new BulkDeleteCommand(conf);
+ LinkedList<String> arguments = new LinkedList<>();
+ String arg1 = "file:///file/name/1";
+ String arg2 = "file:///file/name/1/2";
+ arguments.add(arg1);
+ arguments.add(arg2);
+ LinkedList<PathData> pathData =
bulkDeleteCommand.expandArguments(arguments);
+ Assertions.assertThat(pathData.size()).
+ describedAs("Only one root path must be present").isEqualTo(1);
+ Assertions.assertThat(pathData.get(0).path.toUri().getPath()).
+ describedAs("Base path of the command should match").isEqualTo(new
URI(arg1).getPath());
+ Assertions.assertThat(bulkDeleteCommand.childArgs.size()).
+ describedAs("Only one other argument was passed to the command").
+ isEqualTo(1);
+ Assertions.assertThat(bulkDeleteCommand.childArgs.get(0)).
+ describedAs("Children arguments must match").isEqualTo(arg2);
+ }
- @Test
- public void testLocalFileDeletion() throws IOException {
- String deletionDir = "toDelete";
- String baseFileName = "file_";
- Path baseDir = new Path(testRootDir, deletionDir);
- List<String> listOfPaths = new ArrayList<>();
- for(int i = 0; i < 100; i++) {
- Path p = new Path(baseDir, baseFileName + i);
- lfs.create(p);
- listOfPaths.add(p.toUri().toString());
- }
- List<String> finalCommandList = new ArrayList<>();
- finalCommandList.add("-bulkDelete");
- finalCommandList.add(baseDir.toUri().toString());
- finalCommandList.addAll(listOfPaths);
- shell.run(finalCommandList.toArray(new String[0]));
- Assertions.assertThat(lfs.listFiles(baseDir, false).hasNext())
- .as("All the files should have been deleted").isEqualTo(false);
+ @Test
+ public void testWrongArguments() throws IOException, URISyntaxException {
+ BulkDeleteCommand bulkDeleteCommand = new BulkDeleteCommand(conf);
+ LinkedList<String> arguments = new LinkedList<>();
+ String arg1 = "file:///file/name/1";
+ arguments.add(arg1);
+ Assertions.assertThatThrownBy(() ->
bulkDeleteCommand.expandArguments(arguments)).
+ describedAs("No children to be deleted specified in the command.").
+ isInstanceOf(IOException.class);
+ }
+ @Test
+ public void testLocalFileDeletion() throws IOException {
+ String deletionDir = "toDelete";
+ String baseFileName = "file_";
+ Path baseDir = new Path(testRootDir, deletionDir);
+ List<String> listOfPaths = new ArrayList<>();
+ for (int i = 0; i < 100; i++) {
+ Path p = new Path(baseDir, baseFileName + i);
+ ContractTestUtils.touch(lfs, p);
}
+ RemoteIterator<LocatedFileStatus> remoteIterator = lfs.listFiles(baseDir,
false);
+ while (remoteIterator.hasNext()) {
+ listOfPaths.add(remoteIterator.next().getPath().toUri().toString());
+ }
+ List<String> finalCommandList = new ArrayList<>();
+ finalCommandList.add("-bulkDelete");
+ finalCommandList.add(baseDir.toUri().toString());
+ finalCommandList.addAll(listOfPaths);
+ shell.run(finalCommandList.toArray(new String[0]));
+ Assertions.assertThat(lfs.listFiles(baseDir, false).hasNext())
+ .as("All the files should have been deleted under the path:" +
+ baseDir).isEqualTo(false);
- @Test
- public void testLocalFileDeletionWithFileName() throws IOException {
- String deletionDir = "toDelete";
- String baseFileName = "file_";
- Path baseDir = new Path(testRootDir, deletionDir);
- Path fileWithDeletePaths = new Path(testRootDir,
"fileWithDeletePaths");
- FSDataOutputStream fsDataOutputStream =
lfs.create(fileWithDeletePaths, true);
- BufferedWriter br = new BufferedWriter(new
OutputStreamWriter(fsDataOutputStream));
- for(int i = 0; i < 100; i++) {
- Path p = new Path(baseDir, baseFileName + i);
- lfs.create(p);
- br.write(p.toUri().toString());
- br.newLine();
- }
- br.flush(); // flush the file to write the contents
- br.close(); // close the writer
- List<String> finalCommandList = new ArrayList<>();
- finalCommandList.add("-bulkDelete");
- finalCommandList.add("-readFromFile");
- finalCommandList.add(fileWithDeletePaths.toUri().toString());
- finalCommandList.add(baseDir.toUri().toString());
- shell.run(finalCommandList.toArray(new String[0]));
- Assertions.assertThat(lfs.listFiles(baseDir, false).hasNext())
- .as("All the files should have been deleted").isEqualTo(false);
+ }
+ @Test
+ public void testLocalFileDeletionWithFileName() throws IOException {
+ String deletionDir = "toDelete";
+ String baseFileName = "file_";
+ Path baseDir = new Path(testRootDir, deletionDir);
+ Path fileWithDeletePaths = new Path(testRootDir, "fileWithDeletePaths");
+ FSDataOutputStream fsDataOutputStream = lfs.create(fileWithDeletePaths,
true);
+ BufferedWriter br = new BufferedWriter(new
OutputStreamWriter(fsDataOutputStream));
+ for (int i = 0; i < 100; i++) {
+ Path p = new Path(baseDir, baseFileName + i);
+ ContractTestUtils.touch(lfs, p);
+ br.write(p.toUri().toString());
+ br.newLine();
}
+ br.flush(); // flush the file to write the contents
+ br.close(); // close the writer
+ List<String> finalCommandList = new ArrayList<>();
+ finalCommandList.add("-bulkDelete");
+ finalCommandList.add("-readFromFile");
+ finalCommandList.add(fileWithDeletePaths.toUri().toString());
+ finalCommandList.add(baseDir.toUri().toString());
+ shell.run(finalCommandList.toArray(new String[0]));
+ Assertions.assertThat(lfs.listFiles(baseDir, false).hasNext())
Review Comment:
Oh I see..missed the touch part.
> Implement bulk delete command as hadoop fs command operation
> -------------------------------------------------------------
>
> Key: HADOOP-19254
> URL: https://issues.apache.org/jira/browse/HADOOP-19254
> Project: Hadoop Common
> Issue Type: Improvement
> Components: fs
> Affects Versions: 3.4.1
> Reporter: Mukund Thakur
> Assignee: Harshit Gupta
> Priority: Major
> Labels: pull-request-available
>
> {code}
> hadoop fs -bulkdelete <base-url> <file>
> {code}
> Key uses
> * QE: Testing from python and other scripting languages
> * cluster maintenance: actual bulk deletion operations from the store
> one thought there: we MUST qualify paths with / elements: if a passed in path
> ends in /, it means "delete a marker", not "delete a dir"'. and if it doesn't
> have one then it's an object.. This makes it possible to be used to delete
> surplus markers or where there is a file above another file...cloudstore
> listobjects finds this
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
