This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 3498af81f2 HDDS-11837. Support executing multiple commands in Ozone
CLI (#7727)
3498af81f2 is described below
commit 3498af81f282e07f149a7019c0889da1c12f4e2d
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Mon Jan 27 18:39:19 2025 +0100
HDDS-11837. Support executing multiple commands in Ozone CLI (#7727)
---
.../dist/src/main/smoketest/basic/links.robot | 47 ++++++++++------------
.../src/main/smoketest/basic/ozone-shell-lib.robot | 39 +++++++++---------
.../dist/src/main/smoketest/ozone-lib/shell.robot | 5 +++
.../java/org/apache/hadoop/ozone/shell/REPL.java | 17 ++++++--
.../java/org/apache/hadoop/ozone/shell/Shell.java | 33 +++++++++++++--
5 files changed, 88 insertions(+), 53 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/smoketest/basic/links.robot
b/hadoop-ozone/dist/src/main/smoketest/basic/links.robot
index d4d53c8d3c..7bcec33342 100644
--- a/hadoop-ozone/dist/src/main/smoketest/basic/links.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/basic/links.robot
@@ -31,23 +31,20 @@ Create volumes
${random} = Generate Random String 5 [NUMBERS]
Set Suite Variable ${source} ${random}-source
Set Suite Variable ${target} ${random}-target
- Execute ozone sh volume create ${source}
- Execute ozone sh volume create ${target}
+ Ozone Shell Batch volume create ${source}
+ ... volume create ${target}
Run Keyword if '${SECURITY_ENABLED}' == 'true' Setup ACL tests
Setup ACL tests
- Execute ozone sh bucket create ${source}/readable-bucket
- Execute ozone sh key put
${source}/readable-bucket/key-in-readable-bucket /etc/passwd
- Execute ozone sh bucket create ${source}/unreadable-bucket
-
- Execute ozone sh bucket link ${source}/unreadable-bucket
${target}/link-to-unreadable-bucket
- Execute ozone sh volume addacl --acl user:testuser2:r[DEFAULT]
${target}
-
- Execute ozone sh bucket link ${source}/readable-bucket
${target}/readable-link
- Execute ozone sh bucket link ${source}/readable-bucket
${target}/readable-link2
-
- Execute ozone sh volume addacl --acl user:testuser2:rl
${source}
- Execute ozone sh bucket addacl --acl user:testuser2:rl
${source}/readable-bucket
+ Ozone Shell Batch bucket create ${source}/readable-bucket
+ ... key put
${source}/readable-bucket/key-in-readable-bucket /etc/passwd
+ ... bucket create ${source}/unreadable-bucket
+ ... bucket link ${source}/unreadable-bucket
${target}/link-to-unreadable-bucket
+ ... volume addacl --acl user:testuser2:r[DEFAULT] ${target}
+ ... bucket link ${source}/readable-bucket
${target}/readable-link
+ ... bucket link ${source}/readable-bucket
${target}/readable-link2
+ ... volume addacl --acl user:testuser2:rl ${source}
+ ... bucket addacl --acl user:testuser2:rl
${source}/readable-bucket
Verify Bucket ACL
[arguments] ${source_option} ${object} ${type} ${name}
${acls}
@@ -81,14 +78,14 @@ ACL verified on source and target bucket
Create link loop
Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user
testuser testuser.keytab
- Execute ozone sh bucket link
${target}/loop1 ${target}/loop2
- Execute ozone sh bucket link
${target}/loop2 ${target}/loop3
- Execute ozone sh bucket link
${target}/loop3 ${target}/loop1
+ Ozone Shell Batch bucket link ${target}/loop1
${target}/loop2
+ ... bucket link ${target}/loop2
${target}/loop3
+ ... bucket link ${target}/loop3
${target}/loop1
Delete link loop
- Execute ozone sh bucket delete
${target}/loop1
- Execute ozone sh bucket delete
${target}/loop2
- Execute ozone sh bucket delete
${target}/loop3
+ Ozone Shell Batch bucket delete ${target}/loop1
+ ... bucket delete ${target}/loop2
+ ... bucket delete ${target}/loop3
*** Test Cases ***
Link to non-existent bucket
@@ -97,9 +94,9 @@ Link to non-existent bucket
Should Contain ${result}
BUCKET_NOT_FOUND
Key create passthrough
- Execute ozone sh bucket link
${source}/bucket1 ${target}/link1
- Execute ozone sh bucket create
${source}/bucket1
- Execute ozone sh key put
${target}/link1/key1 /etc/passwd
+ Ozone Shell Batch bucket link
${source}/bucket1 ${target}/link1
+ ... bucket create
${source}/bucket1
+ ... key put
${target}/link1/key1 /etc/passwd
Key Should Match Local File ${target}/link1/key1
/etc/passwd
Key read passthrough
@@ -211,8 +208,8 @@ Loop in link chain is detected
[teardown] Delete link loop
Multiple links to same bucket are allowed
- Execute ozone sh bucket link ${source}/bucket1
${target}/link3
- Execute ozone sh key put ${target}/link3/key3
/etc/group
+ Ozone Shell Batch bucket link ${source}/bucket1
${target}/link3
+ ... key put ${target}/link3/key3 /etc/group
Key Should Match Local File ${target}/link1/key3 /etc/group
Source bucket not affected by deleting link
diff --git a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-lib.robot
b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-lib.robot
index 719cdaf83f..16d8f6febf 100644
--- a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-lib.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-lib.robot
@@ -17,6 +17,7 @@
Documentation Test ozone shell CLI usage
Library OperatingSystem
Resource ../commonlib.robot
+Resource ../ozone-lib/shell.robot
*** Variables ***
${prefix} generated
@@ -168,24 +169,20 @@ Test Bucket Acls
Test key handling
[arguments] ${protocol} ${server} ${volume}
- Execute ozone sh key put
${protocol}${server}/${volume}/bb1/key1 /opt/hadoop/NOTICE.txt
- Execute rm -f /tmp/NOTICE.txt.1
- Execute ozone sh key get
${protocol}${server}/${volume}/bb1/key1 /tmp/NOTICE.txt.1
+ Execute rm -f /tmp/NOTICE.txt.1
/tmp/key1_RATIS /tmp/key1-copy
+ Ozone Shell Batch key put
${protocol}${server}/${volume}/bb1/key1 /opt/hadoop/NOTICE.txt
+ ... key get
${protocol}${server}/${volume}/bb1/key1 /tmp/NOTICE.txt.1
+ ... key put -t RATIS
${protocol}${server}/${volume}/bb1/key1_RATIS /opt/hadoop/NOTICE.txt
+ ... key get
${protocol}${server}/${volume}/bb1/key1_RATIS /tmp/key1_RATIS
+ ... key cp
${protocol}${server}/${volume}/bb1 key1 key1-copy
+ ... key get
${protocol}${server}/${volume}/bb1/key1-copy /tmp/key1-copy
Execute diff -q /opt/hadoop/NOTICE.txt
/tmp/NOTICE.txt.1
-
- Execute ozone sh key put -t RATIS
${protocol}${server}/${volume}/bb1/key1_RATIS /opt/hadoop/NOTICE.txt
- Execute rm -f /tmp/key1_RATIS
- Execute ozone sh key get
${protocol}${server}/${volume}/bb1/key1_RATIS /tmp/key1_RATIS
Execute diff -q /opt/hadoop/NOTICE.txt
/tmp/key1_RATIS
+
${result} = Execute ozone sh key info
${protocol}${server}/${volume}/bb1/key1_RATIS | jq -r '. |
select(.name=="key1_RATIS")'
Should contain ${result} RATIS
- Execute ozone sh key delete
${protocol}${server}/${volume}/bb1/key1_RATIS
- Execute ozone sh key cp
${protocol}${server}/${volume}/bb1 key1 key1-copy
- Execute rm -f /tmp/key1-copy
- Execute ozone sh key get
${protocol}${server}/${volume}/bb1/key1-copy /tmp/key1-copy
Execute diff -q /opt/hadoop/NOTICE.txt
/tmp/key1-copy
- Execute ozone sh key delete
${protocol}${server}/${volume}/bb1/key1-copy
${result} = Execute And Ignore Error ozone sh key get
${protocol}${server}/${volume}/bb1/key1 /tmp/NOTICE.txt.1
Should Contain ${result} NOTICE.txt.1 exists
@@ -199,7 +196,9 @@ Test key handling
Execute ozone sh key rename
${protocol}${server}/${volume}/bb1 key1 key2
${result} = Execute ozone sh key list
${protocol}${server}/${volume}/bb1 | jq -r '.[] | select(.name=="key2") | .name'
Should Be Equal ${result} key2
- Execute ozone sh key delete
${protocol}${server}/${volume}/bb1/key2
+ Ozone Shell Batch key delete
${protocol}${server}/${volume}/bb1/key2
+ ... key delete
${protocol}${server}/${volume}/bb1/key1_RATIS
+ ... key delete
${protocol}${server}/${volume}/bb1/key1-copy
Test key Acls
[arguments] ${protocol} ${server} ${volume}
@@ -274,18 +273,18 @@ Test native authorizer
Test Delete key with Trash
[arguments] ${protocol} ${server} ${volume}
- Execute ozone sh volume create
${protocol}${server}/${volume}
- Execute ozone sh bucket create
${protocol}${server}/${volume}/bfso --layout FILE_SYSTEM_OPTIMIZED
- Execute ozone sh key put -t RATIS
${protocol}${server}/${volume}/bfso/key3 /opt/hadoop/NOTICE.txt
- Execute ozone sh key delete
${protocol}${server}/${volume}/bfso/key3
+ Ozone Shell Batch volume create
${protocol}${server}/${volume}
+ ... bucket create
${protocol}${server}/${volume}/bfso --layout FILE_SYSTEM_OPTIMIZED
+ ... key put -t RATIS
${protocol}${server}/${volume}/bfso/key3 /opt/hadoop/NOTICE.txt
+ ... key delete
${protocol}${server}/${volume}/bfso/key3
${fsokey} = Execute ozone sh key list
${protocol}${server}/${volume}/bfso
${result} = Execute echo '${fsokey}' | jq -r '.[] |
select(.name | startswith(".Trash")) | .name'
Should Contain Any ${result} .Trash/hadoop
.Trash/testuser .Trash/root
Should contain ${result} key3
${result} = Execute echo '${fsokey}' | jq -r '.[] |
select(.name | startswith(".Trash") | not) | .name'
Should Not contain ${result} key3
- Execute ozone sh bucket create
${protocol}${server}/${volume}/obsbkt --layout OBJECT_STORE
- Execute ozone sh key put -t RATIS
${protocol}${server}/${volume}/obsbkt/key2 /opt/hadoop/NOTICE.txt
- Execute ozone sh key delete
${protocol}${server}/${volume}/obsbkt/key2
+ Ozone Shell Batch bucket create
${protocol}${server}/${volume}/obsbkt --layout OBJECT_STORE
+ ... key put -t RATIS
${protocol}${server}/${volume}/obsbkt/key2 /opt/hadoop/NOTICE.txt
+ ... key delete
${protocol}${server}/${volume}/obsbkt/key2
${result} = Execute ozone sh key list
${protocol}${server}/${volume}/obsbkt
Should not contain ${result} key2
diff --git a/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell.robot
b/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell.robot
index d5762f912e..aa288ed865 100644
--- a/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell.robot
@@ -19,6 +19,11 @@ Library String
*** Keywords ***
+Ozone Shell Batch
+ [arguments] @{commands}
+ ${cmd} = Catenate SEPARATOR=' --execute ' @{commands}
+ Run Keyword And Return Execute and checkrc ozone sh --execute
'${cmd}' 0
+
Bucket Exists
[arguments] ${bucket}
${rc} ${output} = Run And Return Rc And Output timeout
15 ozone sh bucket info ${bucket}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/REPL.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/REPL.java
index 1484884634..06cf3958e6 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/REPL.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/REPL.java
@@ -36,6 +36,7 @@
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.List;
import java.util.function.Supplier;
/**
@@ -44,7 +45,7 @@
*/
class REPL {
- REPL(Shell shell, CommandLine cmd, PicocliCommandsFactory factory) {
+ REPL(Shell shell, CommandLine cmd, PicocliCommandsFactory factory,
List<String> lines) {
Parser parser = new DefaultParser();
Supplier<Path> workDir = () -> Paths.get(System.getProperty("user.dir"));
TerminalBuilder terminalBuilder = TerminalBuilder.builder()
@@ -65,12 +66,20 @@ class REPL {
.variable(LineReader.LIST_MAX, 50)
.build();
- TailTipWidgets widgets = new TailTipWidgets(reader,
registry::commandDescription, 5, TipType.COMPLETER);
- widgets.enable();
+ if (!Terminal.TYPE_DUMB.equals(terminal.getType()) &&
!Terminal.TYPE_DUMB_COLOR.equals(terminal.getType())) {
+ TailTipWidgets widgets = new TailTipWidgets(reader,
registry::commandDescription, 5, TipType.COMPLETER);
+ widgets.enable();
+ }
String prompt = shell.prompt() + "> ";
- while (true) {
+ final int batchSize = lines == null ? 0 : lines.size();
+ if (batchSize > 0) {
+ terminal.echo(true);
+ reader.addCommandsInBuffer(lines);
+ }
+
+ for (int i = 0; batchSize == 0 || i < batchSize; i++) {
try {
registry.cleanUp();
String line = reader.readLine(prompt, null, (MaskingCallback) null,
null);
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Shell.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Shell.java
index 515dcec179..b213dc46aa 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Shell.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Shell.java
@@ -25,6 +25,8 @@
import picocli.CommandLine;
import picocli.shell.jline3.PicocliCommands.PicocliCommandsFactory;
+import java.util.List;
+
/**
* Ozone user interface commands.
* <p>
@@ -56,8 +58,16 @@ public abstract class Shell extends GenericCli {
@CommandLine.Spec
private CommandLine.Model.CommandSpec spec;
- @CommandLine.Option(names = { "--interactive" }, description = "Run in
interactive mode")
- private boolean interactive;
+ @CommandLine.ArgGroup
+ private ExecutionMode executionMode;
+
+ private static class ExecutionMode {
+ @CommandLine.Option(names = {"--interactive"}, description = "Run in
interactive mode")
+ private boolean interactive;
+
+ @CommandLine.Option(names = {"--execute"}, description = "Run command as
part of batch")
+ private List<String> command;
+ }
public Shell() {
super(new PicocliCommandsFactory());
@@ -83,9 +93,10 @@ public void run(String[] argv) {
// failure will be reported by regular, non-interactive run
}
- if (interactive) {
+ if (executionMode != null && (executionMode.interactive ||
!executionMode.command.isEmpty())) {
spec.name(""); // use short name (e.g. "token get" instead of "ozone sh
token get")
- new REPL(this, getCmd(), (PicocliCommandsFactory) getCmd().getFactory());
+ installBatchExceptionHandler();
+ new REPL(this, getCmd(), (PicocliCommandsFactory) getCmd().getFactory(),
executionMode.command);
} else {
TracingUtil.initTracing("shell", getOzoneConf());
String spanName = spec.name() + " " + String.join(" ", argv);
@@ -93,6 +104,20 @@ public void run(String[] argv) {
}
}
+ private void installBatchExceptionHandler() {
+ // exit on first error in batch mode
+ if (!executionMode.interactive) {
+ CommandLine.IExecutionExceptionHandler handler =
getCmd().getExecutionExceptionHandler();
+ getCmd().setExecutionExceptionHandler((ex, cli, parseResult) -> {
+ try {
+ return handler.handleExecutionException(ex, cli, parseResult);
+ } finally {
+ System.exit(EXECUTION_ERROR_EXIT_CODE);
+ }
+ });
+ }
+ }
+
@Override
public void printError(Throwable errorArg) {
OMException omException = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
