This is an automated email from the ASF dual-hosted git repository. eolivelli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push: new 57bcc977ce9 PIP-201 : Extensions mechanism for Pulsar Admin CLI tools (#17158) 57bcc977ce9 is described below commit 57bcc977ce956d2bee120d1a88a9f1cb509fe5ec Author: Enrico Olivelli <eolive...@apache.org> AuthorDate: Wed Aug 31 11:20:36 2022 +0200 PIP-201 : Extensions mechanism for Pulsar Admin CLI tools (#17158) --- bin/pulsar-admin-common.sh | 2 +- conf/client.conf | 6 + pom.xml | 4 + .../pom.xml | 58 ++---- .../cli/extensions/CommandExecutionContext.java | 33 +++ .../pulsar/admin/cli/extensions/CustomCommand.java | 56 ++++++ .../admin/cli/extensions/CustomCommandFactory.java | 35 ++++ .../admin/cli/extensions/CustomCommandGroup.java | 47 +++++ .../admin/cli/extensions/ParameterDescriptor.java | 38 ++++ .../pulsar/admin/cli/extensions/ParameterType.java | 26 +++ .../pulsar/admin/cli/extensions/package-info.java | 19 ++ pulsar-client-tools-customcommand-example/pom.xml | 77 +++++++ .../admin/cli/examples/MyCommandFactory.java | 151 ++++++++++++++ .../META-INF/services/command_factory.yml | 19 ++ pulsar-client-tools-test/pom.xml | 26 +++ .../pulsar/admin/cli/PulsarAdminToolTest.java | 127 ++++++++++++ pulsar-client-tools/pom.xml | 10 + .../org/apache/pulsar/admin/cli/CliCommand.java | 2 +- .../pulsar/admin/cli/CmdGenerateDocument.java | 2 +- .../pulsar/admin/cli/CustomCommandsUtils.java | 223 +++++++++++++++++++++ .../apache/pulsar/admin/cli/PulsarAdminTool.java | 63 ++++-- .../cli/utils/CustomCommandFactoryDefinition.java | 37 ++++ .../cli/utils/CustomCommandFactoryDefinitions.java | 28 +++ .../cli/utils/CustomCommandFactoryMetaData.java | 37 ++++ .../cli/utils/CustomCommandFactoryProvider.java | 173 ++++++++++++++++ 25 files changed, 1238 insertions(+), 61 deletions(-) diff --git a/bin/pulsar-admin-common.sh b/bin/pulsar-admin-common.sh index fdfed60beda..7d4b0d861bf 100755 --- a/bin/pulsar-admin-common.sh +++ b/bin/pulsar-admin-common.sh @@ -95,7 +95,7 @@ IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'` # Start --add-opens options # '--add-opens' option is not supported in jdk8 if [[ -z "$IS_JAVA_8" ]]; then - OPTS="$OPTS --add-opens java.base/sun.net=ALL-UNNAMED" + OPTS="$OPTS --add-opens java.base/sun.net=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED" fi OPTS="-cp $PULSAR_CLASSPATH $OPTS" diff --git a/conf/client.conf b/conf/client.conf index 50d9bf374c1..ea1d339a09c 100644 --- a/conf/client.conf +++ b/conf/client.conf @@ -87,3 +87,9 @@ tlsKeyStorePassword= # When TLS authentication with CACert is used, the valid value is either OPENSSL or JDK. # When TLS authentication with KeyStore is used, available options can be SunJSSE, Conscrypt and so on. webserviceTlsProvider= + + + +# Pulsar Admin Custom Commands +#customCommandFactoriesDirectory=commandFactories +#customCommandFactories= diff --git a/pom.xml b/pom.xml index 26273cbf030..e861f9ae0a9 100644 --- a/pom.xml +++ b/pom.xml @@ -2089,7 +2089,9 @@ flexible messaging model and an intuitive client API.</description> <module>pulsar-client-admin-api</module> <module>pulsar-client-admin</module> <module>pulsar-client-admin-shaded</module> + <module>pulsar-client-tools-api</module> <module>pulsar-client-tools</module> + <module>pulsar-client-tools-customcommand-example</module> <module>pulsar-client-tools-test</module> <module>pulsar-client-all</module> <module>pulsar-websocket</module> @@ -2152,7 +2154,9 @@ flexible messaging model and an intuitive client API.</description> <module>pulsar-client</module> <module>pulsar-client-admin-api</module> <module>pulsar-client-admin</module> + <module>pulsar-client-tools-api</module> <module>pulsar-client-tools</module> + <module>pulsar-client-tools-customcommand-example</module> <module>pulsar-client-tools-test</module> <module>pulsar-websocket</module> <module>pulsar-proxy</module> diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-api/pom.xml similarity index 60% copy from pulsar-client-tools-test/pom.xml copy to pulsar-client-tools-api/pom.xml index 7d06e42b05f..302f184e9c2 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-api/pom.xml @@ -19,7 +19,7 @@ --> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.pulsar</groupId> @@ -28,43 +28,27 @@ <relativePath>..</relativePath> </parent> - <artifactId>pulsar-client-tools-test</artifactId> - <name>Pulsar Client Tools Test</name> - <description>Pulsar Client Tools Test</description> + <artifactId>pulsar-client-tools-api</artifactId> + <name>Pulsar Client Tools API</name> + <description>Pulsar Client Tools API</description> <dependencies> <dependency> <groupId>${project.groupId}</groupId> - <artifactId>pulsar-client-tools</artifactId> + <artifactId>pulsar-client-admin-api</artifactId> <version>${project.version}</version> </dependency> - <dependency> - <groupId>${project.groupId}</groupId> - <artifactId>testmocks</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.awaitility</groupId> - <artifactId>awaitility</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>${project.groupId}</groupId> - <artifactId>pulsar-broker</artifactId> - <version>${project.version}</version> - <type>test-jar</type> - <scope>test</scope> - </dependency> - <dependency> - <groupId>${project.groupId}</groupId> - <artifactId>pulsar-broker</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> </dependencies> <build> <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <!-- same version of Pulsar Admin API --> + <release>${pulsar.client.compiler.release}</release> + </configuration> + </plugin> <plugin> <groupId>org.gaul</groupId> <artifactId>modernizer-maven-plugin</artifactId> @@ -85,21 +69,10 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-deploy-plugin</artifactId> - <configuration> - <skip>true</skip> - </configuration> - </plugin> - <plugin> - <groupId>com.github.spotbugs</groupId> - <artifactId>spotbugs-maven-plugin</artifactId> - <version>${spotbugs-maven-plugin.version}</version> - <configuration> - <excludeFilterFile>${basedir}/src/test/resources/findbugsExclude.xml</excludeFilterFile> - </configuration> + <artifactId>maven-checkstyle-plugin</artifactId> <executions> <execution> - <id>spotbugs</id> + <id>checkstyle</id> <phase>verify</phase> <goals> <goal>check</goal> @@ -109,4 +82,5 @@ </plugin> </plugins> </build> + </project> diff --git a/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/CommandExecutionContext.java b/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/CommandExecutionContext.java new file mode 100644 index 00000000000..47f28479ab1 --- /dev/null +++ b/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/CommandExecutionContext.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.extensions; + +import java.util.Properties; +import org.apache.pulsar.client.admin.PulsarAdmin; + +/** + * Access to the Environment. + */ +public interface CommandExecutionContext { + + PulsarAdmin getPulsarAdmin(); + + Properties getConfiguration(); + +} diff --git a/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/CustomCommand.java b/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/CustomCommand.java new file mode 100644 index 00000000000..bae1698eed6 --- /dev/null +++ b/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/CustomCommand.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.extensions; + +import java.util.List; +import java.util.Map; + +/** + * Custom command. + */ +public interface CustomCommand { + + /** + * Name of the command. + * @return the name + */ + String name(); + + /** + * Descritption of the command. + * @return the description + */ + String description(); + + /** + * The parameters for the command. + * @return the parameters + */ + List<ParameterDescriptor> parameters(); + + /** + * Execute the command. + * @param parameters the parameters, one entry per each parameter name + * @param context access the environment + * @return false in case of failure + * @throws Exception + */ + boolean execute(Map<String, Object> parameters, CommandExecutionContext context) throws Exception; + +} diff --git a/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/CustomCommandFactory.java b/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/CustomCommandFactory.java new file mode 100644 index 00000000000..d4412703d0f --- /dev/null +++ b/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/CustomCommandFactory.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.extensions; + +import java.util.List; + +/** + * Entry point to build custom commands for the Pulsar Admin CLI tools. + */ +public interface CustomCommandFactory { + + /** + * Generate the available command groups. + * + * @param context + * @return the list of new commands groups. + */ + List<CustomCommandGroup> commandGroups(CommandExecutionContext context); +} diff --git a/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/CustomCommandGroup.java b/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/CustomCommandGroup.java new file mode 100644 index 00000000000..b20f5c6895c --- /dev/null +++ b/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/CustomCommandGroup.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.extensions; + +import java.util.List; + +/** + * A group of commands. + */ +public interface CustomCommandGroup { + + /** + * The name of the group. + * @return the name + */ + String name(); + + /** + * The description of the group. + * @return the description + */ + String description(); + + /** + * Generate the available commands. + * + * @param context + * @return the list of new commands. + */ + List<CustomCommand> commands(CommandExecutionContext context); +} diff --git a/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/ParameterDescriptor.java b/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/ParameterDescriptor.java new file mode 100644 index 00000000000..21f1074f864 --- /dev/null +++ b/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/ParameterDescriptor.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.extensions; + +import java.util.ArrayList; +import java.util.List; +import lombok.Builder; +import lombok.Getter; + +@Builder +@Getter +public final class ParameterDescriptor { + @Builder.Default + private List<String> names = new ArrayList<>(); + private boolean mainParameter; + @Builder.Default + private String description = ""; + @Builder.Default + private ParameterType type = ParameterType.STRING; + @Builder.Default + private boolean required = false; +} diff --git a/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/ParameterType.java b/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/ParameterType.java new file mode 100644 index 00000000000..a4f491cf492 --- /dev/null +++ b/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/ParameterType.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.extensions; + +public enum ParameterType { + STRING, + INTEGER, + BOOLEAN, + BOOLEAN_FLAG +} diff --git a/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/package-info.java b/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/package-info.java new file mode 100644 index 00000000000..dbefcce7516 --- /dev/null +++ b/pulsar-client-tools-api/src/main/java/org/apache/pulsar/admin/cli/extensions/package-info.java @@ -0,0 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.extensions; diff --git a/pulsar-client-tools-customcommand-example/pom.xml b/pulsar-client-tools-customcommand-example/pom.xml new file mode 100644 index 00000000000..433eaf75ecd --- /dev/null +++ b/pulsar-client-tools-customcommand-example/pom.xml @@ -0,0 +1,77 @@ +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <parent> + <groupId>org.apache.pulsar</groupId> + <artifactId>pulsar</artifactId> + <version>2.11.0-SNAPSHOT</version> + <relativePath>..</relativePath> + </parent> + <modelVersion>4.0.0</modelVersion> + <artifactId>pulsar-client-tools-customcommand-example</artifactId> + <packaging>jar</packaging> + <name>Pulsar CLI Custom command example</name> + <dependencies> + <dependency> + <groupId>org.apache.pulsar</groupId> + <artifactId>pulsar-client-tools-api</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-nar-maven-plugin</artifactId> + <version>1.3.2</version> + <extensions>true</extensions> + <configuration> + <finalName>customCommands</finalName> + <classifier>nar</classifier> + </configuration> + <executions> + <execution> + <id>default-nar</id> + <phase>package</phase> + <goals> + <goal>nar</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-deploy-plugin</artifactId> + <configuration> + <skip>true</skip> + </configuration> + </plugin> + <plugin> + <groupId>org.sonatype.plugins</groupId> + <artifactId>nexus-staging-maven-plugin</artifactId> + <configuration> + <skipNexusStagingDeployMojo>true</skipNexusStagingDeployMojo> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/pulsar-client-tools-customcommand-example/src/main/java/org/apache/pulsar/admin/cli/examples/MyCommandFactory.java b/pulsar-client-tools-customcommand-example/src/main/java/org/apache/pulsar/admin/cli/examples/MyCommandFactory.java new file mode 100644 index 00000000000..e431d002465 --- /dev/null +++ b/pulsar-client-tools-customcommand-example/src/main/java/org/apache/pulsar/admin/cli/examples/MyCommandFactory.java @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.examples; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.admin.cli.extensions.CommandExecutionContext; +import org.apache.pulsar.admin.cli.extensions.CustomCommand; +import org.apache.pulsar.admin.cli.extensions.CustomCommandFactory; +import org.apache.pulsar.admin.cli.extensions.CustomCommandGroup; +import org.apache.pulsar.admin.cli.extensions.ParameterDescriptor; +import org.apache.pulsar.admin.cli.extensions.ParameterType; +import org.apache.pulsar.common.policies.data.TopicStats; + +@Slf4j +public class MyCommandFactory implements CustomCommandFactory { + @Override + public List<CustomCommandGroup> commandGroups(CommandExecutionContext context) { + return Arrays.asList( + new MyCustomCommandGroup()); + } + + private static class MyCustomCommandGroup implements CustomCommandGroup { + @Override + public String name() { + return "customgroup"; + } + + @Override + public String description() { + return "Custom group 1 description"; + } + + @Override + public List<CustomCommand> commands(CommandExecutionContext context) { + return Arrays.asList(new Command1(), new Command2()); + } + + private static class Command1 implements CustomCommand { + @Override + public String name() { + return "command1"; + } + + @Override + public String description() { + return "Command 1 description"; + } + + @Override + public List<ParameterDescriptor> parameters() { + return Arrays.asList( + ParameterDescriptor.builder() + .description("Operation type") + .type(ParameterType.STRING) + .names(Arrays.asList("--type", "-t")) + .required(true) + .build(), + ParameterDescriptor.builder() + .description("Topic") + .type(ParameterType.STRING) + .mainParameter(true) + .names(Arrays.asList("topic")) + .required(true) + .build()); + } + + @Override + public boolean execute( + Map<String, Object> parameters, CommandExecutionContext context) + throws Exception { + System.out.println( + "Execute: " + parameters + " properties " + context.getConfiguration()); + String destination = parameters.getOrDefault("topic", "").toString(); + TopicStats stats = context.getPulsarAdmin().topics().getStats(destination); + System.out.println("Topic stats: " + stats); + return false; + } + } + + private static class Command2 implements CustomCommand { + @Override + public String name() { + return "command2"; + } + + @Override + public String description() { + return "Command 2 description"; + } + + @Override + public List<ParameterDescriptor> parameters() { + return Arrays.asList( + ParameterDescriptor.builder() + .description("mystring") + .type(ParameterType.STRING) + .names(Arrays.asList("-s")) + .build(), + ParameterDescriptor.builder() + .description("myint") + .type(ParameterType.INTEGER) + .names(Arrays.asList("-i")) + .build(), + ParameterDescriptor.builder() + .description("myboolean") + .type(ParameterType.BOOLEAN) + .names(Arrays.asList("-b")) + .build(), + ParameterDescriptor.builder() + .description("mybooleanflag") + .type(ParameterType.BOOLEAN_FLAG) + .names(Arrays.asList("-bf")) + .build(), + ParameterDescriptor.builder() + .description("main") + .type(ParameterType.STRING) + .mainParameter(true) + .names(Arrays.asList("main")) + .build()); + } + + @Override + public boolean execute( + Map<String, Object> parameters, CommandExecutionContext context) + throws Exception { + System.out.println( + "Execute: " + parameters + " properties " + context.getConfiguration()); + return false; + } + } + } +} diff --git a/pulsar-client-tools-customcommand-example/src/main/resources/META-INF/services/command_factory.yml b/pulsar-client-tools-customcommand-example/src/main/resources/META-INF/services/command_factory.yml new file mode 100644 index 00000000000..e6007cb4b09 --- /dev/null +++ b/pulsar-client-tools-customcommand-example/src/main/resources/META-INF/services/command_factory.yml @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +factoryClass: org.apache.pulsar.admin.cli.examples.MyCommandFactory +name: dummy +description: Example of a Custom command factory diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml index 7d06e42b05f..d70184af9b6 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-test/pom.xml @@ -62,6 +62,13 @@ <version>${project.version}</version> <scope>test</scope> </dependency> + <!-- add the dependency in order to let maven build the module before this module --> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>pulsar-client-tools-customcommand-example</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> </dependencies> <build> <plugins> @@ -107,6 +114,25 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <phase>process-test-resources</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <echo>copy filters</echo> + <mkdir dir="${project.build.outputDirectory}/cliExtensions" /> + <copy verbose="true" file="${basedir}/../pulsar-client-tools-customcommand-example/target/customCommands-nar.nar" tofile="${project.build.outputDirectory}/cliExtensions/customCommands-nar.nar" /> + </target> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> </project> diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index 7ca09a2b14d..deda7c41d1b 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.admin.cli; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.longThat; import static org.mockito.Mockito.doReturn; @@ -27,14 +28,19 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; import com.google.common.collect.Sets; + +import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.PrintStream; import java.lang.reflect.Field; import java.net.URL; import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; @@ -44,6 +50,8 @@ import java.util.Optional; import java.util.Properties; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; + +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.admin.cli.utils.SchemaExtractor; import org.apache.pulsar.client.admin.Bookies; import org.apache.pulsar.client.admin.BrokerStats; @@ -57,6 +65,7 @@ import org.apache.pulsar.client.admin.Namespaces; import org.apache.pulsar.client.admin.NonPersistentTopics; import org.apache.pulsar.client.admin.ProxyStats; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminBuilder; import org.apache.pulsar.client.admin.ResourceQuotas; import org.apache.pulsar.client.admin.Schemas; import org.apache.pulsar.client.admin.Tenants; @@ -100,6 +109,7 @@ import org.apache.pulsar.common.policies.data.ResourceQuota; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.protocol.schema.PostSchemaPayload; import org.apache.pulsar.common.util.ObjectMapperFactory; @@ -107,6 +117,7 @@ import org.mockito.ArgumentMatcher; import org.mockito.Mockito; import org.testng.annotations.Test; +@Slf4j public class PulsarAdminToolTest { @Test @@ -2255,6 +2266,122 @@ public class PulsarAdminToolTest { verify(schemas).createSchema("persistent://tn1/ns1/tp1", postSchemaPayload); } + @Test + public void customCommands() throws Exception { + + // see the custom command help in the main help + String logs = runCustomCommand(new String[]{"-h"}); + assertTrue(logs.contains("customgroup")); + assertTrue(logs.contains("Custom group 1 description")); + + logs = runCustomCommand(new String[]{"customgroup"}); + assertTrue(logs.contains("command1")); + assertTrue(logs.contains("Command 1 description")); + assertTrue(logs.contains("command2")); + assertTrue(logs.contains("Command 2 description")); + + logs = runCustomCommand(new String[]{"customgroup", "command1"}); + assertTrue(logs.contains("Command 1 description")); + assertTrue(logs.contains("Usage: command1 [options] Topic")); + + // missing required parameter + logs = runCustomCommand(new String[]{"customgroup", "command1", "mytopic"}); + assertTrue(logs.contains("Command 1 description")); + assertTrue(logs.contains("Usage: command1 [options] Topic")); + assertTrue(logs.contains("The following option is required")); + + // run a comand that uses PulsarAdmin API + logs = runCustomCommand(new String[]{"customgroup", "command1", "--type", "stats", "mytopic"}); + assertTrue(logs.contains("Execute:")); + // parameters + assertTrue(logs.contains("--type=stats")); + // configuration + assertTrue(logs.contains("webServiceUrl=http://localhost:2181")); + // execution of the PulsarAdmin command + assertTrue(logs.contains("Topic stats: MOCK-TOPIC-STATS")); + + + // run a command that uses all parameter types + logs = runCustomCommand(new String[]{"customgroup", "command2", + "-s", "mystring", + "-i", "123", + "-b", "true", // boolean variable, true|false + "-bf", // boolean flag, no arguments + "mainParameterValue"}); + assertTrue(logs.contains("Execute:")); + // parameters + assertTrue(logs.contains("-s=mystring")); + assertTrue(logs.contains("-i=123")); + assertTrue(logs.contains("-b=true")); + assertTrue(logs.contains("-bf=true")); // boolean flag, passed = true + assertTrue(logs.contains("main=mainParameterValue")); + + + // run a command that uses all parameter types, see the default value + logs = runCustomCommand(new String[]{"customgroup", "command2"}); + assertTrue(logs.contains("Execute:")); + // parameters + assertTrue(logs.contains("-s=null")); + assertTrue(logs.contains("-i=0")); + assertTrue(logs.contains("-b=null")); + assertTrue(logs.contains("-bf=false")); // boolean flag, not passed = false + assertTrue(logs.contains("main=null")); + + } + + private static String runCustomCommand(String[] args) throws Exception { + File narFile = new File(PulsarAdminTool.class.getClassLoader() + .getResource("cliExtensions/customCommands-nar.nar").getFile()); + log.info("NAR FILE is {}", narFile); + + PulsarAdminBuilder builder = mock(PulsarAdminBuilder.class); + PulsarAdmin admin = mock(PulsarAdmin.class); + when(builder.build()).thenReturn(admin); + Topics topics = mock(Topics.class); + when(admin.topics()).thenReturn(topics); + TopicStats topicStats = mock(TopicStats.class); + when(topics.getStats(anyString())).thenReturn(topicStats); + when(topicStats.toString()).thenReturn("MOCK-TOPIC-STATS"); + + Properties properties = new Properties(); + properties.put("webServiceUrl", "http://localhost:2181"); + properties.put("cliExtensionsDirectory", narFile.getParentFile().getAbsolutePath()); + properties.put("customCommandFactories", "dummy"); + PulsarAdminTool tool = new PulsarAdminTool(properties) { + @Override + protected PulsarAdminBuilder createAdminBuilder(Properties properties) { + return builder; + } + }; + + // see the custom command help in the main help + StringBuilder logs = new StringBuilder(); + try (CaptureStdOut capture = new CaptureStdOut(logs)){ + tool.run(args); + } + log.info("Captured out: {}", logs); + return logs.toString(); + } + + private static class CaptureStdOut implements AutoCloseable { + final PrintStream currentOut = System.out; + final PrintStream currentErr = System.err; + final ByteArrayOutputStream logs = new ByteArrayOutputStream(); + final PrintStream capturedOut = new PrintStream(logs, true); + final StringBuilder receiver; + public CaptureStdOut(StringBuilder receiver) { + this.receiver = receiver; + System.setOut(capturedOut); + System.setErr(capturedOut); + } + public void close() { + capturedOut.flush(); + System.setOut(currentOut); + System.setErr(currentErr); + receiver.append(logs.toString(StandardCharsets.UTF_8)); + } + } + public static class SchemaDemo { public SchemaDemo() { } diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index 4f16d8ebf84..948e420ee18 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -43,6 +43,11 @@ <artifactId>pulsar-client-admin-api</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>pulsar-client-tools-api</artifactId> + <version>${project.version}</version> + </dependency> <dependency> <groupId>${project.groupId}</groupId> <artifactId>pulsar-client-admin-original</artifactId> @@ -74,6 +79,11 @@ <groupId>org.conscrypt</groupId> <artifactId>conscrypt-openjdk-uber</artifactId> </dependency> + <dependency> + <!-- custom commands --> + <groupId>org.javassist</groupId> + <artifactId>javassist</artifactId> + </dependency> <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java index a609025faa5..1a2136ee967 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java @@ -36,7 +36,7 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.AuthAction; import org.apache.pulsar.common.util.ObjectMapperFactory; -abstract class CliCommand { +public abstract class CliCommand { static String[] validatePropertyCluster(List<String> params) { return splitParameter(params, 2); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdGenerateDocument.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdGenerateDocument.java index 70878f8ef53..c8cc58a83ee 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdGenerateDocument.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdGenerateDocument.java @@ -58,7 +58,7 @@ public class CmdGenerateDocument extends CmdBase { } for (Map.Entry<String, Class<?>> c : tool.commandMap.entrySet()) { try { - if (!c.getKey().equals("documents")) { + if (!c.getKey().equals("documents") && c.getValue() != null) { baseJcommander.addCommand( c.getKey(), c.getValue().getConstructor(Supplier.class).newInstance(admin)); } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CustomCommandsUtils.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CustomCommandsUtils.java new file mode 100644 index 00000000000..9063d0d1d0c --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CustomCommandsUtils.java @@ -0,0 +1,223 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import javassist.ClassPool; +import javassist.CtClass; +import javassist.CtConstructor; +import javassist.CtField; +import javassist.CtNewConstructor; +import javassist.Modifier; +import javassist.bytecode.AnnotationsAttribute; +import javassist.bytecode.ClassFile; +import javassist.bytecode.ConstPool; +import javassist.bytecode.annotation.Annotation; +import javassist.bytecode.annotation.ArrayMemberValue; +import javassist.bytecode.annotation.BooleanMemberValue; +import javassist.bytecode.annotation.IntegerMemberValue; +import javassist.bytecode.annotation.MemberValue; +import javassist.bytecode.annotation.StringMemberValue; +import lombok.Setter; +import org.apache.pulsar.admin.cli.extensions.CommandExecutionContext; +import org.apache.pulsar.admin.cli.extensions.CustomCommand; +import org.apache.pulsar.admin.cli.extensions.CustomCommandGroup; +import org.apache.pulsar.admin.cli.extensions.ParameterDescriptor; +import org.apache.pulsar.admin.cli.extensions.ParameterType; +import org.apache.pulsar.client.admin.PulsarAdmin; + +public final class CustomCommandsUtils { + private CustomCommandsUtils() { + } + + public static Object generateCliCommand(CustomCommandGroup group, CommandExecutionContext context, + Supplier<PulsarAdmin> pulsarAdmin){ + List<CustomCommand> commands = group.commands(context); + String description = group.description(); + + try { + ClassPool pool = ClassPool.getDefault(); + CtClass ctClass = pool.makeClass("CustomCommandGroup" + group + + "_" + System.nanoTime()); + ctClass.setSuperclass(pool.get(CmdBaseAdapter.class.getName())); + + // add class annotation + ClassFile classFile = ctClass.getClassFile(); + ConstPool constpool = classFile.getConstPool(); + AnnotationsAttribute annotationsAttribute = new AnnotationsAttribute(constpool, + AnnotationsAttribute.visibleTag); + Annotation annotation = new Annotation(Parameters.class.getName(), constpool); + annotation.addMemberValue("commandDescription", new StringMemberValue(description, + classFile.getConstPool())); + annotationsAttribute.setAnnotation(annotation); + ctClass.getClassFile().addAttribute(annotationsAttribute); + + // Add a constructor which calls super( ... ); + CtClass[] params = new CtClass[]{ + pool.get(String.class.getName()), + pool.get(Supplier.class.getName()), + pool.get(List.class.getName()), + pool.get(CommandExecutionContext.class.getName()) + }; + final CtConstructor ctor = CtNewConstructor.make(params, null, CtNewConstructor.PASS_PARAMS, + null, null, ctClass); + ctClass.addConstructor(ctor); + + return ctClass.toClass().getConstructor(String.class, Supplier.class, List.class, + CommandExecutionContext.class) + .newInstance(group.name(), pulsarAdmin, commands, context); + } catch (Throwable t) { + throw new RuntimeException(t); + } + } + + public static class CmdBaseAdapter extends CmdBase { + public CmdBaseAdapter(String cmdName, Supplier<PulsarAdmin> adminSupplier, + List<CustomCommand> customCommands, CommandExecutionContext context) { + super(cmdName, adminSupplier); + for (CustomCommand command : customCommands) { + String name = command.name(); + DecoratedCommand commandImpl = generateCustomCommand(cmdName, name, command); + commandImpl.setCommand(command); + commandImpl.setContext(context); + jcommander.addCommand(name, commandImpl); + } + } + } + + + @Setter + public static class DecoratedCommand extends CliCommand { + + private CustomCommand command; + private CommandExecutionContext context; + + public DecoratedCommand() { + } + + @Override + public void run() throws Exception { + Map<String, Object> parameters = new HashMap<>(); + for (Field f : this.getClass().getFields()) { + parameters.put(f.getName(), f.get(this)); + } + command.execute(parameters, context); + } + } + + private static DecoratedCommand generateCustomCommand(String group, String name, CustomCommand command) { + try { + String description = command.description(); + ClassPool pool = ClassPool.getDefault(); + CtClass ctClass = pool.makeClass("CustomCommand" + group + + "_" + name + "_" + System.nanoTime()); + ctClass.setSuperclass(pool.get(DecoratedCommand.class.getName())); + + // add class annotation + + ClassFile classFile = ctClass.getClassFile(); + ConstPool constpool = classFile.getConstPool(); + + AnnotationsAttribute annotationsAttribute = new AnnotationsAttribute(constpool, + AnnotationsAttribute.visibleTag); + Annotation annotation = new Annotation(Parameters.class.getName(), constpool); + annotation.addMemberValue("commandDescription", + new StringMemberValue(description, classFile.getConstPool())); + annotationsAttribute.setAnnotation(annotation); + ctClass.getClassFile().addAttribute(annotationsAttribute); + + + // add fields + List<ParameterDescriptor> parameters = command.parameters(); + for (ParameterDescriptor parameterDescriptor : parameters) { + CtClass fieldType; + switch (parameterDescriptor.getType()) { + case BOOLEAN_FLAG: + // command -parameter + fieldType = CtClass.booleanType; + break; + case BOOLEAN: + // command -parameter true|false + fieldType = pool.get(Boolean.class.getName()); + break; + case INTEGER: + // command -parameter 123 + fieldType = CtClass.intType; + break; + case STRING: + // command -parameter foo + fieldType = pool.get(String.class.getName()); + break; + default: + throw new IllegalStateException(); + } + List<String> parameterNames = parameterDescriptor.getNames(); + if (parameterNames == null || parameterNames.isEmpty()) { + // ignore + continue; + } + String fieldName = parameterNames.get(0); + CtField field = new CtField(fieldType, fieldName, ctClass); + + AnnotationsAttribute fieldAnnotationsAttribute = new AnnotationsAttribute(constpool, + AnnotationsAttribute.visibleTag); + Annotation fieldAnnotation = new Annotation(Parameter.class.getName(), constpool); + + // in JCommander if you don't set the "names" property then you want to get all the other + // parameters + if (!parameterDescriptor.isMainParameter()) { + MemberValue[] memberValues = new MemberValue[parameterNames.size()]; + int i = 0; + for (String parameterName : parameterNames) { + memberValues[i++] = new StringMemberValue(parameterName, classFile.getConstPool()); + } + ArrayMemberValue arrayMemberValue = new ArrayMemberValue(classFile.getConstPool()); + arrayMemberValue.setValue(memberValues); + fieldAnnotation.addMemberValue("names", arrayMemberValue); + } + + fieldAnnotation.addMemberValue("description", + new StringMemberValue(parameterDescriptor.getDescription(), classFile.getConstPool())); + fieldAnnotation.addMemberValue("required", + new BooleanMemberValue(parameterDescriptor.isRequired(), classFile.getConstPool())); + if (parameterDescriptor.getType() == ParameterType.BOOLEAN) { + fieldAnnotation.addMemberValue("arity", + new IntegerMemberValue(classFile.getConstPool(), 1)); + } + fieldAnnotationsAttribute.setAnnotation(fieldAnnotation); + field.getFieldInfo().addAttribute(fieldAnnotationsAttribute); + field.setModifiers(Modifier.PUBLIC); + + ctClass.addField(field); + } + + + return (DecoratedCommand) ctClass.toClass().getConstructor().newInstance(); + } catch (Throwable t) { + t.printStackTrace(System.out); + throw new RuntimeException(t); + } + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java index b4a0e04439f..16c9d58efe1 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java @@ -25,8 +25,10 @@ import com.beust.jcommander.Parameter; import com.google.common.annotations.VisibleForTesting; import java.io.FileInputStream; import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.TimeUnit; @@ -34,6 +36,10 @@ import java.util.function.Function; import java.util.function.Supplier; import lombok.Getter; import org.apache.pulsar.PulsarVersion; +import org.apache.pulsar.admin.cli.extensions.CommandExecutionContext; +import org.apache.pulsar.admin.cli.extensions.CustomCommandFactory; +import org.apache.pulsar.admin.cli.extensions.CustomCommandGroup; +import org.apache.pulsar.admin.cli.utils.CustomCommandFactoryProvider; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; import org.apache.pulsar.client.admin.internal.PulsarAdminImpl; @@ -44,6 +50,7 @@ public class PulsarAdminTool { private static int lastExitCode = Integer.MIN_VALUE; + protected List<CustomCommandFactory> customCommandFactories = new ArrayList(); protected Map<String, Class<?>> commandMap; protected JCommander jcommander; protected final PulsarAdminBuilder adminBuilder; @@ -167,21 +174,34 @@ public class PulsarAdminTool { } } - protected void setupCommands(Function<PulsarAdminBuilder, ? extends PulsarAdmin> adminFactory) { + public void setupCommands(Function<PulsarAdminBuilder, ? extends PulsarAdmin> adminFactory) { try { - adminBuilder.serviceHttpUrl(rootParams.serviceUrl); - adminBuilder.authentication(rootParams.authPluginClassName, rootParams.authParams); - adminBuilder.requestTimeout(rootParams.requestTimeout, TimeUnit.SECONDS); - if (isBlank(rootParams.tlsProvider)) { - rootParams.tlsProvider = properties.getProperty("webserviceTlsProvider"); - } - if (isNotBlank(rootParams.tlsProvider)) { - adminBuilder.sslProvider(rootParams.tlsProvider); - } Supplier<PulsarAdmin> admin = new PulsarAdminSupplier(adminBuilder, adminFactory); for (Map.Entry<String, Class<?>> c : commandMap.entrySet()) { addCommand(c, admin); } + + CommandExecutionContext context = new CommandExecutionContext() { + @Override + public PulsarAdmin getPulsarAdmin() { + return admin.get(); + } + + @Override + public Properties getConfiguration() { + return properties; + } + }; + loadCustomCommandFactories(); + + for (CustomCommandFactory factory : customCommandFactories) { + List<CustomCommandGroup> customCommandGroups = factory.commandGroups(context); + for (CustomCommandGroup group : customCommandGroups) { + Object generated = CustomCommandsUtils.generateCliCommand(group, context, admin); + jcommander.addCommand(group.name(), generated); + commandMap.put(group.name(), null); + } + } } catch (Exception e) { Throwable cause; if (e instanceof InvocationTargetException && null != e.getCause()) { @@ -194,6 +214,11 @@ public class PulsarAdminTool { } } + private void loadCustomCommandFactories() throws Exception { + customCommandFactories.addAll(CustomCommandFactoryProvider.createCustomCommandFactories(properties)); + } + + private void addCommand(Map.Entry<String, Class<?>> c, Supplier<PulsarAdmin> admin) throws Exception { // To remain backwards compatibility for "source" and "sink" commands // TODO eventually remove this @@ -215,8 +240,8 @@ public class PulsarAdminTool { } boolean run(String[] args, Function<PulsarAdminBuilder, ? extends PulsarAdmin> adminFactory) { + setupCommands(adminFactory); if (args.length == 0) { - setupCommands(adminFactory); jcommander.usage(); return false; } @@ -230,10 +255,20 @@ public class PulsarAdminTool { try { jcommander.parse(Arrays.copyOfRange(args, 0, Math.min(cmdPos, args.length))); + + //rootParams are populated by jcommander.parse + adminBuilder.serviceHttpUrl(rootParams.serviceUrl); + adminBuilder.authentication(rootParams.authPluginClassName, rootParams.authParams); + adminBuilder.requestTimeout(rootParams.requestTimeout, TimeUnit.SECONDS); + if (isBlank(rootParams.tlsProvider)) { + rootParams.tlsProvider = properties.getProperty("webserviceTlsProvider"); + } + if (isNotBlank(rootParams.tlsProvider)) { + adminBuilder.sslProvider(rootParams.tlsProvider); + } } catch (Exception e) { System.err.println(e.getMessage()); System.err.println(); - setupCommands(adminFactory); jcommander.usage(); return false; } @@ -250,17 +285,14 @@ public class PulsarAdminTool { } if (rootParams.help) { - setupCommands(adminFactory); jcommander.usage(); return true; } if (cmdPos == args.length) { - setupCommands(adminFactory); jcommander.usage(); return false; } else { - setupCommands(adminFactory); String cmd = args[cmdPos]; // To remain backwards compatibility for "source" and "sink" commands @@ -391,7 +423,6 @@ public class PulsarAdminTool { // Automatically generate documents for pulsar-admin commandMap.put("documents", CmdGenerateDocument.class); - // To remain backwards compatibility for "source" and "sink" commands // TODO eventually remove this commandMap.put("source", CmdSources.class); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/CustomCommandFactoryDefinition.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/CustomCommandFactoryDefinition.java new file mode 100644 index 00000000000..f91f4bb6dd4 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/CustomCommandFactoryDefinition.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +public class CustomCommandFactoryDefinition { + + /** + * The name of the command factory. + */ + private String name; + + /** + * The class name for factory. + */ + private String factoryClass; +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/CustomCommandFactoryDefinitions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/CustomCommandFactoryDefinitions.java new file mode 100644 index 00000000000..fdd73c2fc37 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/CustomCommandFactoryDefinitions.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils; + +import java.util.Map; +import java.util.TreeMap; +import lombok.Data; + +@Data +public class CustomCommandFactoryDefinitions { + private final Map<String, CustomCommandFactoryMetaData> factories = new TreeMap<>(); +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/CustomCommandFactoryMetaData.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/CustomCommandFactoryMetaData.java new file mode 100644 index 00000000000..fce33157435 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/CustomCommandFactoryMetaData.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils; + +import java.nio.file.Path; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +public class CustomCommandFactoryMetaData { + /** + * The definition of the entry filter. + */ + private CustomCommandFactoryDefinition definition; + + /** + * The path to the handler package. + */ + private Path archivePath; +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/CustomCommandFactoryProvider.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/CustomCommandFactoryProvider.java new file mode 100644 index 00000000000..34155efd4a5 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/CustomCommandFactoryProvider.java @@ -0,0 +1,173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils; + +import static com.google.common.base.Preconditions.checkArgument; +import com.google.common.annotations.VisibleForTesting; +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.admin.cli.extensions.CustomCommandFactory; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; +import org.apache.pulsar.common.util.ObjectMapperFactory; + +@Slf4j +public class CustomCommandFactoryProvider { + + @VisibleForTesting + static final String COMMAND_FACTORY_ENTRY = "command_factory"; + + /** + * create a Command Factory. + */ + public static List<CustomCommandFactory> createCustomCommandFactories( + Properties conf) throws IOException { + String names = conf.getProperty("customCommandFactories", ""); + List<CustomCommandFactory> result = new ArrayList<>(); + if (names.isEmpty()) { + // early exit + return result; + } + + String directory = conf.getProperty("cliExtensionsDirectory", "cliExtensions"); + String narExtractionDirectory = NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR; + CustomCommandFactoryDefinitions definitions = searchForCustomCommandFactories(directory, + narExtractionDirectory); + for (String name : names.split(",")) { + CustomCommandFactoryMetaData metaData = definitions.getFactories().get(name); + if (null == metaData) { + throw new RuntimeException("No factory is found for name `" + name + + "`. Available names are : " + definitions.getFactories()); + } + CustomCommandFactory factory = load(metaData, narExtractionDirectory); + if (factory != null) { + result.add(factory); + } + log.debug("Successfully loaded command factory for name `{}`", name); + } + return result; + } + + private static CustomCommandFactoryDefinitions searchForCustomCommandFactories(String directory, + String narExtractionDirectory) + throws IOException { + Path path = Paths.get(directory).toAbsolutePath(); + log.debug("Searching for command factories in {}", path); + + CustomCommandFactoryDefinitions customCommandFactoryDefinitions = new CustomCommandFactoryDefinitions(); + if (!path.toFile().exists()) { + log.error("Pulsar command factories directory not found"); + return customCommandFactoryDefinitions; + } + + try (DirectoryStream<Path> stream = Files.newDirectoryStream(path, "*.nar")) { + for (Path archive : stream) { + try { + CustomCommandFactoryDefinition def = + getCustomCommandFactoryDefinition(archive.toString(), narExtractionDirectory); + log.debug("Found command factory from {} : {}", archive, def); + + checkArgument(StringUtils.isNotBlank(def.getName())); + checkArgument(StringUtils.isNotBlank(def.getFactoryClass())); + + CustomCommandFactoryMetaData metadata = new CustomCommandFactoryMetaData(); + metadata.setDefinition(def); + metadata.setArchivePath(archive); + + customCommandFactoryDefinitions.getFactories().put(def.getName(), metadata); + } catch (Throwable t) { + log.warn("Failed to load command factories from {}." + + " It is OK however if you want to use this command factory," + + " please make sure you put the correct NAR" + + " package in the directory.", archive, t); + } + } + } + + return customCommandFactoryDefinitions; + } + + private static CustomCommandFactoryDefinition getCustomCommandFactoryDefinition(String narPath, + String narExtractionDirectory) + throws IOException { + try (NarClassLoader ncl = NarClassLoaderBuilder.builder() + .narFile(new File(narPath)) + .extractionDirectory(narExtractionDirectory) + .build()) { + return getCustomCommandFactoryDefinition(ncl); + } + } + + @VisibleForTesting + static CustomCommandFactoryDefinition getCustomCommandFactoryDefinition(NarClassLoader ncl) throws IOException { + String configStr; + + try { + configStr = ncl.getServiceDefinition(COMMAND_FACTORY_ENTRY + ".yaml"); + } catch (NoSuchFileException e) { + configStr = ncl.getServiceDefinition(COMMAND_FACTORY_ENTRY + ".yml"); + } + + return ObjectMapperFactory.getThreadLocalYaml().readValue( + configStr, CustomCommandFactoryDefinition.class + ); + } + + private static CustomCommandFactory load(CustomCommandFactoryMetaData metadata, + String narExtractionDirectory) + throws IOException { + final File narFile = metadata.getArchivePath().toAbsolutePath().toFile(); + NarClassLoader ncl = NarClassLoaderBuilder.builder() + .narFile(narFile) + .parentClassLoader(CustomCommandFactory.class.getClassLoader()) + .extractionDirectory(narExtractionDirectory) + .build(); + CustomCommandFactoryDefinition def = getCustomCommandFactoryDefinition(ncl); + if (StringUtils.isBlank(def.getFactoryClass())) { + throw new IOException("Command Factory `" + def.getName() + "` does NOT provide a Command Factory" + + " implementation"); + } + + try { + Class commandFactoryClass = ncl.loadClass(def.getFactoryClass()); + Object factory = commandFactoryClass.getDeclaredConstructor().newInstance(); + if (!(factory instanceof CustomCommandFactory)) { + throw new IOException("Class " + def.getFactoryClass() + + " does not implement CustomCommandFactory interface"); + } + return (CustomCommandFactory) factory; + } catch (Exception e) { + if (e instanceof IOException) { + throw (IOException) e; + } + log.error("Failed to load class {}", def.getFactoryClass(), e); + throw new IOException(e); + } + } +}