This is an automated email from the ASF dual-hosted git repository.
amagyar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/knox.git
The following commit(s) were added to refs/heads/master by this push:
new bb5d265d8 KNOX-3002 - KnoxCLI command for generating descriptor for a
role type from a list of hosts (#835)
bb5d265d8 is described below
commit bb5d265d861489925f158faff761090d672205db
Author: Attila Magyar <[email protected]>
AuthorDate: Mon Feb 26 10:37:00 2024 +0100
KNOX-3002 - KnoxCLI command for generating descriptor for a role type from
a list of hosts (#835)
---
.../knox/gateway/util/DescriptorGenerator.java | 84 ++++++++++++++++++++
.../java/org/apache/knox/gateway/util/KnoxCLI.java | 91 +++++++++++++++++++++-
.../org/apache/knox/gateway/util/ServiceUrls.java | 57 ++++++++++++++
.../knox/gateway/util/DescriptorGeneratorTest.java | 81 +++++++++++++++++++
4 files changed, 309 insertions(+), 4 deletions(-)
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/util/DescriptorGenerator.java
b/gateway-server/src/main/java/org/apache/knox/gateway/util/DescriptorGenerator.java
new file mode 100644
index 000000000..5f31f11b2
--- /dev/null
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/util/DescriptorGenerator.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.knox.gateway.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.knox.gateway.model.DescriptorConfiguration;
+import org.apache.knox.gateway.model.Topology;
+
+public class DescriptorGenerator {
+ private static final ObjectMapper mapper = new ObjectMapper();
+ private final String descriptorName;
+ private final String providerName;
+ private final String serviceName;
+ private final ServiceUrls serviceUrls;
+ private final Map<String, String> params;
+
+ static {
+ /* skip printing out null fields */
+ mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
+ }
+
+ public DescriptorGenerator(String descriptorName, String providerName,
String serviceName, ServiceUrls serviceUrls, Map<String, String> params) {
+ this.descriptorName = descriptorName;
+ this.providerName = providerName;
+ this.serviceName = serviceName.toUpperCase(Locale.ROOT);
+ this.serviceUrls = serviceUrls;
+ this.params = params;
+ }
+
+ public void saveDescriptor(File outputDir, boolean forceOverwrite) {
+ File outputFile = new File(outputDir, descriptorName);
+ if (outputFile.exists() && !forceOverwrite) {
+ throw new IllegalArgumentException(outputFile + " already exists");
+ }
+ DescriptorConfiguration descriptor = new DescriptorConfiguration();
+ descriptor.setName(FilenameUtils.removeExtension(descriptorName));
+ descriptor.setProviderConfig(FilenameUtils.removeExtension(providerName));
+ Topology.Service service = new Topology.Service();
+ service.setRole(serviceName);
+ service.setUrls(serviceUrls.toList());
+ setParams(service, params);
+ descriptor.setServices(Arrays.asList(service));
+ try {
+ mapper.writerWithDefaultPrettyPrinter()
+ .writeValue(outputFile, descriptor);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ private void setParams(Topology.Service service, Map<String, String> params)
{
+ List<Topology.Param> paramList = new ArrayList<>();
+ for (Map.Entry<String, String> each : params.entrySet()) {
+ paramList.add(new Topology.Param(each.getKey(), each.getValue()));
+ }
+ service.setParams(paramList);
+ }
+}
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/util/KnoxCLI.java
b/gateway-server/src/main/java/org/apache/knox/gateway/util/KnoxCLI.java
index 71fde73fa..4ce440f4a 100644
--- a/gateway-server/src/main/java/org/apache/knox/gateway/util/KnoxCLI.java
+++ b/gateway-server/src/main/java/org/apache/knox/gateway/util/KnoxCLI.java
@@ -43,6 +43,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import java.util.TreeMap;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -134,6 +135,7 @@ public class KnoxCLI extends Configured implements Tool {
" [" + RemoteRegistryGetACLCommand.USAGE + "]\n" +
" [" + TopologyConverter.USAGE + "]\n" +
" [" + JWKGenerator.USAGE + "]\n" +
+ " [" + GenerateDescriptorCommand.USAGE + "]\n" +
" [" + TokenMigration.USAGE + "]\n";
/** allows stdout to be captured if necessary */
@@ -173,6 +175,9 @@ public class KnoxCLI extends Configured implements Tool {
private String discoveryUser;
private String discoveryPasswordAlias;
private String discoveryType;
+ private String serviceName;
+ private String urlsFilePath;
+ private final Map<String, String> params = new TreeMap<>();
// For testing only
private String master;
@@ -388,11 +393,23 @@ public class KnoxCLI extends Configured implements Tool {
}
this.topologyName = args[++i];
} else if (args[i].equals("--descriptor-name")) {
- if( i+1 >= args.length || args[i+1].startsWith( "-" ) ) {
+ if (i + 1 >= args.length || args[i + 1].startsWith("-")) {
printKnoxShellUsage();
return -1;
}
this.descriptorName = args[++i];
+ } else if (args[i].equals("--service-name")) {
+ if (i + 1 >= args.length || args[i + 1].startsWith("-")) {
+ printKnoxShellUsage();
+ return -1;
+ }
+ this.serviceName = args[++i];
+ } else if (args[i].equals("--service-urls-file")) {
+ if (i + 1 >= args.length || args[i + 1].startsWith("-")) {
+ printKnoxShellUsage();
+ return -1;
+ }
+ this.urlsFilePath = args[++i];
} else if (args[i].equals("--output-dir")) {
if( i+1 >= args.length || args[i+1].startsWith( "-" ) ) {
printKnoxShellUsage();
@@ -519,8 +536,21 @@ public class KnoxCLI extends Configured implements Tool {
} else if (args[i].equalsIgnoreCase("convert-topology")) {
if (args.length >= 5) {
command = new TopologyConverter();
+ } else {
+ printKnoxShellUsage();
+ return -1;
}
- else {
+ } else if (args[i].equalsIgnoreCase("generate-descriptor")) {
+ if (args.length >= 7) {
+ command = new GenerateDescriptorCommand();
+ } else {
+ printKnoxShellUsage();
+ return -1;
+ }
+ } else if (args[i].equalsIgnoreCase("--param")) {
+ if (i + 2 < args.length) {
+ params.put(args[++i], args[++i]);
+ } else {
printKnoxShellUsage();
return -1;
}
@@ -2315,7 +2345,7 @@ public class KnoxCLI extends Configured implements Tool {
public static final String USAGE =
"convert-topology --path \"path/to/topology.xml\" --provider-name
my-provider.json [--descriptor-name my-descriptor.json] "
- + "[--topology-name topologyName] [--output-path
\"path/to/configs/\"] [--force] [--cluster clusterName] [--discovery-url url] "
+ + "[--topology-name topologyName] [--output-dir
\"path/to/configs/\"] [--force] [--cluster clusterName] [--discovery-url url] "
+ "[--discovery-user discoveryUser] [--discovery-pwd-alias
discoveryPasswordAlias] [--discovery-type discoveryType]";
public static final String DESC =
"Convert Knox topology file to provider and descriptor config files \n"
@@ -2325,7 +2355,7 @@ public class KnoxCLI extends Configured implements Tool {
+ "--descriptor-name (optional) name of descriptor json config
file (including .json extension) \n"
+ "--topology-name (optional) topology-name can be use instead of
--path option, if used, KnoxCLI will attempt to find topology from deployed
topologies.\n"
+ "\t if not provided topology name will be used as descriptor
name \n"
- + "--output-dir (optional) output directory to save provider and
descriptor config files \n"
+ + "--output-dir (optional) output directory to save provider and
descriptor config files. Default is the current working directory. \n"
+ "\t if not provided config files will be saved in appropriate
Knox config directory \n"
+ "--force (optional) force rewriting of existing files, if not
used, command will fail when the configs files with same name already exist. \n"
+ "--cluster (optional) cluster name, required for service
discovery \n"
@@ -2409,6 +2439,59 @@ public class KnoxCLI extends Configured implements Tool {
}
+ public class GenerateDescriptorCommand extends Command {
+
+ public static final String USAGE =
+ "generate-descriptor --service-urls-file \"path/to/urls.txt\"
--service-name SERVICE_NAME \n" +
+ "--provider-name my-provider.json --descriptor-name
my-descriptor.json \n" +
+ "[--output-dir /path/to/output_dir] \n" +
+ "[--param key1 value1] \n" +
+ "[--param key2 value2] \n" +
+ "[--force] \n";
+ public static final String DESC =
+ "Create Knox topology descriptor file for one service\n"
+ + "Options are as follows: \n"
+ + "--service-urls-file (required) path to a text file
containing service urls \n"
+ + "--service-name (required) the name of the service, such
as WEBHDFS, IMPALAUI or HIVE \n"
+ + "--descriptor-name (required) name of descriptor to be
created \n"
+ + "--provider-name (required) name of the referenced
shared provider \n"
+ + "--output-dir (optional) output directory to save the
descriptor file \n"
+ + "--param (optional) service param name and value \n"
+ + "--force (optional) force rewriting of existing files,
if not used, command will fail when the configs files with same name already
exist. \n";
+
+ @Override
+ public void execute() throws Exception {
+ validateParams();
+ File output = StringUtils.isBlank(outputDir) ? new File(".") : new
File(outputDir);
+ DescriptorGenerator generator =
+ new DescriptorGenerator(descriptorName, providerName,
serviceName, ServiceUrls.fromFile(urlsFilePath), params);
+ generator.saveDescriptor(output, force);
+ out.println("Descriptor " + descriptorName + " was successfully saved to
" + output.getAbsolutePath() + "\n");
+ }
+
+ private void validateParams() {
+ if (StringUtils.isBlank(FilenameUtils.getExtension(providerName))
+ ||
StringUtils.isBlank(FilenameUtils.getExtension(descriptorName))) {
+ throw new IllegalArgumentException("JSON extension is required for
provider and descriptor file names");
+ }
+ if (StringUtils.isBlank(urlsFilePath) ) {
+ throw new IllegalArgumentException("Missing --service-urls-file");
+ }
+ if (!new File(urlsFilePath).isFile()) {
+ throw new IllegalArgumentException(urlsFilePath + " does not exist");
+ }
+ if (StringUtils.isBlank(serviceName)) {
+ throw new IllegalArgumentException("Missing --service-name");
+ }
+ }
+
+ @Override
+ public String getUsage() {
+ return USAGE + ":\n\n" + DESC;
+ }
+
+ }
+
public class JWKGenerator extends Command {
public static final String USAGE = "generate-jwk [--jwkAlg
HS256|HS384|HS512] [--saveAlias alias] [--topology topology]";
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/util/ServiceUrls.java
b/gateway-server/src/main/java/org/apache/knox/gateway/util/ServiceUrls.java
new file mode 100644
index 000000000..3f244698a
--- /dev/null
+++ b/gateway-server/src/main/java/org/apache/knox/gateway/util/ServiceUrls.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.knox.gateway.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.Charset;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+
+public class ServiceUrls {
+ private final List<String> urls;
+
+ public static ServiceUrls fromFile(String urlsFilePath) {
+ return fromFile(new File(urlsFilePath));
+ }
+
+ public static ServiceUrls fromFile(File urlsFilePath) {
+ try {
+ List<String> lines = FileUtils.readLines(urlsFilePath,
Charset.defaultCharset()).stream()
+ .map(String::trim)
+ .filter(StringUtils::isNotBlank)
+ .collect(Collectors.toList());
+ return new ServiceUrls(lines);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public ServiceUrls(List<String> urls) {
+ this.urls = urls;
+ }
+
+ public List<String> toList() {
+ return Collections.unmodifiableList(urls);
+ }
+}
diff --git
a/gateway-server/src/test/java/org/apache/knox/gateway/util/DescriptorGeneratorTest.java
b/gateway-server/src/test/java/org/apache/knox/gateway/util/DescriptorGeneratorTest.java
new file mode 100644
index 000000000..8beaf938f
--- /dev/null
+++
b/gateway-server/src/test/java/org/apache/knox/gateway/util/DescriptorGeneratorTest.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.knox.gateway.util;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.knox.gateway.model.DescriptorConfiguration;
+import org.apache.knox.gateway.model.Topology;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class DescriptorGeneratorTest {
+ private static final ObjectMapper mapper = new ObjectMapper();
+ private static final String TEST_DESC_1 = "test_desc1.json";
+ private static final String TEST_PROV_1 = "test_prov1.json";
+ private static final String IMPALA_UI = "IMPALAUI";
+ private static final List<String> URLS =
+ Arrays.asList("http://amagyar-1.test.site:25000/";,
"http://amagyar-2.test.site:25000";);
+
+ private static final Map<String,String> PARAMS = new HashMap<>();
+ static { PARAMS.put("KEY_1", "VAL_1"); }
+
+ @Rule
+ public TemporaryFolder folder= new TemporaryFolder();
+
+ @Test
+ public void testCreateDescriptor() throws Exception {
+ DescriptorGenerator generator = new DescriptorGenerator(TEST_DESC_1,
TEST_PROV_1, IMPALA_UI, new ServiceUrls(URLS), PARAMS);
+ File outputDir = folder.newFolder().getAbsoluteFile();
+ File outputFile = new File(outputDir, TEST_DESC_1);
+ generator.saveDescriptor(outputDir, false);
+ System.out.println(FileUtils.readFileToString(outputFile,
Charset.defaultCharset()));
+ DescriptorConfiguration result = mapper
+ .readerFor(DescriptorConfiguration.class)
+ .readValue(outputFile);
+ assertEquals(FilenameUtils.removeExtension(TEST_PROV_1),
result.getProviderConfig());
+ assertEquals(FilenameUtils.removeExtension(TEST_DESC_1), result.getName());
+ assertEquals(1, result.getServices().size());
+ Topology.Service service = result.getServices().get(0);
+ assertEquals(IMPALA_UI, service.getRole());
+ assertEquals(URLS, service.getUrls());
+ assertEquals(1, service.getParams().size());
+ assertEquals("KEY_1", service.getParams().get(0).getName());
+ assertEquals("VAL_1", service.getParams().get(0).getValue());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testOutputAlreadyExists() throws Exception {
+ DescriptorGenerator generator = new DescriptorGenerator(TEST_DESC_1,
TEST_PROV_1, IMPALA_UI, new ServiceUrls(URLS), PARAMS);
+ File outputDir = folder.newFolder().getAbsoluteFile();
+ File outputFile = new File(outputDir, TEST_DESC_1);
+ outputFile.createNewFile();
+ generator.saveDescriptor(outputDir, false);
+ }
+}
\ No newline at end of file
