This is an automated email from the ASF dual-hosted git repository. gnodet pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/camel.git
commit 41ed1d42641fd0246324dccfe56953833f66ba15 Author: Guillaume Nodet <[email protected]> AuthorDate: Sat Jan 25 15:09:33 2020 +0100 [CAMEL-14437] Remove usage of JSonSchemaHelper in the tooling # Conflicts: # tooling/camel-tooling-util/src/main/java/org/apache/camel/tooling/util/JSonSchemaHelper.java --- components/camel-as2/camel-as2-component/pom.xml | 23 - components/camel-salesforce/pom.xml | 26 - components/camel-servicenow/pom.xml | 26 - .../camel/tooling/util/JSonSchemaHelper.java | 122 -- .../apache/camel/tooling/util/PackageHelper.java | 18 +- .../camel/tooling/util/PackageHelperTest.java | 24 +- .../camel/maven/packaging/PrepareCatalogMojo.java | 1478 ++++++++------------ .../maven/packaging/PrepareUserGuideMojo.java | 47 +- .../camel/maven/packaging/ValidateHelper.java | 101 +- 9 files changed, 646 insertions(+), 1219 deletions(-) diff --git a/components/camel-as2/camel-as2-component/pom.xml b/components/camel-as2/camel-as2-component/pom.xml index 7b2cd19..0204c6c 100644 --- a/components/camel-as2/camel-as2-component/pom.xml +++ b/components/camel-as2/camel-as2-component/pom.xml @@ -170,29 +170,6 @@ </executions> </plugin> - <!-- generate components meta-data and validate component includes documentation etc --> - <plugin> - <groupId>org.apache.camel</groupId> - <artifactId>camel-package-maven-plugin</artifactId> - <version>${project.version}</version> - <executions> - <execution> - <id>prepare</id> - <goals> - <goal>prepare-components</goal> - </goals> - <phase>generate-resources</phase> - </execution> - <execution> - <id>validate</id> - <goals> - <goal>validate-components</goal> - </goals> - <phase>prepare-package</phase> - </execution> - </executions> - </plugin> - <!-- add generated source and test source to build --> <plugin> <groupId>org.codehaus.mojo</groupId> diff --git a/components/camel-salesforce/pom.xml b/components/camel-salesforce/pom.xml index ca4648b..af8f92d 100644 --- a/components/camel-salesforce/pom.xml +++ b/components/camel-salesforce/pom.xml @@ -41,30 +41,4 @@ <salesforce.component.root>${project.basedir}</salesforce.component.root> </properties> - <build> - <plugins> - <plugin> - <groupId>org.apache.camel</groupId> - <artifactId>camel-package-maven-plugin</artifactId> - <version>${project.version}</version> - <executions> - <execution> - <id>prepare</id> - <goals> - <goal>prepare-components</goal> - </goals> - <phase>generate-resources</phase> - </execution> - <execution> - <id>validate</id> - <goals> - <goal>validate-components</goal> - </goals> - <phase>prepare-package</phase> - </execution> - </executions> - </plugin> - </plugins> - </build> - </project> diff --git a/components/camel-servicenow/pom.xml b/components/camel-servicenow/pom.xml index 159835e..cbce2f6 100644 --- a/components/camel-servicenow/pom.xml +++ b/components/camel-servicenow/pom.xml @@ -41,30 +41,4 @@ <servicenow.component.root>${project.basedir}</servicenow.component.root> </properties> - <build> - <plugins> - <plugin> - <groupId>org.apache.camel</groupId> - <artifactId>camel-package-maven-plugin</artifactId> - <version>${project.version}</version> - <executions> - <execution> - <id>prepare</id> - <goals> - <goal>prepare-components</goal> - </goals> - <phase>generate-resources</phase> - </execution> - <execution> - <id>validate</id> - <goals> - <goal>validate-components</goal> - </goals> - <phase>prepare-package</phase> - </execution> - </executions> - </plugin> - </plugins> - </build> - </project> diff --git a/tooling/camel-tooling-util/src/main/java/org/apache/camel/tooling/util/JSonSchemaHelper.java b/tooling/camel-tooling-util/src/main/java/org/apache/camel/tooling/util/JSonSchemaHelper.java deleted file mode 100644 index 9d04cc9..0000000 --- a/tooling/camel-tooling-util/src/main/java/org/apache/camel/tooling/util/JSonSchemaHelper.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.camel.tooling.util; - -import java.io.File; -import java.net.URI; -import java.net.URL; -import java.util.ArrayList; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import org.apache.camel.util.json.JsonObject; -import org.apache.camel.util.json.Jsoner; - -/** - * A helper class for <a href="http://json-schema.org/";>JSON schema</a>. - */ -public final class JSonSchemaHelper { - - private JSonSchemaHelper() { - } - - /** - * Parses the json schema to split it into a list or rows, where each row contains key value pairs with the metadata - * - * @param group the group to parse from such as <tt>component</tt>, <tt>componentProperties</tt>, or <tt>properties</tt>. - * @param json the json - * @return a list of all the rows, where each row is a set of key value pairs with metadata - */ - @SuppressWarnings("unchecked") - public static List<Map<String, String>> parseJsonSchema(String group, String json, boolean parseProperties) { - List<Map<String, String>> answer = new ArrayList<>(); - if (json == null) { - return answer; - } - - // convert into a List<Map<String, String>> structure which is expected as output from this parser - try { - JsonObject output = (JsonObject) Jsoner.deserialize(json); - for (String key : output.keySet()) { - Map<?, ?> row = output.getMap(key); - if (key.equals(group)) { - if (parseProperties) { - // flatten each entry in the row with name as they key, and its value as the content (its a map also) - for (Object obj : row.entrySet()) { - Map.Entry<?, ?> entry = (Map.Entry<?, ?>) obj; - Map<String, String> newRow = new LinkedHashMap<>(); - newRow.put("name", entry.getKey().toString()); - - Map<String, String> newData = transformMap((Map<?, ?>) entry.getValue()); - newRow.putAll(newData); - answer.add(newRow); - } - } else { - // flattern each entry in the row as a list of single Map<key, value> elements - Map<?, ?> newData = transformMap(row); - for (Object obj : newData.entrySet()) { - Map.Entry<?, ?> entry = (Map.Entry<?, ?>) obj; - Map<String, String> newRow = new LinkedHashMap<>(); - newRow.put(entry.getKey().toString(), entry.getValue().toString()); - answer.add(newRow); - } - } - } - } - } catch (Exception e) { - // wrap parsing exceptions as runtime - throw new RuntimeException("Cannot parse json", e); - } - - return answer; - } - - private static String escapeJson(String value) { - // need to safe encode \r as \\r so its escaped - // need to safe encode \n as \\n so its escaped - // need to safe encode \t as \\t so its escaped - return value - .replace("\\r", "\\\\r") - .replace("\\n", "\\\\n") - .replace("\\t", "\\\\t"); - } - - private static Map<String, String> transformMap(Map<?, ?> jsonMap) { - Map<String, String> answer = new LinkedHashMap<>(); - - for (Object rowObj : jsonMap.entrySet()) { - Map.Entry<?, ?> rowEntry = (Map.Entry<?, ?>) rowObj; - // if its a list type then its an enum, and we need to parse it as a single line separated with comma - // to be backwards compatible - Object newValue = rowEntry.getValue(); - if (newValue instanceof List) { - List<?> list = (List<?>) newValue; - newValue = list.stream().map(Object::toString) - .collect(Collectors.joining(",")); - } - // ensure value is escaped - String value = escapeJson(newValue.toString()); - answer.put(rowEntry.getKey().toString(), value); - } - - return answer; - } - -} diff --git a/tooling/camel-tooling-util/src/main/java/org/apache/camel/tooling/util/PackageHelper.java b/tooling/camel-tooling-util/src/main/java/org/apache/camel/tooling/util/PackageHelper.java index 19345dd..49c6751 100644 --- a/tooling/camel-tooling-util/src/main/java/org/apache/camel/tooling/util/PackageHelper.java +++ b/tooling/camel-tooling-util/src/main/java/org/apache/camel/tooling/util/PackageHelper.java @@ -18,7 +18,6 @@ package org.apache.camel.tooling.util; import java.io.BufferedReader; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -26,7 +25,6 @@ import java.io.LineNumberReader; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.stream.Stream; @@ -71,7 +69,7 @@ public final class PackageHelper { } public static String loadText(File file) throws IOException { - return loadText(new FileInputStream(file)); + return loadText(file.toPath()); } public static String loadText(Path file) throws IOException { @@ -112,20 +110,20 @@ public final class PackageHelper { return answer; } - public static Set<File> findJsonFiles(File rootDir) { - return findJsonFiles(rootDir, new HashSet<>()); - } - public static Set<File> findJsonFiles(File rootDir, Set<File> files) { findJsonFiles(rootDir.toPath()).forEach(p -> files.add(p.toFile())); return files; } public static Stream<Path> findJsonFiles(Path rootDir) { + return walk(rootDir) + .filter(p -> p.getFileName().toString().endsWith(JSON_SUFIX)); + } + + public static Stream<Path> walk(Path rootDir) { try { if (Files.isDirectory(rootDir)) { - return Files.walk(rootDir) - .filter(p -> p.getFileName().toString().endsWith(JSON_SUFIX)); + return Files.walk(rootDir); } else { return Stream.empty(); } @@ -141,7 +139,7 @@ public final class PackageHelper { public static String asName(Path file) { String name = file.getFileName().toString(); if (name.endsWith(JSON_SUFIX)) { - return name.substring(0, name.length() - 5); + return name.substring(0, name.length() - JSON_SUFIX.length()); } return name; } diff --git a/tooling/camel-tooling-util/src/test/java/org/apache/camel/tooling/util/PackageHelperTest.java b/tooling/camel-tooling-util/src/test/java/org/apache/camel/tooling/util/PackageHelperTest.java index 24c3e3a..f4dd40f 100644 --- a/tooling/camel-tooling-util/src/test/java/org/apache/camel/tooling/util/PackageHelperTest.java +++ b/tooling/camel-tooling-util/src/test/java/org/apache/camel/tooling/util/PackageHelperTest.java @@ -17,6 +17,9 @@ package org.apache.camel.tooling.util; import java.io.File; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -32,23 +35,26 @@ public class PackageHelperTest { @Test public void testFileToString() throws Exception { - assertEquals("dk19i21)@+#(OR\n", PackageHelper.loadText(ResourceUtils.getResourceAsFile("filecontent/a.txt"))); + File file = ResourceUtils.getResourceAsFile("filecontent/a.txt"); + assertEquals("dk19i21)@+#(OR\n", PackageHelper.loadText(file)); } @Test public void testFindJsonFiles() throws Exception { - Set<File> jsons = PackageHelper.findJsonFiles(ResourceUtils.getResourceAsFile("json")); - Map<String, File> jsonFiles = jsons.stream().collect(Collectors.toMap( - file -> file.getName().replace(JSON_SUFIX, ""), file -> file)); - - assertTrue(jsonFiles.containsKey("a"), "Files a.json must be found"); - assertTrue(jsonFiles.containsKey("b"), "Files b.json must be found"); - assertFalse(jsonFiles.containsKey("c"), "File c.txt must not be found"); + Path dir = ResourceUtils.getResourceAsFile("json").toPath(); + List<String> jsonFiles = PackageHelper.findJsonFiles(dir) + .map(PackageHelper::asName) + .collect(Collectors.toList()); + + assertTrue(jsonFiles.contains("a"), "Files a.json must be found"); + assertTrue(jsonFiles.contains("b"), "Files b.json must be found"); + assertFalse(jsonFiles.contains("c"), "File c.txt must not be found"); } @Test public void testGetSchemaKind() throws Exception { - String json = PackageHelper.loadText(ResourceUtils.getResourceAsFile("json/aop.json")); + File file = ResourceUtils.getResourceAsFile("json/aop.json"); + String json = PackageHelper.loadText(file); assertEquals("model", PackageHelper.getSchemaKind(json)); } } diff --git a/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/PrepareCatalogMojo.java b/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/PrepareCatalogMojo.java index 53d0e54..acf27b7 100644 --- a/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/PrepareCatalogMojo.java +++ b/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/PrepareCatalogMojo.java @@ -16,30 +16,40 @@ */ package org.apache.camel.maven.packaging; +import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.FileFilter; -import java.io.FileOutputStream; import java.io.IOException; -import java.nio.charset.Charset; +import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; -import java.util.regex.Matcher; +import java.util.function.BinaryOperator; +import java.util.function.Function; import java.util.regex.Pattern; - +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.camel.tooling.model.BaseModel; +import org.apache.camel.tooling.model.BaseOptionModel; +import org.apache.camel.tooling.model.ComponentModel; +import org.apache.camel.tooling.model.DataFormatModel; +import org.apache.camel.tooling.model.EipModel; +import org.apache.camel.tooling.model.JsonMapper; +import org.apache.camel.tooling.model.LanguageModel; +import org.apache.camel.tooling.model.OtherModel; import org.apache.camel.tooling.util.FileUtil; -import org.apache.camel.tooling.util.JSonSchemaHelper; import org.apache.camel.tooling.util.PackageHelper; -import org.apache.commons.io.FileUtils; +import org.apache.camel.tooling.util.Strings; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugin.MojoFailureException; @@ -50,6 +60,7 @@ import org.apache.maven.project.MavenProject; import org.apache.maven.project.MavenProjectHelper; import org.asciidoctor.Asciidoctor; import org.asciidoctor.OptionsBuilder; +import org.jruby.RubyString; /** * Prepares the camel catalog to include component, data format, and eip @@ -150,6 +161,12 @@ public class PrepareCatalogMojo extends AbstractMojo { protected File baseDir; /** + * The camel-jaxp directory + */ + @Parameter(defaultValue = "${project.build.directory}/../../../core/camel-jaxp") + protected File jaxpDir; + + /** * The directory where the camel-spring XML models are */ @Parameter(defaultValue = "${project.build.directory}/../../../components/camel-spring") @@ -185,6 +202,10 @@ public class PrepareCatalogMojo extends AbstractMojo { @Component private MavenProjectHelper projectHelper; + Collection<Path> allJsonFiles; + Collection<Path> allPropertiesFiles; + Map<Path, BaseModel<?>> allModels; + /** * Execute goal. * @@ -195,305 +216,226 @@ public class PrepareCatalogMojo extends AbstractMojo { */ @Override public void execute() throws MojoExecutionException, MojoFailureException { - executeModel(); - Set<String> components = executeComponents(); - Set<String> dataformats = executeDataFormats(); - Set<String> languages = executeLanguages(); - Set<String> others = executeOthers(); - executeDocuments(components, dataformats, languages, others); - executeArchetypes(); - executeXmlSchemas(); - executeMain(); + try { + allJsonFiles = new TreeSet<>(); + allPropertiesFiles = new TreeSet<>(); + + Stream.concat(list(componentsDir.toPath()), + Stream.of(coreDir.toPath(), baseDir.toPath(), jaxpDir.toPath(), springDir.toPath())) + .filter(dir -> !"target".equals(dir.getFileName().toString())) + .map(this::getComponentPath) + .filter(dir -> Files.isDirectory(dir.resolve("src"))) + .map(p -> p.resolve("target/classes")) + .flatMap(PackageHelper::walk) + .forEach(p -> { + String f = p.getFileName().toString(); + if (f.endsWith(PackageHelper.JSON_SUFIX)) { + allJsonFiles.add(p); + } else if (f.equals("component.properties") + || f.equals("dataformat.properties") + || f.equals("language.properties") + || f.equals("other.properties")) { + allPropertiesFiles.add(p); + } + }); + allModels = allJsonFiles.stream().collect(Collectors.toMap( + p -> p, JsonMapper::generateModel)); + + executeModel(); + Set<String> components = executeComponents(); + Set<String> dataformats = executeDataFormats(); + Set<String> languages = executeLanguages(); + Set<String> others = executeOthers(); + executeDocuments(components, dataformats, languages, others); + executeArchetypes(); + executeXmlSchemas(); + executeMain(); + } catch (Exception e) { + throw new MojoFailureException("Error preparing catalog", e); + } } - protected void executeModel() throws MojoExecutionException, MojoFailureException { + protected void executeModel() throws Exception { + Path coreDir = this.coreDir.toPath(); + Path springDir = this.springDir.toPath(); + Path modelsOutDir = this.modelsOutDir.toPath(); + getLog().info("================================================================================"); getLog().info("Copying all Camel model json descriptors"); // lets use sorted set/maps - Set<File> jsonFiles = new TreeSet<>(); - Set<File> duplicateJsonFiles = new TreeSet<>(); - Set<File> missingLabels = new TreeSet<>(); - Set<File> missingJavaDoc = new TreeSet<>(); + Set<Path> jsonFiles; + Set<Path> duplicateJsonFiles; + Set<Path> missingLabels = new TreeSet<>(); + Set<Path> missingJavaDoc = new TreeSet<>(); Map<String, Set<String>> usedLabels = new TreeMap<>(); - // find all json files in camel-core - if (coreDir != null && coreDir.isDirectory()) { - File target = new File(coreDir, "target/classes/org/apache/camel/model"); - PackageHelper.findJsonFiles(target, jsonFiles); - } - - // find all json files in camel-spring - if (springDir != null && springDir.isDirectory()) { - File target = new File(springDir, "target/classes/org/apache/camel/spring"); - PackageHelper.findJsonFiles(target, jsonFiles); - File target2 = new File(springDir, "target/classes/org/apache/camel/core/xml"); - PackageHelper.findJsonFiles(target2, jsonFiles); - } - + // find all json files in camel-core and camel-spring + Path coreDirTarget = coreDir.resolve("target/classes/org/apache/camel/model"); + Path springTarget1 = springDir.resolve("target/classes/org/apache/camel/spring"); + Path springTarget2 = springDir.resolve("target/classes/org/apache/camel/core/xml"); + jsonFiles = allJsonFiles.stream() + .filter(p -> p.startsWith(coreDirTarget) || p.startsWith(springTarget1) || p.startsWith(springTarget2)) + .collect(Collectors.toCollection(TreeSet::new)); getLog().info("Found " + jsonFiles.size() + " model json files"); // make sure to create out dir - modelsOutDir.mkdirs(); - // we only want to warn for duplicates if its a clean build - boolean warnDups = modelsOutDir.list() == null || modelsOutDir.list().length == 0; - - for (File file : jsonFiles) { - File to = new File(modelsOutDir, file.getName()); - if (to.exists()) { - if (warnDups) { - duplicateJsonFiles.add(to); - getLog().warn("Duplicate model name detected: " + to); - } else if (file.lastModified() < to.lastModified()) { - getLog().debug("Skipping generated file: " + to); - continue; - } else { - getLog().warn("Stale file: " + to); + Files.createDirectories(modelsOutDir); + + duplicateJsonFiles = getDuplicates(jsonFiles); + + // Copy all descriptors + Map<Path, Path> newJsons = map(jsonFiles, p -> p, p -> modelsOutDir.resolve(p.getFileName())); + list(modelsOutDir) + .filter(p -> !newJsons.containsValue(p)) + .forEach(this::delete); + newJsons.forEach(this::copy); + + for (Path file : jsonFiles) { + // check if we have a label as we want the eip to include labels + EipModel model = (EipModel) allModels.get(file); + + String name = asComponentName(file); + + // grab the label, and remember it in the used labels + String label = model.getLabel(); + if (Strings.isNullOrEmpty(label)) { + missingLabels.add(file); + } else { + String[] labels = label.split(","); + for (String s : labels) { + usedLabels.computeIfAbsent(s, k -> new TreeSet<>()).add(name); } } - try { - copyFile(file, to); - } catch (IOException e) { - throw new MojoFailureException("Cannot copy file from " + file + " -> " + to, e); - } - - try { - // check if we have a label as we want the eip to include labels - String text = PackageHelper.loadText(file); - // just do a basic label check - if (text.contains("\"label\": \"\"")) { - missingLabels.add(file); - } else { - String name = asComponentName(file); - Matcher matcher = LABEL_PATTERN.matcher(text); - // grab the label, and remember it in the used labels - if (matcher.find()) { - String label = matcher.group(1); - String[] labels = label.split(","); - for (String s : labels) { - usedLabels.computeIfAbsent(s, k -> new TreeSet<>()).add(name); - } - } - } - // check all the properties if they have description - List<Map<String, String>> rows = JSonSchemaHelper.parseJsonSchema("properties", text, true); - for (Map<String, String> row : rows) { - String name = row.get("name"); - // skip checking these as they have no documentation - if ("outputs".equals(name) || "transforms".equals(name)) { - continue; - } - - String doc = row.get("description"); - if (doc == null || doc.isEmpty()) { - missingJavaDoc.add(file); - break; - } - } - } catch (IOException e) { - // ignore + // check all the properties if they have description + if (model.getOptions().stream() + .filter(option -> !"outputs".equals(option.getName()) && !"transforms".equals(option.getName())) + .map(BaseOptionModel::getDescription) + .anyMatch(Strings::isNullOrEmpty)) { + missingJavaDoc.add(file); } } - File all = new File(modelsOutDir, "../models.properties"); - try { - FileOutputStream fos = new FileOutputStream(all, false); - - String[] names = modelsOutDir.list(); - List<String> models = new ArrayList<>(); - // sort the names - for (String name : names) { - if (name.endsWith(PackageHelper.JSON_SUFIX)) { - // strip out .json from the name - String modelName = name.substring(0, name.length() - 5); - models.add(modelName); - } - } + Path all = modelsOutDir.resolve("../models.properties"); + Set<String> modelNames = jsonFiles.stream() + .map(PrepareCatalogMojo::asComponentName) + .collect(Collectors.toCollection(TreeSet::new)); + FileUtil.updateFile(all, String.join("\n", modelNames) + "\n"); - Collections.sort(models); - for (String name : models) { - fos.write(name.getBytes()); - fos.write("\n".getBytes()); - } - - fos.close(); - - } catch (IOException e) { - throw new MojoFailureException("Error writing to file " + all); - } - - printModelsReport(jsonFiles, duplicateJsonFiles, missingLabels, usedLabels, missingJavaDoc); + printModelsReport( + jsonFiles, + duplicateJsonFiles, + missingLabels, + usedLabels, + missingJavaDoc); } // CHECKSTYLE:OFF - protected Set<String> executeComponents() throws MojoExecutionException, MojoFailureException { + protected Set<String> executeComponents() throws Exception { + Path coreDir = this.coreDir.toPath(); + Path componentsDir = this.componentsDir.toPath(); + Path componentsOutDir = this.componentsOutDir.toPath(); + getLog().info("Copying all Camel component json descriptors"); // lets use sorted set/maps - Set<File> jsonFiles = new TreeSet<>(); - Set<File> duplicateJsonFiles = new TreeSet<>(); - Set<File> componentFiles = new TreeSet<>(); - Set<File> missingComponents = new TreeSet<>(); + Set<Path> jsonFiles; + Set<Path> duplicateJsonFiles; + Set<Path> componentFiles; + Set<Path> missingComponents = new TreeSet<>(); Map<String, Set<String>> usedComponentLabels = new TreeMap<>(); Set<String> usedOptionLabels = new TreeSet<>(); Set<String> unlabeledOptions = new TreeSet<>(); - Set<File> missingFirstVersions = new TreeSet<>(); + Set<Path> missingFirstVersions = new TreeSet<>(); // find all json files in components and camel-core - if (componentsDir != null && componentsDir.isDirectory()) { - File[] components = componentsDir.listFiles(); - if (components != null) { - for (File dir : components) { - if (dir.isDirectory() && !"target".equals(dir.getName())) { - File target = new File(dir, "target/classes"); - - // special for these as they are in sub dir - if ("camel-as2".equals(dir.getName())) { - target = new File(dir, "camel-as2-component/target/classes"); - } else if ("camel-salesforce".equals(dir.getName())) { - target = new File(dir, "camel-salesforce-component/target/classes"); - } else if ("camel-olingo2".equals(dir.getName())) { - target = new File(dir, "camel-olingo2-component/target/classes"); - } else if ("camel-olingo4".equals(dir.getName())) { - target = new File(dir, "camel-olingo4-component/target/classes"); - } else if ("camel-box".equals(dir.getName())) { - target = new File(dir, "camel-box-component/target/classes"); - } else if ("camel-servicenow".equals(dir.getName())) { - target = new File(dir, "camel-servicenow-component/target/classes"); - } else if ("camel-fhir".equals(dir.getName())) { - target = new File(dir, "camel-fhir-component/target/classes"); - } else { - // this module must be active with a source folder - File src = new File(dir, "src"); - boolean active = src.isDirectory() && src.exists(); - if (!active) { - continue; - } - } - - int before = componentFiles.size(); - int before2 = jsonFiles.size(); - - findComponentFilesRecursive(target, jsonFiles, componentFiles, new CamelComponentsFileFilter()); - - int after = componentFiles.size(); - int after2 = jsonFiles.size(); - if (before != after && before2 == after2) { - missingComponents.add(dir); - } + componentFiles = allPropertiesFiles.stream() + .filter(p -> p.endsWith("component.properties")) + .collect(Collectors.toCollection(TreeSet::new)); + jsonFiles = allJsonFiles.stream() + .filter(p -> allModels.get(p) instanceof ComponentModel) + .collect(Collectors.toCollection(TreeSet::new)); + componentFiles.stream() + .filter(p -> p.endsWith("component.properties")) + .forEach(p -> { + Path parent = getModule(p); + List<Path> jsons = jsonFiles.stream() + .filter(f -> f.startsWith(parent)) + .collect(Collectors.toList()); + if (jsons.isEmpty()) { + missingComponents.add(parent); } - } - } - } - if (coreDir != null && coreDir.isDirectory()) { - File target = new File(coreDir, "target/classes"); - - int before = componentFiles.size(); - int before2 = jsonFiles.size(); - - findComponentFilesRecursive(target, jsonFiles, componentFiles, new CamelComponentsFileFilter()); - - int after = componentFiles.size(); - int after2 = jsonFiles.size(); - if (before != after && before2 == after2) { - missingComponents.add(coreDir); - } - } + }); getLog().info("Found " + componentFiles.size() + " component.properties files"); getLog().info("Found " + jsonFiles.size() + " component json files"); // make sure to create out dir - componentsOutDir.mkdirs(); - // we only want to warn for duplicates if its a clean build - boolean warnDups = componentsOutDir.list() == null || componentsOutDir.list().length == 0; + Files.createDirectories(componentsOutDir); - Set<String> alternativeSchemes = new HashSet<>(); + // Check duplicates + duplicateJsonFiles = getDuplicates(jsonFiles); - for (File file : jsonFiles) { - File to = new File(componentsOutDir, file.getName()); - if (to.exists()) { - if (warnDups) { - duplicateJsonFiles.add(to); - getLog().warn("Duplicate component name detected: " + to); - } else if (file.lastModified() < to.lastModified()) { - getLog().debug("Skipping generated file: " + to); - continue; - } else { - getLog().warn("Stale file: " + to); - } - } - try { - copyFile(file, to); - } catch (IOException e) { - throw new MojoFailureException("Cannot copy file from " + file + " -> " + to, e); - } + // Copy all descriptors + Map<Path, Path> newJsons = map(jsonFiles, p -> p, p -> componentsOutDir.resolve(p.getFileName())); + list(componentsOutDir) + .filter(p -> !newJsons.containsValue(p)) + .forEach(this::delete); + newJsons.forEach(this::copy); + Set<String> alternativeSchemes = new HashSet<>(); + + for (Path file : jsonFiles) { // check if we have a component label as we want the components to // include labels try { String text = PackageHelper.loadText(file); + ComponentModel model = JsonMapper.generateComponentModel(text); + String name = asComponentName(file); - Matcher matcher = LABEL_PATTERN.matcher(text); + // grab the label, and remember it in the used labels - if (matcher.find()) { - String label = matcher.group(1); - String[] labels = label.split(","); - for (String s : labels) { - Set<String> components = usedComponentLabels.computeIfAbsent(s, k -> new TreeSet<>()); - components.add(name); - } + String label = model.getLabel(); + String[] labels = label.split(","); + for (String s : labels) { + Set<String> components = usedComponentLabels.computeIfAbsent(s, k -> new TreeSet<>()); + components.add(name); } - // check all the component options and grab the label(s) they - // use - List<Map<String, String>> rows = JSonSchemaHelper.parseJsonSchema("componentProperties", text, true); - for (Map<String, String> row : rows) { - String label = row.get("label"); - - if (label != null && !label.isEmpty()) { - String[] parts = label.split(","); - Collections.addAll(usedOptionLabels, parts); - } - } + // check all the component options and grab the label(s) they use + model.getComponentOptions().stream() + .map(BaseOptionModel::getLabel) + .filter(l -> !Strings.isNullOrEmpty(l)) + .flatMap(l -> Stream.of(label.split(","))) + .forEach(usedOptionLabels::add); // check all the endpoint options and grab the label(s) they use - int unused = 0; - rows = JSonSchemaHelper.parseJsonSchema("properties", text, true); - for (Map<String, String> row : rows) { - String label = row.get("label"); - if (label != null && !label.isEmpty()) { - String[] parts = label.split(","); - usedOptionLabels.addAll(Arrays.asList(parts)); - } else { - unused++; - } - } - + model.getEndpointOptions().stream() + .map(BaseOptionModel::getLabel) + .filter(l -> !Strings.isNullOrEmpty(l)) + .flatMap(l -> Stream.of(label.split(","))) + .forEach(usedOptionLabels::add); + + long unused = model.getEndpointOptions().stream() + .map(BaseOptionModel::getLabel) + .filter(Strings::isNullOrEmpty) + .count(); if (unused >= UNUSED_LABELS_WARN) { unlabeledOptions.add(name); } // remember alternative schemes - rows = JSonSchemaHelper.parseJsonSchema("component", text, false); - for (Map<String, String> row : rows) { - String alternativeScheme = row.get("alternativeSchemes"); - if (alternativeScheme != null && !alternativeScheme.isEmpty()) { - String[] parts = alternativeScheme.split(","); - // skip first as that is the regular scheme - alternativeSchemes.addAll(Arrays.asList(parts).subList(1, parts.length)); - } + String alternativeScheme = model.getAlternativeSchemes(); + if (!Strings.isNullOrEmpty(alternativeScheme)) { + String[] parts = alternativeScheme.split(","); + // skip first as that is the regular scheme + alternativeSchemes.addAll(Arrays.asList(parts).subList(1, parts.length)); } // detect missing first version - String firstVersion = null; - for (Map<String, String> row : rows) { - if (row.get("firstVersion") != null) { - firstVersion = row.get("firstVersion"); - } - } - if (firstVersion == null) { + String firstVersion = model.getFirstVersion(); + if (Strings.isNullOrEmpty(firstVersion)) { missingFirstVersions.add(file); } @@ -502,460 +444,313 @@ public class PrepareCatalogMojo extends AbstractMojo { } } - Set<String> componentNames = generateJsonList(componentsOutDir.toPath(), "../components.properties"); + Path all = componentsOutDir.resolve("../models.properties"); + Set<String> componentNames = jsonFiles.stream() + .map(PrepareCatalogMojo::asComponentName) + .collect(Collectors.toCollection(TreeSet::new)); + FileUtil.updateFile(all, String.join("\n", componentNames) + "\n"); - printComponentsReport(jsonFiles, duplicateJsonFiles, missingComponents, usedComponentLabels, usedOptionLabels, unlabeledOptions, missingFirstVersions); + printComponentsReport( + jsonFiles, + duplicateJsonFiles, + missingComponents, + usedComponentLabels, + usedOptionLabels, + unlabeledOptions, + missingFirstVersions); - // filter out duplicate component names that are alternative scheme - // names + // filter out duplicate component names that are alternative scheme names componentNames.removeAll(alternativeSchemes); return componentNames; } - // CHECKSTYLE:ON - protected Set<String> executeDataFormats() throws MojoExecutionException, MojoFailureException { + protected Set<String> executeDataFormats() throws Exception { + Path dataFormatsOutDir = this.dataFormatsOutDir.toPath(); + getLog().info("Copying all Camel dataformat json descriptors"); // lets use sorted set/maps - Set<File> jsonFiles = new TreeSet<>(); - Set<File> duplicateJsonFiles = new TreeSet<>(); - Set<File> dataFormatFiles = new TreeSet<>(); + Set<Path> jsonFiles; + Set<Path> duplicateJsonFiles; + Set<Path> dataFormatFiles; Map<String, Set<String>> usedLabels = new TreeMap<>(); - Set<File> missingFirstVersions = new TreeSet<>(); + Set<Path> missingFirstVersions = new TreeSet<>(); // find all data formats from the components directory - if (componentsDir != null && componentsDir.isDirectory()) { - File[] dataFormats = componentsDir.listFiles(); - if (dataFormats != null) { - for (File dir : dataFormats) { - // special for this as the data format is in the sub dir - if (dir.isDirectory() && "camel-fhir".equals(dir.getName())) { - dir = new File(dir, "camel-fhir-component"); - } - if (dir.isDirectory() && !"target".equals(dir.getName())) { - File target = new File(dir, "target/classes"); - // this module must be active with a source folder - File src = new File(dir, "src"); - boolean active = src.isDirectory() && src.exists(); - if (active) { - findDataFormatFilesRecursive(target, jsonFiles, dataFormatFiles, new CamelDataFormatsFileFilter()); - } - } - } - } - } - if (coreDir != null && coreDir.isDirectory()) { - File target = new File(coreDir, "target/classes"); - findDataFormatFilesRecursive(target, jsonFiles, dataFormatFiles, new CamelDataFormatsFileFilter()); - } + dataFormatFiles = allPropertiesFiles.stream() + .filter(p -> p.endsWith("dataformat.properties")) + .collect(Collectors.toCollection(TreeSet::new)); + jsonFiles = allJsonFiles.stream() + .filter(p -> allModels.get(p) instanceof DataFormatModel) + .collect(Collectors.toCollection(TreeSet::new)); getLog().info("Found " + dataFormatFiles.size() + " dataformat.properties files"); getLog().info("Found " + jsonFiles.size() + " dataformat json files"); // make sure to create out dir - dataFormatsOutDir.mkdirs(); - // we only want to warn for duplicates if its a clean build - boolean warnDups = dataFormatsOutDir.list() == null || dataFormatsOutDir.list().length == 0; - - for (File file : jsonFiles) { - File to = new File(dataFormatsOutDir, file.getName()); - if (to.exists()) { - if (warnDups) { - duplicateJsonFiles.add(to); - getLog().warn("Duplicate dataformat name detected: " + to); - } else if (file.lastModified() < to.lastModified()) { - getLog().debug("Skipping generated file: " + to); - continue; - } else { - getLog().warn("Stale file: " + to); - } - } - try { - copyFile(file, to); - } catch (IOException e) { - throw new MojoFailureException("Cannot copy file from " + file + " -> " + to, e); - } + Files.createDirectories(dataFormatsOutDir); - // check if we have a label as we want the data format to include - // labels - try { - String text = PackageHelper.loadText(file); - String name = asComponentName(file); - Matcher matcher = LABEL_PATTERN.matcher(text); - // grab the label, and remember it in the used labels - if (matcher.find()) { - String label = matcher.group(1); - String[] labels = label.split(","); - for (String s : labels) { - Set<String> dataFormats = usedLabels.computeIfAbsent(s, k -> new TreeSet<>()); - dataFormats.add(name); - } - } + // Check duplicates + duplicateJsonFiles = getDuplicates(jsonFiles); - // detect missing first version - List<Map<String, String>> rows = JSonSchemaHelper.parseJsonSchema("dataformat", text, false); - String firstVersion = null; - for (Map<String, String> row : rows) { - if (row.get("firstVersion") != null) { - firstVersion = row.get("firstVersion"); - } - } - if (firstVersion == null) { - missingFirstVersions.add(file); - } + // Copy all descriptors + Map<Path, Path> newJsons = map(jsonFiles, p -> p, p -> dataFormatsOutDir.resolve(p.getFileName())); + list(dataFormatsOutDir) + .filter(p -> !newJsons.containsValue(p)) + .forEach(this::delete); + newJsons.forEach(this::copy); - } catch (IOException e) { - // ignore + for (Path file : jsonFiles) { + + DataFormatModel model = (DataFormatModel) allModels.get(file); + + // Check labels + String name = asComponentName(file); + for (String s : model.getLabel().split(",")) { + usedLabels.computeIfAbsent(s, k -> new TreeSet<>()).add(name); + } + + // detect missing first version + String firstVersion = model.getFirstVersion(); + if (Strings.isNullOrEmpty(firstVersion)) { + missingFirstVersions.add(file); } } - Set<String> answer = generateJsonList(dataFormatsOutDir.toPath(), "../dataformats.properties"); + Path all = dataFormatsOutDir.resolve("../dataformats.properties"); + Set<String> dataFormatNames = jsonFiles.stream() + .map(PrepareCatalogMojo::asComponentName) + .collect(Collectors.toCollection(TreeSet::new)); + FileUtil.updateFile(all, String.join("\n", dataFormatNames) + "\n"); printDataFormatsReport(jsonFiles, duplicateJsonFiles, usedLabels, missingFirstVersions); - return answer; + return dataFormatNames; } - protected Set<String> executeLanguages() throws MojoExecutionException, MojoFailureException { + protected Set<String> executeLanguages() throws Exception { + Path languagesOutDir = this.languagesOutDir.toPath(); + getLog().info("Copying all Camel language json descriptors"); // lets use sorted set/maps - Set<File> jsonFiles = new TreeSet<>(); - Set<File> duplicateJsonFiles = new TreeSet<>(); - Set<File> languageFiles = new TreeSet<>(); + Set<Path> jsonFiles; + Set<Path> duplicateJsonFiles; + Set<Path> languageFiles; Map<String, Set<String>> usedLabels = new TreeMap<>(); - Set<File> missingFirstVersions = new TreeSet<>(); + Set<Path> missingFirstVersions = new TreeSet<>(); // find all languages from the components directory - if (componentsDir != null && componentsDir.isDirectory()) { - File[] languages = componentsDir.listFiles(); - if (languages != null) { - for (File dir : languages) { - if (dir.isDirectory() && !"target".equals(dir.getName())) { - File target = new File(dir, "target/classes"); - // this module must be active with a source folder - File src = new File(dir, "src"); - boolean active = src.isDirectory() && src.exists(); - if (active) { - findLanguageFilesRecursive(target, jsonFiles, languageFiles, new CamelLanguagesFileFilter()); - } - } - } - } - } - if (baseDir != null && baseDir.isDirectory()) { - File target = new File(baseDir, "target/classes"); - findLanguageFilesRecursive(target, jsonFiles, languageFiles, new CamelLanguagesFileFilter()); - // also look in camel-jaxp - target = new File(baseDir, "../camel-jaxp/target/classes"); - findLanguageFilesRecursive(target, jsonFiles, languageFiles, new CamelLanguagesFileFilter()); - } + languageFiles = allPropertiesFiles.stream() + .filter(p -> p.endsWith("language.properties")) + .collect(Collectors.toCollection(TreeSet::new)); + jsonFiles = allJsonFiles.stream() + .filter(p -> allModels.get(p) instanceof LanguageModel) + .collect(Collectors.toCollection(TreeSet::new)); getLog().info("Found " + languageFiles.size() + " language.properties files"); getLog().info("Found " + jsonFiles.size() + " language json files"); // make sure to create out dir - languagesOutDir.mkdirs(); - // we only want to warn for duplicates if its a clean build - boolean warnDups = languagesOutDir.list() == null || languagesOutDir.list().length == 0; - - for (File file : jsonFiles) { - File to = new File(languagesOutDir, file.getName()); - if (to.exists()) { - if (warnDups) { - duplicateJsonFiles.add(to); - getLog().warn("Duplicate language name detected: " + to); - } else if (file.lastModified() < to.lastModified()) { - getLog().debug("Skipping generated file: " + to); - continue; - } else { - getLog().warn("Stale file: " + to); - } - } - try { - copyFile(file, to); - } catch (IOException e) { - throw new MojoFailureException("Cannot copy file from " + file + " -> " + to, e); - } + Files.createDirectories(languagesOutDir); - // check if we have a label as we want the data format to include - // labels - try { - String text = PackageHelper.loadText(file); - String name = asComponentName(file); - Matcher matcher = LABEL_PATTERN.matcher(text); - // grab the label, and remember it in the used labels - if (matcher.find()) { - String label = matcher.group(1); - String[] labels = label.split(","); - for (String s : labels) { - Set<String> languages = usedLabels.computeIfAbsent(s, k -> new TreeSet<>()); - languages.add(name); - } - } + // Check duplicates + duplicateJsonFiles = getDuplicates(jsonFiles); - // detect missing first version - List<Map<String, String>> rows = JSonSchemaHelper.parseJsonSchema("language", text, false); - String firstVersion = null; - for (Map<String, String> row : rows) { - if (row.get("firstVersion") != null) { - firstVersion = row.get("firstVersion"); - } - } - if (firstVersion == null) { - missingFirstVersions.add(file); - } + // Copy all descriptors + Map<Path, Path> newJsons = map(jsonFiles, p -> p, p -> languagesOutDir.resolve(p.getFileName())); + list(languagesOutDir) + .filter(p -> !newJsons.containsValue(p)) + .forEach(this::delete); + newJsons.forEach(this::copy); - } catch (IOException e) { - // ignore + for (Path file : jsonFiles) { + + LanguageModel model = (LanguageModel) allModels.get(file); + + // Check labels + String name = asComponentName(file); + for (String s : model.getLabel().split(",")) { + usedLabels.computeIfAbsent(s, k -> new TreeSet<>()).add(name); + } + + // detect missing first version + String firstVersion = model.getFirstVersion(); + if (Strings.isNullOrEmpty(firstVersion)) { + missingFirstVersions.add(file); } } - Set<String> answer = generateJsonList(languagesOutDir.toPath(), "../languages.properties"); + Path all = languagesOutDir.resolve("../dataformats.properties"); + Set<String> languagesNames = jsonFiles.stream() + .map(PrepareCatalogMojo::asComponentName) + .collect(Collectors.toCollection(TreeSet::new)); + FileUtil.updateFile(all, String.join("\n", languagesNames) + "\n"); printLanguagesReport(jsonFiles, duplicateJsonFiles, usedLabels, missingFirstVersions); - return answer; + return languagesNames; } - private Set<String> executeOthers() throws MojoFailureException { + private Set<String> executeOthers() throws Exception { + Path othersOutDir = this.othersOutDir.toPath(); + getLog().info("Copying all Camel other json descriptors"); // lets use sorted set/maps - Set<File> jsonFiles = new TreeSet<>(); - Set<File> duplicateJsonFiles = new TreeSet<>(); - Set<File> otherFiles = new TreeSet<>(); + Set<Path> jsonFiles; + Set<Path> duplicateJsonFiles; + Set<Path> otherFiles; Map<String, Set<String>> usedLabels = new TreeMap<>(); - Set<File> missingFirstVersions = new TreeSet<>(); - - // find all others from the components directory - if (componentsDir != null && componentsDir.isDirectory()) { - File[] others = componentsDir.listFiles(); - if (others != null) { - for (File dir : others) { - - // skip these special cases - boolean special = "camel-core-osgi".equals(dir.getName()) || "camel-core-xml".equals(dir.getName()) || "camel-box".equals(dir.getName()) - || "camel-http-base".equals(dir.getName()) || "camel-http-common".equals(dir.getName()) || "camel-jetty-common".equals(dir.getName()); - boolean special2 = "camel-as2".equals(dir.getName()) || "camel-olingo2".equals(dir.getName()) || "camel-olingo4".equals(dir.getName()) - || "camel-servicenow".equals(dir.getName()) || "camel-salesforce".equals(dir.getName()) || "camel-fhir".equals(dir.getName()); - boolean special3 = "camel-debezium-common".equals(dir.getName()); - if (special || special2 || special3) { - continue; - } - - if (dir.isDirectory() && !"target".equals(dir.getName())) { - File target = new File(dir, "target/classes"); - if (target.exists()) { - // this module must be active with a source folder - File src = new File(dir, "src"); - boolean active = src.isDirectory() && src.exists(); - if (active) { - findOtherFilesRecursive(target, jsonFiles, otherFiles, new CamelOthersFileFilter()); - } - } + Set<Path> missingFirstVersions = new TreeSet<>(); + + otherFiles = allPropertiesFiles.stream() + .filter(p -> p.endsWith("other.properties")) + .collect(Collectors.toCollection(TreeSet::new)); + jsonFiles = allJsonFiles.stream() + .filter(p -> { + Path m = getModule(p); + switch (m.getFileName().toString()) { + case "camel-core-osgi": + case "camel-core-xml": + case "camel-box": + case "camel-http-base": + case "camel-http-common": + case "camel-jetty-common": + case "camel-as2": + case "camel-olingo2": + case "camel-olingo4": + case "camel-servicenow": + case "camel-salesforce": + case "camel-fhir": + case "camel-debezium-common": + return false; + default: + return true; } - } - } - } - // nothing in camel-core + }) + .filter(p -> allModels.get(p) instanceof OtherModel) + .collect(Collectors.toCollection(TreeSet::new)); getLog().info("Found " + otherFiles.size() + " other.properties files"); getLog().info("Found " + jsonFiles.size() + " other json files"); // make sure to create out dir - othersOutDir.mkdirs(); - // we only want to warn for duplicates if its a clean build - boolean warnDups = othersOutDir.list() == null || othersOutDir.list().length == 0; - - for (File file : jsonFiles) { - File to = new File(othersOutDir, file.getName()); - if (to.exists()) { - if (warnDups) { - duplicateJsonFiles.add(to); - getLog().warn("Duplicate other name detected: " + to); - } else if (file.lastModified() < to.lastModified()) { - getLog().debug("Skipping generated file: " + to); - continue; - } else { - getLog().warn("Stale file: " + to); - } - } - try { - copyFile(file, to); - } catch (IOException e) { - throw new MojoFailureException("Cannot copy file from " + file + " -> " + to, e); - } + Files.createDirectories(othersOutDir); - // check if we have a label as we want the other to include labels - try { - String text = PackageHelper.loadText(file); - String name = asComponentName(file); - Matcher matcher = LABEL_PATTERN.matcher(text); - // grab the label, and remember it in the used labels - if (matcher.find()) { - String label = matcher.group(1); - String[] labels = label.split(","); - for (String s : labels) { - Set<String> others = usedLabels.computeIfAbsent(s, k -> new TreeSet<>()); - others.add(name); - } - } + // Check duplicates + duplicateJsonFiles = getDuplicates(jsonFiles); - // detect missing first version - List<Map<String, String>> rows = JSonSchemaHelper.parseJsonSchema("other", text, false); - String firstVersion = null; - for (Map<String, String> row : rows) { - if (row.get("firstVersion") != null) { - firstVersion = row.get("firstVersion"); - } - } - if (firstVersion == null) { - missingFirstVersions.add(file); + // Copy all descriptors + Map<Path, Path> newJsons = map(jsonFiles, p -> p, p -> othersOutDir.resolve(p.getFileName())); + list(othersOutDir) + .filter(p -> !newJsons.containsValue(p)) + .forEach(this::delete); + newJsons.forEach(this::copy); + + for (Path file : jsonFiles) { + + OtherModel model = (OtherModel) allModels.get(file); + + String name = asComponentName(file); + + // grab the label, and remember it in the used labels + String label = model.getLabel(); + if (!Strings.isNullOrEmpty(label)) { + String[] labels = label.split(","); + for (String s : labels) { + usedLabels.computeIfAbsent(s, k -> new TreeSet<>()).add(name); } + } - } catch (IOException e) { - // ignore + // detect missing first version + String firstVersion = model.getFirstVersion(); + if (Strings.isNullOrEmpty(firstVersion)) { + missingFirstVersions.add(file); } } - Set<String> answer = generateJsonList(othersOutDir.toPath(), "../others.properties"); + Path all = othersOutDir.resolve("../others.properties"); + Set<String> otherNames = jsonFiles.stream() + .map(PrepareCatalogMojo::asComponentName) + .collect(Collectors.toCollection(TreeSet::new)); + FileUtil.updateFile(all, String.join("\n", otherNames) + "\n"); printOthersReport(jsonFiles, duplicateJsonFiles, usedLabels, missingFirstVersions); - return answer; + return otherNames; } - protected void executeArchetypes() throws MojoExecutionException, MojoFailureException { + protected void executeArchetypes() throws Exception { + Path archetypesDir = this.archetypesDir.toPath(); + Path archetypesOutDir = this.archetypesOutDir.toPath(); + getLog().info("Copying Archetype Catalog"); // find the generate catalog - File file = new File(archetypesDir, "target/classes/archetype-catalog.xml"); - - // make sure to create out dir - archetypesOutDir.mkdirs(); - - if (file.exists() && file.isFile()) { - File to = new File(archetypesOutDir, file.getName()); - try { - copyFile(file, to); - } catch (IOException e) { - throw new MojoFailureException("Cannot copy file from " + file + " -> " + to, e); - } - } + copyFile(archetypesDir.resolve("target/classes/archetype-catalog.xml"), + archetypesOutDir); } - protected void executeXmlSchemas() throws MojoExecutionException, MojoFailureException { - getLog().info("Copying Spring/Blueprint XML schemas"); + protected void executeXmlSchemas() throws Exception { + Path schemasOutDir = this.schemasOutDir.toPath(); + Path springSchemaDir = this.springSchemaDir.toPath(); + Path blueprintSchemaDir = this.blueprintSchemaDir.toPath(); - schemasOutDir.mkdirs(); + getLog().info("Copying Spring/Blueprint XML schemas"); - File file = new File(springSchemaDir, "camel-spring.xsd"); - if (file.exists() && file.isFile()) { - File to = new File(schemasOutDir, file.getName()); - try { - copyFile(file, to); - } catch (IOException e) { - throw new MojoFailureException("Cannot copy file from " + file + " -> " + to, e); - } - } - file = new File(blueprintSchemaDir, "camel-blueprint.xsd"); - if (file.exists() && file.isFile()) { - File to = new File(schemasOutDir, file.getName()); - try { - copyFile(file, to); - } catch (IOException e) { - throw new MojoFailureException("Cannot copy file from " + file + " -> " + to, e); - } - } + copyFile(springSchemaDir.resolve("camel-spring.xsd"), schemasOutDir); + copyFile(blueprintSchemaDir.resolve("camel-blueprint.xsd"), schemasOutDir); } - protected void executeMain() throws MojoExecutionException, MojoFailureException { + protected void executeMain() throws Exception { getLog().info("Copying camel-main metadata"); - mainOutDir.mkdirs(); - - File file = new File(mainDir, "camel-main-configuration-metadata.json"); - if (file.exists() && file.isFile()) { - File to = new File(mainOutDir, file.getName()); - try { - copyFile(file, to); - } catch (IOException e) { - throw new MojoFailureException("Cannot copy file from " + file + " -> " + to, e); - } - } + copyFile(mainDir.toPath().resolve("camel-main-configuration-metadata.json"), + mainOutDir.toPath()); } - protected void executeDocuments(Set<String> components, Set<String> dataformats, Set<String> languages, Set<String> others) - throws MojoExecutionException, MojoFailureException { + protected void executeDocuments(Set<String> components, Set<String> dataformats, Set<String> languages, Set<String> others) throws Exception { + Path documentsOutDir = this.documentsOutDir.toPath(); + getLog().info("Copying all Camel documents (ascii docs)"); // lets use sorted set/maps - Set<File> adocFiles = new TreeSet<>(); - Set<File> missingAdocFiles = new TreeSet<>(); - Set<File> duplicateAdocFiles = new TreeSet<>(); + Set<Path> adocFiles = new TreeSet<>(); + Set<Path> missingAdocFiles = new TreeSet<>(); + Set<Path> duplicateAdocFiles = new TreeSet<>(); // find all camel maven modules - if (componentsDir != null && componentsDir.isDirectory()) { - File[] componentFiles = componentsDir.listFiles(); - if (componentFiles != null) { - for (File dir : componentFiles) { - if (dir.isDirectory() && !"target".equals(dir.getName()) && !dir.getName().startsWith(".") && !excludeDocumentDir(dir.getName())) { - File target = new File(dir, "src/main/docs"); - - // special for these as they are in sub dir - if ("camel-as2".equals(dir.getName())) { - target = new File(dir, "camel-as2-component/src/main/docs"); - } else if ("camel-salesforce".equals(dir.getName())) { - target = new File(dir, "camel-salesforce-component/src/main/docs"); - } else if ("camel-olingo2".equals(dir.getName())) { - target = new File(dir, "camel-olingo2-component/src/main/docs"); - } else if ("camel-olingo4".equals(dir.getName())) { - target = new File(dir, "camel-olingo4-component/src/main/docs"); - } else if ("camel-box".equals(dir.getName())) { - target = new File(dir, "camel-box-component/src/main/docs"); - } else if ("camel-servicenow".equals(dir.getName())) { - target = new File(dir, "camel-servicenow-component/src/main/docs"); - } else if ("camel-fhir".equals(dir.getName())) { - target = new File(dir, "camel-fhir-component/src/main/docs"); - } else { - // this module must be active with a source folder - File src = new File(dir, "src"); - boolean active = src.isDirectory() && src.exists(); - if (!active) { - continue; - } - } - - int before = adocFiles.size(); - findAsciiDocFilesRecursive(target, adocFiles, new CamelAsciiDocFileFilter()); - int after = adocFiles.size(); - - if (before == after) { - missingAdocFiles.add(dir); - } + Stream.concat( + list(componentsDir.toPath()) + .filter(dir -> !"target".equals(dir.getFileName().toString())) + .map(this::getComponentPath), + Stream.of(coreDir.toPath(), baseDir.toPath(), jaxpDir.toPath())) + .forEach(dir -> { + List<Path> l = PackageHelper.walk(dir.resolve("src/main/docs")) + .filter(f -> f.getFileName().toString().endsWith(".adoc")) + .collect(Collectors.toList()); + if (l.isEmpty()) { + missingAdocFiles.add(dir); } - } - } - } - if (coreDir != null && coreDir.isDirectory()) { - File target = new File(coreDir, "src/main/docs"); - findAsciiDocFilesRecursive(target, adocFiles, new CamelAsciiDocFileFilter()); - } - if (baseDir != null && baseDir.isDirectory()) { - File target = new File(baseDir, "src/main/docs"); - findAsciiDocFilesRecursive(target, adocFiles, new CamelAsciiDocFileFilter()); - // also look in camel-jaxp - target = new File(coreDir, "../camel-jaxp/src/main/docs"); - findAsciiDocFilesRecursive(target, adocFiles, new CamelAsciiDocFileFilter()); - } + adocFiles.addAll(l); + }); getLog().info("Found " + adocFiles.size() + " ascii document files"); // make sure to create out dir - documentsOutDir.mkdirs(); - // we only want to warn for duplicates if its a clean build - boolean warnDups = documentsOutDir.list() == null || documentsOutDir.list().length == 0; + Files.createDirectories(documentsOutDir); + + // Check duplicates + duplicateAdocFiles = getDuplicates(adocFiles); + + // Copy all descriptors + Map<Path, Path> newJsons = map(adocFiles, p -> p, p -> documentsOutDir.resolve(p.getFileName())); + list(documentsOutDir) + .filter(p -> !newJsons.containsValue(p) + && !newJsons.containsValue(p.resolveSibling(p.getFileName().toString().replace(".html", ".adoc")))) + .forEach(this::delete); + newJsons.forEach(this::copy); // use ascii doctor to convert the adoc files to html so we have // documentation in this format as well @@ -963,96 +758,50 @@ public class PrepareCatalogMojo extends AbstractMojo { int converted = 0; - for (File file : adocFiles) { - File to = new File(documentsOutDir, file.getName()); - if (to.exists()) { - if (warnDups) { - duplicateAdocFiles.add(to); - getLog().warn("Duplicate document name detected: " + to); - } else if (file.lastModified() < to.lastModified()) { - getLog().debug("Skipping generated file: " + to); - continue; - } else { - getLog().warn("Stale file: " + to); - } - } - try { - copyFile(file, to); - } catch (IOException e) { - throw new MojoFailureException("Cannot copy file from " + file + " -> " + to, e); + for (Path file : adocFiles) { + // convert adoc to html as well + String fileName = file.getFileName().toString(); + String newName = fileName.substring(0, fileName.length() - ".adoc".length()) + ".html"; + Path toHtml = documentsOutDir.resolve(newName); + + if (Files.isRegularFile(toHtml) + && Files.getLastModifiedTime(file).compareTo(Files.getLastModifiedTime(toHtml)) < 0) { + getLog().debug("Skipping up to date html -> " + toHtml); + continue; } - // convert adoc to html as well - if (file.getName().endsWith(".adoc")) { - String newName = file.getName().substring(0, file.getName().length() - 5) + ".html"; - File toHtml = new File(documentsOutDir, newName); - - getLog().debug("Converting ascii document to html -> " + toHtml); - asciidoctor.convertFile(file, OptionsBuilder.options().toFile(toHtml)); - - converted++; - - try { - // now fix the html file because we don't want to include - // certain lines - List<String> lines = FileUtils.readLines(toHtml, Charset.defaultCharset()); - List<String> output = new ArrayList<>(); - for (String line : lines) { - // skip these lines - if (line.contains("% raw %") || line.contains("% endraw %")) { - continue; - } - output.add(line); - } - if (lines.size() != output.size()) { - FileUtils.writeLines(toHtml, output, false); - } - } catch (IOException e) { - // ignore + getLog().debug("Converting ascii document to html -> " + toHtml); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + asciidoctor.convertFile(file.toFile(), OptionsBuilder.options().toStream(new PrintStream(baos) { + public void write(RubyString str) { + this.append(str); } - } + })); + + converted++; + + // now fix the html file because we don't want to include certain lines + String data = Stream.of(new String(baos.toByteArray()).split("\n")) + .filter(line -> !line.contains("% raw %") && !line.contains("% endraw %")) + .collect(Collectors.joining("\n")) + "\n"; + FileUtil.updateFile(toHtml, data); } if (converted > 0) { getLog().info("Converted " + converted + " ascii documents to HTML"); } - Set<String> docs = new LinkedHashSet<>(); - - File all = new File(documentsOutDir, "../docs.properties"); - try { - FileOutputStream fos = new FileOutputStream(all, false); - - String[] names = documentsOutDir.list(); - List<String> documents = new ArrayList<>(); - // sort the names - for (String name : names) { - if (name.endsWith(".adoc")) { - // strip out .adoc from the name - String documentName = name.substring(0, name.length() - 5); - documents.add(documentName); - } - } - - Collections.sort(documents); - for (String name : documents) { - fos.write(name.getBytes()); - fos.write("\n".getBytes()); - - docs.add(name); - } - - fos.close(); - - } catch (IOException e) { - throw new MojoFailureException("Error writing to file " + all); - } + Path all = documentsOutDir.resolve("../docs.properties"); + Set<String> docNames = adocFiles.stream() + .map(PrepareCatalogMojo::asComponentName) + .collect(Collectors.toCollection(TreeSet::new)); + FileUtil.updateFile(all, String.join("\n", docNames) + "\n"); printDocumentsReport(adocFiles, duplicateAdocFiles, missingAdocFiles); // find out if we have documents for each component / dataformat / // languages / others - printMissingDocumentsReport(docs, components, dataformats, languages, others); + printMissingDocumentsReport(docNames, components, dataformats, languages, others); } private void printMissingDocumentsReport(Set<String> docs, Set<String> components, Set<String> dataformats, Set<String> languages, Set<String> others) { @@ -1063,11 +812,20 @@ public class PrepareCatalogMojo extends AbstractMojo { List<String> missing = new ArrayList<>(); for (String component : components) { // special for mail - if (component.equals("imap") || component.equals("imaps") || component.equals("pop3") || component.equals("pop3s") || component.equals("smtp") - || component.equals("smtps")) { - component = "mail"; - } else if (component.equals("ftp") || component.equals("sftp") || component.equals("ftps")) { - component = "ftp"; + switch (component) { + case "imap": + case "imaps": + case "pop3": + case "pop3s": + case "smtp": + case "smtps": + component = "mail"; + break; + case "ftp": + case "sftp": + case "ftps": + component = "ftp"; + break; } String name = component + "-component"; if (!docs.contains(name) && (!component.equalsIgnoreCase("salesforce") && !component.equalsIgnoreCase("servicenow"))) { @@ -1136,27 +894,27 @@ public class PrepareCatalogMojo extends AbstractMojo { getLog().info("================================================================================"); } - private void printModelsReport(Set<File> json, Set<File> duplicate, Set<File> missingLabels, Map<String, Set<String>> usedLabels, Set<File> missingJavaDoc) { + private void printModelsReport(Set<Path> json, Set<Path> duplicate, Set<Path> missingLabels, Map<String, Set<String>> usedLabels, Set<Path> missingJavaDoc) { getLog().info("================================================================================"); getLog().info(""); getLog().info("Camel model catalog report"); getLog().info(""); getLog().info("\tModels found: " + json.size()); - for (File file : json) { + for (Path file : json) { getLog().info("\t\t" + asComponentName(file)); } if (!duplicate.isEmpty()) { getLog().info(""); getLog().warn("\tDuplicate models detected: " + duplicate.size()); - for (File file : duplicate) { + for (Path file : duplicate) { getLog().warn("\t\t" + asComponentName(file)); } } if (!missingLabels.isEmpty()) { getLog().info(""); getLog().warn("\tMissing labels detected: " + missingLabels.size()); - for (File file : missingLabels) { + for (Path file : missingLabels) { getLog().warn("\t\t" + asComponentName(file)); } } @@ -1173,7 +931,7 @@ public class PrepareCatalogMojo extends AbstractMojo { if (!missingJavaDoc.isEmpty()) { getLog().info(""); getLog().warn("\tMissing javadoc on models: " + missingJavaDoc.size()); - for (File file : missingJavaDoc) { + for (Path file : missingJavaDoc) { getLog().warn("\t\t" + asComponentName(file)); } } @@ -1181,20 +939,20 @@ public class PrepareCatalogMojo extends AbstractMojo { getLog().info("================================================================================"); } - private void printComponentsReport(Set<File> json, Set<File> duplicate, Set<File> missing, Map<String, Set<String>> usedComponentLabels, Set<String> usedOptionsLabels, - Set<String> unusedLabels, Set<File> missingFirstVersions) { + private void printComponentsReport(Set<Path> json, Set<Path> duplicate, Set<Path> missing, Map<String, Set<String>> usedComponentLabels, + Set<String> usedOptionsLabels, Set<String> unusedLabels, Set<Path> missingFirstVersions) { getLog().info("================================================================================"); getLog().info(""); getLog().info("Camel component catalog report"); getLog().info(""); getLog().info("\tComponents found: " + json.size()); - for (File file : json) { + for (Path file : json) { getLog().info("\t\t" + asComponentName(file)); } if (!duplicate.isEmpty()) { getLog().info(""); getLog().warn("\tDuplicate components detected: " + duplicate.size()); - for (File file : duplicate) { + for (Path file : duplicate) { getLog().warn("\t\t" + asComponentName(file)); } } @@ -1225,34 +983,34 @@ public class PrepareCatalogMojo extends AbstractMojo { if (!missing.isEmpty()) { getLog().info(""); getLog().warn("\tMissing components detected: " + missing.size()); - for (File name : missing) { - getLog().warn("\t\t" + name.getName()); + for (Path name : missing) { + getLog().warn("\t\t" + name.getFileName().toString()); } } if (!missingFirstVersions.isEmpty()) { getLog().info(""); getLog().warn("\tComponents without firstVersion defined: " + missingFirstVersions.size()); - for (File name : missingFirstVersions) { - getLog().warn("\t\t" + name.getName()); + for (Path name : missingFirstVersions) { + getLog().warn("\t\t" + name.getFileName().toString()); } } getLog().info(""); getLog().info("================================================================================"); } - private void printDataFormatsReport(Set<File> json, Set<File> duplicate, Map<String, Set<String>> usedLabels, Set<File> missingFirstVersions) { + private void printDataFormatsReport(Set<Path> json, Set<Path> duplicate, Map<String, Set<String>> usedLabels, Set<Path> missingFirstVersions) { getLog().info("================================================================================"); getLog().info(""); getLog().info("Camel data format catalog report"); getLog().info(""); getLog().info("\tDataFormats found: " + json.size()); - for (File file : json) { + for (Path file : json) { getLog().info("\t\t" + asComponentName(file)); } if (!duplicate.isEmpty()) { getLog().info(""); getLog().warn("\tDuplicate dataformat detected: " + duplicate.size()); - for (File file : duplicate) { + for (Path file : duplicate) { getLog().warn("\t\t" + asComponentName(file)); } } @@ -1269,27 +1027,27 @@ public class PrepareCatalogMojo extends AbstractMojo { if (!missingFirstVersions.isEmpty()) { getLog().info(""); getLog().warn("\tDataFormats without firstVersion defined: " + missingFirstVersions.size()); - for (File name : missingFirstVersions) { - getLog().warn("\t\t" + name.getName()); + for (Path name : missingFirstVersions) { + getLog().warn("\t\t" + name.getFileName().toString()); } } getLog().info(""); getLog().info("================================================================================"); } - private void printLanguagesReport(Set<File> json, Set<File> duplicate, Map<String, Set<String>> usedLabels, Set<File> missingFirstVersions) { + private void printLanguagesReport(Set<Path> json, Set<Path> duplicate, Map<String, Set<String>> usedLabels, Set<Path> missingFirstVersions) { getLog().info("================================================================================"); getLog().info(""); getLog().info("Camel language catalog report"); getLog().info(""); getLog().info("\tLanguages found: " + json.size()); - for (File file : json) { + for (Path file : json) { getLog().info("\t\t" + asComponentName(file)); } if (!duplicate.isEmpty()) { getLog().info(""); getLog().warn("\tDuplicate language detected: " + duplicate.size()); - for (File file : duplicate) { + for (Path file : duplicate) { getLog().warn("\t\t" + asComponentName(file)); } } @@ -1306,27 +1064,27 @@ public class PrepareCatalogMojo extends AbstractMojo { if (!missingFirstVersions.isEmpty()) { getLog().info(""); getLog().warn("\tLanguages without firstVersion defined: " + missingFirstVersions.size()); - for (File name : missingFirstVersions) { - getLog().warn("\t\t" + name.getName()); + for (Path name : missingFirstVersions) { + getLog().warn("\t\t" + name.getFileName().toString()); } } getLog().info(""); getLog().info("================================================================================"); } - private void printOthersReport(Set<File> json, Set<File> duplicate, Map<String, Set<String>> usedLabels, Set<File> missingFirstVersions) { + private void printOthersReport(Set<Path> json, Set<Path> duplicate, Map<String, Set<String>> usedLabels, Set<Path> missingFirstVersions) { getLog().info("================================================================================"); getLog().info(""); getLog().info("Camel other catalog report"); getLog().info(""); getLog().info("\tOthers found: " + json.size()); - for (File file : json) { + for (Path file : json) { getLog().info("\t\t" + asComponentName(file)); } if (!duplicate.isEmpty()) { getLog().info(""); getLog().warn("\tDuplicate other detected: " + duplicate.size()); - for (File file : duplicate) { + for (Path file : duplicate) { getLog().warn("\t\t" + asComponentName(file)); } } @@ -1343,27 +1101,27 @@ public class PrepareCatalogMojo extends AbstractMojo { if (!missingFirstVersions.isEmpty()) { getLog().info(""); getLog().warn("\tOthers without firstVersion defined: " + missingFirstVersions.size()); - for (File name : missingFirstVersions) { - getLog().warn("\t\t" + name.getName()); + for (Path name : missingFirstVersions) { + getLog().warn("\t\t" + name.getFileName().toString()); } } getLog().info(""); getLog().info("================================================================================"); } - private void printDocumentsReport(Set<File> docs, Set<File> duplicate, Set<File> missing) { + private void printDocumentsReport(Set<Path> docs, Set<Path> duplicate, Set<Path> missing) { getLog().info("================================================================================"); getLog().info(""); getLog().info("Camel document catalog report"); getLog().info(""); getLog().info("\tDocuments found: " + docs.size()); - for (File file : docs) { + for (Path file : docs) { getLog().info("\t\t" + asComponentName(file)); } if (!duplicate.isEmpty()) { getLog().info(""); getLog().warn("\tDuplicate document detected: " + duplicate.size()); - for (File file : duplicate) { + for (Path file : duplicate) { getLog().warn("\t\t" + asComponentName(file)); } } @@ -1371,236 +1129,138 @@ public class PrepareCatalogMojo extends AbstractMojo { if (!missing.isEmpty()) { getLog().info(""); getLog().warn("\tMissing document detected: " + missing.size()); - for (File name : missing) { - getLog().warn("\t\t" + name.getName()); + for (Path name : missing) { + getLog().warn("\t\t" + name.getFileName().toString()); } } getLog().info(""); getLog().info("================================================================================"); } - private static String asComponentName(File file) { - String name = file.getName(); - if (name.endsWith(PackageHelper.JSON_SUFIX) || name.endsWith(".adoc")) { - return name.substring(0, name.length() - 5); + private static String asComponentName(Path file) { + String name = file.getFileName().toString(); + if (name.endsWith(PackageHelper.JSON_SUFIX)) { + return name.substring(0, name.length() - PackageHelper.JSON_SUFIX.length()); + } else if (name.endsWith(".adoc")) { + return name.substring(0, name.length() - ".adoc".length()); } return name; } - private void findComponentFilesRecursive(File dir, Set<File> found, Set<File> components, FileFilter filter) { - File[] files = dir.listFiles(filter); - if (files != null) { - for (File file : files) { - // skip files in root dirs as Camel does not store information - // there but others may do - boolean rootDir = "classes".equals(dir.getName()) || "META-INF".equals(dir.getName()); - boolean jsonFile = !rootDir && file.isFile() && file.getName().endsWith(PackageHelper.JSON_SUFIX); - boolean componentFile = !rootDir && file.isFile() && file.getName().equals("component.properties"); - if (jsonFile) { - found.add(file); - } else if (componentFile) { - components.add(file); - } else if (file.isDirectory()) { - findComponentFilesRecursive(file, found, components, filter); - } - } - } - } - - private void findDataFormatFilesRecursive(File dir, Set<File> found, Set<File> dataFormats, FileFilter filter) { - File[] files = dir.listFiles(filter); - if (files != null) { - for (File file : files) { - // skip files in root dirs as Camel does not store information - // there but others may do - boolean rootDir = "classes".equals(dir.getName()) || "META-INF".equals(dir.getName()); - boolean jsonFile = !rootDir && file.isFile() && file.getName().endsWith(PackageHelper.JSON_SUFIX); - boolean dataFormatFile = !rootDir && file.isFile() && file.getName().equals("dataformat.properties"); - if (jsonFile) { - found.add(file); - } else if (dataFormatFile) { - dataFormats.add(file); - } else if (file.isDirectory()) { - findDataFormatFilesRecursive(file, found, dataFormats, filter); - } - } - } - } - - private void findLanguageFilesRecursive(File dir, Set<File> found, Set<File> languages, FileFilter filter) { - File[] files = dir.listFiles(filter); - if (files != null) { - for (File file : files) { - // skip files in root dirs as Camel does not store information - // there but others may do - boolean rootDir = "classes".equals(dir.getName()) || "META-INF".equals(dir.getName()); - boolean jsonFile = !rootDir && file.isFile() && file.getName().endsWith(PackageHelper.JSON_SUFIX); - boolean languageFile = !rootDir && file.isFile() && file.getName().equals("language.properties"); - if (jsonFile) { - found.add(file); - } else if (languageFile) { - languages.add(file); - } else if (file.isDirectory()) { - findLanguageFilesRecursive(file, found, languages, filter); - } + private void copyFile(Path file, Path toDir) throws IOException, MojoFailureException { + if (Files.isRegularFile(file)) { + // make sure to create out dir + Files.createDirectories(toDir); + // copy the file + Path to = toDir.resolve(file.getFileName()); + try { + FileUtil.updateFile(file, to); + } catch (IOException e) { + throw new MojoFailureException("Cannot copy file from " + file + " -> " + to, e); } } } - private void findOtherFilesRecursive(File dir, Set<File> found, Set<File> others, FileFilter filter) { - File[] files = dir.listFiles(filter); - if (files != null) { - for (File file : files) { - // skip files in root dirs as Camel does not store information - // there but others may do - boolean rootDir = "classes".equals(dir.getName()) || "META-INF".equals(dir.getName()); - boolean jsonFile = rootDir && file.isFile() && file.getName().endsWith(PackageHelper.JSON_SUFIX); - boolean otherFile = !rootDir && file.isFile() && file.getName().equals("other.properties"); - if (jsonFile) { - found.add(file); - } else if (otherFile) { - others.add(file); - } else if (file.isDirectory()) { - findOtherFilesRecursive(file, found, others, filter); - } + private static boolean excludeDocumentDir(String name) { + for (String exclude : EXCLUDE_DOC_FILES) { + if (exclude.equals(name)) { + return true; } } + return false; } - private void findAsciiDocFilesRecursive(File dir, Set<File> found, FileFilter filter) { - File[] files = dir.listFiles(filter); - if (files != null) { - for (File file : files) { - // skip files in root dirs as Camel does not store information - // there but others may do - boolean rootDir = "classes".equals(dir.getName()) || "META-INF".equals(dir.getName()); - boolean adocFile = !rootDir && file.isFile() && file.getName().endsWith(".adoc"); - if (adocFile) { - found.add(file); - } else if (file.isDirectory()) { - findAsciiDocFilesRecursive(file, found, filter); - } - } - } + private List<Path> concat(List<Path> l1, List<Path> l2) { + return Stream.concat(l1.stream(), l2.stream()).collect(Collectors.toList()); } - private class CamelComponentsFileFilter implements FileFilter { + // CHECKSTYLE:ON - @Override - public boolean accept(File pathname) { - if (pathname.isDirectory() && pathname.getName().equals("model")) { - // do not check the camel-core model packages as there is no - // components there - return false; - } - if (pathname.isFile() && pathname.getName().endsWith(PackageHelper.JSON_SUFIX)) { - // must be a components json file - try { - String json = PackageHelper.loadText(pathname); - return "component".equals(PackageHelper.getSchemaKind(json)); - } catch (IOException e) { - // ignore - } + private Stream<Path> list(Path dir) { + try { + if (Files.isDirectory(dir)) { + return Files.list(dir); + } else { + return Stream.empty(); } - return pathname.isDirectory() || (pathname.isFile() && pathname.getName().equals("component.properties")); + } catch (IOException e) { + throw new RuntimeException("Unable to list files in directory: " + dir, e); } } - private class CamelDataFormatsFileFilter implements FileFilter { - - @Override - public boolean accept(File pathname) { - if (pathname.isDirectory() && pathname.getName().equals("model")) { - // do not check the camel-core model packages as there is no - // components there - return false; - } - if (pathname.isFile() && pathname.getName().endsWith(PackageHelper.JSON_SUFIX)) { - // must be a dataformat json file - try { - String json = PackageHelper.loadText(pathname); - return "dataformat".equals(PackageHelper.getSchemaKind(json)); - } catch (IOException e) { - // ignore - } - } - return pathname.isDirectory() || (pathname.isFile() && pathname.getName().equals("dataformat.properties")); + private void delete(Path dir) { + try { + Files.delete(dir); + } catch (IOException e) { + throw new RuntimeException("Unable to delete file: " + dir, e); } } - private class CamelLanguagesFileFilter implements FileFilter { - - @Override - public boolean accept(File pathname) { - if (pathname.isDirectory() && pathname.getName().equals("model")) { - // do not check the camel-core model packages as there is no - // components there - return false; - } - if (pathname.isFile() && pathname.getName().endsWith(PackageHelper.JSON_SUFIX)) { - // must be a language json file - try { - String json = PackageHelper.loadText(pathname); - return "language".equals(PackageHelper.getSchemaKind(json)); - } catch (IOException e) { - // ignore + private void copy(Path file, Path to) { + try { + try { + BasicFileAttributes af = Files.readAttributes(file, BasicFileAttributes.class); + BasicFileAttributes at = Files.readAttributes(to, BasicFileAttributes.class); + if (af.isRegularFile() && at.isRegularFile() + && af.size() == at.size() + && af.lastModifiedTime().compareTo(at.lastAccessTime()) < 0) { + // if same size and not modified, assume the same + return; } + } catch (IOException e) { + // ignore and copy } - return pathname.isDirectory() || (pathname.isFile() && pathname.getName().equals("language.properties")); + FileUtil.updateFile(file, to); + } catch (IOException e) { + throw new RuntimeException("Cannot copy file from " + file + " -> " + to, e); } } - private class CamelOthersFileFilter implements FileFilter { - - @Override - public boolean accept(File pathname) { - if (pathname.isFile() && pathname.getName().endsWith(PackageHelper.JSON_SUFIX)) { - // must be a language json file - try { - String json = PackageHelper.loadText(pathname); - return "other".equals(PackageHelper.getSchemaKind(json)); - } catch (IOException e) { - // ignore - } - } - return pathname.isDirectory() || (pathname.isFile() && pathname.getName().equals("other.properties")); - } + private <U, K, V> Map<K, V> map(Collection<U> col, Function<U, K> key, Function<U, V> value) { + return col.stream().collect(Collectors.toMap(key, value)); } - private class CamelAsciiDocFileFilter implements FileFilter { - - @Override - public boolean accept(File pathname) { - return pathname.isFile() && pathname.getName().endsWith(".adoc"); - } + private <U, K, V> Map<K, V> map(Collection<U> col, Function<U, K> key, Function<U, V> value, BinaryOperator<V> merger) { + return col.stream().collect(Collectors.toMap(key, value, merger)); } - public static Set<String> generateJsonList(Path outDir, String outFile) throws MojoFailureException { - Set<String> answer; - Path all = outDir.resolve(outFile); - try { - answer = Files.list(outDir).filter(p -> p.getFileName().toString().endsWith(PackageHelper.JSON_SUFIX)).map(p -> p.getFileName().toString()) - // strip out .json from the name - .map(n -> n.substring(0, n.length() - PackageHelper.JSON_SUFIX.length())).sorted().collect(LinkedHashSet::new, LinkedHashSet::add, LinkedHashSet::addAll); - String data = String.join("\n", answer) + "\n"; - FileUtil.updateFile(all, data); - return answer; - } catch (IOException e) { - throw new MojoFailureException("Error writing to file " + all); + private Path getModule(Path p) { + Path parent = p; + while (!parent.endsWith("target")) { + parent = parent.getParent(); } + return parent.getParent(); } - public static void copyFile(File from, File to) throws IOException { - FileUtil.updateFile(from.toPath(), to.toPath()); + private Set<Path> getDuplicates(Set<Path> jsonFiles) { + Map<String, List<Path>> byName = map(jsonFiles, + PrepareCatalogMojo::asComponentName, // key by component name + Collections::singletonList, // value as a singleton list + this::concat); // merge lists + return byName.values().stream() + .flatMap(l -> l.stream().skip(1)) + .collect(Collectors.toCollection(TreeSet::new)); } - private static boolean excludeDocumentDir(String name) { - for (String exclude : EXCLUDE_DOC_FILES) { - if (exclude.equals(name)) { - return true; - } + private Path getComponentPath(Path dir) { + switch (dir.getFileName().toString()) { + case "camel-as2": + return dir.resolve("camel-as2-component"); + case "camel-salesforce": + return dir.resolve("camel-salesforce-component"); + case "camel-olingo2": + return dir.resolve("camel-olingo2-component"); + case "camel-olingo4": + return dir.resolve("camel-olingo4-component"); + case "camel-box": + return dir.resolve("camel-box-component"); + case "camel-servicenow": + return dir.resolve("camel-servicenow-component"); + case "camel-fhir": + return dir.resolve("camel-fhir-component"); + default: + return dir; } - return false; } } diff --git a/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/PrepareUserGuideMojo.java b/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/PrepareUserGuideMojo.java index ef66838..c0f717a 100644 --- a/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/PrepareUserGuideMojo.java +++ b/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/PrepareUserGuideMojo.java @@ -183,37 +183,36 @@ public class PrepareUserGuideMojo extends AbstractMojo { } } - try { - List<OtherModel> models = new ArrayList<>(); - for (File file : otherFiles) { - String json = PackageHelper.loadText(file); + List<OtherModel> models = new ArrayList<>(); + for (File file : otherFiles) { + try { + String json = PackageHelper.loadText(file); OtherModel model = JsonMapper.generateOtherModel(json); models.add(model); + } catch (Exception e) { + throw new MojoFailureException("Error reading file: " + file, e); } + } - // sort the models - models.sort(BaseModel.compareTitle()); - - // the summary file has the TOC - File file = new File(userGuideDir, "SUMMARY.md"); + // sort the models + models.sort(BaseModel.compareTitle()); - // update core components - StringBuilder other = new StringBuilder(); - other.append("* Miscellaneous Components\n"); - for (OtherModel model : models) { - String line = "\t* " + link(model) + "\n"; - other.append(line); - } - boolean updated = updateOthers(file, other.toString()); + // the summary file has the TOC + File file = new File(userGuideDir, "SUMMARY.md"); - if (updated) { - getLog().info("Updated user guide file: " + file); - } else { - getLog().debug("No changes to user guide file: " + file); - } + // update core components + StringBuilder other = new StringBuilder(); + other.append("* Miscellaneous Components\n"); + for (OtherModel model : models) { + String line = "\t* " + link(model) + "\n"; + other.append(line); + } + boolean updated = updateOthers(file, other.toString()); - } catch (IOException e) { - throw new MojoFailureException("Error due " + e.getMessage(), e); + if (updated) { + getLog().info("Updated user guide file: " + file); + } else { + getLog().debug("No changes to user guide file: " + file); } } diff --git a/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/ValidateHelper.java b/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/ValidateHelper.java index a84bbe9..b5a6bc1 100644 --- a/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/ValidateHelper.java +++ b/tooling/maven/camel-package-maven-plugin/src/main/java/org/apache/camel/maven/packaging/ValidateHelper.java @@ -18,12 +18,12 @@ package org.apache.camel.maven.packaging; import java.io.File; import java.io.IOException; -import java.util.List; import java.util.Map; -import org.apache.camel.tooling.util.JSonSchemaHelper; import org.apache.camel.tooling.util.PackageHelper; -import org.apache.camel.tooling.util.Strings; +import org.apache.camel.util.json.DeserializationException; +import org.apache.camel.util.json.JsonObject; +import org.apache.camel.util.json.Jsoner; /** * Validation helper for validating components, data formats and languages @@ -42,89 +42,50 @@ public final class ValidateHelper { public static void validate(File file, ErrorDetail errorDetail) { try { String json = PackageHelper.loadText(file); + JsonObject obj = (JsonObject) Jsoner.deserialize(json); - String kind = PackageHelper.getSchemaKind(json); - boolean isComponent = "component".equals(kind); - boolean isDataFormat = "dataformat".equals(kind); - boolean isLanguage = "language".equals(kind); + Map<String, Object> model; + boolean isComponent = (model = obj.getMap("component")) != null; + boolean isDataFormat = !isComponent && (model = obj.getMap("dataformat")) != null; + boolean isLanguage = !isComponent && !isDataFormat && (model = obj.getMap("language")) != null; // only check these kind if (!isComponent && !isDataFormat && !isLanguage) { return; } + errorDetail.setKind((String) model.get("kind")); + errorDetail.setMissingDescription(isNullOrEmpty(model.get("description"))); + errorDetail.setMissingLabel(isNullOrEmpty(model.get("label"))); if (isComponent) { - errorDetail.setKind("component"); - } else if (isDataFormat) { - errorDetail.setKind("dataformat"); - } else if (isLanguage) { - errorDetail.setKind("language"); - } - - List<Map<String, String>> rows = JSonSchemaHelper.parseJsonSchema(errorDetail.getKind(), json, false); - boolean label = false; - boolean description = false; - boolean syntax = false; - for (Map<String, String> row : rows) { - String value = row.get("label"); - if (!Strings.isEmpty(value)) { - label = true; - } - value = row.get("description"); - if (!Strings.isEmpty(value)) { - description = true; - } - value = row.get("syntax"); - if (!Strings.isEmpty(value)) { - syntax = true; - } - } - - if (!label) { - errorDetail.setMissingLabel(true); - } - - if (!description) { - errorDetail.setMissingDescription(true); - } - - // syntax check is only for the components - if (!syntax && isComponent) { - errorDetail.setMissingSyntax(true); - } - - if (isComponent) { - // check all the component properties if they have description - rows = JSonSchemaHelper.parseJsonSchema("componentProperties", json, true); - for (Map<String, String> row : rows) { - String key = row.get("name"); - String doc = row.get("description"); - if (doc == null || doc.isEmpty()) { - errorDetail.addMissingComponentDoc(key); + errorDetail.setMissingSyntax(isNullOrEmpty(model.get("syntax"))); + Map<String, Object> componentProps = obj.getMap("componentProperties"); + for (Map.Entry<String, Object> entry : componentProps.entrySet()) { + if (isNullOrEmpty(((JsonObject) entry.getValue()).get("description"))) { + errorDetail.addMissingComponentDoc(entry.getKey()); } } } - - // check all the endpoint properties if they have description - rows = JSonSchemaHelper.parseJsonSchema("properties", json, true); + Map<String, Object> props = obj.getMap("properties"); boolean path = false; - for (Map<String, String> row : rows) { - String key = row.get("name"); - String doc = row.get("description"); - if (doc == null || doc.isEmpty()) { - errorDetail.addMissingEndpointDoc(key); + for (Map.Entry<String, Object> entry : props.entrySet()) { + JsonObject value = (JsonObject) entry.getValue(); + if (isNullOrEmpty(value.get("description"))) { + errorDetail.addMissingEndpointDoc(entry.getKey()); } - if ("path".equals(row.get("kind"))) { - path = true; - } - } - if (isComponent && !path) { - // only components can have missing @UriPath - errorDetail.setMissingUriPath(true); + path |= "path".equals(value.get("kind")); } - } catch (IOException e) { + errorDetail.setMissingUriPath(isComponent && !path); + } catch (DeserializationException e) { + // wrap parsing exceptions as runtime + throw new RuntimeException("Cannot parse json", e); + } catch (IOException e) { // ignore } } + private static boolean isNullOrEmpty(Object obj) { + return obj == null || "".equals(obj); + } + }
