github-code-scanning[bot] commented on code in PR #1589: URL: https://github.com/apache/avro/pull/1589#discussion_r1289837430
########## lang/java/idl/src/main/java/org/apache/avro/idl/IdlReader.java: ########## @@ -0,0 +1,1072 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.idl; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Deque; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.BooleanNode; +import com.fasterxml.jackson.databind.node.DoubleNode; +import com.fasterxml.jackson.databind.node.IntNode; +import com.fasterxml.jackson.databind.node.LongNode; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.Token; +import org.apache.avro.JsonProperties; +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Protocol; +import org.apache.avro.Schema; +import org.apache.avro.SchemaParseException; +import org.apache.avro.idl.IdlParser.ArrayTypeContext; +import org.apache.avro.idl.IdlParser.EnumDeclarationContext; +import org.apache.avro.idl.IdlParser.EnumSymbolContext; +import org.apache.avro.idl.IdlParser.FieldDeclarationContext; +import org.apache.avro.idl.IdlParser.FixedDeclarationContext; +import org.apache.avro.idl.IdlParser.FormalParameterContext; +import org.apache.avro.idl.IdlParser.FullTypeContext; +import org.apache.avro.idl.IdlParser.IdentifierContext; +import org.apache.avro.idl.IdlParser.IdlFileContext; +import org.apache.avro.idl.IdlParser.ImportStatementContext; +import org.apache.avro.idl.IdlParser.JsonArrayContext; +import org.apache.avro.idl.IdlParser.JsonLiteralContext; +import org.apache.avro.idl.IdlParser.JsonObjectContext; +import org.apache.avro.idl.IdlParser.JsonPairContext; +import org.apache.avro.idl.IdlParser.JsonValueContext; +import org.apache.avro.idl.IdlParser.MapTypeContext; +import org.apache.avro.idl.IdlParser.MessageDeclarationContext; +import org.apache.avro.idl.IdlParser.NamespaceDeclarationContext; +import org.apache.avro.idl.IdlParser.NullableTypeContext; +import org.apache.avro.idl.IdlParser.PrimitiveTypeContext; +import org.apache.avro.idl.IdlParser.ProtocolDeclarationBodyContext; +import org.apache.avro.idl.IdlParser.ProtocolDeclarationContext; +import org.apache.avro.idl.IdlParser.RecordBodyContext; +import org.apache.avro.idl.IdlParser.RecordDeclarationContext; +import org.apache.avro.idl.IdlParser.ResultTypeContext; +import org.apache.avro.idl.IdlParser.SchemaPropertyContext; +import org.apache.avro.idl.IdlParser.UnionTypeContext; +import org.apache.avro.idl.IdlParser.VariableDeclarationContext; +import org.apache.avro.util.internal.Accessor; +import org.apache.commons.text.StringEscapeUtils; + +import static java.util.Collections.singleton; +import static java.util.Collections.unmodifiableMap; + +public class IdlReader { + /** + * Simple error listener. Throws a runtime exception because ANTLR does not give + * easy access to the (reasonably readable) error message elsewhere. + */ + private static final BaseErrorListener SIMPLE_AVRO_ERROR_LISTENER = new BaseErrorListener() { + @Override + public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, + String msg, RecognitionException e) { + throw new SchemaParseException("line " + line + ":" + charPositionInLine + " " + msg); + } + }; + private static final String OPTIONAL_NULLABLE_TYPE_PROPERTY = "org.apache.avro.idl.Idl.NullableType.optional"; + /** + * Pattern to match the common whitespace indents in a multi-line String. + * Doesn't match a single-line String, fully matches any multi-line String. + * <p> + * To use: match on a {@link String#trim() trimmed} String, and then replace all + * newlines followed by the group "indent" with a newline. + */ + private static final Pattern WS_INDENT = Pattern.compile("(?U).*\\R(?<indent>\\h*).*(?:\\R\\k<indent>.*)*"); + /** + * Pattern to match the whitespace indents plus common stars (1 or 2) in a + * multi-line String. If a String fully matches, replace all occurrences of a + * newline followed by whitespace and then the group "stars" with a newline. + * <p> + * Note: partial matches are invalid. + */ + private static final Pattern STAR_INDENT = Pattern.compile("(?U)(?<stars>\\*{1,2}).*(?:\\R\\h*\\k<stars>.*)*"); + /** + * Predicate to check for valid names. Should probably be delegated to the + * Schema class. + */ + private static final Predicate<String> VALID_NAME = Pattern.compile("[_\\p{L}][_\\p{L}\\p{Digit}]*").asPredicate(); + private static final Set<String> INVALID_TYPE_NAMES = new HashSet<>(Arrays.asList("boolean", "int", "long", "float", + "double", "bytes", "string", "null", "date", "time_ms", "timestamp_ms", "localtimestamp_ms", "uuid")); + private static final String CLASSPATH_SCHEME = "classpath"; + + private final Set<URI> readLocations; + private final Map<String, Schema> names; + + public IdlReader() { + readLocations = new HashSet<>(); + names = new LinkedHashMap<>(); + } + + public Map<String, Schema> getTypes() { + return unmodifiableMap(names); + } + + private Schema namedSchemaOrUnresolved(String fullName) { + Schema schema = names.get(fullName); + if (schema == null) { + schema = SchemaResolver.unresolvedSchema(fullName); + } + return schema; + } + + private void setTypes(Map<String, Schema> types) { + names.clear(); + for (Schema schema : types.values()) { + addSchema(schema); + } + } + + public void addTypes(Map<String, Schema> types) { + for (Schema schema : types.values()) { + addSchema(schema); + } + } + + private void addSchema(Schema schema) { + String fullName = schema.getFullName(); + if (names.containsKey(fullName)) { + throw new SchemaParseException("Can't redefine: " + fullName); + } + names.put(fullName, schema); + } + + public IdlFile parse(Path location) throws IOException { + return parse(location.toUri()); + } + + IdlFile parse(URI location) throws IOException { + try (InputStream stream = location.toURL().openStream()) { + readLocations.add(location); + URI inputDir = location; + if ("jar".equals(location.getScheme())) { + String jarUriAsString = location.toString(); + String pathFromJarRoot = jarUriAsString.substring(jarUriAsString.indexOf("!/") + 2); + inputDir = URI.create(CLASSPATH_SCHEME + ":/" + pathFromJarRoot); + } + inputDir = inputDir.resolve("."); + + return parse(inputDir, CharStreams.fromStream(stream, StandardCharsets.UTF_8)); + } + } + + /** + * Parse an IDL file from a stream. This method cannot handle imports. + */ + public IdlFile parse(InputStream stream) throws IOException { + return parse(null, CharStreams.fromStream(stream, StandardCharsets.UTF_8)); + } + + private IdlFile parse(URI inputDir, CharStream charStream) { + IdlLexer lexer = new IdlLexer(charStream); + CommonTokenStream tokenStream = new CommonTokenStream(lexer); + + IdlParserListener parseListener = new IdlParserListener(inputDir, tokenStream); + + IdlParser parser = new IdlParser(tokenStream); + parser.removeErrorListeners(); + parser.addErrorListener(SIMPLE_AVRO_ERROR_LISTENER); + parser.addParseListener(parseListener); + parser.setTrace(false); + parser.setBuildParseTree(false); + + try { + // Trigger parsing. + parser.idlFile(); + } catch (RuntimeException e) { + e.printStackTrace(); + throw e; + } + + return parseListener.getIdlFile(); + } + + /* Package private to facilitate testing */ + static String stripIndents(String docComment) { + Matcher starMatcher = STAR_INDENT.matcher(docComment); + if (starMatcher.matches()) { + return docComment.replaceAll("(?U)(?:^|(\\R)\\h*)\\Q" + starMatcher.group("stars") + "\\E\\h?", "$1"); + } + + Matcher whitespaceMatcher = WS_INDENT.matcher(docComment); + if (whitespaceMatcher.matches()) { + return docComment.replaceAll("(?U)(\\R)" + whitespaceMatcher.group("indent"), "$1"); + } + + return docComment; + } + + private static SchemaParseException error(String message, Token token) { + return error(message, token, null); + } + + private static SchemaParseException error(String message, Token token, Throwable cause) { + SchemaParseException exception = new SchemaParseException( + message + ", at line " + token.getLine() + ", column " + token.getCharPositionInLine()); + if (cause != null) { + exception.initCause(cause); + } + return exception; + } + + private class IdlParserListener extends IdlBaseListener { + private final URI inputDir; + private final CommonTokenStream tokenStream; + private int hiddenTokensProcessedIndex; + private final List<String> warnings; + + private IdlFile result; + private Schema mainSchema; + private Protocol protocol; + private final Deque<String> namespaces; + private final List<String> enumSymbols; + private String enumDefaultSymbol; + private Schema schema; + private String defaultVariableDocComment; + private final List<Schema.Field> fields; + private final Deque<Schema> typeStack; + private final Deque<JsonNode> jsonValues; + private final Deque<SchemaProperties> propertiesStack; + private String messageDocComment; + + public IdlParserListener(URI inputDir, CommonTokenStream tokenStream) { + this.inputDir = inputDir; + this.tokenStream = tokenStream; + hiddenTokensProcessedIndex = -1; + warnings = new ArrayList<>(); + + result = null; + mainSchema = null; + protocol = null; + namespaces = new ArrayDeque<>(); + enumSymbols = new ArrayList<>(); + enumDefaultSymbol = null; + schema = null; + defaultVariableDocComment = null; + fields = new ArrayList<>(); + typeStack = new ArrayDeque<>(); + propertiesStack = new ArrayDeque<>(); + jsonValues = new ArrayDeque<>(); + messageDocComment = null; + } + + public IdlFile getIdlFile() { + return result; + } + + private String getDocComment(ParserRuleContext ctx) { + int newHiddenTokensProcessedIndex = ctx.start.getTokenIndex(); + List<Token> docCommentTokens = tokenStream.getHiddenTokensToLeft(newHiddenTokensProcessedIndex, -1); + int searchEndIndex = newHiddenTokensProcessedIndex; + + Token docCommentToken = null; + if (docCommentTokens != null) { + // There's at least one element + docCommentToken = docCommentTokens.get(docCommentTokens.size() - 1); + searchEndIndex = docCommentToken.getTokenIndex() - 1; + } + + Set<Integer> allHiddenTokens = singleton(IdlParser.DocComment); + if (searchEndIndex >= 0) { + List<Token> hiddenTokens = tokenStream.getTokens(hiddenTokensProcessedIndex + 1, searchEndIndex, + allHiddenTokens); + if (hiddenTokens != null) { + for (Token token : hiddenTokens) { + warnings.add(String.format( + "Line %d, char %d: Ignoring out-of-place documentation comment.%n" + + "Did you mean to use a multiline comment ( /* ... */ ) instead?", + token.getLine(), token.getCharPositionInLine() + 1)); + } + } + } + hiddenTokensProcessedIndex = newHiddenTokensProcessedIndex; + + if (docCommentToken == null) { + return null; + } + String comment = docCommentToken.getText(); + String text = comment.substring(3, comment.length() - 2); // Strip /** & */ + return stripIndents(text.trim()); + } + + private void pushNamespace(String namespace) { + namespaces.push(namespace == null ? "" : namespace); + } + + private String currentNamespace() { + String namespace = namespaces.peek(); + return namespace == null || namespace.isEmpty() ? null : namespace; + } + + private void popNamespace() { + namespaces.pop(); + } + + @Override + public void exitIdlFile(IdlFileContext ctx) { + IdlFile unresolved; + if (protocol == null) { + unresolved = new IdlFile(currentNamespace(), mainSchema, getTypes().values(), warnings); + } else { + unresolved = new IdlFile(protocol, warnings); + } + result = SchemaResolver.resolve(unresolved, OPTIONAL_NULLABLE_TYPE_PROPERTY); + } + + @Override + public void enterProtocolDeclaration(ProtocolDeclarationContext ctx) { + propertiesStack.push(new SchemaProperties(null, true, false, false)); + } + + @Override + public void enterProtocolDeclarationBody(ProtocolDeclarationBodyContext ctx) { + ProtocolDeclarationContext protocolCtx = (ProtocolDeclarationContext) ctx.parent; + SchemaProperties properties = propertiesStack.pop(); + String protocolIdentifier = identifier(protocolCtx.name); + pushNamespace(namespace(protocolIdentifier, properties.namespace())); + + String protocolName = name(protocolIdentifier); + String docComment = getDocComment(protocolCtx); + String protocolNamespace = currentNamespace(); + protocol = properties.copyProperties(new Protocol(protocolName, docComment, protocolNamespace)); + } + + @Override + public void exitProtocolDeclaration(ProtocolDeclarationContext ctx) { + if (protocol != null) + protocol.setTypes(getTypes().values()); + if (!namespaces.isEmpty()) + popNamespace(); + } + + @Override + public void exitNamespaceDeclaration(NamespaceDeclarationContext ctx) { + pushNamespace(namespace("", identifier(ctx.namespace))); + } + + @Override + public void exitMainSchemaDeclaration(IdlParser.MainSchemaDeclarationContext ctx) { + mainSchema = typeStack.pop(); + assert typeStack.isEmpty(); + } + + @Override + public void enterSchemaProperty(SchemaPropertyContext ctx) { + assert jsonValues.isEmpty(); + } + + @Override + public void exitSchemaProperty(SchemaPropertyContext ctx) { + String name = identifier(ctx.name); + JsonNode value = jsonValues.pop(); + Token firstToken = ctx.value.start; + + propertiesStack.element().addProperty(name, value, firstToken); + super.exitSchemaProperty(ctx); + } + + @Override + public void exitImportStatement(ImportStatementContext importContext) { + String importFile = getString(importContext.location); + try { + URI importLocation = findImport(importFile); + if (!readLocations.add(importLocation)) { + // Already imported + return; + } + switch (importContext.importType.getType()) { + case IdlParser.IDL: + // Note that the parse(URI) method uses the same known schema collection + IdlFile idlFile = parse(importLocation); + if (protocol != null && idlFile.getProtocol() != null) { + protocol.getMessages().putAll(idlFile.getProtocol().getMessages()); + } + warnings.addAll(idlFile.getWarnings(importFile)); + break; + case IdlParser.Protocol: + try (InputStream stream = importLocation.toURL().openStream()) { + Protocol importProtocol = Protocol.parse(stream); + for (Schema s : importProtocol.getTypes()) { + addSchema(s); + } + if (protocol != null) { + protocol.getMessages().putAll(importProtocol.getMessages()); + } + } + break; + case IdlParser.Schema: + try (InputStream stream = importLocation.toURL().openStream()) { + Schema.Parser parser = new Schema.Parser(); + parser.addTypes(getTypes()); // inherit names Review Comment: ## Deprecated method or constructor invocation Invoking [Parser.addTypes](1) should be avoided because it has been deprecated. [Show more details](https://github.com/apache/avro/security/code-scanning/3103) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
