diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index dcc78a5416ab..eb8a57d639e5 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -359,6 +359,8 @@ class BeamModulePlugin implements Plugin<Project> { activemq_junit : "org.apache.activemq.tooling:activemq-junit:5.13.1", activemq_kahadb_store : "org.apache.activemq:activemq-kahadb-store:5.13.1", activemq_mqtt : "org.apache.activemq:activemq-mqtt:5.13.1", + antlr : "org.antlr:antlr4:4.7", + antlr_runtime : "org.antlr:antlr4-runtime:4.7", apex_common : "org.apache.apex:apex-common:$apex_core_version", apex_engine : "org.apache.apex:apex-engine:$apex_core_version", args4j : "args4j:args4j:2.33", @@ -626,7 +628,7 @@ class BeamModulePlugin implements Plugin<Project> { '-Xlint:all', '-Werror', '-XepDisableWarningsInGeneratedCode', - '-XepExcludedPaths:(.*/)?(build/generated.*avro-java|build/generated)/.*', + '-XepExcludedPaths:(.*/)?(build/generated-src|build/generated.*avro-java|build/generated)/.*', '-Xep:MutableConstantField:OFF' // Guava's immutable collections cannot appear on API surface. ] + (defaultLintSuppressions + configuration.disableLintWarnings).collect { "-Xlint:-${it}" }) @@ -1442,6 +1444,20 @@ class BeamModulePlugin implements Plugin<Project> { // or be left here. project.ext.applyAvroNature = { project.apply plugin: "com.commercehub.gradle.plugin.avro" } + project.ext.applyAntlrNature = { + project.apply plugin: 'antlr' + def generatedDir = "${project.buildDir}/generated/source-src/antlr/main/java/" + project.sourceSets { + generated { java.srcDir generatedDir } + } + project.idea { + module { + sourceDirs += project.file(generatedDir) + generatedSourceDirs += project.file(generatedDir) + } + } + } + // Creates a task to run the quickstart for a runner. // Releases version and URL, can be overriden for a RC release with // ./gradlew :release:runJavaExamplesValidationTask -Pver=2.3.0 -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-1027 diff --git a/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml b/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml index c8aa2f291719..545c2f8cc26c 100644 --- a/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml +++ b/sdks/java/build-tools/src/main/resources/beam/findbugs-filter.xml @@ -410,6 +410,13 @@ <Package name="org.apache.beam.sdk.extensions.sql.impl.parser.impl"/> </Match> + <Match> + <!-- + Classes in this package is auto-generated, let's disable the findbugs for it. + --> + <Package name="org.apache.beam.sdk.schemas.parser.generated"/> + </Match> + <Match> <!-- Classes in this package is auto-generated, let's disable the findbugs for it. diff --git a/sdks/java/build-tools/src/main/resources/beam/suppressions.xml b/sdks/java/build-tools/src/main/resources/beam/suppressions.xml index e3ab90dff46a..d9cdc52b40da 100644 --- a/sdks/java/build-tools/src/main/resources/beam/suppressions.xml +++ b/sdks/java/build-tools/src/main/resources/beam/suppressions.xml @@ -27,6 +27,7 @@ <!-- suppress all checks in the generated directories --> <suppress checks=".*" files=".+[\\\/]generated[\\\/].+\.java" /> + <suppress checks=".*" files=".+[\\\/]generated-src[\\\/].+\.java" /> <suppress checks=".*" files=".+[\\\/]generated-sources[\\\/].+\.java" /> <suppress checks=".*" files=".+[\\\/]generated-test-sources[\\\/].+\.java" /> </suppressions> diff --git a/sdks/java/core/build.gradle b/sdks/java/core/build.gradle index 689efea49ac2..c6d29fc615fd 100644 --- a/sdks/java/core/build.gradle +++ b/sdks/java/core/build.gradle @@ -22,13 +22,20 @@ applyJavaNature(shadowClosure: DEFAULT_SHADOW_CLOSURE << { include(dependency(library.java.protobuf_java)) include(dependency(library.java.byte_buddy)) include(dependency("org.apache.commons:.*")) + include(dependency(library.java.antlr_runtime)) } relocate "com.google.thirdparty", getJavaRelocatedPath("com.google.thirdparty") relocate "com.google.protobuf", getJavaRelocatedPath("com.google.protobuf") relocate "net.bytebuddy", getJavaRelocatedPath("net.bytebuddy") relocate "org.apache.commons", getJavaRelocatedPath("org.apache.commons") + relocate "org.antlr.v4", getJavaRelocatedPath("org.antlr.v4") }) applyAvroNature() +applyAntlrNature() + +generateGrammarSource { + arguments += ["-visitor"] +} description = "Apache Beam :: SDKs :: Java :: Core" ext.summary = """Beam SDK Java All provides a simple, Java-based @@ -51,9 +58,11 @@ test { } dependencies { + antlr library.java.antlr // Required to load constants from the model, e.g. max timestamp for global window shadow project(path: ":beam-model-pipeline", configuration: "shadow") shadow library.java.vendored_guava_20_0 + compile library.java.antlr_runtime compile library.java.protobuf_java compile library.java.byte_buddy compile library.java.commons_compress diff --git a/sdks/java/core/src/main/antlr/java/org/apache/beam/sdk/schemas/parser/generated/FieldSpecifierNotation.g4 b/sdks/java/core/src/main/antlr/java/org/apache/beam/sdk/schemas/parser/generated/FieldSpecifierNotation.g4 new file mode 100644 index 000000000000..a869304bcf4f --- /dev/null +++ b/sdks/java/core/src/main/antlr/java/org/apache/beam/sdk/schemas/parser/generated/FieldSpecifierNotation.g4 @@ -0,0 +1,52 @@ +grammar FieldSpecifierNotation; + +@header { +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.schemas.parser.generated; +} + +fieldSpecifier: dotExpression; + +dotExpression: dotExpressionComponent ('.' dotExpressionComponent)*; + +dotExpressionComponent: qualifiedComponent # qualifyComponent + | IDENTIFIER # simpleIdentifier + | WILDCARD # wildcard + ; + +qualifiedComponent: IDENTIFIER qualifierList + ; + +qualifierList: arrayQualifier (qualifierList)* # arrayQualifierList + | mapQualifier (qualifierList)* # mapQualifierList + ; + +arrayQualifier: '[]' + | '[*]' + ; + +mapQualifier: '{}' + | '{*}' + ; + +IDENTIFIER: [a-zA-Z0-9]+ ; + +WILDCARD: '*'; + +WS: [ \t\n\r]+ -> skip ; \ No newline at end of file diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/parser/FieldAccessDescriptorParser.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/parser/FieldAccessDescriptorParser.java new file mode 100644 index 000000000000..85d8dcf837c2 --- /dev/null +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/parser/FieldAccessDescriptorParser.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.schemas.parser; + +import javax.annotation.Nullable; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.TokenStream; +import org.apache.beam.sdk.schemas.FieldAccessDescriptor; +import org.apache.beam.sdk.schemas.parser.generated.FieldSpecifierNotationBaseVisitor; +import org.apache.beam.sdk.schemas.parser.generated.FieldSpecifierNotationLexer; +import org.apache.beam.sdk.schemas.parser.generated.FieldSpecifierNotationParser; +import org.apache.beam.sdk.schemas.parser.generated.FieldSpecifierNotationParser.DotExpressionContext; +import org.apache.beam.sdk.schemas.parser.generated.FieldSpecifierNotationParser.FieldSpecifierContext; +import org.apache.beam.sdk.schemas.parser.generated.FieldSpecifierNotationParser.QualifiedComponentContext; +import org.apache.beam.sdk.schemas.parser.generated.FieldSpecifierNotationParser.QualifyComponentContext; +import org.apache.beam.sdk.schemas.parser.generated.FieldSpecifierNotationParser.SimpleIdentifierContext; +import org.apache.beam.sdk.schemas.parser.generated.FieldSpecifierNotationParser.WildcardContext; + +/** Parse a textual representation of a field access descriptor. */ +public class FieldAccessDescriptorParser { + @Nullable + /** Parse the string. */ + public static FieldAccessDescriptor parse(String expr) { + CharStream charStream = CharStreams.fromString(expr); + FieldSpecifierNotationLexer lexer = new FieldSpecifierNotationLexer(charStream); + TokenStream tokens = new CommonTokenStream(lexer); + FieldSpecifierNotationParser parser = new FieldSpecifierNotationParser(tokens); + return new BuildFieldAccessDescriptor().visit(parser.dotExpression()); + } + + private static class BuildFieldAccessDescriptor + extends FieldSpecifierNotationBaseVisitor<FieldAccessDescriptor> { + + @Nullable + @Override + public FieldAccessDescriptor visitFieldSpecifier(FieldSpecifierContext ctx) { + return ctx.dotExpression().accept(this); + } + + @Nullable + @Override + public FieldAccessDescriptor visitDotExpression(DotExpressionContext ctx) { + return null; + } + + @Nullable + @Override + public FieldAccessDescriptor visitQualifyComponent(QualifyComponentContext ctx) { + return ctx.qualifiedComponent().accept(this); + } + + @Nullable + @Override + public FieldAccessDescriptor visitSimpleIdentifier(SimpleIdentifierContext ctx) { + return null; + } + + @Nullable + @Override + public FieldAccessDescriptor visitWildcard(WildcardContext ctx) { + return null; + } + + @Nullable + @Override + public FieldAccessDescriptor visitQualifiedComponent(QualifiedComponentContext ctx) { + return null; + } + } +} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/parser/generated/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/parser/generated/package-info.java new file mode 100644 index 000000000000..f24eb8c8b0d7 --- /dev/null +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/parser/generated/package-info.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Defines utilities for deailing with schemas. + * + * <p>For further details, see the documentation for each class in this package. + */ +@DefaultAnnotation(NonNull.class) +package org.apache.beam.sdk.schemas.parser.generated; + +import edu.umd.cs.findbugs.annotations.DefaultAnnotation; +import edu.umd.cs.findbugs.annotations.NonNull; diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/parser/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/parser/package-info.java new file mode 100644 index 000000000000..0751713c77e9 --- /dev/null +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/parser/package-info.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Defines utilities for deailing with schemas. + * + * <p>For further details, see the documentation for each class in this package. + */ +@DefaultAnnotation(NonNull.class) +package org.apache.beam.sdk.schemas.parser; + +import edu.umd.cs.findbugs.annotations.DefaultAnnotation; +import edu.umd.cs.findbugs.annotations.NonNull; diff --git a/sdks/java/core/src/test/java/org/apache/beam/SdkCoreApiSurfaceTest.java b/sdks/java/core/src/test/java/org/apache/beam/SdkCoreApiSurfaceTest.java deleted file mode 100644 index 834e41a50cf6..000000000000 --- a/sdks/java/core/src/test/java/org/apache/beam/SdkCoreApiSurfaceTest.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam; - -import static org.apache.beam.sdk.util.ApiSurface.containsOnlyPackages; -import static org.hamcrest.MatcherAssert.assertThat; - -import java.io.IOException; -import java.util.Set; -import org.apache.beam.sdk.util.ApiSurface; -import org.apache.beam.vendor.guava.v20_0.com.google.common.collect.ImmutableSet; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** API surface verification for {@link org.apache.beam}. */ -@RunWith(JUnit4.class) -public class SdkCoreApiSurfaceTest { - - /** - * All classes transitively reachable via only public method signatures of the SDK. - * - * <p>Note that our idea of "public" does not include various internal-only APIs. - */ - public static ApiSurface getSdkApiSurface(final ClassLoader classLoader) throws IOException { - return ApiSurface.ofPackage("org.apache.beam", classLoader) - .pruningPattern("org[.]apache[.]beam[.].*Test") - // Exposes Guava, but not intended for users - .pruningClassName("org.apache.beam.sdk.util.common.ReflectHelpers") - // test only - .pruningClassName("org.apache.beam.sdk.testing.InterceptingUrlClassLoader") - // test only - .pruningPrefix("org.apache.beam.model.") - .pruningPrefix("org.apache.beam.vendor.") - .pruningPrefix("java"); - } - - @Test - public void testSdkApiSurface() throws Exception { - - @SuppressWarnings("unchecked") - final Set<String> allowed = - ImmutableSet.of( - "org.apache.beam", - "com.fasterxml.jackson.annotation", - "com.fasterxml.jackson.core", - "com.fasterxml.jackson.databind", - "org.apache.avro", - "org.hamcrest", - // via DataflowMatchers - "org.codehaus.jackson", - // via Avro - "org.joda.time", - "org.junit"); - - assertThat(getSdkApiSurface(getClass().getClassLoader()), containsOnlyPackages(allowed)); - } -} diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/FieldAccessDescriptorParserTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/FieldAccessDescriptorParserTest.java new file mode 100644 index 000000000000..bcca3e88c0d6 --- /dev/null +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/FieldAccessDescriptorParserTest.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.schemas; + +import org.apache.beam.sdk.schemas.parser.FieldAccessDescriptorParser; +import org.junit.Test; + +/** Tests for {@link FieldAccessDescriptorParser}. */ +public class FieldAccessDescriptorParserTest { + @Test + public void testSmoke() { + FieldAccessDescriptorParser.parse("a.b[][*].c.*"); + } +}
With regards, Apache Git Services