This is an automated email from the ASF dual-hosted git repository. anton pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new c7fad9e [SQL] Move HCatalogTableProvider into its own module new 2cd2677 Merge pull request #8345 from akedin/move-hcatalogtableprovider c7fad9e is described below commit c7fad9e36c4e78bce9d8f7fb2b7aa6916a1b3940 Author: akedin <ke...@google.com> AuthorDate: Wed Apr 17 22:53:32 2019 -0700 [SQL] Move HCatalogTableProvider into its own module --- sdks/java/extensions/sql/build.gradle | 16 --------- sdks/java/extensions/sql/hcatalog/build.gradle | 40 ++++++++++++++++++++++ .../meta/provider/hcatalog/DatabaseProvider.java | 0 .../sql/meta/provider/hcatalog/HCatalogTable.java | 0 .../provider/hcatalog/HCatalogTableProvider.java | 0 .../sql/meta/provider/hcatalog/package-info.java | 0 .../provider/hcatalog}/BeamSqlHiveSchemaTest.java | 4 +-- settings.gradle | 2 ++ 8 files changed, 44 insertions(+), 18 deletions(-) diff --git a/sdks/java/extensions/sql/build.gradle b/sdks/java/extensions/sql/build.gradle index 703bc76..2039389 100644 --- a/sdks/java/extensions/sql/build.gradle +++ b/sdks/java/extensions/sql/build.gradle @@ -60,8 +60,6 @@ configurations { def calcite_version = "1.19.0" def avatica_version = "1.13.0" -def hive_version = "2.1.0" -def netty_version = "4.1.30.Final" dependencies { javacc "net.java.dev.javacc:javacc:4.0" @@ -85,7 +83,6 @@ dependencies { shadow "com.alibaba:fastjson:1.2.49" shadow "com.jayway.jsonpath:json-path:2.4.0" shadow project(path: ":beam-runners-direct-java", configuration: "shadow") - provided project(path: ":beam-sdks-java-io-hcatalog", configuration: "shadow") provided project(path: ":beam-sdks-java-io-kafka", configuration: "shadow") provided project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadow") provided library.java.kafka_clients @@ -96,18 +93,6 @@ dependencies { shadowTest library.java.mockito_core shadowTest library.java.quickcheck_core - // Needed for HCatalogTableProvider tests, - // they use HCat* types - shadowTest "io.netty:netty-all:$netty_version" - shadowTest("org.apache.hive.hcatalog:hive-hcatalog-core:$hive_version") { - // Hive brings full Calcite 1.6 + Avatica with JDBC driver which - // gets registered and gets started instead of ours, - // and a ton of other stuff. - exclude group: "org.apache.calcite", module:"calcite-avatica" - exclude group: "com.google.guava", module: "guava" - exclude group: "com.google.protobuf", module: "protobuf-java" - } - // Dependencies that we don't directly reference permitUnusedDeclared "com.jayway.jsonpath:json-path:2.4.0" permitUnusedDeclared library.java.jackson_dataformat_yaml @@ -209,7 +194,6 @@ task integrationTest(type: Test) { .sourceSets .test .runtimeClasspath - .filter { it.name != "hive-exec-${hive_version}.jar" } testClassesDirs = files(project(":beam-sdks-java-extensions-sql").sourceSets.test.output.classesDirs) useJUnit { } } diff --git a/sdks/java/extensions/sql/hcatalog/build.gradle b/sdks/java/extensions/sql/hcatalog/build.gradle new file mode 100644 index 0000000..1aedb02 --- /dev/null +++ b/sdks/java/extensions/sql/hcatalog/build.gradle @@ -0,0 +1,40 @@ +import groovy.json.JsonOutput + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id 'org.apache.beam.module' } + +applyJavaNature() + +def hive_version = "2.1.0" +def netty_version = "4.1.30.Final" + +dependencies { + provided project(path: ":beam-sdks-java-extensions-sql", configuration: "shadow") + provided project(path: ":beam-sdks-java-io-hcatalog", configuration: "shadow") + + // Needed for HCatalogTableProvider tests, + // they use HCat* types + shadowTest "io.netty:netty-all:$netty_version" + shadowTest("org.apache.hive.hcatalog:hive-hcatalog-core:$hive_version") { + // Hive brings full Calcite 1.6 + Avatica with JDBC driver which + // gets registered and gets started instead of ours + exclude group: "org.apache.calcite", module:"calcite-avatica" + } +} diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/DatabaseProvider.java b/sdks/java/extensions/sql/hcatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/DatabaseProvider.java similarity index 100% rename from sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/DatabaseProvider.java rename to sdks/java/extensions/sql/hcatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/DatabaseProvider.java diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/HCatalogTable.java b/sdks/java/extensions/sql/hcatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/HCatalogTable.java similarity index 100% rename from sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/HCatalogTable.java rename to sdks/java/extensions/sql/hcatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/HCatalogTable.java diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/HCatalogTableProvider.java b/sdks/java/extensions/sql/hcatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/HCatalogTableProvider.java similarity index 100% rename from sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/HCatalogTableProvider.java rename to sdks/java/extensions/sql/hcatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/HCatalogTableProvider.java diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/package-info.java b/sdks/java/extensions/sql/hcatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/package-info.java similarity index 100% rename from sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/package-info.java rename to sdks/java/extensions/sql/hcatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/package-info.java diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlHiveSchemaTest.java b/sdks/java/extensions/sql/hcatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java similarity index 98% rename from sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlHiveSchemaTest.java rename to sdks/java/extensions/sql/hcatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java index 1e5063f..7c84202 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlHiveSchemaTest.java +++ b/sdks/java/extensions/sql/hcatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.sdk.extensions.sql; +package org.apache.beam.sdk.extensions.sql.meta.provider.hcatalog; import static org.apache.beam.sdk.io.hcatalog.test.HCatalogIOTestUtils.TEST_DATABASE; import static org.apache.beam.sdk.io.hcatalog.test.HCatalogIOTestUtils.TEST_RECORDS_COUNT; @@ -26,10 +26,10 @@ import static org.apache.beam.sdk.io.hcatalog.test.HCatalogIOTestUtils.insertTes import java.io.IOException; import java.io.Serializable; import java.util.Arrays; +import org.apache.beam.sdk.extensions.sql.SqlTransform; import org.apache.beam.sdk.extensions.sql.impl.schema.BeamPCollectionTable; import org.apache.beam.sdk.extensions.sql.meta.provider.ReadOnlyTableProvider; import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider; -import org.apache.beam.sdk.extensions.sql.meta.provider.hcatalog.HCatalogTableProvider; import org.apache.beam.sdk.io.hcatalog.test.EmbeddedMetastoreService; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.testing.PAssert; diff --git a/settings.gradle b/settings.gradle index 980acb8..65e67fa 100644 --- a/settings.gradle +++ b/settings.gradle @@ -123,6 +123,8 @@ include "beam-sdks-java-extensions-sql-jdbc" project(":beam-sdks-java-extensions-sql-jdbc").dir = file("sdks/java/extensions/sql/jdbc") include "beam-sdks-java-extensions-sql-shell" project(":beam-sdks-java-extensions-sql-shell").dir = file("sdks/java/extensions/sql/shell") +include "beam-sdks-java-extensions-sql-hcatalog" +project(":beam-sdks-java-extensions-sql-hcatalog").dir = file("sdks/java/extensions/sql/hcatalog") include "beam-sdks-java-fn-execution" project(":beam-sdks-java-fn-execution").dir = file("sdks/java/fn-execution") include "beam-sdks-java-harness"