This is an automated email from the ASF dual-hosted git repository.
diwu pushed a commit to branch catalog-hive
in repository https://gitbox.apache.org/repos/asf/doris-shade.git
The following commit(s) were added to refs/heads/catalog-hive by this push:
new 63fe7e5 Add new hive catalog shade package to resolve version
conflicts (#7)
63fe7e5 is described below
commit 63fe7e53e2c192916e5d6c9bb6e148cf81476c63
Author: Calvin Kirs <[email protected]>
AuthorDate: Fri Mar 31 11:42:40 2023 +0800
Add new hive catalog shade package to resolve version conflicts (#7)
- The thrift used internally by hive is not compatible with the thrift
version used internally by our doris
---
hadoop-shade-2/pom.xml | 202 ---------------------------
{hive-shade-3 => hive-catalog-shade}/pom.xml | 95 +++++++++++--
hudi-mr-shade-0.13/pom.xml | 81 -----------
pom.xml | 6 +-
spring-shade-5/pom.xml | 71 ----------
thrift-shade-0.13/pom.xml | 75 ----------
6 files changed, 86 insertions(+), 444 deletions(-)
diff --git a/hadoop-shade-2/pom.xml b/hadoop-shade-2/pom.xml
deleted file mode 100644
index f8c737c..0000000
--- a/hadoop-shade-2/pom.xml
+++ /dev/null
@@ -1,202 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.doris</groupId>
- <artifactId>doris-shade</artifactId>
- <version>1.0.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>hadoop-shade-2</artifactId>
-
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <hadoop.version>2.10.2</hadoop.version>
- </properties>
- <dependencies>
- <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws
-->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-aws</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- <!--
https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-huaweicloud -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-huaweicloud</artifactId>
- <version>2.8.3</version>
- <exclusions>
- <exclusion>
- <artifactId>jackson-databind</artifactId>
- <groupId>com.fasterxml.jackson.core</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <!--
https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aliyun -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-aliyun</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
-
- <!--
https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-annotations -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-auth
-->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- <!--
https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs
-->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <artifactId>netty-all</artifactId>
- <groupId>io.netty</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jackson-databind</artifactId>
- <groupId>com.fasterxml.jackson.core</groupId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <!--
https://mvnrepository.com/artifact/com.huaweicloud/esdk-obs-java-bundle -->
- <dependency>
- <groupId>com.huaweicloud</groupId>
- <artifactId>esdk-obs-java-bundle</artifactId>
- <version>3.21.8</version>
- </dependency>
-
-
- <!-- https://mvnrepository.com/artifact/com.qcloud.cos/hadoop-cos -->
- <dependency>
- <groupId>com.qcloud.cos</groupId>
- <artifactId>hadoop-cos</artifactId>
- <version>2.8.5-8.1.8</version>
- </dependency>
- </dependencies>
- <profiles>
- <!-- for custom internal repository -->
- <profile>
- <id>custom-env</id>
- <activation>
- <property>
- <name>env.CUSTOM_MAVEN_REPO</name>
- </property>
- </activation>
- <repositories>
- <repository>
- <id>custom-nexus</id>
- <url>${env.CUSTOM_MAVEN_REPO}</url>
- </repository>
- </repositories>
- <pluginRepositories>
- <pluginRepository>
- <id>custom-nexus</id>
- <url>${env.CUSTOM_MAVEN_REPO}</url>
- </pluginRepository>
- </pluginRepositories>
- </profile>
- <!-- for general repository -->
- <profile>
- <id>general-env</id>
- <activation>
- <property>
- <name>!env.CUSTOM_MAVEN_REPO</name>
- </property>
- </activation>
- <repositories>
- <!-- for java-cup -->
- <repository>
- <id>cloudera-thirdparty</id>
-
<url>https://repository.cloudera.com/content/repositories/third-party/</url>
- </repository>
- <!-- for bdb je -->
- <repository>
- <id>oracleReleases</id>
- <url>http://download.oracle.com/maven</url>
- </repository>
- <!-- for huawei obs sdk -->
- <repository>
- <id>huawei-obs-sdk</id>
-
<url>https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/</url>
- </repository>
- </repositories>
- <pluginRepositories>
- <!-- for cup-maven-plugin -->
- <pluginRepository>
- <id>spring-plugins</id>
- <url>http://repo.spring.io/plugins-release/</url>
- </pluginRepository>
- </pluginRepositories>
- </profile>
- </profiles>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>shade</goal>
- </goals>
- <phase>package</phase>
- <configuration>
-
<createDependencyReducedPom>true</createDependencyReducedPom>
-
<dependencyReducedPomLocation>${project.basedir}/target/dependency-reduced-pom.xml</dependencyReducedPomLocation>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- <relocations>
- <relocation>
- <pattern>com.google.common</pattern>
-
<shadedPattern>shade.doris.hdfs.com.google.common</shadedPattern>
- </relocation>
- </relocations>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
diff --git a/hive-shade-3/pom.xml b/hive-catalog-shade/pom.xml
similarity index 69%
rename from hive-shade-3/pom.xml
rename to hive-catalog-shade/pom.xml
index 5828dc4..e5166b8 100644
--- a/hive-shade-3/pom.xml
+++ b/hive-catalog-shade/pom.xml
@@ -25,13 +25,40 @@ under the License.
<version>1.0.1-SNAPSHOT</version>
</parent>
- <artifactId>hive-shade-3</artifactId>
+ <artifactId>hive-catalog-shade</artifactId>
<properties>
<hive.version>3.1.3</hive.version>
<hive-storage-api.version>2.8.1</hive-storage-api.version>
+ <!--We rewritten some hive methods, so we need to keep the same hadoop
version as the main library-->
+ <hadoop.version>3.3.4</hadoop.version>
+ <iceberg-hive-metastore.version>1.1.0</iceberg-hive-metastore.version>
</properties>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.hive</groupId>
@@ -58,9 +85,12 @@ under the License.
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</exclusion>
+ <exclusion>
+ <artifactId>hadoop-common</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
</exclusions>
</dependency>
- <!--
https://mvnrepository.com/artifact/org.apache.hive/hive-storage-api -->
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
@@ -70,7 +100,6 @@ under the License.
<groupId>org.apache.hive</groupId>
<artifactId>hive-metastore</artifactId>
<version>${hive.version}</version>
- <scope>provided</scope>
<exclusions>
<exclusion>
<artifactId>servlet-api</artifactId>
@@ -98,8 +127,12 @@ under the License.
<artifactId>hbase-client</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
+ <artifactId>netty-all</artifactId>
+ <groupId>io.netty</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>log4j</artifactId>
+ <groupId>log4j</groupId>
</exclusion>
</exclusions>
</dependency>
@@ -150,20 +183,57 @@ under the License.
<artifactId>zookeeper</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
+ <groupId>org.apache.velocity</groupId>
+ <artifactId>velocity</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>commons-httpclient</artifactId>
+ <groupId>commons-httpclient</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>calcite-core</artifactId>
+ <groupId>org.apache.calcite</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>calcite-druid</artifactId>
+ <groupId>org.apache.calcite</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>calcite-linq4j</artifactId>
+ <groupId>org.apache.calcite</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>avatica</artifactId>
+ <groupId>org.apache.calcite.avatica</groupId>
</exclusion>
</exclusions>
</dependency>
+ <!--Since we access through the hive catalog, and the thrift version
used by hive is not compatible with the
+current internal version used by doris, this jar must be introduced to
complete to rename.-->
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-hive-metastore</artifactId>
+ <version>${iceberg-hive-metastore.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-standalone-metastore</artifactId>
- <version>3.1.3</version>
+ <version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-classification</artifactId>
- <version>3.1.3</version>
+ <version>${hive.version}</version>
</dependency>
</dependencies>
@@ -180,7 +250,8 @@ under the License.
<phase>package</phase>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
-
<dependencyReducedPomLocation>${project.basedir}/target/dependency-reduced-pom.xml</dependencyReducedPomLocation>
+
<dependencyReducedPomLocation>${project.basedir}/target/dependency-reduced-pom.xml
+ </dependencyReducedPomLocation>
<filters>
<filter>
<artifact>*:*</artifact>
@@ -208,6 +279,10 @@ under the License.
<pattern>com.google.gson</pattern>
<shadedPattern>shade.doris.hive.com.google.gson</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>org.apache.thrift</pattern>
+
<shadedPattern>shade.doris.hive.org.apache.thrift</shadedPattern>
+ </relocation>
</relocations>
</configuration>
</execution>
diff --git a/hudi-mr-shade-0.13/pom.xml b/hudi-mr-shade-0.13/pom.xml
deleted file mode 100644
index f7258d2..0000000
--- a/hudi-mr-shade-0.13/pom.xml
+++ /dev/null
@@ -1,81 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.doris</groupId>
- <artifactId>doris-shade</artifactId>
- <version>1.0.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>hudi-mr-shade-0.13</artifactId>
-
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <hudi.version>0.13.0</hudi.version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-hadoop-mr</artifactId>
- <version>${hudi.version}</version>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>shade</goal>
- </goals>
- <phase>package</phase>
- <configuration>
-
<createDependencyReducedPom>true</createDependencyReducedPom>
-
<dependencyReducedPomLocation>${project.basedir}/target/dependency-reduced-pom.xml</dependencyReducedPomLocation>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- <relocations>
- <relocation>
- <pattern>com.google.common</pattern>
-
<shadedPattern>shade.doris.hudi.mr.com.google.common</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google.gson</pattern>
-
<shadedPattern>shade.doris.hudi.mr.com.google.gson</shadedPattern>
- </relocation>
- </relocations>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 8539309..6391f9e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -44,11 +44,7 @@ under the License.
</scm>
<modules>
- <module>thrift-shade-0.13</module>
- <module>hadoop-shade-2</module>
- <module>hudi-mr-shade-0.13</module>
- <module>hive-shade-3</module>
- <module>spring-shade-5</module>
+ <module>hive-catalog-shade</module>
</modules>
<properties>
diff --git a/spring-shade-5/pom.xml b/spring-shade-5/pom.xml
deleted file mode 100644
index 3a745a1..0000000
--- a/spring-shade-5/pom.xml
+++ /dev/null
@@ -1,71 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.doris</groupId>
- <artifactId>doris-shade</artifactId>
- <version>1.0.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>spring-shade-5</artifactId>
-
- <properties>
- <spring.version>5.3.25</spring.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-web</artifactId>
- <version>${spring.version}</version>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>shade</goal>
- </goals>
- <phase>package</phase>
- <configuration>
-
<createDependencyReducedPom>true</createDependencyReducedPom>
-
<dependencyReducedPomLocation>${project.basedir}/target/dependency-reduced-pom.xml</dependencyReducedPomLocation>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
diff --git a/thrift-shade-0.13/pom.xml b/thrift-shade-0.13/pom.xml
deleted file mode 100644
index 31ac858..0000000
--- a/thrift-shade-0.13/pom.xml
+++ /dev/null
@@ -1,75 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.doris</groupId>
- <artifactId>doris-shade</artifactId>
- <version>1.0.1-SNAPSHOT</version>
- </parent>
- <artifactId>thrift-shade-0.13</artifactId>
- <version>1.0.1-SNAPSHOT</version>
- <properties>
- <libethrift.version>0.13.0</libethrift.version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- <version>${libethrift.version}</version>
- <exclusions>
- <exclusion>
- <artifactId>httpclient</artifactId>
- <groupId>org.apache.httpcomponents</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>shade</goal>
- </goals>
- <phase>package</phase>
- <configuration>
-
<createDependencyReducedPom>true</createDependencyReducedPom>
-
<dependencyReducedPomLocation>${project.basedir}/target/dependency-reduced-pom.xml
- </dependencyReducedPomLocation>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
