This is an automated email from the ASF dual-hosted git repository.
jinsongzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new 2c83bbded [Improvement] Make RocksDB dependency optional (#3018)
2c83bbded is described below
commit 2c83bbded52b36159a5d70fc0d0848df71c7007c
Author: ZhouJinsong <[email protected]>
AuthorDate: Wed Jul 10 19:51:32 2024 +0800
[Improvement] Make RocksDB dependency optional (#3018)
* Add enable-disk-storage profile to include rocksdb independent manually
* Improve some maven poms
* Change build guide
* Polish som poms
* Make support-disk-storage active by default
* Support disk storage by default, add no-extended-disk-storage profile
---
.github/workflows/trino-ci.yml | 4 +--
README.md | 3 +-
amoro-ams/amoro-ams-server/pom.xml | 40 ++++++++++++++--------
amoro-ams/dist/pom.xml | 5 ---
amoro-core/pom.xml | 20 +++++------
.../amoro-mixed-format-trino/pom.xml | 2 +-
pom.xml | 36 ++++++++++++-------
tools/releasing/create_binary_release.sh | 2 +-
8 files changed, 65 insertions(+), 47 deletions(-)
diff --git a/.github/workflows/trino-ci.yml b/.github/workflows/trino-ci.yml
index 309842216..b1238ffcd 100644
--- a/.github/workflows/trino-ci.yml
+++ b/.github/workflows/trino-ci.yml
@@ -39,10 +39,10 @@ jobs:
cache: maven
- name: Validate checkstyle first
- run: mvn validate -P trino-spotless
+ run: mvn validate -P format-mixed-format-trino
- name: Build trino module with Maven
- run: mvn clean install -pl
'amoro-mixed-format/amoro-mixed-format-trino' -am -B -P
hadoop2,trino-spotless,build-mixed-format-trino
+ run: mvn clean install -pl
'amoro-mixed-format/amoro-mixed-format-trino' -am -B -P
hadoop2,format-mixed-format-trino,build-mixed-format-trino
- name: Code coverage
uses: codecov/codecov-action@v3
diff --git a/README.md b/README.md
index 6416f03a5..0748a8832 100644
--- a/README.md
+++ b/README.md
@@ -115,11 +115,12 @@ Amoro is built using Maven with JDK 8 and JDK 17(only for
`amoro-mixed-format/am
* Build all modules without `amoro-mixed-format-trino`: `mvn clean package`
* Build and skip tests: `mvn clean package -DskipTests`
* Build and skip dashboard: `mvn clean package -Pskip-dashboard-build`
+* Build and support disk storage, RocksDB will be introduced to avoid memory
overflow: `mvn clean package -DskipTests -Psupport-disk-storage`
* Build with hadoop 2.x(the default is 3.x) dependencies: `mvn clean package
-DskipTests -Phadoop2`
* Specify Flink version for Flink optimizer(the default is 1.18.1): `mvn clean
package -DskipTests -Dflink-optimizer.flink-version=1.15.4`
* If the version of Flink is below 1.15.0, you also need to add the
`-Pflink-optimizer-pre-1.15` parameter: `mvn clean package -DskipTests
-Pflink-optimizer-pre-1.15 -Dflink-optimizer.flink-version=1.14.6`
* Specify Spark version for Spark optimizer(the default is 3.3.3): `mvn clean
package -DskipTests -Dspark-optimizer.spark-version=3.2.2`
-* Build `amoro-mixed-format-trino` module under JDK 17: `mvn clean package
-DskipTests -Ptrino-spotless,build-mixed-format-trino -pl
'amoro-mixed-format/amoro-mixed-format-trino' -am`.
+* Build `amoro-mixed-format-trino` module under JDK 17: `mvn clean package
-DskipTests -Pformat-mixed-format-trino,build-mixed-format-trino -pl
'amoro-mixed-format/amoro-mixed-format-trino' -am`.
* Build all modules: `mvn clean package -DskipTests
-Ptoolchain,build-mixed-format-trino`, besides you need config `toolchains.xml`
in `${user.home}/.m2/` dir with content below.
```
diff --git a/amoro-ams/amoro-ams-server/pom.xml
b/amoro-ams/amoro-ams-server/pom.xml
index 77e66a2c0..4e58dd9af 100644
--- a/amoro-ams/amoro-ams-server/pom.xml
+++ b/amoro-ams/amoro-ams-server/pom.xml
@@ -96,6 +96,10 @@
<groupId>org.apache.zookeeper</groupId>
<artifactId>*</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.rocksdb</groupId>
+ <artifactId>rocksdbjni</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -120,6 +124,10 @@
<groupId>org.apache.orc</groupId>
<artifactId>orc-mapreduce</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.rocksdb</groupId>
+ <artifactId>rocksdbjni</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -209,26 +217,22 @@
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
- <scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
- <scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
- <scope>compile</scope>
</dependency>
<dependency>
<!-- API bridge between log4j 1 and 2 -->
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
- <scope>compile</scope>
</dependency>
<dependency>
@@ -273,14 +277,19 @@
</dependency>
<!-- flink dependencies -->
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-clients</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime-web</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-rpc-akka-loader</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-shaded-zookeeper-3</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- runtime dependencies -->
@@ -292,21 +301,22 @@
</dependency>
<dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-spark-common</artifactId>
- <version>${paimon.version}</version>
+ <groupId>org.apache.amoro</groupId>
+ <artifactId>amoro-optimizer-standalone</artifactId>
+ <version>${project.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-s3</artifactId>
+ <scope>runtime</scope>
</dependency>
<dependency>
- <groupId>org.apache.amoro</groupId>
- <artifactId>amoro-optimizer-standalone</artifactId>
- <version>${project.version}</version>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-spark-common</artifactId>
+ <version>${paimon.version}</version>
<scope>runtime</scope>
</dependency>
diff --git a/amoro-ams/dist/pom.xml b/amoro-ams/dist/pom.xml
index fe6951a7b..00a6ca3f9 100644
--- a/amoro-ams/dist/pom.xml
+++ b/amoro-ams/dist/pom.xml
@@ -30,11 +30,6 @@
<name>Amoro Project AMS Dist</name>
<url>https://amoro.apache.org</url>
- <properties>
- <maven.compiler.source>8</maven.compiler.source>
- <maven.compiler.target>8</maven.compiler.target>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.amoro</groupId>
diff --git a/amoro-core/pom.xml b/amoro-core/pom.xml
index ef5187e13..808a82fc6 100644
--- a/amoro-core/pom.xml
+++ b/amoro-core/pom.xml
@@ -69,11 +69,6 @@
<artifactId>commons-pool2</artifactId>
</dependency>
- <dependency>
- <groupId>org.rocksdb</groupId>
- <artifactId>rocksdbjni</artifactId>
- </dependency>
-
<dependency>
<groupId>com.esotericsoftware</groupId>
<artifactId>kryo</artifactId>
@@ -145,17 +140,22 @@
<artifactId>RoaringBitmap</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-core</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-bundled-guava</artifactId>
<version>${iceberg.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.rocksdb</groupId>
+ <artifactId>rocksdbjni</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ </dependency>
+
<!-- test dependencies -->
<dependency>
<groupId>org.apache.iceberg</groupId>
diff --git a/amoro-mixed-format/amoro-mixed-format-trino/pom.xml
b/amoro-mixed-format/amoro-mixed-format-trino/pom.xml
index 0991808cb..9e6ce1686 100644
--- a/amoro-mixed-format/amoro-mixed-format-trino/pom.xml
+++ b/amoro-mixed-format/amoro-mixed-format-trino/pom.xml
@@ -655,7 +655,7 @@
</build>
</profile>
<profile>
- <id>trino-spotless</id>
+ <id>format-mixed-format-trino</id>
<build>
<plugins>
<plugin>
diff --git a/pom.xml b/pom.xml
index ace21dddd..b867e83e0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -140,6 +140,9 @@
<amoro-shade-thrift.version>0.20.0</amoro-shade-thrift.version>
<annotation-api.version>1.3.2</annotation-api.version>
<guava.version>32.1.1-jre</guava.version>
+
+ <rocksdb-dependency-scope>compile</rocksdb-dependency-scope>
+ <lucene-dependency-scope>compile</lucene-dependency-scope>
</properties>
<dependencies>
@@ -490,18 +493,6 @@
<version>${awssdk.version}</version>
</dependency>
- <dependency>
- <groupId>io.prometheus</groupId>
- <artifactId>simpleclient</artifactId>
- <version>${prometheus.version}</version>
- </dependency>
-
- <dependency>
- <groupId>io.prometheus</groupId>
- <artifactId>simpleclient_httpserver</artifactId>
- <version>${prometheus.version}</version>
- </dependency>
-
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>dynamodb</artifactId>
@@ -520,6 +511,18 @@
<version>${awssdk.version}</version>
</dependency>
+ <dependency>
+ <groupId>io.prometheus</groupId>
+ <artifactId>simpleclient</artifactId>
+ <version>${prometheus.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.prometheus</groupId>
+ <artifactId>simpleclient_httpserver</artifactId>
+ <version>${prometheus.version}</version>
+ </dependency>
+
<dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
@@ -614,6 +617,7 @@
<groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId>
<version>${rocksdb.version}</version>
+ <scope>${rocksdb-dependency-scope}</scope>
</dependency>
<dependency>
@@ -780,6 +784,7 @@
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>${lucene.version}</version>
+ <scope>${lucene-dependency-scope}</scope>
</dependency>
<!-- flink dependencies -->
@@ -1283,5 +1288,12 @@
<terminal.spark.major.version>3.2</terminal.spark.major.version>
</properties>
</profile>
+ <profile>
+ <id>no-extended-disk-storage</id>
+ <properties>
+ <rocksdb-dependency-scope>provided</rocksdb-dependency-scope>
+ <lucene-dependency-scope>provided</lucene-dependency-scope>
+ </properties>
+ </profile>
</profiles>
</project>
diff --git a/tools/releasing/create_binary_release.sh
b/tools/releasing/create_binary_release.sh
index 74baf6ec4..cad448d3c 100644
--- a/tools/releasing/create_binary_release.sh
+++ b/tools/releasing/create_binary_release.sh
@@ -65,7 +65,7 @@ make_binary_release() {
echo "Creating ${HADOOP_VERSION} binary release"
# enable release profile here (to check for the maven version)
- $MVN clean package ${HADOOP_PROFILE} -pl ':dist' -am -Dgpg.skip
-Dcheckstyle.skip=true -DskipTests
+ $MVN clean package ${HADOOP_PROFILE} -Pno-extended-disk-storage -pl ':dist'
-am -Dgpg.skip -Dcheckstyle.skip=true -DskipTests
local
TARGET_FILE="apache-amoro-${RELEASE_VERSION}-bin-${HADOOP_VERSION}.tar.gz"
cp amoro-ams/dist/target/apache-amoro-${RELEASE_VERSION}-bin.tar.gz
${RELEASE_DIR}/${TARGET_FILE}
