This is an automated email from the ASF dual-hosted git repository. voonhous pushed a commit to tag rfc-105-pre-cleanup in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 8fa5aba2f54783493bf7c4983834c9fab803eb08 Author: voon <[email protected]> AuthorDate: Thu May 28 19:14:41 2026 +0800 chore(trino): drop trino docker modules and bundle references The integ-test docker images hudi-hadoop-trinobase-docker / hudi-hadoop-trinocoordinator-docker / hudi-hadoop-trinoworker-docker copied packaging/hudi-trino-bundle/target/hudi-trino-bundle-*.jar into a Trino plugin directory. The bundle was removed per RFC-105 (connector ships as the regular non-shaded hudi-trino artifact, consumed from the Trino-side shim), so the antrun copy step crashes when the integration-tests profile is activated. Removes the three docker module directories, unregisters them in docker/hoodie/hadoop/pom.xml, drops the trino services from the amd64 and arm64 docker-compose files, and removes the now-dead hudi-trino-bundle entries from the azure pipeline ignore list and the release bundle validation script. Any docker-based Trino integ test setup belongs with the Trino-side connector going forward. --- azure-pipelines-20230430.yml | 1 - ...r-compose_hadoop284_hive2310_spark353_amd64.yml | 30 ------ ...r-compose_hadoop284_hive2310_spark353_arm64.yml | 28 ----- docker/hoodie/hadoop/pom.xml | 3 - docker/hoodie/hadoop/trinobase/Dockerfile | 67 ------------ docker/hoodie/hadoop/trinobase/pom.xml | 116 --------------------- docker/hoodie/hadoop/trinobase/scripts/trino.sh | 25 ----- docker/hoodie/hadoop/trinocoordinator/Dockerfile | 29 ------ .../trinocoordinator/etc/catalog/hive.properties | 22 ---- .../hadoop/trinocoordinator/etc/config.properties | 26 ----- .../hoodie/hadoop/trinocoordinator/etc/jvm.config | 27 ----- .../hadoop/trinocoordinator/etc/log.properties | 19 ---- .../hadoop/trinocoordinator/etc/node.properties | 21 ---- docker/hoodie/hadoop/trinocoordinator/pom.xml | 96 ----------------- docker/hoodie/hadoop/trinoworker/Dockerfile | 29 ------ .../hadoop/trinoworker/etc/catalog/hive.properties | 22 ---- .../hadoop/trinoworker/etc/config.properties | 24 ----- docker/hoodie/hadoop/trinoworker/etc/jvm.config | 27 ----- .../hoodie/hadoop/trinoworker/etc/log.properties | 19 ---- .../hoodie/hadoop/trinoworker/etc/node.properties | 21 ---- docker/hoodie/hadoop/trinoworker/pom.xml | 96 ----------------- scripts/release/validate_staged_bundles.sh | 2 +- 22 files changed, 1 insertion(+), 749 deletions(-) diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml index 3edac6046b1a..0574bfa6704c 100644 --- a/azure-pipelines-20230430.yml +++ b/azure-pipelines-20230430.yml @@ -97,7 +97,6 @@ parameters: - '!packaging/hudi-presto-bundle' - '!packaging/hudi-spark-bundle' - '!packaging/hudi-timeline-server-bundle' - - '!packaging/hudi-trino-bundle' - '!packaging/hudi-utilities-slim-bundle' variables: diff --git a/docker/compose/docker-compose_hadoop284_hive2310_spark353_amd64.yml b/docker/compose/docker-compose_hadoop284_hive2310_spark353_amd64.yml index 1d84417b378a..6e7c3c778e77 100644 --- a/docker/compose/docker-compose_hadoop284_hive2310_spark353_amd64.yml +++ b/docker/compose/docker-compose_hadoop284_hive2310_spark353_amd64.yml @@ -228,36 +228,6 @@ services: - ${HUDI_WS}:/var/hoodie/ws command: worker - trino-coordinator-1: - container_name: trino-coordinator-1 - hostname: trino-coordinator-1 - image: apachehudi/hudi-hadoop_2.8.4-trinocoordinator_368:latest - platform: linux/amd64 - ports: - - "8091:8091" - links: - - "hivemetastore" - volumes: - - ${HUDI_WS}:/var/hoodie/ws - command: http://trino-coordinator-1:8091 trino-coordinator-1 - - trino-worker-1: - container_name: trino-worker-1 - hostname: trino-worker-1 - image: apachehudi/hudi-hadoop_2.8.4-trinoworker_368:latest - platform: linux/amd64 - depends_on: [ "trino-coordinator-1" ] - ports: - - "8092:8092" - links: - - "hivemetastore" - - "hiveserver" - - "hive-metastore-postgresql" - - "namenode" - volumes: - - ${HUDI_WS}:/var/hoodie/ws - command: http://trino-coordinator-1:8091 trino-worker-1 - graphite: container_name: graphite hostname: graphite diff --git a/docker/compose/docker-compose_hadoop284_hive2310_spark353_arm64.yml b/docker/compose/docker-compose_hadoop284_hive2310_spark353_arm64.yml index b995859c8022..678bc33f0449 100644 --- a/docker/compose/docker-compose_hadoop284_hive2310_spark353_arm64.yml +++ b/docker/compose/docker-compose_hadoop284_hive2310_spark353_arm64.yml @@ -219,34 +219,6 @@ services: - ${HUDI_WS}:/var/hoodie/ws command: worker - trino-coordinator-1: - container_name: trino-coordinator-1 - hostname: trino-coordinator-1 - image: apachehudi/hudi-hadoop_2.8.4-trinocoordinator_368:latest - ports: - - "8091:8091" - links: - - "hivemetastore" - volumes: - - ${HUDI_WS}:/var/hoodie/ws - command: http://trino-coordinator-1:8091 trino-coordinator-1 - - trino-worker-1: - container_name: trino-worker-1 - hostname: trino-worker-1 - image: apachehudi/hudi-hadoop_2.8.4-trinoworker_368:latest - depends_on: [ "trino-coordinator-1" ] - ports: - - "8092:8092" - links: - - "hivemetastore" - - "hiveserver" - - "hive-metastore-postgresql" - - "namenode" - volumes: - - ${HUDI_WS}:/var/hoodie/ws - command: http://trino-coordinator-1:8091 trino-worker-1 - graphite: container_name: graphite hostname: graphite diff --git a/docker/hoodie/hadoop/pom.xml b/docker/hoodie/hadoop/pom.xml index 45bab9947f90..25a934fafd3e 100644 --- a/docker/hoodie/hadoop/pom.xml +++ b/docker/hoodie/hadoop/pom.xml @@ -39,9 +39,6 @@ <module>sparkworker</module> <module>sparkadhoc</module> <module>prestobase</module> - <module>trinobase</module> - <module>trinocoordinator</module> - <module>trinoworker</module> </modules> <dependencies> diff --git a/docker/hoodie/hadoop/trinobase/Dockerfile b/docker/hoodie/hadoop/trinobase/Dockerfile deleted file mode 100644 index 0700fa2f6bfb..000000000000 --- a/docker/hoodie/hadoop/trinobase/Dockerfile +++ /dev/null @@ -1,67 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Trino docker setup is adapted from https://github.com/Lewuathe/docker-trino-cluster - -ARG HADOOP_VERSION=2.8.4 -ARG HIVE_VERSION=2.3.3 -ARG BASE_IMAGE_TAG=java11 -FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base-${BASE_IMAGE_TAG}:latest as hadoop-base - -ENV TRINO_VERSION=368 -ENV TRINO_HOME=/usr/local/trino -ENV BASE_URL=https://repo1.maven.org/maven2 - -RUN apt-get update -RUN apt-get install -y \ - curl \ - tar \ - sudo \ - rsync \ - python \ - wget \ - python3-pip \ - python-dev \ - build-essential \ - uuid-runtime \ - less - -ENV JAVA_HOME /usr/java/default -ENV PATH $PATH:$JAVA_HOME/bin - -WORKDIR /usr/local/bin -RUN wget -q ${BASE_URL}/io/trino/trino-cli/${TRINO_VERSION}/trino-cli-${TRINO_VERSION}-executable.jar -RUN chmod +x trino-cli-${TRINO_VERSION}-executable.jar -RUN mv trino-cli-${TRINO_VERSION}-executable.jar trino-cli - -WORKDIR /usr/local -RUN wget -q ${BASE_URL}/io/trino/trino-server/${TRINO_VERSION}/trino-server-${TRINO_VERSION}.tar.gz -RUN tar xvzf trino-server-${TRINO_VERSION}.tar.gz -C /usr/local/ -RUN ln -s /usr/local/trino-server-${TRINO_VERSION} $TRINO_HOME - -ENV TRINO_BASE_WS /var/hoodie/ws/docker/hoodie/hadoop/trinobase -RUN mkdir -p ${TRINO_BASE_WS}/target/ -ADD target/ ${TRINO_BASE_WS}/target/ -ENV HUDI_TRINO_BUNDLE ${TRINO_BASE_WS}/target/hudi-trino-bundle.jar -RUN cp ${HUDI_TRINO_BUNDLE} ${TRINO_HOME}/plugin/hive/ - -ADD scripts ${TRINO_HOME}/scripts -RUN chmod +x ${TRINO_HOME}/scripts/trino.sh - -RUN mkdir -p $TRINO_HOME/data -VOLUME ["$TRINO_HOME/data"] diff --git a/docker/hoodie/hadoop/trinobase/pom.xml b/docker/hoodie/hadoop/trinobase/pom.xml deleted file mode 100644 index d4cda51a2edf..000000000000 --- a/docker/hoodie/hadoop/trinobase/pom.xml +++ /dev/null @@ -1,116 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, - ~ software distributed under the License is distributed on an - ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - ~ KIND, either express or implied. See the License for the - ~ specific language governing permissions and limitations - ~ under the License. - --> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <parent> - <artifactId>hudi-hadoop-docker</artifactId> - <groupId>org.apache.hudi</groupId> - <version>1.3.0-SNAPSHOT</version> - </parent> - <modelVersion>4.0.0</modelVersion> - <packaging>pom</packaging> - <artifactId>hudi-hadoop-trinobase-docker</artifactId> - <description>Trino Base Docker Image with Hudi</description> - - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <checkstyle.skip>true</checkstyle.skip> - <main.basedir>${project.parent.parent.basedir}</main.basedir> - </properties> - - <dependencyManagement> - <dependencies> - <dependency> - <groupId>org.apache.hudi</groupId> - <artifactId>hudi-hadoop-base-java11-docker</artifactId> - <version>${project.version}</version> - <type>pom</type> - <scope>import</scope> - </dependency> - </dependencies> - </dependencyManagement> - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> - <version>1.7</version> - <executions> - <execution> - <phase>package</phase> - <configuration> - <target> - <copy file="${project.basedir}/../../../../packaging/hudi-trino-bundle/target/hudi-trino-bundle-${project.version}.jar" - tofile="target/hudi-trino-bundle.jar"/> - </target> - </configuration> - <goals> - <goal>run</goal> - </goals> - </execution> - </executions> - </plugin> - <!-- Build Docker image --> - <plugin> - <groupId>com.spotify</groupId> - <artifactId>dockerfile-maven-plugin</artifactId> - <version>${dockerfile.maven.version}</version> - <executions> - <execution> - <id>tag-latest</id> - <phase>pre-integration-test</phase> - <goals> - <goal>build</goal> - <goal>tag</goal> - </goals> - <configuration> - <skip>${docker.build.skip}</skip> - <pullNewerImage>false</pullNewerImage> - <repository> - apachehudi/hudi-hadoop_${docker.hadoop.version}-trinobase_${docker.trino.version} - </repository> - <force>true</force> - <tag>latest</tag> - </configuration> - </execution> - <execution> - <id>tag-version</id> - <phase>pre-integration-test</phase> - <goals> - <goal>build</goal> - <goal>tag</goal> - <!-- <goal>push</goal> --> - </goals> - <configuration> - <skip>${docker.build.skip}</skip> - <pullNewerImage>false</pullNewerImage> - <repository> - apachehudi/hudi-hadoop_${docker.hadoop.version}-trinobase_${docker.trino.version} - </repository> - <force>true</force> - <tag>${project.version}</tag> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> diff --git a/docker/hoodie/hadoop/trinobase/scripts/trino.sh b/docker/hoodie/hadoop/trinobase/scripts/trino.sh deleted file mode 100644 index 4efaed0cd8d3..000000000000 --- a/docker/hoodie/hadoop/trinobase/scripts/trino.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Copy the trino bundle at run time so that locally built bundle overrides the one that is present in the image -echo "Copying trino bundle to ${TRINO_HOME}/plugin/hive/" -cp ${HUDI_TRINO_BUNDLE} ${TRINO_HOME}/plugin/hive/ - -/usr/local/trino/bin/launcher run diff --git a/docker/hoodie/hadoop/trinocoordinator/Dockerfile b/docker/hoodie/hadoop/trinocoordinator/Dockerfile deleted file mode 100644 index 67a31448d7a6..000000000000 --- a/docker/hoodie/hadoop/trinocoordinator/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Trino docker setup is adapted from https://github.com/Lewuathe/docker-trino-cluster - -ARG HADOOP_VERSION=2.8.4 -ARG TRINO_VERSION=368 -FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-trinobase_${TRINO_VERSION}:latest as trino-base - -ADD etc /usr/local/trino/etc -EXPOSE 8091 - -WORKDIR /usr/local/trino -ENTRYPOINT [ "./scripts/trino.sh" ] diff --git a/docker/hoodie/hadoop/trinocoordinator/etc/catalog/hive.properties b/docker/hoodie/hadoop/trinocoordinator/etc/catalog/hive.properties deleted file mode 100644 index ed7fce1b3e64..000000000000 --- a/docker/hoodie/hadoop/trinocoordinator/etc/catalog/hive.properties +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -connector.name=hive -hive.metastore.uri=thrift://hivemetastore:9083 -hive.config.resources=/etc/hadoop/core-site.xml,/etc/hadoop/hdfs-site.xml -hive.hdfs.authentication.type=NONE diff --git a/docker/hoodie/hadoop/trinocoordinator/etc/config.properties b/docker/hoodie/hadoop/trinocoordinator/etc/config.properties deleted file mode 100644 index 9876a0fe0f00..000000000000 --- a/docker/hoodie/hadoop/trinocoordinator/etc/config.properties +++ /dev/null @@ -1,26 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -coordinator=true -node-scheduler.include-coordinator=false -http-server.http.port=8091 -query.max-memory=50GB -query.max-memory-per-node=1GB -query.max-total-memory-per-node=2GB -discovery-server.enabled=true -discovery.uri=http://trino-coordinator-1:8091 diff --git a/docker/hoodie/hadoop/trinocoordinator/etc/jvm.config b/docker/hoodie/hadoop/trinocoordinator/etc/jvm.config deleted file mode 100644 index fb17203ca211..000000000000 --- a/docker/hoodie/hadoop/trinocoordinator/etc/jvm.config +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# --server --Xmx16G --XX:+UseG1GC --XX:G1HeapRegionSize=32M --XX:+UseGCOverheadLimit --XX:+ExplicitGCInvokesConcurrent --XX:+HeapDumpOnOutOfMemoryError --XX:OnOutOfMemoryError=kill -9 %p --Djdk.attach.allowAttachSelf=true diff --git a/docker/hoodie/hadoop/trinocoordinator/etc/log.properties b/docker/hoodie/hadoop/trinocoordinator/etc/log.properties deleted file mode 100644 index 23b063080b4f..000000000000 --- a/docker/hoodie/hadoop/trinocoordinator/etc/log.properties +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -io.trinosql=INFO diff --git a/docker/hoodie/hadoop/trinocoordinator/etc/node.properties b/docker/hoodie/hadoop/trinocoordinator/etc/node.properties deleted file mode 100644 index d97d54748599..000000000000 --- a/docker/hoodie/hadoop/trinocoordinator/etc/node.properties +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -node.environment=development -node.id=3044b958-f077-4fce-87ed-ca8308f800b6 -node.data-dir=/usr/local/trino/data diff --git a/docker/hoodie/hadoop/trinocoordinator/pom.xml b/docker/hoodie/hadoop/trinocoordinator/pom.xml deleted file mode 100644 index 048b4c104f32..000000000000 --- a/docker/hoodie/hadoop/trinocoordinator/pom.xml +++ /dev/null @@ -1,96 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, - ~ software distributed under the License is distributed on an - ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - ~ KIND, either express or implied. See the License for the - ~ specific language governing permissions and limitations - ~ under the License. - --> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <parent> - <artifactId>hudi-hadoop-docker</artifactId> - <groupId>org.apache.hudi</groupId> - <version>1.3.0-SNAPSHOT</version> - </parent> - <modelVersion>4.0.0</modelVersion> - <packaging>pom</packaging> - <artifactId>hudi-hadoop-trinocoordinator-docker</artifactId> - <description>Trino Coordinator Docker Image with Hudi</description> - - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <checkstyle.skip>true</checkstyle.skip> - <main.basedir>${project.parent.parent.basedir}</main.basedir> - </properties> - - <dependencyManagement> - <dependencies> - <dependency> - <groupId>org.apache.hudi</groupId> - <artifactId>hudi-hadoop-trinobase-docker</artifactId> - <version>${project.version}</version> - <type>pom</type> - </dependency> - </dependencies> - </dependencyManagement> - - <build> - <plugins> - <!-- Build Docker image --> - <plugin> - <groupId>com.spotify</groupId> - <artifactId>dockerfile-maven-plugin</artifactId> - <version>${dockerfile.maven.version}</version> - <executions> - <execution> - <id>tag-latest</id> - <phase>pre-integration-test</phase> - <goals> - <goal>build</goal> - <goal>tag</goal> - </goals> - <configuration> - <skip>${docker.build.skip}</skip> - <pullNewerImage>false</pullNewerImage> - <repository> - apachehudi/hudi-hadoop_${docker.hadoop.version}-trinocoordinator_${docker.trino.version} - </repository> - <force>true</force> - <tag>latest</tag> - </configuration> - </execution> - <execution> - <id>tag-version</id> - <phase>pre-integration-test</phase> - <goals> - <goal>build</goal> - <goal>tag</goal> - <!-- <goal>push</goal> --> - </goals> - <configuration> - <skip>${docker.build.skip}</skip> - <pullNewerImage>false</pullNewerImage> - <repository> - apachehudi/hudi-hadoop_${docker.hadoop.version}-trinocoordinator_${docker.trino.version} - </repository> - <force>true</force> - <tag>${project.version}</tag> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> diff --git a/docker/hoodie/hadoop/trinoworker/Dockerfile b/docker/hoodie/hadoop/trinoworker/Dockerfile deleted file mode 100644 index ae5b2766dc9d..000000000000 --- a/docker/hoodie/hadoop/trinoworker/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Trino docker setup is adapted from https://github.com/Lewuathe/docker-trino-cluster - -ARG HADOOP_VERSION=2.8.4 -ARG TRINO_VERSION=368 -FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-trinobase_${TRINO_VERSION}:latest as trino-base - -ADD etc /usr/local/trino/etc -EXPOSE 8092 - -WORKDIR /usr/local/trino -ENTRYPOINT [ "./scripts/trino.sh" ] diff --git a/docker/hoodie/hadoop/trinoworker/etc/catalog/hive.properties b/docker/hoodie/hadoop/trinoworker/etc/catalog/hive.properties deleted file mode 100644 index ed7fce1b3e64..000000000000 --- a/docker/hoodie/hadoop/trinoworker/etc/catalog/hive.properties +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -connector.name=hive -hive.metastore.uri=thrift://hivemetastore:9083 -hive.config.resources=/etc/hadoop/core-site.xml,/etc/hadoop/hdfs-site.xml -hive.hdfs.authentication.type=NONE diff --git a/docker/hoodie/hadoop/trinoworker/etc/config.properties b/docker/hoodie/hadoop/trinoworker/etc/config.properties deleted file mode 100644 index 0e15d3d7c1e9..000000000000 --- a/docker/hoodie/hadoop/trinoworker/etc/config.properties +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -coordinator=false -http-server.http.port=8091 -query.max-memory=50GB -query.max-memory-per-node=1GB -query.max-total-memory-per-node=2GB -discovery.uri=http://trino-coordinator-1:8091 diff --git a/docker/hoodie/hadoop/trinoworker/etc/jvm.config b/docker/hoodie/hadoop/trinoworker/etc/jvm.config deleted file mode 100644 index fb17203ca211..000000000000 --- a/docker/hoodie/hadoop/trinoworker/etc/jvm.config +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# --server --Xmx16G --XX:+UseG1GC --XX:G1HeapRegionSize=32M --XX:+UseGCOverheadLimit --XX:+ExplicitGCInvokesConcurrent --XX:+HeapDumpOnOutOfMemoryError --XX:OnOutOfMemoryError=kill -9 %p --Djdk.attach.allowAttachSelf=true diff --git a/docker/hoodie/hadoop/trinoworker/etc/log.properties b/docker/hoodie/hadoop/trinoworker/etc/log.properties deleted file mode 100644 index 23b063080b4f..000000000000 --- a/docker/hoodie/hadoop/trinoworker/etc/log.properties +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -io.trinosql=INFO diff --git a/docker/hoodie/hadoop/trinoworker/etc/node.properties b/docker/hoodie/hadoop/trinoworker/etc/node.properties deleted file mode 100644 index 6cfebf995602..000000000000 --- a/docker/hoodie/hadoop/trinoworker/etc/node.properties +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -node.environment=development -node.id=6606f0b3-6ae7-4152-a4b1-ddadb6345fe6 -node.data-dir=/var/trino/data diff --git a/docker/hoodie/hadoop/trinoworker/pom.xml b/docker/hoodie/hadoop/trinoworker/pom.xml deleted file mode 100644 index 0bb55cf5d1f9..000000000000 --- a/docker/hoodie/hadoop/trinoworker/pom.xml +++ /dev/null @@ -1,96 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, - ~ software distributed under the License is distributed on an - ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - ~ KIND, either express or implied. See the License for the - ~ specific language governing permissions and limitations - ~ under the License. - --> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <parent> - <artifactId>hudi-hadoop-docker</artifactId> - <groupId>org.apache.hudi</groupId> - <version>1.3.0-SNAPSHOT</version> - </parent> - <modelVersion>4.0.0</modelVersion> - <packaging>pom</packaging> - <artifactId>hudi-hadoop-trinoworker-docker</artifactId> - <description>Trino Worker Docker Image with Hudi</description> - - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <checkstyle.skip>true</checkstyle.skip> - <main.basedir>${project.parent.parent.basedir}</main.basedir> - </properties> - - <dependencyManagement> - <dependencies> - <dependency> - <groupId>org.apache.hudi</groupId> - <artifactId>hudi-hadoop-trinobase-docker</artifactId> - <version>${project.version}</version> - <type>pom</type> - </dependency> - </dependencies> - </dependencyManagement> - - <build> - <plugins> - <!-- Build Docker image --> - <plugin> - <groupId>com.spotify</groupId> - <artifactId>dockerfile-maven-plugin</artifactId> - <version>${dockerfile.maven.version}</version> - <executions> - <execution> - <id>tag-latest</id> - <phase>pre-integration-test</phase> - <goals> - <goal>build</goal> - <goal>tag</goal> - </goals> - <configuration> - <skip>${docker.build.skip}</skip> - <pullNewerImage>false</pullNewerImage> - <repository> - apachehudi/hudi-hadoop_${docker.hadoop.version}-trinoworker_${docker.trino.version} - </repository> - <force>true</force> - <tag>latest</tag> - </configuration> - </execution> - <execution> - <id>tag-version</id> - <phase>pre-integration-test</phase> - <goals> - <goal>build</goal> - <goal>tag</goal> - <!-- <goal>push</goal> --> - </goals> - <configuration> - <skip>${docker.build.skip}</skip> - <pullNewerImage>false</pullNewerImage> - <repository> - apachehudi/hudi-hadoop_${docker.hadoop.version}-trinoworker_${docker.trino.version} - </repository> - <force>true</force> - <tag>${project.version}</tag> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> diff --git a/scripts/release/validate_staged_bundles.sh b/scripts/release/validate_staged_bundles.sh index 142b1664112d..ea723474df56 100755 --- a/scripts/release/validate_staged_bundles.sh +++ b/scripts/release/validate_staged_bundles.sh @@ -88,7 +88,7 @@ declare -a bundles=("hudi-aws-bundle" "hudi-azure-bundle" "hudi-cli-bundle_2.12" "hudi-flink2.0-bundle" "hudi-flink2.1-bundle" "hudi-gcp-bundle" "hudi-hadoop-mr-bundle" "hudi-hive-sync-bundle" "hudi-integ-test-bundle" "hudi-kafka-connect-bundle" "hudi-metaserver-server-bundle" "hudi-presto-bundle" "hudi-spark3.4-bundle_2.12" "hudi-spark3.5-bundle_2.12" -"hudi-spark3.5-bundle_2.13" "hudi-spark4.0-bundle_2.13" "hudi-spark4.1-bundle_2.13" "hudi-timeline-server-bundle" "hudi-trino-bundle" +"hudi-spark3.5-bundle_2.13" "hudi-spark4.0-bundle_2.13" "hudi-spark4.1-bundle_2.13" "hudi-timeline-server-bundle" "hudi-utilities-bundle_2.12" "hudi-utilities-bundle_2.13" "hudi-utilities-slim-bundle_2.12" "hudi-utilities-slim-bundle_2.13")
