hemanthumashankar0511 commented on code in PR #6640: URL: https://github.com/apache/hive/pull/6640#discussion_r4316883875
########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java: ########## @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.tez.yarn; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.SafeMode; +import org.apache.hadoop.fs.SafeModeAction; +import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; +import org.testcontainers.images.builder.ImageFromDockerfile; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; + +/** + * Starts an HDFS + YARN cluster from docker-compose.yml via Testcontainers ComposeContainer. + * Fixed published ports plus custom_hosts_file let the host JVM reach namenode/resourcemanager. + * HDFS and YARN are driven through the Hadoop client APIs rather than in-container CLIs. + */ +public class TezYarnClusterContainer { + + private static final Logger LOG = LoggerFactory.getLogger(TezYarnClusterContainer.class); + + public static final String CONTAINER_JAVA_HOME = "/opt/jdk21"; + + private final String hadoop_image; + + private static final int NN_RPC_PORT = 8020; + private static final int NN_HTTP_PORT = 9870; + private static final int RM_RPC_PORT = 8032; + private static final int RM_HTTP_PORT = 8088; + private static final int NM_HTTP_PORT = 8042; + // Tez AM client RPC port range, published by the NM container so the host JVM can reach the AM. + public static final int AM_CLIENT_PORT_START = 41000; + public static final int AM_CLIENT_PORT_END = 41020; + + // Compose V2 names service containers "<service>-1"; used by getContainerByServiceName(). + private static final int SERVICE_INSTANCE = 1; + + private static final Duration CLUSTER_READY_TIMEOUT = Duration.ofMinutes(2); + private static final Duration CLUSTER_READY_POLL_INTERVAL = Duration.ofSeconds(3); + private static final long RM_CONNECT_MAX_WAIT_MS = 2000; + private static final long RM_CONNECT_RETRY_INTERVAL_MS = 500; + + private final ComposeContainer compose; + + public TezYarnClusterContainer() { + hadoop_image = buildHadoopImage(); + String basedir = System.getProperty("basedir", "."); + File composeFile = Paths.get(basedir, "src/test/docker/hadoop-yarn/docker-compose.yml").toFile(); + LOG.info("Starting Tez-on-YARN cluster from {} (image {})", composeFile, hadoop_image); + // withLocalCompose(false): no local docker-compose binary required on CI. + compose = new ComposeContainer(composeFile) + .withLocalCompose(false); + } + + public void start() throws Exception { + try { + compose.start(); + // Avoid flakiness: wait until HDFS has left safemode before running HDFS operations. + waitForSafemodeExit(); + waitForNodeManagerRegistration(); + verifyJava21InNodeManager(); + } catch (Exception e) { + try { + stop(); + } catch (Exception stopException) { + LOG.info("Failed to stop TezYarnClusterContainer after startup failure", stopException); + } + throw e; + } + } + + public void stop() { + compose.stop(); + } + + private void verifyJava21InNodeManager() throws Exception { + Container.ExecResult r = nodeManagerContainer().execInContainer( + CONTAINER_JAVA_HOME + "/bin/java", "-version"); + if (r.getExitCode() == 0) { + LOG.info("Java 21 is functional in NodeManager ({}): {}", + CONTAINER_JAVA_HOME, r.getStderr().trim()); + } else { + throw new IllegalStateException( + "Java check failed in NodeManager (exit " + r.getExitCode() + + "). stderr: " + r.getStderr()); + } + } Review Comment: Will remove this check entirely. waitForNodeManagerRegistration() already validates NM readiness, and /opt/jdk21/bin/java -version is already validated during image build, so this runtime check was redundant -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
