This is an automated email from the ASF dual-hosted git repository.
xkrogen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 32925d0 HDFS-14640. [Dynamometer] Fix TestDynamometerInfra failure.
Contributed by Erik Krogen.
32925d0 is described below
commit 32925d04d96548308ea14df9ab0dd68473fae60d
Author: Erik Krogen <[email protected]>
AuthorDate: Tue Jul 9 15:15:08 2019 -0700
HDFS-14640. [Dynamometer] Fix TestDynamometerInfra failure. Contributed by
Erik Krogen.
---
.../hadoop-dynamometer-infra/pom.xml | 10 ++++++++++
.../src/main/bash/create-slim-hadoop-tar.sh | 2 +-
.../src/main/bash/upload-fsimage.sh | 4 ++--
.../src/main/resources/start-component.sh | 9 +++------
.../tools/dynamometer/TestDynamometerInfra.java | 21 ++++++++++++++++++---
.../src/main/bash/parse-start-timestamp.sh | 4 ++--
6 files changed, 36 insertions(+), 14 deletions(-)
diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml
b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml
index 4aaf639..00f313c 100644
--- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml
+++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml
@@ -128,6 +128,16 @@
</excludes>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <forkedProcessTimeoutInSeconds>1800</forkedProcessTimeoutInSeconds>
+ <environmentVariables>
+ <JAVA_HOME>${java.home}</JAVA_HOME>
+ </environmentVariables>
+ </configuration>
+ </plugin>
</plugins>
</build>
diff --git
a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/bash/create-slim-hadoop-tar.sh
b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/bash/create-slim-hadoop-tar.sh
index 4904a31..12669a3 100755
---
a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/bash/create-slim-hadoop-tar.sh
+++
b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/bash/create-slim-hadoop-tar.sh
@@ -36,7 +36,7 @@ hadoopTarTmp="$hadoopTar.temporary"
mkdir -p "$hadoopTarTmp"
tar xzf "$hadoopTar" -C "$hadoopTarTmp"
-baseDir="$(find -H "$hadoopTarTmp" -depth 1 -type d | head -n 1)" # Should
only be one subdir
+baseDir="$(find -H "$hadoopTarTmp" -maxdepth 1 -mindepth 1 -type d | head -n
1)" # Should only be one subdir
hadoopShare="$baseDir/share/hadoop"
# Remove unnecessary files
diff --git
a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/bash/upload-fsimage.sh
b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/bash/upload-fsimage.sh
index 862dd5b..8308adb 100755
---
a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/bash/upload-fsimage.sh
+++
b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/bash/upload-fsimage.sh
@@ -38,12 +38,12 @@ else
name_dir="$(pwd)"
fi
-image_file_count="$(find -H "${name_dir}" -depth 1 -name
"fsimage_*$image_txid" -type f | wc -l)"
+image_file_count="$(find -H "${name_dir}" -maxdepth 1 -mindepth 1 -name
"fsimage_*$image_txid" -type f | wc -l)"
if [[ "$image_file_count" != 1 ]]; then
echo "Error; found $image_file_count matching fsimage files."
exit 1
fi
-image_file="$(find -H "${name_dir}" -depth 1 -name "fsimage_*$image_txid"
-type f)"
+image_file="$(find -H "${name_dir}" -maxdepth 1 -mindepth 1 -name
"fsimage_*$image_txid" -type f)"
image_file_name="$(basename "${image_file}")"
echo "Using fsimage: $image_file_name"
image_file_md5="${image_file}.md5"
diff --git
a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh
b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh
index 08b3c85..e1974ca 100644
---
a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh
+++
b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh
@@ -75,7 +75,7 @@ chmod 755 "$baseDir"
chmod 700 "$pidDir"
# Set Hadoop variables for component
-hadoopHome="$(find -H "$(pwd)/hadoopBinary" -depth 1 -type d | head -n 1)"
+hadoopHome="$(find -H "$(pwd)/hadoopBinary" -maxdepth 1 -mindepth 1 -type d |
head -n 1)"
# Save real environment for later
hadoopConfOriginal=${HADOOP_CONF_DIR:-$confDir}
hadoopHomeOriginal=${HADOOP_HOME:-$hadoopHome}
@@ -252,11 +252,8 @@ EOF
rm -rf "$nameDir" "$editsDir" "$checkpointDir"
mkdir -p "$nameDir/current" "$editsDir/current" "$checkpointDir"
chmod -R 700 "$nameDir" "$editsDir" "$checkpointDir"
- fsImageFile="$(find "$(pwd)" -depth 1 -name "fsimage_*" | tail -n 1)"
- fsImageMD5File="$(find "$(pwd)" -depth 1 -name "fsimage_*.md5" | tail -n 1)"
- ln -snf "$fsImageFile" "$nameDir/current/$(basename "$fsImageFile")"
- ln -snf "$fsImageMD5File" "$nameDir/current/$(basename "$fsImageMD5File")"
- ln -snf "$(pwd)/VERSION" "$nameDir/current/VERSION"
+ # Link all of the fsimage files into the name dir
+ find "$(pwd)" -maxdepth 1 -mindepth 1 \( -name "fsimage_*" -or -name
"VERSION" \) -execdir ln -snf "$(pwd)/{}" "$nameDir/current/{}" \;
chmod 700 "$nameDir"/current/*
namenodeConfigs=(
diff --git
a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/java/org/apache/hadoop/tools/dynamometer/TestDynamometerInfra.java
b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/java/org/apache/hadoop/tools/dynamometer/TestDynamometerInfra.java
index b5b780b..5d5ccf1 100644
---
a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/java/org/apache/hadoop/tools/dynamometer/TestDynamometerInfra.java
+++
b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/java/org/apache/hadoop/tools/dynamometer/TestDynamometerInfra.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.tools.dynamometer;
import com.google.common.collect.Sets;
import java.util.Optional;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import org.apache.hadoop.test.PlatformAssumptions;
import
org.apache.hadoop.tools.dynamometer.workloadgenerator.audit.AuditLogDirectParser;
@@ -315,16 +316,30 @@ public class TestDynamometerInfra {
awaitApplicationStartup();
- Supplier<Boolean> falseSupplier = () -> false;
+ long startTime = System.currentTimeMillis();
+ long maxWaitTimeMs = TimeUnit.MINUTES.toMillis(10);
+ Supplier<Boolean> exitCheckSupplier = () -> {
+ if (System.currentTimeMillis() - startTime > maxWaitTimeMs) {
+ // Wait at most 10 minutes for the NameNode to start and be ready
+ return true;
+ }
+ try {
+ // Exit immediately if the YARN app fails
+ return yarnClient.getApplicationReport(infraAppId)
+ .getYarnApplicationState() == YarnApplicationState.FAILED;
+ } catch (IOException | YarnException e) {
+ return true;
+ }
+ };
Optional<Properties> namenodeProperties = DynoInfraUtils
- .waitForAndGetNameNodeProperties(falseSupplier, localConf,
+ .waitForAndGetNameNodeProperties(exitCheckSupplier, localConf,
client.getNameNodeInfoPath(), LOG);
if (!namenodeProperties.isPresent()) {
fail("Unable to fetch NameNode properties");
}
DynoInfraUtils.waitForNameNodeReadiness(namenodeProperties.get(), 3, false,
- falseSupplier, localConf, LOG);
+ exitCheckSupplier, localConf, LOG);
assertClusterIsFunctional(localConf, namenodeProperties.get());
diff --git
a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/bash/parse-start-timestamp.sh
b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/bash/parse-start-timestamp.sh
index 5b4a827..f7e299e 100755
---
a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/bash/parse-start-timestamp.sh
+++
b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/bash/parse-start-timestamp.sh
@@ -38,12 +38,12 @@ else
edits_dir="$(pwd)"
fi
-edits_file_count="$(find -H "${edits_dir}" -depth 1 -type f -name
"edits_*-*$image_txid" | wc -l)"
+edits_file_count="$(find -H "${edits_dir}" -maxdepth 1 -type f -name
"edits_*-*$image_txid" | wc -l)"
if [[ "$edits_file_count" != 1 ]]; then
echo "Error; found $edits_file_count matching edit files."
exit 1
fi
-edits_file="$(find -H "${edits_dir}" -depth 1 -type f -name
"edits_*-*$image_txid")"
+edits_file="$(find -H "${edits_dir}" -maxdepth 1 -type f -name
"edits_*-*$image_txid")"
# Shellcheck complains about the $ in the single-quote because it won't
expand, but this is intentional
# shellcheck disable=SC2016
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
