This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new d2ce078f2d8 HIVE-27295: Improve docker logging in AbstractExternalDB
and DatabaseRule (#4268) (Laszlo Bodor reviewed by Stamatis Zampetakis)
d2ce078f2d8 is described below
commit d2ce078f2d8584f39fbf8329c88c544f019464f8
Author: Bodor Laszlo <[email protected]>
AuthorDate: Thu Apr 27 09:08:09 2023 +0200
HIVE-27295: Improve docker logging in AbstractExternalDB and DatabaseRule
(#4268) (Laszlo Bodor reviewed by Stamatis Zampetakis)
---
.../hive/ql/externalDB/AbstractExternalDB.java | 39 ++++++++++++++--------
.../metastore/dbinstall/rules/DatabaseRule.java | 34 +++++++++++++------
2 files changed, 48 insertions(+), 25 deletions(-)
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/AbstractExternalDB.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/AbstractExternalDB.java
index f328bfc4bc6..48da3344277 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/AbstractExternalDB.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/AbstractExternalDB.java
@@ -83,14 +83,12 @@ public abstract class AbstractExternalDB {
return new String[] { "docker", "logs", getDockerContainerName() };
}
-
private ProcessResults runCmd(String[] cmd, long secondsToWait)
throws IOException, InterruptedException {
LOG.info("Going to run: " + String.join(" ", cmd));
Process proc = Runtime.getRuntime().exec(cmd);
- if (!proc.waitFor(secondsToWait, TimeUnit.SECONDS)) {
- throw new RuntimeException(
- "Process " + cmd[0] + " failed to run in " + secondsToWait
+ " seconds");
+ if (!proc.waitFor(Math.abs(secondsToWait), TimeUnit.SECONDS)) {
+ throw new RuntimeException("Process " + cmd[0] + " failed to run in
" + secondsToWait + " seconds");
}
BufferedReader reader = new BufferedReader(new
InputStreamReader(proc.getInputStream()));
final StringBuilder lines = new StringBuilder();
@@ -99,41 +97,54 @@ public abstract class AbstractExternalDB {
reader = new BufferedReader(new
InputStreamReader(proc.getErrorStream()));
final StringBuilder errLines = new StringBuilder();
reader.lines().forEach(s -> errLines.append(s).append('\n'));
- LOG.info("Result size: " + lines.length() + ";" + errLines.length());
+ LOG.info("Result lines#: {}(stdout);{}(stderr)",lines.length(),
errLines.length());
return new ProcessResults(lines.toString(), errLines.toString(),
proc.exitValue());
}
- private int runCmdAndPrintStreams(String[] cmd, long secondsToWait)
+ private ProcessResults runCmdAndPrintStreams(String[] cmd, long
secondsToWait)
throws InterruptedException, IOException {
ProcessResults results = runCmd(cmd, secondsToWait);
LOG.info("Stdout from proc: " + results.stdout);
LOG.info("Stderr from proc: " + results.stderr);
- return results.rc;
+ return results;
}
public void launchDockerContainer() throws Exception {
runCmdAndPrintStreams(buildRmCmd(), 600);
- if (runCmdAndPrintStreams(buildRunCmd(), 600) != 0) {
- throw new RuntimeException("Unable to start docker container");
+ if (runCmdAndPrintStreams(buildRunCmd(), 600).rc != 0) {
+ printDockerEvents();
+ throw new RuntimeException("Unable to start docker container");
}
long startTime = System.currentTimeMillis();
ProcessResults pr;
do {
Thread.sleep(1000);
- pr = runCmd(buildLogCmd(), 30);
+ pr = runCmdAndPrintStreams(buildLogCmd(), 30);
if (pr.rc != 0) {
- throw new RuntimeException("Failed to get docker logs");
+ printDockerEvents();
+ throw new RuntimeException("Failed to get docker logs");
}
} while (startTime + MAX_STARTUP_WAIT >= System.currentTimeMillis() &&
!isContainerReady(pr));
if (startTime + MAX_STARTUP_WAIT < System.currentTimeMillis()) {
- throw new RuntimeException("Container failed to be ready in " +
MAX_STARTUP_WAIT/1000 +
- " seconds");
+ printDockerEvents();
+ throw new RuntimeException(
+ String.format("Container initialization failed within %d
seconds. Please check the hive logs.",
+ MAX_STARTUP_WAIT / 1000));
}
+ }
+
+ protected void printDockerEvents() {
+ try {
+ runCmdAndPrintStreams(new String[] { "docker", "events", "--since",
"24h", "--until", "0s" }, 3);
+ } catch (Exception e) {
+ LOG.warn("A problem was encountered while attempting to retrieve
Docker events (the system made an analytical"
+ + " best effort to list the events to reveal the root cause). No
further actions are necessary.", e);
+ }
}
public void cleanupDockerContainer() throws IOException,
InterruptedException {
- if (runCmdAndPrintStreams(buildRmCmd(), 600) != 0) {
+ if (runCmdAndPrintStreams(buildRmCmd(), 600).rc != 0) {
throw new RuntimeException("Unable to remove docker container");
}
}
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/DatabaseRule.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/DatabaseRule.java
index 4fc8d50bb76..23115ab916a 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/DatabaseRule.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/DatabaseRule.java
@@ -125,21 +125,25 @@ public abstract class DatabaseRule extends
ExternalResource {
@Override
public void before() throws Exception { //runDockerContainer
runCmdAndPrintStreams(buildRmCmd(), 600);
- if (runCmdAndPrintStreams(buildRunCmd(), 600) != 0) {
+ if (runCmdAndPrintStreams(buildRunCmd(), 600).rc != 0) {
+ printDockerEvents();
throw new RuntimeException("Unable to start docker container");
}
long startTime = System.currentTimeMillis();
ProcessResults pr;
do {
Thread.sleep(1000);
- pr = runCmd(buildLogCmd(), 30);
+ pr = runCmdAndPrintStreams(buildLogCmd(), 30);
if (pr.rc != 0) {
+ printDockerEvents();
throw new RuntimeException("Failed to get docker logs");
}
} while (startTime + MAX_STARTUP_WAIT >= System.currentTimeMillis() &&
!isContainerReady(pr));
if (startTime + MAX_STARTUP_WAIT < System.currentTimeMillis()) {
- throw new RuntimeException("Container failed to be ready in " +
MAX_STARTUP_WAIT/1000 +
- " seconds");
+ printDockerEvents();
+ throw new RuntimeException(
+ String.format("Container initialization failed within %d seconds.
Please check the hive logs.",
+ MAX_STARTUP_WAIT / 1000));
}
MetastoreSchemaTool.setHomeDirForTesting();
}
@@ -152,7 +156,7 @@ public abstract class DatabaseRule extends ExternalResource
{
return;
}
try {
- if (runCmdAndPrintStreams(buildRmCmd(), 600) != 0) {
+ if (runCmdAndPrintStreams(buildRmCmd(), 600).rc != 0) {
throw new RuntimeException("Unable to remove docker container");
}
} catch (InterruptedException | IOException e) {
@@ -174,9 +178,8 @@ public abstract class DatabaseRule extends ExternalResource
{
throws IOException, InterruptedException {
LOG.info("Going to run: " + StringUtils.join(cmd, " "));
Process proc = Runtime.getRuntime().exec(cmd);
- if (!proc.waitFor(secondsToWait, TimeUnit.SECONDS)) {
- throw new RuntimeException(
- "Process " + cmd[0] + " failed to run in " + secondsToWait + "
seconds");
+ if (!proc.waitFor(Math.abs(secondsToWait), TimeUnit.SECONDS)) {
+ throw new RuntimeException("Process " + cmd[0] + " failed to run in " +
secondsToWait + " seconds");
}
BufferedReader reader = new BufferedReader(new
InputStreamReader(proc.getInputStream()));
final StringBuilder lines = new StringBuilder();
@@ -185,16 +188,25 @@ public abstract class DatabaseRule extends
ExternalResource {
reader = new BufferedReader(new InputStreamReader(proc.getErrorStream()));
final StringBuilder errLines = new StringBuilder();
reader.lines().forEach(s -> errLines.append(s).append('\n'));
- LOG.info("Result size: " + lines.length() + ";" + errLines.length());
+ LOG.info("Result lines#: {}(stdout);{}(stderr)",lines.length(),
errLines.length());
return new ProcessResults(lines.toString(), errLines.toString(),
proc.exitValue());
}
- private int runCmdAndPrintStreams(String[] cmd, long secondsToWait)
+ private ProcessResults runCmdAndPrintStreams(String[] cmd, long
secondsToWait)
throws InterruptedException, IOException {
ProcessResults results = runCmd(cmd, secondsToWait);
LOG.info("Stdout from proc: " + results.stdout);
LOG.info("Stderr from proc: " + results.stderr);
- return results.rc;
+ return results;
+ }
+
+ protected void printDockerEvents() {
+ try {
+ runCmdAndPrintStreams(new String[] { "docker", "events", "--since",
"24h", "--until", "0s" }, 3);
+ } catch (Exception e) {
+ LOG.warn("A problem was encountered while attempting to retrieve Docker
events (the system made an analytical"
+ + " best effort to list the events to reveal the root cause). No
further actions are necessary.", e);
+ }
}
private String[] buildRunCmd() {
