fix #1958: better test report organizing
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/f048d6f2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/f048d6f2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/f048d6f2 Branch: refs/heads/master Commit: f048d6f2112ab1ee1024def9f1ceec9dc347cc42 Parents: 058bbf5 Author: Qian Xu <[email protected]> Authored: Fri Feb 5 15:23:53 2016 +0800 Committer: Qian Xu <[email protected]> Committed: Fri Feb 5 15:33:00 2016 +0800 ---------------------------------------------------------------------- integrationtest/autorun/autorun.sh | 17 ++++----- .../minicluster/BaseContainer.scala | 14 ++++++-- .../minicluster/RestClient.scala | 38 ++++++++++++-------- 3 files changed, 43 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f048d6f2/integrationtest/autorun/autorun.sh ---------------------------------------------------------------------- diff --git a/integrationtest/autorun/autorun.sh b/integrationtest/autorun/autorun.sh index e9944e0..4963995 100644 --- a/integrationtest/autorun/autorun.sh +++ b/integrationtest/autorun/autorun.sh @@ -34,6 +34,8 @@ git checkout -q master # Define the variables DIST_DIR=output/target/pack +LOG_DIR=/tmp/gearpump +ATTACHMENT=everything.zip REPORT_DIR=report COMMIT_REV='' @@ -51,7 +53,7 @@ function run_test_if_new_commit_found { COMMIT_REV=$NEW_COMMIT_REV COMMIT_LOG=$(git log -1) - rm -rf $DIST_DIR + rm -Rf $DIST_DIR $LOG_DIR $ATTACHMENT echo "Rebuild the project ..." sbt clean assembly packArchiveZip if [ $? -ne 0 ]; then @@ -62,9 +64,9 @@ function run_test_if_new_commit_found { fi echo "Run tests ... (it will take couple of minutes or longer)" - sbt "it:test-only *Suite* -- -h $REPORT_DIR" | tee console.log + sbt "it:test-only *Suite* -- -h $REPORT_DIR" | tee $REPORT_DIR/console.out if [ $? -eq 0 ]; then - grep -q "sbt.TestsFailedException: Tests unsuccessful" console.log + grep -q "sbt.TestsFailedException: Tests unsuccessful" $REPORT_DIR/console.out if [ $? -ne 0 ]; then mail_to \ "Gearpump test passed $COMMIT_REV" \ @@ -74,15 +76,14 @@ function run_test_if_new_commit_found { fi echo "Copy test logs into report directory ..." - SELECTION="$REPORT_DIR $DIST_DIR/logs console.log" - ZIP_FILES="$NEW_COMMIT_REV.zip" - zip -q -r $ZIP_FILES "$SELECTION" - rm -rf "$SELECTION" + SELECTION="$REPORT_DIR $LOG_DIR" + zip -q -r $ATTACHMENT $SELECTION + rm -Rf $SELECTION mail_to \ "Gearpump test failed $COMMIT_REV" \ "Integration test failed. Please check attached files.\n\n$COMMIT_LOG" \ - -a $ZIP_FILES + -a $ATTACHMENT } while true; do http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f048d6f2/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/BaseContainer.scala ---------------------------------------------------------------------- diff --git a/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/BaseContainer.scala b/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/BaseContainer.scala index b283295..1d7ddd9 100644 --- a/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/BaseContainer.scala +++ b/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/BaseContainer.scala @@ -29,8 +29,14 @@ class BaseContainer(val host: String, command: String, tunnelPorts: Set[Int] = Set.empty) { private val IMAGE_NAME = "stanleyxu2005/gearpump-launcher" - private val SUT_LOCAL_PATH = "pwd".!!.trim + "/output/target/pack" - val SUT_HOME = "/opt/gearpump" + private val IMAGE_SUT_HOME = "/opt/gearpump" + private val IMAGE_LOG_HOME = "/var/log/gearpump" + private val LOCAL_SUT_HOME = "pwd".!!.trim + "/output/target/pack" + private val LOCAL_LOG_HOME = { + val dir = "/tmp/gearpump" + s"mkdir -p $dir".!! + s"mktemp -p $dir -d".!!.trim + } private val CLUSTER_OPTS = { masterAddrs.zipWithIndex.map { case (hostPort, index) => @@ -41,7 +47,9 @@ class BaseContainer(val host: String, command: String, def createAndStart(): String = { Docker.createAndStartContainer(host, IMAGE_NAME, command, environ = Map("JAVA_OPTS" -> CLUSTER_OPTS), - volumes = Map(SUT_LOCAL_PATH -> SUT_HOME), + volumes = Map( + LOCAL_SUT_HOME -> IMAGE_SUT_HOME, + LOCAL_LOG_HOME -> IMAGE_LOG_HOME), knownHosts = masterAddrs.map(_._1).filter(_ != host).toSet, tunnelPorts = tunnelPorts) } http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/f048d6f2/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/RestClient.scala ---------------------------------------------------------------------- diff --git a/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/RestClient.scala b/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/RestClient.scala index 07e9975..2b3003c 100644 --- a/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/RestClient.scala +++ b/integrationtest/core/src/main/scala/io/gearpump/integrationtest/minicluster/RestClient.scala @@ -53,13 +53,21 @@ class RestClient(host: String, port: Int) { Graph(vertexList, edgeList) } + private def decodeAs[T: upickle.default.Reader](expr: String): T = try { + read[T](expr) + } catch { + case ex: Throwable => + LOG.error(ex) + throw ex + } + def queryVersion(): String = { callFromRoot("version") } def listWorkers(): Array[WorkerSummary] = { val resp = callApi("master/workerlist") - read[List[WorkerSummary]](resp).toArray + decodeAs[List[WorkerSummary]](resp).toArray } def listRunningWorkers(): Array[WorkerSummary] = { @@ -68,7 +76,7 @@ class RestClient(host: String, port: Int) { def listApps(): Array[AppMasterData] = { val resp = callApi("master/applist") - read[AppMastersData](resp).appMasters.toArray + decodeAs[AppMastersData](resp).appMasters.toArray } def listRunningApps(): Array[AppMasterData] = { @@ -89,7 +97,7 @@ class RestClient(host: String, port: Int) { options :+= s"conf=@$config" } val resp = callApi(endpoint, options.map("-F " + _).mkString(" ")) - val result = read[AppSubmissionResult](resp) + val result = decodeAs[AppSubmissionResult](resp) assert(result.success) true } catch { @@ -100,7 +108,7 @@ class RestClient(host: String, port: Int) { def queryApp(appId: Int): AppMasterData = { val resp = callApi(s"appmaster/$appId") - read[AppMasterData](resp) + decodeAs[AppMasterData](resp) } def queryAppMasterConfig(appId: Int): Config = { @@ -110,18 +118,18 @@ class RestClient(host: String, port: Int) { def queryStreamingAppDetail(appId: Int): StreamAppMasterSummary = { val resp = callApi(s"appmaster/$appId?detail=true") - upickle.default.read[StreamAppMasterSummary](resp) + decodeAs[StreamAppMasterSummary](resp) } def queryStreamingAppMetrics(appId: Int, current: Boolean, path: String = "processor*"): HistoryMetrics = { val args = if (current) "?readLatest=true" else "" val resp = callApi(s"appmaster/$appId/metrics/app$appId.$path$args") - upickle.default.read[HistoryMetrics](resp) + decodeAs[HistoryMetrics](resp) } def queryExecutorSummary(appId: Int, executorId: Int): ExecutorSummary = { val resp = callApi(s"appmaster/$appId/executor/$executorId") - upickle.default.read[ExecutorSummary](resp) + decodeAs[ExecutorSummary](resp) } def queryExecutorBrief(appId: Int): Array[ExecutorBrief] = { @@ -131,7 +139,7 @@ class RestClient(host: String, port: Int) { def queryExecutorMetrics(appId: Int, current: Boolean): HistoryMetrics = { val args = if (current) "?readLatest=true" else "" val resp = callApi(s"appmaster/$appId/metrics/app$appId.executor*$args") - upickle.default.read[HistoryMetrics](resp) + decodeAs[HistoryMetrics](resp) } def queryExecutorConfig(appId: Int, executorId: Int): Config = { @@ -141,13 +149,13 @@ class RestClient(host: String, port: Int) { def queryMaster(): MasterSummary = { val resp = callApi("master") - read[MasterData](resp).masterDescription + decodeAs[MasterData](resp).masterDescription } def queryMasterMetrics(current: Boolean): HistoryMetrics = { val args = if (current) "?readLatest=true" else "" val resp = callApi(s"master/metrics/master?$args") - upickle.default.read[HistoryMetrics](resp) + decodeAs[HistoryMetrics](resp) } def queryMasterConfig(): Config = { @@ -158,7 +166,7 @@ class RestClient(host: String, port: Int) { def queryWorkerMetrics(workerId: Int, current: Boolean): HistoryMetrics = { val args = if (current) "?readLatest=true" else "" val resp = callApi(s"worker/$workerId/metrics/worker$workerId?$args") - upickle.default.read[HistoryMetrics](resp) + decodeAs[HistoryMetrics](resp) } def queryWorkerConfig(workerId: Int): Config = { @@ -168,19 +176,19 @@ class RestClient(host: String, port: Int) { def queryBuiltInPartitioners(): Array[String] = { val resp = callApi("master/partitioners") - upickle.default.read[BuiltinPartitioners](resp).partitioners + decodeAs[BuiltinPartitioners](resp).partitioners } def uploadJar(localFilePath: String): AppJar = { val resp = callApi(s"master/uploadjar -F jar=@$localFilePath", CRUD_POST) - upickle.default.read[AppJar](resp) + decodeAs[AppJar](resp) } def replaceStreamingAppProcessor(appId: Int, replaceMe: ProcessorDescription): Boolean = try { val replaceOperation = new ReplaceProcessor(replaceMe.id, replaceMe) val args = upickle.default.write(replaceOperation) val resp = callApi(s"appmaster/$appId/dynamicdag?args=" + Util.encodeUriComponent(args), CRUD_POST) - upickle.default.read[DAGOperationResult](resp) + decodeAs[DAGOperationResult](resp) true } catch { case ex: Throwable => @@ -214,7 +222,7 @@ class RestClient(host: String, port: Int) { def restartApp(appId: Int): Boolean = try { val resp = callApi(s"appmaster/$appId/restart", CRUD_POST) - upickle.default.read[Status](resp).success + decodeAs[Status](resp).success } catch { case ex: Throwable => LOG.warn(s"swallowed an exception: $ex")
