This is an automated email from the ASF dual-hosted git repository.
evansye pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 538778b BIGTOP-3217. Added smoke tests for Livy (#534)
538778b is described below
commit 538778b57c262cd2df8588bfe748439ce36bf8fc
Author: Aki Tanaka <[email protected]>
AuthorDate: Wed Jul 24 10:06:24 2019 -0700
BIGTOP-3217. Added smoke tests for Livy (#534)
---
bigtop-deploy/puppet/manifests/cluster.pp | 1 +
.../puppet/modules/livy/manifests/init.pp | 2 +-
bigtop-packages/src/common/hadoop/init-hcfs.json | 1 +
bigtop-tests/smoke-tests/livy/TestLivy.groovy | 112 +++++++++++++++++++++
bigtop-tests/smoke-tests/livy/build.gradle | 38 +++++++
bigtop.bom | 2 +-
provisioner/utils/smoke-tests.sh | 1 +
7 files changed, 155 insertions(+), 2 deletions(-)
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp
b/bigtop-deploy/puppet/manifests/cluster.pp
index 6e4f1d0..31d4950 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -209,6 +209,7 @@ class node_with_roles ($roles = hiera("bigtop::roles"))
inherits hadoop_cluster_
"sqoop2",
"hadoop_zookeeper",
"hcatalog",
+ "livy",
"mahout",
"solr",
"spark",
diff --git a/bigtop-deploy/puppet/modules/livy/manifests/init.pp
b/bigtop-deploy/puppet/modules/livy/manifests/init.pp
index b17a0e1..c832ef4 100644
--- a/bigtop-deploy/puppet/modules/livy/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/livy/manifests/init.pp
@@ -32,7 +32,7 @@ class livy {
}
file { '/etc/livy/conf/livy.conf':
- content => template('livy/livy.json'),
+ content => template('livy/livy.conf'),
require => Package['livy'],
}
diff --git a/bigtop-packages/src/common/hadoop/init-hcfs.json
b/bigtop-packages/src/common/hadoop/init-hcfs.json
index 6225a69..15a23af 100644
--- a/bigtop-packages/src/common/hadoop/init-hcfs.json
+++ b/bigtop-packages/src/common/hadoop/init-hcfs.json
@@ -73,6 +73,7 @@
["/user/hive","777",null,null],
["/user/root","777","root",null],
["/user/hue","777","hue","hue"],
+ ["/user/livy","777","livy","livy"],
["/user/sqoop","777","sqoop",null],
["/user/oozie","777","oozie"],
["/user/oozie/share",null,null,null],
diff --git a/bigtop-tests/smoke-tests/livy/TestLivy.groovy
b/bigtop-tests/smoke-tests/livy/TestLivy.groovy
new file mode 100644
index 0000000..8e6f577
--- /dev/null
+++ b/bigtop-tests/smoke-tests/livy/TestLivy.groovy
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.bigtop.itest.hadoop.livy
+
+import com.google.gson.Gson;
+import org.apache.bigtop.itest.shell.Shell
+import org.apache.commons.logging.Log
+import org.apache.commons.logging.LogFactory
+import org.junit.Test
+
+import static org.junit.Assert.assertEquals
+
+class TestLivy {
+ static private Log LOG = LogFactory.getLog(Object.class)
+ static Shell sh = new Shell("/bin/bash -s")
+ static String cmdPrefix = "export
HADOOP_CONF_DIR=/etc/hadoop/conf;HADOOP_USER_NAME=hdfs"
+ static Gson gson = new Gson();
+ static private final String config_file = "/etc/livy/conf/livy.conf";
+
+ private static void execCommand(String cmd) {
+ LOG.info(cmd)
+ sh.exec("$cmdPrefix $cmd")
+ }
+
+ private LivyResponse fromJSON(String data) {
+ LivyResponse response = gson.fromJson(data, LivyResponse.class);
+ return response;
+ }
+
+ private boolean waitUntilReady(String uri) {
+ int retryCount = 30;
+ for (int i=0;i<retryCount;i++) {
+ execCommand("curl " + uri);
+ String state = fromJSON(sh.out.join('\n')).getState();
+ if (state.equals("idle") || state.equals("available")) {
+ return true;
+ }
+ try {
+ Thread.sleep(3000);
+ } catch (InterruptedException e) {
+ }
+ }
+ return false;
+ }
+
+ @Test
+ void testCheckLivyRestAPIs() {
+ // read Livy host and port from config file
+ execCommand("awk '{if(/^livy.server.port/) print \$2}' < "+ config_file);
+ String port = sh.out.join('\n');
+ if (port.equals("")) {
+ port = "8998";
+ }
+ execCommand("awk '{if(/^livy.server.host:/) print \$2}' < "+config_file);
+ String host = sh.out.join('\n');
+ if (host.equals("")) {
+ host = "127.0.0.1";
+ }
+ String baseURL = "http://"; + host + ":" + port;
+
+ execCommand("curl " + baseURL + "/ui");
+ String result = sh.out.join('\n');
+ assert(result.contains("Livy"));
+
+ // 1. Create Livy Session
+ execCommand("curl -X POST --data '{\"kind\": \"pyspark\"}' -H
\"Content-Type: application/json\" " + baseURL + "/sessions")
+ result = sh.out.join('\n');
+ assert(result.contains("starting"));
+ int sessionId = fromJSON(result).getId();
+ assert(waitUntilReady(baseURL + "/sessions/" + sessionId));
+
+ // 2. Launch Spark job
+ execCommand("curl " + baseURL + "/sessions/" + sessionId + "/statements -X
POST -H 'Content-Type: application/json' -d '{\"code\":\"1 + 1\"}'");
+ result = sh.out.join('\n');
+ int statementId = fromJSON(result).getId();
+ assert(waitUntilReady(baseURL + "/sessions/" + sessionId + "/statements/"
+ statementId));
+
+ // 3. Kill Livy Session
+ execCommand("curl localhost:8998/sessions/" + sessionId + " -X DELETE")
+ result = sh.out.join('\n');
+ assert(result.contains("deleted"));
+ }
+
+ private class LivyResponse {
+ public int id;
+ public String state;
+
+ private int getId() {
+ return id;
+ }
+
+ private String getState() {
+ return state;
+ }
+ }
+}
diff --git a/bigtop-tests/smoke-tests/livy/build.gradle
b/bigtop-tests/smoke-tests/livy/build.gradle
new file mode 100644
index 0000000..cd36e61
--- /dev/null
+++ b/bigtop-tests/smoke-tests/livy/build.gradle
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+def tests_to_include() {
+ return [
+ "TestLivy.groovy"
+ ];
+}
+
+dependencies {
+ compile 'com.google.code.gson:gson:2.3'
+}
+
+sourceSets {
+ test {
+ groovy {
+ srcDirs = ["${BIGTOP_HOME}/bigtop-tests/smoke-tests/livy/"]
+ }
+ }
+}
+
+test.doFirst {
+ checkEnv(["LIVY_HOME"])
+}
diff --git a/bigtop.bom b/bigtop.bom
index fed4eaa..bafca95 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -118,7 +118,7 @@ bigtop {
hbase:['phoenix','giraph','ycsb','hive'],
hive:['oozie', 'zeppelin'],
'ignite-hadoop':['zeppelin'],
- spark:['zeppelin']
+ spark:['livy', 'zeppelin']
]
components {
diff --git a/provisioner/utils/smoke-tests.sh b/provisioner/utils/smoke-tests.sh
index 9a0921d..1ef75ce 100755
--- a/provisioner/utils/smoke-tests.sh
+++ b/provisioner/utils/smoke-tests.sh
@@ -49,6 +49,7 @@ export SQOOP_HOME=${SQOOP_HOME:-/usr/lib/sqoop}
export ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/usr/lib/zookeeper}
export GIRAPH_HOME=${GIRAPH_HOME:-/usr/lib/giraph}
export FLINK_HOME=${FLINK_HOME:-/usr/lib/flink}
+export LIVY_HOME=${LIVY_HOME:-/usr/lib/livy}
echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n"
