Repository: bigtop Updated Branches: refs/heads/master f482fd9fb -> f7d471b41
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f7d471b4/bigtop-packages/src/charm/zeppelin/layer-zeppelin/resources/hdfs-tutorial/note.json ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/resources/hdfs-tutorial/note.json b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/resources/hdfs-tutorial/note.json new file mode 100644 index 0000000..7cb4a5f --- /dev/null +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/resources/hdfs-tutorial/note.json @@ -0,0 +1,340 @@ +{ + "paragraphs": [ + { + "text": "%md\n## Welcome to Zeppelin.\n##### This is a live tutorial, you can run the code yourself. (Shift-Enter to Run)", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "editorHide": false + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423836981412_-1007008116", + "id": "20150213-231621_168813393", + "result": { + "code": "SUCCESS", + "type": "HTML", + "msg": "\u003ch2\u003eWelcome to Zeppelin.\u003c/h2\u003e\n\u003ch5\u003eThis is a live tutorial, you can run the code yourself. (Shift-Enter to Run)\u003c/h5\u003e\n" + }, + "dateCreated": "Feb 13, 2015 11:16:21 PM", + "dateStarted": "Apr 1, 2015 9:11:09 PM", + "dateFinished": "Apr 1, 2015 9:11:10 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "title": "Prepare data", + "text": "import sys.process._\n//you will need \u0027wget\u0027 tool to download\n\"wget https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv\"; !\n\"rm -rf zeppelin-hdfs-tutorial\" !\n\"mkdir zeppelin-hdfs-tutorial\" !\n\"mv bank.csv zeppelin-hdfs-tutorial\" !\n\"hdfs dfs -rm -r /user/zeppelin/zeppelin-hdfs-tutorial\" !\n\"hdfs dfs -copyFromLocal zeppelin-hdfs-tutorial /user/zeppelin\" !", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "title": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1417656535623_-196593192", + "id": "20141204-102855_1590713432", + "result": { + "code": "SUCCESS", + "type": "TEXT", + "msg": "import sys.process._\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres1: Int \u003d 0\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres2: Int \u003d 0\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres3: Int \u003d 0\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres4: Int \u003d 0\n" + }, + "dateCreated": "Dec 4, 2014 10:28:55 AM", + "dateStarted": "Apr 1, 2015 9:11:12 PM", + "dateFinished": "Apr 1, 2015 9:11:22 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "title": "Load data into table", + "text": "// sc is an existing SparkContext.\nval sqlContext \u003d new org.apache.spark.sql.SQLContext(sc)\nval bankText \u003d sc.textFile(s\"hdfs:///user/zeppelin/zeppelin-hdfs-tutorial/bank.csv\")\n\ncase class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)\n\nval bank \u003d bankText.map(s \u003d\u003e s.split(\";\")).filter(s \u003d\u003e s(0) !\u003d \"\\\"age\\\"\").map(\n s \u003d\u003e Bank(s(0).toInt, \n s(1).replaceAll(\"\\\"\", \"\"),\n s(2).replaceAll(\"\\\"\", \"\"),\n s(3).replaceAll(\"\\\"\", \"\"),\n s(5).replaceAll(\"\\\"\", \"\").toInt\n )\n).toDF()\nbank.registerTempTable(\"bank\")\n\n", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "title": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423500779206_-1502780787", + "id": "20150210-015259_1403135953", + "result": { + "code": "SUCCESS", + "type": "TEXT", + "msg": "import sys.process._\nsqlContext: org.apache.spark.sql.SQLContext \u003d org.apache.spark.sql.SQLContext@2c91e2d6\nwarning: there were 1 feature warning(s); re-run with -feature for details\nzeppelinHome: String \u003d /home/langley/lab/incubator-zeppelin\nbankText: org.apache.spark.rdd.RDD[String] \u003d /home/langley/lab/incubator-zeppelin/data/bank-full.csv MapPartitionsRDD[1] at textFile at \u003cconsole\u003e:31\ndefined class Bank\nbank: org.apache.spark.sql.DataFrame \u003d [age: int, job: string, marital: string, education: string, balance: int]\n" + }, + "dateCreated": "Feb 10, 2015 1:52:59 AM", + "dateStarted": "Apr 1, 2015 9:11:28 PM", + "dateFinished": "Apr 1, 2015 9:11:39 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value\nfrom bank \nwhere age \u003c 30 \ngroup by age \norder by age", + "config": { + "colWidth": 4.0, + "graph": { + "mode": "multiBarChart", + "height": 300.0, + "optionOpen": false, + "keys": [ + { + "name": "age", + "index": 0.0, + "aggr": "sum" + } + ], + "values": [ + { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + ], + "groups": [], + "scatter": { + "xAxis": { + "name": "age", + "index": 0.0, + "aggr": "sum" + }, + "yAxis": { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + } + } + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423500782552_-1439281894", + "id": "20150210-015302_1492795503", + "result": { + "code": "SUCCESS", + "type": "TABLE", + "msg": "age\tvalue\n18\t12\n19\t35\n20\t50\n21\t79\n22\t129\n23\t202\n24\t302\n25\t527\n26\t805\n27\t909\n28\t1038\n29\t1185\n" + }, + "dateCreated": "Feb 10, 2015 1:53:02 AM", + "dateStarted": "Apr 1, 2015 9:11:43 PM", + "dateFinished": "Apr 1, 2015 9:11:45 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere age \u003c ${maxAge\u003d30} \ngroup by age \norder by age", + "config": { + "colWidth": 4.0, + "graph": { + "mode": "multiBarChart", + "height": 300.0, + "optionOpen": false, + "keys": [ + { + "name": "age", + "index": 0.0, + "aggr": "sum" + } + ], + "values": [ + { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + ], + "groups": [], + "scatter": { + "xAxis": { + "name": "age", + "index": 0.0, + "aggr": "sum" + }, + "yAxis": { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + } + } + }, + "settings": { + "params": { + "maxAge": "35" + }, + "forms": { + "maxAge": { + "name": "maxAge", + "defaultValue": "30", + "hidden": false + } + } + }, + "jobName": "paragraph_1423720444030_-1424110477", + "id": "20150212-145404_867439529", + "result": { + "code": "SUCCESS", + "type": "TABLE", + "msg": "age\tvalue\n18\t12\n19\t35\n20\t50\n21\t79\n22\t129\n23\t202\n24\t302\n25\t527\n26\t805\n27\t909\n28\t1038\n29\t1185\n30\t1757\n31\t1996\n32\t2085\n33\t1972\n34\t1930\n" + }, + "dateCreated": "Feb 12, 2015 2:54:04 PM", + "dateStarted": "Apr 1, 2015 9:12:03 PM", + "dateFinished": "Apr 1, 2015 9:12:03 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere marital\u003d\"${marital\u003dsingle,single|divorced|married}\" \ngroup by age \norder by age", + "config": { + "colWidth": 4.0, + "graph": { + "mode": "multiBarChart", + "height": 300.0, + "optionOpen": false, + "keys": [ + { + "name": "age", + "index": 0.0, + "aggr": "sum" + } + ], + "values": [ + { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + ], + "groups": [], + "scatter": { + "xAxis": { + "name": "age", + "index": 0.0, + "aggr": "sum" + }, + "yAxis": { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + } + } + }, + "settings": { + "params": { + "marital": "single" + }, + "forms": { + "marital": { + "name": "marital", + "defaultValue": "single", + "options": [ + { + "value": "single" + }, + { + "value": "divorced" + }, + { + "value": "married" + } + ], + "hidden": false + } + } + }, + "jobName": "paragraph_1423836262027_-210588283", + "id": "20150213-230422_1600658137", + "result": { + "code": "SUCCESS", + "type": "TABLE", + "msg": "age\tvalue\n18\t12\n19\t35\n20\t47\n21\t74\n22\t120\n23\t175\n24\t248\n25\t423\n26\t615\n27\t658\n28\t697\n29\t683\n30\t1012\n31\t1017\n32\t941\n33\t746\n34\t650\n35\t631\n36\t538\n37\t453\n38\t394\n39\t346\n40\t257\n41\t241\n42\t218\n43\t183\n44\t170\n45\t146\n46\t130\n47\t100\n48\t124\n49\t101\n50\t76\n51\t72\n52\t62\n53\t71\n54\t55\n55\t54\n56\t45\n57\t38\n58\t35\n59\t36\n60\t27\n61\t5\n63\t2\n66\t5\n67\t3\n68\t4\n69\t2\n70\t1\n71\t1\n72\t5\n73\t2\n77\t1\n83\t2\n86\t1\n" + }, + "dateCreated": "Feb 13, 2015 11:04:22 PM", + "dateStarted": "Apr 1, 2015 9:12:10 PM", + "dateFinished": "Apr 1, 2015 9:12:10 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%md\n## Congratulations, it\u0027s done.\n##### You can create your own notebook in \u0027Notebook\u0027 menu. Good luck!", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + } + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423836268492_216498320", + "id": "20150213-230428_1231780373", + "result": { + "code": "SUCCESS", + "type": "HTML", + "msg": "\u003ch2\u003eCongratulations, it\u0027s done.\u003c/h2\u003e\n\u003ch5\u003eYou can create your own notebook in \u0027Notebook\u0027 menu. Good luck!\u003c/h5\u003e\n" + }, + "dateCreated": "Feb 13, 2015 11:04:28 PM", + "dateStarted": "Apr 1, 2015 9:12:18 PM", + "dateFinished": "Apr 1, 2015 9:12:18 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "config": {}, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1427420818407_872443482", + "id": "20150326-214658_12335843", + "dateCreated": "Mar 26, 2015 9:46:58 PM", + "status": "READY", + "progressUpdateIntervalMs": 500 + } + ], + "name": "Zeppelin HDFS Tutorial", + "id": "hdfs-tutorial", + "config": { + "looknfeel": "default" + }, + "info": {} +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/f7d471b4/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/01-zeppelin-smoke.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/01-zeppelin-smoke.py b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/01-zeppelin-smoke.py new file mode 100755 index 0000000..2cd1a11 --- /dev/null +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/01-zeppelin-smoke.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import amulet + + +class TestDeploy(unittest.TestCase): + """ + Deployment and smoke test for the Apache Bigtop Zeppelin service. + """ + @classmethod + def setUpClass(cls): + cls.d = amulet.Deployment(series='trusty') + cls.d.add('zeppelin', 'zeppelin') + cls.d.add('resourcemanager', 'hadoop-resourcemanager') + cls.d.add('namenode', 'hadoop-namenode') + cls.d.add('slave', 'hadoop-slave') + cls.d.add('plugin', 'hadoop-plugin') + + cls.d.relate('plugin:hadoop-plugin', 'zeppelin:hadoop') + cls.d.relate('plugin:namenode', 'namenode:namenode') + cls.d.relate('plugin:resourcemanager', 'resourcemanager:resourcemanager') + cls.d.relate('slave:namenode', 'namenode:datanode') + cls.d.relate('slave:resourcemanager', 'resourcemanager:nodemanager') + cls.d.relate('namenode:namenode', 'resourcemanager:namenode') + + cls.d.setup(timeout=3600) + cls.d.sentry.wait_for_messages({'zeppelin': 'ready'}, timeout=3600) + cls.zeppelin = cls.d.sentry['zeppelin'][0] + + def test_zeppelin(self): + """ + Validate Zeppelin by running the smoke-test action. + """ + uuid = self.zeppelin.action_do('smoke-test') + output = self.d.action_fetch(uuid, full_output=True) + assert "completed" in output['status'] + + +if __name__ == '__main__': + unittest.main() http://git-wip-us.apache.org/repos/asf/bigtop/blob/f7d471b4/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/tests.yaml ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/tests.yaml b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/tests.yaml new file mode 100644 index 0000000..3b6ce3e --- /dev/null +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/tests.yaml @@ -0,0 +1,3 @@ +reset: false +packages: + - amulet http://git-wip-us.apache.org/repos/asf/bigtop/blob/f7d471b4/bigtop-packages/src/charm/zeppelin/layer-zeppelin/wheelhouse.txt ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/wheelhouse.txt b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/wheelhouse.txt new file mode 100644 index 0000000..d555101 --- /dev/null +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/wheelhouse.txt @@ -0,0 +1 @@ +requests>=2.0.0,<3.0.0
