http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-plugins/notebookrepo/github/src/test/resources/notebook/my_project/my_note1_2A94M5J1Z.zpln ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/github/src/test/resources/notebook/my_project/my_note1_2A94M5J1Z.zpln b/zeppelin-plugins/notebookrepo/github/src/test/resources/notebook/my_project/my_note1_2A94M5J1Z.zpln new file mode 100644 index 0000000..8d7d2df --- /dev/null +++ b/zeppelin-plugins/notebookrepo/github/src/test/resources/notebook/my_project/my_note1_2A94M5J1Z.zpln @@ -0,0 +1,369 @@ +{ + "paragraphs": [ + { + "text": "%md\n## Welcome to Zeppelin.\n##### This is a live tutorial, you can run the code yourself. (Shift-Enter to Run)", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "editorHide": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423836981412_-1007008116", + "id": "20150213-231621_168813393", + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "HTML", + "data": "\u003ch2\u003eWelcome to Zeppelin.\u003c/h2\u003e\n\u003ch5\u003eThis is a live tutorial, you can run the code yourself. (Shift-Enter to Run)\u003c/h5\u003e\n" + } + ] + }, + "dateCreated": "Feb 13, 2015 11:16:21 PM", + "dateStarted": "Apr 1, 2015 9:11:09 PM", + "dateFinished": "Apr 1, 2015 9:11:10 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "title": "Load data into table", + "text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load bank data\nval bankText \u003d sc.parallelize(\n IOUtils.toString(\n new URL(\"https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv\";),\n Charset.forName(\"utf8\")).split(\"\\n\"))\n\ncase class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)\n\nval bank \u003d bankText.map(s \u003d\u003e s.split(\";\")).filter(s \u003d\u003e s(0) !\u003d \"\\\"age\\\"\").map(\n s \u003d\u003e Bank(s(0).toInt, \n s(1).replaceAll(\"\\\"\", \"\"),\n s(2).replaceAll(\"\\\"\", \"\"),\n s(3).replaceAll(\"\\\"\", \"\"),\n s(5).replaceAll(\"\\\"\", \"\").toInt\n )\n).toDF()\nbank.registerTempTable(\"bank\")", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "title": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423500779206_-1502780787", + "id": "20150210-015259_1403135953", + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "TEXT", + "data": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nbankText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[32] at parallelize at \u003cconsole\u003e:65\ndefined class Bank\nbank: org.apache.spark.sql.DataFrame \u003d [age: int, job: string, marital: string, education: string, balance: int]\n" + } + ] + }, + "dateCreated": "Feb 10, 2015 1:52:59 AM", + "dateStarted": "Jul 3, 2015 1:43:40 PM", + "dateFinished": "Jul 3, 2015 1:43:45 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value\nfrom bank \nwhere age \u003c 30 \ngroup by age \norder by age", + "config": { + "colWidth": 4.0, + "graph": { + "mode": "multiBarChart", + "height": 300.0, + "optionOpen": false, + "keys": [ + { + "name": "age", + "index": 0.0, + "aggr": "sum" + } + ], + "values": [ + { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + ], + "groups": [], + "scatter": { + "xAxis": { + "name": "age", + "index": 0.0, + "aggr": "sum" + }, + "yAxis": { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + } + } + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423500782552_-1439281894", + "id": "20150210-015302_1492795503", + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "TABLE", + "data": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n" + } + ] + }, + "dateCreated": "Feb 10, 2015 1:53:02 AM", + "dateStarted": "Jul 3, 2015 1:43:17 PM", + "dateFinished": "Jul 3, 2015 1:43:23 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere age \u003c ${maxAge\u003d30} \ngroup by age \norder by age", + "config": { + "colWidth": 4.0, + "graph": { + "mode": "multiBarChart", + "height": 300.0, + "optionOpen": false, + "keys": [ + { + "name": "age", + "index": 0.0, + "aggr": "sum" + } + ], + "values": [ + { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + ], + "groups": [], + "scatter": { + "xAxis": { + "name": "age", + "index": 0.0, + "aggr": "sum" + }, + "yAxis": { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + } + } + }, + "settings": { + "params": { + "maxAge": "35" + }, + "forms": { + "maxAge": { + "name": "maxAge", + "defaultValue": "30", + "hidden": false + } + } + }, + "jobName": "paragraph_1423720444030_-1424110477", + "id": "20150212-145404_867439529", + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "TABLE", + "data": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n30\t150\n31\t199\n32\t224\n33\t186\n34\t231\n" + } + ] + }, + "dateCreated": "Feb 12, 2015 2:54:04 PM", + "dateStarted": "Jul 3, 2015 1:43:28 PM", + "dateFinished": "Jul 3, 2015 1:43:29 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere marital\u003d\"${marital\u003dsingle,single|divorced|married}\" \ngroup by age \norder by age", + "config": { + "colWidth": 4.0, + "graph": { + "mode": "multiBarChart", + "height": 300.0, + "optionOpen": false, + "keys": [ + { + "name": "age", + "index": 0.0, + "aggr": "sum" + } + ], + "values": [ + { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + ], + "groups": [], + "scatter": { + "xAxis": { + "name": "age", + "index": 0.0, + "aggr": "sum" + }, + "yAxis": { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + } + } + }, + "settings": { + "params": { + "marital": "single" + }, + "forms": { + "marital": { + "name": "marital", + "defaultValue": "single", + "options": [ + { + "value": "single" + }, + { + "value": "divorced" + }, + { + "value": "married" + } + ], + "hidden": false + } + } + }, + "jobName": "paragraph_1423836262027_-210588283", + "id": "20150213-230422_1600658137", + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "TABLE", + "data": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t17\n24\t13\n25\t33\n26\t56\n27\t64\n28\t78\n29\t56\n30\t92\n31\t86\n32\t105\n33\t61\n34\t75\n35\t46\n36\t50\n37\t43\n38\t44\n39\t30\n40\t25\n41\t19\n42\t23\n43\t21\n44\t20\n45\t15\n46\t14\n47\t12\n48\t12\n49\t11\n50\t8\n51\t6\n52\t9\n53\t4\n55\t3\n56\t3\n57\t2\n58\t7\n59\t2\n60\t5\n66\t2\n69\t1\n" + } + ] + }, + "dateCreated": "Feb 13, 2015 11:04:22 PM", + "dateStarted": "Jul 3, 2015 1:43:33 PM", + "dateFinished": "Jul 3, 2015 1:43:34 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%md\n## Congratulations, it\u0027s done.\n##### You can create your own notebook in \u0027Notebook\u0027 menu. Good luck!", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "editorHide": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423836268492_216498320", + "id": "20150213-230428_1231780373", + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "HTML", + "data": "\u003ch2\u003eCongratulations, it\u0027s done.\u003c/h2\u003e\n\u003ch5\u003eYou can create your own notebook in \u0027Notebook\u0027 menu. Good luck!\u003c/h5\u003e\n" + } + ] + }, + "dateCreated": "Feb 13, 2015 11:04:28 PM", + "dateStarted": "Apr 1, 2015 9:12:18 PM", + "dateFinished": "Apr 1, 2015 9:12:18 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%md\n\nAbout bank data\n\n```\nCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n```";, + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "editorHide": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1427420818407_872443482", + "id": "20150326-214658_12335843", + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "HTML", + "data": "\u003cp\u003eAbout bank data\u003c/p\u003e\n\u003cpre\u003e\u003ccode\u003eCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n\u003c/code\u003e\u003c/pre\u003e\n"; + } + ] + }, + "dateCreated": "Mar 26, 2015 9:46:58 PM", + "dateStarted": "Jul 3, 2015 1:44:56 PM", + "dateFinished": "Jul 3, 2015 1:44:56 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "config": {}, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1435955447812_-158639899", + "id": "20150703-133047_853701097", + "dateCreated": "Jul 3, 2015 1:30:47 PM", + "status": "READY", + "progressUpdateIntervalMs": 500 + } + ], + "id": "2A94M5J1Z", + "name": "my_note1", + "angularObjects": {}, + "config": { + "looknfeel": "default" + }, + "info": {} +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-plugins/notebookrepo/mongodb/pom.xml ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/mongodb/pom.xml b/zeppelin-plugins/notebookrepo/mongodb/pom.xml deleted file mode 100644 index 9df8f02..0000000 --- a/zeppelin-plugins/notebookrepo/mongodb/pom.xml +++ /dev/null @@ -1,63 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one or more - ~ contributor license agreements. See the NOTICE file distributed with - ~ this work for additional information regarding copyright ownership. - ~ The ASF licenses this file to You under the Apache License, Version 2.0 - ~ (the "License"); you may not use this file except in compliance with - ~ the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - - <modelVersion>4.0.0</modelVersion> - - <parent> - <artifactId>zengine-plugins-parent</artifactId> - <groupId>org.apache.zeppelin</groupId> - <version>0.9.0-SNAPSHOT</version> - <relativePath>../../../zeppelin-plugins</relativePath> - </parent> - - <groupId>org.apache.zeppelin</groupId> - <artifactId>notebookrepo-mongodb</artifactId> - <packaging>jar</packaging> - <version>0.9.0-SNAPSHOT</version> - <name>Zeppelin: Plugin MongoNotebookRepo</name> - <description>NotebookRepo implementation based on Mongodb</description> - - <properties> - <plugin.name>NotebookRepo/MongoNotebookRepo</plugin.name> - </properties> - - <dependencies> - <dependency> - <groupId>org.apache.zeppelin</groupId> - <artifactId>notebookrepo-vfs</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.mongodb</groupId> - <artifactId>mongo-java-driver</artifactId> - <version>3.4.1</version> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <artifactId>maven-dependency-plugin</artifactId> - </plugin> - </plugins> - </build> -</project> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java b/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java deleted file mode 100644 index 618568d..0000000 --- a/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java +++ /dev/null @@ -1,222 +0,0 @@ -package org.apache.zeppelin.notebook.repo; - -import static com.mongodb.client.model.Filters.eq; -import static com.mongodb.client.model.Filters.in; -import static com.mongodb.client.model.Filters.type; - -import com.mongodb.MongoBulkWriteException; -import com.mongodb.MongoClient; -import com.mongodb.MongoClientURI; -import com.mongodb.bulk.BulkWriteError; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoCursor; -import com.mongodb.client.MongoDatabase; -import com.mongodb.client.model.InsertManyOptions; -import com.mongodb.client.model.UpdateOptions; -import java.io.IOException; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.notebook.Note; -import org.apache.zeppelin.notebook.NoteInfo; -import org.apache.zeppelin.user.AuthenticationInfo; -import org.bson.BsonType; -import org.bson.Document; -import org.bson.types.ObjectId; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Backend for storing Notebook on MongoDB - */ -public class MongoNotebookRepo implements NotebookRepo { - private static final Logger LOG = LoggerFactory.getLogger(MongoNotebookRepo.class); - - private ZeppelinConfiguration conf; - private MongoClient mongo; - private MongoDatabase db; - private MongoCollection<Document> coll; - - public MongoNotebookRepo() { - - } - - public void init(ZeppelinConfiguration conf) throws IOException { - this.conf = conf; - - mongo = new MongoClient(new MongoClientURI(conf.getMongoUri())); - db = mongo.getDatabase(conf.getMongoDatabase()); - coll = db.getCollection(conf.getMongoCollection()); - - if (conf.getMongoAutoimport()) { - // import local notes into MongoDB - insertFileSystemNotes(); - } - } - - /** - * If environment variable ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT is true, - * this method will insert local notes into MongoDB on startup. - * If a note already exists in MongoDB, skip it. - */ - private void insertFileSystemNotes() throws IOException { - LinkedList<Document> docs = new LinkedList<>(); // docs to be imported - NotebookRepo vfsRepo = new VFSNotebookRepo(); - vfsRepo.init(conf); - List<NoteInfo> infos = vfsRepo.list(null); - // collect notes to be imported - for (NoteInfo info : infos) { - Note note = vfsRepo.get(info.getId(), null); - Document doc = noteToDocument(note); - docs.add(doc); - } - - /* - * 'ordered(false)' option allows to proceed bulk inserting even though - * there are duplicated documents. The duplicated documents will be skipped - * and print a WARN log. - */ - try { - coll.insertMany(docs, new InsertManyOptions().ordered(false)); - } catch (MongoBulkWriteException e) { - printDuplicatedException(e); //print duplicated document warning log - } - - vfsRepo.close(); // it does nothing for now but maybe in the future... - } - - /** - * MongoBulkWriteException contains error messages that inform - * which documents were duplicated. This method catches those ID and print them. - * @param e - */ - private void printDuplicatedException(MongoBulkWriteException e) { - List<BulkWriteError> errors = e.getWriteErrors(); - for (BulkWriteError error : errors) { - String msg = error.getMessage(); - Pattern pattern = Pattern.compile("[A-Z0-9]{9}"); // regex for note ID - Matcher matcher = pattern.matcher(msg); - if (matcher.find()) { // if there were a note ID - String noteId = matcher.group(); - LOG.warn("Note " + noteId + " not inserted since already exists in MongoDB"); - } - } - } - - @Override - public List<NoteInfo> list(AuthenticationInfo subject) throws IOException { - syncId(); - - List<NoteInfo> infos = new LinkedList<>(); - MongoCursor<Document> cursor = coll.find().iterator(); - - while (cursor.hasNext()) { - Document doc = cursor.next(); - Note note = documentToNote(doc); - NoteInfo info = new NoteInfo(note); - infos.add(info); - } - - cursor.close(); - - return infos; - } - - /** - * Find documents of which type of _id is object ID, and change it to note ID. - * Since updating _id field is not allowed, remove original documents and insert - * new ones with string _id(note ID) - */ - private void syncId() { - // find documents whose id type is object id - MongoCursor<Document> cursor = coll.find(type("_id", BsonType.OBJECT_ID)).iterator(); - // if there is no such document, exit - if (!cursor.hasNext()) - return; - - List<ObjectId> oldDocIds = new LinkedList<>(); // document ids need to update - List<Document> updatedDocs = new LinkedList<>(); // new documents to be inserted - - while (cursor.hasNext()) { - Document doc = cursor.next(); - // store original _id - ObjectId oldId = doc.getObjectId("_id"); - oldDocIds.add(oldId); - // store the document with string _id (note id) - String noteId = doc.getString("id"); - doc.put("_id", noteId); - updatedDocs.add(doc); - } - - coll.insertMany(updatedDocs); - coll.deleteMany(in("_id", oldDocIds)); - - cursor.close(); - } - - /** - * Convert document to note - */ - private Note documentToNote(Document doc) { - // document to JSON - String json = doc.toJson(); - // JSON to note - return Note.fromJson(json); - } - - /** - * Convert note to document - */ - private Document noteToDocument(Note note) { - // note to JSON - String json = note.toJson(); - // JSON to document - Document doc = Document.parse(json); - // set object id as note id - doc.put("_id", note.getId()); - return doc; - } - - @Override - public Note get(String noteId, AuthenticationInfo subject) throws IOException { - Document doc = coll.find(eq("_id", noteId)).first(); - - if (doc == null) { - throw new IOException("Note " + noteId + "not found"); - } - - return documentToNote(doc); - } - - @Override - public void save(Note note, AuthenticationInfo subject) throws IOException { - Document doc = noteToDocument(note); - coll.replaceOne(eq("_id", note.getId()), doc, new UpdateOptions().upsert(true)); - } - - @Override - public void remove(String noteId, AuthenticationInfo subject) throws IOException { - coll.deleteOne(eq("_id", noteId)); - } - - @Override - public void close() { - mongo.close(); - } - - @Override - public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) { - LOG.warn("Method not implemented"); - return Collections.emptyList(); - } - - @Override - public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { - LOG.warn("Method not implemented"); - } - -} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java index 364943c..c86c627 100644 --- a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java @@ -24,7 +24,7 @@ import java.io.InputStream; import java.io.OutputStreamWriter; import java.io.Writer; import java.util.Collections; -import java.util.LinkedList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -35,8 +35,6 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteInfo; -import org.apache.zeppelin.notebook.Paragraph; -import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.user.AuthenticationInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,7 +64,7 @@ import com.amazonaws.services.s3.model.S3ObjectSummary; * Backend for storing Notebooks on S3 */ public class S3NotebookRepo implements NotebookRepo { - private static final Logger LOG = LoggerFactory.getLogger(S3NotebookRepo.class); + private static final Logger LOGGER = LoggerFactory.getLogger(S3NotebookRepo.class); // Use a credential provider chain so that instance profiles can be utilized // on an EC2 instance. The order of locations where credentials are searched @@ -87,6 +85,7 @@ public class S3NotebookRepo implements NotebookRepo { private String user; private boolean useServerSideEncryption; private ZeppelinConfiguration conf; + private String rootFolder; public S3NotebookRepo() { @@ -96,6 +95,7 @@ public class S3NotebookRepo implements NotebookRepo { this.conf = conf; bucketName = conf.getS3BucketName(); user = conf.getS3User(); + rootFolder = user + "/notebook"; useServerSideEncryption = conf.isS3ServerSideEncryption(); // always use the default provider chain @@ -174,9 +174,8 @@ public class S3NotebookRepo implements NotebookRepo { } @Override - public List<NoteInfo> list(AuthenticationInfo subject) throws IOException { - List<NoteInfo> infos = new LinkedList<>(); - NoteInfo info; + public Map<String, NoteInfo> list(AuthenticationInfo subject) throws IOException { + Map<String, NoteInfo> notesInfo = new HashMap<>(); try { ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) @@ -185,10 +184,12 @@ public class S3NotebookRepo implements NotebookRepo { do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { - if (objectSummary.getKey().endsWith("note.json")) { - info = getNoteInfo(objectSummary.getKey()); - if (info != null) { - infos.add(info); + if (objectSummary.getKey().endsWith(".zpln")) { + try { + NoteInfo info = getNoteInfo(objectSummary.getKey()); + notesInfo.put(info.getId(), info); + } catch (IOException e) { + LOGGER.warn(e.getMessage()); } } } @@ -197,54 +198,45 @@ public class S3NotebookRepo implements NotebookRepo { } catch (AmazonClientException ace) { throw new IOException("Unable to list objects in S3: " + ace, ace); } - return infos; + return notesInfo; } - private Note getNote(String key) throws IOException { + private NoteInfo getNoteInfo(String key) throws IOException { + return new NoteInfo(getNoteId(key), getNotePath(rootFolder, key)); + } + + @Override + public Note get(String noteId, String notePath, AuthenticationInfo subject) throws IOException { S3Object s3object; try { - s3object = s3client.getObject(new GetObjectRequest(bucketName, key)); + s3object = s3client.getObject(new GetObjectRequest(bucketName, + rootFolder + "/" + buildNoteFileName(noteId, notePath))); } catch (AmazonClientException ace) { throw new IOException("Unable to retrieve object from S3: " + ace, ace); } - try (InputStream ins = s3object.getObjectContent()) { String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING)); return Note.fromJson(json); } } - private NoteInfo getNoteInfo(String key) throws IOException { - Note note = getNote(key); - return new NoteInfo(note); - } - - @Override - public Note get(String noteId, AuthenticationInfo subject) throws IOException { - return getNote(user + "/" + "notebook" + "/" + noteId + "/" + "note.json"); - } - @Override public void save(Note note, AuthenticationInfo subject) throws IOException { String json = note.toJson(); - String key = user + "/" + "notebook" + "/" + note.getId() + "/" + "note.json"; - - File file = File.createTempFile("note", "json"); + String key = rootFolder + "/" + buildNoteFileName(note); + File file = File.createTempFile("note", "zpln"); try { Writer writer = new OutputStreamWriter(new FileOutputStream(file)); writer.write(json); writer.close(); - PutObjectRequest putRequest = new PutObjectRequest(bucketName, key, file); - if (useServerSideEncryption) { // Request server-side encryption. ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); putRequest.setMetadata(objectMetadata); } - s3client.putObject(putRequest); } catch (AmazonClientException ace) { @@ -256,11 +248,25 @@ public class S3NotebookRepo implements NotebookRepo { } @Override - public void remove(String noteId, AuthenticationInfo subject) throws IOException { - String key = user + "/" + "notebook" + "/" + noteId; + public void move(String noteId, String notePath, String newNotePath, + AuthenticationInfo subject) throws IOException { + String key = rootFolder + "/" + buildNoteFileName(noteId, notePath); + String newKey = rootFolder + "/" + buildNoteFileName(noteId, newNotePath); + s3client.copyObject(bucketName, key, bucketName, newKey); + s3client.deleteObject(bucketName, key); + } + + @Override + public void move(String folderPath, String newFolderPath, AuthenticationInfo subject) { + + } + + @Override + public void remove(String noteId, String notePath, AuthenticationInfo subject) + throws IOException { + String key = rootFolder + "/" + buildNoteFileName(noteId, notePath); final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName).withPrefix(key); - try { ObjectListing objects = s3client.listObjects(listObjectsRequest); do { @@ -276,19 +282,24 @@ public class S3NotebookRepo implements NotebookRepo { } @Override + public void remove(String folderPath, AuthenticationInfo subject) { + + } + + @Override public void close() { //no-op } @Override public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) { - LOG.warn("Method not implemented"); + LOGGER.warn("Method not implemented"); return Collections.emptyList(); } @Override public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { - LOG.warn("Method not implemented"); + LOGGER.warn("Method not implemented"); } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-plugins/notebookrepo/vfs/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/vfs/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-plugins/notebookrepo/vfs/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java index 4294b86..3f62bff 100644 --- a/zeppelin-plugins/notebookrepo/vfs/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/vfs/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java @@ -17,24 +17,21 @@ package org.apache.zeppelin.notebook.repo; -import com.google.common.collect.Lists; import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedList; +import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; -import org.apache.commons.vfs2.FileContent; import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSystemException; import org.apache.commons.vfs2.FileSystemManager; -import org.apache.commons.vfs2.FileType; import org.apache.commons.vfs2.NameScope; import org.apache.commons.vfs2.Selectors; import org.apache.commons.vfs2.VFS; @@ -47,14 +44,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** -* +* NotebookRepo implementation based on apache vfs */ public class VFSNotebookRepo implements NotebookRepo { - private static final Logger LOG = LoggerFactory.getLogger(VFSNotebookRepo.class); + private static final Logger LOGGER = LoggerFactory.getLogger(VFSNotebookRepo.class); - private FileSystemManager fsManager; - private URI filesystemRoot; protected ZeppelinConfiguration conf; + protected FileSystemManager fsManager; + protected FileObject rootNotebookFileObject; + protected String rootNotebookFolder; public VFSNotebookRepo() { @@ -63,181 +61,140 @@ public class VFSNotebookRepo implements NotebookRepo { @Override public void init(ZeppelinConfiguration conf) throws IOException { this.conf = conf; - setNotebookDirectory(conf.getNotebookDir()); + setNotebookDirectory(conf.getRelativeDir(conf.getNotebookDir())); } protected void setNotebookDirectory(String notebookDirPath) throws IOException { + URI filesystemRoot = null; try { - LOG.info("Using notebookDir: " + notebookDirPath); + LOGGER.info("Using notebookDir: " + notebookDirPath); if (conf.isWindowsPath(notebookDirPath)) { filesystemRoot = new File(notebookDirPath).toURI(); } else { filesystemRoot = new URI(notebookDirPath); } - } catch (URISyntaxException e1) { - throw new IOException(e1); + } catch (URISyntaxException e) { + throw new IOException(e); } if (filesystemRoot.getScheme() == null) { // it is local path File f = new File(conf.getRelativeDir(filesystemRoot.getPath())); - this.filesystemRoot = f.toURI(); + filesystemRoot = f.toURI(); } - - fsManager = VFS.getManager(); - FileObject file = fsManager.resolveFile(filesystemRoot.getPath()); - if (!file.exists()) { - LOG.info("Notebook dir doesn't exist, create on is {}.", file.getName()); - file.createFolder(); - } - } - - private String getNotebookDirPath() { - return filesystemRoot.getPath().toString(); - } - - private String getPath(String path) { - if (path == null || path.trim().length() == 0) { - return filesystemRoot.toString(); - } - if (path.startsWith("/")) { - return filesystemRoot.toString() + path; - } else { - return filesystemRoot.toString() + "/" + path; - } - } - - private boolean isDirectory(FileObject fo) throws IOException { - if (fo == null) return false; - if (fo.getType() == FileType.FOLDER) { - return true; - } else { - return false; + this.fsManager = VFS.getManager(); + this.rootNotebookFileObject = fsManager.resolveFile(filesystemRoot); + if (!this.rootNotebookFileObject.exists()) { + this.rootNotebookFileObject.createFolder(); + LOGGER.info("Notebook dir doesn't exist: {}, creating it.", + rootNotebookFileObject.getName().getPath()); } + this.rootNotebookFolder = rootNotebookFileObject.getName().getPath(); } @Override - public List<NoteInfo> list(AuthenticationInfo subject) throws IOException { - FileObject rootDir = getRootDir(); - - FileObject[] children = rootDir.getChildren(); - - List<NoteInfo> infos = new LinkedList<>(); - for (FileObject f : children) { - String fileName = f.getName().getBaseName(); - if (f.isHidden() - || fileName.startsWith(".") - || fileName.startsWith("#") - || fileName.startsWith("~")) { - // skip hidden, temporary files - continue; - } - - if (!isDirectory(f)) { - // currently single note is saved like, [NOTE_ID]/note.json. - // so it must be a directory - continue; + public Map<String, NoteInfo> list(AuthenticationInfo subject) throws IOException { + // Must to create rootNotebookFileObject each time when call method list, otherwise we can not + // get the updated data under this folder. + this.rootNotebookFileObject = fsManager.resolveFile(this.rootNotebookFolder); + return listFolder(rootNotebookFileObject); + } + + private Map<String, NoteInfo> listFolder(FileObject fileObject) throws IOException { + Map<String, NoteInfo> noteInfos = new HashMap<>(); + if (fileObject.isFolder()) { + for (FileObject child : fileObject.getChildren()) { + noteInfos.putAll(listFolder(child)); } - - NoteInfo info = null; - - try { - info = getNoteInfo(f); - if (info != null) { - infos.add(info); + } else { + String noteFileName = fileObject.getName().getPath(); + if (noteFileName.endsWith(".zpln")) { + try { + String noteId = getNoteId(noteFileName); + String notePath = getNotePath(rootNotebookFolder, noteFileName); + noteInfos.put(noteId, new NoteInfo(noteId, notePath)); + } catch (IOException e) { + LOGGER.warn(e.getMessage()); } - } catch (Exception e) { - LOG.error("Can't read note " + f.getName().toString(), e); + + } else { + LOGGER.debug("Unrecognized note file: " + noteFileName); } } - - return infos; + return noteInfos; } - private Note getNote(FileObject noteDir) throws IOException { - if (!isDirectory(noteDir)) { - throw new IOException(noteDir.getName().toString() + " is not a directory"); - } - - FileObject noteJson = noteDir.resolveFile("note.json", NameScope.CHILD); - if (!noteJson.exists()) { - throw new IOException(noteJson.getName().toString() + " not found"); - } - - FileContent content = noteJson.getContent(); - InputStream ins = content.getInputStream(); - String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING)); - ins.close(); - - return Note.fromJson(json); + @Override + public Note get(String noteId, String notePath, AuthenticationInfo subject) throws IOException { + FileObject noteFile = rootNotebookFileObject.resolveFile(buildNoteFileName(noteId, notePath), + NameScope.DESCENDENT); + String json = IOUtils.toString(noteFile.getContent().getInputStream(), + conf.getString(ConfVars.ZEPPELIN_ENCODING)); + Note note = Note.fromJson(json); + // setPath here just for testing, because actually NoteManager will setPath + note.setPath(notePath); + return note; } - private NoteInfo getNoteInfo(FileObject noteDir) throws IOException { - Note note = getNote(noteDir); - return new NoteInfo(note); + @Override + public synchronized void save(Note note, AuthenticationInfo subject) throws IOException { + LOGGER.info("Saving note " + note.getId() + " to " + buildNoteFileName(note)); + // write to tmp file first, then rename it to the {note_name}_{note_id}.zpln + FileObject noteJson = rootNotebookFileObject.resolveFile( + buildNoteTempFileName(note), NameScope.DESCENDENT); + OutputStream out = null; + try { + out = noteJson.getContent().getOutputStream(false); + IOUtils.write(note.toJson().getBytes(conf.getString(ConfVars.ZEPPELIN_ENCODING)), out); + } finally { + if (out != null) { + out.close(); + } + } + noteJson.moveTo(rootNotebookFileObject.resolveFile( + buildNoteFileName(note), NameScope.DESCENDENT)); } @Override - public Note get(String noteId, AuthenticationInfo subject) throws IOException { - FileObject rootDir = fsManager.resolveFile(getPath("/")); - FileObject noteDir = rootDir.resolveFile(noteId, NameScope.CHILD); - - return getNote(noteDir); + public void move(String noteId, String notePath, String newNotePath, + AuthenticationInfo subject) throws IOException { + LOGGER.info("Move note " + noteId + " from " + notePath + " to " + newNotePath); + FileObject fileObject = rootNotebookFileObject.resolveFile( + buildNoteFileName(noteId, notePath), NameScope.DESCENDENT); + FileObject destFileObject = rootNotebookFileObject.resolveFile( + buildNoteFileName(noteId, newNotePath), NameScope.DESCENDENT); + // create parent folder first, otherwise move operation will fail + destFileObject.getParent().createFolder(); + fileObject.moveTo(destFileObject); } - protected FileObject getRootDir() throws IOException { - FileObject rootDir = fsManager.resolveFile(getPath("/")); - - if (!rootDir.exists()) { - throw new IOException("Root path does not exists"); - } - - if (!isDirectory(rootDir)) { - throw new IOException("Root path is not a directory"); - } - - return rootDir; + @Override + public void move(String folderPath, String newFolderPath, + AuthenticationInfo subject) throws IOException{ + LOGGER.info("Move folder from " + folderPath + " to " + newFolderPath); + FileObject fileObject = rootNotebookFileObject.resolveFile( + folderPath.substring(1), NameScope.DESCENDENT); + FileObject destFileObject = rootNotebookFileObject.resolveFile( + newFolderPath.substring(1), NameScope.DESCENDENT); + // create parent folder first, otherwise move operation will fail + destFileObject.getParent().createFolder(); + fileObject.moveTo(destFileObject); } @Override - public synchronized void save(Note note, AuthenticationInfo subject) throws IOException { - LOG.info("Saving note:" + note.getId()); - String json = note.toJson(); - - FileObject rootDir = getRootDir(); - - FileObject noteDir = rootDir.resolveFile(note.getId(), NameScope.CHILD); - - if (!noteDir.exists()) { - noteDir.createFolder(); - } - if (!isDirectory(noteDir)) { - throw new IOException(noteDir.getName().toString() + " is not a directory"); - } - - FileObject noteJson = noteDir.resolveFile(".note.json", NameScope.CHILD); - // false means not appending. creates file if not exists - OutputStream out = noteJson.getContent().getOutputStream(false); - out.write(json.getBytes(conf.getString(ConfVars.ZEPPELIN_ENCODING))); - out.close(); - noteJson.moveTo(noteDir.resolveFile("note.json", NameScope.CHILD)); + public void remove(String noteId, String notePath, AuthenticationInfo subject) + throws IOException { + LOGGER.info("Remove note: " + noteId + " +, notePath: " + notePath); + FileObject noteFile = rootNotebookFileObject.resolveFile( + buildNoteFileName(noteId, notePath), NameScope.DESCENDENT); + noteFile.delete(Selectors.SELECT_SELF); } @Override - public void remove(String noteId, AuthenticationInfo subject) throws IOException { - FileObject rootDir = fsManager.resolveFile(getPath("/")); - FileObject noteDir = rootDir.resolveFile(noteId, NameScope.CHILD); - - if (!noteDir.exists()) { - // nothing to do - return; - } - - if (!isDirectory(noteDir)) { - // it is not look like zeppelin note savings - throw new IOException("Can not remove " + noteDir.getName().toString()); - } - - noteDir.delete(Selectors.SELECT_SELF_AND_CHILDREN); + public void remove(String folderPath, AuthenticationInfo subject) throws IOException { + LOGGER.info("Remove folder: " + folderPath); + FileObject folderObject = rootNotebookFileObject.resolveFile( + folderPath.substring(1), NameScope.DESCENDENT); + folderObject.deleteAll(); } @Override @@ -252,7 +209,7 @@ public class VFSNotebookRepo implements NotebookRepo { repoSetting.name = "Notebook Path"; repoSetting.type = NotebookRepoSettingsInfo.Type.INPUT; repoSetting.value = Collections.emptyList(); - repoSetting.selected = getNotebookDirPath(); + repoSetting.selected = rootNotebookFileObject.getName().getPath(); settings.add(repoSetting); return settings; @@ -261,7 +218,7 @@ public class VFSNotebookRepo implements NotebookRepo { @Override public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { if (settings == null || settings.isEmpty()) { - LOG.error("Cannot update {} with empty settings", this.getClass().getName()); + LOGGER.error("Cannot update {} with empty settings", this.getClass().getName()); return; } String newNotebookDirectotyPath = StringUtils.EMPTY; @@ -270,15 +227,15 @@ public class VFSNotebookRepo implements NotebookRepo { } if (StringUtils.isBlank(newNotebookDirectotyPath)) { - LOG.error("Notebook path is invalid"); + LOGGER.error("Notebook path is invalid"); return; } - LOG.warn("{} will change notebook dir from {} to {}", - subject.getUser(), getNotebookDirPath(), newNotebookDirectotyPath); + LOGGER.warn("{} will change notebook dir from {} to {}", + subject.getUser(), this.rootNotebookFolder, newNotebookDirectotyPath); try { setNotebookDirectory(newNotebookDirectotyPath); } catch (IOException e) { - LOG.error("Cannot update notebook directory", e); + LOGGER.error("Cannot update notebook directory", e); } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-plugins/notebookrepo/vfs/src/test/java/org/apache/zeppelin/notebook/repo/TestVFSNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/vfs/src/test/java/org/apache/zeppelin/notebook/repo/TestVFSNotebookRepo.java b/zeppelin-plugins/notebookrepo/vfs/src/test/java/org/apache/zeppelin/notebook/repo/TestVFSNotebookRepo.java deleted file mode 100644 index 452adc0..0000000 --- a/zeppelin-plugins/notebookrepo/vfs/src/test/java/org/apache/zeppelin/notebook/repo/TestVFSNotebookRepo.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.notebook.repo; - -import com.google.common.collect.ImmutableMap; -import org.apache.commons.io.FileUtils; -import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.notebook.Note; -import org.apache.zeppelin.notebook.Paragraph; -import org.apache.zeppelin.user.AuthenticationInfo; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -public class TestVFSNotebookRepo { - - private ZeppelinConfiguration zConf; - private VFSNotebookRepo notebookRepo; - private String notebookDir = "/tmp/zeppelin/vfs_notebookrepo/"; - - @Before - public void setUp() throws IOException { - notebookRepo = new VFSNotebookRepo(); - FileUtils.forceMkdir(new File(notebookDir)); - System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir); - zConf = new ZeppelinConfiguration(); - notebookRepo.init(zConf); - } - - @After - public void tearDown() throws IOException { - FileUtils.deleteDirectory(new File(notebookDir)); - } - - @Test - public void testBasics() throws IOException { - assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); - - Note note1 = new Note(); - Paragraph p1 = note1.insertNewParagraph(0, AuthenticationInfo.ANONYMOUS); - p1.setText("%md hello world"); - p1.setTitle("my title"); - notebookRepo.save(note1, AuthenticationInfo.ANONYMOUS); - - assertEquals(1, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); - Note note2 = notebookRepo.get(note1.getId(), AuthenticationInfo.ANONYMOUS); - assertEquals(note1.getParagraphCount(), note2.getParagraphCount()); - - Paragraph p2 = note2.getParagraph(p1.getId()); - assertEquals(p1.getText(), p2.getText()); - assertEquals(p1.getTitle(), p2.getTitle()); - - notebookRepo.remove(note1.getId(), AuthenticationInfo.ANONYMOUS); - assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); - } - - @Test - public void testInvalidJson() throws IOException { - assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); - - // invalid note will be ignored - createNewNote("invalid_content", "id_1"); - assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); - - // only valid note will be fetched - createNewNote("{}", "id_2"); - assertEquals(1, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); - } - - @Test - public void testUpdateSettings() throws IOException { - List<NotebookRepoSettingsInfo> repoSettings = notebookRepo.getSettings(AuthenticationInfo.ANONYMOUS); - assertEquals(1, repoSettings.size()); - NotebookRepoSettingsInfo settingInfo = repoSettings.get(0); - assertEquals("Notebook Path", settingInfo.name); - assertEquals(notebookDir, settingInfo.selected); - - createNewNote("{}", "id_2"); - assertEquals(1, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); - - String newNotebookDir = "/tmp/zeppelin/vfs_notebookrepo2"; - FileUtils.forceMkdir(new File(newNotebookDir)); - Map<String, String> newSettings = ImmutableMap.of("Notebook Path", newNotebookDir); - notebookRepo.updateSettings(newSettings, AuthenticationInfo.ANONYMOUS); - assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); - } - - private void createNewNote(String content, String noteId) throws IOException { - FileUtils.writeStringToFile(new File(notebookDir + "/" + noteId, "note.json"), content); - } -} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-plugins/notebookrepo/vfs/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/vfs/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/vfs/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java new file mode 100644 index 0000000..6207592 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/vfs/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import com.google.common.collect.ImmutableMap; +import com.google.common.io.Files; +import org.apache.commons.io.FileUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.NoteInfo; +import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +public class VFSNotebookRepoTest { + + private ZeppelinConfiguration zConf; + private VFSNotebookRepo notebookRepo; + private File notebookDir = Files.createTempDir(); + + @Before + public void setUp() throws IOException { + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), + notebookDir.getAbsolutePath()); + notebookRepo = new VFSNotebookRepo(); + zConf = new ZeppelinConfiguration(); + notebookRepo.init(zConf); + } + + @After + public void tearDown() throws IOException { + FileUtils.deleteDirectory(notebookDir); + } + + @Test + public void testBasics() throws IOException { + assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); + + // create note1 + Note note1 = new Note(); + note1.setPath("/my_project/my_note1"); + Paragraph p1 = note1.insertNewParagraph(0, AuthenticationInfo.ANONYMOUS); + p1.setText("%md hello world"); + p1.setTitle("my title"); + notebookRepo.save(note1, AuthenticationInfo.ANONYMOUS); + + Map<String, NoteInfo> noteInfos = notebookRepo.list(AuthenticationInfo.ANONYMOUS); + assertEquals(1, noteInfos.size()); + assertEquals(note1.getId(), noteInfos.get(note1.getId()).getId()); + assertEquals(note1.getName(), noteInfos.get(note1.getId()).getNoteName()); + + // create note2 + Note note2 = new Note(); + note2.setPath("/my_note2"); + Paragraph p2 = note2.insertNewParagraph(0, AuthenticationInfo.ANONYMOUS); + p2.setText("%md hello world2"); + p2.setTitle("my title2"); + notebookRepo.save(note2, AuthenticationInfo.ANONYMOUS); + + noteInfos = notebookRepo.list(AuthenticationInfo.ANONYMOUS); + assertEquals(2, noteInfos.size()); + + // move note2 + String newPath = "/my_project2/my_note2"; + notebookRepo.move(note2.getId(), note2.getPath(), "/my_project2/my_note2", AuthenticationInfo.ANONYMOUS); + + Note note3 = notebookRepo.get(note2.getId(), newPath, AuthenticationInfo.ANONYMOUS); + assertEquals(note2, note3); + + // move folder + notebookRepo.move("/my_project2", "/my_project3/my_project2", AuthenticationInfo.ANONYMOUS); + noteInfos = notebookRepo.list(AuthenticationInfo.ANONYMOUS); + assertEquals(2, noteInfos.size()); + + Note note4 = notebookRepo.get(note3.getId(), "/my_project3/my_project2/my_note2", AuthenticationInfo.ANONYMOUS); + assertEquals(note3, note4); + + // remote note1 + notebookRepo.remove(note1.getId(), note1.getPath(), AuthenticationInfo.ANONYMOUS); + assertEquals(1, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); + } + + @Test + public void testUpdateSettings() throws IOException { + List<NotebookRepoSettingsInfo> repoSettings = notebookRepo.getSettings(AuthenticationInfo.ANONYMOUS); + assertEquals(1, repoSettings.size()); + NotebookRepoSettingsInfo settingInfo = repoSettings.get(0); + assertEquals("Notebook Path", settingInfo.name); + assertEquals(notebookDir.getAbsolutePath(), settingInfo.selected); + + createNewNote("{}", "id2", "my_project/name2"); + assertEquals(1, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); + + String newNotebookDir = "/tmp/zeppelin/vfs_notebookrepo2"; + FileUtils.forceMkdir(new File(newNotebookDir)); + Map<String, String> newSettings = ImmutableMap.of("Notebook Path", newNotebookDir); + notebookRepo.updateSettings(newSettings, AuthenticationInfo.ANONYMOUS); + assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size()); + } + + private void createNewNote(String content, String noteId, String noteName) throws IOException { + FileUtils.writeStringToFile( + new File(notebookDir + "/" + noteName + "_" + noteId + ".zpln"), content); + } +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java index 9dd9fbf..3475c53 100644 --- a/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java +++ b/zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -174,22 +175,26 @@ public class ZeppelinHubRepo implements NotebookRepoWithVersionControl { } @Override - public List<NoteInfo> list(AuthenticationInfo subject) throws IOException { + public Map<String, NoteInfo> list(AuthenticationInfo subject) throws IOException { if (!isSubjectValid(subject)) { - return Collections.emptyList(); + return Collections.emptyMap(); } String token = getUserToken(subject.getUser()); String response = restApiClient.get(token, StringUtils.EMPTY); List<NoteInfo> notes = GSON.fromJson(response, new TypeToken<List<NoteInfo>>() {}.getType()); if (notes == null) { - return Collections.emptyList(); + return Collections.emptyMap(); } LOG.info("ZeppelinHub REST API listing notes "); - return notes; + Map<String, NoteInfo> notesInfo = new HashMap<>(); + for (NoteInfo noteInfo : notes) { + notesInfo.put(noteInfo.getId(), noteInfo); + } + return notesInfo; } @Override - public Note get(String noteId, AuthenticationInfo subject) throws IOException { + public Note get(String noteId, String noteName, AuthenticationInfo subject) throws IOException { if (StringUtils.isBlank(noteId) || !isSubjectValid(subject)) { return EMPTY_NOTE; } @@ -215,7 +220,17 @@ public class ZeppelinHubRepo implements NotebookRepoWithVersionControl { } @Override - public void remove(String noteId, AuthenticationInfo subject) throws IOException { + public void move(String noteId, String notePath, String newNotePath, AuthenticationInfo subject) { + + } + + @Override + public void move(String folderPath, String newFolderPath, AuthenticationInfo subject) { + + } + + @Override + public void remove(String noteId, String notePath, AuthenticationInfo subject) throws IOException { if (StringUtils.isBlank(noteId) || !isSubjectValid(subject)) { throw new IOException("Zeppelinhub failed to remove note"); } @@ -225,13 +240,18 @@ public class ZeppelinHubRepo implements NotebookRepoWithVersionControl { } @Override + public void remove(String folderPath, AuthenticationInfo subject) { + + } + + @Override public void close() { websocketClient.stop(); restApiClient.close(); } @Override - public Revision checkpoint(String noteId, String checkpointMsg, AuthenticationInfo subject) + public Revision checkpoint(String noteId, String notePath, String checkpointMsg, AuthenticationInfo subject) throws IOException { if (StringUtils.isBlank(noteId) || !isSubjectValid(subject)) { return Revision.EMPTY; @@ -246,7 +266,7 @@ public class ZeppelinHubRepo implements NotebookRepoWithVersionControl { } @Override - public Note get(String noteId, String revId, AuthenticationInfo subject) throws IOException { + public Note get(String noteId, String notePath, String revId, AuthenticationInfo subject) throws IOException { if (StringUtils.isBlank(noteId) || StringUtils.isBlank(revId) || !isSubjectValid(subject)) { return EMPTY_NOTE; } @@ -263,7 +283,7 @@ public class ZeppelinHubRepo implements NotebookRepoWithVersionControl { } @Override - public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject) { + public List<Revision> revisionHistory(String noteId, String notePath, AuthenticationInfo subject) { if (StringUtils.isBlank(noteId) || !isSubjectValid(subject)) { return Collections.emptyList(); } @@ -376,7 +396,7 @@ public class ZeppelinHubRepo implements NotebookRepoWithVersionControl { } @Override - public Note setNoteRevision(String noteId, String revId, AuthenticationInfo subject) + public Note setNoteRevision(String noteId, String notePath, String revId, AuthenticationInfo subject) throws IOException { // Auto-generated method stub return null; http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-plugins/pom.xml ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/pom.xml b/zeppelin-plugins/pom.xml index f67ac9d..b9dc388 100644 --- a/zeppelin-plugins/pom.xml +++ b/zeppelin-plugins/pom.xml @@ -43,7 +43,6 @@ <module>notebookrepo/github</module> <module>notebookrepo/azure</module> <module>notebookrepo/gcs</module> - <module>notebookrepo/mongodb</module> <module>notebookrepo/zeppelin-hub</module> <module>notebookrepo/filesystem</module> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index cc35293..db188b4 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -26,6 +26,7 @@ import org.apache.zeppelin.annotation.ZeppelinApi; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.NoteInfo; import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.notebook.NotebookAuthorization; import org.apache.zeppelin.notebook.Paragraph; @@ -283,7 +284,7 @@ public class NotebookRestApi extends AbstractRestApi { notebookAuthorization.getReaders(noteId), notebookAuthorization.getRunners(noteId), notebookAuthorization.getWriters(noteId)); AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); - note.persist(subject); + notebook.saveNote(note, subject); notebookServer.broadcastNote(note); notebookServer.broadcastNoteList(subject, userAndRoles); return new JsonResponse<>(Status.OK).build(); @@ -292,8 +293,8 @@ public class NotebookRestApi extends AbstractRestApi { @GET @ZeppelinApi public Response getNoteList() throws IOException { - List<Map<String, String>> notesInfo = notebookService.listNotes(false, getServiceContext(), - new RestServiceCallback()); + List<NoteInfo> notesInfo = notebookService.listNotesInfo(false, getServiceContext(), + new RestServiceCallback<List<NoteInfo>>()); return new JsonResponse<>(Status.OK, "", notesInfo).build(); } @@ -442,7 +443,7 @@ public class NotebookRestApi extends AbstractRestApi { LOG.warn("Trying to rename notebook {} with empty name parameter", noteId); throw new BadRequestException("name can not be empty"); } - notebookService.renameNote(noteId, request.getName(), getServiceContext(), + notebookService.renameNote(noteId, request.getName(), false, getServiceContext(), new RestServiceCallback<Note>(){ @Override public void onSuccess(Note note, ServiceContext context) throws IOException { @@ -482,7 +483,7 @@ public class NotebookRestApi extends AbstractRestApi { p = note.insertNewParagraph(indexDouble.intValue(), subject); } initParagraph(p, request, user); - note.persist(subject); + notebook.saveNote(note, subject); notebookServer.broadcastNote(note); return new JsonResponse<>(Status.OK, "", p.getId()).build(); } @@ -539,7 +540,7 @@ public class NotebookRestApi extends AbstractRestApi { } AuthenticationInfo subject = new AuthenticationInfo(user); - note.persist(subject); + notebook.saveNote(note, subject); notebookServer.broadcastParagraph(note, p); return new JsonResponse<>(Status.OK, "").build(); } @@ -562,7 +563,7 @@ public class NotebookRestApi extends AbstractRestApi { Map<String, Object> newConfig = gson.fromJson(message, HashMap.class); configureParagraph(p, newConfig, user); AuthenticationInfo subject = new AuthenticationInfo(user); - note.persist(subject); + notebook.saveNote(note, subject); return new JsonResponse<>(Status.OK, "", p).build(); } @@ -1006,7 +1007,7 @@ public class NotebookRestApi extends AbstractRestApi { if (paramsForUpdating != null) { paragraph.settings.getParams().putAll(paramsForUpdating); AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); - note.persist(subject); + notebook.saveNote(note, subject); } } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index b0ece52..9c680ba 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -197,7 +197,8 @@ public class ZeppelinServer extends ResourceConfig { heliumApplicationFactory.setApplicationEventListener(notebookWsServer); notebook.addNotebookEventListener(heliumApplicationFactory); - notebook.addNotebookEventListener(notebookWsServer.getNotebookInformationListener()); + notebook.addNotebookEventListener(notebookWsServer); + // Register MBean if ("true".equals(System.getenv("ZEPPELIN_JMX_ENABLE"))) {
