Repository: couchdb-nmo Updated Branches: refs/heads/master 6860b3985 -> 7b9bf5e5e
Import csv into couchdb This adds import-csv command to import csv files into couchdb Project: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/commit/7b9bf5e5 Tree: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/tree/7b9bf5e5 Diff: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/diff/7b9bf5e5 Branch: refs/heads/master Commit: 7b9bf5e5e78cb6d6de7fe150e32b1fd14328e05e Parents: 6860b39 Author: Garren Smith <[email protected]> Authored: Thu Aug 27 17:05:57 2015 +0200 Committer: Garren Smith <[email protected]> Committed: Fri Sep 4 13:42:34 2015 +0200 ---------------------------------------------------------------------- bin/nmo-cli.js | 4 +- doc/api/nmo-import-csv.md | 17 +++++++ doc/cli/nmo-import-csv.md | 27 ++++++++++ package.json | 9 +++- src/import-csv.js | 60 ++++++++++++++++++++++ src/nmo.js | 3 +- test/fixtures/fake.csv | 3 ++ test/import-csv.js | 110 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 230 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/7b9bf5e5/bin/nmo-cli.js ---------------------------------------------------------------------- diff --git a/bin/nmo-cli.js b/bin/nmo-cli.js index 449e9b1..8696009 100755 --- a/bin/nmo-cli.js +++ b/bin/nmo-cli.js @@ -10,7 +10,9 @@ var fs = require('fs'); var nmo = require('../lib/nmo.js'); var parsed = nopt({ 'json': [Boolean], - 'force': [Boolean] + 'force': [Boolean], + 'delimiter': [String], + 'columns': [Boolean] }, {'v': 'v'}, process.argv, 2); var cmd = parsed.argv.remain.shift(); http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/7b9bf5e5/doc/api/nmo-import-csv.md ---------------------------------------------------------------------- diff --git a/doc/api/nmo-import-csv.md b/doc/api/nmo-import-csv.md new file mode 100644 index 0000000..a056ce6 --- /dev/null +++ b/doc/api/nmo-import-csv.md @@ -0,0 +1,17 @@ +nmo-import-csv(3) -- configuration +============================== + +## SYNOPSIS + + nmo.commands.importcsv(file, url, [csvOptions]) + + + +## DESCRIPTION + +Import csv file into CouchDB + + - importcsv: + +Accepts the file, url and csv options to import a file into CouchDB. +It returns a promise. http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/7b9bf5e5/doc/cli/nmo-import-csv.md ---------------------------------------------------------------------- diff --git a/doc/cli/nmo-import-csv.md b/doc/cli/nmo-import-csv.md new file mode 100644 index 0000000..498a1b4 --- /dev/null +++ b/doc/cli/nmo-import-csv.md @@ -0,0 +1,27 @@ +nmo-import-csv(1) -- Bulk import CSV files +================================= + +## SYNOPSIS + + nmo import-csv <file>, <couchdb-url> [delimiter=','] [columns=true] + +## DESCRIPTION + +Imports a csv file into CouchDB. + +## CONFIGURATION: + + delimiter + * Default: ',' + * Type: String + The csv delimiter + + columns + * Default: true + * Type: Boolean + Whether to use the first row of the csv to define the key fields for each document + + +## EXAMPLE + + nmo import-csv /path/to/file.cvs http://couchdb-url:5984 --delimiter=',' --columns=true http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/7b9bf5e5/package.json ---------------------------------------------------------------------- diff --git a/package.json b/package.json index 701530d..ce4dad3 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "scripts": { "transpile": "babel src -d lib", "prepublish": "make && npm run transpile", - "test": "lab -v --transform node_modules/lab-babel -t 100 -S" + "test": "lab -v --transform node_modules/lab-babel -t 100 -S", + "lint": "jshint ./src ./test" }, "author": "Robert Kowalski <[email protected]>", "license": "Apache 2.0", @@ -25,8 +26,12 @@ }, "dependencies": { "bluebird": "~2.9.24", + "bulkbadger": "^1.0.0", "config-chain": "~1.1.8", + "couchbulkimporter": "^1.0.0", + "csv-parse": "^1.0.0", "ini": "~1.3.3", + "lodash": "^3.10.1", "nopt": "~3.0.1", "npmlog": "~1.2.0", "osenv": "~0.1.0", @@ -47,10 +52,12 @@ ], "devDependencies": { "babel": "^5.1.10", + "jshint": "^2.8.0", "lab": "~5.5.1", "lab-babel": "1.0.0", "less": "~2.5.0", "marked-man": "~0.1.4", + "nock": "^2.10.0", "node-doc-generator": "robertkowalski/node-documentation-generator#sloppy" } } http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/7b9bf5e5/src/import-csv.js ---------------------------------------------------------------------- diff --git a/src/import-csv.js b/src/import-csv.js new file mode 100644 index 0000000..5b72f1e --- /dev/null +++ b/src/import-csv.js @@ -0,0 +1,60 @@ +import fs from 'fs'; +import Promise from 'bluebird'; +import CouchBulkImporter from 'couchbulkimporter'; +import parse from 'csv-parse'; +import BulkBadger from 'bulkbadger'; +import {checkUrl} from './utils'; +import nmo from './nmo.js'; + +export function cli (file, url, ...csvOptions) { + if (!file) { + const msg = [ + 'Usage:', + '', + 'nmo import-csv [file] [couchdb-url] [...<csv options> <pairs>]', + ].join('\n'); + const err = new Error(msg); + err.type = 'EUSAGE'; + + throw err; + } + + const er = checkUrl(url); + if (er) { + throw er; + } + + const opts = {delimiter: nmo.config.get('delimiter'), columns: nmo.config.get('columns')}; + return importcsv.apply(importcsv, [file, url, opts]); +} + +export function importcsv (file, url, {delimiter= ',', columns= true}) { + return new Promise((resolve, reject) => { + const input = fs.createReadStream(file) + .on('error', (err) => { + err.message = 'Error reading file - ' + err.message; + reject(err); + }); + + const parser = parse({ + delimiter: delimiter, + columns: columns + }); + + input + .pipe(parser) + .pipe(new BulkBadger()) + .pipe(new CouchBulkImporter({ + url: url + })) + .on('error', function (err) { + err.message = 'Error uploading - ' + err.message; + reject(err); + }) + .on('finish', function () { + console.log('Upload complete!'); + resolve(); + }); + + }); +} http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/7b9bf5e5/src/nmo.js ---------------------------------------------------------------------- diff --git a/src/nmo.js b/src/nmo.js index 874cec9..c60c038 100644 --- a/src/nmo.js +++ b/src/nmo.js @@ -7,7 +7,8 @@ const commands = [ 'help', 'config', 'cluster', - 'v' + 'v', + 'import-csv' ]; const nmo = { http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/7b9bf5e5/test/fixtures/fake.csv ---------------------------------------------------------------------- diff --git a/test/fixtures/fake.csv b/test/fixtures/fake.csv new file mode 100644 index 0000000..3338248 --- /dev/null +++ b/test/fixtures/fake.csv @@ -0,0 +1,3 @@ +name,surname,email +John, Rambo, [email protected] +Eddie, Vedder, [email protected] http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/7b9bf5e5/test/import-csv.js ---------------------------------------------------------------------- diff --git a/test/import-csv.js b/test/import-csv.js new file mode 100644 index 0000000..e1eac3c --- /dev/null +++ b/test/import-csv.js @@ -0,0 +1,110 @@ +import assert from 'assert'; + +import Lab from 'lab'; +export const lab = Lab.script(); +import nock from 'nock'; +import nmo from '../src/nmo.js'; +import {cli, importcsv} from '../src/import-csv.js'; + +const docs = { + "docs":[ + { + "name":"John", + "surname":" Rambo", + "email":" [email protected]" + }, + { + name: 'Eddie', + surname: ' Vedder', + email: ' [email protected]' + } + ]}; + +lab.experiment('import csv', () => { + lab.beforeEach((done) => { + nmo + .load({nmoconf: __dirname + '/fixtures/randomini'}) + .then(() => done()) + .catch(() => done()); + + }); + + lab.experiment('cli', () => { + + lab.test('throws error if no inputs', (done) => { + + try { + cli(); + } catch(e) { + assert.deepEqual(e.type, 'EUSAGE'); + } + done(); + }); + + lab.test('throws error if bad url', (done) => { + + try { + cli('file', 'bad-url'); + } catch(e) { + assert.ok(/not a valid url/.test(e.message)); + } + done(); + }); + + lab.test('full integration works', (done) => { + nock('http://127.0.0.1:5984') + .put('/fake-csv') + .reply(200) + .post('/fake-csv/_bulk_docs') + .reply(200); + + cli(__dirname + '/fixtures/fake.csv', 'http://127.0.0.1:5984/fake-csv', 'delimiter=","') + .then(done); + }); + }); + + lab.experiment('upload to couchdb', () => { + + lab.test('reports bad file', (done) => { + const url = 'http://127.0.0.1:5984'; + importcsv('bad-fake.csv', url + '/csv-upload', {}).catch(function (err) { + assert.ok(/Error reading file -/.test(err)); + done(); + }); + + }); + + lab.test('logs error for failed request', (done) => { + const url = 'http://127.0.0.1:5984'; + + nock(url) + .put('/csv-upload') + .reply(501); + + importcsv(__dirname + '/fixtures/fake.csv', url + '/csv-upload', {}).catch(function (err) { + assert.ok(/CouchDB server answered:/.test(err)); + done(); + }); + + }); + + lab.test('Uploads csv file to CouchDB', (done) => { + const url = 'http://127.0.0.1:5984'; + + nock(url) + .put('/csv-upload') + .reply(200) + .post('/csv-upload/_bulk_docs', docs) + .reply(200); + + importcsv(__dirname + '/fixtures/fake.csv', url + '/csv-upload', {delimiter: ',', columns: true}).then(function () { + done(); + }).catch(function (err) { + throw 'error ' + err; + }); + + }); + + }); + +});
