Repository: couchdb-nmo Updated Branches: refs/heads/master 909b275a7 -> dbe4e44b8
Import mongo collections into CouchDB New command `import-mongo` which imports a mongo collection into a database. Project: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/commit/dbe4e44b Tree: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/tree/dbe4e44b Diff: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/diff/dbe4e44b Branch: refs/heads/master Commit: dbe4e44b8eabd506f57721ddb7ebf54f5bd466e4 Parents: 909b275 Author: Garren Smith <[email protected]> Authored: Mon Nov 30 14:41:34 2015 +0200 Committer: Garren Smith <[email protected]> Committed: Tue Jan 12 13:36:31 2016 +0200 ---------------------------------------------------------------------- doc/api/nmo-import-mongo.md | 11 ++++ doc/cli/nmo-import-mongo.md | 24 +++++++++ package.json | 1 + src/import-mongo.js | 92 +++++++++++++++++++++++++++++++ src/nmo.js | 1 + test/import-mongo.js | 113 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 242 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/dbe4e44b/doc/api/nmo-import-mongo.md ---------------------------------------------------------------------- diff --git a/doc/api/nmo-import-mongo.md b/doc/api/nmo-import-mongo.md new file mode 100644 index 0000000..9b0d9b3 --- /dev/null +++ b/doc/api/nmo-import-mongo.md @@ -0,0 +1,11 @@ +nmo-import-mongo(3) -- import-mongo +============================== + +## SYNOPSIS + + nmo.commands.import-mongo(<mongourl>, <collection>, [<url> || <cluster>], <database>) + + +## DESCRIPTION + +Bulk import a defined collection from a MongoDB instance into CouchDB. http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/dbe4e44b/doc/cli/nmo-import-mongo.md ---------------------------------------------------------------------- diff --git a/doc/cli/nmo-import-mongo.md b/doc/cli/nmo-import-mongo.md new file mode 100644 index 0000000..408ccfb --- /dev/null +++ b/doc/cli/nmo-import-mongo.md @@ -0,0 +1,24 @@ +nmo-import-mongo(1) -- Bulk import a MongoDB collection +=========================================== + +## SYNOPSIS + + nmo import-mongo <clustername> <database> <MongoDB-url> <collection> + nmo import-mongo <url> <database> <MongoDB-url> <collection> + + +## DESCRIPTION + +Import a MongoDB collection into CouchDB. + +Example: + +This will import the collection `restaurants` from the MongoDB url `mongodb://localhost:27017/test` into the database +`mydatabase` in the cluster `mycluster`. + + nmo import-mongo mycluster mydatabase mongodb://localhost:27017/test restaurants + +This will import the collection `restaurants` from the MongoDB url `mongodb://localhost:27017/test` into the database +`mydatabase` at the url `http://127.0.0.1:15984`. + + nmo import-mongo http://127.0.0.1:15984 mydatabase mongodb://localhost:27017/test restaurants http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/dbe4e44b/package.json ---------------------------------------------------------------------- diff --git a/package.json b/package.json index a89f3a5..f917a12 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "couchbulkimporter": "^1.0.0", "csv-parse": "^1.0.0", "ini": "~1.3.3", + "mongodb": "^2.0.49", "nopt": "~3.0.1", "npmlog": "~2.0.0", "osenv": "~0.1.0", http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/dbe4e44b/src/import-mongo.js ---------------------------------------------------------------------- diff --git a/src/import-mongo.js b/src/import-mongo.js new file mode 100644 index 0000000..7b03e7f --- /dev/null +++ b/src/import-mongo.js @@ -0,0 +1,92 @@ +import Promise from 'bluebird'; +import CouchBulkImporter from 'couchbulkimporter'; +import mongo from 'mongodb'; +import BulkBadger from 'bulkbadger'; +import { getUrlFromCluster } from './utils'; + +export function cli (cluster, database, mongourl, collection) { + if (!cluster || !database || !mongourl || !collection) { + const msg = [ + 'Usage:', + '', + 'nmo import-mongo <clustername> <database> <MongoDB-url> <collection>', + 'nmo import-mongo <url> <database> <MongoDB-url> <collection>' + ].join('\n'); + const err = new Error(msg); + err.type = 'EUSAGE'; + throw err; + } + + return importmongo(cluster, database, mongourl, collection); +} + +export function validateMongoUrl (url) { + return /mongodb:\/\//.test(url); +} + + +export default importmongo; +function importmongo (cluster, database, mongourl, collection) { + return new Promise((resolve, reject) => { + if (!validateMongoUrl(mongourl)) { + const err = new Error('Invalid MongoDB url, url must start with mongodb://'); + err.type = 'EUSAGE'; + reject(err); + return; + } + + mongo.connect(mongourl, function (err, db) { + if (err) { + reject(err); + return; + } + + const clusterUrl = getUrlFromCluster(cluster); + const col = db.collection(collection); + col.count(function (errCount, noOfDocs) { + if (err) { + reject(err); + return; + } + + if (noOfDocs === 0 || errCount) { + const err = new Error([ + 'There are 0 documents in this collection. That could mean that', + 'the collection does not exist or that the database does not exist.' + ].join('')); + err.type = 'EUSAGE'; + reject(err); + return; + } + + console.log('Migration started!'); + col.find({}, {}) + .on('error', function (err) { + err.message = 'Error fetching collection - ' + err.message; + reject(err); + }) + .pipe(new BulkBadger()) + .on('error', function (err) { + err.message = 'Error - ' + err.message; + if (/CouchDB server answered/.test(err.message)) { + err.type = 'EUSAGE'; + } + reject(err); + }) + .pipe(new CouchBulkImporter({ + url: clusterUrl + '/' + database + })) + .on('error', function (err) { + err.message = 'Error migration incomplete - ' + err.message; + reject(err); + }) + .on('finish', function () { + db.close(); + console.log('Migration complete!'); + resolve(); + }); + }); + }); + + }); +} http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/dbe4e44b/src/nmo.js ---------------------------------------------------------------------- diff --git a/src/nmo.js b/src/nmo.js index b5124b8..54b96f1 100644 --- a/src/nmo.js +++ b/src/nmo.js @@ -14,6 +14,7 @@ const commands = [ 'savetofile', 'replicate-from', 'replicate-to', + 'import-mongo', 'query' ]; http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/dbe4e44b/test/import-mongo.js ---------------------------------------------------------------------- diff --git a/test/import-mongo.js b/test/import-mongo.js new file mode 100644 index 0000000..d6231fa --- /dev/null +++ b/test/import-mongo.js @@ -0,0 +1,113 @@ +import assert from 'assert'; +import nock from 'nock'; +import { createConfigFile } from './common'; + +import nmo from '../src/nmo.js'; +import importmongo, { validateMongoUrl, cli } from '../src/import-mongo.js'; +import mongo from 'mongodb'; +import JSONStream from 'JSONStream'; + +//mongodb mock +const mongoDocs = [{a: 1}, {b: 2}]; +const col = { + find: function () { + const stream = JSONStream.stringify(); + setTimeout(function () { + stream.write(mongoDocs); + stream.end(); + }, 10); + return stream; + }, + count: function (cb) { + cb(null, mongoDocs.length); + } +}; + +mongo.connect = function (url, cb) { + const db = { + collection: function () { + return col; + }, + + close: function () { + + } + }; + + cb(null, db); +}; + +describe('import-mongo', () => { + createConfigFile(); + beforeEach(() => { + nmo + .load({nmoconf: __dirname + '/fixtures/randomini'}); + }); + + describe('cli', () => { + it('throws error if no inputs', (done) => { + + try { + cli(); + } catch(e) { + assert.deepEqual(e.type, 'EUSAGE'); + } + done(); + }); + + }); + + describe('validateMongoUrl', () => { + it('returns false for bad url', () => { + assert.ok(!validateMongoUrl('bad-url')); + }); + + it('returns true for valid url', () => { + assert.ok(validateMongoUrl('mongodb://localhost:27017/test')); + }); + }); + + describe('importmongo', done => { + + it('imports from mongodb to couchdb', () => { + nock('http://127.0.0.1') + .put('/fake-mongo') + .reply(200) + .post('/fake-mongo/_bulk_docs', {docs:[ '[\n[{"a":1},{"b":2}]', '\n]\n' ] }) + .reply(200); + + return importmongo('clusterone', 'fake-mongo', 'mongodb://localhost:27017/test', 'restaurants'); + + }); + + it('error for bad mongodb url', () => { + return importmongo('clusterone', 'fake-mongo', 'bad-mongo-url', 'restaurants') + .catch(function (err) { + assert.ok(/Invalid MongoDB/.test(err.message)); + }); + }); + + it('rejects on an error', () => { + nock('http://127.0.0.1') + .put('/fake-mongo') + .reply(200) + .post('/fake-mongo/_bulk_docs') + .reply(500, 'error with docs'); + + return importmongo('clusterone', 'fake-mongo', 'mongodb://localhost:27017/test', 'restaurants') + .catch(function (err) { + assert.ok(/error with docs/.test(err.message)); + }); + }); + + it('returns error for no docs in collection', () => { + col.count = (cb) => {cb(null, 0);}; + + return importmongo('clusterone', 'fake-mongo', 'mongodb://localhost:27017/test', 'restaurants') + .catch(function (err) { + assert.ok(/There are 0 documents/.test(err.message)); + }); + + }); + }); +});
