Added: avro/trunk/lang/js/lib/schemas.js URL: http://svn.apache.org/viewvc/avro/trunk/lang/js/lib/schemas.js?rev=1717850&view=auto ============================================================================== --- avro/trunk/lang/js/lib/schemas.js (added) +++ avro/trunk/lang/js/lib/schemas.js Thu Dec 3 21:35:44 2015 @@ -0,0 +1,2197 @@ +/* jshint node: true */ + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +'use strict'; + +var utils = require('./utils'), + buffer = require('buffer'), // For `SlowBuffer`. + crypto = require('crypto'), + util = require('util'); + +// Convenience imports. +var Tap = utils.Tap; +var f = util.format; + +// All Avro types. +var TYPES = { + 'array': ArrayType, + 'boolean': BooleanType, + 'bytes': BytesType, + 'double': DoubleType, + 'enum': EnumType, + 'error': RecordType, + 'fixed': FixedType, + 'float': FloatType, + 'int': IntType, + 'long': LongType, + 'map': MapType, + 'null': NullType, + 'record': RecordType, + 'string': StringType, + 'union': UnionType +}; + +// Valid (field, type, and symbol) name regex. +var NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; + +// Random generator. +var RANDOM = new utils.Lcg(); + +// Encoding tap (shared for performance). +var TAP = new Tap(new buffer.SlowBuffer(1024)); + +// Path prefix for validity checks (shared for performance). +var PATH = []; + +// Currently active logical type, used for name redirection. +var LOGICAL_TYPE = null; + + +/** + * Schema parsing entry point. + * + * It isn't exposed directly but called from `parse` inside `index.js` (node) + * or `avro.js` (browserify) which each add convenience functionality. + * + */ +function createType(attrs, opts) { + if (attrs instanceof Type) { + return attrs; + } + + opts = getOpts(attrs, opts); + + var type; + if (typeof attrs == 'string') { // Type reference. + if (opts.namespace && !~attrs.indexOf('.') && !isPrimitive(attrs)) { + attrs = opts.namespace + '.' + attrs; + } + type = opts.registry[attrs]; + if (type) { + // Type was already defined, return it. + return type; + } + if (isPrimitive(attrs)) { + // Reference to a primitive type. These are also defined names by default + // so we create the appropriate type and it to the registry for future + // reference. + return opts.registry[attrs] = createType({type: attrs}, opts); + } + throw new Error(f('undefined type name: %s', attrs)); + } + + if (opts.typeHook && (type = opts.typeHook(attrs, opts))) { + if (!(type instanceof Type)) { + throw new Error(f('invalid typehook return value: %j', type)); + } + return type; + } + + if (attrs.logicalType && !LOGICAL_TYPE) { + var DerivedType = opts.logicalTypes[attrs.logicalType]; + if (DerivedType) { + var registry = {}; + Object.keys(opts.registry).forEach(function (key) { + registry[key] = opts.registry[key]; + }); + try { + return new DerivedType(attrs, opts); + } catch (err) { + if (opts.assertLogicalType) { + // The spec mandates that we fall through to the underlying type if + // the logical type is invalid. We provide this option to ease + // debugging. + throw err; + } + LOGICAL_TYPE = null; + opts.registry = registry; // In case any names were registered. + } + } + } + + if (attrs instanceof Array) { // Union. + type = new UnionType(attrs, opts); + } else { // New type definition. + type = (function (typeName) { + var Type = TYPES[typeName]; + if (Type === undefined) { + throw new Error(f('unknown type: %j', typeName)); + } + return new Type(attrs, opts); + })(attrs.type); + } + return type; +} + +/** + * "Abstract" base Avro type class. + * + * This class' constructor will register any named types to support + * recursive schemas. + * + * All type values are represented in memory similarly to their JSON + * representation, except for `bytes` and `fixed` which are represented as + * `Buffer`s. See individual subclasses for details. + * + */ +function Type(registry) { + var name = this._name; + var type = LOGICAL_TYPE || this; + LOGICAL_TYPE = null; + + if (registry === undefined || name === undefined) { + return; + } + + var prev = registry[name]; + if (prev !== undefined) { + throw new Error(f('duplicate type name: %s', name)); + } + registry[name] = type; +} + +Type.__reset = function (size) { TAP.buf = new buffer.SlowBuffer(size); }; + +Type.prototype.createResolver = function (type, opts) { + if (!(type instanceof Type)) { + // More explicit error message than the "incompatible type" thrown + // otherwise (especially because of the overridden `toJSON` method). + throw new Error(f('not a type: %j', type)); + } + + if (type instanceof LogicalType && !(this instanceof LogicalType)) { + // Trying to read a logical type as a built-in: unwrap the logical type. + return this.createResolver(type._underlyingType, opts); + } + + opts = opts || {}; + opts.registry = opts.registry || {}; + + var resolver, key; + if (this instanceof RecordType && type instanceof RecordType) { + key = this._name + ':' + type._name; // ':' is illegal in Avro type names. + resolver = opts.registry[key]; + if (resolver) { + return resolver; + } + } + + resolver = new Resolver(this); + if (key) { // Register resolver early for recursive schemas. + opts.registry[key] = resolver; + } + + if (type instanceof UnionType) { + var resolvers = type._types.map(function (t) { + return this.createResolver(t, opts); + }, this); + resolver._read = function (tap) { + var index = tap.readLong(); + var resolver = resolvers[index]; + if (resolver === undefined) { + throw new Error(f('invalid union index: %s', index)); + } + return resolvers[index]._read(tap); + }; + } else { + this._updateResolver(resolver, type, opts); + } + + if (!resolver._read) { + throw new Error(f('cannot read %s as %s', type, this)); + } + return resolver; +}; + +Type.prototype.decode = function (buf, pos, resolver) { + var tap = new Tap(buf); + tap.pos = pos | 0; + var val = readValue(this, tap, resolver); + if (!tap.isValid()) { + return {value: undefined, offset: -1}; + } + return {value: val, offset: tap.pos}; +}; + +Type.prototype.encode = function (val, buf, pos) { + var tap = new Tap(buf); + tap.pos = pos | 0; + this._write(tap, val); + if (!tap.isValid()) { + // Don't throw as there is no way to predict this. We also return the + // number of missing bytes to ease resizing. + return buf.length - tap.pos; + } + return tap.pos; +}; + +Type.prototype.fromBuffer = function (buf, resolver, noCheck) { + var tap = new Tap(buf); + var val = readValue(this, tap, resolver, noCheck); + if (!tap.isValid()) { + throw new Error('truncated buffer'); + } + if (!noCheck && tap.pos < buf.length) { + throw new Error('trailing data'); + } + return val; +}; + +Type.prototype.toBuffer = function (val) { + TAP.pos = 0; + this._write(TAP, val); + if (!TAP.isValid()) { + Type.__reset(2 * TAP.pos); + TAP.pos = 0; + this._write(TAP, val); + } + var buf = new Buffer(TAP.pos); + TAP.buf.copy(buf, 0, 0, TAP.pos); + return buf; +}; + +Type.prototype.fromString = function (str) { + return this._copy(JSON.parse(str), {coerce: 2}); +}; + +Type.prototype.toString = function (val) { + if (val === undefined) { + // Consistent behavior with standard `toString` expectations. + return this.getSchema(true); + } + return JSON.stringify(this._copy(val, {coerce: 3})); +}; + +Type.prototype.clone = function (val, opts) { + if (opts) { + opts = { + coerce: !!opts.coerceBuffers | 0, // Coerce JSON to Buffer. + fieldHook: opts.fieldHook, + wrap: !!opts.wrapUnions | 0 // Wrap first match into union. + }; + } + return this._copy(val, opts); +}; + +Type.prototype.isValid = function (val, opts) { + while (PATH.length) { + // In case the previous `isValid` call didn't complete successfully (e.g. + // if an exception was thrown, but then caught in client code), `PATH` + // might be non-empty, we must manually clear it. + PATH.pop(); + } + return this._check(val, opts && opts.errorHook); +}; + +Type.prototype.compareBuffers = function (buf1, buf2) { + return this._match(new Tap(buf1), new Tap(buf2)); +}; + +Type.prototype.getName = function () { return this._name; }; + +Type.prototype.getSchema = function (noDeref) { + // Since JS objects are unordered, this implementation (unfortunately) + // relies on engines returning properties in the same order that they are + // inserted in. This is not in the JS spec, but can be "somewhat" safely + // assumed (more here: http://stackoverflow.com/q/5525795/1062617). + return (function (type, registry) { + return JSON.stringify(type, function (key, value) { + if (value instanceof Field) { + return {name: value._name, type: value._type}; + } else if (value && value.name) { + var name = value.name; + if (noDeref || registry[name]) { + return name; + } + registry[name] = true; + } + return value; + }); + })(this, {}); +}; + +Type.prototype.getFingerprint = function (algorithm) { + algorithm = algorithm || 'md5'; + var hash = crypto.createHash(algorithm); + hash.end(this.getSchema()); + return hash.read(); +}; + +Type.prototype.inspect = function () { + if (this instanceof PrimitiveType) { + return f('<%s>', this.constructor.name); + } else { + var obj = JSON.parse(this.getSchema(true)); // Slow, only for debugging. + if (typeof obj == 'object') { + obj.type = undefined; // Would be redundant with constructor name. + } + return f('<%s %j>', this.constructor.name, obj); + } +}; + +Type.prototype._check = utils.abstractFunction; +Type.prototype._copy = utils.abstractFunction; +Type.prototype._match = utils.abstractFunction; +Type.prototype._read = utils.abstractFunction; +Type.prototype._skip = utils.abstractFunction; +Type.prototype._updateResolver = utils.abstractFunction; +Type.prototype._write = utils.abstractFunction; +Type.prototype.compare = utils.abstractFunction; +Type.prototype.random = utils.abstractFunction; + +// Implementations. + +/** + * Base primitive Avro type. + * + * Most of the primitive types share the same cloning and resolution + * mechanisms, provided by this class. This class also lets us conveniently + * check whether a type is a primitive using `instanceof`. + * + */ +function PrimitiveType() { Type.call(this); } +util.inherits(PrimitiveType, Type); +PrimitiveType.prototype._updateResolver = function (resolver, type) { + if (type.constructor === this.constructor) { + resolver._read = this._read; + } +}; +PrimitiveType.prototype._copy = function (val) { + this._check(val, throwInvalidError); + return val; +}; +PrimitiveType.prototype.compare = utils.compare; + +/** + * Nulls. + * + */ +function NullType() { PrimitiveType.call(this); } +util.inherits(NullType, PrimitiveType); +NullType.prototype._check = function (val, cb) { + var b = val === null; + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; +NullType.prototype._read = function () { return null; }; +NullType.prototype._skip = function () {}; +NullType.prototype._write = function (tap, val) { + if (val !== null) { + throwInvalidError(null, val, this); + } +}; +NullType.prototype._match = function () { return 0; }; +NullType.prototype.compare = NullType.prototype._match; +NullType.prototype.random = NullType.prototype._read; +NullType.prototype.toJSON = function () { return 'null'; }; + +/** + * Booleans. + * + */ +function BooleanType() { PrimitiveType.call(this); } +util.inherits(BooleanType, PrimitiveType); +BooleanType.prototype._check = function (val, cb) { + var b = typeof val == 'boolean'; + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; +BooleanType.prototype._read = function (tap) { return tap.readBoolean(); }; +BooleanType.prototype._skip = function (tap) { tap.skipBoolean(); }; +BooleanType.prototype._write = function (tap, val) { + if (typeof val != 'boolean') { + throwInvalidError(null, val, this); + } + tap.writeBoolean(val); +}; +BooleanType.prototype._match = function (tap1, tap2) { + return tap1.matchBoolean(tap2); +}; +BooleanType.prototype.random = function () { return RANDOM.nextBoolean(); }; +BooleanType.prototype.toJSON = function () { return 'boolean'; }; + +/** + * Integers. + * + */ +function IntType() { PrimitiveType.call(this); } +util.inherits(IntType, PrimitiveType); +IntType.prototype._check = function (val, cb) { + var b = val === (val | 0); + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; +IntType.prototype._read = function (tap) { return tap.readInt(); }; +IntType.prototype._skip = function (tap) { tap.skipInt(); }; +IntType.prototype._write = function (tap, val) { + if (val !== (val | 0)) { + throwInvalidError(null, val, this); + } + tap.writeInt(val); +}; +IntType.prototype._match = function (tap1, tap2) { + return tap1.matchInt(tap2); +}; +IntType.prototype.random = function () { return RANDOM.nextInt(1000) | 0; }; +IntType.prototype.toJSON = function () { return 'int'; }; + +/** + * Longs. + * + * We can't capture all the range unfortunately since JavaScript represents all + * numbers internally as `double`s, so the default implementation plays safe + * and throws rather than potentially silently change the data. See `using` or + * `AbstractLongType` below for a way to implement a custom long type. + * + */ +function LongType() { PrimitiveType.call(this); } +util.inherits(LongType, PrimitiveType); +LongType.prototype._check = function (val, cb) { + var b = typeof val == 'number' && val % 1 === 0 && isSafeLong(val); + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; +LongType.prototype._read = function (tap) { + var n = tap.readLong(); + if (!isSafeLong(n)) { + throw new Error('potential precision loss'); + } + return n; +}; +LongType.prototype._skip = function (tap) { tap.skipLong(); }; +LongType.prototype._write = function (tap, val) { + if (typeof val != 'number' || val % 1 || !isSafeLong(val)) { + throwInvalidError(null, val, this); + } + tap.writeLong(val); +}; +LongType.prototype._match = function (tap1, tap2) { + return tap1.matchLong(tap2); +}; +LongType.prototype._updateResolver = function (resolver, type) { + if (type instanceof LongType || type instanceof IntType) { + resolver._read = type._read; + } +}; +LongType.prototype.random = function () { return RANDOM.nextInt(); }; +LongType.prototype.toJSON = function () { return 'long'; }; +LongType.using = function (methods, noUnpack) { + methods = methods || {}; // Will give a more helpful error message. + // We map some of the methods to a different name to be able to intercept + // their input and output (otherwise we wouldn't be able to perform any + // unpacking logic, and the type wouldn't work when nested). + var mapping = { + toBuffer: '_toBuffer', + fromBuffer: '_fromBuffer', + fromJSON: '_fromJSON', + toJSON: '_toJSON', + isValid: '_isValid', + compare: 'compare' + }; + var type = new AbstractLongType(noUnpack); + Object.keys(mapping).forEach(function (name) { + if (methods[name] === undefined) { + throw new Error(f('missing method implementation: %s', name)); + } + type[mapping[name]] = methods[name]; + }); + return type; +}; + +/** + * Floats. + * + */ +function FloatType() { PrimitiveType.call(this); } +util.inherits(FloatType, PrimitiveType); +FloatType.prototype._check = function (val, cb) { + var b = typeof val == 'number'; + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; +FloatType.prototype._read = function (tap) { return tap.readFloat(); }; +FloatType.prototype._skip = function (tap) { tap.skipFloat(); }; +FloatType.prototype._write = function (tap, val) { + if (typeof val != 'number') { + throwInvalidError(null, val, this); + } + tap.writeFloat(val); +}; +FloatType.prototype._match = function (tap1, tap2) { + return tap1.matchFloat(tap2); +}; +FloatType.prototype._updateResolver = function (resolver, type) { + if ( + type instanceof FloatType || + type instanceof LongType || + type instanceof IntType + ) { + resolver._read = type._read; + } +}; +FloatType.prototype.random = function () { return RANDOM.nextFloat(1e3); }; +FloatType.prototype.toJSON = function () { return 'float'; }; + +/** + * Doubles. + * + */ +function DoubleType() { PrimitiveType.call(this); } +util.inherits(DoubleType, PrimitiveType); +DoubleType.prototype._check = function (val, cb) { + var b = typeof val == 'number'; + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; +DoubleType.prototype._read = function (tap) { return tap.readDouble(); }; +DoubleType.prototype._skip = function (tap) { tap.skipDouble(); }; +DoubleType.prototype._write = function (tap, val) { + if (typeof val != 'number') { + throwInvalidError(null, val, this); + } + tap.writeDouble(val); +}; +DoubleType.prototype._match = function (tap1, tap2) { + return tap1.matchDouble(tap2); +}; +DoubleType.prototype._updateResolver = function (resolver, type) { + if ( + type instanceof DoubleType || + type instanceof FloatType || + type instanceof LongType || + type instanceof IntType + ) { + resolver._read = type._read; + } +}; +DoubleType.prototype.random = function () { return RANDOM.nextFloat(); }; +DoubleType.prototype.toJSON = function () { return 'double'; }; + +/** + * Strings. + * + */ +function StringType() { PrimitiveType.call(this); } +util.inherits(StringType, PrimitiveType); +StringType.prototype._check = function (val, cb) { + var b = typeof val == 'string'; + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; +StringType.prototype._read = function (tap) { return tap.readString(); }; +StringType.prototype._skip = function (tap) { tap.skipString(); }; +StringType.prototype._write = function (tap, val) { + if (typeof val != 'string') { + throwInvalidError(null, val, this); + } + tap.writeString(val); +}; +StringType.prototype._match = function (tap1, tap2) { + return tap1.matchString(tap2); +}; +StringType.prototype._updateResolver = function (resolver, type) { + if (type instanceof StringType || type instanceof BytesType) { + resolver._read = this._read; + } +}; +StringType.prototype.random = function () { + return RANDOM.nextString(RANDOM.nextInt(32)); +}; +StringType.prototype.toJSON = function () { return 'string'; }; + +/** + * Bytes. + * + * These are represented in memory as `Buffer`s rather than binary-encoded + * strings. This is more efficient (when decoding/encoding from bytes, the + * common use-case), idiomatic, and convenient. + * + * Note the coercion in `_copy`. + * + */ +function BytesType() { PrimitiveType.call(this); } +util.inherits(BytesType, PrimitiveType); +BytesType.prototype._check = function (val, cb) { + var b = Buffer.isBuffer(val); + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; +BytesType.prototype._read = function (tap) { return tap.readBytes(); }; +BytesType.prototype._skip = function (tap) { tap.skipBytes(); }; +BytesType.prototype._write = function (tap, val) { + if (!Buffer.isBuffer(val)) { + throwInvalidError(null, val, this); + } + tap.writeBytes(val); +}; +BytesType.prototype._match = function (tap1, tap2) { + return tap1.matchBytes(tap2); +}; +BytesType.prototype._updateResolver = StringType.prototype._updateResolver; +BytesType.prototype._copy = function (obj, opts) { + var buf; + switch ((opts && opts.coerce) | 0) { + case 3: // Coerce buffers to strings. + this._check(obj, throwInvalidError); + return obj.toString('binary'); + case 2: // Coerce strings to buffers. + if (typeof obj != 'string') { + throw new Error(f('cannot coerce to buffer: %j', obj)); + } + buf = new Buffer(obj, 'binary'); + this._check(buf, throwInvalidError); + return buf; + case 1: // Coerce buffer JSON representation to buffers. + if (!obj || obj.type !== 'Buffer' || !(obj.data instanceof Array)) { + throw new Error(f('cannot coerce to buffer: %j', obj)); + } + buf = new Buffer(obj.data); + this._check(buf, throwInvalidError); + return buf; + default: // Copy buffer. + this._check(obj, throwInvalidError); + return new Buffer(obj); + } +}; +BytesType.prototype.compare = Buffer.compare; +BytesType.prototype.random = function () { + return RANDOM.nextBuffer(RANDOM.nextInt(32)); +}; +BytesType.prototype.toJSON = function () { return 'bytes'; }; + +/** + * Avro unions. + * + * Unions are represented in memory similarly to their JSON representation + * (i.e. inside an object with single key the name of the contained type). + * + * This is not ideal, but is the most efficient way to unambiguously support + * all unions. Here are a few reasons why the wrapping object is necessary: + * + * + Unions with multiple number types would have undefined behavior, unless + * numbers are wrapped (either everywhere, leading to large performance and + * convenience costs; or only when necessary inside unions, making it hard to + * understand when numbers are wrapped or not). + * + Fixed types would have to be wrapped to be distinguished from bytes. + * + Using record's constructor names would work (after a slight change to use + * the fully qualified name), but would mean that generic objects could no + * longer be valid records (making it inconvenient to do simple things like + * creating new records). + * + * Lore: In the past (until d304cab), there used to be an "unwrapped union + * type" which directly exposed its values, without the wrapping object + * (similarly to Avro's python implementation). It was removed to keep all + * representations consistent and make this library simpler to understand + * (conversions, e.g. for schema evolution, between representations were + * particularly confusing). Encoding was also much slower (worst case + * complexity linear in the number of types in the union). + * + */ +function UnionType(attrs, opts) { + if (!(attrs instanceof Array)) { + throw new Error(f('non-array union schema: %j', attrs)); + } + if (!attrs.length) { + throw new Error('empty union'); + } + + opts = getOpts(attrs, opts); + Type.call(this); + this._types = attrs.map(function (obj) { return createType(obj, opts); }); + + this._indices = {}; + this._types.forEach(function (type, i) { + if (type instanceof UnionType) { + throw new Error('unions cannot be directly nested'); + } + var name = type._name || getTypeName(type); + if (this._indices[name] !== undefined) { + throw new Error(f('duplicate union name: %j', name)); + } + this._indices[name] = i; + }, this); + + this._constructors = this._types.map(function (type) { + // jshint -W054 + var name = type._name || getTypeName(type); + if (name === 'null') { + return null; + } + var body; + if (~name.indexOf('.')) { // Qualified name. + body = 'this[\'' + name + '\'] = val;'; + } else { + body = 'this.' + name + ' = val;'; + } + return new Function('val', body); + }); +} +util.inherits(UnionType, Type); + +UnionType.prototype._check = function (val, cb) { + var b = false; + if (val === null) { + // Shortcut type lookup in this case. + b = this._indices['null'] !== undefined; + } else if (typeof val == 'object') { + var keys = Object.keys(val); + if (keys.length === 1) { + // We require a single key here to ensure that writes are correct and + // efficient as soon as a record passes this check. + var name = keys[0]; + var index = this._indices[name]; + if (index !== undefined) { + PATH.push(name); + b = this._types[index]._check(val[name], cb); + PATH.pop(); + return b; + } + } + } + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; + +UnionType.prototype._read = function (tap) { + var index = tap.readLong(); + var Class = this._constructors[index]; + if (Class) { + return new Class(this._types[index]._read(tap)); + } else if (Class === null) { + return null; + } else { + throw new Error(f('invalid union index: %s', index)); + } +}; + +UnionType.prototype._skip = function (tap) { + this._types[tap.readLong()]._skip(tap); +}; + +UnionType.prototype._write = function (tap, val) { + var index, keys, name; + if (val === null) { + index = this._indices['null']; + if (index === undefined) { + throwInvalidError(null, val, this); + } + tap.writeLong(index); + } else { + keys = Object.keys(val); + if (keys.length === 1) { + name = keys[0]; + index = this._indices[name]; + } + if (index === undefined) { + throwInvalidError(null, val, this); + } + tap.writeLong(index); + this._types[index]._write(tap, val[name]); + } +}; + +UnionType.prototype._match = function (tap1, tap2) { + var n1 = tap1.readLong(); + var n2 = tap2.readLong(); + if (n1 === n2) { + return this._types[n1]._match(tap1, tap2); + } else { + return n1 < n2 ? -1 : 1; + } +}; + +UnionType.prototype._updateResolver = function (resolver, type, opts) { + // jshint -W083 + // (The loop exits after the first function is created.) + var i, l, typeResolver, Class; + for (i = 0, l = this._types.length; i < l; i++) { + try { + typeResolver = this._types[i].createResolver(type, opts); + } catch (err) { + continue; + } + Class = this._constructors[i]; + if (Class) { + resolver._read = function (tap) { + return new Class(typeResolver._read(tap)); + }; + } else { + resolver._read = function () { return null; }; + } + return; + } +}; + +UnionType.prototype._copy = function (val, opts) { + var wrap = opts && opts.wrap | 0; + if (wrap === 2) { + // Promote into first type (used for schema defaults). + if (val === null && this._constructors[0] === null) { + return null; + } + return new this._constructors[0](this._types[0]._copy(val, opts)); + } + if (val === null && this._indices['null'] !== undefined) { + return null; + } + var i, l, obj; + if (wrap === 1) { + // Promote into first match (convenience, slow). + i = 0; + l = this._types.length; + while (i < l && obj === undefined) { + try { + obj = this._types[i]._copy(val, opts); + } catch (err) { + i++; + } + } + } else if (typeof val == 'object') { + var keys = Object.keys(val); + if (keys.length === 1) { + var name = keys[0]; + i = this._indices[name]; + if (i === undefined) { + // We are a bit more flexible than in `_check` here since we have + // to deal with other serializers being less strict, so we fall + // back to looking up unqualified names. + var j, type; + for (j = 0, l = this._types.length; j < l; j++) { + type = this._types[j]; + if (type._name && name === unqualify(type._name)) { + i = j; + break; + } + } + } + if (i !== undefined) { + obj = this._types[i]._copy(val[name], opts); + } + } + } + if (obj !== undefined) { + return new this._constructors[i](obj); + } + throwInvalidError(null, val, this); +}; + +UnionType.prototype.compare = function (val1, val2) { + var name1 = val1 === null ? 'null' : Object.keys(val1)[0]; + var name2 = val2 === null ? 'null' : Object.keys(val2)[0]; + var index = this._indices[name1]; + if (name1 === name2) { + return name1 === 'null' ? + 0 : + this._types[index].compare(val1[name1], val2[name1]); + } else { + return utils.compare(index, this._indices[name2]); + } +}; + +UnionType.prototype.getTypes = function () { return this._types.slice(); }; + +UnionType.prototype.random = function () { + var index = RANDOM.nextInt(this._types.length); + var Class = this._constructors[index]; + if (!Class) { + return null; + } + return new Class(this._types[index].random()); +}; + +UnionType.prototype.toJSON = function () { return this._types; }; + +/** + * Avro enum type. + * + * Represented as strings (with allowed values from the set of symbols). Using + * integers would be a reasonable option, but the performance boost is arguably + * offset by the legibility cost and the extra deviation from the JSON encoding + * convention. + * + * An integer representation can still be used (e.g. for compatibility with + * TypeScript `enum`s) by overriding the `EnumType` with a `LongType` (e.g. via + * `parse`'s registry). + * + */ +function EnumType(attrs, opts) { + if (!(attrs.symbols instanceof Array) || !attrs.symbols.length) { + throw new Error(f('invalid %j enum symbols: %j', attrs.name, attrs)); + } + + opts = getOpts(attrs, opts); + var resolutions = resolveNames(attrs, opts.namespace); + this._name = resolutions.name; + this._symbols = attrs.symbols; + this._aliases = resolutions.aliases; + Type.call(this, opts.registry); + + this._indices = {}; + this._symbols.forEach(function (symbol, i) { + if (!NAME_PATTERN.test(symbol)) { + throw new Error(f('invalid %s symbol: %j', this, symbol)); + } + if (this._indices[symbol] !== undefined) { + throw new Error(f('duplicate %s symbol: %j', this, symbol)); + } + this._indices[symbol] = i; + }, this); +} +util.inherits(EnumType, Type); + +EnumType.prototype._check = function (val, cb) { + var b = this._indices[val] !== undefined; + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; + +EnumType.prototype._read = function (tap) { + var index = tap.readLong(); + var symbol = this._symbols[index]; + if (symbol === undefined) { + throw new Error(f('invalid %s enum index: %s', this._name, index)); + } + return symbol; +}; + +EnumType.prototype._skip = function (tap) { tap.skipLong(); }; + +EnumType.prototype._write = function (tap, val) { + var index = this._indices[val]; + if (index === undefined) { + throwInvalidError(null, val, this); + } + tap.writeLong(index); +}; + +EnumType.prototype._match = function (tap1, tap2) { + return tap1.matchLong(tap2); +}; + +EnumType.prototype.compare = function (val1, val2) { + return utils.compare(this._indices[val1], this._indices[val2]); +}; + +EnumType.prototype._updateResolver = function (resolver, type) { + var symbols = this._symbols; + if ( + type instanceof EnumType && + ~getAliases(this).indexOf(type._name) && + type._symbols.every(function (s) { return ~symbols.indexOf(s); }) + ) { + resolver._symbols = type._symbols; + resolver._read = type._read; + } +}; + +EnumType.prototype._copy = function (val) { + this._check(val, throwInvalidError); + return val; +}; + +EnumType.prototype.getAliases = function () { return this._aliases; }; + +EnumType.prototype.getSymbols = function () { return this._symbols.slice(); }; + +EnumType.prototype.random = function () { + return RANDOM.choice(this._symbols); +}; + +EnumType.prototype.toJSON = function () { + return {name: this._name, type: 'enum', symbols: this._symbols}; +}; + +/** + * Avro fixed type. + * + * Represented simply as a `Buffer`. + * + */ +function FixedType(attrs, opts) { + if (attrs.size !== (attrs.size | 0) || attrs.size < 1) { + throw new Error(f('invalid %j fixed size: %j', attrs.name, attrs.size)); + } + + opts = getOpts(attrs, opts); + var resolutions = resolveNames(attrs, opts.namespace); + this._name = resolutions.name; + this._size = attrs.size | 0; + this._aliases = resolutions.aliases; + Type.call(this, opts.registry); +} +util.inherits(FixedType, Type); + +FixedType.prototype._check = function (val, cb) { + var b = Buffer.isBuffer(val) && val.length === this._size; + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; + +FixedType.prototype._read = function (tap) { + return tap.readFixed(this._size); +}; + +FixedType.prototype._skip = function (tap) { + tap.skipFixed(this._size); +}; + +FixedType.prototype._write = function (tap, val) { + if (!Buffer.isBuffer(val) || val.length !== this._size) { + throwInvalidError(null, val, this); + } + tap.writeFixed(val, this._size); +}; + +FixedType.prototype._match = function (tap1, tap2) { + return tap1.matchFixed(tap2, this._size); +}; + +FixedType.prototype.compare = Buffer.compare; + +FixedType.prototype._updateResolver = function (resolver, type) { + if ( + type instanceof FixedType && + this._size === type._size && + ~getAliases(this).indexOf(type._name) + ) { + resolver._size = this._size; + resolver._read = this._read; + } +}; + +FixedType.prototype._copy = BytesType.prototype._copy; + +FixedType.prototype.getAliases = function () { return this._aliases; }; + +FixedType.prototype.getSize = function () { return this._size; }; + +FixedType.prototype.random = function () { + return RANDOM.nextBuffer(this._size); +}; + +FixedType.prototype.toJSON = function () { + return {name: this._name, type: 'fixed', size: this._size}; +}; + +/** + * Avro map. + * + * Represented as vanilla objects. + * + */ +function MapType(attrs, opts) { + if (!attrs.values) { + throw new Error(f('missing map values: %j', attrs)); + } + + opts = getOpts(attrs, opts); + Type.call(this); + this._values = createType(attrs.values, opts); +} +util.inherits(MapType, Type); + +MapType.prototype.getValuesType = function () { return this._values; }; + +MapType.prototype._check = function (val, cb) { + if (!val || typeof val != 'object' || val instanceof Array) { + if (cb) { + cb(PATH.slice(), val, this); + } + return false; + } + + var keys = Object.keys(val); + var b = true; + var i, l, j, key; + if (cb) { + // Slow path. + j = PATH.length; + PATH.push(''); + for (i = 0, l = keys.length; i < l; i++) { + key = PATH[j] = keys[i]; + if (!this._values._check(val[key], cb)) { + b = false; + } + } + PATH.pop(); + } else { + for (i = 0, l = keys.length; i < l; i++) { + if (!this._values._check(val[keys[i]], cb)) { + return false; + } + } + } + return b; +}; + +MapType.prototype._read = function (tap) { + var values = this._values; + var val = {}; + var n; + while ((n = readArraySize(tap))) { + while (n--) { + var key = tap.readString(); + val[key] = values._read(tap); + } + } + return val; +}; + +MapType.prototype._skip = function (tap) { + var values = this._values; + var len, n; + while ((n = tap.readLong())) { + if (n < 0) { + len = tap.readLong(); + tap.pos += len; + } else { + while (n--) { + tap.skipString(); + values._skip(tap); + } + } + } +}; + +MapType.prototype._write = function (tap, val) { + if (!val || typeof val != 'object' || val instanceof Array) { + throwInvalidError(null, val, this); + } + + var values = this._values; + var keys = Object.keys(val); + var n = keys.length; + var i, key; + if (n) { + tap.writeLong(n); + for (i = 0; i < n; i++) { + key = keys[i]; + tap.writeString(key); + values._write(tap, val[key]); + } + } + tap.writeLong(0); +}; + +MapType.prototype._match = function () { + throw new Error('maps cannot be compared'); +}; + +MapType.prototype._updateResolver = function (resolver, type, opts) { + if (type instanceof MapType) { + resolver._values = this._values.createResolver(type._values, opts); + resolver._read = this._read; + } +}; + +MapType.prototype._copy = function (val, opts) { + if (val && typeof val == 'object' && !(val instanceof Array)) { + var values = this._values; + var keys = Object.keys(val); + var i, l, key; + var copy = {}; + for (i = 0, l = keys.length; i < l; i++) { + key = keys[i]; + copy[key] = values._copy(val[key], opts); + } + return copy; + } + throwInvalidError(null, val, this); +}; + +MapType.prototype.compare = MapType.prototype._match; + +MapType.prototype.random = function () { + var val = {}; + var i, l; + for (i = 0, l = RANDOM.nextInt(10); i < l; i++) { + val[RANDOM.nextString(RANDOM.nextInt(20))] = this._values.random(); + } + return val; +}; + +MapType.prototype.toJSON = function () { + return {type: 'map', values: this._values}; +}; + +/** + * Avro array. + * + * Represented as vanilla arrays. + * + */ +function ArrayType(attrs, opts) { + if (!attrs.items) { + throw new Error(f('missing array items: %j', attrs)); + } + + opts = getOpts(attrs, opts); + + this._items = createType(attrs.items, opts); + Type.call(this); +} +util.inherits(ArrayType, Type); + +ArrayType.prototype._check = function (val, cb) { + if (!(val instanceof Array)) { + if (cb) { + cb(PATH.slice(), val, this); + } + return false; + } + + var b = true; + var i, l, j; + if (cb) { + // Slow path. + j = PATH.length; + PATH.push(''); + for (i = 0, l = val.length; i < l; i++) { + PATH[j] = '' + i; + if (!this._items._check(val[i], cb)) { + b = false; + } + } + PATH.pop(); + } else { + for (i = 0, l = val.length; i < l; i++) { + if (!this._items._check(val[i], cb)) { + return false; + } + } + } + return b; +}; + +ArrayType.prototype._read = function (tap) { + var items = this._items; + var val = []; + var n; + while ((n = tap.readLong())) { + if (n < 0) { + n = -n; + tap.skipLong(); // Skip size. + } + while (n--) { + val.push(items._read(tap)); + } + } + return val; +}; + +ArrayType.prototype._skip = function (tap) { + var len, n; + while ((n = tap.readLong())) { + if (n < 0) { + len = tap.readLong(); + tap.pos += len; + } else { + while (n--) { + this._items._skip(tap); + } + } + } +}; + +ArrayType.prototype._write = function (tap, val) { + if (!(val instanceof Array)) { + throwInvalidError(null, val, this); + } + + var n = val.length; + var i; + if (n) { + tap.writeLong(n); + for (i = 0; i < n; i++) { + this._items._write(tap, val[i]); + } + } + tap.writeLong(0); +}; + +ArrayType.prototype._match = function (tap1, tap2) { + var n1 = tap1.readLong(); + var n2 = tap2.readLong(); + var f; + while (n1 && n2) { + f = this._items._match(tap1, tap2); + if (f) { + return f; + } + if (!--n1) { + n1 = readArraySize(tap1); + } + if (!--n2) { + n2 = readArraySize(tap2); + } + } + return utils.compare(n1, n2); +}; + +ArrayType.prototype._updateResolver = function (resolver, type, opts) { + if (type instanceof ArrayType) { + resolver._items = this._items.createResolver(type._items, opts); + resolver._read = this._read; + } +}; + +ArrayType.prototype._copy = function (val, opts) { + if (!(val instanceof Array)) { + throwInvalidError(null, val, this); + } + var items = []; + var i, l; + for (i = 0, l = val.length; i < l; i++) { + items.push(this._items._copy(val[i], opts)); + } + return items; +}; + +ArrayType.prototype.compare = function (val1, val2) { + var n1 = val1.length; + var n2 = val2.length; + var i, l, f; + for (i = 0, l = Math.min(n1, n2); i < l; i++) { + if ((f = this._items.compare(val1[i], val2[i]))) { + return f; + } + } + return utils.compare(n1, n2); +}; + +ArrayType.prototype.getItemsType = function () { return this._items; }; + +ArrayType.prototype.random = function () { + var arr = []; + var i, l; + for (i = 0, l = RANDOM.nextInt(10); i < l; i++) { + arr.push(this._items.random()); + } + return arr; +}; + +ArrayType.prototype.toJSON = function () { + return {type: 'array', items: this._items}; +}; + +/** + * Avro record. + * + * Values are represented as instances of a programmatically generated + * constructor (similar to a "specific record"), available via the + * `getRecordConstructor` method. This "specific record class" gives + * significant speedups over using generics objects. + * + * Note that vanilla objects are still accepted as valid as long as their + * fields match (this makes it much more convenient to do simple things like + * update nested records). + * + */ +function RecordType(attrs, opts) { + opts = getOpts(attrs, opts); + + var resolutions = resolveNames(attrs, opts.namespace); + this._name = resolutions.name; + this._aliases = resolutions.aliases; + Type.call(this, opts.registry); + + if (!(attrs.fields instanceof Array)) { + throw new Error(f('non-array %s fields', this._name)); + } + this._fields = attrs.fields.map(function (f) { + return new Field(f, opts); + }); + if (utils.hasDuplicates(attrs.fields, function (f) { return f.name; })) { + throw new Error(f('duplicate %s field name', this._name)); + } + + var isError = attrs.type === 'error'; + this._constructor = this._createConstructor(isError); + this._read = this._createReader(); + this._skip = this._createSkipper(); + this._write = this._createWriter(); + this._check = this._createChecker(); +} +util.inherits(RecordType, Type); + +RecordType.prototype._createConstructor = function (isError) { + // jshint -W054 + var outerArgs = []; + var innerArgs = []; + var ds = []; // Defaults. + var innerBody = isError ? ' Error.call(this);\n' : ''; + // Not calling `Error.captureStackTrace` because this wouldn't be compatible + // with browsers other than Chrome. + var i, l, field, name, getDefault; + for (i = 0, l = this._fields.length; i < l; i++) { + field = this._fields[i]; + getDefault = field.getDefault; + name = field._name; + innerArgs.push('v' + i); + innerBody += ' '; + if (getDefault() === undefined) { + innerBody += 'this.' + name + ' = v' + i + ';\n'; + } else { + innerBody += 'if (v' + i + ' === undefined) { '; + innerBody += 'this.' + name + ' = d' + ds.length + '(); '; + innerBody += '} else { this.' + name + ' = v' + i + '; }\n'; + outerArgs.push('d' + ds.length); + ds.push(getDefault); + } + } + var outerBody = 'return function ' + unqualify(this._name) + '('; + outerBody += innerArgs.join() + ') {\n' + innerBody + '};'; + var Record = new Function(outerArgs.join(), outerBody).apply(undefined, ds); + + var self = this; + Record.getType = function () { return self; }; + Record.prototype = { + constructor: Record, + $clone: function (opts) { return self.clone(this, opts); }, + $compare: function (val) { return self.compare(this, val); }, + $getType: Record.getType, + $isValid: function (opts) { return self.isValid(this, opts); }, + $toBuffer: function () { return self.toBuffer(this); }, + $toString: function (noCheck) { return self.toString(this, noCheck); } + }; + // The names of these properties added to the prototype are prefixed with `$` + // because it is an invalid property name in Avro but not in JavaScript. + // (This way we are guaranteed not to be stepped over!) + if (isError) { + util.inherits(Record, Error); + // Not setting the name on the prototype to be consistent with how object + // fields are mapped to (only if defined in the schema as a field). + } + + return Record; +}; + +RecordType.prototype._createChecker = function () { + // jshint -W054 + var names = ['t', 'P']; + var values = [this, PATH]; + var body = 'return function check' + unqualify(this._name) + '(val, cb) {\n'; + body += ' if (val === null || typeof val != \'object\') {\n'; + body += ' if (cb) { cb(P.slice(), val, t); }\n'; + body += ' return false;\n'; + body += ' }\n'; + if (!this._fields.length) { + // Special case, empty record. We handle this directly. + body += ' return true;\n'; + } else { + for (i = 0, l = this._fields.length; i < l; i++) { + field = this._fields[i]; + names.push('t' + i); + values.push(field._type); + if (field.getDefault() !== undefined) { + body += ' var v' + i + ' = val.' + field._name + ';\n'; + } + } + body += ' if (cb) {\n'; + body += ' var b = 1;\n'; + body += ' var j = P.length;\n'; + body += ' P.push(\'\');\n'; + var i, l, field; + for (i = 0, l = this._fields.length; i < l; i++) { + field = this._fields[i]; + body += ' P[j] = \'' + field._name + '\';\n'; + if (field.getDefault() === undefined) { + body += ' b &= t' + i + '._check(val.' + field._name + ', cb);\n'; + } else { + body += ' b &= v' + i + ' === undefined || '; + body += 't' + i + '._check(v' + i + ', cb);\n'; + } + } + body += ' P.pop();\n'; + body += ' return !!b;\n'; + body += ' } else {\n return (\n '; + body += this._fields.map(function (field, i) { + if (field.getDefault() === undefined) { + return 't' + i + '._check(val.' + field._name + ')'; + } else { + return '(v' + i + ' === undefined || t' + i + '._check(v' + i + '))'; + } + }).join(' &&\n '); + body += '\n );\n }\n'; + } + body += '};'; + return new Function(names.join(), body).apply(undefined, values); +}; + +RecordType.prototype._createReader = function () { + // jshint -W054 + var uname = unqualify(this._name); + var names = []; + var values = [this._constructor]; + var i, l; + for (i = 0, l = this._fields.length; i < l; i++) { + names.push('t' + i); + values.push(this._fields[i]._type); + } + var body = 'return function read' + uname + '(tap) {\n'; + body += ' return new ' + uname + '('; + body += names.map(function (t) { return t + '._read(tap)'; }).join(); + body += ');\n};'; + names.unshift(uname); + // We can do this since the JS spec guarantees that function arguments are + // evaluated from left to right. + return new Function(names.join(), body).apply(undefined, values); +}; + +RecordType.prototype._createSkipper = function () { + // jshint -W054 + var args = []; + var body = 'return function skip' + unqualify(this._name) + '(tap) {\n'; + var values = []; + var i, l; + for (i = 0, l = this._fields.length; i < l; i++) { + args.push('t' + i); + values.push(this._fields[i]._type); + body += ' t' + i + '._skip(tap);\n'; + } + body += '}'; + return new Function(args.join(), body).apply(undefined, values); +}; + +RecordType.prototype._createWriter = function () { + // jshint -W054 + // We still do default handling here, in case a normal JS object is passed. + var args = []; + var body = 'return function write' + unqualify(this._name) + '(tap, val) {\n'; + var values = []; + var i, l, field, value; + for (i = 0, l = this._fields.length; i < l; i++) { + field = this._fields[i]; + args.push('t' + i); + values.push(field._type); + body += ' '; + if (field.getDefault() === undefined) { + body += 't' + i + '._write(tap, val.' + field._name + ');\n'; + } else { + value = field._type.toBuffer(field.getDefault()).toString('binary'); + // Convert the default value to a binary string ahead of time. We aren't + // converting it to a buffer to avoid retaining too much memory. If we + // had our own buffer pool, this could be an idea in the future. + args.push('d' + i); + values.push(value); + body += 'var v' + i + ' = val.' + field._name + '; '; + body += 'if (v' + i + ' === undefined) { '; + body += 'tap.writeBinary(d' + i + ', ' + value.length + ');'; + body += ' } else { t' + i + '._write(tap, v' + i + '); }\n'; + } + } + body += '}'; + return new Function(args.join(), body).apply(undefined, values); +}; + +RecordType.prototype._updateResolver = function (resolver, type, opts) { + // jshint -W054 + if (!~getAliases(this).indexOf(type._name)) { + throw new Error(f('no alias for %s in %s', type._name, this._name)); + } + + var rFields = this._fields; + var wFields = type._fields; + var wFieldsMap = utils.toMap(wFields, function (f) { return f._name; }); + + var innerArgs = []; // Arguments for reader constructor. + var resolvers = {}; // Resolvers keyed by writer field name. + var i, j, field, name, names, matches; + for (i = 0; i < rFields.length; i++) { + field = rFields[i]; + names = getAliases(field); + matches = []; + for (j = 0; j < names.length; j++) { + name = names[j]; + if (wFieldsMap[name]) { + matches.push(name); + } + } + if (matches.length > 1) { + throw new Error(f('multiple matches for %s', field.name)); + } + if (!matches.length) { + if (field.getDefault() === undefined) { + throw new Error(f('no match for default-less %s', field.name)); + } + innerArgs.push('undefined'); + } else { + name = matches[0]; + resolvers[name] = { + resolver: field._type.createResolver(wFieldsMap[name]._type, opts), + name: field._name // Reader field name. + }; + innerArgs.push(field._name); + } + } + + // See if we can add a bypass for unused fields at the end of the record. + var lazyIndex = -1; + i = wFields.length; + while (i && resolvers[wFields[--i]._name] === undefined) { + lazyIndex = i; + } + + var uname = unqualify(this._name); + var args = [uname]; + var values = [this._constructor]; + var body = ' return function read' + uname + '(tap,lazy) {\n'; + for (i = 0; i < wFields.length; i++) { + if (i === lazyIndex) { + body += ' if (!lazy) {\n'; + } + field = type._fields[i]; + name = field._name; + body += (~lazyIndex && i >= lazyIndex) ? ' ' : ' '; + if (resolvers[name] === undefined) { + args.push('t' + i); + values.push(field._type); + body += 't' + i + '._skip(tap);\n'; + } else { + args.push('t' + i); + values.push(resolvers[name].resolver); + body += 'var ' + resolvers[name].name + ' = '; + body += 't' + i + '._read(tap);\n'; + } + } + if (~lazyIndex) { + body += ' }\n'; + } + body += ' return new ' + uname + '(' + innerArgs.join() + ');\n};'; + + resolver._read = new Function(args.join(), body).apply(undefined, values); +}; + +RecordType.prototype._match = function (tap1, tap2) { + var fields = this._fields; + var i, l, field, order, type; + for (i = 0, l = fields.length; i < l; i++) { + field = fields[i]; + order = field._order; + type = field._type; + if (order) { + order *= type._match(tap1, tap2); + if (order) { + return order; + } + } else { + type._skip(tap1); + type._skip(tap2); + } + } + return 0; +}; + +RecordType.prototype._copy = function (val, opts) { + // jshint -W058 + var hook = opts && opts.fieldHook; + var values = [undefined]; + var i, l, field, value; + for (i = 0, l = this._fields.length; i < l; i++) { + field = this._fields[i]; + value = field._type._copy(val[field._name], opts); + if (hook) { + value = hook(field, value, this); + } + values.push(value); + } + return new (this._constructor.bind.apply(this._constructor, values)); +}; + +RecordType.prototype.compare = function (val1, val2) { + var fields = this._fields; + var i, l, field, name, order, type; + for (i = 0, l = fields.length; i < l; i++) { + field = fields[i]; + name = field._name; + order = field._order; + type = field._type; + if (order) { + order *= type.compare(val1[name], val2[name]); + if (order) { + return order; + } + } + } + return 0; +}; + +RecordType.prototype.random = function () { + // jshint -W058 + var fields = this._fields.map(function (f) { return f._type.random(); }); + fields.unshift(undefined); + return new (this._constructor.bind.apply(this._constructor, fields)); +}; + +RecordType.prototype.getAliases = function () { return this._aliases; }; + +RecordType.prototype.getFields = function () { return this._fields.slice(); }; + +RecordType.prototype.getRecordConstructor = function () { + return this._constructor; +}; + +RecordType.prototype.toJSON = function () { + return {name: this._name, type: 'record', fields: this._fields}; +}; + +/** + * Derived type abstract class. + * + */ +function LogicalType(attrs, opts, Types) { + Type.call(this); + LOGICAL_TYPE = this; + this._underlyingType = createType(attrs, opts); + + // Convenience type check. + if (Types && !~Types.indexOf(this._underlyingType.constructor)) { + var lType = attrs.logicalType; + var uType = this._underlyingType; + throw new Error(f('invalid underlying type for %s: %s', lType, uType)); + } +} +util.inherits(LogicalType, Type); + +LogicalType.prototype.getUnderlyingType = function () { + return this._underlyingType; +}; + +LogicalType.prototype._read = function (tap) { + return this._fromValue(this._underlyingType._read(tap)); +}; + +LogicalType.prototype._write = function (tap, val) { + this._underlyingType._write(tap, this._toValue(val)); +}; + +LogicalType.prototype._check = function (val, cb) { + return this._underlyingType._check(this._toValue(val), cb); +}; + +LogicalType.prototype._copy = function (val, opts) { + var type = this._underlyingType; + switch (opts && opts.coerce) { + case 3: // To string. + return type._copy(this._toValue(val), opts); + case 2: // From string. + return this._fromValue(type._copy(val, opts)); + default: // Normal copy. + return this._fromValue(type._copy(this._toValue(val), opts)); + } +}; + +LogicalType.prototype._updateResolver = function (resolver, type, opts) { + var _fromValue = this._resolve(type, opts); + if (_fromValue) { + resolver._read = function (tap) { return _fromValue(type._read(tap)); }; + } +}; + +LogicalType.prototype.random = function () { + return this._fromValue(this._underlyingType.random()); +}; + +LogicalType.prototype.compare = function (obj1, obj2) { + var val1 = this._toValue(obj1); + var val2 = this._toValue(obj2); + return this._underlyingType.compare(val1, val2); +}; + +LogicalType.prototype.toJSON = function () { + return this._underlyingType.toJSON(); +}; + +// Methods to be implemented. +LogicalType.prototype._fromValue = utils.abstractFunction; +LogicalType.prototype._toValue = utils.abstractFunction; +LogicalType.prototype._resolve = utils.abstractFunction; + + +// General helpers. + +/** + * Customizable long. + * + * This allows support of arbitrarily large long (e.g. larger than + * `Number.MAX_SAFE_INTEGER`). See `LongType.using` method above. + * + */ +function AbstractLongType(noUnpack) { + LongType.call(this); + this._noUnpack = !!noUnpack; +} +util.inherits(AbstractLongType, LongType); + +AbstractLongType.prototype._check = function (val, cb) { + var b = this._isValid(val); + if (!b && cb) { + cb(PATH.slice(), val, this); + } + return b; +}; + +AbstractLongType.prototype._read = function (tap) { + var buf, pos; + if (this._noUnpack) { + pos = tap.pos; + tap.skipLong(); + buf = tap.buf.slice(pos, tap.pos); + } else { + buf = tap.unpackLongBytes(tap); + } + if (tap.isValid()) { + return this._fromBuffer(buf); + } +}; + +AbstractLongType.prototype._write = function (tap, val) { + if (!this._isValid(val)) { + throwInvalidError(null, val, this); + } + var buf = this._toBuffer(val); + if (this._noUnpack) { + tap.writeFixed(buf); + } else { + tap.packLongBytes(buf); + } +}; + +AbstractLongType.prototype._copy = function (val, opts) { + switch (opts && opts.coerce) { + case 3: // To string. + return this._toJSON(val); + case 2: // From string. + return this._fromJSON(val); + default: // Normal copy. + // Slow but guarantees most consistent results. Faster alternatives would + // require assumptions on the long class used (e.g. immutability). + return this._fromJSON(JSON.parse(JSON.stringify(this._toJSON(val)))); + } +}; + +AbstractLongType.prototype.random = function () { + return this._fromJSON(LongType.prototype.random()); +}; + +// Methods to be implemented by the user. +AbstractLongType.prototype._fromBuffer = utils.abstractFunction; +AbstractLongType.prototype._toBuffer = utils.abstractFunction; +AbstractLongType.prototype._fromJSON = utils.abstractFunction; +AbstractLongType.prototype._toJSON = utils.abstractFunction; +AbstractLongType.prototype._isValid = utils.abstractFunction; +AbstractLongType.prototype.compare = utils.abstractFunction; + +/** + * Field. + * + * @param attrs {Object} The field's schema. + * @para opts {Object} Schema parsing options (the same as `Type`s'). + * + */ +function Field(attrs, opts) { + var name = attrs.name; + if (typeof name != 'string' || !NAME_PATTERN.test(name)) { + throw new Error(f('invalid field name: %s', name)); + } + + this._name = name; + this._type = createType(attrs.type, opts); + this._aliases = attrs.aliases || []; + + this._order = (function (order) { + switch (order) { + case 'ascending': + return 1; + case 'descending': + return -1; + case 'ignore': + return 0; + default: + throw new Error(f('invalid order: %j', order)); + } + })(attrs.order === undefined ? 'ascending' : attrs.order); + + var value = attrs['default']; + if (value !== undefined) { + // We need to convert defaults back to a valid format (unions are + // disallowed in default definitions, only the first type of each union is + // allowed instead). + // http://apache-avro.679487.n3.nabble.com/field-union-default-in-Java-td1175327.html + var type = this._type; + var val = type._copy(value, {coerce: 2, wrap: 2}); + // The clone call above will throw an error if the default is invalid. + if (type instanceof PrimitiveType && !(type instanceof BytesType)) { + // These are immutable. + this.getDefault = function () { return val; }; + } else { + this.getDefault = function () { return type._copy(val); }; + } + } +} + +Field.prototype.getAliases = function () { return this._aliases; }; + +Field.prototype.getDefault = function () {}; // Undefined default. + +Field.prototype.getName = function () { return this._name; }; + +Field.prototype.getOrder = function () { + return ['descending', 'ignore', 'ascending'][this._order + 1]; +}; + +Field.prototype.getType = function () { return this._type; }; + +Field.prototype.inspect = function () { return f('<Field %j>', this._name); }; + +/** + * Resolver to read a writer's schema as a new schema. + * + * @param readerType {Type} The type to convert to. + * + */ +function Resolver(readerType) { + // Add all fields here so that all resolvers share the same hidden class. + this._readerType = readerType; + this._items = null; + this._read = null; + this._size = 0; + this._symbols = null; + this._values = null; +} + +Resolver.prototype.inspect = function () { return '<Resolver>'; }; + +/** + * Read a value from a tap. + * + * @param type {Type} The type to decode. + * @param tap {Tap} The tap to read from. No checks are performed here. + * @param resolver {Resolver} Optional resolver. It must match the input type. + * @param lazy {Boolean} Skip trailing fields when using a resolver. + * + */ +function readValue(type, tap, resolver, lazy) { + if (resolver) { + if (resolver._readerType !== type) { + throw new Error('invalid resolver'); + } + return resolver._read(tap, lazy); + } else { + return type._read(tap); + } +} + +/** + * Create default parsing options. + * + * @param attrs {Object} Schema to populate options with. + * @param opts {Object} Base options. + * + */ +function getOpts(attrs, opts) { + if (attrs === null) { + // Let's be helpful for this common error. + throw new Error('invalid type: null (did you mean "null"?)'); + } + opts = opts || {}; + opts.registry = opts.registry || {}; + opts.namespace = attrs.namespace || opts.namespace; + opts.logicalTypes = opts.logicalTypes || {}; + return opts; +} + +/** + * Resolve a schema's name and aliases. + * + * @param attrs {Object} True schema (can't be a string). + * @param namespace {String} Optional parent namespace. + * @param key {String} Key where the name should be looked up (defaults to + * `name`). + * + */ +function resolveNames(attrs, namespace, key) { + namespace = attrs.namespace || namespace; + key = key || 'name'; + + var name = attrs[key]; + if (!name) { + throw new Error(f('missing %s property in schema: %j', key, attrs)); + } + return { + name: qualify(name), + aliases: attrs.aliases ? attrs.aliases.map(qualify) : [] + }; + + function qualify(name) { + if (!~name.indexOf('.') && namespace) { + name = namespace + '.' + name; + } + var tail = unqualify(name); + if (isPrimitive(tail)) { + // Primitive types cannot be defined in any namespace. + throw new Error(f('cannot rename primitive type: %j', tail)); + } + name.split('.').forEach(function (part) { + if (!NAME_PATTERN.test(part)) { + throw new Error(f('invalid name: %j', name)); + } + }); + return name; + } +} + +/** + * Remove namespace from a name. + * + * @param name {String} Full or short name. + * + */ +function unqualify(name) { + var parts = name.split('.'); + return parts[parts.length - 1]; +} + +/** + * Get all aliases for a type (including its name). + * + * @param obj {Type|Object} Typically a type or a field. Its aliases property + * must exist and be an array. + * + */ +function getAliases(obj) { + var names = [obj._name]; + var aliases = obj._aliases; + var i, l; + for (i = 0, l = aliases.length; i < l; i++) { + names.push(aliases[i]); + } + return names; +} + +/** + * Get a type's "type" (as a string, e.g. `'record'`, `'string'`). + * + * @param type {Type} Any type. + * + */ +function getTypeName(type) { + var obj = type.toJSON(); + return typeof obj == 'string' ? obj : obj.type; +} + +/** + * Check whether a type's name is a primitive. + * + * @param name {String} Type name (e.g. `'string'`, `'array'`). + * + */ +function isPrimitive(name) { + var type = TYPES[name]; + return type !== undefined && type.prototype instanceof PrimitiveType; +} + +/** + * Get the number of elements in an array block. + * + * @param tap {Tap} A tap positioned at the beginning of an array block. + * + */ +function readArraySize(tap) { + var n = tap.readLong(); + if (n < 0) { + n = -n; + tap.skipLong(); // Skip size. + } + return n; +} + +/** + * Check whether a long can be represented without precision loss. + * + * @param n {Number} The number. + * + * Two things to note: + * + * + We are not using the `Number` constants for compatibility with older + * browsers. + * + We must remove one from each bound because of rounding errors. + * + */ +function isSafeLong(n) { + return n >= -9007199254740990 && n <= 9007199254740990; +} + +/** + * Throw a somewhat helpful error on invalid object. + * + * @param path {Array} Passed from hook, but unused (because empty where this + * function is used, since we aren't keeping track of it for effiency). + * @param val {...} The object to reject. + * @param type {Type} The type to check against. + * + * This method is mostly used from `_write` to signal an invalid object for a + * given type. Note that this provides less information than calling `isValid` + * with a hook since the path is not propagated (for efficiency reasons). + * + */ +function throwInvalidError(path, val, type) { + throw new Error(f('invalid %s: %j', type, val)); +} + + +module.exports = { + createType: createType, + resolveNames: resolveNames, // Protocols use the same name resolution logic. + types: (function () { + // Export the base types along with all concrete implementations. + var obj = {Type: Type, LogicalType: LogicalType}; + var types = Object.keys(TYPES); + var i, l, Class; + for (i = 0, l = types.length; i < l; i++) { + Class = TYPES[types[i]]; + obj[Class.name] = Class; + } + return obj; + })() +};
Added: avro/trunk/lang/js/lib/utils.js URL: http://svn.apache.org/viewvc/avro/trunk/lang/js/lib/utils.js?rev=1717850&view=auto ============================================================================== --- avro/trunk/lang/js/lib/utils.js (added) +++ avro/trunk/lang/js/lib/utils.js Thu Dec 3 21:35:44 2015 @@ -0,0 +1,632 @@ +/* jshint node: true */ + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +'use strict'; + +/** + * Uppercase the first letter of a string. + * + * @param s {String} The string. + * + */ +function capitalize(s) { return s.charAt(0).toUpperCase() + s.slice(1); } + +/** + * Compare two numbers. + * + * @param n1 {Number} The first one. + * @param n2 {Number} The second one. + * + */ +function compare(n1, n2) { return n1 === n2 ? 0 : (n1 < n2 ? -1 : 1); } + +/** + * Find index of value in array. + * + * @param arr {Array} Can also be a false-ish value. + * @param v {Object} Value to find. + * + * Returns -1 if not found, -2 if found multiple times. + * + */ +function singleIndexOf(arr, v) { + var pos = -1; + var i, l; + if (!arr) { + return -1; + } + for (i = 0, l = arr.length; i < l; i++) { + if (arr[i] === v) { + if (pos >= 0) { + return -2; + } + pos = i; + } + } + return pos; +} + +/** + * Convert array to map. + * + * @param arr {Array} Elements. + * @param fn {Function} Function returning an element's key. + * + */ +function toMap(arr, fn) { + var obj = {}; + var i, elem; + for (i = 0; i < arr.length; i++) { + elem = arr[i]; + obj[fn(elem)] = elem; + } + return obj; +} + +/** + * Check whether an array has duplicates. + * + * @param arr {Array} The array. + * @param fn {Function} Optional function to apply to each element. + * + */ +function hasDuplicates(arr, fn) { + var obj = {}; + var i, l, elem; + for (i = 0, l = arr.length; i < l; i++) { + elem = arr[i]; + if (fn) { + elem = fn(elem); + } + if (obj[elem]) { + return true; + } + obj[elem] = true; + } + return false; +} + +/** + * "Abstract" function to help with "subclassing". + * + */ +function abstractFunction() { throw new Error('abstract'); } + +/** + * Generator of random things. + * + * Inspired by: http://stackoverflow.com/a/424445/1062617 + * + */ +function Lcg(seed) { + var a = 1103515245; + var c = 12345; + var m = Math.pow(2, 31); + var state = Math.floor(seed || Math.random() * (m - 1)); + + this._max = m; + this._nextInt = function () { return state = (a * state + c) % m; }; +} + +Lcg.prototype.nextBoolean = function () { + // jshint -W018 + return !!(this._nextInt() % 2); +}; + +Lcg.prototype.nextInt = function (start, end) { + if (end === undefined) { + end = start; + start = 0; + } + end = end === undefined ? this._max : end; + return start + Math.floor(this.nextFloat() * (end - start)); +}; + +Lcg.prototype.nextFloat = function (start, end) { + if (end === undefined) { + end = start; + start = 0; + } + end = end === undefined ? 1 : end; + return start + (end - start) * this._nextInt() / this._max; +}; + +Lcg.prototype.nextString = function(len, flags) { + len |= 0; + flags = flags || 'aA'; + var mask = ''; + if (flags.indexOf('a') > -1) { + mask += 'abcdefghijklmnopqrstuvwxyz'; + } + if (flags.indexOf('A') > -1) { + mask += 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'; + } + if (flags.indexOf('#') > -1) { + mask += '0123456789'; + } + if (flags.indexOf('!') > -1) { + mask += '~`!@#$%^&*()_+-={}[]:";\'<>?,./|\\'; + } + var result = []; + for (var i = 0; i < len; i++) { + result.push(this.choice(mask)); + } + return result.join(''); +}; + +Lcg.prototype.nextBuffer = function (len) { + var arr = []; + var i; + for (i = 0; i < len; i++) { + arr.push(this.nextInt(256)); + } + return new Buffer(arr); +}; + +Lcg.prototype.choice = function (arr) { + var len = arr.length; + if (!len) { + throw new Error('choosing from empty array'); + } + return arr[this.nextInt(len)]; +}; + +/** + * Ordered queue which returns items consecutively. + * + * This is actually a heap by index, with the added requirements that elements + * can only be retrieved consecutively. + * + */ +function OrderedQueue() { + this._index = 0; + this._items = []; +} + +OrderedQueue.prototype.push = function (item) { + var items = this._items; + var i = items.length | 0; + var j; + items.push(item); + while (i > 0 && items[i].index < items[j = ((i - 1) >> 1)].index) { + item = items[i]; + items[i] = items[j]; + items[j] = item; + i = j; + } +}; + +OrderedQueue.prototype.pop = function () { + var items = this._items; + var len = (items.length - 1) | 0; + var first = items[0]; + if (!first || first.index > this._index) { + return null; + } + this._index++; + if (!len) { + items.pop(); + return first; + } + items[0] = items.pop(); + var mid = len >> 1; + var i = 0; + var i1, i2, j, item, c, c1, c2; + while (i < mid) { + item = items[i]; + i1 = (i << 1) + 1; + i2 = (i + 1) << 1; + c1 = items[i1]; + c2 = items[i2]; + if (!c2 || c1.index <= c2.index) { + c = c1; + j = i1; + } else { + c = c2; + j = i2; + } + if (c.index >= item.index) { + break; + } + items[j] = item; + items[i] = c; + i = j; + } + return first; +}; + +/** + * A tap is a buffer which remembers what has been already read. + * + * It is optimized for performance, at the cost of failing silently when + * overflowing the buffer. This is a purposeful trade-off given the expected + * rarity of this case and the large performance hit necessary to enforce + * validity. See `isValid` below for more information. + * + */ +function Tap(buf, pos) { + this.buf = buf; + this.pos = pos | 0; +} + +/** + * Check that the tap is in a valid state. + * + * For efficiency reasons, none of the methods below will fail if an overflow + * occurs (either read, skip, or write). For this reason, it is up to the + * caller to always check that the read, skip, or write was valid by calling + * this method. + * + */ +Tap.prototype.isValid = function () { return this.pos <= this.buf.length; }; + +// Read, skip, write methods. +// +// These should fail silently when the buffer overflows. Note this is only +// required to be true when the functions are decoding valid objects. For +// example errors will still be thrown if a bad count is read, leading to a +// negative position offset (which will typically cause a failure in +// `readFixed`). + +Tap.prototype.readBoolean = function () { return !!this.buf[this.pos++]; }; + +Tap.prototype.skipBoolean = function () { this.pos++; }; + +Tap.prototype.writeBoolean = function (b) { this.buf[this.pos++] = !!b; }; + +Tap.prototype.readInt = Tap.prototype.readLong = function () { + var n = 0; + var k = 0; + var buf = this.buf; + var b, h, f, fk; + + do { + b = buf[this.pos++]; + h = b & 0x80; + n |= (b & 0x7f) << k; + k += 7; + } while (h && k < 28); + + if (h) { + // Switch to float arithmetic, otherwise we might overflow. + f = n; + fk = 268435456; // 2 ** 28. + do { + b = buf[this.pos++]; + f += (b & 0x7f) * fk; + fk *= 128; + } while (b & 0x80); + return (f % 2 ? -(f + 1) : f) / 2; + } + + return (n >> 1) ^ -(n & 1); +}; + +Tap.prototype.skipInt = Tap.prototype.skipLong = function () { + var buf = this.buf; + while (buf[this.pos++] & 0x80) {} +}; + +Tap.prototype.writeInt = Tap.prototype.writeLong = function (n) { + var buf = this.buf; + var f, m; + + if (n >= -1073741824 && n < 1073741824) { + // Won't overflow, we can use integer arithmetic. + m = n >= 0 ? n << 1 : (~n << 1) | 1; + do { + buf[this.pos] = m & 0x7f; + m >>= 7; + } while (m && (buf[this.pos++] |= 0x80)); + } else { + // We have to use slower floating arithmetic. + f = n >= 0 ? n * 2 : (-n * 2) - 1; + do { + buf[this.pos] = f & 0x7f; + f /= 128; + } while (f >= 1 && (buf[this.pos++] |= 0x80)); + } + this.pos++; +}; + +Tap.prototype.readFloat = function () { + var buf = this.buf; + var pos = this.pos; + this.pos += 4; + if (this.pos > buf.length) { + return; + } + return this.buf.readFloatLE(pos); +}; + +Tap.prototype.skipFloat = function () { this.pos += 4; }; + +Tap.prototype.writeFloat = function (f) { + var buf = this.buf; + var pos = this.pos; + this.pos += 4; + if (this.pos > buf.length) { + return; + } + return this.buf.writeFloatLE(f, pos); +}; + +Tap.prototype.readDouble = function () { + var buf = this.buf; + var pos = this.pos; + this.pos += 8; + if (this.pos > buf.length) { + return; + } + return this.buf.readDoubleLE(pos); +}; + +Tap.prototype.skipDouble = function () { this.pos += 8; }; + +Tap.prototype.writeDouble = function (d) { + var buf = this.buf; + var pos = this.pos; + this.pos += 8; + if (this.pos > buf.length) { + return; + } + return this.buf.writeDoubleLE(d, pos); +}; + +Tap.prototype.readFixed = function (len) { + var pos = this.pos; + this.pos += len; + if (this.pos > this.buf.length) { + return; + } + var fixed = new Buffer(len); + this.buf.copy(fixed, 0, pos, pos + len); + return fixed; +}; + +Tap.prototype.skipFixed = function (len) { this.pos += len; }; + +Tap.prototype.writeFixed = function (buf, len) { + len = len || buf.length; + var pos = this.pos; + this.pos += len; + if (this.pos > this.buf.length) { + return; + } + buf.copy(this.buf, pos, 0, len); +}; + +Tap.prototype.readBytes = function () { + return this.readFixed(this.readLong()); +}; + +Tap.prototype.skipBytes = function () { + var len = this.readLong(); + this.pos += len; +}; + +Tap.prototype.writeBytes = function (buf) { + var len = buf.length; + this.writeLong(len); + this.writeFixed(buf, len); +}; + +Tap.prototype.readString = function () { + var len = this.readLong(); + var pos = this.pos; + var buf = this.buf; + this.pos += len; + if (this.pos > buf.length) { + return; + } + return this.buf.utf8Slice(pos, pos + len); +}; + +Tap.prototype.skipString = function () { + var len = this.readLong(); + this.pos += len; +}; + +Tap.prototype.writeString = function (s) { + var len = Buffer.byteLength(s); + this.writeLong(len); + var pos = this.pos; + this.pos += len; + if (this.pos > this.buf.length) { + return; + } + this.buf.utf8Write(s, pos, len); +}; + +// Helper used to speed up writing defaults. + +Tap.prototype.writeBinary = function (str, len) { + var pos = this.pos; + this.pos += len; + if (this.pos > this.buf.length) { + return; + } + this.buf.binaryWrite(str, pos, len); +}; + +// Binary comparison methods. +// +// These are not guaranteed to consume the objects they are comparing when +// returning a non-zero result (allowing for performance benefits), so no other +// operations should be done on either tap after a compare returns a non-zero +// value. Also, these methods do not have the same silent failure requirement +// as read, skip, and write since they are assumed to be called on valid +// buffers. + +Tap.prototype.matchBoolean = function (tap) { + return this.buf[this.pos++] - tap.buf[tap.pos++]; +}; + +Tap.prototype.matchInt = Tap.prototype.matchLong = function (tap) { + var n1 = this.readLong(); + var n2 = tap.readLong(); + return n1 === n2 ? 0 : (n1 < n2 ? -1 : 1); +}; + +Tap.prototype.matchFloat = function (tap) { + var n1 = this.readFloat(); + var n2 = tap.readFloat(); + return n1 === n2 ? 0 : (n1 < n2 ? -1 : 1); +}; + +Tap.prototype.matchDouble = function (tap) { + var n1 = this.readDouble(); + var n2 = tap.readDouble(); + return n1 === n2 ? 0 : (n1 < n2 ? -1 : 1); +}; + +Tap.prototype.matchFixed = function (tap, len) { + return this.readFixed(len).compare(tap.readFixed(len)); +}; + +Tap.prototype.matchBytes = Tap.prototype.matchString = function (tap) { + var l1 = this.readLong(); + var p1 = this.pos; + this.pos += l1; + var l2 = tap.readLong(); + var p2 = tap.pos; + tap.pos += l2; + var b1 = this.buf.slice(p1, this.pos); + var b2 = tap.buf.slice(p2, tap.pos); + return b1.compare(b2); +}; + +// Functions for supporting custom long classes. +// +// The two following methods allow the long implementations to not have to +// worry about Avro's zigzag encoding, we directly expose longs as unpacked. + +Tap.prototype.unpackLongBytes = function () { + var res = new Buffer(8); + var n = 0; + var i = 0; // Byte index in target buffer. + var j = 6; // Bit offset in current target buffer byte. + var buf = this.buf; + var b, neg; + + b = buf[this.pos++]; + neg = b & 1; + res.fill(0); + + n |= (b & 0x7f) >> 1; + while (b & 0x80) { + b = buf[this.pos++]; + n |= (b & 0x7f) << j; + j += 7; + if (j >= 8) { + // Flush byte. + j -= 8; + res[i++] = n; + n >>= 8; + } + } + res[i] = n; + + if (neg) { + invert(res, 8); + } + + return res; +}; + +Tap.prototype.packLongBytes = function (buf) { + var neg = (buf[7] & 0x80) >> 7; + var res = this.buf; + var j = 1; + var k = 0; + var m = 3; + var n; + + if (neg) { + invert(buf, 8); + n = 1; + } else { + n = 0; + } + + var parts = [ + buf.readUIntLE(0, 3), + buf.readUIntLE(3, 3), + buf.readUIntLE(6, 2) + ]; + // Not reading more than 24 bits because we need to be able to combine the + // "carry" bits from the previous part and JavaScript only supports bitwise + // operations on 32 bit integers. + while (m && !parts[--m]) {} // Skip trailing 0s. + + // Leading parts (if any), we never bail early here since we need the + // continuation bit to be set. + while (k < m) { + n |= parts[k++] << j; + j += 24; + while (j > 7) { + res[this.pos++] = (n & 0x7f) | 0x80; + n >>= 7; + j -= 7; + } + } + + // Final part, similar to normal packing aside from the initial offset. + n |= parts[m] << j; + do { + res[this.pos] = n & 0x7f; + n >>= 7; + } while (n && (res[this.pos++] |= 0x80)); + this.pos++; + + // Restore original buffer (could make this optional?). + if (neg) { + invert(buf, 8); + } +}; + +// Helpers. + +/** + * Invert all bits in a buffer. + * + * @param buf {Buffer} Non-empty buffer to invert. + * @param len {Number} Buffer length (must be positive). + * + */ +function invert(buf, len) { + while (len--) { + buf[len] = ~buf[len]; + } +} + + +module.exports = { + abstractFunction: abstractFunction, + capitalize: capitalize, + compare: compare, + toMap: toMap, + singleIndexOf: singleIndexOf, + hasDuplicates: hasDuplicates, + Lcg: Lcg, + OrderedQueue: OrderedQueue, + Tap: Tap +}; Modified: avro/trunk/lang/js/package.json URL: http://svn.apache.org/viewvc/avro/trunk/lang/js/package.json?rev=1717850&r1=1717849&r2=1717850&view=diff ============================================================================== --- avro/trunk/lang/js/package.json (original) +++ avro/trunk/lang/js/package.json Thu Dec 3 21:35:44 2015 @@ -1,17 +1,18 @@ { "name": "avro-js", - "version": "0.0.1", + "version": "1.8.0", "author": "Avro Developers <[email protected]>", - "description": "Avro validator for Javascript", + "description": "JavaScript Avro implementation", "contributors": [ { + "name": "Matthieu Monsch", + "email": "[email protected]" + }, + { "name": "Quinn Slack", "email": "[email protected]" - } + } ], - "scripts": { - "test": "grunt test" - }, "repository": { "type": "svn", "url": "http://svn.apache.org/repos/asf/avro/trunk/lang/js/"; @@ -20,18 +21,34 @@ "avro", "json" ], - "dependencies" : { - "underscore" : "*" + "files": [ + "LICENSE", + "NOTICE", + "lib", + "etc/browser", + "etc/deprecated/validator.js" + ], + "main": "./lib", + "browser": { + "./lib": "./etc/browser/avro.js", + "crypto": "./etc/browser/crypto.js" + }, + "scripts": { + "test": "mocha --ui tdd --reporter dot", + "clean": "rm -rf node_modules" + }, + "dependencies": { + "underscore": "*" }, - "devDependencies" : { - "grunt" : "*", - "grunt-contrib-jshint" : "*", - "grunt-contrib-nodeunit" : "*", - "grunt-contrib-watch" : "*" + "devDependencies": { + "coveralls": "^2.11.4", + "istanbul": "^0.3.19", + "mocha": "^2.3.2", + "tmp": "^0.0.28" }, "noAnalyze": true, - "license": "Apache", + "license": "Apache-2.0", "engine": { - "node": ">=0.4" + "node": ">=0.11" } } Added: avro/trunk/lang/js/test/dat/Id.avsc URL: http://svn.apache.org/viewvc/avro/trunk/lang/js/test/dat/Id.avsc?rev=1717850&view=auto ============================================================================== --- avro/trunk/lang/js/test/dat/Id.avsc (added) +++ avro/trunk/lang/js/test/dat/Id.avsc Thu Dec 3 21:35:44 2015 @@ -0,0 +1,6 @@ +{ + "type": "fixed", + "name": "Id", + "namespace": "id", + "size": 64 +} Added: avro/trunk/lang/js/test/dat/Person.avsc URL: http://svn.apache.org/viewvc/avro/trunk/lang/js/test/dat/Person.avsc?rev=1717850&view=auto ============================================================================== --- avro/trunk/lang/js/test/dat/Person.avsc (added) +++ avro/trunk/lang/js/test/dat/Person.avsc Thu Dec 3 21:35:44 2015 @@ -0,0 +1,20 @@ +{ + "name": "Person", + "type": "record", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "age", "type": ["null", "int"], "default": null}, + { + "name": "gender", + "type": {"name": "Gender", "type": "enum", "symbols": ["FEMALE", "MALE"]} + }, + { + "name": "address", + "type": { + "name": "Address", + "type": "record", + "fields": [{"name": "zipcode", "type": "int"}] + } + } + ] +} Added: avro/trunk/lang/js/test/dat/person-10.avro URL: http://svn.apache.org/viewvc/avro/trunk/lang/js/test/dat/person-10.avro?rev=1717850&view=auto ============================================================================== Binary files avro/trunk/lang/js/test/dat/person-10.avro (added) and avro/trunk/lang/js/test/dat/person-10.avro Thu Dec 3 21:35:44 2015 differ Added: avro/trunk/lang/js/test/dat/person-10.avro.raw URL: http://svn.apache.org/viewvc/avro/trunk/lang/js/test/dat/person-10.avro.raw?rev=1717850&view=auto ============================================================================== Binary files avro/trunk/lang/js/test/dat/person-10.avro.raw (added) and avro/trunk/lang/js/test/dat/person-10.avro.raw Thu Dec 3 21:35:44 2015 differ Added: avro/trunk/lang/js/test/dat/person-10.no-codec.avro URL: http://svn.apache.org/viewvc/avro/trunk/lang/js/test/dat/person-10.no-codec.avro?rev=1717850&view=auto ============================================================================== Binary files avro/trunk/lang/js/test/dat/person-10.no-codec.avro (added) and avro/trunk/lang/js/test/dat/person-10.no-codec.avro Thu Dec 3 21:35:44 2015 differ
