Author: tomwhite
Date: Wed Jul 8 14:31:50 2015
New Revision: 1689889
URL: http://svn.apache.org/r1689889
Log:
AVRO-1693. Ruby: Allow writing arbitrary metadata to data files. Contributed by
Daniel Schierbeck.
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/ruby/lib/avro/data_file.rb
avro/trunk/lang/ruby/test/test_datafile.rb
Modified: avro/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1689889&r1=1689888&r2=1689889&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Wed Jul 8 14:31:50 2015
@@ -90,6 +90,9 @@ Trunk (not yet released)
AVRO-1645. Ruby: Improved handling of missing named types.
(Daniel Schierbeck via tomwhite)
+ AVRO-1693. Ruby: Allow writing arbitrary metadata to data files.
+ (Daniel Schierbeck via tomwhite)
+
BUG FIXES
AVRO-1553. Java: MapReduce never uses MapOutputValueSchema (tomwhite)
Modified: avro/trunk/lang/ruby/lib/avro/data_file.rb
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/ruby/lib/avro/data_file.rb?rev=1689889&r1=1689888&r2=1689889&view=diff
==============================================================================
--- avro/trunk/lang/ruby/lib/avro/data_file.rb (original)
+++ avro/trunk/lang/ruby/lib/avro/data_file.rb Wed Jul 8 14:31:50 2015
@@ -93,23 +93,22 @@ module Avro
attr_reader :writer, :encoder, :datum_writer, :buffer_writer,
:buffer_encoder, :sync_marker, :meta, :codec
attr_accessor :block_count
- def initialize(writer, datum_writer, writers_schema=nil, codec=nil)
+ def initialize(writer, datum_writer, writers_schema=nil, codec=nil,
meta={})
# If writers_schema is not present, presume we're appending
@writer = writer
@encoder = IO::BinaryEncoder.new(@writer)
@datum_writer = datum_writer
+ @meta = meta
@buffer_writer = StringIO.new('', 'w')
@buffer_writer.set_encoding('BINARY') if
@buffer_writer.respond_to?(:set_encoding)
@buffer_encoder = IO::BinaryEncoder.new(@buffer_writer)
@block_count = 0
- @meta = {}
-
if writers_schema
@sync_marker = Writer.generate_sync_marker
@codec = DataFile.get_codec(codec)
- meta['avro.codec'] = @codec.codec_name.to_s
- meta['avro.schema'] = writers_schema.to_s
+ @meta['avro.codec'] = @codec.codec_name.to_s
+ @meta['avro.schema'] = writers_schema.to_s
datum_writer.writers_schema = writers_schema
write_header
else
@@ -119,12 +118,12 @@ module Avro
# FIXME(jmhodges): collect arbitrary metadata
# collect metadata
@sync_marker = dfr.sync_marker
- meta['avro.codec'] = dfr.meta['avro.codec']
+ @meta['avro.codec'] = dfr.meta['avro.codec']
@codec = DataFile.get_codec(meta['avro.codec'])
# get schema used to write existing file
schema_from_file = dfr.meta['avro.schema']
- meta['avro.schema'] = schema_from_file
+ @meta['avro.schema'] = schema_from_file
datum_writer.writers_schema = Schema.parse(schema_from_file)
# seek to the end of the file and prepare for writing
Modified: avro/trunk/lang/ruby/test/test_datafile.rb
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/ruby/test/test_datafile.rb?rev=1689889&r1=1689888&r2=1689889&view=diff
==============================================================================
--- avro/trunk/lang/ruby/test/test_datafile.rb (original)
+++ avro/trunk/lang/ruby/test/test_datafile.rb Wed Jul 8 14:31:50 2015
@@ -185,4 +185,17 @@ JSON
end
assert_equal records, ['a' * 10_000, 'b' * 10_000]
end
+
+ def test_custom_meta
+ meta = { 'x.greeting' => 'yo' }
+
+ schema = Avro::Schema.parse('"string"')
+ writer = Avro::IO::DatumWriter.new(schema)
+ file = Avro::DataFile::Writer.new(File.open('data.avr', 'wb'), writer,
schema, nil, meta)
+ file.close
+
+ Avro::DataFile.open('data.avr') do |reader|
+ assert_equal 'yo', reader.meta['x.greeting']
+ end
+ end
end
