Author: tomwhite
Date: Thu Jun 25 15:38:03 2015
New Revision: 1687559
URL: http://svn.apache.org/r1687559
Log:
AVRO-1685. Allow specifying sync in DataFileWriter.create. Contributed by
Sehrope Sarkuni.
Added:
avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestDataFileCustomSync.java
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java
Modified: avro/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1687559&r1=1687558&r2=1687559&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Thu Jun 25 15:38:03 2015
@@ -44,6 +44,9 @@ Trunk (not yet released)
AVRO-1497. Java: Add support for logical types. (blue)
+ AVRO-1685. Java: Allow specifying sync in DataFileWriter.create
+ (Sehrope Sarkuni via tomwhite)
+
OPTIMIZATIONS
IMPROVEMENTS
Modified:
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java?rev=1687559&r1=1687558&r2=1687559&view=diff
==============================================================================
---
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java
(original)
+++
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java
Thu Jun 25 15:38:03 2015
@@ -124,19 +124,31 @@ public class DataFileWriter<D> implement
return this;
}
- /** Open a new file for data matching a schema. */
+ /** Open a new file for data matching a schema with a random sync. */
public DataFileWriter<D> create(Schema schema, File file) throws IOException
{
- return create(schema, new SyncableFileOutputStream(file));
+ return create(schema, new SyncableFileOutputStream(file), null);
}
- /** Open a new file for data matching a schema. */
+ /** Open a new file for data matching a schema with a random sync. */
public DataFileWriter<D> create(Schema schema, OutputStream outs)
throws IOException {
+ return create(schema, outs, null);
+ }
+
+ /** Open a new file for data matching a schema with an explicit sync. */
+ public DataFileWriter<D> create(Schema schema, OutputStream outs, byte[]
sync)
+ throws IOException {
assertNotOpen();
this.schema = schema;
setMetaInternal(DataFileConstants.SCHEMA, schema.toString());
- this.sync = generateSync();
+ if (sync == null ) {
+ this.sync = generateSync();
+ } else if (sync.length == 16) {
+ this.sync = sync;
+ } else {
+ throw new IOException("sync must be exactly 16 bytes");
+ }
init(outs);
@@ -150,7 +162,7 @@ public class DataFileWriter<D> implement
vout.writeBytes(entry.getValue());
}
vout.writeMapEnd();
- vout.writeFixed(sync); // write initial sync
+ vout.writeFixed(this.sync); // write initial sync
vout.flush(); //vout may be buffered, flush before writing to out
return this;
}
Added:
avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestDataFileCustomSync.java
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestDataFileCustomSync.java?rev=1687559&view=auto
==============================================================================
---
avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestDataFileCustomSync.java
(added)
+++
avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestDataFileCustomSync.java
Thu Jun 25 15:38:03 2015
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.avro;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.UUID;
+import org.apache.avro.Schema.Type;
+import org.apache.avro.file.DataFileConstants;
+import org.apache.avro.file.DataFileReader;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.util.Utf8;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class TestDataFileCustomSync {
+ private byte[] createDataFile(byte[] sync) throws IOException {
+ Schema schema = Schema.create(Type.STRING);
+ DataFileWriter<Utf8> w = new DataFileWriter<Utf8>(new
GenericDatumWriter<Utf8>(schema));
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ w.create(schema, baos, sync);
+ w.append(new Utf8("apple"));
+ w.append(new Utf8("banana"));
+ w.sync();
+ w.append(new Utf8("celery"));
+ w.append(new Utf8("date"));
+ w.sync();
+ w.append(new Utf8("endive"));
+ w.append(new Utf8("fig"));
+ w.close();
+ return baos.toByteArray();
+ }
+
+ private static byte[] generateSync() {
+ try {
+ MessageDigest digester = MessageDigest.getInstance("MD5");
+ long time = System.currentTimeMillis();
+ digester.update((UUID.randomUUID()+"@"+time).getBytes());
+ return digester.digest();
+ } catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Test(expected = IOException.class)
+ public void testInvalidSync() throws IOException {
+ // Invalid size (must be 16):
+ byte[] sync = new byte[8];
+ createDataFile(sync);
+ }
+
+ @Test
+ public void testRandomSync() throws IOException {
+ byte[] sync = generateSync();
+ byte[] randSyncFile = createDataFile(null);
+ byte[] customSyncFile = createDataFile(sync);
+ assertFalse(Arrays.equals(randSyncFile, customSyncFile));
+ }
+
+ @Test
+ public void testCustomSync() throws IOException {
+ byte[] sync = generateSync();
+ byte[] customSyncFile = createDataFile(sync);
+ byte[] sameCustomSyncFile = createDataFile(sync);
+ assertTrue(Arrays.equals(customSyncFile, sameCustomSyncFile));
+ }
+}
