This is an automated email from the ASF dual-hosted git repository.
dkulp pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/master by this push:
new b09726d [AVRO-1738] add java tool for outputting schema fingerprints
Patch from Sean Busbey applied (with modifications for current code formatting
and other updates)
b09726d is described below
commit b09726d066111d3e884872960f511f97a0215d34
Author: Daniel Kulp <[email protected]>
AuthorDate: Fri Apr 5 10:54:22 2019 -0400
[AVRO-1738] add java tool for outputting schema fingerprints
Patch from Sean Busbey applied (with modifications for current code
formatting and other updates)
---
NOTICE.txt | 2 +-
.../src/main/java/org/apache/avro/tool/Main.java | 2 +-
.../apache/avro/tool/SchemaFingerprintTool.java | 87 ++++++++++++++++++++++
.../src/main/java/org/apache/avro/tool/Util.java | 33 ++++++++
.../test/java/org/apache/avro/tool/TestMain.java | 1 +
5 files changed, 123 insertions(+), 2 deletions(-)
diff --git a/NOTICE.txt b/NOTICE.txt
index 2a93cb3..57572f0 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -1,5 +1,5 @@
Apache Avro
-Copyright 2010-2015 The Apache Software Foundation
+Copyright 2010-2019 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
diff --git a/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java
b/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java
index 45ce360..ff0b1b2 100644
--- a/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java
+++ b/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java
@@ -42,7 +42,7 @@ public class Main {
new DataFileRepairTool(), new IdlTool(), new IdlToSchemataTool(), new
RecodecTool(), new ConcatTool(),
new RpcReceiveTool(), new RpcSendTool(), new RpcProtocolTool(), new
FromTextTool(), new ToTextTool(),
new ToTrevniTool(), new TetherTool(), new TrevniCreateRandomTool(),
new TrevniMetadataTool(),
- new TrevniToJsonTool(), new SchemaNormalizationTool() }) {
+ new TrevniToJsonTool(), new SchemaNormalizationTool(), new
SchemaFingerprintTool() }) {
Tool prev = tools.put(tool.getName(), tool);
if (prev != null) {
throw new AssertionError("Two tools with identical names: " + tool +
", " + prev);
diff --git
a/lang/java/tools/src/main/java/org/apache/avro/tool/SchemaFingerprintTool.java
b/lang/java/tools/src/main/java/org/apache/avro/tool/SchemaFingerprintTool.java
new file mode 100644
index 0000000..489e1fb
--- /dev/null
+++
b/lang/java/tools/src/main/java/org/apache/avro/tool/SchemaFingerprintTool.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.avro.tool;
+
+import java.io.InputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+
+import java.util.List;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaNormalization;
+
+/**
+ * Utility to generate fingerprint(s) from a schema.
+ */
+public class SchemaFingerprintTool implements Tool {
+
+ @Override
+ public int run(InputStream in, PrintStream out, PrintStream err,
List<String> args) throws Exception {
+ final OptionParser optParser = new OptionParser();
+ final OptionSpec<String> fingerprintOpt = optParser
+ .accepts("fingerprint",
+ "Fingerprint algorithm to use. Recommended Avro practice dictiates
"
+ + "that \"CRC-64-AVRO\" is used for 64-bit fingerprints,
\"MD5\" is "
+ + "used for 128-bit fingerprints, and \"SHA-256\" is used for
256-bit " + "fingerprints.")
+ .withRequiredArg().ofType(String.class).defaultsTo("CRC-64-AVRO");
+
+ final OptionSet opts = optParser.parse(args.toArray(new String[0]));
+ final Schema.Parser parser = new Schema.Parser();
+ final List<String> nargs = (List<String>) opts.nonOptionArguments();
+ if (nargs.size() < 1) {
+ printHelp(out, optParser);
+ return 0;
+ }
+
+ for (final String fileOrStdin : (List<String>) opts.nonOptionArguments()) {
+ final InputStream input = Util.fileOrStdin(fileOrStdin, in);
+ try {
+ final Schema schema = parser.parse(input);
+ final byte[] fingerprint =
SchemaNormalization.parsingFingerprint(opts.valueOf(fingerprintOpt), schema);
+ out.format("%s %s%n", Util.encodeHex(fingerprint), fileOrStdin);
+ } finally {
+ Util.close(input);
+ }
+ }
+
+ return 0;
+ }
+
+ @Override
+ public String getName() {
+ return "fingerprint";
+ }
+
+ @Override
+ public String getShortDescription() {
+ return "Returns the fingerprint for the schemas.";
+ }
+
+ private void printHelp(PrintStream out, OptionParser optParser) throws
IOException {
+ out.println("fingerprint [--fingerprint <fingerprint>] input-file
[inputfile [inputfile...]]");
+ out.println();
+ out.println("generates fingerprints based on Avro specification.");
+ optParser.printHelpOn(out);
+ out.println("A dash ('-') can be given to read a schema from stdin");
+ }
+}
diff --git a/lang/java/tools/src/main/java/org/apache/avro/tool/Util.java
b/lang/java/tools/src/main/java/org/apache/avro/tool/Util.java
index f92aa67..f795352 100644
--- a/lang/java/tools/src/main/java/org/apache/avro/tool/Util.java
+++ b/lang/java/tools/src/main/java/org/apache/avro/tool/Util.java
@@ -14,6 +14,9 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
+ *
+ * The section demarcated by 'copied from Apache commons-codec' is
+ * from Apache Commons Codec v1.9.
*/
package org.apache.avro.tool;
@@ -271,4 +274,34 @@ class Util {
return CodecFactory.fromString(codec.value(opts));
}
}
+
+ // Below copied from Apache commons-codec version 1.9
+ // org.apache.commons.codec.binary.Hex, see NOTICE.
+ /**
+ * Used to build output as Hex
+ */
+ private static final char[] DIGITS_LOWER = { '0', '1', '2', '3', '4', '5',
'6', '7', '8', '9', 'a', 'b', 'c', 'd',
+ 'e', 'f' };
+
+ /**
+ * Converts an array of bytes into an array of characters representing the
+ * hexadecimal values of each byte in order. The returned array will be
double
+ * the length of the passed array, as it takes two characters to represent
any
+ * given byte.
+ *
+ * @param data a byte[] to convert to Hex characters
+ * @param toDigits the output alphabet
+ * @return A char[] containing hexadecimal characters
+ */
+ static String encodeHex(final byte[] data) {
+ final int l = data.length;
+ final char[] out = new char[l << 1];
+ // two characters form the hex value.
+ for (int i = 0, j = 0; i < l; i++) {
+ out[j++] = DIGITS_LOWER[(0xF0 & data[i]) >>> 4];
+ out[j++] = DIGITS_LOWER[0x0F & data[i]];
+ }
+ return new String(out);
+ }
+ // end copied from Apache commons-codec
}
diff --git a/lang/java/tools/src/test/java/org/apache/avro/tool/TestMain.java
b/lang/java/tools/src/test/java/org/apache/avro/tool/TestMain.java
index e1d25c7..03d0278 100644
--- a/lang/java/tools/src/test/java/org/apache/avro/tool/TestMain.java
+++ b/lang/java/tools/src/test/java/org/apache/avro/tool/TestMain.java
@@ -27,6 +27,7 @@ public class TestMain {
public void testToolDescriptionLength() {
Main m = new Main();
for (Tool t : m.tools.values()) {
+ // System.out.println(t.getName() + ": " +
t.getShortDescription().length());
if (m.maxLen + 2 + t.getShortDescription().length() > 80) {
fail("Tool description too long: " + t.getName());
}
