Repository: nifi Updated Branches: refs/heads/master 2938a5353 -> d54427488
NIFI-2531: Fixed JdbcCommon handling of BigInteger objects for Avro This closes #823. Signed-off-by: Bryan Bende <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/d5442748 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/d5442748 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/d5442748 Branch: refs/heads/master Commit: d544274881f738d387ffad51b85c344a8f097238 Parents: 2938a53 Author: Matt Burgess <[email protected]> Authored: Wed Aug 10 14:54:05 2016 -0400 Committer: Bryan Bende <[email protected]> Committed: Wed Aug 10 15:19:34 2016 -0400 ---------------------------------------------------------------------- .../processors/standard/util/JdbcCommon.java | 38 ++++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/d5442748/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java index 3b74c19..5ece149 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java @@ -68,6 +68,8 @@ import org.apache.commons.lang3.StringUtils; */ public class JdbcCommon { + private static final int MAX_DIGITS_IN_BIGINT = 19; + public static long convertToAvroStream(final ResultSet rs, final OutputStream outStream) throws SQLException, IOException { return convertToAvroStream(rs, outStream, null, null); } @@ -114,10 +116,32 @@ public class JdbcCommon { // org.apache.avro.AvroRuntimeException: Unknown datum type java.lang.Byte rec.put(i - 1, ((Byte) value).intValue()); - } else if (value instanceof BigDecimal || value instanceof BigInteger) { - // Avro can't handle BigDecimal and BigInteger as numbers - it will throw an AvroRuntimeException such as: "Unknown datum type: java.math.BigDecimal: 38" + } else if (value instanceof BigDecimal) { + // Avro can't handle BigDecimal as a number - it will throw an AvroRuntimeException such as: "Unknown datum type: java.math.BigDecimal: 38" rec.put(i - 1, value.toString()); + } else if (value instanceof BigInteger) { + // Check the precision of the BIGINT. Some databases allow arbitrary precision (> 19), but Avro won't handle that. + // It the SQL type is BIGINT and the precision is between 0 and 19 (inclusive); if so, the BigInteger is likely a + // long (and the schema says it will be), so try to get its value as a long. + // Otherwise, Avro can't handle BigInteger as a number - it will throw an AvroRuntimeException + // such as: "Unknown datum type: java.math.BigInteger: 38". In this case the schema is expecting a string. + int precision = meta.getPrecision(i); + if (javaSqlType == BIGINT) { + if (precision < 0 || precision > MAX_DIGITS_IN_BIGINT) { + rec.put(i - 1, value.toString()); + } else { + try { + rec.put(i - 1, ((BigInteger) value).longValueExact()); + } catch (ArithmeticException ae) { + // Since the value won't fit in a long, convert it to a string + rec.put(i - 1, value.toString()); + } + } + } else { + rec.put(i - 1, value.toString()); + } + } else if (value instanceof Number || value instanceof Boolean) { rec.put(i - 1, value); @@ -196,7 +220,15 @@ public class JdbcCommon { break; case BIGINT: - builder.name(meta.getColumnName(i)).type().unionOf().nullBuilder().endNull().and().longType().endUnion().noDefault(); + // Check the precision of the BIGINT. Some databases allow arbitrary precision (> 19), but Avro won't handle that. + // If the precision > 19 (or is negative), use a string for the type, otherwise use a long. The object(s) will be converted + // to strings as necessary + int precision = meta.getPrecision(i); + if (precision < 0 || precision > MAX_DIGITS_IN_BIGINT) { + builder.name(meta.getColumnName(i)).type().unionOf().nullBuilder().endNull().and().stringType().endUnion().noDefault(); + } else { + builder.name(meta.getColumnName(i)).type().unionOf().nullBuilder().endNull().and().longType().endUnion().noDefault(); + } break; // java.sql.RowId is interface, is seems to be database
