[
https://issues.apache.org/jira/browse/HIVE-21218?focusedWorklogId=398132&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-398132
]
ASF GitHub Bot logged work on HIVE-21218:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 05/Mar/20 04:59
Start Date: 05/Mar/20 04:59
Worklog Time Spent: 10m
Work Description: cricket007 commented on pull request #933: HIVE-21218:
Adding support for Confluent Kafka Avro message format
URL: https://github.com/apache/hive/pull/933#discussion_r388078182
##########
File path:
kafka-handler/src/test/org/apache/hadoop/hive/kafka/AvroBytesConverterTest.java
##########
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.kafka;
+
+import com.google.common.collect.Maps;
+import io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient;
+import io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig;
+import io.confluent.kafka.serializers.KafkaAvroSerializer;
+import org.apache.avro.Schema;
+import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Test class for Hive Kafka Avro SerDe with variable bytes skipped.
+ */
+public class AvroBytesConverterTest {
+ private static SimpleRecord simpleRecord =
SimpleRecord.newBuilder().setId("123").setName("test").build();
+ private static byte[] simpleRecordConfluentBytes;
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ /**
+ * Use the KafkaAvroSerializer from Confluent to serialize the simpleRecord.
+ */
+ @BeforeClass
+ public static void setUp() {
+ Map<String, String> config = Maps.newHashMap();
+ config.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG,
"http://localhost:8081";);
+ KafkaAvroSerializer avroSerializer = new KafkaAvroSerializer(new
MockSchemaRegistryClient());
+ avroSerializer.configure(config, false);
+ simpleRecordConfluentBytes = avroSerializer.serialize("temp",
simpleRecord);
+ }
+
+ private void runConversionTest(KafkaSerDe.AvroBytesConverter conv, byte[]
serializedSimpleRecord) {
+ AvroGenericRecordWritable simpleRecordWritable =
conv.getWritable(serializedSimpleRecord);
+
+ Assert.assertNotNull(simpleRecordWritable);
+ Assert.assertEquals(SimpleRecord.class,
simpleRecordWritable.getRecord().getClass());
+
+ SimpleRecord simpleRecordDeserialized = (SimpleRecord)
simpleRecordWritable.getRecord();
+
+ Assert.assertNotNull(simpleRecordDeserialized);
+ Assert.assertEquals(simpleRecord, simpleRecordDeserialized);
+ }
+
+ /**
+ * Tests the default case of no skipped bytes per record works properly.
+ */
+ @Test
+ public void convertWithAvroBytesConverter() {
+ // Since the serialized version was created by Confluent, lets remove the
first five bytes to get the actual message.
+ byte[] simpleRecordWithNoOffset =
Arrays.copyOfRange(simpleRecordConfluentBytes, 5,
simpleRecordConfluentBytes.length);
+
+ Schema schema = SimpleRecord.getClassSchema();
+ KafkaSerDe.AvroBytesConverter conv = new
KafkaSerDe.AvroBytesConverter(schema);
+ runConversionTest(conv, simpleRecordWithNoOffset);
+ }
+
+ /**
+ * Tests that the skip converter skips 5 bytes properly, which matches what
Confluent needs.
+ */
+ @Test
+ public void convertWithConfluentAvroBytesConverter() {
+ Schema schema = SimpleRecord.getClassSchema();
+ KafkaSerDe.AvroSkipBytesConverter conv = new
KafkaSerDe.AvroSkipBytesConverter(schema, 5);
+ runConversionTest(conv, simpleRecordConfluentBytes);
+ }
+
+ /**
+ * Tests that the skip converter skips a custom number of bytes properly.
+ */
+ @Test
+ public void convertWithCustomAvroSkipBytesConverter() {
+ int offset = 2;
+ // Remove all but two bytes of the five byte offset which Confluent adds,
+ // to simulate a message with only 2 bytes in front of each message.
+ byte[] simpleRecordAsOffsetBytes =
Arrays.copyOfRange(simpleRecordConfluentBytes, 5 - offset,
simpleRecordConfluentBytes.length);
+
+ Schema schema = SimpleRecord.getClassSchema();
+ KafkaSerDe.AvroSkipBytesConverter conv = new
KafkaSerDe.AvroSkipBytesConverter(schema, offset);
+ runConversionTest(conv, simpleRecordAsOffsetBytes);
+ }
+
+ /**
+ * Test that when we skip more bytes than are in the message, we throw an
exception properly.
+ */
+ @Test
+ public void skipBytesLargerThanMessageSizeConverter() {
+ // The simple record we are serializing is two strings, that combine to be
7 characters or 14 bytes.
+ // Adding in the 5 byte offset, we get 19 bytes. To make sure we go bigger
than that, we are setting
+ // the offset to ten times that value.
+ int offset = 190;
+
+ Schema schema = SimpleRecord.getClassSchema();
+ KafkaSerDe.AvroSkipBytesConverter conv = new
KafkaSerDe.AvroSkipBytesConverter(schema, offset);
+
+ exception.expect(RuntimeException.class);
+ exception.expectMessage("org.apache.hadoop.hive.serde2.SerDeException: " +
+ "Skip bytes value is larger than the message length.");
+ runConversionTest(conv, simpleRecordConfluentBytes);
+ }
+
+ /**
+ * Test that we properly parse the converter type, no matter the casing.
+ */
+ @Test
+ public void bytesConverterTypeParseTest() {
+ Map<String, KafkaSerDe.BytesConverterType> testCases = new HashMap<String,
KafkaSerDe.BytesConverterType>() {{
Review comment:
Nit: double brace.
This could be implemented as a parameterized test
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 398132)
Time Spent: 12h (was: 11h 50m)
> KafkaSerDe doesn't support topics created via Confluent Avro serializer
> -----------------------------------------------------------------------
>
> Key: HIVE-21218
> URL: https://issues.apache.org/jira/browse/HIVE-21218
> Project: Hive
> Issue Type: Bug
> Components: kafka integration, Serializers/Deserializers
> Affects Versions: 3.1.1
> Reporter: Milan Baran
> Assignee: David McGinnis
> Priority: Major
> Labels: pull-request-available
> Attachments: HIVE-21218.2.patch, HIVE-21218.3.patch,
> HIVE-21218.4.patch, HIVE-21218.5.patch, HIVE-21218.patch
>
> Time Spent: 12h
> Remaining Estimate: 0h
>
> According to [Google
> groups|https://groups.google.com/forum/#!topic/confluent-platform/JYhlXN0u9_A]
> the Confluent avro serialzier uses propertiary format for kafka value -
> <magic_byte 0x00><4 bytes of schema ID><regular avro bytes for object that
> conforms to schema>.
> This format does not cause any problem for Confluent kafka deserializer which
> respect the format however for hive kafka handler its bit a problem to
> correctly deserialize kafka value, because Hive uses custom deserializer from
> bytes to objects and ignores kafka consumer ser/deser classes provided via
> table property.
> It would be nice to support Confluent format with magic byte.
> Also it would be great to support Schema registry as well.
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
