[incubator-pinot] 01/01: Fix extract method in AvroRecordExtractor
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a commit to branch fix-AvroRecordExtractor
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit bfb59545cdd13e94f34c882be579afd917fbdd80
Author: Jack Li(Analytics Engineering)
AuthorDate: Fri Sep 11 10:33:09 2020 -0700
Fix extract method in AvroRecordExtractor
---
.../plugin/inputformat/avro/AvroRecordExtractor.java| 17 -
.../main/java/org/apache/pinot/spi/utils/JsonUtils.java | 13 -
2 files changed, 8 insertions(+), 22 deletions(-)
diff --git
a/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractor.java
b/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractor.java
index 339ab67..ede8586 100644
---
a/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractor.java
+++
b/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractor.java
@@ -18,14 +18,14 @@
*/
package org.apache.pinot.plugin.inputformat.avro;
-import java.util.Map;
+import java.util.List;
import java.util.Set;
import javax.annotation.Nullable;
+import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.data.readers.RecordExtractor;
import org.apache.pinot.spi.data.readers.RecordExtractorConfig;
-import org.apache.pinot.spi.utils.JsonUtils;
/**
@@ -46,14 +46,13 @@ public class AvroRecordExtractor implements
RecordExtractor {
@Override
public GenericRow extract(GenericRecord from, GenericRow to) {
if (_extractAll) {
- Map jsonMap = JsonUtils.genericRecordToJson(from);
- jsonMap.forEach((fieldName, value) -> to.putValue(fieldName,
AvroUtils.convert(value)));
+ List fields = from.getSchema().getFields();
+ fields.forEach(field -> {
+String fieldName = field.name();
+to.putValue(fieldName, AvroUtils.convert(from.get(fieldName)));
+ });
} else {
- for (String fieldName : _fields) {
-Object value = from.get(fieldName);
-Object convertedValue = AvroUtils.convert(value);
-to.putValue(fieldName, convertedValue);
- }
+ _fields.forEach(fieldName -> to.putValue(fieldName,
AvroUtils.convert(from.get(fieldName;
}
return to;
}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/JsonUtils.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/JsonUtils.java
index f5bf9d3..419b5ce 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/JsonUtils.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/JsonUtils.java
@@ -193,17 +193,4 @@ public class JsonUtils {
throw new IllegalArgumentException(String.format("Unsupported data
type %s", dataType));
}
}
-
- /**
- * Converts from a GenericRecord to a json map
- */
- public static Map genericRecordToJson(GenericRecord
genericRecord) {
-try {
- String jsonString = genericRecord.toString();
- return DEFAULT_MAPPER.readValue(jsonString, new
TypeReference>() {
- });
-} catch (IOException e) {
- throw new IllegalStateException("Caught exception when converting
generic record " + genericRecord + " to JSON");
-}
- }
}
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] 01/01: Fix extract method in AvroRecordExtractor
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a commit to branch fix-AvroRecordExtractor
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit f4add59f9518b37c638d90457a92e18f81108545
Author: Jack Li(Analytics Engineering)
AuthorDate: Fri Sep 11 10:33:09 2020 -0700
Fix extract method in AvroRecordExtractor
---
.../plugin/inputformat/avro/AvroRecordExtractor.java | 15 ---
.../main/java/org/apache/pinot/spi/utils/JsonUtils.java | 13 -
2 files changed, 8 insertions(+), 20 deletions(-)
diff --git
a/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractor.java
b/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractor.java
index 339ab67..cec3e5f 100644
---
a/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractor.java
+++
b/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractor.java
@@ -18,9 +18,11 @@
*/
package org.apache.pinot.plugin.inputformat.avro;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.annotation.Nullable;
+import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.data.readers.RecordExtractor;
@@ -46,14 +48,13 @@ public class AvroRecordExtractor implements
RecordExtractor {
@Override
public GenericRow extract(GenericRecord from, GenericRow to) {
if (_extractAll) {
- Map jsonMap = JsonUtils.genericRecordToJson(from);
- jsonMap.forEach((fieldName, value) -> to.putValue(fieldName,
AvroUtils.convert(value)));
+ List fields = from.getSchema().getFields();
+ fields.forEach(field -> {
+String fieldName = field.name();
+to.putValue(fieldName, AvroUtils.convert(from.get(fieldName)));
+ });
} else {
- for (String fieldName : _fields) {
-Object value = from.get(fieldName);
-Object convertedValue = AvroUtils.convert(value);
-to.putValue(fieldName, convertedValue);
- }
+ _fields.forEach(fieldName -> to.putValue(fieldName,
AvroUtils.convert(from.get(fieldName;
}
return to;
}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/JsonUtils.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/JsonUtils.java
index f5bf9d3..419b5ce 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/JsonUtils.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/JsonUtils.java
@@ -193,17 +193,4 @@ public class JsonUtils {
throw new IllegalArgumentException(String.format("Unsupported data
type %s", dataType));
}
}
-
- /**
- * Converts from a GenericRecord to a json map
- */
- public static Map genericRecordToJson(GenericRecord
genericRecord) {
-try {
- String jsonString = genericRecord.toString();
- return DEFAULT_MAPPER.readValue(jsonString, new
TypeReference>() {
- });
-} catch (IOException e) {
- throw new IllegalStateException("Caught exception when converting
generic record " + genericRecord + " to JSON");
-}
- }
}
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] 01/01: Fix extract method in AvroRecordExtractor
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a commit to branch fix-AvroRecordExtractor
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit fd1cb00a4f8d0f475e114add27b81d11ef651372
Author: Jack Li(Analytics Engineering)
AuthorDate: Fri Sep 11 10:33:09 2020 -0700
Fix extract method in AvroRecordExtractor
---
.../apache/pinot/plugin/inputformat/avro/AvroRecordExtractor.java | 8 ++--
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git
a/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractor.java
b/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractor.java
index 339ab67..05cab34 100644
---
a/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractor.java
+++
b/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractor.java
@@ -47,13 +47,9 @@ public class AvroRecordExtractor implements
RecordExtractor {
public GenericRow extract(GenericRecord from, GenericRow to) {
if (_extractAll) {
Map jsonMap = JsonUtils.genericRecordToJson(from);
- jsonMap.forEach((fieldName, value) -> to.putValue(fieldName,
AvroUtils.convert(value)));
+ jsonMap.forEach((fieldName, value) -> to.putValue(fieldName,
AvroUtils.convert(from.get(fieldName;
} else {
- for (String fieldName : _fields) {
-Object value = from.get(fieldName);
-Object convertedValue = AvroUtils.convert(value);
-to.putValue(fieldName, convertedValue);
- }
+ _fields.forEach(fieldName -> to.putValue(fieldName,
AvroUtils.convert(from.get(fieldName;
}
return to;
}
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
