bamaer commented on code in PR #9561:
URL: https://github.com/apache/hop/pull/9561#discussion_r4626766738


##########
core/src/main/java/org/apache/hop/core/row/value/ValueMetaAvroRecord.java:
##########
@@ -146,33 +164,185 @@ public static String 
convertGenericRecordToString(GenericRecord genericRecord)
       throws HopValueException {
     try {
       Schema schema = genericRecord.getSchema();
-      String schemaJson = schema.toString();
-      String dataJson = genericRecord.toString();
+      ByteArrayOutputStream data = new ByteArrayOutputStream();
+      JsonEncoder encoder = EncoderFactory.get().jsonEncoder(schema, data);
+      new GenericDatumWriter<GenericRecord>(schema).write(genericRecord, 
encoder);
+      encoder.flush();
 
-      return "{ \"schema\" : " + schemaJson + ", \"data\" : " + dataJson + " 
}";
+      return "{ \"schema\" : "
+          + schema
+          + ", \"data\" : "
+          + data.toString(StandardCharsets.UTF_8)
+          + " }";
     } catch (Exception e) {
       throw new HopValueException(
           "Unable to convert an Avro record to a JSON String using the 
provided schema", e);
     }
   }
 
+  /** Read a record from the schema and data JSON {@link 
#convertGenericRecordToString} writes. */
   public static GenericRecord convertStringToGenericRecord(String jsonString)
       throws HopValueException {
+    return convertStringToGenericRecord(jsonString, null);
+  }
+
+  /**
+   * Read a record from text: either the schema and data JSON {@link 
#convertGenericRecordToString}
+   * writes, or only the data (as a record's toString() writes it) when a 
schema is given. The data
+   * can be in Avro's JSON encoding or plain JSON values.
+   *
+   * @param jsonString The text to read
+   * @param schema The schema of data without one, or null
+   * @return The record
+   */
+  public static GenericRecord convertStringToGenericRecord(String jsonString, 
Schema schema)
+      throws HopValueException {
+    JsonNode json;
     try {
-      // Convert schema AND data to JSON...
-      //
-      JSONObject json = (JSONObject) new JSONParser().parse(jsonString);
-      JSONObject schemaObject = (JSONObject) json.get(CONST_SCHEMA);
-      JSONObject dataObject = (JSONObject) json.get("data");
+      json = JSON_MAPPER.readTree(jsonString);
+    } catch (Exception e) {
+      throw new HopValueException("Unable to read an Avro record from text: 
not valid JSON", e);
+    }
+    if (json != null && json.has(CONST_SCHEMA) && json.has(CONST_DATA)) {
+      Schema textSchema;
+      try {
+        textSchema = new 
Schema.Parser().parse(json.get(CONST_SCHEMA).toString());
+      } catch (Exception e) {
+        throw new HopValueException("Unable to read the Avro schema in the 
text", e);
+      }
+      return (GenericRecord) jsonToAvro(json.get(CONST_DATA), textSchema);
+    }
+    if (schema == null) {
+      throw new HopValueException(
+          "Unable to read an Avro record from text without a schema: the text 
holds no"
+              + " \"schema\" and the field has none");
+    }
+    return (GenericRecord) jsonToAvro(json, schema);
+  }
 
-      Schema schema = new Schema.Parser().parse(schemaObject.toJSONString());
+  /**
+   * One JSON value as the Avro value of the schema. A union value can be 
wrapped in its type, as
+   * Avro's JSON encoding writes it, or plain, as a record's toString() writes 
it: then the first
+   * type it fits is used.
+   */
+  private static Object jsonToAvro(JsonNode node, Schema schema) throws 
HopValueException {
+    boolean isNull = node == null || node.isNull();
+    switch (schema.getType()) {
+      case NULL:
+        if (isNull) {
+          return null;
+        }
+        break;
+      case BOOLEAN:
+        if (!isNull && node.isBoolean()) {
+          return node.booleanValue();
+        }
+        break;
+      case INT:
+        if (!isNull && node.isIntegralNumber() && node.canConvertToInt()) {
+          return node.intValue();
+        }
+        break;
+      case LONG:
+        if (!isNull && node.isIntegralNumber() && node.canConvertToLong()) {
+          return node.longValue();
+        }
+        break;
+      case FLOAT:
+        if (!isNull && node.isNumber()) {
+          return node.floatValue();
+        }
+        break;
+      case DOUBLE:
+        if (!isNull && node.isNumber()) {
+          return node.doubleValue();
+        }
+        break;
+      case STRING:
+        if (!isNull && node.isTextual()) {
+          return new Utf8(node.textValue());
+        }
+        break;
+      case BYTES:
+        if (!isNull && node.isTextual()) {
+          return 
ByteBuffer.wrap(node.textValue().getBytes(StandardCharsets.ISO_8859_1));
+        }
+        break;
+      case FIXED:
+        if (!isNull && node.isTextual()) {
+          return new GenericData.Fixed(
+              schema, node.textValue().getBytes(StandardCharsets.ISO_8859_1));
+        }
+        break;
+      case ENUM:
+        if (!isNull && node.isTextual() && 
schema.hasEnumSymbol(node.textValue())) {
+          return new GenericData.EnumSymbol(schema, node.textValue());
+        }
+        break;
+      case ARRAY:
+        if (!isNull && node.isArray()) {
+          GenericData.Array<Object> array = new 
GenericData.Array<>(node.size(), schema);
+          for (JsonNode element : node) {
+            array.add(jsonToAvro(element, schema.getElementType()));
+          }
+          return array;
+        }
+        break;
+      case MAP:
+        if (!isNull && node.isObject()) {
+          Map<Utf8, Object> map = new LinkedHashMap<>();
+          for (Map.Entry<String, JsonNode> entry : node.properties()) {
+            map.put(new Utf8(entry.getKey()), jsonToAvro(entry.getValue(), 
schema.getValueType()));
+          }
+          return map;
+        }
+        break;
+      case RECORD:
+        if (!isNull && node.isObject()) {
+          GenericData.Record avroRecord = new GenericData.Record(schema);
+          for (Schema.Field field : schema.getFields()) {
+            JsonNode value = node.get(field.name());
+            if (value == null && field.hasDefaultValue()) {
+              avroRecord.put(field.pos(), 
GenericData.get().getDefaultValue(field));
+            } else {
+              avroRecord.put(field.pos(), jsonToAvro(value, field.schema()));
+            }
+          }
+          return avroRecord;
+        }
+        break;
+      case UNION:
+        return unionToAvro(node, schema);
+      default:
+        break;
+    }
+    throw new HopValueException(
+        "Unable to read " + (isNull ? "null" : node.toString()) + " as Avro 
type " + schema);
+  }
 
-      JsonDecoder jsonDecoder = DecoderFactory.get().jsonDecoder(schema, 
dataObject.toJSONString());
-      GenericDatumReader<GenericRecord> genericDatumReader = new 
GenericDatumReader<>(schema);
-      return genericDatumReader.read(null, jsonDecoder);
-    } catch (Exception e) {
-      throw new HopValueException("Unable to convert a String to an Avro 
record", e);
+  private static Object unionToAvro(JsonNode node, Schema union) throws 
HopValueException {
+    // Avro's JSON encoding: {"type name": value}
+    if (node != null && node.isObject() && node.size() == 1) {
+      String typeName = node.fieldNames().next();
+      for (Schema type : union.getTypes()) {
+        if (typeName.equals(type.getFullName()) || 
typeName.equals(type.getName())) {
+          return jsonToAvro(node.get(typeName), type);
+        }
+      }
     }
+    // Plain JSON: the first type the value fits

Review Comment:
   Fixed in 58469c35e7, thanks @hansva. The union-of-records value now reads 
back as B; `aUnionOfRecordsReadsBackAsTheRightRecord` covers it.



##########
core/src/main/java/org/apache/hop/core/row/value/ValueMetaAvroRecord.java:
##########
@@ -146,33 +164,185 @@ public static String 
convertGenericRecordToString(GenericRecord genericRecord)
       throws HopValueException {
     try {
       Schema schema = genericRecord.getSchema();
-      String schemaJson = schema.toString();
-      String dataJson = genericRecord.toString();
+      ByteArrayOutputStream data = new ByteArrayOutputStream();
+      JsonEncoder encoder = EncoderFactory.get().jsonEncoder(schema, data);
+      new GenericDatumWriter<GenericRecord>(schema).write(genericRecord, 
encoder);
+      encoder.flush();
 
-      return "{ \"schema\" : " + schemaJson + ", \"data\" : " + dataJson + " 
}";
+      return "{ \"schema\" : "
+          + schema
+          + ", \"data\" : "
+          + data.toString(StandardCharsets.UTF_8)
+          + " }";
     } catch (Exception e) {
       throw new HopValueException(
           "Unable to convert an Avro record to a JSON String using the 
provided schema", e);
     }
   }
 
+  /** Read a record from the schema and data JSON {@link 
#convertGenericRecordToString} writes. */
   public static GenericRecord convertStringToGenericRecord(String jsonString)
       throws HopValueException {
+    return convertStringToGenericRecord(jsonString, null);
+  }
+
+  /**
+   * Read a record from text: either the schema and data JSON {@link 
#convertGenericRecordToString}
+   * writes, or only the data (as a record's toString() writes it) when a 
schema is given. The data
+   * can be in Avro's JSON encoding or plain JSON values.
+   *
+   * @param jsonString The text to read
+   * @param schema The schema of data without one, or null
+   * @return The record
+   */
+  public static GenericRecord convertStringToGenericRecord(String jsonString, 
Schema schema)
+      throws HopValueException {
+    JsonNode json;
     try {
-      // Convert schema AND data to JSON...
-      //
-      JSONObject json = (JSONObject) new JSONParser().parse(jsonString);
-      JSONObject schemaObject = (JSONObject) json.get(CONST_SCHEMA);
-      JSONObject dataObject = (JSONObject) json.get("data");
+      json = JSON_MAPPER.readTree(jsonString);
+    } catch (Exception e) {
+      throw new HopValueException("Unable to read an Avro record from text: 
not valid JSON", e);
+    }
+    if (json != null && json.has(CONST_SCHEMA) && json.has(CONST_DATA)) {
+      Schema textSchema;
+      try {
+        textSchema = new 
Schema.Parser().parse(json.get(CONST_SCHEMA).toString());
+      } catch (Exception e) {
+        throw new HopValueException("Unable to read the Avro schema in the 
text", e);
+      }
+      return (GenericRecord) jsonToAvro(json.get(CONST_DATA), textSchema);
+    }
+    if (schema == null) {
+      throw new HopValueException(
+          "Unable to read an Avro record from text without a schema: the text 
holds no"
+              + " \"schema\" and the field has none");
+    }
+    return (GenericRecord) jsonToAvro(json, schema);
+  }
 
-      Schema schema = new Schema.Parser().parse(schemaObject.toJSONString());
+  /**
+   * One JSON value as the Avro value of the schema. A union value can be 
wrapped in its type, as
+   * Avro's JSON encoding writes it, or plain, as a record's toString() writes 
it: then the first
+   * type it fits is used.
+   */
+  private static Object jsonToAvro(JsonNode node, Schema schema) throws 
HopValueException {
+    boolean isNull = node == null || node.isNull();
+    switch (schema.getType()) {
+      case NULL:
+        if (isNull) {
+          return null;
+        }
+        break;
+      case BOOLEAN:
+        if (!isNull && node.isBoolean()) {
+          return node.booleanValue();
+        }
+        break;
+      case INT:
+        if (!isNull && node.isIntegralNumber() && node.canConvertToInt()) {
+          return node.intValue();
+        }
+        break;
+      case LONG:
+        if (!isNull && node.isIntegralNumber() && node.canConvertToLong()) {
+          return node.longValue();
+        }
+        break;
+      case FLOAT:
+        if (!isNull && node.isNumber()) {
+          return node.floatValue();
+        }
+        break;
+      case DOUBLE:
+        if (!isNull && node.isNumber()) {
+          return node.doubleValue();
+        }
+        break;
+      case STRING:
+        if (!isNull && node.isTextual()) {
+          return new Utf8(node.textValue());
+        }
+        break;
+      case BYTES:
+        if (!isNull && node.isTextual()) {
+          return 
ByteBuffer.wrap(node.textValue().getBytes(StandardCharsets.ISO_8859_1));
+        }
+        break;
+      case FIXED:

Review Comment:
   Fixed in 58469c35e7, thanks @hansva. Fixed values and NaN/±Infinity now read 
back from both the `toString()` text and the schema + data text.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to