This is an automated email from the ASF dual-hosted git repository.

bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git


The following commit(s) were added to refs/heads/main by this push:
     new effb726a4f An Avro record converts from text, with its schema or the 
field's, fi… (#9561)
effb726a4f is described below

commit effb726a4ffe9cf4cf25f7f91941cc2c07b3e746
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Thu Nov 19 11:01:03 2026 +0100

    An Avro record converts from text, with its schema or the field's, fi… 
(#9561)
    
    * An Avro record converts from text, with its schema or the field's, fixes 
#9560
    
    * fix remarks
---
 core/src/main/java/org/apache/hop/core/Const.java  |  13 +
 .../hop/core/row/value/ValueMetaAvroRecord.java    | 285 +++++++++++++++++++--
 .../core/row/value/ValueMetaAvroRecordTest.java    | 198 ++++++++++++++
 .../modules/ROOT/pages/variables.adoc              |   1 +
 .../transforms/0131-avro-record-convert.hpl        | 198 ++++++++++++++
 .../datasets/golden-0131-avro-record-convert.csv   |   4 +
 .../transforms/main-0131-avro-record-convert.hwf   |  80 ++++++
 .../dataset/golden-0131-avro-record-convert.json   |  24 ++
 .../unit-test/0131-avro-record-convert UNIT.json   |  32 +++
 9 files changed, 817 insertions(+), 18 deletions(-)

diff --git a/core/src/main/java/org/apache/hop/core/Const.java 
b/core/src/main/java/org/apache/hop/core/Const.java
index a467f14645..72aeee47d2 100644
--- a/core/src/main/java/org/apache/hop/core/Const.java
+++ b/core/src/main/java/org/apache/hop/core/Const.java
@@ -639,6 +639,19 @@ public class Const {
   public static final String HOP_EMPTY_STRING_DIFFERS_FROM_NULL =
       "HOP_EMPTY_STRING_DIFFERS_FROM_NULL";
 
+  /**
+   * Write an Avro record as text with its schema, so the text can be 
converted back to a record
+   * without one.
+   */
+  @Variable(
+      value = "N",
+      description =
+          "Set this variable to Y to write an Avro record as text with its 
schema and its data in"
+              + " Avro's JSON encoding, so the text converts back to the 
record without a schema."
+              + " With N only the data is written")
+  public static final String HOP_AVRO_RECORD_STRING_WITH_SCHEMA =
+      "HOP_AVRO_RECORD_STRING_WITH_SCHEMA";
+
   /**
    * System wide flag to allow non-strict string to number conversion for 
backward compatibility. If
    * this setting is set to "Y", an string starting with digits will be 
converted successfully into
diff --git 
a/core/src/main/java/org/apache/hop/core/row/value/ValueMetaAvroRecord.java 
b/core/src/main/java/org/apache/hop/core/row/value/ValueMetaAvroRecord.java
index 6ee5deac50..7ee522cb5f 100644
--- a/core/src/main/java/org/apache/hop/core/row/value/ValueMetaAvroRecord.java
+++ b/core/src/main/java/org/apache/hop/core/row/value/ValueMetaAvroRecord.java
@@ -17,6 +17,8 @@
 
 package org.apache.hop.core.row.value;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
@@ -25,7 +27,10 @@ import java.io.EOFException;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.net.SocketTimeoutException;
+import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
+import java.util.LinkedHashMap;
+import java.util.Map;
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericDatumReader;
@@ -35,8 +40,10 @@ import org.apache.avro.io.BinaryDecoder;
 import org.apache.avro.io.BinaryEncoder;
 import org.apache.avro.io.DecoderFactory;
 import org.apache.avro.io.EncoderFactory;
-import org.apache.avro.io.JsonDecoder;
+import org.apache.avro.io.JsonEncoder;
+import org.apache.avro.util.Utf8;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.Const;
 import org.apache.hop.core.exception.HopEofException;
 import org.apache.hop.core.exception.HopException;
 import org.apache.hop.core.exception.HopFileException;
@@ -57,6 +64,8 @@ public class ValueMetaAvroRecord extends ValueMetaBase {
 
   private Schema schema;
   private static final String CONST_SCHEMA = "schema";
+  private static final String CONST_DATA = "data";
+  private static final ObjectMapper JSON_MAPPER = new ObjectMapper();
   private static final String CONST_SPECIFIED = " specified.";
   private static final String CONST_UNKNOWN_STORAGE_TYPE = " : Unknown storage 
type ";
   private static final String CONST_SCHEMA_NEEDED =
@@ -65,6 +74,13 @@ public class ValueMetaAvroRecord extends ValueMetaBase {
   /** Reject a corrupt length instead of allocating it. Sample rows are far 
smaller than this. */
   private static final int MAX_AVRO_BYTES = 32 * 1024 * 1024;
 
+  /**
+   * Write a record as text with its schema (see {@link 
#convertGenericRecordToString}), so it can
+   * be read back without one. Off by default: the text is then only the data, 
as before.
+   */
+  private final boolean stringWithSchema =
+      
Const.toBoolean(System.getProperty(Const.HOP_AVRO_RECORD_STRING_WITH_SCHEMA, 
"N"));
+
   public ValueMetaAvroRecord() {
     super(null, IValueMeta.TYPE_AVRO);
   }
@@ -119,7 +135,7 @@ public class ValueMetaAvroRecord extends ValueMetaBase {
           case STORAGE_TYPE_NORMAL:
             try {
               String jsonString = (String) object;
-              return convertStringToGenericRecord(jsonString);
+              return convertStringToGenericRecord(jsonString, schema);
             } catch (Exception e) {
               throw new HopValueException(
                   "Error converting a JSON representation of an Avro 
GenericRecord to a native representation",
@@ -136,7 +152,9 @@ public class ValueMetaAvroRecord extends ValueMetaBase {
   }
 
   /**
-   * Convert the record to both schema and data in a single JSON block...
+   * Convert the record to both schema and data in a single JSON block, the 
data in Avro's JSON
+   * encoding: <code>{ "schema" : ..., "data" : ... }</code>. {@link
+   * #convertStringToGenericRecord(String)} reads it back.
    *
    * @param genericRecord The record to convert to JSON
    * @return The JSON representation of a generic Avro record
@@ -146,33 +164,237 @@ public class ValueMetaAvroRecord extends ValueMetaBase {
       throws HopValueException {
     try {
       Schema schema = genericRecord.getSchema();
-      String schemaJson = schema.toString();
-      String dataJson = genericRecord.toString();
+      ByteArrayOutputStream data = new ByteArrayOutputStream();
+      JsonEncoder encoder = EncoderFactory.get().jsonEncoder(schema, data);
+      new GenericDatumWriter<GenericRecord>(schema).write(genericRecord, 
encoder);
+      encoder.flush();
 
-      return "{ \"schema\" : " + schemaJson + ", \"data\" : " + dataJson + " 
}";
+      return "{ \"schema\" : "
+          + schema
+          + ", \"data\" : "
+          + data.toString(StandardCharsets.UTF_8)
+          + " }";
     } catch (Exception e) {
       throw new HopValueException(
           "Unable to convert an Avro record to a JSON String using the 
provided schema", e);
     }
   }
 
+  /** Read a record from the schema and data JSON {@link 
#convertGenericRecordToString} writes. */
   public static GenericRecord convertStringToGenericRecord(String jsonString)
       throws HopValueException {
+    return convertStringToGenericRecord(jsonString, null);
+  }
+
+  /**
+   * Read a record from text: either the schema and data JSON {@link 
#convertGenericRecordToString}
+   * writes, or only the data (as a record's toString() writes it) when a 
schema is given. The data
+   * can be in Avro's JSON encoding or plain JSON values.
+   *
+   * @param jsonString The text to read
+   * @param schema The schema of data without one, or null
+   * @return The record
+   */
+  public static GenericRecord convertStringToGenericRecord(String jsonString, 
Schema schema)
+      throws HopValueException {
+    JsonNode json;
     try {
-      // Convert schema AND data to JSON...
-      //
-      JSONObject json = (JSONObject) new JSONParser().parse(jsonString);
-      JSONObject schemaObject = (JSONObject) json.get(CONST_SCHEMA);
-      JSONObject dataObject = (JSONObject) json.get("data");
+      json = JSON_MAPPER.readTree(jsonString);
+    } catch (Exception e) {
+      throw new HopValueException("Unable to read an Avro record from text: 
not valid JSON", e);
+    }
+    if (json != null && json.has(CONST_SCHEMA) && json.has(CONST_DATA)) {
+      Schema textSchema;
+      try {
+        textSchema = new 
Schema.Parser().parse(json.get(CONST_SCHEMA).toString());
+      } catch (Exception e) {
+        throw new HopValueException("Unable to read the Avro schema in the 
text", e);
+      }
+      return (GenericRecord) jsonToAvro(json.get(CONST_DATA), textSchema);
+    }
+    if (schema == null) {
+      throw new HopValueException(
+          "Unable to read an Avro record from text without a schema: the text 
holds no"
+              + " \"schema\" and the field has none");
+    }
+    return (GenericRecord) jsonToAvro(json, schema);
+  }
+
+  /**
+   * One JSON value as the Avro value of the schema. A union value can be 
wrapped in its type, as
+   * Avro's JSON encoding writes it, or plain, as a record's toString() writes 
it: then the first
+   * type it fits is used.
+   */
+  private static Object jsonToAvro(JsonNode node, Schema schema) throws 
HopValueException {
+    boolean isNull = node == null || node.isNull();
+    switch (schema.getType()) {
+      case NULL:
+        if (isNull) {
+          return null;
+        }
+        break;
+      case BOOLEAN:
+        if (!isNull && node.isBoolean()) {
+          return node.booleanValue();
+        }
+        break;
+      case INT:
+        if (!isNull && node.isIntegralNumber() && node.canConvertToInt()) {
+          return node.intValue();
+        }
+        break;
+      case LONG:
+        if (!isNull && node.isIntegralNumber() && node.canConvertToLong()) {
+          return node.longValue();
+        }
+        break;
+      case FLOAT:
+        if (!isNull && node.isNumber()) {
+          return node.floatValue();
+        }
+        if (nonFiniteNumber(node) instanceof Double nonFinite) {
+          return nonFinite.floatValue();
+        }
+        break;
+      case DOUBLE:
+        if (!isNull && node.isNumber()) {
+          return node.doubleValue();
+        }
+        if (nonFiniteNumber(node) instanceof Double nonFinite) {
+          return nonFinite;
+        }
+        break;
+      case STRING:
+        if (!isNull && node.isTextual()) {
+          return new Utf8(node.textValue());
+        }
+        break;
+      case BYTES:
+        if (!isNull && node.isTextual()) {
+          return 
ByteBuffer.wrap(node.textValue().getBytes(StandardCharsets.ISO_8859_1));
+        }
+        break;
+      case FIXED:
+        if (!isNull && node.isTextual()) {
+          return new GenericData.Fixed(
+              schema, node.textValue().getBytes(StandardCharsets.ISO_8859_1));
+        }
+        // A record's toString() writes the bytes as an array of numbers: [1, 
2, 3, -6]
+        if (fixedBytes(node, schema.getFixedSize()) instanceof byte[] bytes) {
+          return new GenericData.Fixed(schema, bytes);
+        }
+        break;
+      case ENUM:
+        if (!isNull && node.isTextual() && 
schema.hasEnumSymbol(node.textValue())) {
+          return new GenericData.EnumSymbol(schema, node.textValue());
+        }
+        break;
+      case ARRAY:
+        if (!isNull && node.isArray()) {
+          GenericData.Array<Object> array = new 
GenericData.Array<>(node.size(), schema);
+          for (JsonNode element : node) {
+            array.add(jsonToAvro(element, schema.getElementType()));
+          }
+          return array;
+        }
+        break;
+      case MAP:
+        if (!isNull && node.isObject()) {
+          Map<Utf8, Object> map = new LinkedHashMap<>();
+          for (Map.Entry<String, JsonNode> entry : node.properties()) {
+            map.put(new Utf8(entry.getKey()), jsonToAvro(entry.getValue(), 
schema.getValueType()));
+          }
+          return map;
+        }
+        break;
+      case RECORD:
+        // Only an object with no keys the record doesn't have: in a union of 
records written by
+        // toString(), a record whose fields are all optional would otherwise 
match any object.
+        if (!isNull && node.isObject() && onlyFieldsOf(node, schema)) {
+          GenericData.Record avroRecord = new GenericData.Record(schema);
+          for (Schema.Field field : schema.getFields()) {
+            JsonNode value = node.get(field.name());
+            if (value == null && field.hasDefaultValue()) {
+              avroRecord.put(field.pos(), 
GenericData.get().getDefaultValue(field));
+            } else {
+              avroRecord.put(field.pos(), jsonToAvro(value, field.schema()));
+            }
+          }
+          return avroRecord;
+        }
+        break;
+      case UNION:
+        return unionToAvro(node, schema);
+      default:
+        break;
+    }
+    throw new HopValueException(
+        "Unable to read " + (isNull ? "null" : node.toString()) + " as Avro 
type " + schema);
+  }
 
-      Schema schema = new Schema.Parser().parse(schemaObject.toJSONString());
+  /** NaN and the infinities, which JSON writers (toString(), Avro's encoder) 
write as text. */
+  private static Double nonFiniteNumber(JsonNode node) {
+    if (node == null || !node.isTextual()) {
+      return null;
+    }
+    return switch (node.textValue()) {
+      case "NaN" -> Double.NaN;
+      case "Infinity" -> Double.POSITIVE_INFINITY;
+      case "-Infinity" -> Double.NEGATIVE_INFINITY;
+      default -> null;
+    };
+  }
 
-      JsonDecoder jsonDecoder = DecoderFactory.get().jsonDecoder(schema, 
dataObject.toJSONString());
-      GenericDatumReader<GenericRecord> genericDatumReader = new 
GenericDatumReader<>(schema);
-      return genericDatumReader.read(null, jsonDecoder);
-    } catch (Exception e) {
-      throw new HopValueException("Unable to convert a String to an Avro 
record", e);
+  private static byte[] fixedBytes(JsonNode node, int size) {
+    if (node == null || !node.isArray() || node.size() != size) {
+      return null;
+    }
+    byte[] bytes = new byte[size];
+    for (int i = 0; i < size; i++) {
+      JsonNode element = node.get(i);
+      if (!element.isIntegralNumber()
+          || !element.canConvertToInt()
+          || element.intValue() < Byte.MIN_VALUE
+          || element.intValue() > Byte.MAX_VALUE) {
+        return null;
+      }
+      bytes[i] = (byte) element.intValue();
+    }
+    return bytes;
+  }
+
+  private static boolean onlyFieldsOf(JsonNode node, Schema recordSchema) {
+    for (Map.Entry<String, JsonNode> entry : node.properties()) {
+      if (recordSchema.getField(entry.getKey()) == null) {
+        return false;
+      }
     }
+    return true;
+  }
+
+  private static Object unionToAvro(JsonNode node, Schema union) throws 
HopValueException {
+    // Avro's JSON encoding: {"type name": value}
+    if (node != null && node.isObject() && node.size() == 1) {
+      String typeName = node.fieldNames().next();
+      for (Schema type : union.getTypes()) {
+        if (typeName.equals(type.getFullName()) || 
typeName.equals(type.getName())) {
+          return jsonToAvro(node.get(typeName), type);
+        }
+      }
+    }
+    // Plain JSON: the first type the value fits
+    for (Schema type : union.getTypes()) {
+      try {
+        return jsonToAvro(node, type);
+      } catch (HopValueException e) {
+        // Try the next type of the union
+      }
+    }
+    throw new HopValueException(
+        "Unable to read "
+            + (node == null ? "null" : node.toString())
+            + " as any type of Avro union "
+            + union);
   }
 
   /**
@@ -260,7 +482,12 @@ public class ValueMetaAvroRecord extends ValueMetaBase {
         case IValueMeta.TYPE_AVRO:
           string =
               switch (storageType) {
-                case STORAGE_TYPE_NORMAL -> object == null ? null : 
object.toString();
+                case STORAGE_TYPE_NORMAL ->
+                    object == null
+                        ? null
+                        : stringWithSchema
+                            ? convertGenericRecordToString((GenericRecord) 
object)
+                            : object.toString();
                 default ->
                     throw new HopValueException(
                         this
@@ -293,6 +520,28 @@ public class ValueMetaAvroRecord extends ValueMetaBase {
     }
   }
 
+  /**
+   * An Avro record from another value: a record is copied. Text is read as 
the schema and data JSON
+   * {@link #convertGenericRecordToString} writes, or as only the data when 
this field has a schema.
+   */
+  @Override
+  public Object convertData(IValueMeta meta2, Object data2) throws 
HopValueException {
+    if (data2 == null) {
+      return null;
+    }
+    if (meta2 != null && meta2.getType() == TYPE_AVRO) {
+      return cloneValueData(data2);
+    }
+    if (meta2 != null && meta2.isString()) {
+      String json = meta2.getString(data2);
+      if (StringUtils.isEmpty(json)) {
+        return null;
+      }
+      return convertStringToGenericRecord(json, schema);
+    }
+    throw new HopValueException(this + " : unable to convert " + meta2 + " 
data to an Avro record");
+  }
+
   @Override
   public Object cloneValueData(Object object) throws HopValueException {
     if (object == null) {
diff --git 
a/core/src/test/java/org/apache/hop/core/row/value/ValueMetaAvroRecordTest.java 
b/core/src/test/java/org/apache/hop/core/row/value/ValueMetaAvroRecordTest.java
index cc5618fca2..db6ebffb7f 100644
--- 
a/core/src/test/java/org/apache/hop/core/row/value/ValueMetaAvroRecordTest.java
+++ 
b/core/src/test/java/org/apache/hop/core/row/value/ValueMetaAvroRecordTest.java
@@ -21,6 +21,7 @@ package org.apache.hop.core.row.value;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotSame;
 import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import com.fasterxml.jackson.databind.JsonNode;
@@ -33,8 +34,11 @@ import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.avro.util.Utf8;
+import org.apache.hop.core.Const;
 import org.apache.hop.core.HopClientEnvironment;
+import org.apache.hop.core.exception.HopValueException;
 import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
 import org.apache.hop.core.row.RowMeta;
 import org.apache.hop.core.util.TestUtil;
 import org.json.simple.JSONObject;
@@ -262,6 +266,200 @@ class ValueMetaAvroRecordTest {
     assertEquals(5L, loaded[2]);
   }
 
+  /** By default the text is only the data: a field with a schema reads it 
back. */
+  @Test
+  void aRecordComesBackFromItsTextWithTheFieldsSchema() throws Exception {
+    GenericRecord genericRecord = generateGenericRecord();
+    ValueMetaAvroRecord valueMeta = new ValueMetaAvroRecord("test", 
genericRecord.getSchema());
+    String text = valueMeta.getString(genericRecord);
+    assertEquals(genericRecord.toString(), text);
+
+    Object back = valueMeta.convertData(new ValueMetaString("text"), text);
+
+    assertEquals(genericRecord.getSchema(), ((GenericRecord) 
back).getSchema());
+    verifyGenericRecords(genericRecord, (GenericRecord) back);
+  }
+
+  /** Only the data and no schema on the field: nothing to read it with. */
+  @Test
+  void textWithoutASchemaNeedsTheFieldsSchema() throws Exception {
+    GenericRecord genericRecord = generateGenericRecord();
+    String text = genericRecord.toString();
+
+    ValueMetaAvroRecord withoutSchema = new ValueMetaAvroRecord("test");
+    HopValueException thrown =
+        assertThrows(
+            HopValueException.class,
+            () -> withoutSchema.convertData(new ValueMetaString("text"), 
text));
+    assertTrue(thrown.getMessage().contains("schema"), thrown.getMessage());
+  }
+
+  /** With HOP_AVRO_RECORD_STRING_WITH_SCHEMA=Y the text carries the schema: 
no schema needed. */
+  @Test
+  void textWithItsSchemaComesBackWithoutOne() throws Exception {
+    GenericRecord genericRecord = generateGenericRecord();
+    String text;
+    System.setProperty(Const.HOP_AVRO_RECORD_STRING_WITH_SCHEMA, "Y");
+    try {
+      text = new ValueMetaAvroRecord("test", 
genericRecord.getSchema()).getString(genericRecord);
+    } finally {
+      System.clearProperty(Const.HOP_AVRO_RECORD_STRING_WITH_SCHEMA);
+    }
+    assertTrue(text.startsWith("{ \"schema\" : "), text);
+
+    Object back = new ValueMetaAvroRecord("test").convertData(new 
ValueMetaString("text"), text);
+
+    assertEquals(genericRecord.getSchema(), ((GenericRecord) 
back).getSchema());
+    verifyGenericRecords(genericRecord, (GenericRecord) back);
+  }
+
+  /** The schema and data helpers round trip, also for a nullable (union) 
field. */
+  @Test
+  void schemaAndDataTextRoundTrips() throws Exception {
+    GenericRecord genericRecord = generateGenericRecord();
+
+    GenericRecord back =
+        ValueMetaAvroRecord.convertStringToGenericRecord(
+            ValueMetaAvroRecord.convertGenericRecordToString(genericRecord));
+
+    verifyGenericRecords(genericRecord, back);
+  }
+
+  /** Schema and data text as earlier versions wrote it, the data as 
toString() gives it. */
+  @Test
+  void schemaAndDataTextWithPlainDataIsRead() throws Exception {
+    GenericRecord genericRecord = generateGenericRecord();
+    String text =
+        "{ \"schema\" : " + genericRecord.getSchema() + ", \"data\" : " + 
genericRecord + " }";
+
+    GenericRecord back = 
ValueMetaAvroRecord.convertStringToGenericRecord(text);
+
+    verifyGenericRecords(genericRecord, back);
+  }
+
+  /**
+   * toString() doesn't say which record of a union a value is: the reader 
must not take a record
+   * whose fields are all optional for an object with other keys.
+   */
+  @Test
+  void aUnionOfRecordsReadsBackAsTheRightRecord() throws Exception {
+    Schema schema =
+        new Schema.Parser()
+            .parse(
+                
"{\"type\":\"record\",\"name\":\"Outer\",\"fields\":[{\"name\":\"two\",\"type\":["
+                    + 
"{\"type\":\"record\",\"name\":\"A\",\"fields\":[{\"name\":\"a\","
+                    + "\"type\":[\"null\",\"string\"],\"default\":null}]},"
+                    + 
"{\"type\":\"record\",\"name\":\"B\",\"fields\":[{\"name\":\"b\","
+                    + "\"type\":\"string\"}]}]}]}");
+    Schema recordB = schema.getField("two").schema().getTypes().get(1);
+    GenericRecord b = new GenericData.Record(recordB);
+    b.put("b", "bee");
+    GenericRecord outer = new GenericData.Record(schema);
+    outer.put("two", b);
+
+    GenericRecord back = 
ValueMetaAvroRecord.convertStringToGenericRecord(outer.toString(), schema);
+
+    GenericRecord two = (GenericRecord) back.get("two");
+    assertEquals("B", two.getSchema().getName());
+    assertEquals("bee", two.get("b").toString());
+  }
+
+  /** A fixed value, as toString() writes it ([1, 2, 3, -6]) and as Avro's 
JSON encoding does. */
+  @Test
+  void aFixedValueReadsBackFromBothTexts() throws Exception {
+    Schema schema =
+        new Schema.Parser()
+            .parse(
+                
"{\"type\":\"record\",\"name\":\"R\",\"fields\":[{\"name\":\"hash\",\"type\":"
+                    + "{\"type\":\"fixed\",\"name\":\"Hash\",\"size\":4}}]}");
+    GenericRecord original = new GenericData.Record(schema);
+    original.put(
+        "hash", new GenericData.Fixed(schema.getField("hash").schema(), new 
byte[] {1, 2, 3, -6}));
+
+    GenericRecord fromPlain =
+        ValueMetaAvroRecord.convertStringToGenericRecord(original.toString(), 
schema);
+    GenericRecord fromEncoded =
+        ValueMetaAvroRecord.convertStringToGenericRecord(
+            ValueMetaAvroRecord.convertGenericRecordToString(original));
+
+    assertEquals(original, fromPlain);
+    assertEquals(original, fromEncoded);
+  }
+
+  /** NaN and the infinities are written as text by toString() and by Avro's 
JSON encoding. */
+  @Test
+  void nanAndInfinityReadBackFromBothTexts() throws Exception {
+    Schema schema =
+        new Schema.Parser()
+            .parse(
+                "{\"type\":\"record\",\"name\":\"R\",\"fields\":["
+                    + "{\"name\":\"d\",\"type\":\"double\"},"
+                    + "{\"name\":\"f\",\"type\":\"float\"},"
+                    + "{\"name\":\"n\",\"type\":[\"null\",\"double\"]}]}");
+    GenericRecord original = new GenericData.Record(schema);
+    original.put("d", Double.NaN);
+    original.put("f", Float.NEGATIVE_INFINITY);
+    original.put("n", Double.POSITIVE_INFINITY);
+
+    GenericRecord fromPlain =
+        ValueMetaAvroRecord.convertStringToGenericRecord(original.toString(), 
schema);
+    GenericRecord fromEncoded =
+        ValueMetaAvroRecord.convertStringToGenericRecord(
+            ValueMetaAvroRecord.convertGenericRecordToString(original));
+
+    for (GenericRecord back : new GenericRecord[] {fromPlain, fromEncoded}) {
+      assertTrue(Double.isNaN((Double) back.get("d")));
+      assertEquals(Float.NEGATIVE_INFINITY, back.get("f"));
+      assertEquals(Double.POSITIVE_INFINITY, back.get("n"));
+    }
+  }
+
+  /** Text file input, CSV input and the like read an Avro record field this 
way. */
+  @Test
+  void aRecordIsReadFromText() throws Exception {
+    GenericRecord genericRecord = generateGenericRecord();
+    ValueMetaAvroRecord valueMeta = new ValueMetaAvroRecord("test", 
genericRecord.getSchema());
+    String text = valueMeta.getString(genericRecord);
+
+    Object read =
+        valueMeta.convertDataFromString(
+            text, new ValueMetaString("text"), null, null, 
IValueMeta.TRIM_TYPE_NONE);
+
+    verifyGenericRecords(genericRecord, (GenericRecord) read);
+  }
+
+  @Test
+  void aRecordIsCopied() throws Exception {
+    GenericRecord genericRecord = generateGenericRecord();
+    ValueMetaAvroRecord valueMeta = new ValueMetaAvroRecord("test", 
genericRecord.getSchema());
+
+    Object copy =
+        valueMeta.convertData(
+            new ValueMetaAvroRecord("other", genericRecord.getSchema()), 
genericRecord);
+
+    assertNotSame(genericRecord, copy);
+    verifyGenericRecords(genericRecord, (GenericRecord) copy);
+  }
+
+  @Test
+  void emptyTextIsNoRecord() throws Exception {
+    ValueMetaAvroRecord valueMeta = new ValueMetaAvroRecord("test");
+
+    assertNull(valueMeta.convertData(new ValueMetaString("text"), ""));
+    assertNull(valueMeta.convertData(new ValueMetaString("text"), null));
+  }
+
+  @Test
+  void textThatIsNoRecordOrAnotherTypeIsAnError() {
+    ValueMetaAvroRecord valueMeta = new ValueMetaAvroRecord("test");
+
+    assertThrows(
+        HopValueException.class,
+        () -> valueMeta.convertData(new ValueMetaString("text"), "not json"));
+    assertThrows(
+        HopValueException.class, () -> valueMeta.convertData(new 
ValueMetaInteger("n"), 1L));
+  }
+
   private GenericRecord generateGenericRecord() {
     Schema schema = new Schema.Parser().parse(SCHEMA_JSON);
     GenericRecord genericRecord = new GenericData.Record(schema);
diff --git a/docs/hop-user-manual/modules/ROOT/pages/variables.adoc 
b/docs/hop-user-manual/modules/ROOT/pages/variables.adoc
index 15850ea4a1..88d3349409 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/variables.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/variables.adoc
@@ -245,6 +245,7 @@ Otherwise by default NULL is ignored by the MIN aggregate 
and MIN is set to the
 See also the variable HOP_AGGREGATION_ALL_NULLS_ARE_ZERO.
 |HOP_ALLOW_EMPTY_FIELD_NAMES_AND_TYPES|N|Set this variable to Y to allow your 
pipeline to pass 'null' fields and/or empty types.
 |HOP_ALLOW_UNCONSUMED_MAIN_INPUT|N|Set this variable to Y to start a pipeline 
that has main hops into transforms that do not consume input (for example 
existing files whose upstream produces no rows). Default N fails init and stops 
on leftover input. Verify still reports an error. See 
xref:pipeline/pipeline-sources.adoc[Pipeline sources].
+|HOP_AVRO_RECORD_STRING_WITH_SCHEMA|N|Set this variable to Y to write an Avro 
record as text with its schema and its data in Avro's JSON encoding: `{ 
"schema" : ..., "data" : ... }`. That text converts back to the record without 
a schema, for example in xref:pipeline/transforms/selectvalues.adoc[Select 
values]. With N only the data is written, as a record's `toString()` gives it; 
that converts back only to a field that has a schema.
 |HOP_BINARY_FIELDS_AVOID_HEX_PREVIEW|false|Set this variable to true to show 
the content of binary fields as text, for example for CLOB database fields, in 
preview windows, in the results of the 
xref:hop-gui/perspective-database.adoc[Database perspective] and in the preview 
of a Parquet file.
 By default a binary value is shown as its hexadecimal representation.
 Hop only reads this variable from the configuration perspective.
diff --git a/integration-tests/transforms/0131-avro-record-convert.hpl 
b/integration-tests/transforms/0131-avro-record-convert.hpl
new file mode 100644
index 0000000000..8ff698c74b
--- /dev/null
+++ b/integration-tests/transforms/0131-avro-record-convert.hpl
@@ -0,0 +1,198 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+  <info>
+    <name>0131-avro-record-convert</name>
+    <name_sync_with_filename>Y</name_sync_with_filename>
+    <description>Select values converts text holding an Avro record with its 
schema to an Avro Record field and back to text (#9560)</description>
+    <extended_description/>
+    <pipeline_version/>
+    <pipeline_type>Normal</pipeline_type>
+    <parameters>
+    </parameters>
+    <capture_transform_performance>N</capture_transform_performance>
+    
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+    
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+    <created_user>-</created_user>
+    <created_date>2026/11/18 12:00:00.000</created_date>
+    <modified_user>-</modified_user>
+    <modified_date>2026/11/18 12:00:00.000</modified_date>
+  </info>
+  <notepads>
+  </notepads>
+  <order>
+    <hop>
+      <from>text</from>
+      <to>text to record</to>
+      <enabled>Y</enabled>
+    </hop>
+    <hop>
+      <from>text to record</from>
+      <to>record to text</to>
+      <enabled>Y</enabled>
+    </hop>
+    <hop>
+      <from>record to text</from>
+      <to>validate</to>
+      <enabled>Y</enabled>
+    </hop>
+  </order>
+  <transform>
+    <name>text</name>
+    <type>DataGrid</type>
+    <description/>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <fields>
+      <field>
+        <set_empty_string>N</set_empty_string>
+        <length>-1</length>
+        <name>id</name>
+        <precision>-1</precision>
+        <type>Integer</type>
+      </field>
+      <field>
+        <set_empty_string>N</set_empty_string>
+        <length>-1</length>
+        <name>text</name>
+        <precision>-1</precision>
+        <type>String</type>
+      </field>
+    </fields>
+    <data>
+      <line>
+        <item>1</item>
+        <item>{ "schema" : 
{"type":"record","name":"person","namespace":"hop.it","fields":[{"name":"id","type":"long"},{"name":"name","type":["null","string"]},{"name":"score","type":["null","double"]}]},
 "data" : {"id":1,"name":{"string":"Ada"},"score":{"double":9.5}} }</item>
+      </line>
+      <line>
+        <item>2</item>
+        <item>{ "schema" : 
{"type":"record","name":"person","namespace":"hop.it","fields":[{"name":"id","type":"long"},{"name":"name","type":["null","string"]},{"name":"score","type":["null","double"]}]},
 "data" : {"id": 2, "name": "Grace", "score": null} }</item>
+      </line>
+      <line>
+        <item>3</item>
+        <item/>
+      </line>
+    </data>
+    <attributes/>
+    <GUI>
+      <xloc>96</xloc>
+      <yloc>96</yloc>
+    </GUI>
+  </transform>
+  <transform>
+    <name>text to record</name>
+    <type>SelectValues</type>
+    <description/>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <fields>
+      <select_unspecified>N</select_unspecified>
+      <meta>
+        <name>text</name>
+        <rename>record</rename>
+        <type>Avro Record</type>
+        <length>-2</length>
+        <precision>-2</precision>
+        <conversion_mask/>
+        <date_format_lenient>false</date_format_lenient>
+        <date_format_locale/>
+        <date_format_timezone/>
+        <lenient_string_to_number>false</lenient_string_to_number>
+        <encoding/>
+        <decimal_symbol/>
+        <grouping_symbol/>
+        <currency_symbol/>
+        <storage_type/>
+      </meta>
+    </fields>
+    <attributes/>
+    <GUI>
+      <xloc>288</xloc>
+      <yloc>96</yloc>
+    </GUI>
+  </transform>
+  <transform>
+    <name>record to text</name>
+    <type>SelectValues</type>
+    <description/>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <fields>
+      <select_unspecified>N</select_unspecified>
+      <meta>
+        <name>record</name>
+        <rename>text_out</rename>
+        <type>String</type>
+        <length>-2</length>
+        <precision>-2</precision>
+        <conversion_mask/>
+        <date_format_lenient>false</date_format_lenient>
+        <date_format_locale/>
+        <date_format_timezone/>
+        <lenient_string_to_number>false</lenient_string_to_number>
+        <encoding/>
+        <decimal_symbol/>
+        <grouping_symbol/>
+        <currency_symbol/>
+        <storage_type/>
+      </meta>
+    </fields>
+    <attributes/>
+    <GUI>
+      <xloc>480</xloc>
+      <yloc>96</yloc>
+    </GUI>
+  </transform>
+  <transform>
+    <name>validate</name>
+    <type>Dummy</type>
+    <description/>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <attributes/>
+    <GUI>
+      <xloc>672</xloc>
+      <yloc>96</yloc>
+    </GUI>
+  </transform>
+  <transform_error_handling>
+  </transform_error_handling>
+  <attributes/>
+</pipeline>
diff --git 
a/integration-tests/transforms/datasets/golden-0131-avro-record-convert.csv 
b/integration-tests/transforms/datasets/golden-0131-avro-record-convert.csv
new file mode 100644
index 0000000000..7eca3f585e
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-0131-avro-record-convert.csv
@@ -0,0 +1,4 @@
+id,text_out
+1,"{""id"": 1, ""name"": ""Ada"", ""score"": 9.5}"
+2,"{""id"": 2, ""name"": ""Grace"", ""score"": null}"
+3,
diff --git a/integration-tests/transforms/main-0131-avro-record-convert.hwf 
b/integration-tests/transforms/main-0131-avro-record-convert.hwf
new file mode 100644
index 0000000000..d26df2acfd
--- /dev/null
+++ b/integration-tests/transforms/main-0131-avro-record-convert.hwf
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+  <name>main-0131-avro-record-convert</name>
+  <name_sync_with_filename>Y</name_sync_with_filename>
+  <description/>
+  <extended_description/>
+  <workflow_version/>
+  <created_user>-</created_user>
+  <created_date>2026/11/18 12:00:00.000</created_date>
+  <modified_user>-</modified_user>
+  <modified_date>2026/11/18 12:00:00.000</modified_date>
+  <parameters>
+    </parameters>
+  <actions>
+    <action>
+      <name>Start</name>
+      <description/>
+      <type>SPECIAL</type>
+      <attributes/>
+      <DayOfMonth>1</DayOfMonth>
+      <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+      <hour>12</hour>
+      <intervalMinutes>60</intervalMinutes>
+      <intervalSeconds>0</intervalSeconds>
+      <minutes>0</minutes>
+      <repeat>N</repeat>
+      <schedulerType>0</schedulerType>
+      <weekDay>1</weekDay>
+      <parallel>N</parallel>
+      <xloc>96</xloc>
+      <yloc>64</yloc>
+      <attributes_hac/>
+    </action>
+    <action>
+      <name>0131-avro-record-convert UNIT</name>
+      <description/>
+      <type>RunPipelineTests</type>
+      <attributes/>
+      <test_names>
+        <test_name>
+          <name>0131-avro-record-convert UNIT</name>
+        </test_name>
+      </test_names>
+      <parallel>N</parallel>
+      <xloc>288</xloc>
+      <yloc>64</yloc>
+      <attributes_hac/>
+    </action>
+  </actions>
+  <hops>
+    <hop>
+      <from>Start</from>
+      <to>0131-avro-record-convert UNIT</to>
+      <enabled>Y</enabled>
+      <evaluation>Y</evaluation>
+      <unconditional>Y</unconditional>
+    </hop>
+  </hops>
+  <notepads>
+  </notepads>
+  <attributes/>
+</workflow>
diff --git 
a/integration-tests/transforms/metadata/dataset/golden-0131-avro-record-convert.json
 
b/integration-tests/transforms/metadata/dataset/golden-0131-avro-record-convert.json
new file mode 100644
index 0000000000..3bdcb05bc5
--- /dev/null
+++ 
b/integration-tests/transforms/metadata/dataset/golden-0131-avro-record-convert.json
@@ -0,0 +1,24 @@
+{
+  "base_filename": "golden-0131-avro-record-convert.csv",
+  "name": "golden-0131-avro-record-convert",
+  "description": "Avro records read from schema and data text, written back as 
their data; empty text stays empty",
+  "dataset_fields": [
+    {
+      "field_comment": "",
+      "field_length": -1,
+      "field_type": 5,
+      "field_precision": -1,
+      "field_name": "id",
+      "field_format": ""
+    },
+    {
+      "field_comment": "",
+      "field_length": -1,
+      "field_type": 2,
+      "field_precision": -1,
+      "field_name": "text_out",
+      "field_format": ""
+    }
+  ],
+  "folder_name": ""
+}
diff --git 
a/integration-tests/transforms/metadata/unit-test/0131-avro-record-convert 
UNIT.json 
b/integration-tests/transforms/metadata/unit-test/0131-avro-record-convert 
UNIT.json
new file mode 100644
index 0000000000..2c019cd4c8
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0131-avro-record-convert 
UNIT.json        
@@ -0,0 +1,32 @@
+{
+  "database_replacements": [],
+  "autoOpening": true,
+  "description": "",
+  "persist_filename": "",
+  "test_type": "UNIT_TEST",
+  "variableValues": [],
+  "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+  "golden_data_sets": [
+    {
+      "field_mappings": [
+        {
+          "transform_field": "id",
+          "data_set_field": "id"
+        },
+        {
+          "transform_field": "text_out",
+          "data_set_field": "text_out"
+        }
+      ],
+      "field_order": [
+        "id"
+      ],
+      "data_set_name": "golden-0131-avro-record-convert",
+      "transform_name": "validate"
+    }
+  ],
+  "input_data_sets": [],
+  "name": "0131-avro-record-convert UNIT",
+  "trans_test_tweaks": [],
+  "pipeline_filename": "./0131-avro-record-convert.hpl"
+}

Reply via email to