This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-testing.git
The following commit(s) were added to refs/heads/master by this push:
new e7bea05 Add JSON and BSON logical type test files (#118)
e7bea05 is described below
commit e7bea05fa5bc18a032f3687e8a71128f14bb7331
Author: Neelesh Salian <[email protected]>
AuthorDate: Thu Aug 6 14:19:06 2026 -0700
Add JSON and BSON logical type test files (#118)
---
data/README.md | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
data/bson.parquet | Bin 0 -> 412 bytes
data/json.parquet | Bin 0 -> 402 bytes
3 files changed, 107 insertions(+)
diff --git a/data/README.md b/data/README.md
index eb823e1..ea93558 100644
--- a/data/README.md
+++ b/data/README.md
@@ -63,6 +63,8 @@
| int96_from_spark.parquet | Single column of (deprecated) int96 values that
originated as Apache Spark microsecond-resolution timestamps. Some values are
outside the range typically representable by 64-bit nanosecond-resolution
timestamps. See [int96_from_spark.md](int96_from_spark.md) for details. |
| int96_timestamp_order.parquet | Single `required int96` column written with
the `INT96_TIMESTAMP_ORDER` column order ([parquet-format
#584](https://github.com/apache/parquet-format/pull/584)). Values are chosen so
a byte-wise comparison disagrees with the chronological order, so the min/max
statistics (and column index) are only correct for a reader that honors the new
order. See [int96_timestamp_order.md](int96_timestamp_order.md) for details. |
| binary_truncated_min_max.parquet | A file containing six columns with exact,
fully-truncated and partially-truncated max and min statistics and with the
expected is_{min/max}_value_exact. (see
[note](Binary-truncated-min-and-max-statistics)).|
+| json.parquet | A single optional BYTE_ARRAY column annotated with the JSON
logical type (also carries the legacy converted type JSON). Four rows:
`{"a":1}`, `{"a":1,"b":null}` (null field value inside a non-null document),
`[1,null,3]` (null element inside an array), and a NULL row. See
[note](#json-and-bson-logical-types) below. |
+| bson.parquet | A single optional BYTE_ARRAY column annotated with the BSON
logical type (also carries the legacy converted type BSON). Three rows: BSON
`{"a":1}`, `{"a":1,"b":null}` (null field value inside a non-null document),
and a NULL row. See [note](#json-and-bson-logical-types) below. |
TODO: Document what each file is in the table above.
@@ -589,3 +591,108 @@ java -jar
parquet-cli/target/parquet-cli-1.16.0-SNAPSHOT-runtime.jar cat /home/r
{"utf8_full_truncation": "Julia Roberts", "binary_full_truncation": "Julia
Roberts", "utf8_partial_truncation": "Julia Roberts",
"binary_partial_truncation": "Julia Roberts", "utf8_no_truncation": "Julia
Roberts", "binary_no_truncation": "Julia Roberts"}
{"utf8_full_truncation": "Kevin Bacon", "binary_full_truncation": "Kevin
Bacon", "utf8_partial_truncation": "🚀Kevin Bacon", "binary_partial_truncation":
"ÿÿ\u0001\u0002", "utf8_no_truncation": "Ke", "binary_no_truncation": "Ke"}
```
+
+## JSON and BSON logical types
+
+`json.parquet` and `bson.parquet` each contain a single optional `BYTE_ARRAY`
+column annotated with the `JSON` and `BSON` logical types respectively (both
also
+carry the equivalent legacy converted type). They give reader implementations a
+small fixture for the JSON/BSON annotations, which no other file in this repo
+exercises. The data is intentionally small and includes a NULL value, documents
+with a null field value, and (for JSON) a null array element.
+
+`json.parquet` (4 rows): `{"a":1}`, `{"a":1,"b":null}` (null field value in a
+non-null document), `[1,null,3]` (null element in an array), and one NULL row.
+
+`bson.parquet` (3 rows): BSON `{"a":1}` (`0c0000001061000100000000`),
+`{"a":1,"b":null}` (`0f000000106100010000000a620000`), and one NULL row.
+
+Both files are generated by parquet-mr 1.18.0-SNAPSHOT using the following
code:
+
+`json.parquet`:
+
+```java
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.example.data.simple.SimpleGroupFactory;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.example.ExampleParquetWriter;
+import org.apache.parquet.hadoop.example.GroupWriteSupport;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.apache.parquet.schema.Types;
+
+public class JsonFixture {
+ public static void main(String[] args) throws Exception {
+ MessageType schema = Types.buildMessage()
+ .optional(PrimitiveTypeName.BINARY)
+ .as(LogicalTypeAnnotation.jsonType())
+ .named("json_field")
+ .named("json_fixture");
+ Configuration conf = new Configuration();
+ GroupWriteSupport.setSchema(schema, conf);
+ SimpleGroupFactory factory = new SimpleGroupFactory(schema);
+ String[] docs = {"{\"a\":1}", "{\"a\":1,\"b\":null}", "[1,null,3]"};
+ try (ParquetWriter<Group> writer = ExampleParquetWriter.builder(new
Path("json.parquet"))
+ .withConf(conf)
+ .withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+ .build()) {
+ for (String d : docs) {
+ writer.write(factory.newGroup().append("json_field", d)); // String
-> UTF-8 BYTE_ARRAY
+ }
+ writer.write(factory.newGroup()); // optional field omitted -> NULL row
+ }
+ }
+}
+```
+
+`bson.parquet`:
+
+```java
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.example.data.simple.SimpleGroupFactory;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.example.ExampleParquetWriter;
+import org.apache.parquet.hadoop.example.GroupWriteSupport;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.apache.parquet.schema.Types;
+
+public class BsonFixture {
+ public static void main(String[] args) throws Exception {
+ MessageType schema = Types.buildMessage()
+ .optional(PrimitiveTypeName.BINARY)
+ .as(LogicalTypeAnnotation.bsonType())
+ .named("bson_field")
+ .named("bson_fixture");
+ Configuration conf = new Configuration();
+ GroupWriteSupport.setSchema(schema, conf);
+ SimpleGroupFactory factory = new SimpleGroupFactory(schema);
+ byte[] doc1 =
{0x0c,0x00,0x00,0x00,0x10,0x61,0x00,0x01,0x00,0x00,0x00,0x00};
// {"a":1}
+ byte[] doc2 =
{0x0f,0x00,0x00,0x00,0x10,0x61,0x00,0x01,0x00,0x00,0x00,0x0a,0x62,0x00,0x00};
// {"a":1,"b":null}
+ try (ParquetWriter<Group> writer = ExampleParquetWriter.builder(new
Path("bson.parquet"))
+ .withConf(conf)
+ .withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+ .build()) {
+ writer.write(factory.newGroup().append("bson_field",
Binary.fromConstantByteArray(doc1)));
+ writer.write(factory.newGroup().append("bson_field",
Binary.fromConstantByteArray(doc2)));
+ writer.write(factory.newGroup()); // optional field omitted -> NULL row
+ }
+ }
+}
+```
+
+Column annotations read back with pyarrow (both files: `null_count` = 1):
+
+```
+json.parquet | physical: BYTE_ARRAY | logical: JSON | converted: JSON
+bson.parquet | physical: BYTE_ARRAY | logical: BSON | converted: BSON
+```
diff --git a/data/bson.parquet b/data/bson.parquet
new file mode 100644
index 0000000..5d97a9b
Binary files /dev/null and b/data/bson.parquet differ
diff --git a/data/json.parquet b/data/json.parquet
new file mode 100644
index 0000000..855ddcb
Binary files /dev/null and b/data/json.parquet differ
