paleolimbot commented on code in PR #334:
URL: https://github.com/apache/arrow-nanoarrow/pull/334#discussion_r1424308208
##########
extensions/nanoarrow_ipc/src/nanoarrow/nanoarrow_ipc_files_test.cc:
##########
@@ -70,31 +71,124 @@ class TestFile {
return Err(ENODATA, path, message);
}
- void Test(std::string dir_prefix) {
+ std::string CheckJSONGzFile() {
+ size_t dot_pos = path_.find('.');
+ return path_.substr(0, dot_pos) + std::string(".json.gz");
+ }
+
+ ArrowErrorCode GetArrowArrayStreamIPC(const std::string& dir_prefix,
+ ArrowArrayStream* out, ArrowError*
error) {
std::stringstream path_builder;
path_builder << dir_prefix << "/" << path_;
- // Read the whole file into an ArrowBuffer. We need the whole thing in
memory
- // to avoid requiring Arrow C++ with filesystem.
- std::ifstream infile(path_builder.str(), std::ios::in | std::ios::binary);
+ // Read using nanoarrow_ipc
nanoarrow::UniqueBuffer content;
+ NANOARROW_RETURN_NOT_OK(ReadFileBuffer(path_builder.str(), content.get(),
error));
+
+ struct ArrowIpcInputStream input;
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(
+ ArrowIpcInputStreamInitBuffer(&input, content.get()), error);
+ NANOARROW_RETURN_NOT_OK_WITH_ERROR(
+ ArrowIpcArrayStreamReaderInit(out, &input, nullptr), error);
+ return NANOARROW_OK;
+ }
+
+ ArrowErrorCode GetArrowArrayStreamCheckJSON(const std::string& dir_prefix,
+ ArrowArrayStream* out,
ArrowError* error) {
+ std::stringstream path_builder;
+ path_builder << dir_prefix << "/" << CheckJSONGzFile();
+
+ // Read .json.gz file into a buffer
+ nanoarrow::UniqueBuffer json_gz_content;
+ NANOARROW_RETURN_NOT_OK(
+ ReadFileBuffer(path_builder.str(), json_gz_content.get(), error));
+
+ // Decompress into a JSON string
+ nanoarrow::UniqueBuffer json_content;
+ NANOARROW_RETURN_NOT_OK(UnGZIP(json_gz_content.get(), json_content.get(),
error));
+
+ std::string json_string(reinterpret_cast<char*>(json_content->data),
+ json_content->size_bytes);
+
+ // Use testing util to populate the array stream
+ nanoarrow::testing::TestingJSONReader reader;
+ NANOARROW_RETURN_NOT_OK(reader.ReadDataFile(json_string, out, error));
+ return NANOARROW_OK;
+ }
+
+ // Read a whole file into an ArrowBuffer
+ static ArrowErrorCode ReadFileBuffer(const std::string& path, ArrowBuffer*
content,
Review Comment:
Eventually I need the "read gzipped JSON" half of this to not use Arrow C++
(so that it is possible/easier to run tests on platforms where Arrow C++
doesn't build), so I'm hesitant to lean too far into the Arrow C++ API for any
piece of this test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
