WZhuo commented on code in PR #1144: URL: https://github.com/apache/iceberg-cpp/pull/1144#discussion_r4639459757
########## src/iceberg/test/verification/types_verification_test.cc: ########## @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Runs the `table-spec/types` surface: parse(input) == decoded, and where the case +// carries `canonical`, re-serializing the parsed type reproduces it byte for byte. + +#include <array> +#include <exception> +#include <format> +#include <memory> +#include <optional> +#include <string> +#include <string_view> +#include <vector> + +#include <gtest/gtest.h> +#include <nlohmann/json.hpp> + +#include "iceberg/json_serde_internal.h" +#include "iceberg/schema_field.h" +#include "iceberg/test/verification/verification_util.h" +#include "iceberg/type.h" +#include "iceberg/util/checked_cast.h" + +namespace iceberg::verification { + +namespace { + +constexpr std::array<std::string_view, 4> kSubdirs = {"primitive", "nested", "variant", + "geospatial"}; + +// Fields specific to this surface: the case's `canonical` and the rest from the +// `decoded` projection. +constexpr std::string_view kCanonical = "canonical"; +constexpr std::string_view kCrs = "crs"; +constexpr std::string_view kAlgorithm = "algorithm"; +constexpr std::string_view kElementId = "element-id"; +constexpr std::string_view kElementRequired = "element-required"; +constexpr std::string_view kElement = "element"; +constexpr std::string_view kKeyId = "key-id"; +constexpr std::string_view kKey = "key"; +constexpr std::string_view kValueId = "value-id"; +constexpr std::string_view kValueRequired = "value-required"; +constexpr std::string_view kValue = "value"; + +/// \brief Map a type to the language-neutral `decoded` shape of the types surface. +nlohmann::json ProjectType(const Type& type) { + nlohmann::json json{{kType, std::string(ToString(type.type_id()))}}; + switch (type.type_id()) { + case TypeId::kDecimal: { + const auto& decimal = internal::checked_cast<const DecimalType&>(type); + json[kPrecision] = decimal.precision(); + json[kScale] = decimal.scale(); + break; + } + case TypeId::kFixed: + json[kLength] = internal::checked_cast<const FixedType&>(type).length(); + break; + case TypeId::kGeometry: + json[kCrs] = std::string(internal::checked_cast<const GeometryType&>(type).crs()); + break; + case TypeId::kGeography: { + const auto& geography = internal::checked_cast<const GeographyType&>(type); + json[kCrs] = std::string(geography.crs()); + json[kAlgorithm] = std::string(ToString(geography.algorithm())); + break; + } + case TypeId::kStruct: { + json[kFields] = nlohmann::json::array(); + for (const auto& field : internal::checked_cast<const StructType&>(type).fields()) { + json[kFields].push_back({{kId, field.field_id()}, + {kName, std::string(field.name())}, + {kRequired, !field.optional()}, + {kType, ProjectType(*field.type())}}); + } + break; + } + case TypeId::kList: { + const auto& element = internal::checked_cast<const ListType&>(type).element(); + json[kElementId] = element.field_id(); + json[kElementRequired] = !element.optional(); + json[kElement] = ProjectType(*element.type()); + break; + } + case TypeId::kMap: { + const auto& map = internal::checked_cast<const MapType&>(type); + json[kKeyId] = map.key().field_id(); + json[kKey] = ProjectType(*map.key().type()); + json[kValueId] = map.value().field_id(); + json[kValueRequired] = !map.value().optional(); + json[kValue] = ProjectType(*map.value().type()); + break; + } + default: + break; + } + return json; +} + +std::vector<std::string> CheckCase(const nlohmann::json& c) { + std::optional<Result<std::unique_ptr<Type>>> parsed; + try { + parsed = TypeFromJson(c.at(kInput)); + } catch (const std::exception& e) { + return {std::format("parse threw instead of returning an error: {}", e.what())}; + } + + if (!c.at(kValid).get<bool>()) { + if (parsed->has_value()) { + return {std::format("expected rejection, but parsed as {}", + parsed->value()->ToString())}; + } + return {}; + } + if (!parsed->has_value()) { + return {std::format("parse failed: {}", parsed->error().message)}; + } + + const auto& type = *parsed->value(); + std::vector<std::string> diffs; + DiffJson(c.at(kDecoded), ProjectType(type), "", &diffs); + if (c.contains(kCanonical)) { + auto written = ToJson(type); + if (!written.has_value()) { Review Comment: `ToJson` returns a `Result`, so dereferencing it unchecked would be UB (or throw) if serialization ever failed. The check is defensive: a type that parses but cannot be written back shows up as a labeled failure of this case (`serialize failed: ...`) instead of crashing the test binary. ########## src/iceberg/test/verification/verification_util.cc: ########## @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/test/verification/verification_util.h" + +#include <algorithm> +#include <cctype> +#include <exception> +#include <filesystem> +#include <format> +#include <fstream> +#include <utility> + +#include "iceberg/test/test_config.h" + +namespace iceberg::verification { + +namespace { + +// Fixture fields read only by the loader and the outcome report. +constexpr std::string_view kCases = "cases"; +constexpr std::string_view kNormativeLevel = "normative_level"; + +std::filesystem::path TableSpecDir() { + return std::filesystem::path(ICEBERG_VERIFICATION_DIR) / "table-spec"; +} + +std::string JoinDiffs(const std::vector<std::string>& diffs) { + std::string out; + for (const auto& diff : diffs) { + out += "\n "; + out += diff; + } + return out; +} + +} // namespace + +bool FixturesAvailable() { return std::filesystem::is_directory(TableSpecDir()); } + +std::vector<VerificationCase> LoadCases(std::string_view surface, + std::span<const std::string_view> subdirs) { + std::vector<VerificationCase> cases; + if (!FixturesAvailable()) { + return cases; + } + for (auto subdir : subdirs) { + auto path = TableSpecDir() / surface / subdir / "cases.json"; + auto load_error = [&](std::string message) { + cases.push_back({.subdir = std::string(subdir), + .id = "load", + .load_error = std::format("{}: {}", path.string(), message)}); + }; + std::ifstream file(path); + if (!file) { + load_error("cannot open file"); + continue; + } + nlohmann::json fixture; + try { + fixture = nlohmann::json::parse(file); + } catch (const std::exception& e) { + load_error(e.what()); + continue; + } + if (!fixture.contains(kCases) || !fixture[kCases].is_array()) { + load_error("no `cases` array"); + continue; + } + for (auto& json : fixture[kCases]) { + cases.push_back({.subdir = std::string(subdir), + .id = json.value(kId, std::string()), + .json = std::move(json)}); + } + } + return cases; +} + +std::string CaseName(const ::testing::TestParamInfo<VerificationCase>& info) { + std::string name = info.param.Label(); + for (auto& c : name) { + if (!std::isalnum(static_cast<unsigned char>(c))) { + c = '_'; + } + } + return name; +} + +void DiffJson(const nlohmann::json& expected, const nlohmann::json& actual, + const std::string& path, std::vector<std::string>* diffs) { + if (expected.is_object() && actual.is_object()) { + for (const auto& [key, value] : expected.items()) { + auto child = path.empty() ? key : path + "." + key; + if (!actual.contains(key)) { + diffs->push_back(std::format("{}: missing, expected {}", child, value.dump())); + } else { + DiffJson(value, actual.at(key), child, diffs); + } + } + for (const auto& [key, value] : actual.items()) { + if (!expected.contains(key)) { + auto child = path.empty() ? key : path + "." + key; + diffs->push_back(std::format("{}: unexpected {}", child, value.dump())); + } + } + return; Review Comment: Done in c9c7a843. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
