lgbo-ustc commented on code in PR #8558:
URL: https://github.com/apache/incubator-gluten/pull/8558#discussion_r1952070799
##########
cpp-ch/local-engine/Functions/SparkFunctionGetJsonObject.h:
##########
@@ -697,30 +786,36 @@ class FlattenJSONStringOnRequiredFunction : public
DB::IFunction
bool path_parsed = true;
for (const auto & field : tokenizer)
{
- auto normalized_field = JSONPathNormalizer::normalize(field);
- // LOG_ERROR(getLogger("JSONPatch"), "xxx field {} -> {}",
field, normalized_field);
+ auto normalized_field = field.find('#') != std::string::npos ?
field : JSONPathNormalizer::normalize(field);
if(normalized_field.find("[*]") != std::string::npos)
path_has_asterisk.emplace_back(true);
else
path_has_asterisk.emplace_back(false);
- required_fields.push_back(normalized_field);
- tuple_columns.emplace_back(str_type->createColumn());
-
- const char * query_begin = reinterpret_cast<const char
*>(required_fields.back().c_str());
- const char * query_end = required_fields.back().c_str() +
required_fields.back().size();
- DB::Tokens tokens(query_begin, query_end);
- UInt32 max_parser_depth =
static_cast<UInt32>(context->getSettingsRef()[DB::Setting::max_parser_depth]);
- UInt32 max_parser_backtracks =
static_cast<UInt32>(context->getSettingsRef()[DB::Setting::max_parser_backtracks]);
- DB::IParser::Pos token_iterator(tokens, max_parser_depth,
max_parser_backtracks);
- DB::ASTPtr json_path_ast;
- DB::ParserJSONPath path_parser;
- DB::Expected expected;
- if (!path_parser.parse(token_iterator, json_path_ast,
expected))
+ Poco::StringTokenizer sub_tokenizer(normalized_field, "#");
+ std::vector<String> sub_required_fields;
+ std::vector<DB::ASTPtr> sub_json_path_asts;
+ for (const auto & sub_field : sub_tokenizer)
{
- path_parsed = false;
+
sub_required_fields.push_back(JSONPathNormalizer::normalize(sub_field));
+ const char * query_begin = reinterpret_cast<const char
*>(sub_field.c_str());
+ const char * query_end = sub_field.c_str() +
sub_field.size();
Review Comment:
why not used the normalized one?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
