ocean-zhc commented on code in PR #9103: URL: https://github.com/apache/seatunnel/pull/9103#discussion_r2080878429
########## seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/util/JsonPathUtils.java: ########## @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.http.util; + +import org.apache.seatunnel.connectors.seatunnel.http.config.JsonField; +import org.apache.seatunnel.connectors.seatunnel.http.exception.HttpConnectorErrorCode; +import org.apache.seatunnel.connectors.seatunnel.http.exception.HttpConnectorException; + +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.Option; +import com.jayway.jsonpath.ReadContext; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Utility class for JsonPath operations. */ +public class JsonPathUtils { + + private static final Option[] DEFAULT_OPTIONS = { + Option.SUPPRESS_EXCEPTIONS, Option.ALWAYS_RETURN_LIST, Option.DEFAULT_PATH_LEAF_TO_NULL + }; + + private static final Configuration JSON_CONFIGURATION = + Configuration.defaultConfiguration().addOptions(DEFAULT_OPTIONS); + + /** + * Creates a ReadContext from a JSON string. + * + * @param json The JSON string + * @return A ReadContext for the JSON + */ + public static ReadContext parseJson(String json) { + return JsonPath.using(JSON_CONFIGURATION).parse(json); + } + + /** + * Extracts data from JSON using JsonPaths. + * + * @param json The JSON string to parse + * @param jsonField The JsonField containing path mappings + * @param allowNullFields Whether to allow missing fields + * @return List of data extracted from JSON + */ + public static List<Map<String, String>> extractData( + String json, JsonField jsonField, boolean allowNullFields) { + // Parse JSON into ReadContext + ReadContext jsonReadContext = parseJson(json); + + // Convert JsonField to JsonPath array + JsonPath[] jsonPaths = createJsonPaths(jsonField); + + // Get appropriate processor with jsonFiledMissedReturnNull flag + JsonPathProcessor processor = + JsonPathProcessorFactory.getProcessor(jsonPaths, allowNullFields); + + // Process JSON data based on the processor + List<List<String>> results = processor.processJsonData(jsonReadContext, jsonPaths); + + // Convert results to maps + return parseToMap(results, jsonField); + } + + /** + * Creates JsonPath array from JsonField. + * + * @param jsonField The JsonField to convert + * @return Array of JsonPath objects + */ + public static JsonPath[] createJsonPaths(JsonField jsonField) { + if (jsonField == null || jsonField.getFields() == null || jsonField.getFields().isEmpty()) { + throw new HttpConnectorException( + HttpConnectorErrorCode.FIELD_DATA_IS_INCONSISTENT, + "JsonField cannot be null or empty"); + } + + JsonPath[] jsonPaths = new JsonPath[jsonField.getFields().size()]; + int index = 0; + for (String pathString : jsonField.getFields().values()) { + jsonPaths[index++] = JsonPath.compile(pathString); + } + + return jsonPaths; + } + + /** + * Converts parsed data to a list of maps. + * + * @param data The raw data (list of lists) + * @param jsonField The JsonField containing field names + * @return List of maps with field names as keys + */ + public static List<Map<String, String>> parseToMap( + List<List<String>> data, JsonField jsonField) { + List<Map<String, String>> resultList = new ArrayList<>(data.size()); + String[] keys = jsonField.getFields().keySet().toArray(new String[0]); + + for (List<String> row : data) { + Map<String, String> resultMap = new HashMap<>(jsonField.getFields().size()); + for (int i = 0; i < row.size(); i++) { + resultMap.put(keys[i], row.get(i)); + } + resultList.add(resultMap); + } + + return resultList; + } + + /** + * Extracts the common parent path from JsonPaths. + * + * @param paths Array of JsonPath objects + * @return The common parent path + */ + public static String extractCommonParentPath(JsonPath[] paths) { + return JsonPathProcessorFactory.getProcessor(paths).extractCommonParentPath(paths); + } + + /** + * Gets a relative path from a parent path and a full path. + * + * @param parentPath The parent path + * @param fullPath The full path + * @return The relative path + */ + public static String getRelativePath(String parentPath, String fullPath) { + return JsonPathProcessorFactory.getProcessor(fullPath) + .getRelativePath(parentPath, fullPath); + } Review Comment: Moved out ########## seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/source/HttpSourceReader.java: ########## @@ -467,8 +447,8 @@ private String getPartOfJson(String data) { } private List<List<String>> dataFlip(List<List<String>> results) { Review Comment: Moved out ########## seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/util/JsonPathUtils.java: ########## @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.http.util; + +import org.apache.seatunnel.connectors.seatunnel.http.config.JsonField; +import org.apache.seatunnel.connectors.seatunnel.http.exception.HttpConnectorErrorCode; +import org.apache.seatunnel.connectors.seatunnel.http.exception.HttpConnectorException; + +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.Option; +import com.jayway.jsonpath.ReadContext; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Utility class for JsonPath operations. */ +public class JsonPathUtils { + + private static final Option[] DEFAULT_OPTIONS = { + Option.SUPPRESS_EXCEPTIONS, Option.ALWAYS_RETURN_LIST, Option.DEFAULT_PATH_LEAF_TO_NULL + }; + + private static final Configuration JSON_CONFIGURATION = + Configuration.defaultConfiguration().addOptions(DEFAULT_OPTIONS); + + /** + * Creates a ReadContext from a JSON string. + * + * @param json The JSON string + * @return A ReadContext for the JSON + */ + public static ReadContext parseJson(String json) { + return JsonPath.using(JSON_CONFIGURATION).parse(json); + } + + /** + * Extracts data from JSON using JsonPaths. + * + * @param json The JSON string to parse + * @param jsonField The JsonField containing path mappings + * @param allowNullFields Whether to allow missing fields + * @return List of data extracted from JSON + */ + public static List<Map<String, String>> extractData( Review Comment: Moved out -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
